diff options
Diffstat (limited to 'bpc/cmd')
| -rw-r--r-- | bpc/cmd/bpc/main.go | 48 |
1 files changed, 0 insertions, 48 deletions
diff --git a/bpc/cmd/bpc/main.go b/bpc/cmd/bpc/main.go deleted file mode 100644 index 567b68a..0000000 --- a/bpc/cmd/bpc/main.go +++ /dev/null @@ -1,48 +0,0 @@ -package main - -import ( - "log" - "os" - - "github.com/jonasknobloch/mbpe" - - "go.jknobloc.com/x/bpc" - "go.jknobloc.com/x/gpt2" -) - -func main() { - m := model() - - if err := m.Init(); err != nil { - log.Fatal(err) - } - - t := tokenizer() - - bpc.Run(m, t) - - if err := m.Destroy(); err != nil { - log.Fatal(err) - } -} - -func model() *gpt2.Model { - return gpt2.NewModel("../gpt2/models/base/model.onnx", os.Getenv("BPC_CUDA_DEVICE_ID")) -} - -func tokenizer() *mbpe.Tokenizer { - m := mbpe.NewMBPE() - - if err := m.Load("../gpt2/models/base/vocab.json", "../gpt2/models/base/merges.txt"); err != nil { - panic(err) - } - - t := mbpe.NewTokenizer(m) - - byteLevel := mbpe.NewByteLevel(false) - - t.SetPreTokenizer(byteLevel) - t.SetDecoder(byteLevel) - - return t -} |
