summaryrefslogtreecommitdiff
path: root/bpc/cmd
diff options
context:
space:
mode:
Diffstat (limited to 'bpc/cmd')
-rw-r--r--bpc/cmd/bpc/main.go48
1 files changed, 0 insertions, 48 deletions
diff --git a/bpc/cmd/bpc/main.go b/bpc/cmd/bpc/main.go
deleted file mode 100644
index 567b68a..0000000
--- a/bpc/cmd/bpc/main.go
+++ /dev/null
@@ -1,48 +0,0 @@
-package main
-
-import (
- "log"
- "os"
-
- "github.com/jonasknobloch/mbpe"
-
- "go.jknobloc.com/x/bpc"
- "go.jknobloc.com/x/gpt2"
-)
-
-func main() {
- m := model()
-
- if err := m.Init(); err != nil {
- log.Fatal(err)
- }
-
- t := tokenizer()
-
- bpc.Run(m, t)
-
- if err := m.Destroy(); err != nil {
- log.Fatal(err)
- }
-}
-
-func model() *gpt2.Model {
- return gpt2.NewModel("../gpt2/models/base/model.onnx", os.Getenv("BPC_CUDA_DEVICE_ID"))
-}
-
-func tokenizer() *mbpe.Tokenizer {
- m := mbpe.NewMBPE()
-
- if err := m.Load("../gpt2/models/base/vocab.json", "../gpt2/models/base/merges.txt"); err != nil {
- panic(err)
- }
-
- t := mbpe.NewTokenizer(m)
-
- byteLevel := mbpe.NewByteLevel(false)
-
- t.SetPreTokenizer(byteLevel)
- t.SetDecoder(byteLevel)
-
- return t
-}