diff options
| -rw-r--r-- | llm/cmd/eval/main.go | 3 | ||||
| -rw-r--r-- | tokenizer/bpe/bytelevel.go | 1 |
2 files changed, 2 insertions, 2 deletions
diff --git a/llm/cmd/eval/main.go b/llm/cmd/eval/main.go index ec9b7e9..bcc6fb4 100644 --- a/llm/cmd/eval/main.go +++ b/llm/cmd/eval/main.go @@ -3,10 +3,9 @@ package main import ( "log" - "go.jknobloc.com/x/tokenizer/bpe" - "go.jknobloc.com/x/dataset" "go.jknobloc.com/x/gpt2" + "go.jknobloc.com/x/tokenizer/bpe" ) func main() { diff --git a/tokenizer/bpe/bytelevel.go b/tokenizer/bpe/bytelevel.go index 4059d0d..dfecb53 100644 --- a/tokenizer/bpe/bytelevel.go +++ b/tokenizer/bpe/bytelevel.go @@ -2,6 +2,7 @@ package bpe import ( "github.com/jonasknobloch/mbpe" + "go.jknobloc.com/x/tokenizer/bpe/split" ) |