diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-24 11:49:26 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-24 11:50:02 +0200 |
| commit | 12f15fdd8c048e5958e294f12a9f163f6f950b3e (patch) | |
| tree | 16bb23c134f1fe476d977dd846e6dda6c7667c48 /research/sander | |
| parent | 408536516bdfedc93d93226dc08280ed224361fc (diff) | |
Recover tokenization based on config
Diffstat (limited to 'research/sander')
| -rw-r--r-- | research/sander/cmd/sander/main.go | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/research/sander/cmd/sander/main.go b/research/sander/cmd/sander/main.go index 6f68434..56f1488 100644 --- a/research/sander/cmd/sander/main.go +++ b/research/sander/cmd/sander/main.go @@ -28,7 +28,11 @@ func main() { } func run(src, dst string) error { - t := must(bpe.NewTokenizerFromFiles(path.Join(src, "vocab.json"), path.Join(src, "merges.txt"))) + cfg := bpe.Config{ + Recover: true, + } + + t := must(bpe.NewTokenizerFromFiles(path.Join(src, "vocab.json"), path.Join(src, "merges.txt"), cfg)) e := must(sander.NewExperiment(dst, path.Join(src, "model.onnx"), sander.UnusedTokensMBPE(t))) return e.Run() |