diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-06-27 20:24:38 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-06-27 20:41:40 +0200 |
| commit | c87625c71737f41f42f0188ff87b8d9313c8548a (patch) | |
| tree | 0d9666eaabe04d29a01b5417e2fa59c96811a17e /llm/cmd/eval/perplexity.go | |
| parent | e1e4eab2e1a682d633125f3c71cc09c3e95b30d3 (diff) | |
Pad token buffer
Diffstat (limited to 'llm/cmd/eval/perplexity.go')
| -rw-r--r-- | llm/cmd/eval/perplexity.go | 8 |
1 files changed, 6 insertions, 2 deletions
diff --git a/llm/cmd/eval/perplexity.go b/llm/cmd/eval/perplexity.go index dd4dae3..b84b659 100644 --- a/llm/cmd/eval/perplexity.go +++ b/llm/cmd/eval/perplexity.go @@ -43,8 +43,11 @@ func perplexity() { n := 0 cfg := llm.TokenBufferConfig{ - Window: 1024, - Stride: 512, + Window: 1024, + Stride: 512, + PadLeft: false, // probably not fine since we don't adjust model inputs for padding + PadRight: true, // fine since we remove padded log probs anyway + PadTokenID: 50256, } if err := e.RunAndCollect("Perplexity", d, cfg, func(r pplResult) error { @@ -60,6 +63,7 @@ func perplexity() { ppl := math.Exp(avg) fmt.Println(ppl) + fmt.Printf("%d tokens\n", n) if err := m.Destroy(); err != nil { log.Fatal(err) |