From e1e4eab2e1a682d633125f3c71cc09c3e95b30d3 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Sat, 27 Jun 2026 19:08:39 +0200 Subject: Add token buffer config --- llm/tokenbuffer.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'llm/tokenbuffer.go') diff --git a/llm/tokenbuffer.go b/llm/tokenbuffer.go index d19de4a..e2d8a91 100644 --- a/llm/tokenbuffer.go +++ b/llm/tokenbuffer.go @@ -12,22 +12,27 @@ type TokenBuffer struct { includeTail bool } +type TokenBufferConfig struct { + Window int + Stride int +} + type TokenWindow struct { Document int Tokens []int64 Seen int } -func NewTokenBuffer(tokenizer Tokenizer, window, stride int) *TokenBuffer { - if stride > window { +func NewTokenBuffer(tokenizer Tokenizer, cfg TokenBufferConfig) *TokenBuffer { + if cfg.Stride > cfg.Window { panic("stride exceeds window") } return &TokenBuffer{ tokenizer: tokenizer, - window: window, - stride: stride, - buffer: make([]int64, 0, 2*window), + window: cfg.Window, + stride: cfg.Stride, + buffer: make([]int64, 0, 2*cfg.Window), document: -1, position: 0, includeTail: true, -- cgit v1.2.3