From e1e4eab2e1a682d633125f3c71cc09c3e95b30d3 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Sat, 27 Jun 2026 19:08:39 +0200 Subject: Add token buffer config --- llm/tokenbuffer_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'llm/tokenbuffer_test.go') diff --git a/llm/tokenbuffer_test.go b/llm/tokenbuffer_test.go index 789caa6..c07c371 100644 --- a/llm/tokenbuffer_test.go +++ b/llm/tokenbuffer_test.go @@ -49,7 +49,7 @@ func TestTokenBuffer_Push(t *testing.T) { fmt.Sprintf("window%d_stride%d", tt.window, tt.stride), func(t *testing.T) { - tb := NewTokenBuffer(byteTokenizer{}, tt.window, tt.stride) + tb := NewTokenBuffer(byteTokenizer{}, TokenBufferConfig{Window: tt.window, Stride: tt.stride}) tb.SetIncludeTail(false) @@ -74,7 +74,7 @@ func TestTokenBuffer_Push(t *testing.T) { } func TestTokenBuffer_PushAccumulates(t *testing.T) { - tb := NewTokenBuffer(byteTokenizer{}, 4, 4) + tb := NewTokenBuffer(byteTokenizer{}, TokenBufferConfig{Window: 4, Stride: 4}) var got [][]int64 @@ -104,7 +104,7 @@ func TestTokenBuffer_PushAccumulates(t *testing.T) { } func TestTokenBuffer_Tail(t *testing.T) { - tb := NewTokenBuffer(byteTokenizer{}, 5, 5) + tb := NewTokenBuffer(byteTokenizer{}, TokenBufferConfig{Window: 5, Stride: 5}) tb.SetIncludeTail(true) @@ -132,7 +132,7 @@ func TestTokenBuffer_Tail(t *testing.T) { } func TestTokenBuffer_Position(t *testing.T) { - tb := NewTokenBuffer(byteTokenizer{}, 4, 2) + tb := NewTokenBuffer(byteTokenizer{}, TokenBufferConfig{Window: 4, Stride: 2}) tb.SetIncludeTail(false) @@ -162,7 +162,7 @@ func TestTokenBuffer_Position(t *testing.T) { } func TestTokenBuffer_DocumentBoundaryYieldsTail(t *testing.T) { - tb := NewTokenBuffer(byteTokenizer{}, 10, 10) + tb := NewTokenBuffer(byteTokenizer{}, TokenBufferConfig{Window: 10, Stride: 10}) tb.SetIncludeTail(true) @@ -194,7 +194,7 @@ func TestTokenBuffer_DocumentBoundaryYieldsTail(t *testing.T) { } func TestTokenBuffer_TailPosition(t *testing.T) { - tb := NewTokenBuffer(byteTokenizer{}, 4, 2) + tb := NewTokenBuffer(byteTokenizer{}, TokenBufferConfig{Window: 4, Stride: 2}) tb.SetIncludeTail(true) -- cgit v1.2.3