summaryrefslogtreecommitdiff
path: root/research/lesci/context.go
diff options
context:
space:
mode:
Diffstat (limited to 'research/lesci/context.go')
-rw-r--r--research/lesci/context.go14
1 files changed, 3 insertions, 11 deletions
diff --git a/research/lesci/context.go b/research/lesci/context.go
index 7f98946..8201e6f 100644
--- a/research/lesci/context.go
+++ b/research/lesci/context.go
@@ -84,27 +84,19 @@ func (e *Experiment) BuildContext(db *sql.DB) error {
document: job.Document,
token: token,
value: logProbs[i],
- offset: job.Position*512 + job.Seen + i, // TODO refactor
+ offset: job.Position*e.tokenBufferConfig.Stride + job.Seen + i,
}
}
return r
- }, llm.EvaluatorConfig{
- BatchSize: 32,
- NumWorkers: 64,
- })
+ }, e.evaluatorConfig)
if err := e.ensureContext(db); err != nil {
return err
}
- cfg := llm.TokenBufferConfig{
- Window: 1024,
- Stride: 512,
- }
-
return AppendRows(db, "context", func(append AppendFunc) error {
- return eval.RunAndCollect("Context", e.data, cfg, func(r []logProb) error {
+ return eval.RunAndCollect("Context", e.data, e.tokenBufferConfig, func(r []logProb) error {
for _, l := range r {
if err := append([]driver.Value{l.document, l.token, l.value, l.offset}); err != nil {
return err