diff options
Diffstat (limited to 'research/lesci/context.go')
| -rw-r--r-- | research/lesci/context.go | 14 |
1 files changed, 3 insertions, 11 deletions
diff --git a/research/lesci/context.go b/research/lesci/context.go index 7f98946..8201e6f 100644 --- a/research/lesci/context.go +++ b/research/lesci/context.go @@ -84,27 +84,19 @@ func (e *Experiment) BuildContext(db *sql.DB) error { document: job.Document, token: token, value: logProbs[i], - offset: job.Position*512 + job.Seen + i, // TODO refactor + offset: job.Position*e.tokenBufferConfig.Stride + job.Seen + i, } } return r - }, llm.EvaluatorConfig{ - BatchSize: 32, - NumWorkers: 64, - }) + }, e.evaluatorConfig) if err := e.ensureContext(db); err != nil { return err } - cfg := llm.TokenBufferConfig{ - Window: 1024, - Stride: 512, - } - return AppendRows(db, "context", func(append AppendFunc) error { - return eval.RunAndCollect("Context", e.data, cfg, func(r []logProb) error { + return eval.RunAndCollect("Context", e.data, e.tokenBufferConfig, func(r []logProb) error { for _, l := range r { if err := append([]driver.Value{l.document, l.token, l.value, l.offset}); err != nil { return err |
