From b1ba8ff474afc003e956de7bf371887859fb1df1 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Sat, 27 Jun 2026 15:06:57 +0200 Subject: Bump github.com/jonasknobloch/mbpe to v0.1.2 --- tokenizer/go.mod | 13 ++++++++++--- tokenizer/go.sum | 36 ++++++++++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 9 deletions(-) (limited to 'tokenizer') diff --git a/tokenizer/go.mod b/tokenizer/go.mod index dcac6cc..2db675b 100644 --- a/tokenizer/go.mod +++ b/tokenizer/go.mod @@ -3,10 +3,12 @@ module go.jknobloc.com/x/tokenizer go 1.25.0 require ( - github.com/jonasknobloch/mbpe v0.1.1 - go.jknobloc.com/x/dataset v0.0.0-20260324194423-87bbece7e040 + github.com/jonasknobloch/mbpe v0.1.2 + go.jknobloc.com/x/dataset v0.0.0-20260424095002-12f15fdd8c04 go.jknobloc.com/x/llm v0.0.0-20260324194423-87bbece7e040 - go.jknobloc.com/x/tui v0.0.0-20260324194423-87bbece7e040 + go.jknobloc.com/x/profile v0.0.0-20260626223026-e3713a4b0402 + go.jknobloc.com/x/shelf v0.0.0-20260519111645-306baa3f2a00 + go.jknobloc.com/x/tui v0.0.0-20260424095002-12f15fdd8c04 ) require ( @@ -19,11 +21,15 @@ require ( github.com/apache/arrow-go/v18 v18.5.1 // indirect github.com/apache/thrift v0.22.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.8.0 // indirect + github.com/go-git/go-git/v5 v5.18.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/google/uuid v1.6.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.18.3 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect @@ -46,4 +52,5 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect google.golang.org/grpc v1.78.0 // indirect google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/tokenizer/go.sum b/tokenizer/go.sum index c12403c..49b4759 100644 --- a/tokenizer/go.sum +++ b/tokenizer/go.sum @@ -29,6 +29,12 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0= +github.com/go-git/go-billy/v5 v5.8.0/go.mod h1:RpvI/rw4Vr5QA+Z60c6d6LXH0rYJo0uD5SqfmrrheCY= +github.com/go-git/go-git/v5 v5.18.0 h1:O831KI+0PR51hM2kep6T8k+w0/LIAD490gvqMCvL5hM= +github.com/go-git/go-git/v5 v5.18.0/go.mod h1:pW/VmeqkanRFqR6AljLcs7EA7FbZaN5MQqO7oZADXpo= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -47,8 +53,10 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/jonasknobloch/mbpe v0.1.1 h1:eXUrMdM7Wt6kPTA8cyX+t1PsInk3QIXpt430pE8CJEs= -github.com/jonasknobloch/mbpe v0.1.1/go.mod h1:2qW/5BfAu7GKAXXNYJzMrIPYk/8z5+/TbabaAb+vPv8= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jonasknobloch/mbpe v0.1.2 h1:BRAVv/Tr246rtFAv5JRRfHptFvPgid8ZUWpdh7yJRnY= +github.com/jonasknobloch/mbpe v0.1.2/go.mod h1:/gcmDVpejj0J5f/onTzCzrtLgkhuhkIsWq8cbwjVP6o= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= @@ -56,14 +64,22 @@ github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+ github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -75,12 +91,16 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= -go.jknobloc.com/x/dataset v0.0.0-20260324194423-87bbece7e040 h1:sDDUfVV0ZLHDB8jSYs2j7Le6rs+J8aUVAH1QDKLzKEc= -go.jknobloc.com/x/dataset v0.0.0-20260324194423-87bbece7e040/go.mod h1:UZypBoGqi23LRt29l2z9NtVD2pKBHd4unV/sztHRz4o= +go.jknobloc.com/x/dataset v0.0.0-20260424095002-12f15fdd8c04 h1:vPmulcQEwluqrIfmL4yH9mRsXooGKcRznv79FehGP50= +go.jknobloc.com/x/dataset v0.0.0-20260424095002-12f15fdd8c04/go.mod h1:UZypBoGqi23LRt29l2z9NtVD2pKBHd4unV/sztHRz4o= go.jknobloc.com/x/llm v0.0.0-20260324194423-87bbece7e040 h1:XlRLdIUmIHy8T6Pe27+drvMDJoyGm6RcsFQM6+3HEFc= go.jknobloc.com/x/llm v0.0.0-20260324194423-87bbece7e040/go.mod h1:4oqwJDP4X+3ehI3pF8UOCklKUWub3XiPv9vhFNSfYgs= -go.jknobloc.com/x/tui v0.0.0-20260324194423-87bbece7e040 h1:7Ago/qaKuXIJ7/+/XDRO8Yb/4VWzBLKEcbAq2jD5k3Q= -go.jknobloc.com/x/tui v0.0.0-20260324194423-87bbece7e040/go.mod h1:kblmBsWO7WlRCo4sWpstaVV9ukNfYP3PeAPFhcoJvKA= +go.jknobloc.com/x/profile v0.0.0-20260626223026-e3713a4b0402 h1:32q61rY/IF/Klm/i2PheGGJD8Na6gfaD+mwwt+ymveE= +go.jknobloc.com/x/profile v0.0.0-20260626223026-e3713a4b0402/go.mod h1:FQSHymbvsWdLUJp21RlDBf51agRmf3WRsIdXoyRJT0c= +go.jknobloc.com/x/shelf v0.0.0-20260519111645-306baa3f2a00 h1:wf1MCh+A9icxS98XUSMv1igLxI4k2OrrM0DpodQZfWY= +go.jknobloc.com/x/shelf v0.0.0-20260519111645-306baa3f2a00/go.mod h1:j+bN57GbQXhfjqGJ0KEeGjGsJ3QpOy9FriNBigqT5xQ= +go.jknobloc.com/x/tui v0.0.0-20260424095002-12f15fdd8c04 h1:ry/wi9v24wk6JFfIBaTQ7X29bCccGs5tKBtDF3NvGmc= +go.jknobloc.com/x/tui v0.0.0-20260424095002-12f15fdd8c04/go.mod h1:kblmBsWO7WlRCo4sWpstaVV9ukNfYP3PeAPFhcoJvKA= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= @@ -144,6 +164,10 @@ google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -- cgit v1.2.3