Add lesci module

2 files changed, 34 insertions, 0 deletions

diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go
index f47c073..fbe99fc 100644
--- a/tokenizer/bpe/utility.go
+++ b/tokenizer/bpe/utility.go
@@ -18,3 +18,23 @@ func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) {
 
 	return NewTokenizer(tokenizer), nil
 }
+
+func Vocab(t *Tokenizer) []string {
+	m, ok := t.mbpe.Model().(*mbpe.MBPE)
+
+	if !ok {
+		panic("unimplemented")
+	}
+
+	return m.Vocab()
+}
+
+func Merges(t *Tokenizer) [][2]string {
+	m, ok := t.mbpe.Model().(*mbpe.MBPE)
+
+	if !ok {
+		panic("unimplemented")
+	}
+
+	return m.Merges()
+}
diff --git a/tokenizer/bpe/validate.go b/tokenizer/bpe/validate.go
index f9bd265..84910cf 100644
--- a/tokenizer/bpe/validate.go
+++ b/tokenizer/bpe/validate.go
@@ -106,3 +106,17 @@ func ByteCoverage(t *Tokenizer) bool {
 
 	return covered
 }
+
+func ReachableMerges(t *Tokenizer, merges [][2]string) []bool {
+	mask := make([]bool, len(merges))
+
+	for i, merge := range merges {
+		a := t.Tokenize(merge[0])
+		b := t.Tokenize(merge[1])
+		c := t.Tokenize(merge[0] + merge[1])
+
+		mask[i] = len(a) == 1 && len(b) == 1 && len(c) == 1
+	}
+
+	return mask
+}