1package tokenize 2 3import ( 4 "testing" 5 6 "github.com/stretchr/testify/assert" 7) 8 9func TestTreebankWordTokenizer(t *testing.T) { 10 input, output := getWordData("treebank_words.json") 11 word := NewTreebankWordTokenizer() 12 for i, s := range input { 13 assert.Equal(t, output[i], word.Tokenize(s)) 14 } 15} 16 17func BenchmarkTreebankWordTokenizer(b *testing.B) { 18 word := NewTreebankWordTokenizer() 19 for n := 0; n < b.N; n++ { 20 for _, s := range getWordBenchData() { 21 word.Tokenize(s) 22 } 23 } 24} 25