1package tokenize
2
3import (
4	"testing"
5
6	"github.com/stretchr/testify/assert"
7)
8
9func TestTreebankWordTokenizer(t *testing.T) {
10	input, output := getWordData("treebank_words.json")
11	word := NewTreebankWordTokenizer()
12	for i, s := range input {
13		assert.Equal(t, output[i], word.Tokenize(s))
14	}
15}
16
17func BenchmarkTreebankWordTokenizer(b *testing.B) {
18	word := NewTreebankWordTokenizer()
19	for n := 0; n < b.N; n++ {
20		for _, s := range getWordBenchData() {
21			word.Tokenize(s)
22		}
23	}
24}
25