1package textseg 2 3import ( 4 "bufio" 5 "reflect" 6 "testing" 7) 8 9func TestAllTokens(t *testing.T) { 10 tests := []struct { 11 input string 12 want []string 13 }{ 14 { 15 ``, 16 []string{}, 17 }, 18 { 19 `hello`, 20 []string{ 21 `hello`, 22 }, 23 }, 24 { 25 `hello world`, 26 []string{ 27 `hello`, 28 `world`, 29 }, 30 }, 31 { 32 `hello worldly world`, 33 []string{ 34 `hello`, 35 `worldly`, 36 `world`, 37 }, 38 }, 39 } 40 41 for _, test := range tests { 42 t.Run(test.input, func(t *testing.T) { 43 gotBytes, err := AllTokens([]byte(test.input), bufio.ScanWords) 44 if err != nil { 45 t.Fatalf("unexpected error: %s", err) 46 } 47 48 got := make([]string, len(gotBytes)) 49 for i, buf := range gotBytes { 50 got[i] = string(buf) 51 } 52 53 if !reflect.DeepEqual(got, test.want) { 54 wantBytes := make([][]byte, len(test.want)) 55 for i, str := range test.want { 56 wantBytes[i] = []byte(str) 57 } 58 59 t.Errorf( 60 "wrong result\ninput: %s\ngot: %s\nwant: %s", 61 formatBytes([]byte(test.input)), 62 formatByteRanges(gotBytes), 63 formatByteRanges(wantBytes), 64 ) 65 } 66 }) 67 } 68} 69