1package textseg
2
3import (
4	"bufio"
5	"reflect"
6	"testing"
7)
8
9func TestAllTokens(t *testing.T) {
10	tests := []struct {
11		input string
12		want  []string
13	}{
14		{
15			``,
16			[]string{},
17		},
18		{
19			`hello`,
20			[]string{
21				`hello`,
22			},
23		},
24		{
25			`hello world`,
26			[]string{
27				`hello`,
28				`world`,
29			},
30		},
31		{
32			`hello worldly world`,
33			[]string{
34				`hello`,
35				`worldly`,
36				`world`,
37			},
38		},
39	}
40
41	for _, test := range tests {
42		t.Run(test.input, func(t *testing.T) {
43			gotBytes, err := AllTokens([]byte(test.input), bufio.ScanWords)
44			if err != nil {
45				t.Fatalf("unexpected error: %s", err)
46			}
47
48			got := make([]string, len(gotBytes))
49			for i, buf := range gotBytes {
50				got[i] = string(buf)
51			}
52
53			if !reflect.DeepEqual(got, test.want) {
54				wantBytes := make([][]byte, len(test.want))
55				for i, str := range test.want {
56					wantBytes[i] = []byte(str)
57				}
58
59				t.Errorf(
60					"wrong result\ninput: %s\ngot:   %s\nwant:  %s",
61					formatBytes([]byte(test.input)),
62					formatByteRanges(gotBytes),
63					formatByteRanges(wantBytes),
64				)
65			}
66		})
67	}
68}
69