1//  Copyright (c) 2014 Couchbase, Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// 		http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package analysis
16
17import (
18	"bufio"
19	"bytes"
20	"io"
21	"io/ioutil"
22	"strings"
23)
24
25type TokenMap map[string]bool
26
27func NewTokenMap() TokenMap {
28	return make(TokenMap, 0)
29}
30
31// LoadFile reads in a list of tokens from a text file,
32// one per line.
33// Comments are supported using `#` or `|`
34func (t TokenMap) LoadFile(filename string) error {
35	data, err := ioutil.ReadFile(filename)
36	if err != nil {
37		return err
38	}
39	return t.LoadBytes(data)
40}
41
42// LoadBytes reads in a list of tokens from memory,
43// one per line.
44// Comments are supported using `#` or `|`
45func (t TokenMap) LoadBytes(data []byte) error {
46	bytesReader := bytes.NewReader(data)
47	bufioReader := bufio.NewReader(bytesReader)
48	line, err := bufioReader.ReadString('\n')
49	for err == nil {
50		t.LoadLine(line)
51		line, err = bufioReader.ReadString('\n')
52	}
53	// if the err was EOF we still need to process the last value
54	if err == io.EOF {
55		t.LoadLine(line)
56		return nil
57	}
58	return err
59}
60
61func (t TokenMap) LoadLine(line string) {
62	// find the start of a comment, if any
63	startComment := strings.IndexAny(line, "#|")
64	if startComment >= 0 {
65		line = line[:startComment]
66	}
67
68	tokens := strings.Fields(line)
69	for _, token := range tokens {
70		t.AddToken(token)
71	}
72}
73
74func (t TokenMap) AddToken(token string) {
75	t[token] = true
76}
77