1// Copyright (c) 2014 Couchbase, Inc. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package analysis 16 17import ( 18 "bufio" 19 "bytes" 20 "io" 21 "io/ioutil" 22 "strings" 23) 24 25type TokenMap map[string]bool 26 27func NewTokenMap() TokenMap { 28 return make(TokenMap, 0) 29} 30 31// LoadFile reads in a list of tokens from a text file, 32// one per line. 33// Comments are supported using `#` or `|` 34func (t TokenMap) LoadFile(filename string) error { 35 data, err := ioutil.ReadFile(filename) 36 if err != nil { 37 return err 38 } 39 return t.LoadBytes(data) 40} 41 42// LoadBytes reads in a list of tokens from memory, 43// one per line. 44// Comments are supported using `#` or `|` 45func (t TokenMap) LoadBytes(data []byte) error { 46 bytesReader := bytes.NewReader(data) 47 bufioReader := bufio.NewReader(bytesReader) 48 line, err := bufioReader.ReadString('\n') 49 for err == nil { 50 t.LoadLine(line) 51 line, err = bufioReader.ReadString('\n') 52 } 53 // if the err was EOF we still need to process the last value 54 if err == io.EOF { 55 t.LoadLine(line) 56 return nil 57 } 58 return err 59} 60 61func (t TokenMap) LoadLine(line string) { 62 // find the start of a comment, if any 63 startComment := strings.IndexAny(line, "#|") 64 if startComment >= 0 { 65 line = line[:startComment] 66 } 67 68 tokens := strings.Fields(line) 69 for _, token := range tokens { 70 t.AddToken(token) 71 } 72} 73 74func (t TokenMap) AddToken(token string) { 75 t[token] = true 76} 77