1// Copyright (c) 2015 Couchbase, Inc. 2// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 3// except in compliance with the License. You may obtain a copy of the License at 4// http://www.apache.org/licenses/LICENSE-2.0 5// Unless required by applicable law or agreed to in writing, software distributed under the 6// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 7// either express or implied. See the License for the specific language governing permissions 8// and limitations under the License. 9 10package segment 11 12import ( 13 "errors" 14 "io" 15) 16 17// Autogenerate the following: 18// 1. Ragel rules from subset of Unicode script properties 19// 2. Ragel rules from Unicode word segmentation properties 20// 3. Ragel machine for word segmentation 21// 4. Test tables from Unicode 22// 23// Requires: 24// 1. Ruby (to generate ragel rules from unicode spec) 25// 2. Ragel (only v6.9 tested) 26// 3. sed (to rewrite build tags) 27// 28//go:generate ragel/unicode2ragel.rb -u http://www.unicode.org/Public/8.0.0/ucd/Scripts.txt -m SCRIPTS -p Hangul,Han,Hiragana -o ragel/uscript.rl 29//go:generate ragel/unicode2ragel.rb -u http://www.unicode.org/Public/8.0.0/ucd/auxiliary/WordBreakProperty.txt -m WB -p Double_Quote,Single_Quote,Hebrew_Letter,CR,LF,Newline,Extend,Format,Katakana,ALetter,MidLetter,MidNum,MidNumLet,Numeric,ExtendNumLet,Regional_Indicator -o ragel/uwb.rl 30//go:generate ragel -T1 -Z segment_words.rl -o segment_words.go 31//go:generate sed -i "" -e "s/BUILDTAGS/!prod/" segment_words.go 32//go:generate sed -i "" -e "s/RAGELFLAGS/-T1/" segment_words.go 33//go:generate ragel -G2 -Z segment_words.rl -o segment_words_prod.go 34//go:generate sed -i "" -e "s/BUILDTAGS/prod/" segment_words_prod.go 35//go:generate sed -i "" -e "s/RAGELFLAGS/-G2/" segment_words_prod.go 36//go:generate go run maketesttables.go -output tables_test.go 37 38// NewWordSegmenter returns a new Segmenter to read from r. 39func NewWordSegmenter(r io.Reader) *Segmenter { 40 return NewSegmenter(r) 41} 42 43// NewWordSegmenterDirect returns a new Segmenter to work directly with buf. 44func NewWordSegmenterDirect(buf []byte) *Segmenter { 45 return NewSegmenterDirect(buf) 46} 47 48func SplitWords(data []byte, atEOF bool) (int, []byte, error) { 49 advance, token, _, err := SegmentWords(data, atEOF) 50 return advance, token, err 51} 52 53func SegmentWords(data []byte, atEOF bool) (int, []byte, int, error) { 54 vals := make([][]byte, 0, 1) 55 types := make([]int, 0, 1) 56 tokens, types, advance, err := segmentWords(data, 1, atEOF, vals, types) 57 if len(tokens) > 0 { 58 return advance, tokens[0], types[0], err 59 } 60 return advance, nil, 0, err 61} 62 63func SegmentWordsDirect(data []byte, val [][]byte, types []int) ([][]byte, []int, int, error) { 64 return segmentWords(data, -1, true, val, types) 65} 66 67// *** Core Segmenter 68 69const maxConsecutiveEmptyReads = 100 70 71// NewSegmenter returns a new Segmenter to read from r. 72// Defaults to segment using SegmentWords 73func NewSegmenter(r io.Reader) *Segmenter { 74 return &Segmenter{ 75 r: r, 76 segment: SegmentWords, 77 maxTokenSize: MaxScanTokenSize, 78 buf: make([]byte, 4096), // Plausible starting size; needn't be large. 79 } 80} 81 82// NewSegmenterDirect returns a new Segmenter to work directly with buf. 83// Defaults to segment using SegmentWords 84func NewSegmenterDirect(buf []byte) *Segmenter { 85 return &Segmenter{ 86 segment: SegmentWords, 87 maxTokenSize: MaxScanTokenSize, 88 buf: buf, 89 start: 0, 90 end: len(buf), 91 err: io.EOF, 92 } 93} 94 95// Segmenter provides a convenient interface for reading data such as 96// a file of newline-delimited lines of text. Successive calls to 97// the Segment method will step through the 'tokens' of a file, skipping 98// the bytes between the tokens. The specification of a token is 99// defined by a split function of type SplitFunc; the default split 100// function breaks the input into lines with line termination stripped. Split 101// functions are defined in this package for scanning a file into 102// lines, bytes, UTF-8-encoded runes, and space-delimited words. The 103// client may instead provide a custom split function. 104// 105// Segmenting stops unrecoverably at EOF, the first I/O error, or a token too 106// large to fit in the buffer. When a scan stops, the reader may have 107// advanced arbitrarily far past the last token. Programs that need more 108// control over error handling or large tokens, or must run sequential scans 109// on a reader, should use bufio.Reader instead. 110// 111type Segmenter struct { 112 r io.Reader // The reader provided by the client. 113 segment SegmentFunc // The function to split the tokens. 114 maxTokenSize int // Maximum size of a token; modified by tests. 115 token []byte // Last token returned by split. 116 buf []byte // Buffer used as argument to split. 117 start int // First non-processed byte in buf. 118 end int // End of data in buf. 119 typ int // The token type 120 err error // Sticky error. 121} 122 123// SegmentFunc is the signature of the segmenting function used to tokenize the 124// input. The arguments are an initial substring of the remaining unprocessed 125// data and a flag, atEOF, that reports whether the Reader has no more data 126// to give. The return values are the number of bytes to advance the input 127// and the next token to return to the user, plus an error, if any. If the 128// data does not yet hold a complete token, for instance if it has no newline 129// while scanning lines, SegmentFunc can return (0, nil, nil) to signal the 130// Segmenter to read more data into the slice and try again with a longer slice 131// starting at the same point in the input. 132// 133// If the returned error is non-nil, segmenting stops and the error 134// is returned to the client. 135// 136// The function is never called with an empty data slice unless atEOF 137// is true. If atEOF is true, however, data may be non-empty and, 138// as always, holds unprocessed text. 139type SegmentFunc func(data []byte, atEOF bool) (advance int, token []byte, segmentType int, err error) 140 141// Errors returned by Segmenter. 142var ( 143 ErrTooLong = errors.New("bufio.Segmenter: token too long") 144 ErrNegativeAdvance = errors.New("bufio.Segmenter: SplitFunc returns negative advance count") 145 ErrAdvanceTooFar = errors.New("bufio.Segmenter: SplitFunc returns advance count beyond input") 146) 147 148const ( 149 // Maximum size used to buffer a token. The actual maximum token size 150 // may be smaller as the buffer may need to include, for instance, a newline. 151 MaxScanTokenSize = 64 * 1024 152) 153 154// Err returns the first non-EOF error that was encountered by the Segmenter. 155func (s *Segmenter) Err() error { 156 if s.err == io.EOF { 157 return nil 158 } 159 return s.err 160} 161 162func (s *Segmenter) Type() int { 163 return s.typ 164} 165 166// Bytes returns the most recent token generated by a call to Segment. 167// The underlying array may point to data that will be overwritten 168// by a subsequent call to Segment. It does no allocation. 169func (s *Segmenter) Bytes() []byte { 170 return s.token 171} 172 173// Text returns the most recent token generated by a call to Segment 174// as a newly allocated string holding its bytes. 175func (s *Segmenter) Text() string { 176 return string(s.token) 177} 178 179// Segment advances the Segmenter to the next token, which will then be 180// available through the Bytes or Text method. It returns false when the 181// scan stops, either by reaching the end of the input or an error. 182// After Segment returns false, the Err method will return any error that 183// occurred during scanning, except that if it was io.EOF, Err 184// will return nil. 185func (s *Segmenter) Segment() bool { 186 // Loop until we have a token. 187 for { 188 // See if we can get a token with what we already have. 189 if s.end > s.start { 190 advance, token, typ, err := s.segment(s.buf[s.start:s.end], s.err != nil) 191 if err != nil { 192 s.setErr(err) 193 return false 194 } 195 s.typ = typ 196 if !s.advance(advance) { 197 return false 198 } 199 s.token = token 200 if token != nil { 201 return true 202 } 203 } 204 // We cannot generate a token with what we are holding. 205 // If we've already hit EOF or an I/O error, we are done. 206 if s.err != nil { 207 // Shut it down. 208 s.start = 0 209 s.end = 0 210 return false 211 } 212 // Must read more data. 213 // First, shift data to beginning of buffer if there's lots of empty space 214 // or space is needed. 215 if s.start > 0 && (s.end == len(s.buf) || s.start > len(s.buf)/2) { 216 copy(s.buf, s.buf[s.start:s.end]) 217 s.end -= s.start 218 s.start = 0 219 } 220 // Is the buffer full? If so, resize. 221 if s.end == len(s.buf) { 222 if len(s.buf) >= s.maxTokenSize { 223 s.setErr(ErrTooLong) 224 return false 225 } 226 newSize := len(s.buf) * 2 227 if newSize > s.maxTokenSize { 228 newSize = s.maxTokenSize 229 } 230 newBuf := make([]byte, newSize) 231 copy(newBuf, s.buf[s.start:s.end]) 232 s.buf = newBuf 233 s.end -= s.start 234 s.start = 0 235 continue 236 } 237 // Finally we can read some input. Make sure we don't get stuck with 238 // a misbehaving Reader. Officially we don't need to do this, but let's 239 // be extra careful: Segmenter is for safe, simple jobs. 240 for loop := 0; ; { 241 n, err := s.r.Read(s.buf[s.end:len(s.buf)]) 242 s.end += n 243 if err != nil { 244 s.setErr(err) 245 break 246 } 247 if n > 0 { 248 break 249 } 250 loop++ 251 if loop > maxConsecutiveEmptyReads { 252 s.setErr(io.ErrNoProgress) 253 break 254 } 255 } 256 } 257} 258 259// advance consumes n bytes of the buffer. It reports whether the advance was legal. 260func (s *Segmenter) advance(n int) bool { 261 if n < 0 { 262 s.setErr(ErrNegativeAdvance) 263 return false 264 } 265 if n > s.end-s.start { 266 s.setErr(ErrAdvanceTooFar) 267 return false 268 } 269 s.start += n 270 return true 271} 272 273// setErr records the first error encountered. 274func (s *Segmenter) setErr(err error) { 275 if s.err == nil || s.err == io.EOF { 276 s.err = err 277 } 278} 279 280// SetSegmenter sets the segment function for the Segmenter. If called, it must be 281// called before Segment. 282func (s *Segmenter) SetSegmenter(segmenter SegmentFunc) { 283 s.segment = segmenter 284} 285