1 ///////////////////////////////////////////////////////////////////////////// 2 // Copyright (c) 2009-2014 Alan Wright. All rights reserved. 3 // Distributable under the terms of either the Apache License (Version 2.0) 4 // or the GNU Lesser General Public License. 5 ///////////////////////////////////////////////////////////////////////////// 6 7 #include "LuceneInc.h" 8 #include "Tokenizer.h" 9 #include "CharReader.h" 10 11 namespace Lucene { 12 Tokenizer()13Tokenizer::Tokenizer() { 14 } 15 Tokenizer(const ReaderPtr & input)16Tokenizer::Tokenizer(const ReaderPtr& input) { 17 this->input = CharReader::get(input); 18 this->charStream = boost::dynamic_pointer_cast<CharStream>(this->input); 19 } 20 Tokenizer(const AttributeFactoryPtr & factory)21Tokenizer::Tokenizer(const AttributeFactoryPtr& factory) : TokenStream(factory) { 22 } 23 Tokenizer(const AttributeFactoryPtr & factory,const ReaderPtr & input)24Tokenizer::Tokenizer(const AttributeFactoryPtr& factory, const ReaderPtr& input) : TokenStream(factory) { 25 this->input = CharReader::get(input); 26 this->charStream = boost::dynamic_pointer_cast<CharStream>(this->input); 27 } 28 Tokenizer(const AttributeSourcePtr & source)29Tokenizer::Tokenizer(const AttributeSourcePtr& source) : TokenStream(source) { 30 } 31 Tokenizer(const AttributeSourcePtr & source,const ReaderPtr & input)32Tokenizer::Tokenizer(const AttributeSourcePtr& source, const ReaderPtr& input) : TokenStream(source) { 33 this->input = CharReader::get(input); 34 this->charStream = boost::dynamic_pointer_cast<CharStream>(this->input); 35 } 36 ~Tokenizer()37Tokenizer::~Tokenizer() { 38 } 39 close()40void Tokenizer::close() { 41 if (input) { 42 input->close(); 43 input.reset(); // don't hold onto Reader after close 44 } 45 } 46 correctOffset(int32_t currentOff)47int32_t Tokenizer::correctOffset(int32_t currentOff) { 48 return charStream ? charStream->correctOffset(currentOff) : currentOff; 49 } 50 reset(const ReaderPtr & input)51void Tokenizer::reset(const ReaderPtr& input) { 52 this->input = input; 53 } 54 55 } 56