1 /////////////////////////////////////////////////////////////////////////////
2 // Copyright (c) 2009-2014 Alan Wright. All rights reserved.
3 // Distributable under the terms of either the Apache License (Version 2.0)
4 // or the GNU Lesser General Public License.
5 /////////////////////////////////////////////////////////////////////////////
6 
7 #include "LuceneInc.h"
8 #include "Tokenizer.h"
9 #include "CharReader.h"
10 
11 namespace Lucene {
12 
Tokenizer()13 Tokenizer::Tokenizer() {
14 }
15 
Tokenizer(const ReaderPtr & input)16 Tokenizer::Tokenizer(const ReaderPtr& input) {
17     this->input = CharReader::get(input);
18     this->charStream = boost::dynamic_pointer_cast<CharStream>(this->input);
19 }
20 
Tokenizer(const AttributeFactoryPtr & factory)21 Tokenizer::Tokenizer(const AttributeFactoryPtr& factory) : TokenStream(factory) {
22 }
23 
Tokenizer(const AttributeFactoryPtr & factory,const ReaderPtr & input)24 Tokenizer::Tokenizer(const AttributeFactoryPtr& factory, const ReaderPtr& input) : TokenStream(factory) {
25     this->input = CharReader::get(input);
26     this->charStream = boost::dynamic_pointer_cast<CharStream>(this->input);
27 }
28 
Tokenizer(const AttributeSourcePtr & source)29 Tokenizer::Tokenizer(const AttributeSourcePtr& source) : TokenStream(source) {
30 }
31 
Tokenizer(const AttributeSourcePtr & source,const ReaderPtr & input)32 Tokenizer::Tokenizer(const AttributeSourcePtr& source, const ReaderPtr& input) : TokenStream(source) {
33     this->input = CharReader::get(input);
34     this->charStream = boost::dynamic_pointer_cast<CharStream>(this->input);
35 }
36 
~Tokenizer()37 Tokenizer::~Tokenizer() {
38 }
39 
close()40 void Tokenizer::close() {
41     if (input) {
42         input->close();
43         input.reset(); // don't hold onto Reader after close
44     }
45 }
46 
correctOffset(int32_t currentOff)47 int32_t Tokenizer::correctOffset(int32_t currentOff) {
48     return charStream ? charStream->correctOffset(currentOff) : currentOff;
49 }
50 
reset(const ReaderPtr & input)51 void Tokenizer::reset(const ReaderPtr& input) {
52     this->input = input;
53 }
54 
55 }
56