1 /*------------------------------------------------------------------------------
2 * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
3 *
4 * Distributable under the terms of either the Apache License (Version 2.0) or
5 * the GNU Lesser General Public License, as specified in the COPYING file.
6 ------------------------------------------------------------------------------*/
7 #include "CLucene/StdHeader.h"
8 #include "StandardAnalyzer.h"
9
10 #include "CLucene/util/VoidMap.h"
11 #include "CLucene/util/Reader.h"
12 #include "CLucene/analysis/AnalysisHeader.h"
13 #include "CLucene/analysis/Analyzers.h"
14 #include "StandardFilter.h"
15 #include "StandardTokenizer.h"
16
17 CL_NS_USE(util)
CL_NS_USE(analysis)18 CL_NS_USE(analysis)
19
20 CL_NS_DEF2(analysis,standard)
21
22 StandardAnalyzer::StandardAnalyzer():
23 stopSet(false)
24 {
25 StopFilter::fillStopTable( &stopSet,CL_NS(analysis)::StopAnalyzer::ENGLISH_STOP_WORDS);
26 }
27
StandardAnalyzer(const TCHAR ** stopWords)28 StandardAnalyzer::StandardAnalyzer( const TCHAR** stopWords):
29 stopSet(false)
30 {
31 StopFilter::fillStopTable( &stopSet,stopWords );
32 }
33
~StandardAnalyzer()34 StandardAnalyzer::~StandardAnalyzer(){
35 }
36
37
tokenStream(const TCHAR * fieldName,Reader * reader)38 TokenStream* StandardAnalyzer::tokenStream(const TCHAR* fieldName, Reader* reader)
39 {
40 TokenStream* ret = _CLNEW StandardTokenizer(reader);
41 ret = _CLNEW StandardFilter(ret,true);
42 ret = _CLNEW LowerCaseFilter(ret,true);
43 ret = _CLNEW StopFilter(ret,true, &stopSet);
44 return ret;
45 }
46 CL_NS_END2
47