1 // Boost tokenizer.hpp  -----------------------------------------------------//
2 
3 // (c) Copyright Jeremy Siek and John R. Bandela 2001.
4 
5 // Distributed under the Boost Software License, Version 1.0. (See
6 // accompanying file LICENSE_1_0.txt or copy at
7 // http://www.boost.org/LICENSE_1_0.txt)
8 
9 // See http://www.boost.org/libs/tokenizer for documenation
10 
11 // Revision History:
12 // 03 Jul 2003   John Bandela
13 //      Converted to new iterator adapter
14 // 02 Feb 2002   Jeremy Siek
15 //      Removed tabs and a little cleanup.
16 
17 #ifndef BOOST_TOKENIZER_JRB070303_HPP_
18 #define BOOST_TOKENIZER_JRB070303_HPP_
19 
20 #include <boost/token_iterator.hpp>
21 
22 namespace boost {
23 
24 
25   //===========================================================================
26   // A container-view of a tokenized "sequence"
27   template <
28     typename TokenizerFunc = char_delimiters_separator<char>,
29     typename Iterator = std::string::const_iterator,
30     typename Type = std::string
31   >
32   class tokenizer {
33   private:
34     typedef token_iterator_generator<TokenizerFunc,Iterator,Type> TGen;
35 
36     // It seems that MSVC does not like the unqualified use of iterator,
37     // Thus we use iter internally when it is used unqualified and
38     // the users of this class will always qualify iterator.
39     typedef typename TGen::type iter;
40 
41   public:
42 
43     typedef iter iterator;
44     typedef iter const_iterator;
45     typedef Type value_type;
46     typedef value_type& reference;
47     typedef const value_type& const_reference;
48     typedef value_type* pointer;
49     typedef const pointer const_pointer;
50     typedef void size_type;
51     typedef void difference_type;
52 
tokenizer(Iterator first,Iterator last,const TokenizerFunc & f=TokenizerFunc ())53     tokenizer(Iterator first, Iterator last,
54               const TokenizerFunc& f = TokenizerFunc())
55       : first_(first), last_(last), f_(f) { }
56 
57     template <typename Container>
tokenizer(const Container & c)58     tokenizer(const Container& c)
59       : first_(c.begin()), last_(c.end()), f_() { }
60 
61     template <typename Container>
tokenizer(const Container & c,const TokenizerFunc & f)62     tokenizer(const Container& c,const TokenizerFunc& f)
63       : first_(c.begin()), last_(c.end()), f_(f) { }
64 
assign(Iterator first,Iterator last)65     void assign(Iterator first, Iterator last){
66       first_ = first;
67       last_ = last;
68     }
69 
assign(Iterator first,Iterator last,const TokenizerFunc & f)70     void assign(Iterator first, Iterator last, const TokenizerFunc& f){
71       assign(first,last);
72       f_ = f;
73     }
74 
75     template <typename Container>
assign(const Container & c)76     void assign(const Container& c){
77       assign(c.begin(),c.end());
78     }
79 
80 
81     template <typename Container>
assign(const Container & c,const TokenizerFunc & f)82     void assign(const Container& c, const TokenizerFunc& f){
83       assign(c.begin(),c.end(),f);
84     }
85 
begin() const86     iter begin() const { return iter(f_,first_,last_); }
end() const87     iter end() const { return iter(f_,last_,last_); }
88 
89   private:
90     Iterator first_;
91     Iterator last_;
92     TokenizerFunc f_;
93   };
94 
95 
96 } // namespace boost
97 
98 #endif
99