1 /*=============================================================================
2 Boost.Wave: A Standard compliant C++ preprocessor library
3
4 A generic C++ lexer token definition
5
6 http://www.boost.org/
7
8 Copyright (c) 2001-2012 Hartmut Kaiser. Distributed under the Boost
9 Software License, Version 1.0. (See accompanying file
10 LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
11 =============================================================================*/
12
13 #if !defined(BOOST_SLEX_TOKEN_HPP_53A13BD2_FBAA_444B_9B8B_FCB225C2BBA8_INCLUDED)
14 #define BOOST_SLEX_TOKEN_HPP_53A13BD2_FBAA_444B_9B8B_FCB225C2BBA8_INCLUDED
15
16 #include <iomanip>
17 #include <ios>
18
19 #include <boost/wave/wave_config.hpp>
20 #include <boost/wave/token_ids.hpp>
21 #include <boost/wave/language_support.hpp>
22 #include <boost/wave/util/file_position.hpp>
23 #include <boost/optional.hpp>
24
25 ///////////////////////////////////////////////////////////////////////////////
26 namespace boost {
27 namespace wave {
28 namespace cpplexer {
29
30 ///////////////////////////////////////////////////////////////////////////////
31 // forward declaration of the token type
32 template <typename PositionT = boost::wave::util::file_position_type>
33 class slex_token;
34
35 ///////////////////////////////////////////////////////////////////////////////
36 //
37 // lex_token
38 //
39 ///////////////////////////////////////////////////////////////////////////////
40
41 template <typename PositionT>
42 class slex_token
43 {
44 public:
45 typedef BOOST_WAVE_STRINGTYPE string_type;
46 typedef PositionT position_type;
47
slex_token()48 slex_token()
49 : id(T_EOI)
50 {}
51
52 // construct an invalid token
slex_token(int)53 explicit slex_token(int)
54 : id(T_UNKNOWN)
55 {}
56
slex_token(token_id id_,string_type const & value_,PositionT const & pos_)57 slex_token(token_id id_, string_type const &value_, PositionT const &pos_)
58 : id(id_), value(value_), pos(pos_)
59 {}
60
61 // accessors
operator token_id() const62 operator token_id() const { return id; }
get_value() const63 string_type const &get_value() const { return value; }
get_position() const64 position_type const &get_position() const { return pos; }
get_expand_position() const65 position_type const &get_expand_position() const {
66 if (expand_pos)
67 return *expand_pos;
68 else
69 return pos;
70 }
is_eoi() const71 bool is_eoi() const { return id == T_EOI; }
is_valid() const72 bool is_valid() const { return id != T_UNKNOWN; }
73
set_token_id(token_id id_)74 void set_token_id (token_id id_) { id = id_; }
set_value(string_type const & newval)75 void set_value (string_type const &newval) { value = newval; }
set_position(position_type const & pos_)76 void set_position (position_type const &pos_) { pos = pos_; }
set_expand_position(position_type const & pos_)77 void set_expand_position (position_type const &pos_) { expand_pos = pos_; }
78
operator ==(slex_token const & lhs,slex_token const & rhs)79 friend bool operator== (slex_token const& lhs, slex_token const& rhs)
80 {
81 // two tokens are considered equal even if they contain different
82 // positions
83 return (lhs.id == rhs.id && lhs.value == rhs.value) ? true : false;
84 }
85
86 // debug support
87 #if BOOST_WAVE_DUMP_PARSE_TREE != 0
88 // access functions for the tree_to_xml functionality
get_token_id(slex_token const & t)89 static int get_token_id(slex_token const &t)
90 { return ID_FROM_TOKEN(token_id(t)); }
get_token_value(slex_token const & t)91 static string_type get_token_value(slex_token const &t)
92 { return t.get_value(); }
93 #endif
94
95 // print support
print(std::ostream & stream) const96 void print (std::ostream &stream) const
97 {
98 using namespace std;
99 using namespace boost::wave;
100
101 stream << std::setw(16)
102 << std::left << boost::wave::get_token_name(id) << " ("
103 << "#" << token_id(BASEID_FROM_TOKEN(*this))
104 << ") at " << get_position().get_file() << " ("
105 << std::setw(3) << std::right << get_position().get_line() << "/"
106 << std::setw(2) << std::right << get_position().get_column()
107 << "): >";
108
109 for (std::size_t i = 0; i < value.size(); ++i) {
110 switch (value[i]) {
111 case '\r': stream << "\\r"; break;
112 case '\n': stream << "\\n"; break;
113 case '\t': stream << "\\t"; break;
114 default:
115 stream << value[i];
116 break;
117 }
118 }
119 stream << "<";
120 }
121
122 private:
123 boost::wave::token_id id; // the token id
124 string_type value; // the text, which was parsed into this token
125 PositionT pos; // the original file position
126 boost::optional<PositionT> expand_pos;
127 };
128
129 template <typename PositionT>
130 inline std::ostream &
operator <<(std::ostream & stream,slex_token<PositionT> const & object)131 operator<< (std::ostream &stream, slex_token<PositionT> const &object)
132 {
133 object.print(stream);
134 return stream;
135 }
136
137 ///////////////////////////////////////////////////////////////////////////////
138 // This overload is needed by the multi_pass/functor_input_policy to
139 // validate a token instance. It has to be defined in the same namespace
140 // as the token class itself to allow ADL to find it.
141 ///////////////////////////////////////////////////////////////////////////////
142 template <typename Position>
143 inline bool
token_is_valid(slex_token<Position> const & t)144 token_is_valid(slex_token<Position> const& t)
145 {
146 return t.is_valid();
147 }
148
149 ///////////////////////////////////////////////////////////////////////////////
150 } // namespace cpplexer
151 } // namespace wave
152 } // namespace boost
153
154 #endif // !defined(BOOST_SLEX_TOKEN_HPP_53A13BD2_FBAA_444B_9B8B_FCB225C2BBA8_INCLUDED)
155