1 // (C) Copyright 2016 Jethro G. Beekman
2 //
3 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
4 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
5 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
6 // option. This file may not be copied, modified, or distributed
7 // except according to those terms.
8 //! Representation of a C token
9 //!
10 //! This is designed to map onto a libclang CXToken.
11
12 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
13 #[allow(missing_docs)]
14 pub enum Kind {
15 Punctuation,
16 Keyword,
17 Identifier,
18 Literal,
19 Comment,
20 }
21
22 /// A single token in a C expression.
23 #[derive(Debug, Clone, PartialEq, Eq)]
24 pub struct Token {
25 /// The type of this token.
26 pub kind: Kind,
27 /// The bytes that make up the token.
28 pub raw: Box<[u8]>,
29 }
30
31 impl<'a> From<(Kind, &'a [u8])> for Token {
from((kind, value): (Kind, &'a [u8])) -> Token32 fn from((kind, value): (Kind, &'a [u8])) -> Token {
33 Token {
34 kind,
35 raw: value.to_owned().into_boxed_slice(),
36 }
37 }
38 }
39
40 /// Remove all comment tokens from a vector of tokens
remove_comments(v: &mut Vec<Token>) -> &mut Vec<Token>41 pub fn remove_comments(v: &mut Vec<Token>) -> &mut Vec<Token> {
42 v.retain(|t| t.kind != Kind::Comment);
43 v
44 }
45