1{-# LANGUAGE CPP, DeriveDataTypeable #-} 2----------------------------------------------------------------------------- 3-- | 4-- Module : Language.Python.Common.Token 5-- Copyright : (c) 2009 Bernie Pope 6-- License : BSD-style 7-- Maintainer : bjpop@csse.unimelb.edu.au 8-- Stability : experimental 9-- Portability : ghc 10-- 11-- Lexical tokens for the Python lexer. Contains the superset of tokens from 12-- version 2 and version 3 of Python (they are mostly the same). 13----------------------------------------------------------------------------- 14 15module Language.JavaScript.Parser.Token 16 ( 17 -- * The tokens 18 Token (..) 19 , CommentAnnotation (..) 20 -- * String conversion 21 , debugTokenString 22 -- * Classification 23 -- TokenClass (..), 24 ) where 25 26import Data.Data 27import Language.JavaScript.Parser.SrcLocation 28 29data CommentAnnotation 30 = CommentA TokenPosn String 31 | WhiteSpace TokenPosn String 32 | NoComment 33 deriving (Eq, Show, Typeable, Data, Read) 34 35-- | Lexical tokens. 36-- Each may be annotated with any comment occurring between the prior token and this one 37data Token 38 -- Comment 39 = CommentToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } -- ^ Single line comment. 40 | WsToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } -- ^ White space, for preservation. 41 42 -- Identifiers 43 | IdentifierToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } -- ^ Identifier. 44 45 -- Javascript Literals 46 47 | DecimalToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 48 -- ^ Literal: Decimal 49 | HexIntegerToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 50 -- ^ Literal: Hexadecimal Integer 51 | OctalToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 52 -- ^ Literal: Octal Integer 53 | StringToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 54 -- ^ Literal: string, delimited by either single or double quotes 55 | RegExToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 56 -- ^ Literal: Regular Expression 57 58 -- Keywords 59 | AsyncToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 60 | AwaitToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 61 | BreakToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 62 | CaseToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 63 | CatchToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 64 | ClassToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 65 | ConstToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 66 | LetToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 67 | ContinueToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 68 | DebuggerToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 69 | DefaultToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 70 | DeleteToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 71 | DoToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 72 | ElseToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 73 | EnumToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 74 | ExtendsToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 75 | FalseToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 76 | FinallyToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 77 | ForToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 78 | FunctionToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 79 | FromToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 80 | IfToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 81 | InToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 82 | InstanceofToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 83 | NewToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 84 | NullToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 85 | OfToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 86 | ReturnToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 87 | StaticToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 88 | SuperToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 89 | SwitchToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 90 | ThisToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 91 | ThrowToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 92 | TrueToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 93 | TryToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 94 | TypeofToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 95 | VarToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 96 | VoidToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 97 | WhileToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 98 | YieldToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 99 | ImportToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 100 | WithToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 101 | ExportToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 102 -- Future reserved words 103 | FutureToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 104 -- Needed, not sure what they are though. 105 | GetToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 106 | SetToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 107 108 -- Delimiters 109 -- Operators 110 | AutoSemiToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 111 | SemiColonToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 112 | CommaToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 113 | HookToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 114 | ColonToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 115 | OrToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 116 | AndToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 117 | BitwiseOrToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 118 | BitwiseXorToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 119 | BitwiseAndToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 120 | StrictEqToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 121 | EqToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 122 | TimesAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 123 | DivideAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 124 | ModAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 125 | PlusAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 126 | MinusAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 127 | LshAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 128 | RshAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 129 | UrshAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 130 | AndAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 131 | XorAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 132 | OrAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 133 | SimpleAssignToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 134 | StrictNeToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 135 | NeToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 136 | LshToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 137 | LeToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 138 | LtToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 139 | UrshToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 140 | RshToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 141 | GeToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 142 | GtToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 143 | IncrementToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 144 | DecrementToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 145 | PlusToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 146 | MinusToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 147 | MulToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 148 | DivToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 149 | ModToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 150 | NotToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 151 | BitwiseNotToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 152 | ArrowToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 153 | SpreadToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 154 | DotToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 155 | LeftBracketToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 156 | RightBracketToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 157 | LeftCurlyToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 158 | RightCurlyToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 159 | LeftParenToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 160 | RightParenToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 161 | CondcommentEndToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } 162 163 -- Template literal lexical components 164 | NoSubstitutionTemplateToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 165 | TemplateHeadToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 166 | TemplateMiddleToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 167 | TemplateTailToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 168 169 -- Special cases 170 | AsToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] } 171 | TailToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } -- ^ Stuff between last JS and EOF 172 | EOFToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation] } -- ^ End of file 173 deriving (Eq, Show, Typeable) 174 175 176-- | Produce a string from a token containing detailed information. Mainly intended for debugging. 177debugTokenString :: Token -> String 178debugTokenString = takeWhile (/= ' ') . show 179