1*760c2415Smrg
2*760c2415Smrg /* Compiler implementation of the D programming language
3*760c2415Smrg * Copyright (C) 1999-2019 by The D Language Foundation, All Rights Reserved
4*760c2415Smrg * written by Walter Bright
5*760c2415Smrg * http://www.digitalmars.com
6*760c2415Smrg * Distributed under the Boost Software License, Version 1.0.
7*760c2415Smrg * http://www.boost.org/LICENSE_1_0.txt
8*760c2415Smrg * https://github.com/D-Programming-Language/dmd/blob/master/src/lexer.c
9*760c2415Smrg */
10*760c2415Smrg
11*760c2415Smrg #include "root/dsystem.h"
12*760c2415Smrg
13*760c2415Smrg #include "tokens.h"
14*760c2415Smrg #include "root/rmem.h"
15*760c2415Smrg #include "root/outbuffer.h"
16*760c2415Smrg #include "id.h"
17*760c2415Smrg #include "identifier.h"
18*760c2415Smrg #include "utf.h"
19*760c2415Smrg
20*760c2415Smrg /************************* Token **********************************************/
21*760c2415Smrg
22*760c2415Smrg Token *Token::freelist = NULL;
23*760c2415Smrg
24*760c2415Smrg const char *Token::tochars[TOKMAX];
25*760c2415Smrg
alloc()26*760c2415Smrg Token *Token::alloc()
27*760c2415Smrg {
28*760c2415Smrg if (Token::freelist)
29*760c2415Smrg {
30*760c2415Smrg Token *t = freelist;
31*760c2415Smrg freelist = t->next;
32*760c2415Smrg t->next = NULL;
33*760c2415Smrg return t;
34*760c2415Smrg }
35*760c2415Smrg
36*760c2415Smrg return new Token();
37*760c2415Smrg }
38*760c2415Smrg
free()39*760c2415Smrg void Token::free()
40*760c2415Smrg {
41*760c2415Smrg next = freelist;
42*760c2415Smrg freelist = this;
43*760c2415Smrg }
44*760c2415Smrg
toChars()45*760c2415Smrg const char *Token::toChars() const
46*760c2415Smrg {
47*760c2415Smrg static char buffer[3 + 3 * sizeof(floatvalue) + 1];
48*760c2415Smrg
49*760c2415Smrg const char *p = &buffer[0];
50*760c2415Smrg switch (value)
51*760c2415Smrg {
52*760c2415Smrg case TOKint32v:
53*760c2415Smrg sprintf(&buffer[0],"%d",(d_int32)int64value);
54*760c2415Smrg break;
55*760c2415Smrg
56*760c2415Smrg case TOKuns32v:
57*760c2415Smrg case TOKcharv:
58*760c2415Smrg case TOKwcharv:
59*760c2415Smrg case TOKdcharv:
60*760c2415Smrg sprintf(&buffer[0],"%uU",(d_uns32)uns64value);
61*760c2415Smrg break;
62*760c2415Smrg
63*760c2415Smrg case TOKint64v:
64*760c2415Smrg sprintf(&buffer[0],"%lldL",(longlong)int64value);
65*760c2415Smrg break;
66*760c2415Smrg
67*760c2415Smrg case TOKuns64v:
68*760c2415Smrg sprintf(&buffer[0],"%lluUL",(ulonglong)uns64value);
69*760c2415Smrg break;
70*760c2415Smrg
71*760c2415Smrg case TOKfloat32v:
72*760c2415Smrg CTFloat::sprint(&buffer[0], 'g', floatvalue);
73*760c2415Smrg strcat(&buffer[0], "f");
74*760c2415Smrg break;
75*760c2415Smrg
76*760c2415Smrg case TOKfloat64v:
77*760c2415Smrg CTFloat::sprint(&buffer[0], 'g', floatvalue);
78*760c2415Smrg break;
79*760c2415Smrg
80*760c2415Smrg case TOKfloat80v:
81*760c2415Smrg CTFloat::sprint(&buffer[0], 'g', floatvalue);
82*760c2415Smrg strcat(&buffer[0], "L");
83*760c2415Smrg break;
84*760c2415Smrg
85*760c2415Smrg case TOKimaginary32v:
86*760c2415Smrg CTFloat::sprint(&buffer[0], 'g', floatvalue);
87*760c2415Smrg strcat(&buffer[0], "fi");
88*760c2415Smrg break;
89*760c2415Smrg
90*760c2415Smrg case TOKimaginary64v:
91*760c2415Smrg CTFloat::sprint(&buffer[0], 'g', floatvalue);
92*760c2415Smrg strcat(&buffer[0], "i");
93*760c2415Smrg break;
94*760c2415Smrg
95*760c2415Smrg case TOKimaginary80v:
96*760c2415Smrg CTFloat::sprint(&buffer[0], 'g', floatvalue);
97*760c2415Smrg strcat(&buffer[0], "Li");
98*760c2415Smrg break;
99*760c2415Smrg
100*760c2415Smrg case TOKstring:
101*760c2415Smrg {
102*760c2415Smrg OutBuffer buf;
103*760c2415Smrg buf.writeByte('"');
104*760c2415Smrg for (size_t i = 0; i < len; )
105*760c2415Smrg {
106*760c2415Smrg unsigned c;
107*760c2415Smrg utf_decodeChar((utf8_t *)ustring, len, &i, &c);
108*760c2415Smrg switch (c)
109*760c2415Smrg {
110*760c2415Smrg case 0:
111*760c2415Smrg break;
112*760c2415Smrg
113*760c2415Smrg case '"':
114*760c2415Smrg case '\\':
115*760c2415Smrg buf.writeByte('\\');
116*760c2415Smrg /* fall through */
117*760c2415Smrg default:
118*760c2415Smrg if (c <= 0x7F)
119*760c2415Smrg {
120*760c2415Smrg if (isprint(c))
121*760c2415Smrg buf.writeByte(c);
122*760c2415Smrg else
123*760c2415Smrg buf.printf("\\x%02x", c);
124*760c2415Smrg }
125*760c2415Smrg else if (c <= 0xFFFF)
126*760c2415Smrg buf.printf("\\u%04x", c);
127*760c2415Smrg else
128*760c2415Smrg buf.printf("\\U%08x", c);
129*760c2415Smrg continue;
130*760c2415Smrg }
131*760c2415Smrg break;
132*760c2415Smrg }
133*760c2415Smrg buf.writeByte('"');
134*760c2415Smrg if (postfix)
135*760c2415Smrg buf.writeByte(postfix);
136*760c2415Smrg p = buf.extractString();
137*760c2415Smrg }
138*760c2415Smrg break;
139*760c2415Smrg
140*760c2415Smrg case TOKxstring:
141*760c2415Smrg {
142*760c2415Smrg OutBuffer buf;
143*760c2415Smrg buf.writeByte('x');
144*760c2415Smrg buf.writeByte('"');
145*760c2415Smrg for (size_t i = 0; i < len; i++)
146*760c2415Smrg {
147*760c2415Smrg if (i)
148*760c2415Smrg buf.writeByte(' ');
149*760c2415Smrg buf.printf("%02x", ustring[i]);
150*760c2415Smrg }
151*760c2415Smrg buf.writeByte('"');
152*760c2415Smrg if (postfix)
153*760c2415Smrg buf.writeByte(postfix);
154*760c2415Smrg buf.writeByte(0);
155*760c2415Smrg p = (char *)buf.extractData();
156*760c2415Smrg break;
157*760c2415Smrg }
158*760c2415Smrg
159*760c2415Smrg case TOKidentifier:
160*760c2415Smrg case TOKenum:
161*760c2415Smrg case TOKstruct:
162*760c2415Smrg case TOKimport:
163*760c2415Smrg case TOKwchar: case TOKdchar:
164*760c2415Smrg case TOKbool: case TOKchar:
165*760c2415Smrg case TOKint8: case TOKuns8:
166*760c2415Smrg case TOKint16: case TOKuns16:
167*760c2415Smrg case TOKint32: case TOKuns32:
168*760c2415Smrg case TOKint64: case TOKuns64:
169*760c2415Smrg case TOKint128: case TOKuns128:
170*760c2415Smrg case TOKfloat32: case TOKfloat64: case TOKfloat80:
171*760c2415Smrg case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:
172*760c2415Smrg case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:
173*760c2415Smrg case TOKvoid:
174*760c2415Smrg p = ident->toChars();
175*760c2415Smrg break;
176*760c2415Smrg
177*760c2415Smrg default:
178*760c2415Smrg p = toChars(value);
179*760c2415Smrg break;
180*760c2415Smrg }
181*760c2415Smrg return p;
182*760c2415Smrg }
183*760c2415Smrg
toChars(TOK value)184*760c2415Smrg const char *Token::toChars(TOK value)
185*760c2415Smrg {
186*760c2415Smrg static char buffer[3 + 3 * sizeof(value) + 1];
187*760c2415Smrg
188*760c2415Smrg const char *p = tochars[value];
189*760c2415Smrg if (!p)
190*760c2415Smrg {
191*760c2415Smrg sprintf(&buffer[0],"TOK%d",value);
192*760c2415Smrg p = &buffer[0];
193*760c2415Smrg }
194*760c2415Smrg return p;
195*760c2415Smrg }
196*760c2415Smrg
197*760c2415Smrg /****************************************
198*760c2415Smrg */
199*760c2415Smrg
200*760c2415Smrg struct Keyword
201*760c2415Smrg {
202*760c2415Smrg const char *name;
203*760c2415Smrg TOK value;
204*760c2415Smrg };
205*760c2415Smrg
206*760c2415Smrg static size_t nkeywords;
207*760c2415Smrg static Keyword keywords[] =
208*760c2415Smrg {
209*760c2415Smrg { "this", TOKthis },
210*760c2415Smrg { "super", TOKsuper },
211*760c2415Smrg { "assert", TOKassert },
212*760c2415Smrg { "null", TOKnull },
213*760c2415Smrg { "true", TOKtrue },
214*760c2415Smrg { "false", TOKfalse },
215*760c2415Smrg { "cast", TOKcast },
216*760c2415Smrg { "new", TOKnew },
217*760c2415Smrg { "delete", TOKdelete },
218*760c2415Smrg { "throw", TOKthrow },
219*760c2415Smrg { "module", TOKmodule },
220*760c2415Smrg { "pragma", TOKpragma },
221*760c2415Smrg { "typeof", TOKtypeof },
222*760c2415Smrg { "typeid", TOKtypeid },
223*760c2415Smrg
224*760c2415Smrg { "template", TOKtemplate },
225*760c2415Smrg
226*760c2415Smrg { "void", TOKvoid },
227*760c2415Smrg { "byte", TOKint8 },
228*760c2415Smrg { "ubyte", TOKuns8 },
229*760c2415Smrg { "short", TOKint16 },
230*760c2415Smrg { "ushort", TOKuns16 },
231*760c2415Smrg { "int", TOKint32 },
232*760c2415Smrg { "uint", TOKuns32 },
233*760c2415Smrg { "long", TOKint64 },
234*760c2415Smrg { "ulong", TOKuns64 },
235*760c2415Smrg { "cent", TOKint128, },
236*760c2415Smrg { "ucent", TOKuns128, },
237*760c2415Smrg { "float", TOKfloat32 },
238*760c2415Smrg { "double", TOKfloat64 },
239*760c2415Smrg { "real", TOKfloat80 },
240*760c2415Smrg
241*760c2415Smrg { "bool", TOKbool },
242*760c2415Smrg { "char", TOKchar },
243*760c2415Smrg { "wchar", TOKwchar },
244*760c2415Smrg { "dchar", TOKdchar },
245*760c2415Smrg
246*760c2415Smrg { "ifloat", TOKimaginary32 },
247*760c2415Smrg { "idouble", TOKimaginary64 },
248*760c2415Smrg { "ireal", TOKimaginary80 },
249*760c2415Smrg
250*760c2415Smrg { "cfloat", TOKcomplex32 },
251*760c2415Smrg { "cdouble", TOKcomplex64 },
252*760c2415Smrg { "creal", TOKcomplex80 },
253*760c2415Smrg
254*760c2415Smrg { "delegate", TOKdelegate },
255*760c2415Smrg { "function", TOKfunction },
256*760c2415Smrg
257*760c2415Smrg { "is", TOKis },
258*760c2415Smrg { "if", TOKif },
259*760c2415Smrg { "else", TOKelse },
260*760c2415Smrg { "while", TOKwhile },
261*760c2415Smrg { "for", TOKfor },
262*760c2415Smrg { "do", TOKdo },
263*760c2415Smrg { "switch", TOKswitch },
264*760c2415Smrg { "case", TOKcase },
265*760c2415Smrg { "default", TOKdefault },
266*760c2415Smrg { "break", TOKbreak },
267*760c2415Smrg { "continue", TOKcontinue },
268*760c2415Smrg { "synchronized", TOKsynchronized },
269*760c2415Smrg { "return", TOKreturn },
270*760c2415Smrg { "goto", TOKgoto },
271*760c2415Smrg { "try", TOKtry },
272*760c2415Smrg { "catch", TOKcatch },
273*760c2415Smrg { "finally", TOKfinally },
274*760c2415Smrg { "with", TOKwith },
275*760c2415Smrg { "asm", TOKasm },
276*760c2415Smrg { "foreach", TOKforeach },
277*760c2415Smrg { "foreach_reverse", TOKforeach_reverse },
278*760c2415Smrg { "scope", TOKscope },
279*760c2415Smrg
280*760c2415Smrg { "struct", TOKstruct },
281*760c2415Smrg { "class", TOKclass },
282*760c2415Smrg { "interface", TOKinterface },
283*760c2415Smrg { "union", TOKunion },
284*760c2415Smrg { "enum", TOKenum },
285*760c2415Smrg { "import", TOKimport },
286*760c2415Smrg { "mixin", TOKmixin },
287*760c2415Smrg { "static", TOKstatic },
288*760c2415Smrg { "final", TOKfinal },
289*760c2415Smrg { "const", TOKconst },
290*760c2415Smrg { "alias", TOKalias },
291*760c2415Smrg { "override", TOKoverride },
292*760c2415Smrg { "abstract", TOKabstract },
293*760c2415Smrg { "debug", TOKdebug },
294*760c2415Smrg { "deprecated", TOKdeprecated },
295*760c2415Smrg { "in", TOKin },
296*760c2415Smrg { "out", TOKout },
297*760c2415Smrg { "inout", TOKinout },
298*760c2415Smrg { "lazy", TOKlazy },
299*760c2415Smrg { "auto", TOKauto },
300*760c2415Smrg
301*760c2415Smrg { "align", TOKalign },
302*760c2415Smrg { "extern", TOKextern },
303*760c2415Smrg { "private", TOKprivate },
304*760c2415Smrg { "package", TOKpackage },
305*760c2415Smrg { "protected", TOKprotected },
306*760c2415Smrg { "public", TOKpublic },
307*760c2415Smrg { "export", TOKexport },
308*760c2415Smrg
309*760c2415Smrg { "invariant", TOKinvariant },
310*760c2415Smrg { "unittest", TOKunittest },
311*760c2415Smrg { "version", TOKversion },
312*760c2415Smrg
313*760c2415Smrg { "__argTypes", TOKargTypes },
314*760c2415Smrg { "__parameters", TOKparameters },
315*760c2415Smrg { "ref", TOKref },
316*760c2415Smrg { "macro", TOKmacro },
317*760c2415Smrg
318*760c2415Smrg { "pure", TOKpure },
319*760c2415Smrg { "nothrow", TOKnothrow },
320*760c2415Smrg { "__gshared", TOKgshared },
321*760c2415Smrg { "__traits", TOKtraits },
322*760c2415Smrg { "__vector", TOKvector },
323*760c2415Smrg { "__overloadset", TOKoverloadset },
324*760c2415Smrg { "__FILE__", TOKfile },
325*760c2415Smrg { "__FILE_FULL_PATH__", TOKfilefullpath },
326*760c2415Smrg { "__LINE__", TOKline },
327*760c2415Smrg { "__MODULE__", TOKmodulestring },
328*760c2415Smrg { "__FUNCTION__", TOKfuncstring },
329*760c2415Smrg { "__PRETTY_FUNCTION__", TOKprettyfunc },
330*760c2415Smrg { "shared", TOKshared },
331*760c2415Smrg { "immutable", TOKimmutable },
332*760c2415Smrg { NULL, TOKreserved }
333*760c2415Smrg };
334*760c2415Smrg
isKeyword()335*760c2415Smrg int Token::isKeyword()
336*760c2415Smrg {
337*760c2415Smrg for (size_t u = 0; u < nkeywords; u++)
338*760c2415Smrg {
339*760c2415Smrg if (keywords[u].value == value)
340*760c2415Smrg return 1;
341*760c2415Smrg }
342*760c2415Smrg return 0;
343*760c2415Smrg }
344*760c2415Smrg
345*760c2415Smrg struct TokenInitializer
346*760c2415Smrg {
347*760c2415Smrg TokenInitializer();
348*760c2415Smrg };
349*760c2415Smrg
350*760c2415Smrg static TokenInitializer tokeninitializer;
351*760c2415Smrg
TokenInitializer()352*760c2415Smrg TokenInitializer::TokenInitializer()
353*760c2415Smrg {
354*760c2415Smrg Identifier::initTable();
355*760c2415Smrg for (nkeywords = 0; keywords[nkeywords].name; nkeywords++)
356*760c2415Smrg {
357*760c2415Smrg //printf("keyword[%d] = '%s'\n",u, keywords[u].name);
358*760c2415Smrg const char *s = keywords[nkeywords].name;
359*760c2415Smrg size_t len = strlen(s);
360*760c2415Smrg TOK v = keywords[nkeywords].value;
361*760c2415Smrg Identifier::idPool(s, len, v);
362*760c2415Smrg
363*760c2415Smrg //printf("tochars[%d] = '%s'\n",v, s);
364*760c2415Smrg Token::tochars[v] = s;
365*760c2415Smrg }
366*760c2415Smrg
367*760c2415Smrg Token::tochars[TOKeof] = "EOF";
368*760c2415Smrg Token::tochars[TOKlcurly] = "{";
369*760c2415Smrg Token::tochars[TOKrcurly] = "}";
370*760c2415Smrg Token::tochars[TOKlparen] = "(";
371*760c2415Smrg Token::tochars[TOKrparen] = ")";
372*760c2415Smrg Token::tochars[TOKlbracket] = "[";
373*760c2415Smrg Token::tochars[TOKrbracket] = "]";
374*760c2415Smrg Token::tochars[TOKsemicolon] = ";";
375*760c2415Smrg Token::tochars[TOKcolon] = ":";
376*760c2415Smrg Token::tochars[TOKcomma] = ",";
377*760c2415Smrg Token::tochars[TOKdot] = ".";
378*760c2415Smrg Token::tochars[TOKxor] = "^";
379*760c2415Smrg Token::tochars[TOKxorass] = "^=";
380*760c2415Smrg Token::tochars[TOKassign] = "=";
381*760c2415Smrg Token::tochars[TOKconstruct] = "=";
382*760c2415Smrg Token::tochars[TOKblit] = "=";
383*760c2415Smrg Token::tochars[TOKlt] = "<";
384*760c2415Smrg Token::tochars[TOKgt] = ">";
385*760c2415Smrg Token::tochars[TOKle] = "<=";
386*760c2415Smrg Token::tochars[TOKge] = ">=";
387*760c2415Smrg Token::tochars[TOKequal] = "==";
388*760c2415Smrg Token::tochars[TOKnotequal] = "!=";
389*760c2415Smrg Token::tochars[TOKnotidentity] = "!is";
390*760c2415Smrg
391*760c2415Smrg Token::tochars[TOKunord] = "!<>=";
392*760c2415Smrg Token::tochars[TOKue] = "!<>";
393*760c2415Smrg Token::tochars[TOKlg] = "<>";
394*760c2415Smrg Token::tochars[TOKleg] = "<>=";
395*760c2415Smrg Token::tochars[TOKule] = "!>";
396*760c2415Smrg Token::tochars[TOKul] = "!>=";
397*760c2415Smrg Token::tochars[TOKuge] = "!<";
398*760c2415Smrg Token::tochars[TOKug] = "!<=";
399*760c2415Smrg
400*760c2415Smrg Token::tochars[TOKnot] = "!";
401*760c2415Smrg Token::tochars[TOKshl] = "<<";
402*760c2415Smrg Token::tochars[TOKshr] = ">>";
403*760c2415Smrg Token::tochars[TOKushr] = ">>>";
404*760c2415Smrg Token::tochars[TOKadd] = "+";
405*760c2415Smrg Token::tochars[TOKmin] = "-";
406*760c2415Smrg Token::tochars[TOKmul] = "*";
407*760c2415Smrg Token::tochars[TOKdiv] = "/";
408*760c2415Smrg Token::tochars[TOKmod] = "%";
409*760c2415Smrg Token::tochars[TOKslice] = "..";
410*760c2415Smrg Token::tochars[TOKdotdotdot] = "...";
411*760c2415Smrg Token::tochars[TOKand] = "&";
412*760c2415Smrg Token::tochars[TOKandand] = "&&";
413*760c2415Smrg Token::tochars[TOKor] = "|";
414*760c2415Smrg Token::tochars[TOKoror] = "||";
415*760c2415Smrg Token::tochars[TOKarray] = "[]";
416*760c2415Smrg Token::tochars[TOKindex] = "[i]";
417*760c2415Smrg Token::tochars[TOKaddress] = "&";
418*760c2415Smrg Token::tochars[TOKstar] = "*";
419*760c2415Smrg Token::tochars[TOKtilde] = "~";
420*760c2415Smrg Token::tochars[TOKdollar] = "$";
421*760c2415Smrg Token::tochars[TOKcast] = "cast";
422*760c2415Smrg Token::tochars[TOKplusplus] = "++";
423*760c2415Smrg Token::tochars[TOKminusminus] = "--";
424*760c2415Smrg Token::tochars[TOKpreplusplus] = "++";
425*760c2415Smrg Token::tochars[TOKpreminusminus] = "--";
426*760c2415Smrg Token::tochars[TOKtype] = "type";
427*760c2415Smrg Token::tochars[TOKquestion] = "?";
428*760c2415Smrg Token::tochars[TOKneg] = "-";
429*760c2415Smrg Token::tochars[TOKuadd] = "+";
430*760c2415Smrg Token::tochars[TOKvar] = "var";
431*760c2415Smrg Token::tochars[TOKaddass] = "+=";
432*760c2415Smrg Token::tochars[TOKminass] = "-=";
433*760c2415Smrg Token::tochars[TOKmulass] = "*=";
434*760c2415Smrg Token::tochars[TOKdivass] = "/=";
435*760c2415Smrg Token::tochars[TOKmodass] = "%=";
436*760c2415Smrg Token::tochars[TOKshlass] = "<<=";
437*760c2415Smrg Token::tochars[TOKshrass] = ">>=";
438*760c2415Smrg Token::tochars[TOKushrass] = ">>>=";
439*760c2415Smrg Token::tochars[TOKandass] = "&=";
440*760c2415Smrg Token::tochars[TOKorass] = "|=";
441*760c2415Smrg Token::tochars[TOKcatass] = "~=";
442*760c2415Smrg Token::tochars[TOKcat] = "~";
443*760c2415Smrg Token::tochars[TOKcall] = "call";
444*760c2415Smrg Token::tochars[TOKidentity] = "is";
445*760c2415Smrg Token::tochars[TOKnotidentity] = "!is";
446*760c2415Smrg
447*760c2415Smrg Token::tochars[TOKorass] = "|=";
448*760c2415Smrg Token::tochars[TOKidentifier] = "identifier";
449*760c2415Smrg Token::tochars[TOKat] = "@";
450*760c2415Smrg Token::tochars[TOKpow] = "^^";
451*760c2415Smrg Token::tochars[TOKpowass] = "^^=";
452*760c2415Smrg Token::tochars[TOKgoesto] = "=>";
453*760c2415Smrg Token::tochars[TOKpound] = "#";
454*760c2415Smrg
455*760c2415Smrg // For debugging
456*760c2415Smrg Token::tochars[TOKerror] = "error";
457*760c2415Smrg Token::tochars[TOKdotid] = "dotid";
458*760c2415Smrg Token::tochars[TOKdottd] = "dottd";
459*760c2415Smrg Token::tochars[TOKdotti] = "dotti";
460*760c2415Smrg Token::tochars[TOKdotvar] = "dotvar";
461*760c2415Smrg Token::tochars[TOKdottype] = "dottype";
462*760c2415Smrg Token::tochars[TOKsymoff] = "symoff";
463*760c2415Smrg Token::tochars[TOKarraylength] = "arraylength";
464*760c2415Smrg Token::tochars[TOKarrayliteral] = "arrayliteral";
465*760c2415Smrg Token::tochars[TOKassocarrayliteral] = "assocarrayliteral";
466*760c2415Smrg Token::tochars[TOKstructliteral] = "structliteral";
467*760c2415Smrg Token::tochars[TOKstring] = "string";
468*760c2415Smrg Token::tochars[TOKdsymbol] = "symbol";
469*760c2415Smrg Token::tochars[TOKtuple] = "tuple";
470*760c2415Smrg Token::tochars[TOKdeclaration] = "declaration";
471*760c2415Smrg Token::tochars[TOKon_scope_exit] = "scope(exit)";
472*760c2415Smrg Token::tochars[TOKon_scope_success] = "scope(success)";
473*760c2415Smrg Token::tochars[TOKon_scope_failure] = "scope(failure)";
474*760c2415Smrg Token::tochars[TOKdelegateptr] = "delegateptr";
475*760c2415Smrg Token::tochars[TOKvectorarray] = "vectorarray";
476*760c2415Smrg }
477