1
2 /* Compiler implementation of the D programming language
3 * Copyright (C) 1999-2019 by The D Language Foundation, All Rights Reserved
4 * written by Walter Bright
5 * http://www.digitalmars.com
6 * Distributed under the Boost Software License, Version 1.0.
7 * http://www.boost.org/LICENSE_1_0.txt
8 * https://github.com/D-Programming-Language/dmd/blob/master/src/lexer.c
9 */
10
11 #include "root/dsystem.h"
12
13 #include "tokens.h"
14 #include "root/rmem.h"
15 #include "root/outbuffer.h"
16 #include "id.h"
17 #include "identifier.h"
18 #include "utf.h"
19
20 /************************* Token **********************************************/
21
22 Token *Token::freelist = NULL;
23
24 const char *Token::tochars[TOKMAX];
25
alloc()26 Token *Token::alloc()
27 {
28 if (Token::freelist)
29 {
30 Token *t = freelist;
31 freelist = t->next;
32 t->next = NULL;
33 return t;
34 }
35
36 return new Token();
37 }
38
free()39 void Token::free()
40 {
41 next = freelist;
42 freelist = this;
43 }
44
toChars()45 const char *Token::toChars() const
46 {
47 static char buffer[3 + 3 * sizeof(floatvalue) + 1];
48
49 const char *p = &buffer[0];
50 switch (value)
51 {
52 case TOKint32v:
53 sprintf(&buffer[0],"%d",(d_int32)int64value);
54 break;
55
56 case TOKuns32v:
57 case TOKcharv:
58 case TOKwcharv:
59 case TOKdcharv:
60 sprintf(&buffer[0],"%uU",(d_uns32)uns64value);
61 break;
62
63 case TOKint64v:
64 sprintf(&buffer[0],"%lldL",(longlong)int64value);
65 break;
66
67 case TOKuns64v:
68 sprintf(&buffer[0],"%lluUL",(ulonglong)uns64value);
69 break;
70
71 case TOKfloat32v:
72 CTFloat::sprint(&buffer[0], 'g', floatvalue);
73 strcat(&buffer[0], "f");
74 break;
75
76 case TOKfloat64v:
77 CTFloat::sprint(&buffer[0], 'g', floatvalue);
78 break;
79
80 case TOKfloat80v:
81 CTFloat::sprint(&buffer[0], 'g', floatvalue);
82 strcat(&buffer[0], "L");
83 break;
84
85 case TOKimaginary32v:
86 CTFloat::sprint(&buffer[0], 'g', floatvalue);
87 strcat(&buffer[0], "fi");
88 break;
89
90 case TOKimaginary64v:
91 CTFloat::sprint(&buffer[0], 'g', floatvalue);
92 strcat(&buffer[0], "i");
93 break;
94
95 case TOKimaginary80v:
96 CTFloat::sprint(&buffer[0], 'g', floatvalue);
97 strcat(&buffer[0], "Li");
98 break;
99
100 case TOKstring:
101 {
102 OutBuffer buf;
103 buf.writeByte('"');
104 for (size_t i = 0; i < len; )
105 {
106 unsigned c;
107 utf_decodeChar((utf8_t *)ustring, len, &i, &c);
108 switch (c)
109 {
110 case 0:
111 break;
112
113 case '"':
114 case '\\':
115 buf.writeByte('\\');
116 /* fall through */
117 default:
118 if (c <= 0x7F)
119 {
120 if (isprint(c))
121 buf.writeByte(c);
122 else
123 buf.printf("\\x%02x", c);
124 }
125 else if (c <= 0xFFFF)
126 buf.printf("\\u%04x", c);
127 else
128 buf.printf("\\U%08x", c);
129 continue;
130 }
131 break;
132 }
133 buf.writeByte('"');
134 if (postfix)
135 buf.writeByte(postfix);
136 p = buf.extractString();
137 }
138 break;
139
140 case TOKxstring:
141 {
142 OutBuffer buf;
143 buf.writeByte('x');
144 buf.writeByte('"');
145 for (size_t i = 0; i < len; i++)
146 {
147 if (i)
148 buf.writeByte(' ');
149 buf.printf("%02x", ustring[i]);
150 }
151 buf.writeByte('"');
152 if (postfix)
153 buf.writeByte(postfix);
154 buf.writeByte(0);
155 p = (char *)buf.extractData();
156 break;
157 }
158
159 case TOKidentifier:
160 case TOKenum:
161 case TOKstruct:
162 case TOKimport:
163 case TOKwchar: case TOKdchar:
164 case TOKbool: case TOKchar:
165 case TOKint8: case TOKuns8:
166 case TOKint16: case TOKuns16:
167 case TOKint32: case TOKuns32:
168 case TOKint64: case TOKuns64:
169 case TOKint128: case TOKuns128:
170 case TOKfloat32: case TOKfloat64: case TOKfloat80:
171 case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:
172 case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:
173 case TOKvoid:
174 p = ident->toChars();
175 break;
176
177 default:
178 p = toChars(value);
179 break;
180 }
181 return p;
182 }
183
toChars(TOK value)184 const char *Token::toChars(TOK value)
185 {
186 static char buffer[3 + 3 * sizeof(value) + 1];
187
188 const char *p = tochars[value];
189 if (!p)
190 {
191 sprintf(&buffer[0],"TOK%d",value);
192 p = &buffer[0];
193 }
194 return p;
195 }
196
197 /****************************************
198 */
199
200 struct Keyword
201 {
202 const char *name;
203 TOK value;
204 };
205
206 static size_t nkeywords;
207 static Keyword keywords[] =
208 {
209 { "this", TOKthis },
210 { "super", TOKsuper },
211 { "assert", TOKassert },
212 { "null", TOKnull },
213 { "true", TOKtrue },
214 { "false", TOKfalse },
215 { "cast", TOKcast },
216 { "new", TOKnew },
217 { "delete", TOKdelete },
218 { "throw", TOKthrow },
219 { "module", TOKmodule },
220 { "pragma", TOKpragma },
221 { "typeof", TOKtypeof },
222 { "typeid", TOKtypeid },
223
224 { "template", TOKtemplate },
225
226 { "void", TOKvoid },
227 { "byte", TOKint8 },
228 { "ubyte", TOKuns8 },
229 { "short", TOKint16 },
230 { "ushort", TOKuns16 },
231 { "int", TOKint32 },
232 { "uint", TOKuns32 },
233 { "long", TOKint64 },
234 { "ulong", TOKuns64 },
235 { "cent", TOKint128, },
236 { "ucent", TOKuns128, },
237 { "float", TOKfloat32 },
238 { "double", TOKfloat64 },
239 { "real", TOKfloat80 },
240
241 { "bool", TOKbool },
242 { "char", TOKchar },
243 { "wchar", TOKwchar },
244 { "dchar", TOKdchar },
245
246 { "ifloat", TOKimaginary32 },
247 { "idouble", TOKimaginary64 },
248 { "ireal", TOKimaginary80 },
249
250 { "cfloat", TOKcomplex32 },
251 { "cdouble", TOKcomplex64 },
252 { "creal", TOKcomplex80 },
253
254 { "delegate", TOKdelegate },
255 { "function", TOKfunction },
256
257 { "is", TOKis },
258 { "if", TOKif },
259 { "else", TOKelse },
260 { "while", TOKwhile },
261 { "for", TOKfor },
262 { "do", TOKdo },
263 { "switch", TOKswitch },
264 { "case", TOKcase },
265 { "default", TOKdefault },
266 { "break", TOKbreak },
267 { "continue", TOKcontinue },
268 { "synchronized", TOKsynchronized },
269 { "return", TOKreturn },
270 { "goto", TOKgoto },
271 { "try", TOKtry },
272 { "catch", TOKcatch },
273 { "finally", TOKfinally },
274 { "with", TOKwith },
275 { "asm", TOKasm },
276 { "foreach", TOKforeach },
277 { "foreach_reverse", TOKforeach_reverse },
278 { "scope", TOKscope },
279
280 { "struct", TOKstruct },
281 { "class", TOKclass },
282 { "interface", TOKinterface },
283 { "union", TOKunion },
284 { "enum", TOKenum },
285 { "import", TOKimport },
286 { "mixin", TOKmixin },
287 { "static", TOKstatic },
288 { "final", TOKfinal },
289 { "const", TOKconst },
290 { "alias", TOKalias },
291 { "override", TOKoverride },
292 { "abstract", TOKabstract },
293 { "debug", TOKdebug },
294 { "deprecated", TOKdeprecated },
295 { "in", TOKin },
296 { "out", TOKout },
297 { "inout", TOKinout },
298 { "lazy", TOKlazy },
299 { "auto", TOKauto },
300
301 { "align", TOKalign },
302 { "extern", TOKextern },
303 { "private", TOKprivate },
304 { "package", TOKpackage },
305 { "protected", TOKprotected },
306 { "public", TOKpublic },
307 { "export", TOKexport },
308
309 { "invariant", TOKinvariant },
310 { "unittest", TOKunittest },
311 { "version", TOKversion },
312
313 { "__argTypes", TOKargTypes },
314 { "__parameters", TOKparameters },
315 { "ref", TOKref },
316 { "macro", TOKmacro },
317
318 { "pure", TOKpure },
319 { "nothrow", TOKnothrow },
320 { "__gshared", TOKgshared },
321 { "__traits", TOKtraits },
322 { "__vector", TOKvector },
323 { "__overloadset", TOKoverloadset },
324 { "__FILE__", TOKfile },
325 { "__FILE_FULL_PATH__", TOKfilefullpath },
326 { "__LINE__", TOKline },
327 { "__MODULE__", TOKmodulestring },
328 { "__FUNCTION__", TOKfuncstring },
329 { "__PRETTY_FUNCTION__", TOKprettyfunc },
330 { "shared", TOKshared },
331 { "immutable", TOKimmutable },
332 { NULL, TOKreserved }
333 };
334
isKeyword()335 int Token::isKeyword()
336 {
337 for (size_t u = 0; u < nkeywords; u++)
338 {
339 if (keywords[u].value == value)
340 return 1;
341 }
342 return 0;
343 }
344
345 struct TokenInitializer
346 {
347 TokenInitializer();
348 };
349
350 static TokenInitializer tokeninitializer;
351
TokenInitializer()352 TokenInitializer::TokenInitializer()
353 {
354 Identifier::initTable();
355 for (nkeywords = 0; keywords[nkeywords].name; nkeywords++)
356 {
357 //printf("keyword[%d] = '%s'\n",u, keywords[u].name);
358 const char *s = keywords[nkeywords].name;
359 size_t len = strlen(s);
360 TOK v = keywords[nkeywords].value;
361 Identifier::idPool(s, len, v);
362
363 //printf("tochars[%d] = '%s'\n",v, s);
364 Token::tochars[v] = s;
365 }
366
367 Token::tochars[TOKeof] = "EOF";
368 Token::tochars[TOKlcurly] = "{";
369 Token::tochars[TOKrcurly] = "}";
370 Token::tochars[TOKlparen] = "(";
371 Token::tochars[TOKrparen] = ")";
372 Token::tochars[TOKlbracket] = "[";
373 Token::tochars[TOKrbracket] = "]";
374 Token::tochars[TOKsemicolon] = ";";
375 Token::tochars[TOKcolon] = ":";
376 Token::tochars[TOKcomma] = ",";
377 Token::tochars[TOKdot] = ".";
378 Token::tochars[TOKxor] = "^";
379 Token::tochars[TOKxorass] = "^=";
380 Token::tochars[TOKassign] = "=";
381 Token::tochars[TOKconstruct] = "=";
382 Token::tochars[TOKblit] = "=";
383 Token::tochars[TOKlt] = "<";
384 Token::tochars[TOKgt] = ">";
385 Token::tochars[TOKle] = "<=";
386 Token::tochars[TOKge] = ">=";
387 Token::tochars[TOKequal] = "==";
388 Token::tochars[TOKnotequal] = "!=";
389 Token::tochars[TOKnotidentity] = "!is";
390
391 Token::tochars[TOKunord] = "!<>=";
392 Token::tochars[TOKue] = "!<>";
393 Token::tochars[TOKlg] = "<>";
394 Token::tochars[TOKleg] = "<>=";
395 Token::tochars[TOKule] = "!>";
396 Token::tochars[TOKul] = "!>=";
397 Token::tochars[TOKuge] = "!<";
398 Token::tochars[TOKug] = "!<=";
399
400 Token::tochars[TOKnot] = "!";
401 Token::tochars[TOKshl] = "<<";
402 Token::tochars[TOKshr] = ">>";
403 Token::tochars[TOKushr] = ">>>";
404 Token::tochars[TOKadd] = "+";
405 Token::tochars[TOKmin] = "-";
406 Token::tochars[TOKmul] = "*";
407 Token::tochars[TOKdiv] = "/";
408 Token::tochars[TOKmod] = "%";
409 Token::tochars[TOKslice] = "..";
410 Token::tochars[TOKdotdotdot] = "...";
411 Token::tochars[TOKand] = "&";
412 Token::tochars[TOKandand] = "&&";
413 Token::tochars[TOKor] = "|";
414 Token::tochars[TOKoror] = "||";
415 Token::tochars[TOKarray] = "[]";
416 Token::tochars[TOKindex] = "[i]";
417 Token::tochars[TOKaddress] = "&";
418 Token::tochars[TOKstar] = "*";
419 Token::tochars[TOKtilde] = "~";
420 Token::tochars[TOKdollar] = "$";
421 Token::tochars[TOKcast] = "cast";
422 Token::tochars[TOKplusplus] = "++";
423 Token::tochars[TOKminusminus] = "--";
424 Token::tochars[TOKpreplusplus] = "++";
425 Token::tochars[TOKpreminusminus] = "--";
426 Token::tochars[TOKtype] = "type";
427 Token::tochars[TOKquestion] = "?";
428 Token::tochars[TOKneg] = "-";
429 Token::tochars[TOKuadd] = "+";
430 Token::tochars[TOKvar] = "var";
431 Token::tochars[TOKaddass] = "+=";
432 Token::tochars[TOKminass] = "-=";
433 Token::tochars[TOKmulass] = "*=";
434 Token::tochars[TOKdivass] = "/=";
435 Token::tochars[TOKmodass] = "%=";
436 Token::tochars[TOKshlass] = "<<=";
437 Token::tochars[TOKshrass] = ">>=";
438 Token::tochars[TOKushrass] = ">>>=";
439 Token::tochars[TOKandass] = "&=";
440 Token::tochars[TOKorass] = "|=";
441 Token::tochars[TOKcatass] = "~=";
442 Token::tochars[TOKcat] = "~";
443 Token::tochars[TOKcall] = "call";
444 Token::tochars[TOKidentity] = "is";
445 Token::tochars[TOKnotidentity] = "!is";
446
447 Token::tochars[TOKorass] = "|=";
448 Token::tochars[TOKidentifier] = "identifier";
449 Token::tochars[TOKat] = "@";
450 Token::tochars[TOKpow] = "^^";
451 Token::tochars[TOKpowass] = "^^=";
452 Token::tochars[TOKgoesto] = "=>";
453 Token::tochars[TOKpound] = "#";
454
455 // For debugging
456 Token::tochars[TOKerror] = "error";
457 Token::tochars[TOKdotid] = "dotid";
458 Token::tochars[TOKdottd] = "dottd";
459 Token::tochars[TOKdotti] = "dotti";
460 Token::tochars[TOKdotvar] = "dotvar";
461 Token::tochars[TOKdottype] = "dottype";
462 Token::tochars[TOKsymoff] = "symoff";
463 Token::tochars[TOKarraylength] = "arraylength";
464 Token::tochars[TOKarrayliteral] = "arrayliteral";
465 Token::tochars[TOKassocarrayliteral] = "assocarrayliteral";
466 Token::tochars[TOKstructliteral] = "structliteral";
467 Token::tochars[TOKstring] = "string";
468 Token::tochars[TOKdsymbol] = "symbol";
469 Token::tochars[TOKtuple] = "tuple";
470 Token::tochars[TOKdeclaration] = "declaration";
471 Token::tochars[TOKon_scope_exit] = "scope(exit)";
472 Token::tochars[TOKon_scope_success] = "scope(success)";
473 Token::tochars[TOKon_scope_failure] = "scope(failure)";
474 Token::tochars[TOKdelegateptr] = "delegateptr";
475 Token::tochars[TOKvectorarray] = "vectorarray";
476 }
477