1 /**
2 * MojoShader; generate shader programs from bytecode of compiled
3 * Direct3D shaders.
4 *
5 * Please see the file LICENSE.txt in the source's root directory.
6 *
7 * This file written by Ryan C. Gordon.
8 */
9
10 #define __MOJOSHADER_INTERNAL__ 1
11 #include "mojoshader_internal.h"
12
13 #if DEBUG_PREPROCESSOR
14 #define print_debug_token(token, len, val) \
15 MOJOSHADER_print_debug_token("PREPROCESSOR", token, len, val)
16 #else
17 #define print_debug_token(token, len, val)
18 #endif
19
20 #if DEBUG_LEXER
debug_preprocessor_lexer(IncludeState * s)21 static Token debug_preprocessor_lexer(IncludeState *s)
22 {
23 const Token retval = preprocessor_lexer(s);
24 MOJOSHADER_print_debug_token("LEXER", s->token, s->tokenlen, retval);
25 return retval;
26 } // debug_preprocessor_lexer
27 #define preprocessor_lexer(s) debug_preprocessor_lexer(s)
28 #endif
29
30 #if DEBUG_TOKENIZER
print_debug_lexing_position(IncludeState * s)31 static void print_debug_lexing_position(IncludeState *s)
32 {
33 if (s != NULL)
34 printf("NOW LEXING %s:%d ...\n", s->filename, s->line);
35 } // print_debug_lexing_position
36 #else
37 #define print_debug_lexing_position(s)
38 #endif
39
40 typedef struct Context
41 {
42 int isfail;
43 int out_of_memory;
44 char failstr[256];
45 int recursion_count;
46 int asm_comments;
47 int parsing_pragma;
48 Conditional *conditional_pool;
49 IncludeState *include_stack;
50 IncludeState *include_pool;
51 Define *define_hashtable[256];
52 Define *define_pool;
53 Define *file_macro;
54 Define *line_macro;
55 StringCache *filename_cache;
56 MOJOSHADER_includeOpen open_callback;
57 MOJOSHADER_includeClose close_callback;
58 MOJOSHADER_malloc malloc;
59 MOJOSHADER_free free;
60 void *malloc_data;
61 } Context;
62
63
64 // Convenience functions for allocators...
65
out_of_memory(Context * ctx)66 static inline void out_of_memory(Context *ctx)
67 {
68 ctx->out_of_memory = 1;
69 } // out_of_memory
70
Malloc(Context * ctx,const size_t len)71 static inline void *Malloc(Context *ctx, const size_t len)
72 {
73 void *retval = ctx->malloc((int) len, ctx->malloc_data);
74 if (retval == NULL)
75 out_of_memory(ctx);
76 return retval;
77 } // Malloc
78
Free(Context * ctx,void * ptr)79 static inline void Free(Context *ctx, void *ptr)
80 {
81 ctx->free(ptr, ctx->malloc_data);
82 } // Free
83
MallocBridge(int bytes,void * data)84 static void *MallocBridge(int bytes, void *data)
85 {
86 return Malloc((Context *) data, (size_t) bytes);
87 } // MallocBridge
88
FreeBridge(void * ptr,void * data)89 static void FreeBridge(void *ptr, void *data)
90 {
91 Free((Context *) data, ptr);
92 } // FreeBridge
93
StrDup(Context * ctx,const char * str)94 static inline char *StrDup(Context *ctx, const char *str)
95 {
96 char *retval = (char *) Malloc(ctx, strlen(str) + 1);
97 if (retval != NULL)
98 strcpy(retval, str);
99 return retval;
100 } // StrDup
101
102 static void failf(Context *ctx, const char *fmt, ...) ISPRINTF(2,3);
failf(Context * ctx,const char * fmt,...)103 static void failf(Context *ctx, const char *fmt, ...)
104 {
105 ctx->isfail = 1;
106 va_list ap;
107 va_start(ap, fmt);
108 vsnprintf(ctx->failstr, sizeof (ctx->failstr), fmt, ap);
109 va_end(ap);
110 } // failf
111
fail(Context * ctx,const char * reason)112 static inline void fail(Context *ctx, const char *reason)
113 {
114 failf(ctx, "%s", reason);
115 } // fail
116
117
118 #if DEBUG_TOKENIZER
MOJOSHADER_print_debug_token(const char * subsystem,const char * token,const unsigned int tokenlen,const Token tokenval)119 void MOJOSHADER_print_debug_token(const char *subsystem, const char *token,
120 const unsigned int tokenlen,
121 const Token tokenval)
122 {
123 printf("%s TOKEN: \"", subsystem);
124 unsigned int i;
125 for (i = 0; i < tokenlen; i++)
126 {
127 if (token[i] == '\n')
128 printf("\\n");
129 else if (token[i] == '\\')
130 printf("\\\\");
131 else
132 printf("%c", token[i]);
133 } // for
134 printf("\" (");
135 switch (tokenval)
136 {
137 #define TOKENCASE(x) case x: printf("%s", #x); break
138 TOKENCASE(TOKEN_UNKNOWN);
139 TOKENCASE(TOKEN_IDENTIFIER);
140 TOKENCASE(TOKEN_INT_LITERAL);
141 TOKENCASE(TOKEN_FLOAT_LITERAL);
142 TOKENCASE(TOKEN_STRING_LITERAL);
143 TOKENCASE(TOKEN_ADDASSIGN);
144 TOKENCASE(TOKEN_SUBASSIGN);
145 TOKENCASE(TOKEN_MULTASSIGN);
146 TOKENCASE(TOKEN_DIVASSIGN);
147 TOKENCASE(TOKEN_MODASSIGN);
148 TOKENCASE(TOKEN_XORASSIGN);
149 TOKENCASE(TOKEN_ANDASSIGN);
150 TOKENCASE(TOKEN_ORASSIGN);
151 TOKENCASE(TOKEN_INCREMENT);
152 TOKENCASE(TOKEN_DECREMENT);
153 TOKENCASE(TOKEN_RSHIFT);
154 TOKENCASE(TOKEN_LSHIFT);
155 TOKENCASE(TOKEN_ANDAND);
156 TOKENCASE(TOKEN_OROR);
157 TOKENCASE(TOKEN_LEQ);
158 TOKENCASE(TOKEN_GEQ);
159 TOKENCASE(TOKEN_EQL);
160 TOKENCASE(TOKEN_NEQ);
161 TOKENCASE(TOKEN_HASH);
162 TOKENCASE(TOKEN_HASHHASH);
163 TOKENCASE(TOKEN_PP_INCLUDE);
164 TOKENCASE(TOKEN_PP_LINE);
165 TOKENCASE(TOKEN_PP_DEFINE);
166 TOKENCASE(TOKEN_PP_UNDEF);
167 TOKENCASE(TOKEN_PP_IF);
168 TOKENCASE(TOKEN_PP_IFDEF);
169 TOKENCASE(TOKEN_PP_IFNDEF);
170 TOKENCASE(TOKEN_PP_ELSE);
171 TOKENCASE(TOKEN_PP_ELIF);
172 TOKENCASE(TOKEN_PP_ENDIF);
173 TOKENCASE(TOKEN_PP_ERROR);
174 TOKENCASE(TOKEN_PP_PRAGMA);
175 TOKENCASE(TOKEN_INCOMPLETE_COMMENT);
176 TOKENCASE(TOKEN_BAD_CHARS);
177 TOKENCASE(TOKEN_EOI);
178 TOKENCASE(TOKEN_PREPROCESSING_ERROR);
179 #undef TOKENCASE
180
181 case ((Token) '\n'):
182 printf("'\\n'");
183 break;
184
185 case ((Token) '\\'):
186 printf("'\\\\'");
187 break;
188
189 default:
190 assert(((int)tokenval) < 256);
191 printf("'%c'", (char) tokenval);
192 break;
193 } // switch
194 printf(")\n");
195 } // MOJOSHADER_print_debug_token
196 #endif
197
198
199
200 #if !MOJOSHADER_FORCE_INCLUDE_CALLBACKS
201
202 // !!! FIXME: most of these _MSC_VER should probably be _WINDOWS?
203 #ifdef _MSC_VER
204 #define WIN32_LEAN_AND_MEAN 1
205 #include <windows.h> // GL headers need this for WINGDIAPI definition.
206 #else
207 #include <sys/stat.h>
208 #include <fcntl.h>
209 #include <unistd.h>
210 #endif
211
MOJOSHADER_internal_include_open(MOJOSHADER_includeType inctype,const char * fname,const char * parent,const char ** outdata,unsigned int * outbytes,MOJOSHADER_malloc m,MOJOSHADER_free f,void * d)212 int MOJOSHADER_internal_include_open(MOJOSHADER_includeType inctype,
213 const char *fname, const char *parent,
214 const char **outdata,
215 unsigned int *outbytes,
216 MOJOSHADER_malloc m, MOJOSHADER_free f,
217 void *d)
218 {
219 #ifdef _MSC_VER
220 WCHAR wpath[MAX_PATH];
221 if (!MultiByteToWideChar(CP_UTF8, 0, fname, -1, wpath, MAX_PATH))
222 return 0;
223
224 const DWORD share = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
225 const HANDLE handle = CreateFileW(wpath, FILE_GENERIC_READ, share,
226 NULL, OPEN_EXISTING, NULL, NULL);
227 if (handle == INVALID_HANDLE_VALUE)
228 return 0;
229
230 const DWORD fileSize = GetFileSize(handle, NULL);
231 if (fileSize == INVALID_FILE_SIZE)
232 {
233 CloseHandle(handle);
234 return 0;
235 } // if
236
237 char *data = (char *) m(fileSize, d);
238 if (data == NULL)
239 {
240 CloseHandle(handle);
241 return 0;
242 } // if
243
244 DWORD readLength = 0;
245 if (!ReadFile(handle, data, fileSize, &readLength, NULL))
246 {
247 CloseHandle(handle);
248 f(data, d);
249 return 0;
250 } // if
251
252 CloseHandle(handle);
253
254 if (readLength != fileSize)
255 {
256 f(data, d);
257 return 0;
258 } // if
259 *outdata = data;
260 *outbytes = fileSize;
261 return 1;
262 #else
263 struct stat statbuf;
264 if (stat(fname, &statbuf) == -1)
265 return 0;
266 char *data = (char *) m(statbuf.st_size, d);
267 if (data == NULL)
268 return 0;
269 const int fd = open(fname, O_RDONLY);
270 if (fd == -1)
271 {
272 f(data, d);
273 return 0;
274 } // if
275 if (read(fd, data, statbuf.st_size) != statbuf.st_size)
276 {
277 f(data, d);
278 close(fd);
279 return 0;
280 } // if
281 close(fd);
282 *outdata = data;
283 *outbytes = (unsigned int) statbuf.st_size;
284 return 1;
285 #endif
286 } // MOJOSHADER_internal_include_open
287
288
MOJOSHADER_internal_include_close(const char * data,MOJOSHADER_malloc m,MOJOSHADER_free f,void * d)289 void MOJOSHADER_internal_include_close(const char *data, MOJOSHADER_malloc m,
290 MOJOSHADER_free f, void *d)
291 {
292 f((void *) data, d);
293 } // MOJOSHADER_internal_include_close
294 #endif // !MOJOSHADER_FORCE_INCLUDE_CALLBACKS
295
296
297 // !!! FIXME: maybe use these pool magic elsewhere?
298 // !!! FIXME: maybe just get rid of this? (maybe the fragmentation isn't a big deal?)
299
300 // Pool stuff...
301 // ugh, I hate this macro salsa.
302 #define FREE_POOL(type, poolname) \
303 static void free_##poolname##_pool(Context *ctx) { \
304 type *item = ctx->poolname##_pool; \
305 while (item != NULL) { \
306 type *next = item->next; \
307 Free(ctx, item); \
308 item = next; \
309 } \
310 }
311
312 #define GET_POOL(type, poolname) \
313 static type *get_##poolname(Context *ctx) { \
314 type *retval = ctx->poolname##_pool; \
315 if (retval != NULL) \
316 ctx->poolname##_pool = retval->next; \
317 else \
318 retval = (type *) Malloc(ctx, sizeof (type)); \
319 if (retval != NULL) \
320 memset(retval, '\0', sizeof (type)); \
321 return retval; \
322 }
323
324 #define PUT_POOL(type, poolname) \
325 static void put_##poolname(Context *ctx, type *item) { \
326 item->next = ctx->poolname##_pool; \
327 ctx->poolname##_pool = item; \
328 }
329
330 #define IMPLEMENT_POOL(type, poolname) \
331 FREE_POOL(type, poolname) \
332 GET_POOL(type, poolname) \
333 PUT_POOL(type, poolname)
334
IMPLEMENT_POOL(Conditional,conditional)335 IMPLEMENT_POOL(Conditional, conditional)
336 IMPLEMENT_POOL(IncludeState, include)
337 IMPLEMENT_POOL(Define, define)
338
339
340 // Preprocessor define hashtable stuff...
341
342 // !!! FIXME: why isn't this using mojoshader_common.c's code?
343
344 // this is djb's xor hashing function.
345 static inline uint32 hash_string_djbxor(const char *sym)
346 {
347 register uint32 hash = 5381;
348 while (*sym)
349 hash = ((hash << 5) + hash) ^ *(sym++);
350 return hash;
351 } // hash_string_djbxor
352
hash_define(const char * sym)353 static inline uint8 hash_define(const char *sym)
354 {
355 return (uint8) hash_string_djbxor(sym);
356 } // hash_define
357
358
add_define(Context * ctx,const char * sym,const char * val,char ** parameters,int paramcount)359 static int add_define(Context *ctx, const char *sym, const char *val,
360 char **parameters, int paramcount)
361 {
362 const uint8 hash = hash_define(sym);
363 Define *bucket = ctx->define_hashtable[hash];
364 while (bucket)
365 {
366 if (strcmp(bucket->identifier, sym) == 0)
367 {
368 failf(ctx, "'%s' already defined", sym); // !!! FIXME: warning?
369 // !!! FIXME: gcc reports the location of previous #define here.
370 return 0;
371 } // if
372 bucket = bucket->next;
373 } // while
374
375 bucket = get_define(ctx);
376 if (bucket == NULL)
377 return 0;
378
379 bucket->definition = val;
380 bucket->original = NULL;
381 bucket->identifier = sym;
382 bucket->parameters = (const char **) parameters;
383 bucket->paramcount = paramcount;
384 bucket->next = ctx->define_hashtable[hash];
385 ctx->define_hashtable[hash] = bucket;
386 return 1;
387 } // add_define
388
389
free_define(Context * ctx,Define * def)390 static void free_define(Context *ctx, Define *def)
391 {
392 if (def != NULL)
393 {
394 int i;
395 for (i = 0; i < def->paramcount; i++)
396 Free(ctx, (void *) def->parameters[i]);
397 Free(ctx, (void *) def->parameters);
398 Free(ctx, (void *) def->identifier);
399 Free(ctx, (void *) def->definition);
400 Free(ctx, (void *) def->original);
401 put_define(ctx, def);
402 } // if
403 } // free_define
404
405
remove_define(Context * ctx,const char * sym)406 static int remove_define(Context *ctx, const char *sym)
407 {
408 const uint8 hash = hash_define(sym);
409 Define *bucket = ctx->define_hashtable[hash];
410 Define *prev = NULL;
411 while (bucket)
412 {
413 if (strcmp(bucket->identifier, sym) == 0)
414 {
415 if (prev == NULL)
416 ctx->define_hashtable[hash] = bucket->next;
417 else
418 prev->next = bucket->next;
419 free_define(ctx, bucket);
420 return 1;
421 } // if
422 prev = bucket;
423 bucket = bucket->next;
424 } // while
425
426 return 0;
427 } // remove_define
428
429
find_define(Context * ctx,const char * sym)430 static const Define *find_define(Context *ctx, const char *sym)
431 {
432 if ( (ctx->file_macro) && (strcmp(sym, "__FILE__") == 0) )
433 {
434 Free(ctx, (char *) ctx->file_macro->definition);
435 const IncludeState *state = ctx->include_stack;
436 const char *fname = state ? state->filename : "";
437 const size_t len = strlen(fname) + 2;
438 char *str = (char *) Malloc(ctx, len);
439 if (!str)
440 return NULL;
441 str[0] = '\"';
442 memcpy(str + 1, fname, len - 2);
443 str[len - 1] = '\"';
444 ctx->file_macro->definition = str;
445 return ctx->file_macro;
446 } // if
447
448 else if ( (ctx->line_macro) && (strcmp(sym, "__LINE__") == 0) )
449 {
450 Free(ctx, (char *) ctx->line_macro->definition);
451 const IncludeState *state = ctx->include_stack;
452 const size_t bufsize = 32;
453 char *str = (char *) Malloc(ctx, bufsize);
454 if (!str)
455 return 0;
456
457 const size_t len = snprintf(str, bufsize, "%u", state->line);
458 assert(len < bufsize);
459 ctx->line_macro->definition = str;
460 return ctx->line_macro;
461 } // else
462
463 const uint8 hash = hash_define(sym);
464 Define *bucket = ctx->define_hashtable[hash];
465 while (bucket)
466 {
467 if (strcmp(bucket->identifier, sym) == 0)
468 return bucket;
469 bucket = bucket->next;
470 } // while
471 return NULL;
472 } // find_define
473
474
find_define_by_token(Context * ctx)475 static const Define *find_define_by_token(Context *ctx)
476 {
477 IncludeState *state = ctx->include_stack;
478 assert(state->tokenval == TOKEN_IDENTIFIER);
479 char *sym = (char *) alloca(state->tokenlen+1);
480 memcpy(sym, state->token, state->tokenlen);
481 sym[state->tokenlen] = '\0';
482 return find_define(ctx, sym);
483 } // find_define_by_token
484
485
find_macro_arg(const IncludeState * state,const Define * defines)486 static const Define *find_macro_arg(const IncludeState *state,
487 const Define *defines)
488 {
489 const Define *def = NULL;
490 char *sym = (char *) alloca(state->tokenlen + 1);
491 memcpy(sym, state->token, state->tokenlen);
492 sym[state->tokenlen] = '\0';
493
494 for (def = defines; def != NULL; def = def->next)
495 {
496 assert(def->parameters == NULL); // args can't have args!
497 assert(def->paramcount == 0); // args can't have args!
498 if (strcmp(def->identifier, sym) == 0)
499 break;
500 } // while
501
502 return def;
503 } // find_macro_arg
504
505
put_all_defines(Context * ctx)506 static void put_all_defines(Context *ctx)
507 {
508 size_t i;
509 for (i = 0; i < STATICARRAYLEN(ctx->define_hashtable); i++)
510 {
511 Define *bucket = ctx->define_hashtable[i];
512 ctx->define_hashtable[i] = NULL;
513 while (bucket)
514 {
515 Define *next = bucket->next;
516 free_define(ctx, bucket);
517 bucket = next;
518 } // while
519 } // for
520 } // put_all_defines
521
522
push_source(Context * ctx,const char * fname,const char * source,unsigned int srclen,unsigned int linenum,MOJOSHADER_includeClose close_callback)523 static int push_source(Context *ctx, const char *fname, const char *source,
524 unsigned int srclen, unsigned int linenum,
525 MOJOSHADER_includeClose close_callback)
526 {
527 IncludeState *state = get_include(ctx);
528 if (state == NULL)
529 return 0;
530
531 if (fname != NULL)
532 {
533 state->filename = stringcache(ctx->filename_cache, fname);
534 if (state->filename == NULL)
535 {
536 put_include(ctx, state);
537 return 0;
538 } // if
539 } // if
540
541 state->close_callback = close_callback;
542 state->source_base = source;
543 state->source = source;
544 state->token = source;
545 state->tokenval = ((Token) '\n');
546 state->orig_length = srclen;
547 state->bytes_left = srclen;
548 state->line = linenum;
549 state->next = ctx->include_stack;
550 state->asm_comments = ctx->asm_comments;
551
552 print_debug_lexing_position(state);
553
554 ctx->include_stack = state;
555
556 return 1;
557 } // push_source
558
559
pop_source(Context * ctx)560 static void pop_source(Context *ctx)
561 {
562 IncludeState *state = ctx->include_stack;
563 assert(state != NULL); // more pops than pushes!
564 if (state == NULL)
565 return;
566
567 if (state->close_callback)
568 {
569 state->close_callback(state->source_base, ctx->malloc,
570 ctx->free, ctx->malloc_data);
571 } // if
572
573 // state->filename is a pointer to the filename cache; don't free it here!
574
575 Conditional *cond = state->conditional_stack;
576 while (cond)
577 {
578 Conditional *next = cond->next;
579 put_conditional(ctx, cond);
580 cond = next;
581 } // while
582
583 ctx->include_stack = state->next;
584
585 print_debug_lexing_position(ctx->include_stack);
586
587 put_include(ctx, state);
588 } // pop_source
589
590
close_define_include(const char * data,MOJOSHADER_malloc m,MOJOSHADER_free f,void * d)591 static void close_define_include(const char *data, MOJOSHADER_malloc m,
592 MOJOSHADER_free f, void *d)
593 {
594 f((void *) data, d);
595 } // close_define_include
596
597
preprocessor_start(const char * fname,const char * source,unsigned int sourcelen,MOJOSHADER_includeOpen open_callback,MOJOSHADER_includeClose close_callback,const MOJOSHADER_preprocessorDefine * defines,unsigned int define_count,int asm_comments,MOJOSHADER_malloc m,MOJOSHADER_free f,void * d)598 Preprocessor *preprocessor_start(const char *fname, const char *source,
599 unsigned int sourcelen,
600 MOJOSHADER_includeOpen open_callback,
601 MOJOSHADER_includeClose close_callback,
602 const MOJOSHADER_preprocessorDefine *defines,
603 unsigned int define_count, int asm_comments,
604 MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
605 {
606 int okay = 1;
607 unsigned int i = 0;
608
609 // the preprocessor is internal-only, so we verify all these are != NULL.
610 assert(m != NULL);
611 assert(f != NULL);
612
613 Context *ctx = (Context *) m(sizeof (Context), d);
614 if (ctx == NULL)
615 return NULL;
616
617 memset(ctx, '\0', sizeof (Context));
618 ctx->malloc = m;
619 ctx->free = f;
620 ctx->malloc_data = d;
621 ctx->open_callback = open_callback;
622 ctx->close_callback = close_callback;
623 ctx->asm_comments = asm_comments;
624
625 ctx->filename_cache = stringcache_create(MallocBridge, FreeBridge, ctx);
626 okay = ((okay) && (ctx->filename_cache != NULL));
627
628 ctx->file_macro = get_define(ctx);
629 okay = ((okay) && (ctx->file_macro != NULL));
630 if ((okay) && (ctx->file_macro))
631 okay = ((ctx->file_macro->identifier = StrDup(ctx, "__FILE__")) != 0);
632
633 ctx->line_macro = get_define(ctx);
634 okay = ((okay) && (ctx->line_macro != NULL));
635 if ((okay) && (ctx->line_macro))
636 okay = ((ctx->line_macro->identifier = StrDup(ctx, "__LINE__")) != 0);
637
638 // let the usual preprocessor parser sort these out.
639 char *define_include = NULL;
640 unsigned int define_include_len = 0;
641 if ((okay) && (define_count > 0))
642 {
643 Buffer *predefbuf = buffer_create(256, MallocBridge, FreeBridge, ctx);
644 okay = okay && (predefbuf != NULL);
645 for (i = 0; okay && (i < define_count); i++)
646 {
647 okay = okay && buffer_append_fmt(predefbuf, "#define %s %s\n",
648 defines[i].identifier, defines[i].definition);
649 } // for
650
651 define_include_len = buffer_size(predefbuf);
652 if (define_include_len > 0)
653 {
654 define_include = buffer_flatten(predefbuf);
655 okay = okay && (define_include != NULL);
656 } // if
657 buffer_destroy(predefbuf);
658 } // if
659
660 if ((okay) && (!push_source(ctx,fname,source,sourcelen,1,NULL)))
661 okay = 0;
662
663 if ((okay) && (define_include_len > 0))
664 {
665 assert(define_include != NULL);
666 okay = push_source(ctx, "<predefined macros>", define_include,
667 define_include_len, 1, close_define_include);
668 } // if
669
670 if (!okay)
671 {
672 preprocessor_end((Preprocessor *) ctx);
673 return NULL;
674 } // if
675
676 return (Preprocessor *) ctx;
677 } // preprocessor_start
678
679
preprocessor_end(Preprocessor * _ctx)680 void preprocessor_end(Preprocessor *_ctx)
681 {
682 Context *ctx = (Context *) _ctx;
683 if (ctx == NULL)
684 return;
685
686 while (ctx->include_stack != NULL)
687 pop_source(ctx);
688
689 put_all_defines(ctx);
690
691 if (ctx->filename_cache != NULL)
692 stringcache_destroy(ctx->filename_cache);
693
694 free_define(ctx, ctx->file_macro);
695 free_define(ctx, ctx->line_macro);
696 free_define_pool(ctx);
697 free_conditional_pool(ctx);
698 free_include_pool(ctx);
699
700 Free(ctx, ctx);
701 } // preprocessor_end
702
703
preprocessor_outofmemory(Preprocessor * _ctx)704 int preprocessor_outofmemory(Preprocessor *_ctx)
705 {
706 Context *ctx = (Context *) _ctx;
707 return ctx->out_of_memory;
708 } // preprocessor_outofmemory
709
710
pushback(IncludeState * state)711 static inline void pushback(IncludeState *state)
712 {
713 #if DEBUG_PREPROCESSOR
714 printf("PREPROCESSOR PUSHBACK\n");
715 #endif
716 assert(!state->pushedback);
717 state->pushedback = 1;
718 } // pushback
719
720
lexer(IncludeState * state)721 static Token lexer(IncludeState *state)
722 {
723 if (!state->pushedback)
724 return preprocessor_lexer(state);
725 state->pushedback = 0;
726 return state->tokenval;
727 } // lexer
728
729
730 // !!! FIXME: parsing fails on preprocessor directives should skip rest of line.
require_newline(IncludeState * state)731 static int require_newline(IncludeState *state)
732 {
733 const Token token = lexer(state);
734 pushback(state); // rewind no matter what.
735 return ( (token == TOKEN_INCOMPLETE_COMMENT) || // call it an eol.
736 (token == ((Token) '\n')) || (token == TOKEN_EOI) );
737 } // require_newline
738
739 // !!! FIXME: didn't we implement this by hand elsewhere?
token_to_int(IncludeState * state)740 static int token_to_int(IncludeState *state)
741 {
742 assert(state->tokenval == TOKEN_INT_LITERAL);
743 char *buf = (char *) alloca(state->tokenlen+1);
744 memcpy(buf, state->token, state->tokenlen);
745 buf[state->tokenlen] = '\0';
746 return atoi(buf);
747 } // token_to_int
748
749
handle_pp_include(Context * ctx)750 static void handle_pp_include(Context *ctx)
751 {
752 IncludeState *state = ctx->include_stack;
753 Token token = lexer(state);
754 MOJOSHADER_includeType incltype;
755 char *filename = NULL;
756 int bogus = 0;
757
758 if (token == TOKEN_STRING_LITERAL)
759 incltype = MOJOSHADER_INCLUDETYPE_LOCAL;
760 else if (token == ((Token) '<'))
761 {
762 incltype = MOJOSHADER_INCLUDETYPE_SYSTEM;
763 // can't use lexer, since every byte between the < > pair is
764 // considered part of the filename. :/
765 while (!bogus)
766 {
767 if ( !(bogus = (state->bytes_left == 0)) )
768 {
769 const char ch = *state->source;
770 if ( !(bogus = ((ch == '\r') || (ch == '\n'))) )
771 {
772 state->source++;
773 state->bytes_left--;
774
775 if (ch == '>')
776 break;
777 } // if
778 } // if
779 } // while
780 } // else if
781 else
782 {
783 bogus = 1;
784 } // else
785
786 if (!bogus)
787 {
788 state->token++; // skip '<' or '\"'...
789 const unsigned int len = ((unsigned int) (state->source-state->token));
790 filename = (char *) alloca(len);
791 memcpy(filename, state->token, len-1);
792 filename[len-1] = '\0';
793 bogus = !require_newline(state);
794 } // if
795
796 if (bogus)
797 {
798 fail(ctx, "Invalid #include directive");
799 return;
800 } // else
801
802 const char *newdata = NULL;
803 unsigned int newbytes = 0;
804 if ((ctx->open_callback == NULL) || (ctx->close_callback == NULL))
805 {
806 fail(ctx, "Saw #include, but no include callbacks defined");
807 return;
808 } // if
809
810 if (!ctx->open_callback(incltype, filename, state->source_base,
811 &newdata, &newbytes, ctx->malloc,
812 ctx->free, ctx->malloc_data))
813 {
814 fail(ctx, "Include callback failed"); // !!! FIXME: better error
815 return;
816 } // if
817
818 MOJOSHADER_includeClose callback = ctx->close_callback;
819 if (!push_source(ctx, filename, newdata, newbytes, 1, callback))
820 {
821 assert(ctx->out_of_memory);
822 ctx->close_callback(newdata, ctx->malloc, ctx->free, ctx->malloc_data);
823 } // if
824 } // handle_pp_include
825
826
handle_pp_line(Context * ctx)827 static void handle_pp_line(Context *ctx)
828 {
829 IncludeState *state = ctx->include_stack;
830 char *filename = NULL;
831 int linenum = 0;
832 int bogus = 0;
833
834 if (lexer(state) != TOKEN_INT_LITERAL)
835 bogus = 1;
836 else
837 linenum = token_to_int(state);
838
839 if (!bogus)
840 {
841 Token t = lexer(state);
842 if (t == ((Token) '\n'))
843 {
844 state->line = linenum;
845 return;
846 }
847 bogus = (t != TOKEN_STRING_LITERAL);
848 }
849
850 if (!bogus)
851 {
852 state->token++; // skip '\"'...
853 filename = (char *) alloca(state->tokenlen);
854 memcpy(filename, state->token, state->tokenlen-1);
855 filename[state->tokenlen-1] = '\0';
856 bogus = !require_newline(state);
857 } // if
858
859 if (bogus)
860 {
861 fail(ctx, "Invalid #line directive");
862 return;
863 } // if
864
865 const char *cached = stringcache(ctx->filename_cache, filename);
866 state->filename = cached; // may be NULL if stringcache() failed.
867 state->line = linenum;
868 } // handle_pp_line
869
870
handle_pp_error(Context * ctx)871 static void handle_pp_error(Context *ctx)
872 {
873 IncludeState *state = ctx->include_stack;
874 char *ptr = ctx->failstr;
875 int avail = sizeof (ctx->failstr) - 1;
876 int cpy = 0;
877 int done = 0;
878
879 const char *prefix = "#error";
880 const size_t prefixlen = strlen(prefix);
881 strcpy(ctx->failstr, prefix);
882 avail -= prefixlen;
883 ptr += prefixlen;
884
885 state->report_whitespace = 1;
886 while (!done)
887 {
888 const Token token = lexer(state);
889 switch (token)
890 {
891 case ((Token) '\n'):
892 state->line--; // make sure error is on the right line.
893 // fall through!
894 case TOKEN_INCOMPLETE_COMMENT:
895 case TOKEN_EOI:
896 pushback(state); // move back so we catch this later.
897 done = 1;
898 break;
899
900 case ((Token) ' '):
901 if (!avail)
902 break;
903 *(ptr++) = ' ';
904 avail--;
905 break;
906
907 default:
908 cpy = Min(avail, (int) state->tokenlen);
909 if (cpy)
910 memcpy(ptr, state->token, cpy);
911 ptr += cpy;
912 avail -= cpy;
913 break;
914 } // switch
915 } // while
916
917 *ptr = '\0';
918 state->report_whitespace = 0;
919 ctx->isfail = 1;
920 } // handle_pp_error
921
922
handle_pp_define(Context * ctx)923 static void handle_pp_define(Context *ctx)
924 {
925 IncludeState *state = ctx->include_stack;
926 int done = 0;
927
928 if (lexer(state) != TOKEN_IDENTIFIER)
929 {
930 fail(ctx, "Macro names must be identifiers");
931 return;
932 } // if
933
934 char *definition = NULL;
935 char *sym = (char *) Malloc(ctx, state->tokenlen+1);
936 if (sym == NULL)
937 return;
938 memcpy(sym, state->token, state->tokenlen);
939 sym[state->tokenlen] = '\0';
940
941 if (strcmp(sym, "defined") == 0)
942 {
943 Free(ctx, sym);
944 fail(ctx, "'defined' cannot be used as a macro name");
945 return;
946 } // if
947
948 // Don't treat these symbols as special anymore if they get (re)#defined.
949 if (strcmp(sym, "__FILE__") == 0)
950 {
951 if (ctx->file_macro)
952 {
953 failf(ctx, "'%s' already defined", sym); // !!! FIXME: warning?
954 free_define(ctx, ctx->file_macro);
955 ctx->file_macro = NULL;
956 } // if
957 } // if
958 else if (strcmp(sym, "__LINE__") == 0)
959 {
960 if (ctx->line_macro)
961 {
962 failf(ctx, "'%s' already defined", sym); // !!! FIXME: warning?
963 free_define(ctx, ctx->line_macro);
964 ctx->line_macro = NULL;
965 } // if
966 } // else if
967
968 // #define a(b) is different than #define a (b) :(
969 state->report_whitespace = 1;
970 lexer(state);
971 state->report_whitespace = 0;
972
973 int params = 0;
974 char **idents = NULL;
975 static const char space = ' ';
976
977 if (state->tokenval == ((Token) ' '))
978 lexer(state); // skip it.
979 else if (state->tokenval == ((Token) '('))
980 {
981 IncludeState saved;
982 memcpy(&saved, state, sizeof (IncludeState));
983 while (1)
984 {
985 if (lexer(state) != TOKEN_IDENTIFIER)
986 break;
987 params++;
988 if (lexer(state) != ((Token) ','))
989 break;
990 } // while
991
992 if (state->tokenval != ((Token) ')'))
993 {
994 fail(ctx, "syntax error in macro parameter list");
995 goto handle_pp_define_failed;
996 } // if
997
998 if (params == 0) // special case for void args: "#define a() b"
999 params = -1;
1000 else
1001 {
1002 idents = (char **) Malloc(ctx, sizeof (char *) * params);
1003 if (idents == NULL)
1004 goto handle_pp_define_failed;
1005
1006 // roll all the way back, do it again.
1007 memcpy(state, &saved, sizeof (IncludeState));
1008 memset(idents, '\0', sizeof (char *) * params);
1009
1010 int i;
1011 for (i = 0; i < params; i++)
1012 {
1013 lexer(state);
1014 assert(state->tokenval == TOKEN_IDENTIFIER);
1015
1016 char *dst = (char *) Malloc(ctx, state->tokenlen+1);
1017 if (dst == NULL)
1018 break;
1019
1020 memcpy(dst, state->token, state->tokenlen);
1021 dst[state->tokenlen] = '\0';
1022 idents[i] = dst;
1023
1024 if (i < (params-1))
1025 {
1026 lexer(state);
1027 assert(state->tokenval == ((Token) ','));
1028 } // if
1029 } // for
1030
1031 if (i != params)
1032 {
1033 assert(ctx->out_of_memory);
1034 goto handle_pp_define_failed;
1035 } // if
1036
1037 lexer(state);
1038 assert(state->tokenval == ((Token) ')'));
1039 } // else
1040
1041 lexer(state);
1042 } // else if
1043
1044 pushback(state);
1045
1046 Buffer *buffer = buffer_create(128, MallocBridge, FreeBridge, ctx);
1047
1048 state->report_whitespace = 1;
1049 while ((!done) && (!ctx->out_of_memory))
1050 {
1051 const Token token = lexer(state);
1052 switch (token)
1053 {
1054 case TOKEN_INCOMPLETE_COMMENT:
1055 case TOKEN_EOI:
1056 pushback(state); // move back so we catch this later.
1057 done = 1;
1058 break;
1059
1060 case ((Token) '\n'):
1061 done = 1;
1062 break;
1063
1064 case ((Token) ' '): // may not actually point to ' '.
1065 assert(buffer_size(buffer) > 0);
1066 buffer_append(buffer, &space, 1);
1067 break;
1068
1069 default:
1070 buffer_append(buffer, state->token, state->tokenlen);
1071 break;
1072 } // switch
1073 } // while
1074 state->report_whitespace = 0;
1075
1076 size_t buflen = buffer_size(buffer) + 1;
1077 if (!ctx->out_of_memory)
1078 definition = buffer_flatten(buffer);
1079
1080 buffer_destroy(buffer);
1081
1082 if (ctx->out_of_memory)
1083 goto handle_pp_define_failed;
1084
1085 int hashhash_error = 0;
1086 if ((buflen > 2) && (definition[0] == '#') && (definition[1] == '#'))
1087 {
1088 hashhash_error = 1;
1089 buflen -= 2;
1090 memmove(definition, definition + 2, buflen);
1091 } // if
1092
1093 if (buflen > 2)
1094 {
1095 char *ptr = (definition + buflen) - 2;
1096 if (*ptr == ' ')
1097 {
1098 ptr--;
1099 buflen--;
1100 } // if
1101 if ((buflen > 2) && (ptr[0] == '#') && (ptr[-1] == '#'))
1102 {
1103 hashhash_error = 1;
1104 buflen -= 2;
1105 ptr[-1] = '\0';
1106 } // if
1107 } // if
1108
1109 if (hashhash_error)
1110 fail(ctx, "'##' cannot appear at either end of a macro expansion");
1111
1112 assert(done);
1113
1114 if (!add_define(ctx, sym, definition, idents, params))
1115 goto handle_pp_define_failed;
1116
1117 return;
1118
1119 handle_pp_define_failed:
1120 Free(ctx, sym);
1121 Free(ctx, definition);
1122 if (idents != NULL)
1123 {
1124 while (params--)
1125 Free(ctx, idents[params]);
1126 } // if
1127 Free(ctx, idents);
1128 } // handle_pp_define
1129
1130
handle_pp_undef(Context * ctx)1131 static void handle_pp_undef(Context *ctx)
1132 {
1133 IncludeState *state = ctx->include_stack;
1134
1135 if (lexer(state) != TOKEN_IDENTIFIER)
1136 {
1137 fail(ctx, "Macro names must be indentifiers");
1138 return;
1139 } // if
1140
1141 char *sym = (char *) alloca(state->tokenlen+1);
1142 memcpy(sym, state->token, state->tokenlen);
1143 sym[state->tokenlen] = '\0';
1144
1145 if (!require_newline(state))
1146 {
1147 fail(ctx, "Invalid #undef directive");
1148 return;
1149 } // if
1150
1151 if (strcmp(sym, "__FILE__") == 0)
1152 {
1153 if (ctx->file_macro)
1154 {
1155 failf(ctx, "undefining \"%s\"", sym); // !!! FIXME: should be warning.
1156 free_define(ctx, ctx->file_macro);
1157 ctx->file_macro = NULL;
1158 } // if
1159 } // if
1160 else if (strcmp(sym, "__LINE__") == 0)
1161 {
1162 if (ctx->line_macro)
1163 {
1164 failf(ctx, "undefining \"%s\"", sym); // !!! FIXME: should be warning.
1165 free_define(ctx, ctx->line_macro);
1166 ctx->line_macro = NULL;
1167 } // if
1168 } // if
1169
1170 remove_define(ctx, sym);
1171 } // handle_pp_undef
1172
1173
_handle_pp_ifdef(Context * ctx,const Token type)1174 static Conditional *_handle_pp_ifdef(Context *ctx, const Token type)
1175 {
1176 IncludeState *state = ctx->include_stack;
1177
1178 assert((type == TOKEN_PP_IFDEF) || (type == TOKEN_PP_IFNDEF));
1179
1180 if (lexer(state) != TOKEN_IDENTIFIER)
1181 {
1182 fail(ctx, "Macro names must be indentifiers");
1183 return NULL;
1184 } // if
1185
1186 char *sym = (char *) alloca(state->tokenlen+1);
1187 memcpy(sym, state->token, state->tokenlen);
1188 sym[state->tokenlen] = '\0';
1189
1190 if (!require_newline(state))
1191 {
1192 if (type == TOKEN_PP_IFDEF)
1193 fail(ctx, "Invalid #ifdef directive");
1194 else
1195 fail(ctx, "Invalid #ifndef directive");
1196 return NULL;
1197 } // if
1198
1199 Conditional *conditional = get_conditional(ctx);
1200 assert((conditional != NULL) || (ctx->out_of_memory));
1201 if (conditional == NULL)
1202 return NULL;
1203
1204 Conditional *parent = state->conditional_stack;
1205 const int found = (find_define(ctx, sym) != NULL);
1206 const int chosen = (type == TOKEN_PP_IFDEF) ? found : !found;
1207 const int skipping = ( (((parent) && (parent->skipping))) || (!chosen) );
1208
1209 conditional->type = type;
1210 conditional->linenum = state->line - 1;
1211 conditional->skipping = skipping;
1212 conditional->chosen = chosen;
1213 conditional->next = parent;
1214 state->conditional_stack = conditional;
1215 return conditional;
1216 } // _handle_pp_ifdef
1217
1218
handle_pp_ifdef(Context * ctx)1219 static inline void handle_pp_ifdef(Context *ctx)
1220 {
1221 _handle_pp_ifdef(ctx, TOKEN_PP_IFDEF);
1222 } // handle_pp_ifdef
1223
1224
handle_pp_ifndef(Context * ctx)1225 static inline void handle_pp_ifndef(Context *ctx)
1226 {
1227 _handle_pp_ifdef(ctx, TOKEN_PP_IFNDEF);
1228 } // handle_pp_ifndef
1229
1230
replace_and_push_macro(Context * ctx,const Define * def,const Define * params)1231 static int replace_and_push_macro(Context *ctx, const Define *def,
1232 const Define *params)
1233 {
1234 char *final = NULL;
1235
1236 // We push the #define and lex it, building a buffer with argument
1237 // replacement, stringification, and concatenation.
1238 Buffer *buffer = buffer_create(128, MallocBridge, FreeBridge, ctx);
1239 if (buffer == NULL)
1240 return 0;
1241
1242 IncludeState *state = ctx->include_stack;
1243 if (!push_source(ctx, state->filename, def->definition,
1244 strlen(def->definition), state->line, NULL))
1245 {
1246 buffer_destroy(buffer);
1247 return 0;
1248 } // if
1249
1250 state = ctx->include_stack;
1251 while (lexer(state) != TOKEN_EOI)
1252 {
1253 int wantorig = 0;
1254 const Define *arg = NULL;
1255
1256 // put a space between tokens if we're not concatenating.
1257 if (state->tokenval == TOKEN_HASHHASH) // concatenate?
1258 {
1259 wantorig = 1;
1260 lexer(state);
1261 assert(state->tokenval != TOKEN_EOI);
1262 } // if
1263 else
1264 {
1265 if (buffer_size(buffer) > 0)
1266 {
1267 if (!buffer_append(buffer, " ", 1))
1268 goto replace_and_push_macro_failed;
1269 } // if
1270 } // else
1271
1272 const char *data = state->token;
1273 unsigned int len = state->tokenlen;
1274
1275 if (state->tokenval == TOKEN_HASH) // stringify?
1276 {
1277 lexer(state);
1278 assert(state->tokenval != TOKEN_EOI); // we checked for this.
1279
1280 if (!buffer_append(buffer, "\"", 1))
1281 goto replace_and_push_macro_failed;
1282
1283 if (state->tokenval == TOKEN_IDENTIFIER)
1284 {
1285 arg = find_macro_arg(state, params);
1286 if (arg != NULL)
1287 {
1288 data = arg->original;
1289 len = strlen(data);
1290 } // if
1291 } // if
1292
1293 if (!buffer_append(buffer, data, len))
1294 goto replace_and_push_macro_failed;
1295
1296 if (!buffer_append(buffer, "\"", 1))
1297 goto replace_and_push_macro_failed;
1298
1299 continue;
1300 } // if
1301
1302 if (state->tokenval == TOKEN_IDENTIFIER)
1303 {
1304 arg = find_macro_arg(state, params);
1305 if (arg != NULL)
1306 {
1307 if (!wantorig)
1308 {
1309 wantorig = (lexer(state) == TOKEN_HASHHASH);
1310 pushback(state);
1311 } // if
1312 data = wantorig ? arg->original : arg->definition;
1313 len = strlen(data);
1314 } // if
1315 } // if
1316
1317 if (!buffer_append(buffer, data, len))
1318 goto replace_and_push_macro_failed;
1319 } // while
1320
1321 final = buffer_flatten(buffer);
1322 if (!final)
1323 goto replace_and_push_macro_failed;
1324
1325 buffer_destroy(buffer);
1326 pop_source(ctx); // ditch the macro.
1327 state = ctx->include_stack;
1328 if (!push_source(ctx, state->filename, final, strlen(final), state->line,
1329 close_define_include))
1330 {
1331 Free(ctx, final);
1332 return 0;
1333 } // if
1334
1335 return 1;
1336
1337 replace_and_push_macro_failed:
1338 pop_source(ctx);
1339 buffer_destroy(buffer);
1340 return 0;
1341 } // replace_and_push_macro
1342
1343
handle_macro_args(Context * ctx,const char * sym,const Define * def)1344 static int handle_macro_args(Context *ctx, const char *sym, const Define *def)
1345 {
1346 int retval = 0;
1347 IncludeState *state = ctx->include_stack;
1348 Define *params = NULL;
1349 const int expected = (def->paramcount < 0) ? 0 : def->paramcount;
1350 int saw_params = 0;
1351 IncludeState saved; // can't pushback, we need the original token.
1352 memcpy(&saved, state, sizeof (IncludeState));
1353 if (lexer(state) != ((Token) '('))
1354 {
1355 memcpy(state, &saved, sizeof (IncludeState));
1356 goto handle_macro_args_failed; // gcc abandons replacement, too.
1357 } // if
1358
1359 state->report_whitespace = 1;
1360
1361 int void_call = 0;
1362 int paren = 1;
1363 while (paren > 0)
1364 {
1365 Buffer *buffer = buffer_create(128, MallocBridge, FreeBridge, ctx);
1366 Buffer *origbuffer = buffer_create(128, MallocBridge, FreeBridge, ctx);
1367
1368 Token t = lexer(state);
1369
1370 assert(!void_call);
1371
1372 while (1)
1373 {
1374 const char *origexpr = state->token;
1375 unsigned int origexprlen = state->tokenlen;
1376 const char *expr = state->token;
1377 unsigned int exprlen = state->tokenlen;
1378
1379 if (t == ((Token) '('))
1380 paren++;
1381
1382 else if (t == ((Token) ')'))
1383 {
1384 paren--;
1385 if (paren < 1) // end of macro?
1386 break;
1387 } // else if
1388
1389 else if (t == ((Token) ','))
1390 {
1391 if (paren == 1) // new macro arg?
1392 break;
1393 } // else if
1394
1395 else if (t == ((Token) ' '))
1396 {
1397 // don't add whitespace to the start, so we recognize
1398 // void calls correctly.
1399 origexpr = expr = " ";
1400 origexprlen = (buffer_size(origbuffer) == 0) ? 0 : 1;
1401 exprlen = (buffer_size(buffer) == 0) ? 0 : 1;
1402 } // else if
1403
1404 else if (t == TOKEN_IDENTIFIER)
1405 {
1406 const Define *def = find_define_by_token(ctx);
1407 // don't replace macros with arguments so they replace correctly, later.
1408 if ((def) && (def->paramcount == 0))
1409 {
1410 expr = def->definition;
1411 exprlen = strlen(def->definition);
1412 } // if
1413 } // else if
1414
1415 else if ((t == TOKEN_INCOMPLETE_COMMENT) || (t == TOKEN_EOI))
1416 {
1417 pushback(state);
1418 fail(ctx, "Unterminated macro list");
1419 goto handle_macro_args_failed;
1420 } // else if
1421
1422 assert(expr != NULL);
1423
1424 if (!buffer_append(buffer, expr, exprlen))
1425 goto handle_macro_args_failed;
1426
1427 if (!buffer_append(origbuffer, origexpr, origexprlen))
1428 goto handle_macro_args_failed;
1429
1430 t = lexer(state);
1431 } // while
1432
1433 if (buffer_size(buffer) == 0)
1434 void_call = ((saw_params == 0) && (paren == 0));
1435
1436 if (saw_params < expected)
1437 {
1438 const int origdeflen = (int) buffer_size(origbuffer);
1439 char *origdefinition = buffer_flatten(origbuffer);
1440 const int deflen = (int) buffer_size(buffer);
1441 char *definition = buffer_flatten(buffer);
1442 Define *p = get_define(ctx);
1443 if ((!origdefinition) || (!definition) || (!p))
1444 {
1445 Free(ctx, origdefinition);
1446 Free(ctx, definition);
1447 buffer_destroy(origbuffer);
1448 buffer_destroy(buffer);
1449 free_define(ctx, p);
1450 goto handle_macro_args_failed;
1451 } // if
1452
1453 // trim any whitespace from the end of the string...
1454 int i;
1455 for (i = deflen - 1; i >= 0; i--)
1456 {
1457 if (definition[i] == ' ')
1458 definition[i] = '\0';
1459 else
1460 break;
1461 } // for
1462
1463 for (i = origdeflen - 1; i >= 0; i--)
1464 {
1465 if (origdefinition[i] == ' ')
1466 origdefinition[i] = '\0';
1467 else
1468 break;
1469 } // for
1470
1471 p->identifier = def->parameters[saw_params];
1472 p->definition = definition;
1473 p->original = origdefinition;
1474 p->next = params;
1475 params = p;
1476 } // if
1477
1478 buffer_destroy(buffer);
1479 buffer_destroy(origbuffer);
1480 saw_params++;
1481 } // while
1482
1483 assert(paren == 0);
1484
1485 // "a()" should match "#define a()" ...
1486 if ((expected == 0) && (saw_params == 1) && (void_call))
1487 {
1488 assert(params == NULL);
1489 saw_params = 0;
1490 } // if
1491
1492 if (saw_params != expected)
1493 {
1494 failf(ctx, "macro '%s' passed %d arguments, but requires %d",
1495 sym, saw_params, expected);
1496 goto handle_macro_args_failed;
1497 } // if
1498
1499 // this handles arg replacement and the '##' and '#' operators.
1500 retval = replace_and_push_macro(ctx, def, params);
1501
1502 handle_macro_args_failed:
1503 while (params)
1504 {
1505 Define *next = params->next;
1506 params->identifier = NULL;
1507 free_define(ctx, params);
1508 params = next;
1509 } // while
1510
1511 state->report_whitespace = 0;
1512 return retval;
1513 } // handle_macro_args
1514
1515
handle_pp_identifier(Context * ctx)1516 static int handle_pp_identifier(Context *ctx)
1517 {
1518 if (ctx->recursion_count++ >= 256) // !!! FIXME: gcc can figure this out.
1519 {
1520 fail(ctx, "Recursing macros");
1521 return 0;
1522 } // if
1523
1524 IncludeState *state = ctx->include_stack;
1525 const char *fname = state->filename;
1526 const unsigned int line = state->line;
1527 char *sym = (char *) alloca(state->tokenlen+1);
1528 memcpy(sym, state->token, state->tokenlen);
1529 sym[state->tokenlen] = '\0';
1530
1531 // Is this identifier #defined?
1532 const Define *def = find_define(ctx, sym);
1533 if (def == NULL)
1534 return 0; // just send the token through unchanged.
1535 else if (def->paramcount != 0)
1536 return handle_macro_args(ctx, sym, def);
1537
1538 const size_t deflen = strlen(def->definition);
1539 return push_source(ctx, fname, def->definition, deflen, line, NULL);
1540 } // handle_pp_identifier
1541
1542
find_precedence(const Token token)1543 static int find_precedence(const Token token)
1544 {
1545 // operator precedence, left and right associative...
1546 typedef struct { int precedence; Token token; } Precedence;
1547 static const Precedence ops[] = {
1548 { 0, TOKEN_OROR }, { 1, TOKEN_ANDAND }, { 2, ((Token) '|') },
1549 { 3, ((Token) '^') }, { 4, ((Token) '&') }, { 5, TOKEN_NEQ },
1550 { 6, TOKEN_EQL }, { 7, ((Token) '<') }, { 7, ((Token) '>') },
1551 { 7, TOKEN_LEQ }, { 7, TOKEN_GEQ }, { 8, TOKEN_LSHIFT },
1552 { 8, TOKEN_RSHIFT }, { 9, ((Token) '-') }, { 9, ((Token) '+') },
1553 { 10, ((Token) '%') }, { 10, ((Token) '/') }, { 10, ((Token) '*') },
1554 { 11, TOKEN_PP_UNARY_PLUS }, { 11, TOKEN_PP_UNARY_MINUS },
1555 { 11, ((Token) '!') }, { 11, ((Token) '~') },
1556 };
1557
1558 size_t i;
1559 for (i = 0; i < STATICARRAYLEN(ops); i++)
1560 {
1561 if (ops[i].token == token)
1562 return ops[i].precedence;
1563 } // for
1564
1565 return -1;
1566 } // find_precedence
1567
1568 // !!! FIXME: we're using way too much stack space here...
1569 typedef struct RpnTokens
1570 {
1571 int isoperator;
1572 int value;
1573 } RpnTokens;
1574
interpret_rpn(const RpnTokens * tokens,int tokencount,int * error)1575 static long interpret_rpn(const RpnTokens *tokens, int tokencount, int *error)
1576 {
1577 long stack[128];
1578 size_t stacksize = 0;
1579
1580 *error = 1;
1581
1582 #define NEED_X_TOKENS(x) do { if (stacksize < x) return 0; } while (0)
1583
1584 #define BINARY_OPERATION(op) do { \
1585 NEED_X_TOKENS(2); \
1586 stack[stacksize-2] = stack[stacksize-2] op stack[stacksize-1]; \
1587 stacksize--; \
1588 } while (0)
1589
1590 #define UNARY_OPERATION(op) do { \
1591 NEED_X_TOKENS(1); \
1592 stack[stacksize-1] = op stack[stacksize-1]; \
1593 } while (0)
1594
1595 while (tokencount-- > 0)
1596 {
1597 if (!tokens->isoperator)
1598 {
1599 assert(stacksize < STATICARRAYLEN(stack));
1600 stack[stacksize++] = (long) tokens->value;
1601 tokens++;
1602 continue;
1603 } // if
1604
1605 // operators.
1606 switch (tokens->value)
1607 {
1608 case '!': UNARY_OPERATION(!); break;
1609 case '~': UNARY_OPERATION(~); break;
1610 case TOKEN_PP_UNARY_MINUS: UNARY_OPERATION(-); break;
1611 case TOKEN_PP_UNARY_PLUS: UNARY_OPERATION(+); break;
1612 case TOKEN_OROR: BINARY_OPERATION(||); break;
1613 case TOKEN_ANDAND: BINARY_OPERATION(&&); break;
1614 case '|': BINARY_OPERATION(|); break;
1615 case '^': BINARY_OPERATION(^); break;
1616 case '&': BINARY_OPERATION(&); break;
1617 case TOKEN_NEQ: BINARY_OPERATION(!=); break;
1618 case TOKEN_EQL: BINARY_OPERATION(==); break;
1619 case '<': BINARY_OPERATION(<); break;
1620 case '>': BINARY_OPERATION(>); break;
1621 case TOKEN_LEQ: BINARY_OPERATION(<=); break;
1622 case TOKEN_GEQ: BINARY_OPERATION(>=); break;
1623 case TOKEN_LSHIFT: BINARY_OPERATION(<<); break;
1624 case TOKEN_RSHIFT: BINARY_OPERATION(>>); break;
1625 case '-': BINARY_OPERATION(-); break;
1626 case '+': BINARY_OPERATION(+); break;
1627 case '%': BINARY_OPERATION(%); break;
1628 case '/': BINARY_OPERATION(/); break;
1629 case '*': BINARY_OPERATION(*); break;
1630 default: return 0;
1631 } // switch
1632
1633 tokens++;
1634 } // while
1635
1636 #undef NEED_X_TOKENS
1637 #undef BINARY_OPERATION
1638 #undef UNARY_OPERATION
1639
1640 if (stacksize != 1)
1641 return 0;
1642
1643 *error = 0;
1644 return stack[0];
1645 } // interpret_rpn
1646
1647 // http://en.wikipedia.org/wiki/Shunting_yard_algorithm
1648 // Convert from infix to postfix, then use this for constant folding.
1649 // Everything that parses should fold down to a constant value: any
1650 // identifiers that aren't resolved as macros become zero. Anything we
1651 // don't explicitly expect becomes a parsing error.
1652 // returns 1 (true), 0 (false), or -1 (error)
reduce_pp_expression(Context * ctx)1653 static int reduce_pp_expression(Context *ctx)
1654 {
1655 IncludeState *orig_state = ctx->include_stack;
1656 RpnTokens output[128];
1657 Token stack[64];
1658 Token previous_token = TOKEN_UNKNOWN;
1659 size_t outputsize = 0;
1660 size_t stacksize = 0;
1661 int matched = 0;
1662 int done = 0;
1663
1664 #define ADD_TO_OUTPUT(op, val) \
1665 assert(outputsize < STATICARRAYLEN(output)); \
1666 output[outputsize].isoperator = op; \
1667 output[outputsize].value = val; \
1668 outputsize++;
1669
1670 #define PUSH_TO_STACK(t) \
1671 assert(stacksize < STATICARRAYLEN(stack)); \
1672 stack[stacksize] = t; \
1673 stacksize++;
1674
1675 while (!done)
1676 {
1677 IncludeState *state = ctx->include_stack;
1678 Token token = lexer(state);
1679 int isleft = 1;
1680 int precedence = -1;
1681
1682 if ( (token == ((Token) '!')) || (token == ((Token) '~')) )
1683 isleft = 0;
1684 else if (token == ((Token) '-'))
1685 {
1686 if ((isleft = (previous_token == TOKEN_INT_LITERAL)) == 0)
1687 token = TOKEN_PP_UNARY_MINUS;
1688 } // else if
1689 else if (token == ((Token) '+'))
1690 {
1691 if ((isleft = (previous_token == TOKEN_INT_LITERAL)) == 0)
1692 token = TOKEN_PP_UNARY_PLUS;
1693 } // else if
1694
1695 if (token != TOKEN_IDENTIFIER)
1696 ctx->recursion_count = 0;
1697
1698 switch (token)
1699 {
1700 case TOKEN_EOI:
1701 if (state != orig_state) // end of a substate, or the expr?
1702 {
1703 pop_source(ctx);
1704 continue; // substate, go again with the parent state.
1705 } // if
1706 done = 1; // the expression itself is done.
1707 break;
1708
1709 case ((Token) '\n'):
1710 done = 1;
1711 break; // we're done!
1712
1713 case TOKEN_IDENTIFIER:
1714 if (handle_pp_identifier(ctx))
1715 continue; // go again with new IncludeState.
1716
1717 if ( (state->tokenlen == 7) &&
1718 (memcmp(state->token, "defined", 7) == 0) )
1719 {
1720 token = lexer(state);
1721 const int paren = (token == ((Token) '('));
1722 if (paren) // gcc doesn't let us nest parens here, either.
1723 token = lexer(state);
1724 if (token != TOKEN_IDENTIFIER)
1725 {
1726 fail(ctx, "operator 'defined' requires an identifier");
1727 return -1;
1728 } // if
1729 const int found = (find_define_by_token(ctx) != NULL);
1730
1731 if (paren)
1732 {
1733 if (lexer(state) != ((Token) ')'))
1734 {
1735 fail(ctx, "Unmatched ')'");
1736 return -1;
1737 } // if
1738 } // if
1739
1740 ADD_TO_OUTPUT(0, found);
1741 continue;
1742 } // if
1743
1744 // can't replace identifier with a number? It becomes zero.
1745 token = TOKEN_INT_LITERAL;
1746 ADD_TO_OUTPUT(0, 0);
1747 break;
1748
1749 case TOKEN_INT_LITERAL:
1750 ADD_TO_OUTPUT(0, token_to_int(state));
1751 break;
1752
1753 case ((Token) '('):
1754 PUSH_TO_STACK((Token) '(');
1755 break;
1756
1757 case ((Token) ')'):
1758 matched = 0;
1759 while (stacksize > 0)
1760 {
1761 const Token t = stack[--stacksize];
1762 if (t == ((Token) '('))
1763 {
1764 matched = 1;
1765 break;
1766 } // if
1767 ADD_TO_OUTPUT(1, t);
1768 } // while
1769
1770 if (!matched)
1771 {
1772 fail(ctx, "Unmatched ')'");
1773 return -1;
1774 } // if
1775 break;
1776
1777 default:
1778 precedence = find_precedence(token);
1779 // bogus token, or two operators together.
1780 if (precedence < 0)
1781 {
1782 pushback(state);
1783 fail(ctx, "Invalid expression");
1784 return -1;
1785 } // if
1786
1787 else // it's an operator.
1788 {
1789 while (stacksize > 0)
1790 {
1791 const Token t = stack[stacksize-1];
1792 const int p = find_precedence(t);
1793 if ( (p >= 0) &&
1794 ( ((isleft) && (precedence <= p)) ||
1795 ((!isleft) && (precedence < p)) ) )
1796 {
1797 stacksize--;
1798 ADD_TO_OUTPUT(1, t);
1799 } // if
1800 else
1801 {
1802 break;
1803 } // else
1804 } // while
1805 PUSH_TO_STACK(token);
1806 } // else
1807 break;
1808 } // switch
1809 previous_token = token;
1810 } // while
1811
1812 while (stacksize > 0)
1813 {
1814 const Token t = stack[--stacksize];
1815 if (t == ((Token) '('))
1816 {
1817 fail(ctx, "Unmatched ')'");
1818 return -1;
1819 } // if
1820 ADD_TO_OUTPUT(1, t);
1821 } // while
1822
1823 #undef ADD_TO_OUTPUT
1824 #undef PUSH_TO_STACK
1825
1826 // okay, you now have some validated data in reverse polish notation.
1827 #if DEBUG_PREPROCESSOR
1828 printf("PREPROCESSOR EXPRESSION RPN:");
1829 int i = 0;
1830 for (i = 0; i < outputsize; i++)
1831 {
1832 if (!output[i].isoperator)
1833 printf(" %d", output[i].value);
1834 else
1835 {
1836 switch (output[i].value)
1837 {
1838 case TOKEN_OROR: printf(" ||"); break;
1839 case TOKEN_ANDAND: printf(" &&"); break;
1840 case TOKEN_NEQ: printf(" !="); break;
1841 case TOKEN_EQL: printf(" =="); break;
1842 case TOKEN_LEQ: printf(" <="); break;
1843 case TOKEN_GEQ: printf(" >="); break;
1844 case TOKEN_LSHIFT: printf(" <<"); break;
1845 case TOKEN_RSHIFT: printf(" >>"); break;
1846 case TOKEN_PP_UNARY_PLUS: printf(" +"); break;
1847 case TOKEN_PP_UNARY_MINUS: printf(" -"); break;
1848 default: printf(" %c", output[i].value); break;
1849 } // switch
1850 } // else
1851 } // for
1852 printf("\n");
1853 #endif
1854
1855 int error = 0;
1856 const long val = interpret_rpn(output, outputsize, &error);
1857
1858 #if DEBUG_PREPROCESSOR
1859 printf("PREPROCESSOR RPN RESULT: %ld%s\n", val, error ? " (ERROR)" : "");
1860 #endif
1861
1862 if (error)
1863 {
1864 fail(ctx, "Invalid expression");
1865 return -1;
1866 } // if
1867
1868 return ((val) ? 1 : 0);
1869 } // reduce_pp_expression
1870
1871
handle_pp_if(Context * ctx)1872 static Conditional *handle_pp_if(Context *ctx)
1873 {
1874 IncludeState *state = ctx->include_stack;
1875 const int result = reduce_pp_expression(ctx);
1876 if (result == -1)
1877 return NULL;
1878
1879 Conditional *conditional = get_conditional(ctx);
1880 assert((conditional != NULL) || (ctx->out_of_memory));
1881 if (conditional == NULL)
1882 return NULL;
1883
1884 Conditional *parent = state->conditional_stack;
1885 const int chosen = result;
1886 const int skipping = ( (((parent) && (parent->skipping))) || (!chosen) );
1887
1888 conditional->type = TOKEN_PP_IF;
1889 conditional->linenum = state->line - 1;
1890 conditional->skipping = skipping;
1891 conditional->chosen = chosen;
1892 conditional->next = parent;
1893 state->conditional_stack = conditional;
1894 return conditional;
1895 } // handle_pp_if
1896
1897
handle_pp_elif(Context * ctx)1898 static void handle_pp_elif(Context *ctx)
1899 {
1900 const int rc = reduce_pp_expression(ctx);
1901 if (rc == -1)
1902 return;
1903
1904 IncludeState *state = ctx->include_stack;
1905 Conditional *cond = state->conditional_stack;
1906 if (cond == NULL)
1907 fail(ctx, "#elif without #if");
1908 else if (cond->type == TOKEN_PP_ELSE)
1909 fail(ctx, "#elif after #else");
1910 else
1911 {
1912 const Conditional *parent = cond->next;
1913 cond->type = TOKEN_PP_ELIF;
1914 cond->skipping = (parent && parent->skipping) || cond->chosen || !rc;
1915 if (!cond->chosen)
1916 cond->chosen = rc;
1917 } // else
1918 } // handle_pp_elif
1919
1920
handle_pp_else(Context * ctx)1921 static void handle_pp_else(Context *ctx)
1922 {
1923 IncludeState *state = ctx->include_stack;
1924 Conditional *cond = state->conditional_stack;
1925
1926 if (!require_newline(state))
1927 fail(ctx, "Invalid #else directive");
1928 else if (cond == NULL)
1929 fail(ctx, "#else without #if");
1930 else if (cond->type == TOKEN_PP_ELSE)
1931 fail(ctx, "#else after #else");
1932 else
1933 {
1934 const Conditional *parent = cond->next;
1935 cond->type = TOKEN_PP_ELSE;
1936 cond->skipping = (parent && parent->skipping) || cond->chosen;
1937 if (!cond->chosen)
1938 cond->chosen = 1;
1939 } // else
1940 } // handle_pp_else
1941
1942
handle_pp_endif(Context * ctx)1943 static void handle_pp_endif(Context *ctx)
1944 {
1945 IncludeState *state = ctx->include_stack;
1946 Conditional *cond = state->conditional_stack;
1947
1948 if (!require_newline(state))
1949 fail(ctx, "Invalid #endif directive");
1950 else if (cond == NULL)
1951 fail(ctx, "Unmatched #endif");
1952 else
1953 {
1954 state->conditional_stack = cond->next; // pop it.
1955 put_conditional(ctx, cond);
1956 } // else
1957 } // handle_pp_endif
1958
1959
unterminated_pp_condition(Context * ctx)1960 static void unterminated_pp_condition(Context *ctx)
1961 {
1962 IncludeState *state = ctx->include_stack;
1963 Conditional *cond = state->conditional_stack;
1964
1965 // !!! FIXME: report the line number where the #if is, not the EOI.
1966 switch (cond->type)
1967 {
1968 case TOKEN_PP_IF: fail(ctx, "Unterminated #if"); break;
1969 case TOKEN_PP_IFDEF: fail(ctx, "Unterminated #ifdef"); break;
1970 case TOKEN_PP_IFNDEF: fail(ctx, "Unterminated #ifndef"); break;
1971 case TOKEN_PP_ELSE: fail(ctx, "Unterminated #else"); break;
1972 case TOKEN_PP_ELIF: fail(ctx, "Unterminated #elif"); break;
1973 default: assert(0 && "Shouldn't hit this case"); break;
1974 } // switch
1975
1976 // pop this conditional, we'll report the next error next time...
1977
1978 state->conditional_stack = cond->next; // pop it.
1979 put_conditional(ctx, cond);
1980 } // unterminated_pp_condition
1981
1982
_preprocessor_nexttoken(Preprocessor * _ctx,unsigned int * _len,Token * _token)1983 static inline const char *_preprocessor_nexttoken(Preprocessor *_ctx,
1984 unsigned int *_len, Token *_token)
1985 {
1986 Context *ctx = (Context *) _ctx;
1987
1988 while (1)
1989 {
1990 if (ctx->isfail)
1991 {
1992 ctx->isfail = 0;
1993 *_token = TOKEN_PREPROCESSING_ERROR;
1994 *_len = strlen(ctx->failstr);
1995 return ctx->failstr;
1996 } // if
1997
1998 IncludeState *state = ctx->include_stack;
1999 if (state == NULL)
2000 {
2001 *_token = TOKEN_EOI;
2002 *_len = 0;
2003 return NULL; // we're done!
2004 } // if
2005
2006 const Conditional *cond = state->conditional_stack;
2007 const int skipping = ((cond != NULL) && (cond->skipping));
2008
2009 const Token token = lexer(state);
2010
2011 if (token != TOKEN_IDENTIFIER)
2012 ctx->recursion_count = 0;
2013
2014 if (token == TOKEN_EOI)
2015 {
2016 assert(state->bytes_left == 0);
2017 if (state->conditional_stack != NULL)
2018 {
2019 unterminated_pp_condition(ctx);
2020 continue; // returns an error.
2021 } // if
2022
2023 pop_source(ctx);
2024 continue; // pick up again after parent's #include line.
2025 } // if
2026
2027 else if (token == TOKEN_INCOMPLETE_COMMENT)
2028 {
2029 fail(ctx, "Incomplete multiline comment");
2030 continue; // will return at top of loop.
2031 } // else if
2032
2033 else if (token == TOKEN_PP_IFDEF)
2034 {
2035 handle_pp_ifdef(ctx);
2036 continue; // get the next thing.
2037 } // else if
2038
2039 else if (token == TOKEN_PP_IFNDEF)
2040 {
2041 handle_pp_ifndef(ctx);
2042 continue; // get the next thing.
2043 } // else if
2044
2045 else if (token == TOKEN_PP_IF)
2046 {
2047 handle_pp_if(ctx);
2048 continue; // get the next thing.
2049 } // else if
2050
2051 else if (token == TOKEN_PP_ELIF)
2052 {
2053 handle_pp_elif(ctx);
2054 continue; // get the next thing.
2055 } // else if
2056
2057 else if (token == TOKEN_PP_ENDIF)
2058 {
2059 handle_pp_endif(ctx);
2060 continue; // get the next thing.
2061 } // else if
2062
2063 else if (token == TOKEN_PP_ELSE)
2064 {
2065 handle_pp_else(ctx);
2066 continue; // get the next thing.
2067 } // else if
2068
2069 // NOTE: Conditionals must be above (skipping) test.
2070 else if (skipping)
2071 continue; // just keep dumping tokens until we get end of block.
2072
2073 else if (token == TOKEN_PP_INCLUDE)
2074 {
2075 handle_pp_include(ctx);
2076 continue; // will return error or use new top of include_stack.
2077 } // else if
2078
2079 else if (token == TOKEN_PP_LINE)
2080 {
2081 handle_pp_line(ctx);
2082 continue; // get the next thing.
2083 } // else if
2084
2085 else if (token == TOKEN_PP_ERROR)
2086 {
2087 handle_pp_error(ctx);
2088 continue; // will return at top of loop.
2089 } // else if
2090
2091 else if (token == TOKEN_PP_DEFINE)
2092 {
2093 handle_pp_define(ctx);
2094 continue; // will return at top of loop.
2095 } // else if
2096
2097 else if (token == TOKEN_PP_UNDEF)
2098 {
2099 handle_pp_undef(ctx);
2100 continue; // will return at top of loop.
2101 } // else if
2102
2103 else if (token == TOKEN_PP_PRAGMA)
2104 {
2105 ctx->parsing_pragma = 1;
2106 } // else if
2107
2108 if (token == TOKEN_IDENTIFIER)
2109 {
2110 if (handle_pp_identifier(ctx))
2111 continue; // pushed the include_stack.
2112 } // else if
2113
2114 else if (token == ((Token) '\n'))
2115 {
2116 print_debug_lexing_position(state);
2117 if (ctx->parsing_pragma) // let this one through.
2118 ctx->parsing_pragma = 0;
2119 else
2120 {
2121 // preprocessor is line-oriented, nothing else gets newlines.
2122 continue; // get the next thing.
2123 } // else
2124 } // else if
2125
2126 assert(!skipping);
2127 *_token = token;
2128 *_len = state->tokenlen;
2129 return state->token;
2130 } // while
2131
2132 assert(0 && "shouldn't hit this code");
2133 *_token = TOKEN_UNKNOWN;
2134 *_len = 0;
2135 return NULL;
2136 } // _preprocessor_nexttoken
2137
2138
preprocessor_nexttoken(Preprocessor * ctx,unsigned int * len,Token * token)2139 const char *preprocessor_nexttoken(Preprocessor *ctx, unsigned int *len,
2140 Token *token)
2141 {
2142 const char *retval = _preprocessor_nexttoken(ctx, len, token);
2143 print_debug_token(retval, *len, *token);
2144 return retval;
2145 } // preprocessor_nexttoken
2146
2147
preprocessor_sourcepos(Preprocessor * _ctx,unsigned int * pos)2148 const char *preprocessor_sourcepos(Preprocessor *_ctx, unsigned int *pos)
2149 {
2150 Context *ctx = (Context *) _ctx;
2151 if (ctx->include_stack == NULL)
2152 {
2153 *pos = 0;
2154 return NULL;
2155 } // if
2156
2157 *pos = ctx->include_stack->line;
2158 return ctx->include_stack->filename;
2159 } // preprocessor_sourcepos
2160
2161
indent_buffer(Buffer * buffer,int n,const int newline)2162 static void indent_buffer(Buffer *buffer, int n, const int newline)
2163 {
2164 static char spaces[4] = { ' ', ' ', ' ', ' ' };
2165 if (newline)
2166 {
2167 while (n--)
2168 {
2169 if (!buffer_append(buffer, spaces, sizeof (spaces)))
2170 return;
2171 } // while
2172 } // if
2173 else
2174 {
2175 if (!buffer_append(buffer, spaces, 1))
2176 return;
2177 } // else
2178 } // indent_buffer
2179
2180
2181 static const MOJOSHADER_preprocessData out_of_mem_data_preprocessor = {
2182 1, &MOJOSHADER_out_of_mem_error, 0, 0, 0, 0, 0
2183 };
2184
2185
2186 // public API...
2187
MOJOSHADER_preprocess(const char * filename,const char * source,unsigned int sourcelen,const MOJOSHADER_preprocessorDefine * defines,unsigned int define_count,MOJOSHADER_includeOpen include_open,MOJOSHADER_includeClose include_close,MOJOSHADER_malloc m,MOJOSHADER_free f,void * d)2188 const MOJOSHADER_preprocessData *MOJOSHADER_preprocess(const char *filename,
2189 const char *source, unsigned int sourcelen,
2190 const MOJOSHADER_preprocessorDefine *defines,
2191 unsigned int define_count,
2192 MOJOSHADER_includeOpen include_open,
2193 MOJOSHADER_includeClose include_close,
2194 MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
2195 {
2196 MOJOSHADER_preprocessData *retval = NULL;
2197 Preprocessor *pp = NULL;
2198 ErrorList *errors = NULL;
2199 Buffer *buffer = NULL;
2200 Token token = TOKEN_UNKNOWN;
2201 const char *tokstr = NULL;
2202 int nl = 1;
2203 int indent = 0;
2204 unsigned int len = 0;
2205 char *output = NULL;
2206 int errcount = 0;
2207 size_t total_bytes = 0;
2208
2209 // !!! FIXME: what's wrong with ENDLINE_STR?
2210 #ifdef _WINDOWS
2211 static const char endline[] = { '\r', '\n' };
2212 #else
2213 static const char endline[] = { '\n' };
2214 #endif
2215
2216 if (!m) m = MOJOSHADER_internal_malloc;
2217 if (!f) f = MOJOSHADER_internal_free;
2218 if (!include_open) include_open = MOJOSHADER_internal_include_open;
2219 if (!include_close) include_close = MOJOSHADER_internal_include_close;
2220
2221 pp = preprocessor_start(filename, source, sourcelen,
2222 include_open, include_close,
2223 defines, define_count, 0, m, f, d);
2224 if (pp == NULL)
2225 goto preprocess_out_of_mem;
2226
2227 errors = errorlist_create(MallocBridge, FreeBridge, pp);
2228 if (errors == NULL)
2229 goto preprocess_out_of_mem;
2230
2231 buffer = buffer_create(4096, MallocBridge, FreeBridge, pp);
2232 if (buffer == NULL)
2233 goto preprocess_out_of_mem;
2234
2235 while ((tokstr = preprocessor_nexttoken(pp, &len, &token)) != NULL)
2236 {
2237 int isnewline = 0;
2238
2239 assert(token != TOKEN_EOI);
2240
2241 if (preprocessor_outofmemory(pp))
2242 goto preprocess_out_of_mem;
2243
2244 // Microsoft's preprocessor is weird.
2245 // It ignores newlines, and then inserts its own around certain
2246 // tokens. For example, after a semicolon. This allows HLSL code to
2247 // be mostly readable, instead of a stream of tokens.
2248 if ( (token == ((Token) '}')) || (token == ((Token) ';')) )
2249 {
2250 if ( (token == ((Token) '}')) && (indent > 0) )
2251 indent--;
2252
2253 indent_buffer(buffer, indent, nl);
2254 buffer_append(buffer, tokstr, len);
2255 buffer_append(buffer, endline, sizeof (endline));
2256
2257 isnewline = 1;
2258 } // if
2259
2260 else if (token == ((Token) '\n'))
2261 {
2262 buffer_append(buffer, endline, sizeof (endline));
2263 isnewline = 1;
2264 } // else if
2265
2266 else if (token == ((Token) '{'))
2267 {
2268 buffer_append(buffer, endline, sizeof (endline));
2269 indent_buffer(buffer, indent, 1);
2270 buffer_append(buffer, "{", 1);
2271 buffer_append(buffer, endline, sizeof (endline));
2272 indent++;
2273 isnewline = 1;
2274 } // else if
2275
2276 else if (token == TOKEN_PREPROCESSING_ERROR)
2277 {
2278 unsigned int pos = 0;
2279 const char *fname = preprocessor_sourcepos(pp, &pos);
2280 errorlist_add(errors, fname, (int) pos, tokstr);
2281 } // else if
2282
2283 else
2284 {
2285 indent_buffer(buffer, indent, nl);
2286 buffer_append(buffer, tokstr, len);
2287 } // else
2288
2289 nl = isnewline;
2290 } // while
2291
2292 assert(token == TOKEN_EOI);
2293
2294 total_bytes = buffer_size(buffer);
2295 output = buffer_flatten(buffer);
2296 buffer_destroy(buffer);
2297 buffer = NULL; // don't free this pointer again.
2298
2299 if (output == NULL)
2300 goto preprocess_out_of_mem;
2301
2302 retval = (MOJOSHADER_preprocessData *) m(sizeof (*retval), d);
2303 if (retval == NULL)
2304 goto preprocess_out_of_mem;
2305
2306 memset(retval, '\0', sizeof (*retval));
2307 errcount = errorlist_count(errors);
2308 if (errcount > 0)
2309 {
2310 retval->error_count = errcount;
2311 retval->errors = errorlist_flatten(errors);
2312 if (retval->errors == NULL)
2313 goto preprocess_out_of_mem;
2314 } // if
2315
2316 retval->output = output;
2317 retval->output_len = total_bytes;
2318 retval->malloc = m;
2319 retval->free = f;
2320 retval->malloc_data = d;
2321
2322 errorlist_destroy(errors);
2323 preprocessor_end(pp);
2324 return retval;
2325
2326 preprocess_out_of_mem:
2327 if (retval != NULL)
2328 f(retval->errors, d);
2329 f(retval, d);
2330 f(output, d);
2331 buffer_destroy(buffer);
2332 errorlist_destroy(errors);
2333 preprocessor_end(pp);
2334 return &out_of_mem_data_preprocessor;
2335 } // MOJOSHADER_preprocess
2336
2337
MOJOSHADER_freePreprocessData(const MOJOSHADER_preprocessData * _data)2338 void MOJOSHADER_freePreprocessData(const MOJOSHADER_preprocessData *_data)
2339 {
2340 MOJOSHADER_preprocessData *data = (MOJOSHADER_preprocessData *) _data;
2341 if ((data == NULL) || (data == &out_of_mem_data_preprocessor))
2342 return;
2343
2344 MOJOSHADER_free f = (data->free == NULL) ? MOJOSHADER_internal_free : data->free;
2345 void *d = data->malloc_data;
2346 int i;
2347
2348 f((void *) data->output, d);
2349
2350 for (i = 0; i < data->error_count; i++)
2351 {
2352 f((void *) data->errors[i].error, d);
2353 f((void *) data->errors[i].filename, d);
2354 } // for
2355 f(data->errors, d);
2356
2357 f(data, d);
2358 } // MOJOSHADER_freePreprocessData
2359
2360
2361 // end of mojoshader_preprocessor.c ...
2362
2363