xref: /dragonfly/contrib/gcc-8.0/libcpp/lex.c (revision 38fd1498)
1 /* CPP Library - lexical analysis.
2    Copyright (C) 2000-2018 Free Software Foundation, Inc.
3    Contributed by Per Bothner, 1994-95.
4    Based on CCCP program by Paul Rubin, June 1986
5    Adapted to ANSI C, Richard Stallman, Jan 1987
6    Broken out to separate file, Zack Weinberg, Mar 2000
7 
8 This program is free software; you can redistribute it and/or modify it
9 under the terms of the GNU General Public License as published by the
10 Free Software Foundation; either version 3, or (at your option) any
11 later version.
12 
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 GNU General Public License for more details.
17 
18 You should have received a copy of the GNU General Public License
19 along with this program; see the file COPYING3.  If not see
20 <http://www.gnu.org/licenses/>.  */
21 
22 #include "config.h"
23 #include "system.h"
24 #include "cpplib.h"
25 #include "internal.h"
26 
27 enum spell_type
28 {
29   SPELL_OPERATOR = 0,
30   SPELL_IDENT,
31   SPELL_LITERAL,
32   SPELL_NONE
33 };
34 
35 struct token_spelling
36 {
37   enum spell_type category;
38   const unsigned char *name;
39 };
40 
41 static const unsigned char *const digraph_spellings[] =
42 { UC"%:", UC"%:%:", UC"<:", UC":>", UC"<%", UC"%>" };
43 
44 #define OP(e, s) { SPELL_OPERATOR, UC s  },
45 #define TK(e, s) { SPELL_ ## s,    UC #e },
46 static const struct token_spelling token_spellings[N_TTYPES] = { TTYPE_TABLE };
47 #undef OP
48 #undef TK
49 
50 #define TOKEN_SPELL(token) (token_spellings[(token)->type].category)
51 #define TOKEN_NAME(token) (token_spellings[(token)->type].name)
52 
53 static void add_line_note (cpp_buffer *, const uchar *, unsigned int);
54 static int skip_line_comment (cpp_reader *);
55 static void skip_whitespace (cpp_reader *, cppchar_t);
56 static void lex_string (cpp_reader *, cpp_token *, const uchar *);
57 static void save_comment (cpp_reader *, cpp_token *, const uchar *, cppchar_t);
58 static void store_comment (cpp_reader *, cpp_token *);
59 static void create_literal (cpp_reader *, cpp_token *, const uchar *,
60 			    unsigned int, enum cpp_ttype);
61 static bool warn_in_comment (cpp_reader *, _cpp_line_note *);
62 static int name_p (cpp_reader *, const cpp_string *);
63 static tokenrun *next_tokenrun (tokenrun *);
64 
65 static _cpp_buff *new_buff (size_t);
66 
67 
68 /* Utility routine:
69 
70    Compares, the token TOKEN to the NUL-terminated string STRING.
71    TOKEN must be a CPP_NAME.  Returns 1 for equal, 0 for unequal.  */
72 int
cpp_ideq(const cpp_token * token,const char * string)73 cpp_ideq (const cpp_token *token, const char *string)
74 {
75   if (token->type != CPP_NAME)
76     return 0;
77 
78   return !ustrcmp (NODE_NAME (token->val.node.node), (const uchar *) string);
79 }
80 
81 /* Record a note TYPE at byte POS into the current cleaned logical
82    line.  */
83 static void
add_line_note(cpp_buffer * buffer,const uchar * pos,unsigned int type)84 add_line_note (cpp_buffer *buffer, const uchar *pos, unsigned int type)
85 {
86   if (buffer->notes_used == buffer->notes_cap)
87     {
88       buffer->notes_cap = buffer->notes_cap * 2 + 200;
89       buffer->notes = XRESIZEVEC (_cpp_line_note, buffer->notes,
90                                   buffer->notes_cap);
91     }
92 
93   buffer->notes[buffer->notes_used].pos = pos;
94   buffer->notes[buffer->notes_used].type = type;
95   buffer->notes_used++;
96 }
97 
98 
99 /* Fast path to find line special characters using optimized character
100    scanning algorithms.  Anything complicated falls back to the slow
101    path below.  Since this loop is very hot it's worth doing these kinds
102    of optimizations.
103 
104    One of the paths through the ifdefs should provide
105 
106      const uchar *search_line_fast (const uchar *s, const uchar *end);
107 
108    Between S and END, search for \n, \r, \\, ?.  Return a pointer to
109    the found character.
110 
111    Note that the last character of the buffer is *always* a newline,
112    as forced by _cpp_convert_input.  This fact can be used to avoid
113    explicitly looking for the end of the buffer.  */
114 
115 /* Configure gives us an ifdef test.  */
116 #ifndef WORDS_BIGENDIAN
117 #define WORDS_BIGENDIAN 0
118 #endif
119 
120 /* We'd like the largest integer that fits into a register.  There's nothing
121    in <stdint.h> that gives us that.  For most hosts this is unsigned long,
122    but MS decided on an LLP64 model.  Thankfully when building with GCC we
123    can get the "real" word size.  */
124 #ifdef __GNUC__
125 typedef unsigned int word_type __attribute__((__mode__(__word__)));
126 #else
127 typedef unsigned long word_type;
128 #endif
129 
130 /* The code below is only expecting sizes 4 or 8.
131    Die at compile-time if this expectation is violated.  */
132 typedef char check_word_type_size
133   [(sizeof(word_type) == 8 || sizeof(word_type) == 4) * 2 - 1];
134 
135 /* Return X with the first N bytes forced to values that won't match one
136    of the interesting characters.  Note that NUL is not interesting.  */
137 
138 static inline word_type
acc_char_mask_misalign(word_type val,unsigned int n)139 acc_char_mask_misalign (word_type val, unsigned int n)
140 {
141   word_type mask = -1;
142   if (WORDS_BIGENDIAN)
143     mask >>= n * 8;
144   else
145     mask <<= n * 8;
146   return val & mask;
147 }
148 
149 /* Return X replicated to all byte positions within WORD_TYPE.  */
150 
151 static inline word_type
acc_char_replicate(uchar x)152 acc_char_replicate (uchar x)
153 {
154   word_type ret;
155 
156   ret = (x << 24) | (x << 16) | (x << 8) | x;
157   if (sizeof(word_type) == 8)
158     ret = (ret << 16 << 16) | ret;
159   return ret;
160 }
161 
162 /* Return non-zero if some byte of VAL is (probably) C.  */
163 
164 static inline word_type
acc_char_cmp(word_type val,word_type c)165 acc_char_cmp (word_type val, word_type c)
166 {
167 #if defined(__GNUC__) && defined(__alpha__)
168   /* We can get exact results using a compare-bytes instruction.
169      Get (val == c) via (0 >= (val ^ c)).  */
170   return __builtin_alpha_cmpbge (0, val ^ c);
171 #else
172   word_type magic = 0x7efefefeU;
173   if (sizeof(word_type) == 8)
174     magic = (magic << 16 << 16) | 0xfefefefeU;
175   magic |= 1;
176 
177   val ^= c;
178   return ((val + magic) ^ ~val) & ~magic;
179 #endif
180 }
181 
182 /* Given the result of acc_char_cmp is non-zero, return the index of
183    the found character.  If this was a false positive, return -1.  */
184 
185 static inline int
acc_char_index(word_type cmp ATTRIBUTE_UNUSED,word_type val ATTRIBUTE_UNUSED)186 acc_char_index (word_type cmp ATTRIBUTE_UNUSED,
187 		word_type val ATTRIBUTE_UNUSED)
188 {
189 #if defined(__GNUC__) && defined(__alpha__) && !WORDS_BIGENDIAN
190   /* The cmpbge instruction sets *bits* of the result corresponding to
191      matches in the bytes with no false positives.  */
192   return __builtin_ctzl (cmp);
193 #else
194   unsigned int i;
195 
196   /* ??? It would be nice to force unrolling here,
197      and have all of these constants folded.  */
198   for (i = 0; i < sizeof(word_type); ++i)
199     {
200       uchar c;
201       if (WORDS_BIGENDIAN)
202 	c = (val >> (sizeof(word_type) - i - 1) * 8) & 0xff;
203       else
204 	c = (val >> i * 8) & 0xff;
205 
206       if (c == '\n' || c == '\r' || c == '\\' || c == '?')
207 	return i;
208     }
209 
210   return -1;
211 #endif
212 }
213 
214 /* A version of the fast scanner using bit fiddling techniques.
215 
216    For 32-bit words, one would normally perform 16 comparisons and
217    16 branches.  With this algorithm one performs 24 arithmetic
218    operations and one branch.  Whether this is faster with a 32-bit
219    word size is going to be somewhat system dependent.
220 
221    For 64-bit words, we eliminate twice the number of comparisons
222    and branches without increasing the number of arithmetic operations.
223    It's almost certainly going to be a win with 64-bit word size.  */
224 
225 static const uchar * search_line_acc_char (const uchar *, const uchar *)
226   ATTRIBUTE_UNUSED;
227 
228 static const uchar *
search_line_acc_char(const uchar * s,const uchar * end ATTRIBUTE_UNUSED)229 search_line_acc_char (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
230 {
231   const word_type repl_nl = acc_char_replicate ('\n');
232   const word_type repl_cr = acc_char_replicate ('\r');
233   const word_type repl_bs = acc_char_replicate ('\\');
234   const word_type repl_qm = acc_char_replicate ('?');
235 
236   unsigned int misalign;
237   const word_type *p;
238   word_type val, t;
239 
240   /* Align the buffer.  Mask out any bytes from before the beginning.  */
241   p = (word_type *)((uintptr_t)s & -sizeof(word_type));
242   val = *p;
243   misalign = (uintptr_t)s & (sizeof(word_type) - 1);
244   if (misalign)
245     val = acc_char_mask_misalign (val, misalign);
246 
247   /* Main loop.  */
248   while (1)
249     {
250       t  = acc_char_cmp (val, repl_nl);
251       t |= acc_char_cmp (val, repl_cr);
252       t |= acc_char_cmp (val, repl_bs);
253       t |= acc_char_cmp (val, repl_qm);
254 
255       if (__builtin_expect (t != 0, 0))
256 	{
257 	  int i = acc_char_index (t, val);
258 	  if (i >= 0)
259 	    return (const uchar *)p + i;
260 	}
261 
262       val = *++p;
263     }
264 }
265 
266 /* Disable on Solaris 2/x86 until the following problem can be properly
267    autoconfed:
268 
269    The Solaris 10+ assembler tags objects with the instruction set
270    extensions used, so SSE4.2 executables cannot run on machines that
271    don't support that extension.  */
272 
273 #if (GCC_VERSION >= 4005) && (__GNUC__ >= 5 || !defined(__PIC__)) && (defined(__i386__) || defined(__x86_64__)) && !(defined(__sun__) && defined(__svr4__))
274 
275 /* Replicated character data to be shared between implementations.
276    Recall that outside of a context with vector support we can't
277    define compatible vector types, therefore these are all defined
278    in terms of raw characters.  */
279 static const char repl_chars[4][16] __attribute__((aligned(16))) = {
280   { '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
281     '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n' },
282   { '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
283     '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r' },
284   { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
285     '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' },
286   { '?', '?', '?', '?', '?', '?', '?', '?',
287     '?', '?', '?', '?', '?', '?', '?', '?' },
288 };
289 
290 /* A version of the fast scanner using MMX vectorized byte compare insns.
291 
292    This uses the PMOVMSKB instruction which was introduced with "MMX2",
293    which was packaged into SSE1; it is also present in the AMD MMX
294    extension.  Mark the function as using "sse" so that we emit a real
295    "emms" instruction, rather than the 3dNOW "femms" instruction.  */
296 
297 static const uchar *
298 #ifndef __SSE__
299 __attribute__((__target__("sse")))
300 #endif
search_line_mmx(const uchar * s,const uchar * end ATTRIBUTE_UNUSED)301 search_line_mmx (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
302 {
303   typedef char v8qi __attribute__ ((__vector_size__ (8)));
304   typedef int __m64 __attribute__ ((__vector_size__ (8), __may_alias__));
305 
306   const v8qi repl_nl = *(const v8qi *)repl_chars[0];
307   const v8qi repl_cr = *(const v8qi *)repl_chars[1];
308   const v8qi repl_bs = *(const v8qi *)repl_chars[2];
309   const v8qi repl_qm = *(const v8qi *)repl_chars[3];
310 
311   unsigned int misalign, found, mask;
312   const v8qi *p;
313   v8qi data, t, c;
314 
315   /* Align the source pointer.  While MMX doesn't generate unaligned data
316      faults, this allows us to safely scan to the end of the buffer without
317      reading beyond the end of the last page.  */
318   misalign = (uintptr_t)s & 7;
319   p = (const v8qi *)((uintptr_t)s & -8);
320   data = *p;
321 
322   /* Create a mask for the bytes that are valid within the first
323      16-byte block.  The Idea here is that the AND with the mask
324      within the loop is "free", since we need some AND or TEST
325      insn in order to set the flags for the branch anyway.  */
326   mask = -1u << misalign;
327 
328   /* Main loop processing 8 bytes at a time.  */
329   goto start;
330   do
331     {
332       data = *++p;
333       mask = -1;
334 
335     start:
336       t = __builtin_ia32_pcmpeqb(data, repl_nl);
337       c = __builtin_ia32_pcmpeqb(data, repl_cr);
338       t = (v8qi) __builtin_ia32_por ((__m64)t, (__m64)c);
339       c = __builtin_ia32_pcmpeqb(data, repl_bs);
340       t = (v8qi) __builtin_ia32_por ((__m64)t, (__m64)c);
341       c = __builtin_ia32_pcmpeqb(data, repl_qm);
342       t = (v8qi) __builtin_ia32_por ((__m64)t, (__m64)c);
343       found = __builtin_ia32_pmovmskb (t);
344       found &= mask;
345     }
346   while (!found);
347 
348   __builtin_ia32_emms ();
349 
350   /* FOUND contains 1 in bits for which we matched a relevant
351      character.  Conversion to the byte index is trivial.  */
352   found = __builtin_ctz(found);
353   return (const uchar *)p + found;
354 }
355 
356 /* A version of the fast scanner using SSE2 vectorized byte compare insns.  */
357 
358 static const uchar *
359 #ifndef __SSE2__
360 __attribute__((__target__("sse2")))
361 #endif
search_line_sse2(const uchar * s,const uchar * end ATTRIBUTE_UNUSED)362 search_line_sse2 (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
363 {
364   typedef char v16qi __attribute__ ((__vector_size__ (16)));
365 
366   const v16qi repl_nl = *(const v16qi *)repl_chars[0];
367   const v16qi repl_cr = *(const v16qi *)repl_chars[1];
368   const v16qi repl_bs = *(const v16qi *)repl_chars[2];
369   const v16qi repl_qm = *(const v16qi *)repl_chars[3];
370 
371   unsigned int misalign, found, mask;
372   const v16qi *p;
373   v16qi data, t;
374 
375   /* Align the source pointer.  */
376   misalign = (uintptr_t)s & 15;
377   p = (const v16qi *)((uintptr_t)s & -16);
378   data = *p;
379 
380   /* Create a mask for the bytes that are valid within the first
381      16-byte block.  The Idea here is that the AND with the mask
382      within the loop is "free", since we need some AND or TEST
383      insn in order to set the flags for the branch anyway.  */
384   mask = -1u << misalign;
385 
386   /* Main loop processing 16 bytes at a time.  */
387   goto start;
388   do
389     {
390       data = *++p;
391       mask = -1;
392 
393     start:
394       t  = __builtin_ia32_pcmpeqb128(data, repl_nl);
395       t |= __builtin_ia32_pcmpeqb128(data, repl_cr);
396       t |= __builtin_ia32_pcmpeqb128(data, repl_bs);
397       t |= __builtin_ia32_pcmpeqb128(data, repl_qm);
398       found = __builtin_ia32_pmovmskb128 (t);
399       found &= mask;
400     }
401   while (!found);
402 
403   /* FOUND contains 1 in bits for which we matched a relevant
404      character.  Conversion to the byte index is trivial.  */
405   found = __builtin_ctz(found);
406   return (const uchar *)p + found;
407 }
408 
409 #ifdef HAVE_SSE4
410 /* A version of the fast scanner using SSE 4.2 vectorized string insns.  */
411 
412 static const uchar *
413 #ifndef __SSE4_2__
414 __attribute__((__target__("sse4.2")))
415 #endif
search_line_sse42(const uchar * s,const uchar * end)416 search_line_sse42 (const uchar *s, const uchar *end)
417 {
418   typedef char v16qi __attribute__ ((__vector_size__ (16)));
419   static const v16qi search = { '\n', '\r', '?', '\\' };
420 
421   uintptr_t si = (uintptr_t)s;
422   uintptr_t index;
423 
424   /* Check for unaligned input.  */
425   if (si & 15)
426     {
427       v16qi sv;
428 
429       if (__builtin_expect (end - s < 16, 0)
430 	  && __builtin_expect ((si & 0xfff) > 0xff0, 0))
431 	{
432 	  /* There are less than 16 bytes left in the buffer, and less
433 	     than 16 bytes left on the page.  Reading 16 bytes at this
434 	     point might generate a spurious page fault.  Defer to the
435 	     SSE2 implementation, which already handles alignment.  */
436 	  return search_line_sse2 (s, end);
437 	}
438 
439       /* ??? The builtin doesn't understand that the PCMPESTRI read from
440 	 memory need not be aligned.  */
441       sv = __builtin_ia32_loaddqu ((const char *) s);
442       index = __builtin_ia32_pcmpestri128 (search, 4, sv, 16, 0);
443 
444       if (__builtin_expect (index < 16, 0))
445 	goto found;
446 
447       /* Advance the pointer to an aligned address.  We will re-scan a
448 	 few bytes, but we no longer need care for reading past the
449 	 end of a page, since we're guaranteed a match.  */
450       s = (const uchar *)((si + 15) & -16);
451     }
452 
453   /* Main loop, processing 16 bytes at a time.  */
454 #ifdef __GCC_ASM_FLAG_OUTPUTS__
455   while (1)
456     {
457       char f;
458 
459       /* By using inline assembly instead of the builtin,
460 	 we can use the result, as well as the flags set.  */
461       __asm ("%vpcmpestri\t$0, %2, %3"
462 	     : "=c"(index), "=@ccc"(f)
463 	     : "m"(*s), "x"(search), "a"(4), "d"(16));
464       if (f)
465 	break;
466 
467       s += 16;
468     }
469 #else
470   s -= 16;
471   /* By doing the whole loop in inline assembly,
472      we can make proper use of the flags set.  */
473   __asm (      ".balign 16\n"
474 	"0:	add $16, %1\n"
475 	"	%vpcmpestri\t$0, (%1), %2\n"
476 	"	jnc 0b"
477 	: "=&c"(index), "+r"(s)
478 	: "x"(search), "a"(4), "d"(16));
479 #endif
480 
481  found:
482   return s + index;
483 }
484 
485 #else
486 /* Work around out-dated assemblers without sse4 support.  */
487 #define search_line_sse42 search_line_sse2
488 #endif
489 
490 /* Check the CPU capabilities.  */
491 
492 #include "../gcc/config/i386/cpuid.h"
493 
494 typedef const uchar * (*search_line_fast_type) (const uchar *, const uchar *);
495 static search_line_fast_type search_line_fast;
496 
497 #define HAVE_init_vectorized_lexer 1
498 static inline void
init_vectorized_lexer(void)499 init_vectorized_lexer (void)
500 {
501   unsigned dummy, ecx = 0, edx = 0;
502   search_line_fast_type impl = search_line_acc_char;
503   int minimum = 0;
504 
505 #if defined(__SSE4_2__)
506   minimum = 3;
507 #elif defined(__SSE2__)
508   minimum = 2;
509 #elif defined(__SSE__)
510   minimum = 1;
511 #endif
512 
513   if (minimum == 3)
514     impl = search_line_sse42;
515   else if (__get_cpuid (1, &dummy, &dummy, &ecx, &edx) || minimum == 2)
516     {
517       if (minimum == 3 || (ecx & bit_SSE4_2))
518         impl = search_line_sse42;
519       else if (minimum == 2 || (edx & bit_SSE2))
520 	impl = search_line_sse2;
521       else if (minimum == 1 || (edx & bit_SSE))
522 	impl = search_line_mmx;
523     }
524   else if (__get_cpuid (0x80000001, &dummy, &dummy, &dummy, &edx))
525     {
526       if (minimum == 1
527 	  || (edx & (bit_MMXEXT | bit_CMOV)) == (bit_MMXEXT | bit_CMOV))
528 	impl = search_line_mmx;
529     }
530 
531   search_line_fast = impl;
532 }
533 
534 #elif defined(_ARCH_PWR8) && defined(__ALTIVEC__)
535 
536 /* A vection of the fast scanner using AltiVec vectorized byte compares
537    and VSX unaligned loads (when VSX is available).  This is otherwise
538    the same as the pre-GCC 5 version.  */
539 
540 ATTRIBUTE_NO_SANITIZE_UNDEFINED
541 static const uchar *
search_line_fast(const uchar * s,const uchar * end ATTRIBUTE_UNUSED)542 search_line_fast (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
543 {
544   typedef __attribute__((altivec(vector))) unsigned char vc;
545 
546   const vc repl_nl = {
547     '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
548     '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'
549   };
550   const vc repl_cr = {
551     '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
552     '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'
553   };
554   const vc repl_bs = {
555     '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
556     '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'
557   };
558   const vc repl_qm = {
559     '?', '?', '?', '?', '?', '?', '?', '?',
560     '?', '?', '?', '?', '?', '?', '?', '?',
561   };
562   const vc zero = { 0 };
563 
564   vc data, t;
565 
566   /* Main loop processing 16 bytes at a time.  */
567   do
568     {
569       vc m_nl, m_cr, m_bs, m_qm;
570 
571       data = __builtin_vec_vsx_ld (0, s);
572       s += 16;
573 
574       m_nl = (vc) __builtin_vec_cmpeq(data, repl_nl);
575       m_cr = (vc) __builtin_vec_cmpeq(data, repl_cr);
576       m_bs = (vc) __builtin_vec_cmpeq(data, repl_bs);
577       m_qm = (vc) __builtin_vec_cmpeq(data, repl_qm);
578       t = (m_nl | m_cr) | (m_bs | m_qm);
579 
580       /* T now contains 0xff in bytes for which we matched one of the relevant
581 	 characters.  We want to exit the loop if any byte in T is non-zero.
582 	 Below is the expansion of vec_any_ne(t, zero).  */
583     }
584   while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t, zero));
585 
586   /* Restore s to to point to the 16 bytes we just processed.  */
587   s -= 16;
588 
589   {
590 #define N  (sizeof(vc) / sizeof(long))
591 
592     union {
593       vc v;
594       /* Statically assert that N is 2 or 4.  */
595       unsigned long l[(N == 2 || N == 4) ? N : -1];
596     } u;
597     unsigned long l, i = 0;
598 
599     u.v = t;
600 
601     /* Find the first word of T that is non-zero.  */
602     switch (N)
603       {
604       case 4:
605 	l = u.l[i++];
606 	if (l != 0)
607 	  break;
608 	s += sizeof(unsigned long);
609 	l = u.l[i++];
610 	if (l != 0)
611 	  break;
612 	s += sizeof(unsigned long);
613 	/* FALLTHRU */
614       case 2:
615 	l = u.l[i++];
616 	if (l != 0)
617 	  break;
618 	s += sizeof(unsigned long);
619 	l = u.l[i];
620       }
621 
622     /* L now contains 0xff in bytes for which we matched one of the
623        relevant characters.  We can find the byte index by finding
624        its bit index and dividing by 8.  */
625 #ifdef __BIG_ENDIAN__
626     l = __builtin_clzl(l) >> 3;
627 #else
628     l = __builtin_ctzl(l) >> 3;
629 #endif
630     return s + l;
631 
632 #undef N
633   }
634 }
635 
636 #elif (GCC_VERSION >= 4005) && defined(__ALTIVEC__) && defined (__BIG_ENDIAN__)
637 
638 /* A vection of the fast scanner using AltiVec vectorized byte compares.
639    This cannot be used for little endian because vec_lvsl/lvsr are
640    deprecated for little endian and the code won't work properly.  */
641 /* ??? Unfortunately, attribute(target("altivec")) is not yet supported,
642    so we can't compile this function without -maltivec on the command line
643    (or implied by some other switch).  */
644 
645 static const uchar *
search_line_fast(const uchar * s,const uchar * end ATTRIBUTE_UNUSED)646 search_line_fast (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
647 {
648   typedef __attribute__((altivec(vector))) unsigned char vc;
649 
650   const vc repl_nl = {
651     '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
652     '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'
653   };
654   const vc repl_cr = {
655     '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
656     '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'
657   };
658   const vc repl_bs = {
659     '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
660     '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'
661   };
662   const vc repl_qm = {
663     '?', '?', '?', '?', '?', '?', '?', '?',
664     '?', '?', '?', '?', '?', '?', '?', '?',
665   };
666   const vc ones = {
667     -1, -1, -1, -1, -1, -1, -1, -1,
668     -1, -1, -1, -1, -1, -1, -1, -1,
669   };
670   const vc zero = { 0 };
671 
672   vc data, mask, t;
673 
674   /* Altivec loads automatically mask addresses with -16.  This lets us
675      issue the first load as early as possible.  */
676   data = __builtin_vec_ld(0, (const vc *)s);
677 
678   /* Discard bytes before the beginning of the buffer.  Do this by
679      beginning with all ones and shifting in zeros according to the
680      mis-alignment.  The LVSR instruction pulls the exact shift we
681      want from the address.  */
682   mask = __builtin_vec_lvsr(0, s);
683   mask = __builtin_vec_perm(zero, ones, mask);
684   data &= mask;
685 
686   /* While altivec loads mask addresses, we still need to align S so
687      that the offset we compute at the end is correct.  */
688   s = (const uchar *)((uintptr_t)s & -16);
689 
690   /* Main loop processing 16 bytes at a time.  */
691   goto start;
692   do
693     {
694       vc m_nl, m_cr, m_bs, m_qm;
695 
696       s += 16;
697       data = __builtin_vec_ld(0, (const vc *)s);
698 
699     start:
700       m_nl = (vc) __builtin_vec_cmpeq(data, repl_nl);
701       m_cr = (vc) __builtin_vec_cmpeq(data, repl_cr);
702       m_bs = (vc) __builtin_vec_cmpeq(data, repl_bs);
703       m_qm = (vc) __builtin_vec_cmpeq(data, repl_qm);
704       t = (m_nl | m_cr) | (m_bs | m_qm);
705 
706       /* T now contains 0xff in bytes for which we matched one of the relevant
707 	 characters.  We want to exit the loop if any byte in T is non-zero.
708 	 Below is the expansion of vec_any_ne(t, zero).  */
709     }
710   while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t, zero));
711 
712   {
713 #define N  (sizeof(vc) / sizeof(long))
714 
715     union {
716       vc v;
717       /* Statically assert that N is 2 or 4.  */
718       unsigned long l[(N == 2 || N == 4) ? N : -1];
719     } u;
720     unsigned long l, i = 0;
721 
722     u.v = t;
723 
724     /* Find the first word of T that is non-zero.  */
725     switch (N)
726       {
727       case 4:
728 	l = u.l[i++];
729 	if (l != 0)
730 	  break;
731 	s += sizeof(unsigned long);
732 	l = u.l[i++];
733 	if (l != 0)
734 	  break;
735 	s += sizeof(unsigned long);
736 	/* FALLTHROUGH */
737       case 2:
738 	l = u.l[i++];
739 	if (l != 0)
740 	  break;
741 	s += sizeof(unsigned long);
742 	l = u.l[i];
743       }
744 
745     /* L now contains 0xff in bytes for which we matched one of the
746        relevant characters.  We can find the byte index by finding
747        its bit index and dividing by 8.  */
748     l = __builtin_clzl(l) >> 3;
749     return s + l;
750 
751 #undef N
752   }
753 }
754 
755 #elif defined (__ARM_NEON) && defined (__ARM_64BIT_STATE)
756 #include "arm_neon.h"
757 
758 /* This doesn't have to be the exact page size, but no system may use
759    a size smaller than this.  ARMv8 requires a minimum page size of
760    4k.  The impact of being conservative here is a small number of
761    cases will take the slightly slower entry path into the main
762    loop.  */
763 
764 #define AARCH64_MIN_PAGE_SIZE 4096
765 
766 static const uchar *
search_line_fast(const uchar * s,const uchar * end ATTRIBUTE_UNUSED)767 search_line_fast (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
768 {
769   const uint8x16_t repl_nl = vdupq_n_u8 ('\n');
770   const uint8x16_t repl_cr = vdupq_n_u8 ('\r');
771   const uint8x16_t repl_bs = vdupq_n_u8 ('\\');
772   const uint8x16_t repl_qm = vdupq_n_u8 ('?');
773   const uint8x16_t xmask = (uint8x16_t) vdupq_n_u64 (0x8040201008040201ULL);
774 
775 #ifdef __ARM_BIG_ENDIAN
776   const int16x8_t shift = {8, 8, 8, 8, 0, 0, 0, 0};
777 #else
778   const int16x8_t shift = {0, 0, 0, 0, 8, 8, 8, 8};
779 #endif
780 
781   unsigned int found;
782   const uint8_t *p;
783   uint8x16_t data;
784   uint8x16_t t;
785   uint16x8_t m;
786   uint8x16_t u, v, w;
787 
788   /* Align the source pointer.  */
789   p = (const uint8_t *)((uintptr_t)s & -16);
790 
791   /* Assuming random string start positions, with a 4k page size we'll take
792      the slow path about 0.37% of the time.  */
793   if (__builtin_expect ((AARCH64_MIN_PAGE_SIZE
794 			 - (((uintptr_t) s) & (AARCH64_MIN_PAGE_SIZE - 1)))
795 			< 16, 0))
796     {
797       /* Slow path: the string starts near a possible page boundary.  */
798       uint32_t misalign, mask;
799 
800       misalign = (uintptr_t)s & 15;
801       mask = (-1u << misalign) & 0xffff;
802       data = vld1q_u8 (p);
803       t = vceqq_u8 (data, repl_nl);
804       u = vceqq_u8 (data, repl_cr);
805       v = vorrq_u8 (t, vceqq_u8 (data, repl_bs));
806       w = vorrq_u8 (u, vceqq_u8 (data, repl_qm));
807       t = vorrq_u8 (v, w);
808       t = vandq_u8 (t, xmask);
809       m = vpaddlq_u8 (t);
810       m = vshlq_u16 (m, shift);
811       found = vaddvq_u16 (m);
812       found &= mask;
813       if (found)
814 	return (const uchar*)p + __builtin_ctz (found);
815     }
816   else
817     {
818       data = vld1q_u8 ((const uint8_t *) s);
819       t = vceqq_u8 (data, repl_nl);
820       u = vceqq_u8 (data, repl_cr);
821       v = vorrq_u8 (t, vceqq_u8 (data, repl_bs));
822       w = vorrq_u8 (u, vceqq_u8 (data, repl_qm));
823       t = vorrq_u8 (v, w);
824       if (__builtin_expect (vpaddd_u64 ((uint64x2_t)t) != 0, 0))
825 	goto done;
826     }
827 
828   do
829     {
830       p += 16;
831       data = vld1q_u8 (p);
832       t = vceqq_u8 (data, repl_nl);
833       u = vceqq_u8 (data, repl_cr);
834       v = vorrq_u8 (t, vceqq_u8 (data, repl_bs));
835       w = vorrq_u8 (u, vceqq_u8 (data, repl_qm));
836       t = vorrq_u8 (v, w);
837     } while (!vpaddd_u64 ((uint64x2_t)t));
838 
839 done:
840   /* Now that we've found the terminating substring, work out precisely where
841      we need to stop.  */
842   t = vandq_u8 (t, xmask);
843   m = vpaddlq_u8 (t);
844   m = vshlq_u16 (m, shift);
845   found = vaddvq_u16 (m);
846   return (((((uintptr_t) p) < (uintptr_t) s) ? s : (const uchar *)p)
847 	  + __builtin_ctz (found));
848 }
849 
850 #elif defined (__ARM_NEON)
851 #include "arm_neon.h"
852 
853 static const uchar *
search_line_fast(const uchar * s,const uchar * end ATTRIBUTE_UNUSED)854 search_line_fast (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
855 {
856   const uint8x16_t repl_nl = vdupq_n_u8 ('\n');
857   const uint8x16_t repl_cr = vdupq_n_u8 ('\r');
858   const uint8x16_t repl_bs = vdupq_n_u8 ('\\');
859   const uint8x16_t repl_qm = vdupq_n_u8 ('?');
860   const uint8x16_t xmask = (uint8x16_t) vdupq_n_u64 (0x8040201008040201ULL);
861 
862   unsigned int misalign, found, mask;
863   const uint8_t *p;
864   uint8x16_t data;
865 
866   /* Align the source pointer.  */
867   misalign = (uintptr_t)s & 15;
868   p = (const uint8_t *)((uintptr_t)s & -16);
869   data = vld1q_u8 (p);
870 
871   /* Create a mask for the bytes that are valid within the first
872      16-byte block.  The Idea here is that the AND with the mask
873      within the loop is "free", since we need some AND or TEST
874      insn in order to set the flags for the branch anyway.  */
875   mask = (-1u << misalign) & 0xffff;
876 
877   /* Main loop, processing 16 bytes at a time.  */
878   goto start;
879 
880   do
881     {
882       uint8x8_t l;
883       uint16x4_t m;
884       uint32x2_t n;
885       uint8x16_t t, u, v, w;
886 
887       p += 16;
888       data = vld1q_u8 (p);
889       mask = 0xffff;
890 
891     start:
892       t = vceqq_u8 (data, repl_nl);
893       u = vceqq_u8 (data, repl_cr);
894       v = vorrq_u8 (t, vceqq_u8 (data, repl_bs));
895       w = vorrq_u8 (u, vceqq_u8 (data, repl_qm));
896       t = vandq_u8 (vorrq_u8 (v, w), xmask);
897       l = vpadd_u8 (vget_low_u8 (t), vget_high_u8 (t));
898       m = vpaddl_u8 (l);
899       n = vpaddl_u16 (m);
900 
901       found = vget_lane_u32 ((uint32x2_t) vorr_u64 ((uint64x1_t) n,
902 	      vshr_n_u64 ((uint64x1_t) n, 24)), 0);
903       found &= mask;
904     }
905   while (!found);
906 
907   /* FOUND contains 1 in bits for which we matched a relevant
908      character.  Conversion to the byte index is trivial.  */
909   found = __builtin_ctz (found);
910   return (const uchar *)p + found;
911 }
912 
913 #else
914 
915 /* We only have one accelerated alternative.  Use a direct call so that
916    we encourage inlining.  */
917 
918 #define search_line_fast  search_line_acc_char
919 
920 #endif
921 
922 /* Initialize the lexer if needed.  */
923 
924 void
_cpp_init_lexer(void)925 _cpp_init_lexer (void)
926 {
927 #ifdef HAVE_init_vectorized_lexer
928   init_vectorized_lexer ();
929 #endif
930 }
931 
932 /* Returns with a logical line that contains no escaped newlines or
933    trigraphs.  This is a time-critical inner loop.  */
934 void
_cpp_clean_line(cpp_reader * pfile)935 _cpp_clean_line (cpp_reader *pfile)
936 {
937   cpp_buffer *buffer;
938   const uchar *s;
939   uchar c, *d, *p;
940 
941   buffer = pfile->buffer;
942   buffer->cur_note = buffer->notes_used = 0;
943   buffer->cur = buffer->line_base = buffer->next_line;
944   buffer->need_line = false;
945   s = buffer->next_line;
946 
947   if (!buffer->from_stage3)
948     {
949       const uchar *pbackslash = NULL;
950 
951       /* Fast path.  This is the common case of an un-escaped line with
952 	 no trigraphs.  The primary win here is by not writing any
953 	 data back to memory until we have to.  */
954       while (1)
955 	{
956 	  /* Perform an optimized search for \n, \r, \\, ?.  */
957 	  s = search_line_fast (s, buffer->rlimit);
958 
959 	  c = *s;
960 	  if (c == '\\')
961 	    {
962 	      /* Record the location of the backslash and continue.  */
963 	      pbackslash = s++;
964 	    }
965 	  else if (__builtin_expect (c == '?', 0))
966 	    {
967 	      if (__builtin_expect (s[1] == '?', false)
968 		   && _cpp_trigraph_map[s[2]])
969 		{
970 		  /* Have a trigraph.  We may or may not have to convert
971 		     it.  Add a line note regardless, for -Wtrigraphs.  */
972 		  add_line_note (buffer, s, s[2]);
973 		  if (CPP_OPTION (pfile, trigraphs))
974 		    {
975 		      /* We do, and that means we have to switch to the
976 		         slow path.  */
977 		      d = (uchar *) s;
978 		      *d = _cpp_trigraph_map[s[2]];
979 		      s += 2;
980 		      goto slow_path;
981 		    }
982 		}
983 	      /* Not a trigraph.  Continue on fast-path.  */
984 	      s++;
985 	    }
986 	  else
987 	    break;
988 	}
989 
990       /* This must be \r or \n.  We're either done, or we'll be forced
991 	 to write back to the buffer and continue on the slow path.  */
992       d = (uchar *) s;
993 
994       if (__builtin_expect (s == buffer->rlimit, false))
995 	goto done;
996 
997       /* DOS line ending? */
998       if (__builtin_expect (c == '\r', false) && s[1] == '\n')
999 	{
1000 	  s++;
1001 	  if (s == buffer->rlimit)
1002 	    goto done;
1003 	}
1004 
1005       if (__builtin_expect (pbackslash == NULL, true))
1006 	goto done;
1007 
1008       /* Check for escaped newline.  */
1009       p = d;
1010       while (is_nvspace (p[-1]))
1011 	p--;
1012       if (p - 1 != pbackslash)
1013 	goto done;
1014 
1015       /* Have an escaped newline; process it and proceed to
1016 	 the slow path.  */
1017       add_line_note (buffer, p - 1, p != d ? ' ' : '\\');
1018       d = p - 2;
1019       buffer->next_line = p - 1;
1020 
1021     slow_path:
1022       while (1)
1023 	{
1024 	  c = *++s;
1025 	  *++d = c;
1026 
1027 	  if (c == '\n' || c == '\r')
1028 	    {
1029 	      /* Handle DOS line endings.  */
1030 	      if (c == '\r' && s != buffer->rlimit && s[1] == '\n')
1031 		s++;
1032 	      if (s == buffer->rlimit)
1033 		break;
1034 
1035 	      /* Escaped?  */
1036 	      p = d;
1037 	      while (p != buffer->next_line && is_nvspace (p[-1]))
1038 		p--;
1039 	      if (p == buffer->next_line || p[-1] != '\\')
1040 		break;
1041 
1042 	      add_line_note (buffer, p - 1, p != d ? ' ': '\\');
1043 	      d = p - 2;
1044 	      buffer->next_line = p - 1;
1045 	    }
1046 	  else if (c == '?' && s[1] == '?' && _cpp_trigraph_map[s[2]])
1047 	    {
1048 	      /* Add a note regardless, for the benefit of -Wtrigraphs.  */
1049 	      add_line_note (buffer, d, s[2]);
1050 	      if (CPP_OPTION (pfile, trigraphs))
1051 		{
1052 		  *d = _cpp_trigraph_map[s[2]];
1053 		  s += 2;
1054 		}
1055 	    }
1056 	}
1057     }
1058   else
1059     {
1060       while (*s != '\n' && *s != '\r')
1061 	s++;
1062       d = (uchar *) s;
1063 
1064       /* Handle DOS line endings.  */
1065       if (*s == '\r' && s != buffer->rlimit && s[1] == '\n')
1066 	s++;
1067     }
1068 
1069  done:
1070   *d = '\n';
1071   /* A sentinel note that should never be processed.  */
1072   add_line_note (buffer, d + 1, '\n');
1073   buffer->next_line = s + 1;
1074 }
1075 
1076 /* Return true if the trigraph indicated by NOTE should be warned
1077    about in a comment.  */
1078 static bool
warn_in_comment(cpp_reader * pfile,_cpp_line_note * note)1079 warn_in_comment (cpp_reader *pfile, _cpp_line_note *note)
1080 {
1081   const uchar *p;
1082 
1083   /* Within comments we don't warn about trigraphs, unless the
1084      trigraph forms an escaped newline, as that may change
1085      behavior.  */
1086   if (note->type != '/')
1087     return false;
1088 
1089   /* If -trigraphs, then this was an escaped newline iff the next note
1090      is coincident.  */
1091   if (CPP_OPTION (pfile, trigraphs))
1092     return note[1].pos == note->pos;
1093 
1094   /* Otherwise, see if this forms an escaped newline.  */
1095   p = note->pos + 3;
1096   while (is_nvspace (*p))
1097     p++;
1098 
1099   /* There might have been escaped newlines between the trigraph and the
1100      newline we found.  Hence the position test.  */
1101   return (*p == '\n' && p < note[1].pos);
1102 }
1103 
1104 /* Process the notes created by add_line_note as far as the current
1105    location.  */
1106 void
_cpp_process_line_notes(cpp_reader * pfile,int in_comment)1107 _cpp_process_line_notes (cpp_reader *pfile, int in_comment)
1108 {
1109   cpp_buffer *buffer = pfile->buffer;
1110 
1111   for (;;)
1112     {
1113       _cpp_line_note *note = &buffer->notes[buffer->cur_note];
1114       unsigned int col;
1115 
1116       if (note->pos > buffer->cur)
1117 	break;
1118 
1119       buffer->cur_note++;
1120       col = CPP_BUF_COLUMN (buffer, note->pos + 1);
1121 
1122       if (note->type == '\\' || note->type == ' ')
1123 	{
1124 	  if (note->type == ' ' && !in_comment)
1125 	    cpp_error_with_line (pfile, CPP_DL_WARNING, pfile->line_table->highest_line, col,
1126 				 "backslash and newline separated by space");
1127 
1128 	  if (buffer->next_line > buffer->rlimit)
1129 	    {
1130 	      cpp_error_with_line (pfile, CPP_DL_PEDWARN, pfile->line_table->highest_line, col,
1131 				   "backslash-newline at end of file");
1132 	      /* Prevent "no newline at end of file" warning.  */
1133 	      buffer->next_line = buffer->rlimit;
1134 	    }
1135 
1136 	  buffer->line_base = note->pos;
1137 	  CPP_INCREMENT_LINE (pfile, 0);
1138 	}
1139       else if (_cpp_trigraph_map[note->type])
1140 	{
1141 	  if (CPP_OPTION (pfile, warn_trigraphs)
1142 	      && (!in_comment || warn_in_comment (pfile, note)))
1143 	    {
1144 	      if (CPP_OPTION (pfile, trigraphs))
1145 		cpp_warning_with_line (pfile, CPP_W_TRIGRAPHS,
1146                                        pfile->line_table->highest_line, col,
1147 				       "trigraph ??%c converted to %c",
1148 				       note->type,
1149 				       (int) _cpp_trigraph_map[note->type]);
1150 	      else
1151 		{
1152 		  cpp_warning_with_line
1153 		    (pfile, CPP_W_TRIGRAPHS,
1154                      pfile->line_table->highest_line, col,
1155 		     "trigraph ??%c ignored, use -trigraphs to enable",
1156 		     note->type);
1157 		}
1158 	    }
1159 	}
1160       else if (note->type == 0)
1161 	/* Already processed in lex_raw_string.  */;
1162       else
1163 	abort ();
1164     }
1165 }
1166 
1167 /* Skip a C-style block comment.  We find the end of the comment by
1168    seeing if an asterisk is before every '/' we encounter.  Returns
1169    nonzero if comment terminated by EOF, zero otherwise.
1170 
1171    Buffer->cur points to the initial asterisk of the comment.  */
1172 bool
_cpp_skip_block_comment(cpp_reader * pfile)1173 _cpp_skip_block_comment (cpp_reader *pfile)
1174 {
1175   cpp_buffer *buffer = pfile->buffer;
1176   const uchar *cur = buffer->cur;
1177   uchar c;
1178 
1179   cur++;
1180   if (*cur == '/')
1181     cur++;
1182 
1183   for (;;)
1184     {
1185       /* People like decorating comments with '*', so check for '/'
1186 	 instead for efficiency.  */
1187       c = *cur++;
1188 
1189       if (c == '/')
1190 	{
1191 	  if (cur[-2] == '*')
1192 	    break;
1193 
1194 	  /* Warn about potential nested comments, but not if the '/'
1195 	     comes immediately before the true comment delimiter.
1196 	     Don't bother to get it right across escaped newlines.  */
1197 	  if (CPP_OPTION (pfile, warn_comments)
1198 	      && cur[0] == '*' && cur[1] != '/')
1199 	    {
1200 	      buffer->cur = cur;
1201 	      cpp_warning_with_line (pfile, CPP_W_COMMENTS,
1202 				     pfile->line_table->highest_line,
1203 				     CPP_BUF_COL (buffer),
1204 				     "\"/*\" within comment");
1205 	    }
1206 	}
1207       else if (c == '\n')
1208 	{
1209 	  unsigned int cols;
1210 	  buffer->cur = cur - 1;
1211 	  _cpp_process_line_notes (pfile, true);
1212 	  if (buffer->next_line >= buffer->rlimit)
1213 	    return true;
1214 	  _cpp_clean_line (pfile);
1215 
1216 	  cols = buffer->next_line - buffer->line_base;
1217 	  CPP_INCREMENT_LINE (pfile, cols);
1218 
1219 	  cur = buffer->cur;
1220 	}
1221     }
1222 
1223   buffer->cur = cur;
1224   _cpp_process_line_notes (pfile, true);
1225   return false;
1226 }
1227 
1228 /* Skip a C++ line comment, leaving buffer->cur pointing to the
1229    terminating newline.  Handles escaped newlines.  Returns nonzero
1230    if a multiline comment.  */
1231 static int
skip_line_comment(cpp_reader * pfile)1232 skip_line_comment (cpp_reader *pfile)
1233 {
1234   cpp_buffer *buffer = pfile->buffer;
1235   source_location orig_line = pfile->line_table->highest_line;
1236 
1237   while (*buffer->cur != '\n')
1238     buffer->cur++;
1239 
1240   _cpp_process_line_notes (pfile, true);
1241   return orig_line != pfile->line_table->highest_line;
1242 }
1243 
1244 /* Skips whitespace, saving the next non-whitespace character.  */
1245 static void
skip_whitespace(cpp_reader * pfile,cppchar_t c)1246 skip_whitespace (cpp_reader *pfile, cppchar_t c)
1247 {
1248   cpp_buffer *buffer = pfile->buffer;
1249   bool saw_NUL = false;
1250 
1251   do
1252     {
1253       /* Horizontal space always OK.  */
1254       if (c == ' ' || c == '\t')
1255 	;
1256       /* Just \f \v or \0 left.  */
1257       else if (c == '\0')
1258 	saw_NUL = true;
1259       else if (pfile->state.in_directive && CPP_PEDANTIC (pfile))
1260 	cpp_error_with_line (pfile, CPP_DL_PEDWARN, pfile->line_table->highest_line,
1261 			     CPP_BUF_COL (buffer),
1262 			     "%s in preprocessing directive",
1263 			     c == '\f' ? "form feed" : "vertical tab");
1264 
1265       c = *buffer->cur++;
1266     }
1267   /* We only want non-vertical space, i.e. ' ' \t \f \v \0.  */
1268   while (is_nvspace (c));
1269 
1270   if (saw_NUL)
1271     cpp_error (pfile, CPP_DL_WARNING, "null character(s) ignored");
1272 
1273   buffer->cur--;
1274 }
1275 
1276 /* See if the characters of a number token are valid in a name (no
1277    '.', '+' or '-').  */
1278 static int
name_p(cpp_reader * pfile,const cpp_string * string)1279 name_p (cpp_reader *pfile, const cpp_string *string)
1280 {
1281   unsigned int i;
1282 
1283   for (i = 0; i < string->len; i++)
1284     if (!is_idchar (string->text[i]))
1285       return 0;
1286 
1287   return 1;
1288 }
1289 
1290 /* After parsing an identifier or other sequence, produce a warning about
1291    sequences not in NFC/NFKC.  */
1292 static void
warn_about_normalization(cpp_reader * pfile,const cpp_token * token,const struct normalize_state * s)1293 warn_about_normalization (cpp_reader *pfile,
1294 			  const cpp_token *token,
1295 			  const struct normalize_state *s)
1296 {
1297   if (CPP_OPTION (pfile, warn_normalize) < NORMALIZE_STATE_RESULT (s)
1298       && !pfile->state.skipping)
1299     {
1300       /* Make sure that the token is printed using UCNs, even
1301 	 if we'd otherwise happily print UTF-8.  */
1302       unsigned char *buf = XNEWVEC (unsigned char, cpp_token_len (token));
1303       size_t sz;
1304 
1305       sz = cpp_spell_token (pfile, token, buf, false) - buf;
1306       if (NORMALIZE_STATE_RESULT (s) == normalized_C)
1307 	cpp_warning_with_line (pfile, CPP_W_NORMALIZE, token->src_loc, 0,
1308 			       "`%.*s' is not in NFKC", (int) sz, buf);
1309       else
1310 	cpp_warning_with_line (pfile, CPP_W_NORMALIZE, token->src_loc, 0,
1311 			       "`%.*s' is not in NFC", (int) sz, buf);
1312       free (buf);
1313     }
1314 }
1315 
1316 /* Returns TRUE if the sequence starting at buffer->cur is invalid in
1317    an identifier.  FIRST is TRUE if this starts an identifier.  */
1318 static bool
forms_identifier_p(cpp_reader * pfile,int first,struct normalize_state * state)1319 forms_identifier_p (cpp_reader *pfile, int first,
1320 		    struct normalize_state *state)
1321 {
1322   cpp_buffer *buffer = pfile->buffer;
1323 
1324   if (*buffer->cur == '$')
1325     {
1326       if (!CPP_OPTION (pfile, dollars_in_ident))
1327 	return false;
1328 
1329       buffer->cur++;
1330       if (CPP_OPTION (pfile, warn_dollars) && !pfile->state.skipping)
1331 	{
1332 	  CPP_OPTION (pfile, warn_dollars) = 0;
1333 	  cpp_error (pfile, CPP_DL_PEDWARN, "'$' in identifier or number");
1334 	}
1335 
1336       return true;
1337     }
1338 
1339   /* Is this a syntactically valid UCN?  */
1340   if (CPP_OPTION (pfile, extended_identifiers)
1341       && *buffer->cur == '\\'
1342       && (buffer->cur[1] == 'u' || buffer->cur[1] == 'U'))
1343     {
1344       cppchar_t s;
1345       buffer->cur += 2;
1346       if (_cpp_valid_ucn (pfile, &buffer->cur, buffer->rlimit, 1 + !first,
1347 			  state, &s, NULL, NULL))
1348 	return true;
1349       buffer->cur -= 2;
1350     }
1351 
1352   return false;
1353 }
1354 
1355 /* Helper function to issue error about improper __VA_OPT__ use.  */
1356 static void
maybe_va_opt_error(cpp_reader * pfile)1357 maybe_va_opt_error (cpp_reader *pfile)
1358 {
1359   if (CPP_PEDANTIC (pfile) && !CPP_OPTION (pfile, va_opt))
1360     {
1361       /* __VA_OPT__ should not be accepted at all, but allow it in
1362 	 system headers.  */
1363       if (!cpp_in_system_header (pfile))
1364 	cpp_error (pfile, CPP_DL_PEDWARN,
1365 		   "__VA_OPT__ is not available until C++2a");
1366     }
1367   else if (!pfile->state.va_args_ok)
1368     {
1369       /* __VA_OPT__ should only appear in the replacement list of a
1370 	 variadic macro.  */
1371       cpp_error (pfile, CPP_DL_PEDWARN,
1372 		 "__VA_OPT__ can only appear in the expansion"
1373 		 " of a C++2a variadic macro");
1374     }
1375 }
1376 
1377 /* Helper function to get the cpp_hashnode of the identifier BASE.  */
1378 static cpp_hashnode *
lex_identifier_intern(cpp_reader * pfile,const uchar * base)1379 lex_identifier_intern (cpp_reader *pfile, const uchar *base)
1380 {
1381   cpp_hashnode *result;
1382   const uchar *cur;
1383   unsigned int len;
1384   unsigned int hash = HT_HASHSTEP (0, *base);
1385 
1386   cur = base + 1;
1387   while (ISIDNUM (*cur))
1388     {
1389       hash = HT_HASHSTEP (hash, *cur);
1390       cur++;
1391     }
1392   len = cur - base;
1393   hash = HT_HASHFINISH (hash, len);
1394   result = CPP_HASHNODE (ht_lookup_with_hash (pfile->hash_table,
1395 					      base, len, hash, HT_ALLOC));
1396 
1397   /* Rarely, identifiers require diagnostics when lexed.  */
1398   if (__builtin_expect ((result->flags & NODE_DIAGNOSTIC)
1399 			&& !pfile->state.skipping, 0))
1400     {
1401       /* It is allowed to poison the same identifier twice.  */
1402       if ((result->flags & NODE_POISONED) && !pfile->state.poisoned_ok)
1403 	cpp_error (pfile, CPP_DL_ERROR, "attempt to use poisoned \"%s\"",
1404 		   NODE_NAME (result));
1405 
1406       /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1407 	 replacement list of a variadic macro.  */
1408       if (result == pfile->spec_nodes.n__VA_ARGS__
1409 	  && !pfile->state.va_args_ok)
1410 	{
1411 	  if (CPP_OPTION (pfile, cplusplus))
1412 	    cpp_error (pfile, CPP_DL_PEDWARN,
1413 		       "__VA_ARGS__ can only appear in the expansion"
1414 		       " of a C++11 variadic macro");
1415 	  else
1416 	    cpp_error (pfile, CPP_DL_PEDWARN,
1417 		       "__VA_ARGS__ can only appear in the expansion"
1418 		       " of a C99 variadic macro");
1419 	}
1420 
1421       if (result == pfile->spec_nodes.n__VA_OPT__)
1422 	maybe_va_opt_error (pfile);
1423 
1424       /* For -Wc++-compat, warn about use of C++ named operators.  */
1425       if (result->flags & NODE_WARN_OPERATOR)
1426 	cpp_warning (pfile, CPP_W_CXX_OPERATOR_NAMES,
1427 		     "identifier \"%s\" is a special operator name in C++",
1428 		     NODE_NAME (result));
1429     }
1430 
1431   return result;
1432 }
1433 
1434 /* Get the cpp_hashnode of an identifier specified by NAME in
1435    the current cpp_reader object.  If none is found, NULL is returned.  */
1436 cpp_hashnode *
_cpp_lex_identifier(cpp_reader * pfile,const char * name)1437 _cpp_lex_identifier (cpp_reader *pfile, const char *name)
1438 {
1439   cpp_hashnode *result;
1440   result = lex_identifier_intern (pfile, (uchar *) name);
1441   return result;
1442 }
1443 
1444 /* Lex an identifier starting at BUFFER->CUR - 1.  */
1445 static cpp_hashnode *
lex_identifier(cpp_reader * pfile,const uchar * base,bool starts_ucn,struct normalize_state * nst,cpp_hashnode ** spelling)1446 lex_identifier (cpp_reader *pfile, const uchar *base, bool starts_ucn,
1447 		struct normalize_state *nst, cpp_hashnode **spelling)
1448 {
1449   cpp_hashnode *result;
1450   const uchar *cur;
1451   unsigned int len;
1452   unsigned int hash = HT_HASHSTEP (0, *base);
1453 
1454   cur = pfile->buffer->cur;
1455   if (! starts_ucn)
1456     {
1457       while (ISIDNUM (*cur))
1458 	{
1459 	  hash = HT_HASHSTEP (hash, *cur);
1460 	  cur++;
1461 	}
1462       NORMALIZE_STATE_UPDATE_IDNUM (nst, *(cur - 1));
1463     }
1464   pfile->buffer->cur = cur;
1465   if (starts_ucn || forms_identifier_p (pfile, false, nst))
1466     {
1467       /* Slower version for identifiers containing UCNs (or $).  */
1468       do {
1469 	while (ISIDNUM (*pfile->buffer->cur))
1470 	  {
1471 	    NORMALIZE_STATE_UPDATE_IDNUM (nst, *pfile->buffer->cur);
1472 	    pfile->buffer->cur++;
1473 	  }
1474       } while (forms_identifier_p (pfile, false, nst));
1475       result = _cpp_interpret_identifier (pfile, base,
1476 					  pfile->buffer->cur - base);
1477       *spelling = cpp_lookup (pfile, base, pfile->buffer->cur - base);
1478     }
1479   else
1480     {
1481       len = cur - base;
1482       hash = HT_HASHFINISH (hash, len);
1483 
1484       result = CPP_HASHNODE (ht_lookup_with_hash (pfile->hash_table,
1485 						  base, len, hash, HT_ALLOC));
1486       *spelling = result;
1487     }
1488 
1489   /* Rarely, identifiers require diagnostics when lexed.  */
1490   if (__builtin_expect ((result->flags & NODE_DIAGNOSTIC)
1491 			&& !pfile->state.skipping, 0))
1492     {
1493       /* It is allowed to poison the same identifier twice.  */
1494       if ((result->flags & NODE_POISONED) && !pfile->state.poisoned_ok)
1495 	cpp_error (pfile, CPP_DL_ERROR, "attempt to use poisoned \"%s\"",
1496 		   NODE_NAME (result));
1497 
1498       /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1499 	 replacement list of a variadic macro.  */
1500       if (result == pfile->spec_nodes.n__VA_ARGS__
1501 	  && !pfile->state.va_args_ok)
1502 	{
1503 	  if (CPP_OPTION (pfile, cplusplus))
1504 	    cpp_error (pfile, CPP_DL_PEDWARN,
1505 		       "__VA_ARGS__ can only appear in the expansion"
1506 		       " of a C++11 variadic macro");
1507 	  else
1508 	    cpp_error (pfile, CPP_DL_PEDWARN,
1509 		       "__VA_ARGS__ can only appear in the expansion"
1510 		       " of a C99 variadic macro");
1511 	}
1512 
1513       /* __VA_OPT__ should only appear in the replacement list of a
1514 	 variadic macro.  */
1515       if (result == pfile->spec_nodes.n__VA_OPT__)
1516 	maybe_va_opt_error (pfile);
1517 
1518       /* For -Wc++-compat, warn about use of C++ named operators.  */
1519       if (result->flags & NODE_WARN_OPERATOR)
1520 	cpp_warning (pfile, CPP_W_CXX_OPERATOR_NAMES,
1521 		     "identifier \"%s\" is a special operator name in C++",
1522 		     NODE_NAME (result));
1523     }
1524 
1525   return result;
1526 }
1527 
1528 /* Lex a number to NUMBER starting at BUFFER->CUR - 1.  */
1529 static void
lex_number(cpp_reader * pfile,cpp_string * number,struct normalize_state * nst)1530 lex_number (cpp_reader *pfile, cpp_string *number,
1531 	    struct normalize_state *nst)
1532 {
1533   const uchar *cur;
1534   const uchar *base;
1535   uchar *dest;
1536 
1537   base = pfile->buffer->cur - 1;
1538   do
1539     {
1540       cur = pfile->buffer->cur;
1541 
1542       /* N.B. ISIDNUM does not include $.  */
1543       while (ISIDNUM (*cur) || *cur == '.' || DIGIT_SEP (*cur)
1544 	     || VALID_SIGN (*cur, cur[-1]))
1545 	{
1546 	  NORMALIZE_STATE_UPDATE_IDNUM (nst, *cur);
1547 	  cur++;
1548 	}
1549       /* A number can't end with a digit separator.  */
1550       while (cur > pfile->buffer->cur && DIGIT_SEP (cur[-1]))
1551 	--cur;
1552 
1553       pfile->buffer->cur = cur;
1554     }
1555   while (forms_identifier_p (pfile, false, nst));
1556 
1557   number->len = cur - base;
1558   dest = _cpp_unaligned_alloc (pfile, number->len + 1);
1559   memcpy (dest, base, number->len);
1560   dest[number->len] = '\0';
1561   number->text = dest;
1562 }
1563 
1564 /* Create a token of type TYPE with a literal spelling.  */
1565 static void
create_literal(cpp_reader * pfile,cpp_token * token,const uchar * base,unsigned int len,enum cpp_ttype type)1566 create_literal (cpp_reader *pfile, cpp_token *token, const uchar *base,
1567 		unsigned int len, enum cpp_ttype type)
1568 {
1569   uchar *dest = _cpp_unaligned_alloc (pfile, len + 1);
1570 
1571   memcpy (dest, base, len);
1572   dest[len] = '\0';
1573   token->type = type;
1574   token->val.str.len = len;
1575   token->val.str.text = dest;
1576 }
1577 
1578 /* Subroutine of lex_raw_string: Append LEN chars from BASE to the buffer
1579    sequence from *FIRST_BUFF_P to LAST_BUFF_P.  */
1580 
1581 static void
bufring_append(cpp_reader * pfile,const uchar * base,size_t len,_cpp_buff ** first_buff_p,_cpp_buff ** last_buff_p)1582 bufring_append (cpp_reader *pfile, const uchar *base, size_t len,
1583 		_cpp_buff **first_buff_p, _cpp_buff **last_buff_p)
1584 {
1585   _cpp_buff *first_buff = *first_buff_p;
1586   _cpp_buff *last_buff = *last_buff_p;
1587 
1588   if (first_buff == NULL)
1589     first_buff = last_buff = _cpp_get_buff (pfile, len);
1590   else if (len > BUFF_ROOM (last_buff))
1591     {
1592       size_t room = BUFF_ROOM (last_buff);
1593       memcpy (BUFF_FRONT (last_buff), base, room);
1594       BUFF_FRONT (last_buff) += room;
1595       base += room;
1596       len -= room;
1597       last_buff = _cpp_append_extend_buff (pfile, last_buff, len);
1598     }
1599 
1600   memcpy (BUFF_FRONT (last_buff), base, len);
1601   BUFF_FRONT (last_buff) += len;
1602 
1603   *first_buff_p = first_buff;
1604   *last_buff_p = last_buff;
1605 }
1606 
1607 
1608 /* Returns true if a macro has been defined.
1609    This might not work if compile with -save-temps,
1610    or preprocess separately from compilation.  */
1611 
1612 static bool
is_macro(cpp_reader * pfile,const uchar * base)1613 is_macro(cpp_reader *pfile, const uchar *base)
1614 {
1615   const uchar *cur = base;
1616   if (! ISIDST (*cur))
1617     return false;
1618   unsigned int hash = HT_HASHSTEP (0, *cur);
1619   ++cur;
1620   while (ISIDNUM (*cur))
1621     {
1622       hash = HT_HASHSTEP (hash, *cur);
1623       ++cur;
1624     }
1625   hash = HT_HASHFINISH (hash, cur - base);
1626 
1627   cpp_hashnode *result = CPP_HASHNODE (ht_lookup_with_hash (pfile->hash_table,
1628 					base, cur - base, hash, HT_NO_INSERT));
1629 
1630   return !result ? false : (result->type == NT_MACRO);
1631 }
1632 
1633 /* Returns true if a literal suffix does not have the expected form
1634    and is defined as a macro.  */
1635 
1636 static bool
is_macro_not_literal_suffix(cpp_reader * pfile,const uchar * base)1637 is_macro_not_literal_suffix(cpp_reader *pfile, const uchar *base)
1638 {
1639   /* User-defined literals outside of namespace std must start with a single
1640      underscore, so assume anything of that form really is a UDL suffix.
1641      We don't need to worry about UDLs defined inside namespace std because
1642      their names are reserved, so cannot be used as macro names in valid
1643      programs.  */
1644   if (base[0] == '_' && base[1] != '_')
1645     return false;
1646   return is_macro (pfile, base);
1647 }
1648 
1649 /* Lexes a raw string.  The stored string contains the spelling, including
1650    double quotes, delimiter string, '(' and ')', any leading
1651    'L', 'u', 'U' or 'u8' and 'R' modifier.  It returns the type of the
1652    literal, or CPP_OTHER if it was not properly terminated.
1653 
1654    The spelling is NUL-terminated, but it is not guaranteed that this
1655    is the first NUL since embedded NULs are preserved.  */
1656 
1657 static void
lex_raw_string(cpp_reader * pfile,cpp_token * token,const uchar * base,const uchar * cur)1658 lex_raw_string (cpp_reader *pfile, cpp_token *token, const uchar *base,
1659 		const uchar *cur)
1660 {
1661   uchar raw_prefix[17];
1662   uchar temp_buffer[18];
1663   const uchar *orig_base;
1664   unsigned int raw_prefix_len = 0, raw_suffix_len = 0;
1665   enum raw_str_phase { RAW_STR_PREFIX, RAW_STR, RAW_STR_SUFFIX };
1666   raw_str_phase phase = RAW_STR_PREFIX;
1667   enum cpp_ttype type;
1668   size_t total_len = 0;
1669   /* Index into temp_buffer during phases other than RAW_STR,
1670      during RAW_STR phase 17 to tell BUF_APPEND that nothing should
1671      be appended to temp_buffer.  */
1672   size_t temp_buffer_len = 0;
1673   _cpp_buff *first_buff = NULL, *last_buff = NULL;
1674   size_t raw_prefix_start;
1675   _cpp_line_note *note = &pfile->buffer->notes[pfile->buffer->cur_note];
1676 
1677   type = (*base == 'L' ? CPP_WSTRING :
1678 	  *base == 'U' ? CPP_STRING32 :
1679 	  *base == 'u' ? (base[1] == '8' ? CPP_UTF8STRING : CPP_STRING16)
1680 	  : CPP_STRING);
1681 
1682 #define BUF_APPEND(STR,LEN)					\
1683       do {							\
1684 	bufring_append (pfile, (const uchar *)(STR), (LEN),	\
1685 			&first_buff, &last_buff);		\
1686 	total_len += (LEN);					\
1687 	if (__builtin_expect (temp_buffer_len < 17, 0)		\
1688 	    && (const uchar *)(STR) != base			\
1689 	    && (LEN) <= 2)					\
1690 	  {							\
1691 	    memcpy (temp_buffer + temp_buffer_len,		\
1692 		    (const uchar *)(STR), (LEN));		\
1693 	    temp_buffer_len += (LEN);				\
1694 	  }							\
1695       } while (0)
1696 
1697   orig_base = base;
1698   ++cur;
1699   raw_prefix_start = cur - base;
1700   for (;;)
1701     {
1702       cppchar_t c;
1703 
1704       /* If we previously performed any trigraph or line splicing
1705 	 transformations, undo them in between the opening and closing
1706 	 double quote.  */
1707       while (note->pos < cur)
1708 	++note;
1709       for (; note->pos == cur; ++note)
1710 	{
1711 	  switch (note->type)
1712 	    {
1713 	    case '\\':
1714 	    case ' ':
1715 	      /* Restore backslash followed by newline.  */
1716 	      BUF_APPEND (base, cur - base);
1717 	      base = cur;
1718 	      BUF_APPEND ("\\", 1);
1719 	    after_backslash:
1720 	      if (note->type == ' ')
1721 		{
1722 		  /* GNU backslash whitespace newline extension.  FIXME
1723 		     could be any sequence of non-vertical space.  When we
1724 		     can properly restore any such sequence, we should mark
1725 		     this note as handled so _cpp_process_line_notes
1726 		     doesn't warn.  */
1727 		  BUF_APPEND (" ", 1);
1728 		}
1729 
1730 	      BUF_APPEND ("\n", 1);
1731 	      break;
1732 
1733 	    case 0:
1734 	      /* Already handled.  */
1735 	      break;
1736 
1737 	    default:
1738 	      if (_cpp_trigraph_map[note->type])
1739 		{
1740 		  /* Don't warn about this trigraph in
1741 		     _cpp_process_line_notes, since trigraphs show up as
1742 		     trigraphs in raw strings.  */
1743 		  uchar type = note->type;
1744 		  note->type = 0;
1745 
1746 		  if (!CPP_OPTION (pfile, trigraphs))
1747 		    /* If we didn't convert the trigraph in the first
1748 		       place, don't do anything now either.  */
1749 		    break;
1750 
1751 		  BUF_APPEND (base, cur - base);
1752 		  base = cur;
1753 		  BUF_APPEND ("??", 2);
1754 
1755 		  /* ??/ followed by newline gets two line notes, one for
1756 		     the trigraph and one for the backslash/newline.  */
1757 		  if (type == '/' && note[1].pos == cur)
1758 		    {
1759 		      if (note[1].type != '\\'
1760 			  && note[1].type != ' ')
1761 			abort ();
1762 		      BUF_APPEND ("/", 1);
1763 		      ++note;
1764 		      goto after_backslash;
1765 		    }
1766 		  else
1767 		    {
1768 		      /* Skip the replacement character.  */
1769 		      base = ++cur;
1770 		      BUF_APPEND (&type, 1);
1771 		      c = type;
1772 		      goto check_c;
1773 		    }
1774 		}
1775 	      else
1776 		abort ();
1777 	      break;
1778 	    }
1779 	}
1780       c = *cur++;
1781       if (__builtin_expect (temp_buffer_len < 17, 0))
1782 	temp_buffer[temp_buffer_len++] = c;
1783 
1784      check_c:
1785       if (phase == RAW_STR_PREFIX)
1786 	{
1787 	  while (raw_prefix_len < temp_buffer_len)
1788 	    {
1789 	      raw_prefix[raw_prefix_len] = temp_buffer[raw_prefix_len];
1790 	      switch (raw_prefix[raw_prefix_len])
1791 		{
1792 		case ' ': case '(': case ')': case '\\': case '\t':
1793 		case '\v': case '\f': case '\n': default:
1794 		  break;
1795 		/* Basic source charset except the above chars.  */
1796 		case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
1797 		case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
1798 		case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
1799 		case 's': case 't': case 'u': case 'v': case 'w': case 'x':
1800 		case 'y': case 'z':
1801 		case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
1802 		case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
1803 		case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
1804 		case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
1805 		case 'Y': case 'Z':
1806 		case '0': case '1': case '2': case '3': case '4': case '5':
1807 		case '6': case '7': case '8': case '9':
1808 		case '_': case '{': case '}': case '#': case '[': case ']':
1809 		case '<': case '>': case '%': case ':': case ';': case '.':
1810 		case '?': case '*': case '+': case '-': case '/': case '^':
1811 		case '&': case '|': case '~': case '!': case '=': case ',':
1812 		case '"': case '\'':
1813 		  if (raw_prefix_len < 16)
1814 		    {
1815 		      raw_prefix_len++;
1816 		      continue;
1817 		    }
1818 		  break;
1819 		}
1820 
1821 	      if (raw_prefix[raw_prefix_len] != '(')
1822 		{
1823 		  int col = CPP_BUF_COLUMN (pfile->buffer, cur) + 1;
1824 		  if (raw_prefix_len == 16)
1825 		    cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc,
1826 					 col, "raw string delimiter longer "
1827 					      "than 16 characters");
1828 		  else if (raw_prefix[raw_prefix_len] == '\n')
1829 		    cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc,
1830 					 col, "invalid new-line in raw "
1831 					      "string delimiter");
1832 		  else
1833 		    cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc,
1834 					 col, "invalid character '%c' in "
1835 					      "raw string delimiter",
1836 					 (int) raw_prefix[raw_prefix_len]);
1837 		  pfile->buffer->cur = orig_base + raw_prefix_start - 1;
1838 		  create_literal (pfile, token, orig_base,
1839 				  raw_prefix_start - 1, CPP_OTHER);
1840 		  if (first_buff)
1841 		    _cpp_release_buff (pfile, first_buff);
1842 		  return;
1843 		}
1844 	      raw_prefix[raw_prefix_len] = '"';
1845 	      phase = RAW_STR;
1846 	      /* Nothing should be appended to temp_buffer during
1847 		 RAW_STR phase.  */
1848 	      temp_buffer_len = 17;
1849 	      break;
1850 	    }
1851 	  continue;
1852 	}
1853       else if (phase == RAW_STR_SUFFIX)
1854 	{
1855 	  while (raw_suffix_len <= raw_prefix_len
1856 		 && raw_suffix_len < temp_buffer_len
1857 		 && temp_buffer[raw_suffix_len] == raw_prefix[raw_suffix_len])
1858 	    raw_suffix_len++;
1859 	  if (raw_suffix_len > raw_prefix_len)
1860 	    break;
1861 	  if (raw_suffix_len == temp_buffer_len)
1862 	    continue;
1863 	  phase = RAW_STR;
1864 	  /* Nothing should be appended to temp_buffer during
1865 	     RAW_STR phase.  */
1866 	  temp_buffer_len = 17;
1867 	}
1868       if (c == ')')
1869 	{
1870 	  phase = RAW_STR_SUFFIX;
1871 	  raw_suffix_len = 0;
1872 	  temp_buffer_len = 0;
1873 	}
1874       else if (c == '\n')
1875 	{
1876 	  if (pfile->state.in_directive
1877 	      || (pfile->state.parsing_args
1878 		  && pfile->buffer->next_line >= pfile->buffer->rlimit))
1879 	    {
1880 	      cur--;
1881 	      type = CPP_OTHER;
1882 	      cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, 0,
1883 				   "unterminated raw string");
1884 	      break;
1885 	    }
1886 
1887 	  BUF_APPEND (base, cur - base);
1888 
1889 	  if (pfile->buffer->cur < pfile->buffer->rlimit)
1890 	    CPP_INCREMENT_LINE (pfile, 0);
1891 	  pfile->buffer->need_line = true;
1892 
1893 	  pfile->buffer->cur = cur-1;
1894 	  _cpp_process_line_notes (pfile, false);
1895 	  if (!_cpp_get_fresh_line (pfile))
1896 	    {
1897 	      source_location src_loc = token->src_loc;
1898 	      token->type = CPP_EOF;
1899 	      /* Tell the compiler the line number of the EOF token.  */
1900 	      token->src_loc = pfile->line_table->highest_line;
1901 	      token->flags = BOL;
1902 	      if (first_buff != NULL)
1903 		_cpp_release_buff (pfile, first_buff);
1904 	      cpp_error_with_line (pfile, CPP_DL_ERROR, src_loc, 0,
1905 				   "unterminated raw string");
1906 	      return;
1907 	    }
1908 
1909 	  cur = base = pfile->buffer->cur;
1910 	  note = &pfile->buffer->notes[pfile->buffer->cur_note];
1911 	}
1912     }
1913 
1914   if (CPP_OPTION (pfile, user_literals))
1915     {
1916       /* If a string format macro, say from inttypes.h, is placed touching
1917 	 a string literal it could be parsed as a C++11 user-defined string
1918 	 literal thus breaking the program.  */
1919       if (is_macro_not_literal_suffix (pfile, cur))
1920 	{
1921 	  /* Raise a warning, but do not consume subsequent tokens.  */
1922 	  if (CPP_OPTION (pfile, warn_literal_suffix) && !pfile->state.skipping)
1923 	    cpp_warning_with_line (pfile, CPP_W_LITERAL_SUFFIX,
1924 				   token->src_loc, 0,
1925 				   "invalid suffix on literal; C++11 requires "
1926 				   "a space between literal and string macro");
1927 	}
1928       /* Grab user defined literal suffix.  */
1929       else if (ISIDST (*cur))
1930 	{
1931 	  type = cpp_userdef_string_add_type (type);
1932 	  ++cur;
1933 
1934 	  while (ISIDNUM (*cur))
1935 	    ++cur;
1936 	}
1937     }
1938 
1939   pfile->buffer->cur = cur;
1940   if (first_buff == NULL)
1941     create_literal (pfile, token, base, cur - base, type);
1942   else
1943     {
1944       uchar *dest = _cpp_unaligned_alloc (pfile, total_len + (cur - base) + 1);
1945 
1946       token->type = type;
1947       token->val.str.len = total_len + (cur - base);
1948       token->val.str.text = dest;
1949       last_buff = first_buff;
1950       while (last_buff != NULL)
1951 	{
1952 	  memcpy (dest, last_buff->base,
1953 		  BUFF_FRONT (last_buff) - last_buff->base);
1954 	  dest += BUFF_FRONT (last_buff) - last_buff->base;
1955 	  last_buff = last_buff->next;
1956 	}
1957       _cpp_release_buff (pfile, first_buff);
1958       memcpy (dest, base, cur - base);
1959       dest[cur - base] = '\0';
1960     }
1961 }
1962 
1963 /* Lexes a string, character constant, or angle-bracketed header file
1964    name.  The stored string contains the spelling, including opening
1965    quote and any leading 'L', 'u', 'U' or 'u8' and optional
1966    'R' modifier.  It returns the type of the literal, or CPP_OTHER
1967    if it was not properly terminated, or CPP_LESS for an unterminated
1968    header name which must be relexed as normal tokens.
1969 
1970    The spelling is NUL-terminated, but it is not guaranteed that this
1971    is the first NUL since embedded NULs are preserved.  */
1972 static void
lex_string(cpp_reader * pfile,cpp_token * token,const uchar * base)1973 lex_string (cpp_reader *pfile, cpp_token *token, const uchar *base)
1974 {
1975   bool saw_NUL = false;
1976   const uchar *cur;
1977   cppchar_t terminator;
1978   enum cpp_ttype type;
1979 
1980   cur = base;
1981   terminator = *cur++;
1982   if (terminator == 'L' || terminator == 'U')
1983     terminator = *cur++;
1984   else if (terminator == 'u')
1985     {
1986       terminator = *cur++;
1987       if (terminator == '8')
1988 	terminator = *cur++;
1989     }
1990   if (terminator == 'R')
1991     {
1992       lex_raw_string (pfile, token, base, cur);
1993       return;
1994     }
1995   if (terminator == '"')
1996     type = (*base == 'L' ? CPP_WSTRING :
1997 	    *base == 'U' ? CPP_STRING32 :
1998 	    *base == 'u' ? (base[1] == '8' ? CPP_UTF8STRING : CPP_STRING16)
1999 			 : CPP_STRING);
2000   else if (terminator == '\'')
2001     type = (*base == 'L' ? CPP_WCHAR :
2002 	    *base == 'U' ? CPP_CHAR32 :
2003 	    *base == 'u' ? (base[1] == '8' ? CPP_UTF8CHAR : CPP_CHAR16)
2004 			 : CPP_CHAR);
2005   else
2006     terminator = '>', type = CPP_HEADER_NAME;
2007 
2008   for (;;)
2009     {
2010       cppchar_t c = *cur++;
2011 
2012       /* In #include-style directives, terminators are not escapable.  */
2013       if (c == '\\' && !pfile->state.angled_headers && *cur != '\n')
2014 	cur++;
2015       else if (c == terminator)
2016 	break;
2017       else if (c == '\n')
2018 	{
2019 	  cur--;
2020 	  /* Unmatched quotes always yield undefined behavior, but
2021 	     greedy lexing means that what appears to be an unterminated
2022 	     header name may actually be a legitimate sequence of tokens.  */
2023 	  if (terminator == '>')
2024 	    {
2025 	      token->type = CPP_LESS;
2026 	      return;
2027 	    }
2028 	  type = CPP_OTHER;
2029 	  break;
2030 	}
2031       else if (c == '\0')
2032 	saw_NUL = true;
2033     }
2034 
2035   if (saw_NUL && !pfile->state.skipping)
2036     cpp_error (pfile, CPP_DL_WARNING,
2037 	       "null character(s) preserved in literal");
2038 
2039   if (type == CPP_OTHER && CPP_OPTION (pfile, lang) != CLK_ASM)
2040     cpp_error (pfile, CPP_DL_PEDWARN, "missing terminating %c character",
2041 	       (int) terminator);
2042 
2043   if (CPP_OPTION (pfile, user_literals))
2044     {
2045       /* If a string format macro, say from inttypes.h, is placed touching
2046 	 a string literal it could be parsed as a C++11 user-defined string
2047 	 literal thus breaking the program.  */
2048       if (is_macro_not_literal_suffix (pfile, cur))
2049 	{
2050 	  /* Raise a warning, but do not consume subsequent tokens.  */
2051 	  if (CPP_OPTION (pfile, warn_literal_suffix) && !pfile->state.skipping)
2052 	    cpp_warning_with_line (pfile, CPP_W_LITERAL_SUFFIX,
2053 				   token->src_loc, 0,
2054 				   "invalid suffix on literal; C++11 requires "
2055 				   "a space between literal and string macro");
2056 	}
2057       /* Grab user defined literal suffix.  */
2058       else if (ISIDST (*cur))
2059 	{
2060 	  type = cpp_userdef_char_add_type (type);
2061 	  type = cpp_userdef_string_add_type (type);
2062           ++cur;
2063 
2064 	  while (ISIDNUM (*cur))
2065 	    ++cur;
2066 	}
2067     }
2068   else if (CPP_OPTION (pfile, cpp_warn_cxx11_compat)
2069 	   && is_macro (pfile, cur)
2070 	   && !pfile->state.skipping)
2071     cpp_warning_with_line (pfile, CPP_W_CXX11_COMPAT,
2072 			   token->src_loc, 0, "C++11 requires a space "
2073 			   "between string literal and macro");
2074 
2075   pfile->buffer->cur = cur;
2076   create_literal (pfile, token, base, cur - base, type);
2077 }
2078 
2079 /* Return the comment table. The client may not make any assumption
2080    about the ordering of the table.  */
2081 cpp_comment_table *
cpp_get_comments(cpp_reader * pfile)2082 cpp_get_comments (cpp_reader *pfile)
2083 {
2084   return &pfile->comments;
2085 }
2086 
2087 /* Append a comment to the end of the comment table. */
2088 static void
store_comment(cpp_reader * pfile,cpp_token * token)2089 store_comment (cpp_reader *pfile, cpp_token *token)
2090 {
2091   int len;
2092 
2093   if (pfile->comments.allocated == 0)
2094     {
2095       pfile->comments.allocated = 256;
2096       pfile->comments.entries = (cpp_comment *) xmalloc
2097 	(pfile->comments.allocated * sizeof (cpp_comment));
2098     }
2099 
2100   if (pfile->comments.count == pfile->comments.allocated)
2101     {
2102       pfile->comments.allocated *= 2;
2103       pfile->comments.entries = (cpp_comment *) xrealloc
2104 	(pfile->comments.entries,
2105 	 pfile->comments.allocated * sizeof (cpp_comment));
2106     }
2107 
2108   len = token->val.str.len;
2109 
2110   /* Copy comment. Note, token may not be NULL terminated. */
2111   pfile->comments.entries[pfile->comments.count].comment =
2112     (char *) xmalloc (sizeof (char) * (len + 1));
2113   memcpy (pfile->comments.entries[pfile->comments.count].comment,
2114 	  token->val.str.text, len);
2115   pfile->comments.entries[pfile->comments.count].comment[len] = '\0';
2116 
2117   /* Set source location. */
2118   pfile->comments.entries[pfile->comments.count].sloc = token->src_loc;
2119 
2120   /* Increment the count of entries in the comment table. */
2121   pfile->comments.count++;
2122 }
2123 
2124 /* The stored comment includes the comment start and any terminator.  */
2125 static void
save_comment(cpp_reader * pfile,cpp_token * token,const unsigned char * from,cppchar_t type)2126 save_comment (cpp_reader *pfile, cpp_token *token, const unsigned char *from,
2127 	      cppchar_t type)
2128 {
2129   unsigned char *buffer;
2130   unsigned int len, clen, i;
2131 
2132   len = pfile->buffer->cur - from + 1; /* + 1 for the initial '/'.  */
2133 
2134   /* C++ comments probably (not definitely) have moved past a new
2135      line, which we don't want to save in the comment.  */
2136   if (is_vspace (pfile->buffer->cur[-1]))
2137     len--;
2138 
2139   /* If we are currently in a directive or in argument parsing, then
2140      we need to store all C++ comments as C comments internally, and
2141      so we need to allocate a little extra space in that case.
2142 
2143      Note that the only time we encounter a directive here is
2144      when we are saving comments in a "#define".  */
2145   clen = ((pfile->state.in_directive || pfile->state.parsing_args)
2146 	  && type == '/') ? len + 2 : len;
2147 
2148   buffer = _cpp_unaligned_alloc (pfile, clen);
2149 
2150   token->type = CPP_COMMENT;
2151   token->val.str.len = clen;
2152   token->val.str.text = buffer;
2153 
2154   buffer[0] = '/';
2155   memcpy (buffer + 1, from, len - 1);
2156 
2157   /* Finish conversion to a C comment, if necessary.  */
2158   if ((pfile->state.in_directive || pfile->state.parsing_args) && type == '/')
2159     {
2160       buffer[1] = '*';
2161       buffer[clen - 2] = '*';
2162       buffer[clen - 1] = '/';
2163       /* As there can be in a C++ comments illegal sequences for C comments
2164          we need to filter them out.  */
2165       for (i = 2; i < (clen - 2); i++)
2166         if (buffer[i] == '/' && (buffer[i - 1] == '*' || buffer[i + 1] == '*'))
2167           buffer[i] = '|';
2168     }
2169 
2170   /* Finally store this comment for use by clients of libcpp. */
2171   store_comment (pfile, token);
2172 }
2173 
2174 /* Returns true if comment at COMMENT_START is a recognized FALLTHROUGH
2175    comment.  */
2176 
2177 static bool
fallthrough_comment_p(cpp_reader * pfile,const unsigned char * comment_start)2178 fallthrough_comment_p (cpp_reader *pfile, const unsigned char *comment_start)
2179 {
2180   const unsigned char *from = comment_start + 1;
2181 
2182   switch (CPP_OPTION (pfile, cpp_warn_implicit_fallthrough))
2183     {
2184       /* For both -Wimplicit-fallthrough=0 and -Wimplicit-fallthrough=5 we
2185 	 don't recognize any comments.  The latter only checks attributes,
2186 	 the former doesn't warn.  */
2187     case 0:
2188     default:
2189       return false;
2190       /* -Wimplicit-fallthrough=1 considers any comment, no matter what
2191 	 content it has.  */
2192     case 1:
2193       return true;
2194     case 2:
2195       /* -Wimplicit-fallthrough=2 looks for (case insensitive)
2196 	 .*falls?[ \t-]*thr(u|ough).* regex.  */
2197       for (; (size_t) (pfile->buffer->cur - from) >= sizeof "fallthru" - 1;
2198 	   from++)
2199 	{
2200 	  /* Is there anything like strpbrk with upper boundary, or
2201 	     memchr looking for 2 characters rather than just one?  */
2202 	  if (from[0] != 'f' && from[0] != 'F')
2203 	    continue;
2204 	  if (from[1] != 'a' && from[1] != 'A')
2205 	    continue;
2206 	  if (from[2] != 'l' && from[2] != 'L')
2207 	    continue;
2208 	  if (from[3] != 'l' && from[3] != 'L')
2209 	    continue;
2210 	  from += sizeof "fall" - 1;
2211 	  if (from[0] == 's' || from[0] == 'S')
2212 	    from++;
2213 	  while (*from == ' ' || *from == '\t' || *from == '-')
2214 	    from++;
2215 	  if (from[0] != 't' && from[0] != 'T')
2216 	    continue;
2217 	  if (from[1] != 'h' && from[1] != 'H')
2218 	    continue;
2219 	  if (from[2] != 'r' && from[2] != 'R')
2220 	    continue;
2221 	  if (from[3] == 'u' || from[3] == 'U')
2222 	    return true;
2223 	  if (from[3] != 'o' && from[3] != 'O')
2224 	    continue;
2225 	  if (from[4] != 'u' && from[4] != 'U')
2226 	    continue;
2227 	  if (from[5] != 'g' && from[5] != 'G')
2228 	    continue;
2229 	  if (from[6] != 'h' && from[6] != 'H')
2230 	    continue;
2231 	  return true;
2232 	}
2233       return false;
2234     case 3:
2235     case 4:
2236       break;
2237     }
2238 
2239   /* Whole comment contents:
2240      -fallthrough
2241      @fallthrough@
2242    */
2243   if (*from == '-' || *from == '@')
2244     {
2245       size_t len = sizeof "fallthrough" - 1;
2246       if ((size_t) (pfile->buffer->cur - from - 1) < len)
2247 	return false;
2248       if (memcmp (from + 1, "fallthrough", len))
2249 	return false;
2250       if (*from == '@')
2251 	{
2252 	  if (from[len + 1] != '@')
2253 	    return false;
2254 	  len++;
2255 	}
2256       from += 1 + len;
2257     }
2258   /* Whole comment contents (regex):
2259      lint -fallthrough[ \t]*
2260    */
2261   else if (*from == 'l')
2262     {
2263       size_t len = sizeof "int -fallthrough" - 1;
2264       if ((size_t) (pfile->buffer->cur - from - 1) < len)
2265 	return false;
2266       if (memcmp (from + 1, "int -fallthrough", len))
2267 	return false;
2268       from += 1 + len;
2269       while (*from == ' ' || *from == '\t')
2270 	from++;
2271     }
2272   /* Whole comment contents (regex):
2273      [ \t]*FALLTHR(U|OUGH)[ \t]*
2274    */
2275   else if (CPP_OPTION (pfile, cpp_warn_implicit_fallthrough) == 4)
2276     {
2277       while (*from == ' ' || *from == '\t')
2278 	from++;
2279       if ((size_t) (pfile->buffer->cur - from)  < sizeof "FALLTHRU" - 1)
2280 	return false;
2281       if (memcmp (from, "FALLTHR", sizeof "FALLTHR" - 1))
2282 	return false;
2283       from += sizeof "FALLTHR" - 1;
2284       if (*from == 'U')
2285 	from++;
2286       else if ((size_t) (pfile->buffer->cur - from)  < sizeof "OUGH" - 1)
2287 	return false;
2288       else if (memcmp (from, "OUGH", sizeof "OUGH" - 1))
2289 	return false;
2290       else
2291 	from += sizeof "OUGH" - 1;
2292       while (*from == ' ' || *from == '\t')
2293 	from++;
2294     }
2295   /* Whole comment contents (regex):
2296      [ \t.!]*(ELSE,? |INTENTIONAL(LY)? )?FALL(S | |-)?THR(OUGH|U)[ \t.!]*(-[^\n\r]*)?
2297      [ \t.!]*(Else,? |Intentional(ly)? )?Fall((s | |-)[Tt]|t)hr(ough|u)[ \t.!]*(-[^\n\r]*)?
2298      [ \t.!]*([Ee]lse,? |[Ii]ntentional(ly)? )?fall(s | |-)?thr(ough|u)[ \t.!]*(-[^\n\r]*)?
2299    */
2300   else
2301     {
2302       while (*from == ' ' || *from == '\t' || *from == '.' || *from == '!')
2303 	from++;
2304       unsigned char f = *from;
2305       bool all_upper = false;
2306       if (f == 'E' || f == 'e')
2307 	{
2308 	  if ((size_t) (pfile->buffer->cur - from)
2309 	      < sizeof "else fallthru" - 1)
2310 	    return false;
2311 	  if (f == 'E' && memcmp (from + 1, "LSE", sizeof "LSE" - 1) == 0)
2312 	    all_upper = true;
2313 	  else if (memcmp (from + 1, "lse", sizeof "lse" - 1))
2314 	    return false;
2315 	  from += sizeof "else" - 1;
2316 	  if (*from == ',')
2317 	    from++;
2318 	  if (*from != ' ')
2319 	    return false;
2320 	  from++;
2321 	  if (all_upper && *from == 'f')
2322 	    return false;
2323 	  if (f == 'e' && *from == 'F')
2324 	    return false;
2325 	  f = *from;
2326 	}
2327       else if (f == 'I' || f == 'i')
2328 	{
2329 	  if ((size_t) (pfile->buffer->cur - from)
2330 	      < sizeof "intentional fallthru" - 1)
2331 	    return false;
2332 	  if (f == 'I' && memcmp (from + 1, "NTENTIONAL",
2333 				  sizeof "NTENTIONAL" - 1) == 0)
2334 	    all_upper = true;
2335 	  else if (memcmp (from + 1, "ntentional",
2336 			   sizeof "ntentional" - 1))
2337 	    return false;
2338 	  from += sizeof "intentional" - 1;
2339 	  if (*from == ' ')
2340 	    {
2341 	      from++;
2342 	      if (all_upper && *from == 'f')
2343 		return false;
2344 	    }
2345 	  else if (all_upper)
2346 	    {
2347 	      if (memcmp (from, "LY F", sizeof "LY F" - 1))
2348 		return false;
2349 	      from += sizeof "LY " - 1;
2350 	    }
2351 	  else
2352 	    {
2353 	      if (memcmp (from, "ly ", sizeof "ly " - 1))
2354 		return false;
2355 	      from += sizeof "ly " - 1;
2356 	    }
2357 	  if (f == 'i' && *from == 'F')
2358 	    return false;
2359 	  f = *from;
2360 	}
2361       if (f != 'F' && f != 'f')
2362 	return false;
2363       if ((size_t) (pfile->buffer->cur - from) < sizeof "fallthru" - 1)
2364 	return false;
2365       if (f == 'F' && memcmp (from + 1, "ALL", sizeof "ALL" - 1) == 0)
2366 	all_upper = true;
2367       else if (all_upper)
2368 	return false;
2369       else if (memcmp (from + 1, "all", sizeof "all" - 1))
2370 	return false;
2371       from += sizeof "fall" - 1;
2372       if (*from == (all_upper ? 'S' : 's') && from[1] == ' ')
2373 	from += 2;
2374       else if (*from == ' ' || *from == '-')
2375 	from++;
2376       else if (*from != (all_upper ? 'T' : 't'))
2377 	return false;
2378       if ((f == 'f' || *from != 'T') && (all_upper || *from != 't'))
2379 	return false;
2380       if ((size_t) (pfile->buffer->cur - from) < sizeof "thru" - 1)
2381 	return false;
2382       if (memcmp (from + 1, all_upper ? "HRU" : "hru", sizeof "hru" - 1))
2383 	{
2384 	  if ((size_t) (pfile->buffer->cur - from) < sizeof "through" - 1)
2385 	    return false;
2386 	  if (memcmp (from + 1, all_upper ? "HROUGH" : "hrough",
2387 		      sizeof "hrough" - 1))
2388 	    return false;
2389 	  from += sizeof "through" - 1;
2390 	}
2391       else
2392 	from += sizeof "thru" - 1;
2393       while (*from == ' ' || *from == '\t' || *from == '.' || *from == '!')
2394 	from++;
2395       if (*from == '-')
2396 	{
2397 	  from++;
2398 	  if (*comment_start == '*')
2399 	    {
2400 	      do
2401 		{
2402 		  while (*from && *from != '*'
2403 			 && *from != '\n' && *from != '\r')
2404 		    from++;
2405 		  if (*from != '*' || from[1] == '/')
2406 		    break;
2407 		  from++;
2408 		}
2409 	      while (1);
2410 	    }
2411 	  else
2412 	    while (*from && *from != '\n' && *from != '\r')
2413 	      from++;
2414 	}
2415     }
2416   /* C block comment.  */
2417   if (*comment_start == '*')
2418     {
2419       if (*from != '*' || from[1] != '/')
2420 	return false;
2421     }
2422   /* C++ line comment.  */
2423   else if (*from != '\n')
2424     return false;
2425 
2426   return true;
2427 }
2428 
2429 /* Allocate COUNT tokens for RUN.  */
2430 void
_cpp_init_tokenrun(tokenrun * run,unsigned int count)2431 _cpp_init_tokenrun (tokenrun *run, unsigned int count)
2432 {
2433   run->base = XNEWVEC (cpp_token, count);
2434   run->limit = run->base + count;
2435   run->next = NULL;
2436 }
2437 
2438 /* Returns the next tokenrun, or creates one if there is none.  */
2439 static tokenrun *
next_tokenrun(tokenrun * run)2440 next_tokenrun (tokenrun *run)
2441 {
2442   if (run->next == NULL)
2443     {
2444       run->next = XNEW (tokenrun);
2445       run->next->prev = run;
2446       _cpp_init_tokenrun (run->next, 250);
2447     }
2448 
2449   return run->next;
2450 }
2451 
2452 /* Return the number of not yet processed token in a given
2453    context.  */
2454 int
_cpp_remaining_tokens_num_in_context(cpp_context * context)2455 _cpp_remaining_tokens_num_in_context (cpp_context *context)
2456 {
2457   if (context->tokens_kind == TOKENS_KIND_DIRECT)
2458     return (LAST (context).token - FIRST (context).token);
2459   else if (context->tokens_kind == TOKENS_KIND_INDIRECT
2460 	   || context->tokens_kind == TOKENS_KIND_EXTENDED)
2461     return (LAST (context).ptoken - FIRST (context).ptoken);
2462   else
2463       abort ();
2464 }
2465 
2466 /* Returns the token present at index INDEX in a given context.  If
2467    INDEX is zero, the next token to be processed is returned.  */
2468 static const cpp_token*
_cpp_token_from_context_at(cpp_context * context,int index)2469 _cpp_token_from_context_at (cpp_context *context, int index)
2470 {
2471   if (context->tokens_kind == TOKENS_KIND_DIRECT)
2472     return &(FIRST (context).token[index]);
2473   else if (context->tokens_kind == TOKENS_KIND_INDIRECT
2474 	   || context->tokens_kind == TOKENS_KIND_EXTENDED)
2475     return FIRST (context).ptoken[index];
2476  else
2477    abort ();
2478 }
2479 
2480 /* Look ahead in the input stream.  */
2481 const cpp_token *
cpp_peek_token(cpp_reader * pfile,int index)2482 cpp_peek_token (cpp_reader *pfile, int index)
2483 {
2484   cpp_context *context = pfile->context;
2485   const cpp_token *peektok;
2486   int count;
2487 
2488   /* First, scan through any pending cpp_context objects.  */
2489   while (context->prev)
2490     {
2491       ptrdiff_t sz = _cpp_remaining_tokens_num_in_context (context);
2492 
2493       if (index < (int) sz)
2494         return _cpp_token_from_context_at (context, index);
2495       index -= (int) sz;
2496       context = context->prev;
2497     }
2498 
2499   /* We will have to read some new tokens after all (and do so
2500      without invalidating preceding tokens).  */
2501   count = index;
2502   pfile->keep_tokens++;
2503 
2504   /* For peeked tokens temporarily disable line_change reporting,
2505      until the tokens are parsed for real.  */
2506   void (*line_change) (cpp_reader *, const cpp_token *, int)
2507     = pfile->cb.line_change;
2508   pfile->cb.line_change = NULL;
2509 
2510   do
2511     {
2512       peektok = _cpp_lex_token (pfile);
2513       if (peektok->type == CPP_EOF)
2514 	{
2515 	  index--;
2516 	  break;
2517 	}
2518     }
2519   while (index--);
2520 
2521   _cpp_backup_tokens_direct (pfile, count - index);
2522   pfile->keep_tokens--;
2523   pfile->cb.line_change = line_change;
2524 
2525   return peektok;
2526 }
2527 
2528 /* Allocate a single token that is invalidated at the same time as the
2529    rest of the tokens on the line.  Has its line and col set to the
2530    same as the last lexed token, so that diagnostics appear in the
2531    right place.  */
2532 cpp_token *
_cpp_temp_token(cpp_reader * pfile)2533 _cpp_temp_token (cpp_reader *pfile)
2534 {
2535   cpp_token *old, *result;
2536   ptrdiff_t sz = pfile->cur_run->limit - pfile->cur_token;
2537   ptrdiff_t la = (ptrdiff_t) pfile->lookaheads;
2538 
2539   old = pfile->cur_token - 1;
2540   /* Any pre-existing lookaheads must not be clobbered.  */
2541   if (la)
2542     {
2543       if (sz <= la)
2544         {
2545           tokenrun *next = next_tokenrun (pfile->cur_run);
2546 
2547           if (sz < la)
2548             memmove (next->base + 1, next->base,
2549                      (la - sz) * sizeof (cpp_token));
2550 
2551           next->base[0] = pfile->cur_run->limit[-1];
2552         }
2553 
2554       if (sz > 1)
2555         memmove (pfile->cur_token + 1, pfile->cur_token,
2556                  MIN (la, sz - 1) * sizeof (cpp_token));
2557     }
2558 
2559   if (!sz && pfile->cur_token == pfile->cur_run->limit)
2560     {
2561       pfile->cur_run = next_tokenrun (pfile->cur_run);
2562       pfile->cur_token = pfile->cur_run->base;
2563     }
2564 
2565   result = pfile->cur_token++;
2566   result->src_loc = old->src_loc;
2567   return result;
2568 }
2569 
2570 /* Lex a token into RESULT (external interface).  Takes care of issues
2571    like directive handling, token lookahead, multiple include
2572    optimization and skipping.  */
2573 const cpp_token *
_cpp_lex_token(cpp_reader * pfile)2574 _cpp_lex_token (cpp_reader *pfile)
2575 {
2576   cpp_token *result;
2577 
2578   for (;;)
2579     {
2580       if (pfile->cur_token == pfile->cur_run->limit)
2581 	{
2582 	  pfile->cur_run = next_tokenrun (pfile->cur_run);
2583 	  pfile->cur_token = pfile->cur_run->base;
2584 	}
2585       /* We assume that the current token is somewhere in the current
2586 	 run.  */
2587       if (pfile->cur_token < pfile->cur_run->base
2588 	  || pfile->cur_token >= pfile->cur_run->limit)
2589 	abort ();
2590 
2591       if (pfile->lookaheads)
2592 	{
2593 	  pfile->lookaheads--;
2594 	  result = pfile->cur_token++;
2595 	}
2596       else
2597 	result = _cpp_lex_direct (pfile);
2598 
2599       if (result->flags & BOL)
2600 	{
2601 	  /* Is this a directive.  If _cpp_handle_directive returns
2602 	     false, it is an assembler #.  */
2603 	  if (result->type == CPP_HASH
2604 	      /* 6.10.3 p 11: Directives in a list of macro arguments
2605 		 gives undefined behavior.  This implementation
2606 		 handles the directive as normal.  */
2607 	      && pfile->state.parsing_args != 1)
2608 	    {
2609 	      if (_cpp_handle_directive (pfile, result->flags & PREV_WHITE))
2610 		{
2611 		  if (pfile->directive_result.type == CPP_PADDING)
2612 		    continue;
2613 		  result = &pfile->directive_result;
2614 		}
2615 	    }
2616 	  else if (pfile->state.in_deferred_pragma)
2617 	    result = &pfile->directive_result;
2618 
2619 	  if (pfile->cb.line_change && !pfile->state.skipping)
2620 	    pfile->cb.line_change (pfile, result, pfile->state.parsing_args);
2621 	}
2622 
2623       /* We don't skip tokens in directives.  */
2624       if (pfile->state.in_directive || pfile->state.in_deferred_pragma)
2625 	break;
2626 
2627       /* Outside a directive, invalidate controlling macros.  At file
2628 	 EOF, _cpp_lex_direct takes care of popping the buffer, so we never
2629 	 get here and MI optimization works.  */
2630       pfile->mi_valid = false;
2631 
2632       if (!pfile->state.skipping || result->type == CPP_EOF)
2633 	break;
2634     }
2635 
2636   return result;
2637 }
2638 
2639 /* Returns true if a fresh line has been loaded.  */
2640 bool
_cpp_get_fresh_line(cpp_reader * pfile)2641 _cpp_get_fresh_line (cpp_reader *pfile)
2642 {
2643   int return_at_eof;
2644 
2645   /* We can't get a new line until we leave the current directive.  */
2646   if (pfile->state.in_directive)
2647     return false;
2648 
2649   for (;;)
2650     {
2651       cpp_buffer *buffer = pfile->buffer;
2652 
2653       if (!buffer->need_line)
2654 	return true;
2655 
2656       if (buffer->next_line < buffer->rlimit)
2657 	{
2658 	  _cpp_clean_line (pfile);
2659 	  return true;
2660 	}
2661 
2662       /* First, get out of parsing arguments state.  */
2663       if (pfile->state.parsing_args)
2664 	return false;
2665 
2666       /* End of buffer.  Non-empty files should end in a newline.  */
2667       if (buffer->buf != buffer->rlimit
2668 	  && buffer->next_line > buffer->rlimit
2669 	  && !buffer->from_stage3)
2670 	{
2671 	  /* Clip to buffer size.  */
2672 	  buffer->next_line = buffer->rlimit;
2673 	}
2674 
2675       return_at_eof = buffer->return_at_eof;
2676       _cpp_pop_buffer (pfile);
2677       if (pfile->buffer == NULL || return_at_eof)
2678 	return false;
2679     }
2680 }
2681 
2682 #define IF_NEXT_IS(CHAR, THEN_TYPE, ELSE_TYPE)		\
2683   do							\
2684     {							\
2685       result->type = ELSE_TYPE;				\
2686       if (*buffer->cur == CHAR)				\
2687 	buffer->cur++, result->type = THEN_TYPE;	\
2688     }							\
2689   while (0)
2690 
2691 /* Lex a token into pfile->cur_token, which is also incremented, to
2692    get diagnostics pointing to the correct location.
2693 
2694    Does not handle issues such as token lookahead, multiple-include
2695    optimization, directives, skipping etc.  This function is only
2696    suitable for use by _cpp_lex_token, and in special cases like
2697    lex_expansion_token which doesn't care for any of these issues.
2698 
2699    When meeting a newline, returns CPP_EOF if parsing a directive,
2700    otherwise returns to the start of the token buffer if permissible.
2701    Returns the location of the lexed token.  */
2702 cpp_token *
_cpp_lex_direct(cpp_reader * pfile)2703 _cpp_lex_direct (cpp_reader *pfile)
2704 {
2705   cppchar_t c;
2706   cpp_buffer *buffer;
2707   const unsigned char *comment_start;
2708   bool fallthrough_comment = false;
2709   cpp_token *result = pfile->cur_token++;
2710 
2711  fresh_line:
2712   result->flags = 0;
2713   buffer = pfile->buffer;
2714   if (buffer->need_line)
2715     {
2716       if (pfile->state.in_deferred_pragma)
2717 	{
2718 	  result->type = CPP_PRAGMA_EOL;
2719 	  pfile->state.in_deferred_pragma = false;
2720 	  if (!pfile->state.pragma_allow_expansion)
2721 	    pfile->state.prevent_expansion--;
2722 	  return result;
2723 	}
2724       if (!_cpp_get_fresh_line (pfile))
2725 	{
2726 	  result->type = CPP_EOF;
2727 	  if (!pfile->state.in_directive)
2728 	    {
2729 	      /* Tell the compiler the line number of the EOF token.  */
2730 	      result->src_loc = pfile->line_table->highest_line;
2731 	      result->flags = BOL;
2732 	    }
2733 	  return result;
2734 	}
2735       if (buffer != pfile->buffer)
2736 	fallthrough_comment = false;
2737       if (!pfile->keep_tokens)
2738 	{
2739 	  pfile->cur_run = &pfile->base_run;
2740 	  result = pfile->base_run.base;
2741 	  pfile->cur_token = result + 1;
2742 	}
2743       result->flags = BOL;
2744       if (pfile->state.parsing_args == 2)
2745 	result->flags |= PREV_WHITE;
2746     }
2747   buffer = pfile->buffer;
2748  update_tokens_line:
2749   result->src_loc = pfile->line_table->highest_line;
2750 
2751  skipped_white:
2752   if (buffer->cur >= buffer->notes[buffer->cur_note].pos
2753       && !pfile->overlaid_buffer)
2754     {
2755       _cpp_process_line_notes (pfile, false);
2756       result->src_loc = pfile->line_table->highest_line;
2757     }
2758   c = *buffer->cur++;
2759 
2760   if (pfile->forced_token_location_p)
2761     result->src_loc = *pfile->forced_token_location_p;
2762   else
2763     result->src_loc = linemap_position_for_column (pfile->line_table,
2764 					  CPP_BUF_COLUMN (buffer, buffer->cur));
2765 
2766   switch (c)
2767     {
2768     case ' ': case '\t': case '\f': case '\v': case '\0':
2769       result->flags |= PREV_WHITE;
2770       skip_whitespace (pfile, c);
2771       goto skipped_white;
2772 
2773     case '\n':
2774       if (buffer->cur < buffer->rlimit)
2775 	CPP_INCREMENT_LINE (pfile, 0);
2776       buffer->need_line = true;
2777       goto fresh_line;
2778 
2779     case '0': case '1': case '2': case '3': case '4':
2780     case '5': case '6': case '7': case '8': case '9':
2781       {
2782 	struct normalize_state nst = INITIAL_NORMALIZE_STATE;
2783 	result->type = CPP_NUMBER;
2784 	lex_number (pfile, &result->val.str, &nst);
2785 	warn_about_normalization (pfile, result, &nst);
2786 	break;
2787       }
2788 
2789     case 'L':
2790     case 'u':
2791     case 'U':
2792     case 'R':
2793       /* 'L', 'u', 'U', 'u8' or 'R' may introduce wide characters,
2794 	 wide strings or raw strings.  */
2795       if (c == 'L' || CPP_OPTION (pfile, rliterals)
2796 	  || (c != 'R' && CPP_OPTION (pfile, uliterals)))
2797 	{
2798 	  if ((*buffer->cur == '\'' && c != 'R')
2799 	      || *buffer->cur == '"'
2800 	      || (*buffer->cur == 'R'
2801 		  && c != 'R'
2802 		  && buffer->cur[1] == '"'
2803 		  && CPP_OPTION (pfile, rliterals))
2804 	      || (*buffer->cur == '8'
2805 		  && c == 'u'
2806 		  && ((buffer->cur[1] == '"' || (buffer->cur[1] == '\''
2807 				&& CPP_OPTION (pfile, utf8_char_literals)))
2808 		      || (buffer->cur[1] == 'R' && buffer->cur[2] == '"'
2809 			  && CPP_OPTION (pfile, rliterals)))))
2810 	    {
2811 	      lex_string (pfile, result, buffer->cur - 1);
2812 	      break;
2813 	    }
2814 	}
2815       /* Fall through.  */
2816 
2817     case '_':
2818     case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
2819     case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
2820     case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
2821     case 's': case 't':           case 'v': case 'w': case 'x':
2822     case 'y': case 'z':
2823     case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
2824     case 'G': case 'H': case 'I': case 'J': case 'K':
2825     case 'M': case 'N': case 'O': case 'P': case 'Q':
2826     case 'S': case 'T':           case 'V': case 'W': case 'X':
2827     case 'Y': case 'Z':
2828       result->type = CPP_NAME;
2829       {
2830 	struct normalize_state nst = INITIAL_NORMALIZE_STATE;
2831 	result->val.node.node = lex_identifier (pfile, buffer->cur - 1, false,
2832 						&nst,
2833 						&result->val.node.spelling);
2834 	warn_about_normalization (pfile, result, &nst);
2835       }
2836 
2837       /* Convert named operators to their proper types.  */
2838       if (result->val.node.node->flags & NODE_OPERATOR)
2839 	{
2840 	  result->flags |= NAMED_OP;
2841 	  result->type = (enum cpp_ttype) result->val.node.node->directive_index;
2842 	}
2843 
2844       /* Signal FALLTHROUGH comment followed by another token.  */
2845       if (fallthrough_comment)
2846 	result->flags |= PREV_FALLTHROUGH;
2847       break;
2848 
2849     case '\'':
2850     case '"':
2851       lex_string (pfile, result, buffer->cur - 1);
2852       break;
2853 
2854     case '/':
2855       /* A potential block or line comment.  */
2856       comment_start = buffer->cur;
2857       c = *buffer->cur;
2858 
2859       if (c == '*')
2860 	{
2861 	  if (_cpp_skip_block_comment (pfile))
2862 	    cpp_error (pfile, CPP_DL_ERROR, "unterminated comment");
2863 	}
2864       else if (c == '/' && ! CPP_OPTION (pfile, traditional))
2865 	{
2866 	  /* Don't warn for system headers.  */
2867 	  if (cpp_in_system_header (pfile))
2868 	    ;
2869 	  /* Warn about comments if pedantically GNUC89, and not
2870 	     in system headers.  */
2871 	  else if (CPP_OPTION (pfile, lang) == CLK_GNUC89
2872 		   && CPP_PEDANTIC (pfile)
2873 		   && ! buffer->warned_cplusplus_comments)
2874 	    {
2875 	      cpp_error (pfile, CPP_DL_PEDWARN,
2876 			 "C++ style comments are not allowed in ISO C90");
2877 	      cpp_error (pfile, CPP_DL_PEDWARN,
2878 			 "(this will be reported only once per input file)");
2879 	      buffer->warned_cplusplus_comments = 1;
2880 	    }
2881 	  /* Or if specifically desired via -Wc90-c99-compat.  */
2882 	  else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0
2883 		   && ! CPP_OPTION (pfile, cplusplus)
2884 		   && ! buffer->warned_cplusplus_comments)
2885 	    {
2886 	      cpp_error (pfile, CPP_DL_WARNING,
2887 			 "C++ style comments are incompatible with C90");
2888 	      cpp_error (pfile, CPP_DL_WARNING,
2889 			 "(this will be reported only once per input file)");
2890 	      buffer->warned_cplusplus_comments = 1;
2891 	    }
2892 	  /* In C89/C94, C++ style comments are forbidden.  */
2893 	  else if ((CPP_OPTION (pfile, lang) == CLK_STDC89
2894 		    || CPP_OPTION (pfile, lang) == CLK_STDC94))
2895 	    {
2896 	      /* But don't be confused about valid code such as
2897 	         - // immediately followed by *,
2898 		 - // in a preprocessing directive,
2899 		 - // in an #if 0 block.  */
2900 	      if (buffer->cur[1] == '*'
2901 		  || pfile->state.in_directive
2902 		  || pfile->state.skipping)
2903 		{
2904 		  result->type = CPP_DIV;
2905 		  break;
2906 		}
2907 	      else if (! buffer->warned_cplusplus_comments)
2908 		{
2909 		  cpp_error (pfile, CPP_DL_ERROR,
2910 			     "C++ style comments are not allowed in ISO C90");
2911 		  cpp_error (pfile, CPP_DL_ERROR,
2912 			     "(this will be reported only once per input "
2913 			     "file)");
2914 		  buffer->warned_cplusplus_comments = 1;
2915 		}
2916 	    }
2917 	  if (skip_line_comment (pfile) && CPP_OPTION (pfile, warn_comments))
2918 	    cpp_warning (pfile, CPP_W_COMMENTS, "multi-line comment");
2919 	}
2920       else if (c == '=')
2921 	{
2922 	  buffer->cur++;
2923 	  result->type = CPP_DIV_EQ;
2924 	  break;
2925 	}
2926       else
2927 	{
2928 	  result->type = CPP_DIV;
2929 	  break;
2930 	}
2931 
2932       if (fallthrough_comment_p (pfile, comment_start))
2933 	fallthrough_comment = true;
2934 
2935       if (pfile->cb.comment)
2936 	{
2937 	  size_t len = pfile->buffer->cur - comment_start;
2938 	  pfile->cb.comment (pfile, result->src_loc, comment_start - 1,
2939 			     len + 1);
2940 	}
2941 
2942       if (!pfile->state.save_comments)
2943 	{
2944 	  result->flags |= PREV_WHITE;
2945 	  goto update_tokens_line;
2946 	}
2947 
2948       if (fallthrough_comment)
2949 	result->flags |= PREV_FALLTHROUGH;
2950 
2951       /* Save the comment as a token in its own right.  */
2952       save_comment (pfile, result, comment_start, c);
2953       break;
2954 
2955     case '<':
2956       if (pfile->state.angled_headers)
2957 	{
2958 	  lex_string (pfile, result, buffer->cur - 1);
2959 	  if (result->type != CPP_LESS)
2960 	    break;
2961 	}
2962 
2963       result->type = CPP_LESS;
2964       if (*buffer->cur == '=')
2965 	buffer->cur++, result->type = CPP_LESS_EQ;
2966       else if (*buffer->cur == '<')
2967 	{
2968 	  buffer->cur++;
2969 	  IF_NEXT_IS ('=', CPP_LSHIFT_EQ, CPP_LSHIFT);
2970 	}
2971       else if (CPP_OPTION (pfile, digraphs))
2972 	{
2973 	  if (*buffer->cur == ':')
2974 	    {
2975 	      /* C++11 [2.5/3 lex.pptoken], "Otherwise, if the next
2976 		 three characters are <:: and the subsequent character
2977 		 is neither : nor >, the < is treated as a preprocessor
2978 		 token by itself".  */
2979 	      if (CPP_OPTION (pfile, cplusplus)
2980 		  && CPP_OPTION (pfile, lang) != CLK_CXX98
2981 		  && CPP_OPTION (pfile, lang) != CLK_GNUCXX
2982 		  && buffer->cur[1] == ':'
2983 		  && buffer->cur[2] != ':' && buffer->cur[2] != '>')
2984 		break;
2985 
2986 	      buffer->cur++;
2987 	      result->flags |= DIGRAPH;
2988 	      result->type = CPP_OPEN_SQUARE;
2989 	    }
2990 	  else if (*buffer->cur == '%')
2991 	    {
2992 	      buffer->cur++;
2993 	      result->flags |= DIGRAPH;
2994 	      result->type = CPP_OPEN_BRACE;
2995 	    }
2996 	}
2997       break;
2998 
2999     case '>':
3000       result->type = CPP_GREATER;
3001       if (*buffer->cur == '=')
3002 	buffer->cur++, result->type = CPP_GREATER_EQ;
3003       else if (*buffer->cur == '>')
3004 	{
3005 	  buffer->cur++;
3006 	  IF_NEXT_IS ('=', CPP_RSHIFT_EQ, CPP_RSHIFT);
3007 	}
3008       break;
3009 
3010     case '%':
3011       result->type = CPP_MOD;
3012       if (*buffer->cur == '=')
3013 	buffer->cur++, result->type = CPP_MOD_EQ;
3014       else if (CPP_OPTION (pfile, digraphs))
3015 	{
3016 	  if (*buffer->cur == ':')
3017 	    {
3018 	      buffer->cur++;
3019 	      result->flags |= DIGRAPH;
3020 	      result->type = CPP_HASH;
3021 	      if (*buffer->cur == '%' && buffer->cur[1] == ':')
3022 		buffer->cur += 2, result->type = CPP_PASTE, result->val.token_no = 0;
3023 	    }
3024 	  else if (*buffer->cur == '>')
3025 	    {
3026 	      buffer->cur++;
3027 	      result->flags |= DIGRAPH;
3028 	      result->type = CPP_CLOSE_BRACE;
3029 	    }
3030 	}
3031       break;
3032 
3033     case '.':
3034       result->type = CPP_DOT;
3035       if (ISDIGIT (*buffer->cur))
3036 	{
3037 	  struct normalize_state nst = INITIAL_NORMALIZE_STATE;
3038 	  result->type = CPP_NUMBER;
3039 	  lex_number (pfile, &result->val.str, &nst);
3040 	  warn_about_normalization (pfile, result, &nst);
3041 	}
3042       else if (*buffer->cur == '.' && buffer->cur[1] == '.')
3043 	buffer->cur += 2, result->type = CPP_ELLIPSIS;
3044       else if (*buffer->cur == '*' && CPP_OPTION (pfile, cplusplus))
3045 	buffer->cur++, result->type = CPP_DOT_STAR;
3046       break;
3047 
3048     case '+':
3049       result->type = CPP_PLUS;
3050       if (*buffer->cur == '+')
3051 	buffer->cur++, result->type = CPP_PLUS_PLUS;
3052       else if (*buffer->cur == '=')
3053 	buffer->cur++, result->type = CPP_PLUS_EQ;
3054       break;
3055 
3056     case '-':
3057       result->type = CPP_MINUS;
3058       if (*buffer->cur == '>')
3059 	{
3060 	  buffer->cur++;
3061 	  result->type = CPP_DEREF;
3062 	  if (*buffer->cur == '*' && CPP_OPTION (pfile, cplusplus))
3063 	    buffer->cur++, result->type = CPP_DEREF_STAR;
3064 	}
3065       else if (*buffer->cur == '-')
3066 	buffer->cur++, result->type = CPP_MINUS_MINUS;
3067       else if (*buffer->cur == '=')
3068 	buffer->cur++, result->type = CPP_MINUS_EQ;
3069       break;
3070 
3071     case '&':
3072       result->type = CPP_AND;
3073       if (*buffer->cur == '&')
3074 	buffer->cur++, result->type = CPP_AND_AND;
3075       else if (*buffer->cur == '=')
3076 	buffer->cur++, result->type = CPP_AND_EQ;
3077       break;
3078 
3079     case '|':
3080       result->type = CPP_OR;
3081       if (*buffer->cur == '|')
3082 	buffer->cur++, result->type = CPP_OR_OR;
3083       else if (*buffer->cur == '=')
3084 	buffer->cur++, result->type = CPP_OR_EQ;
3085       break;
3086 
3087     case ':':
3088       result->type = CPP_COLON;
3089       if (*buffer->cur == ':' && CPP_OPTION (pfile, cplusplus))
3090 	buffer->cur++, result->type = CPP_SCOPE;
3091       else if (*buffer->cur == '>' && CPP_OPTION (pfile, digraphs))
3092 	{
3093 	  buffer->cur++;
3094 	  result->flags |= DIGRAPH;
3095 	  result->type = CPP_CLOSE_SQUARE;
3096 	}
3097       break;
3098 
3099     case '*': IF_NEXT_IS ('=', CPP_MULT_EQ, CPP_MULT); break;
3100     case '=': IF_NEXT_IS ('=', CPP_EQ_EQ, CPP_EQ); break;
3101     case '!': IF_NEXT_IS ('=', CPP_NOT_EQ, CPP_NOT); break;
3102     case '^': IF_NEXT_IS ('=', CPP_XOR_EQ, CPP_XOR); break;
3103     case '#': IF_NEXT_IS ('#', CPP_PASTE, CPP_HASH); result->val.token_no = 0; break;
3104 
3105     case '?': result->type = CPP_QUERY; break;
3106     case '~': result->type = CPP_COMPL; break;
3107     case ',': result->type = CPP_COMMA; break;
3108     case '(': result->type = CPP_OPEN_PAREN; break;
3109     case ')': result->type = CPP_CLOSE_PAREN; break;
3110     case '[': result->type = CPP_OPEN_SQUARE; break;
3111     case ']': result->type = CPP_CLOSE_SQUARE; break;
3112     case '{': result->type = CPP_OPEN_BRACE; break;
3113     case '}': result->type = CPP_CLOSE_BRACE; break;
3114     case ';': result->type = CPP_SEMICOLON; break;
3115 
3116       /* @ is a punctuator in Objective-C.  */
3117     case '@': result->type = CPP_ATSIGN; break;
3118 
3119     case '$':
3120     case '\\':
3121       {
3122 	const uchar *base = --buffer->cur;
3123 	struct normalize_state nst = INITIAL_NORMALIZE_STATE;
3124 
3125 	if (forms_identifier_p (pfile, true, &nst))
3126 	  {
3127 	    result->type = CPP_NAME;
3128 	    result->val.node.node = lex_identifier (pfile, base, true, &nst,
3129 						    &result->val.node.spelling);
3130 	    warn_about_normalization (pfile, result, &nst);
3131 	    break;
3132 	  }
3133 	buffer->cur++;
3134       }
3135       /* FALLTHRU */
3136 
3137     default:
3138       create_literal (pfile, result, buffer->cur - 1, 1, CPP_OTHER);
3139       break;
3140     }
3141 
3142   /* Potentially convert the location of the token to a range.  */
3143   if (result->src_loc >= RESERVED_LOCATION_COUNT
3144       && result->type != CPP_EOF)
3145     {
3146       /* Ensure that any line notes are processed, so that we have the
3147 	 correct physical line/column for the end-point of the token even
3148 	 when a logical line is split via one or more backslashes.  */
3149       if (buffer->cur >= buffer->notes[buffer->cur_note].pos
3150 	  && !pfile->overlaid_buffer)
3151 	_cpp_process_line_notes (pfile, false);
3152 
3153       source_range tok_range;
3154       tok_range.m_start = result->src_loc;
3155       tok_range.m_finish
3156 	= linemap_position_for_column (pfile->line_table,
3157 				       CPP_BUF_COLUMN (buffer, buffer->cur));
3158 
3159       result->src_loc = COMBINE_LOCATION_DATA (pfile->line_table,
3160 					       result->src_loc,
3161 					       tok_range, NULL);
3162     }
3163 
3164   return result;
3165 }
3166 
3167 /* An upper bound on the number of bytes needed to spell TOKEN.
3168    Does not include preceding whitespace.  */
3169 unsigned int
cpp_token_len(const cpp_token * token)3170 cpp_token_len (const cpp_token *token)
3171 {
3172   unsigned int len;
3173 
3174   switch (TOKEN_SPELL (token))
3175     {
3176     default:		len = 6;				break;
3177     case SPELL_LITERAL:	len = token->val.str.len;		break;
3178     case SPELL_IDENT:	len = NODE_LEN (token->val.node.node) * 10;	break;
3179     }
3180 
3181   return len;
3182 }
3183 
3184 /* Parse UTF-8 out of NAMEP and place a \U escape in BUFFER.
3185    Return the number of bytes read out of NAME.  (There are always
3186    10 bytes written to BUFFER.)  */
3187 
3188 static size_t
utf8_to_ucn(unsigned char * buffer,const unsigned char * name)3189 utf8_to_ucn (unsigned char *buffer, const unsigned char *name)
3190 {
3191   int j;
3192   int ucn_len = 0;
3193   int ucn_len_c;
3194   unsigned t;
3195   unsigned long utf32;
3196 
3197   /* Compute the length of the UTF-8 sequence.  */
3198   for (t = *name; t & 0x80; t <<= 1)
3199     ucn_len++;
3200 
3201   utf32 = *name & (0x7F >> ucn_len);
3202   for (ucn_len_c = 1; ucn_len_c < ucn_len; ucn_len_c++)
3203     {
3204       utf32 = (utf32 << 6) | (*++name & 0x3F);
3205 
3206       /* Ill-formed UTF-8.  */
3207       if ((*name & ~0x3F) != 0x80)
3208 	abort ();
3209     }
3210 
3211   *buffer++ = '\\';
3212   *buffer++ = 'U';
3213   for (j = 7; j >= 0; j--)
3214     *buffer++ = "0123456789abcdef"[(utf32 >> (4 * j)) & 0xF];
3215   return ucn_len;
3216 }
3217 
3218 /* Given a token TYPE corresponding to a digraph, return a pointer to
3219    the spelling of the digraph.  */
3220 static const unsigned char *
cpp_digraph2name(enum cpp_ttype type)3221 cpp_digraph2name (enum cpp_ttype type)
3222 {
3223   return digraph_spellings[(int) type - (int) CPP_FIRST_DIGRAPH];
3224 }
3225 
3226 /* Write the spelling of an identifier IDENT, using UCNs, to BUFFER.
3227    The buffer must already contain the enough space to hold the
3228    token's spelling.  Returns a pointer to the character after the
3229    last character written.  */
3230 unsigned char *
_cpp_spell_ident_ucns(unsigned char * buffer,cpp_hashnode * ident)3231 _cpp_spell_ident_ucns (unsigned char *buffer, cpp_hashnode *ident)
3232 {
3233   size_t i;
3234   const unsigned char *name = NODE_NAME (ident);
3235 
3236   for (i = 0; i < NODE_LEN (ident); i++)
3237     if (name[i] & ~0x7F)
3238       {
3239 	i += utf8_to_ucn (buffer, name + i) - 1;
3240 	buffer += 10;
3241       }
3242     else
3243       *buffer++ = name[i];
3244 
3245   return buffer;
3246 }
3247 
3248 /* Write the spelling of a token TOKEN to BUFFER.  The buffer must
3249    already contain the enough space to hold the token's spelling.
3250    Returns a pointer to the character after the last character written.
3251    FORSTRING is true if this is to be the spelling after translation
3252    phase 1 (with the original spelling of extended identifiers), false
3253    if extended identifiers should always be written using UCNs (there is
3254    no option for always writing them in the internal UTF-8 form).
3255    FIXME: Would be nice if we didn't need the PFILE argument.  */
3256 unsigned char *
cpp_spell_token(cpp_reader * pfile,const cpp_token * token,unsigned char * buffer,bool forstring)3257 cpp_spell_token (cpp_reader *pfile, const cpp_token *token,
3258 		 unsigned char *buffer, bool forstring)
3259 {
3260   switch (TOKEN_SPELL (token))
3261     {
3262     case SPELL_OPERATOR:
3263       {
3264 	const unsigned char *spelling;
3265 	unsigned char c;
3266 
3267 	if (token->flags & DIGRAPH)
3268 	  spelling = cpp_digraph2name (token->type);
3269 	else if (token->flags & NAMED_OP)
3270 	  goto spell_ident;
3271 	else
3272 	  spelling = TOKEN_NAME (token);
3273 
3274 	while ((c = *spelling++) != '\0')
3275 	  *buffer++ = c;
3276       }
3277       break;
3278 
3279     spell_ident:
3280     case SPELL_IDENT:
3281       if (forstring)
3282 	{
3283 	  memcpy (buffer, NODE_NAME (token->val.node.spelling),
3284 		  NODE_LEN (token->val.node.spelling));
3285 	  buffer += NODE_LEN (token->val.node.spelling);
3286 	}
3287       else
3288 	buffer = _cpp_spell_ident_ucns (buffer, token->val.node.node);
3289       break;
3290 
3291     case SPELL_LITERAL:
3292       memcpy (buffer, token->val.str.text, token->val.str.len);
3293       buffer += token->val.str.len;
3294       break;
3295 
3296     case SPELL_NONE:
3297       cpp_error (pfile, CPP_DL_ICE,
3298 		 "unspellable token %s", TOKEN_NAME (token));
3299       break;
3300     }
3301 
3302   return buffer;
3303 }
3304 
3305 /* Returns TOKEN spelt as a null-terminated string.  The string is
3306    freed when the reader is destroyed.  Useful for diagnostics.  */
3307 unsigned char *
cpp_token_as_text(cpp_reader * pfile,const cpp_token * token)3308 cpp_token_as_text (cpp_reader *pfile, const cpp_token *token)
3309 {
3310   unsigned int len = cpp_token_len (token) + 1;
3311   unsigned char *start = _cpp_unaligned_alloc (pfile, len), *end;
3312 
3313   end = cpp_spell_token (pfile, token, start, false);
3314   end[0] = '\0';
3315 
3316   return start;
3317 }
3318 
3319 /* Returns a pointer to a string which spells the token defined by
3320    TYPE and FLAGS.  Used by C front ends, which really should move to
3321    using cpp_token_as_text.  */
3322 const char *
cpp_type2name(enum cpp_ttype type,unsigned char flags)3323 cpp_type2name (enum cpp_ttype type, unsigned char flags)
3324 {
3325   if (flags & DIGRAPH)
3326     return (const char *) cpp_digraph2name (type);
3327   else if (flags & NAMED_OP)
3328     return cpp_named_operator2name (type);
3329 
3330   return (const char *) token_spellings[type].name;
3331 }
3332 
3333 /* Writes the spelling of token to FP, without any preceding space.
3334    Separated from cpp_spell_token for efficiency - to avoid stdio
3335    double-buffering.  */
3336 void
cpp_output_token(const cpp_token * token,FILE * fp)3337 cpp_output_token (const cpp_token *token, FILE *fp)
3338 {
3339   switch (TOKEN_SPELL (token))
3340     {
3341     case SPELL_OPERATOR:
3342       {
3343 	const unsigned char *spelling;
3344 	int c;
3345 
3346 	if (token->flags & DIGRAPH)
3347 	  spelling = cpp_digraph2name (token->type);
3348 	else if (token->flags & NAMED_OP)
3349 	  goto spell_ident;
3350 	else
3351 	  spelling = TOKEN_NAME (token);
3352 
3353 	c = *spelling;
3354 	do
3355 	  putc (c, fp);
3356 	while ((c = *++spelling) != '\0');
3357       }
3358       break;
3359 
3360     spell_ident:
3361     case SPELL_IDENT:
3362       {
3363 	size_t i;
3364 	const unsigned char * name = NODE_NAME (token->val.node.node);
3365 
3366 	for (i = 0; i < NODE_LEN (token->val.node.node); i++)
3367 	  if (name[i] & ~0x7F)
3368 	    {
3369 	      unsigned char buffer[10];
3370 	      i += utf8_to_ucn (buffer, name + i) - 1;
3371 	      fwrite (buffer, 1, 10, fp);
3372 	    }
3373 	  else
3374 	    fputc (NODE_NAME (token->val.node.node)[i], fp);
3375       }
3376       break;
3377 
3378     case SPELL_LITERAL:
3379       fwrite (token->val.str.text, 1, token->val.str.len, fp);
3380       break;
3381 
3382     case SPELL_NONE:
3383       /* An error, most probably.  */
3384       break;
3385     }
3386 }
3387 
3388 /* Compare two tokens.  */
3389 int
_cpp_equiv_tokens(const cpp_token * a,const cpp_token * b)3390 _cpp_equiv_tokens (const cpp_token *a, const cpp_token *b)
3391 {
3392   if (a->type == b->type && a->flags == b->flags)
3393     switch (TOKEN_SPELL (a))
3394       {
3395       default:			/* Keep compiler happy.  */
3396       case SPELL_OPERATOR:
3397 	/* token_no is used to track where multiple consecutive ##
3398 	   tokens were originally located.  */
3399 	return (a->type != CPP_PASTE || a->val.token_no == b->val.token_no);
3400       case SPELL_NONE:
3401 	return (a->type != CPP_MACRO_ARG
3402 		|| (a->val.macro_arg.arg_no == b->val.macro_arg.arg_no
3403 		    && a->val.macro_arg.spelling == b->val.macro_arg.spelling));
3404       case SPELL_IDENT:
3405 	return (a->val.node.node == b->val.node.node
3406 		&& a->val.node.spelling == b->val.node.spelling);
3407       case SPELL_LITERAL:
3408 	return (a->val.str.len == b->val.str.len
3409 		&& !memcmp (a->val.str.text, b->val.str.text,
3410 			    a->val.str.len));
3411       }
3412 
3413   return 0;
3414 }
3415 
3416 /* Returns nonzero if a space should be inserted to avoid an
3417    accidental token paste for output.  For simplicity, it is
3418    conservative, and occasionally advises a space where one is not
3419    needed, e.g. "." and ".2".  */
3420 int
cpp_avoid_paste(cpp_reader * pfile,const cpp_token * token1,const cpp_token * token2)3421 cpp_avoid_paste (cpp_reader *pfile, const cpp_token *token1,
3422 		 const cpp_token *token2)
3423 {
3424   enum cpp_ttype a = token1->type, b = token2->type;
3425   cppchar_t c;
3426 
3427   if (token1->flags & NAMED_OP)
3428     a = CPP_NAME;
3429   if (token2->flags & NAMED_OP)
3430     b = CPP_NAME;
3431 
3432   c = EOF;
3433   if (token2->flags & DIGRAPH)
3434     c = digraph_spellings[(int) b - (int) CPP_FIRST_DIGRAPH][0];
3435   else if (token_spellings[b].category == SPELL_OPERATOR)
3436     c = token_spellings[b].name[0];
3437 
3438   /* Quickly get everything that can paste with an '='.  */
3439   if ((int) a <= (int) CPP_LAST_EQ && c == '=')
3440     return 1;
3441 
3442   switch (a)
3443     {
3444     case CPP_GREATER:	return c == '>';
3445     case CPP_LESS:	return c == '<' || c == '%' || c == ':';
3446     case CPP_PLUS:	return c == '+';
3447     case CPP_MINUS:	return c == '-' || c == '>';
3448     case CPP_DIV:	return c == '/' || c == '*'; /* Comments.  */
3449     case CPP_MOD:	return c == ':' || c == '>';
3450     case CPP_AND:	return c == '&';
3451     case CPP_OR:	return c == '|';
3452     case CPP_COLON:	return c == ':' || c == '>';
3453     case CPP_DEREF:	return c == '*';
3454     case CPP_DOT:	return c == '.' || c == '%' || b == CPP_NUMBER;
3455     case CPP_HASH:	return c == '#' || c == '%'; /* Digraph form.  */
3456     case CPP_NAME:	return ((b == CPP_NUMBER
3457 				 && name_p (pfile, &token2->val.str))
3458 				|| b == CPP_NAME
3459 				|| b == CPP_CHAR || b == CPP_STRING); /* L */
3460     case CPP_NUMBER:	return (b == CPP_NUMBER || b == CPP_NAME
3461 				|| c == '.' || c == '+' || c == '-');
3462 				      /* UCNs */
3463     case CPP_OTHER:	return ((token1->val.str.text[0] == '\\'
3464 				 && b == CPP_NAME)
3465 				|| (CPP_OPTION (pfile, objc)
3466 				    && token1->val.str.text[0] == '@'
3467 				    && (b == CPP_NAME || b == CPP_STRING)));
3468     case CPP_STRING:
3469     case CPP_WSTRING:
3470     case CPP_UTF8STRING:
3471     case CPP_STRING16:
3472     case CPP_STRING32:	return (CPP_OPTION (pfile, user_literals)
3473 				&& (b == CPP_NAME
3474 				    || (TOKEN_SPELL (token2) == SPELL_LITERAL
3475 					&& ISIDST (token2->val.str.text[0]))));
3476 
3477     default:		break;
3478     }
3479 
3480   return 0;
3481 }
3482 
3483 /* Output all the remaining tokens on the current line, and a newline
3484    character, to FP.  Leading whitespace is removed.  If there are
3485    macros, special token padding is not performed.  */
3486 void
cpp_output_line(cpp_reader * pfile,FILE * fp)3487 cpp_output_line (cpp_reader *pfile, FILE *fp)
3488 {
3489   const cpp_token *token;
3490 
3491   token = cpp_get_token (pfile);
3492   while (token->type != CPP_EOF)
3493     {
3494       cpp_output_token (token, fp);
3495       token = cpp_get_token (pfile);
3496       if (token->flags & PREV_WHITE)
3497 	putc (' ', fp);
3498     }
3499 
3500   putc ('\n', fp);
3501 }
3502 
3503 /* Return a string representation of all the remaining tokens on the
3504    current line.  The result is allocated using xmalloc and must be
3505    freed by the caller.  */
3506 unsigned char *
cpp_output_line_to_string(cpp_reader * pfile,const unsigned char * dir_name)3507 cpp_output_line_to_string (cpp_reader *pfile, const unsigned char *dir_name)
3508 {
3509   const cpp_token *token;
3510   unsigned int out = dir_name ? ustrlen (dir_name) : 0;
3511   unsigned int alloced = 120 + out;
3512   unsigned char *result = (unsigned char *) xmalloc (alloced);
3513 
3514   /* If DIR_NAME is empty, there are no initial contents.  */
3515   if (dir_name)
3516     {
3517       sprintf ((char *) result, "#%s ", dir_name);
3518       out += 2;
3519     }
3520 
3521   token = cpp_get_token (pfile);
3522   while (token->type != CPP_EOF)
3523     {
3524       unsigned char *last;
3525       /* Include room for a possible space and the terminating nul.  */
3526       unsigned int len = cpp_token_len (token) + 2;
3527 
3528       if (out + len > alloced)
3529 	{
3530 	  alloced *= 2;
3531 	  if (out + len > alloced)
3532 	    alloced = out + len;
3533 	  result = (unsigned char *) xrealloc (result, alloced);
3534 	}
3535 
3536       last = cpp_spell_token (pfile, token, &result[out], 0);
3537       out = last - result;
3538 
3539       token = cpp_get_token (pfile);
3540       if (token->flags & PREV_WHITE)
3541 	result[out++] = ' ';
3542     }
3543 
3544   result[out] = '\0';
3545   return result;
3546 }
3547 
3548 /* Memory buffers.  Changing these three constants can have a dramatic
3549    effect on performance.  The values here are reasonable defaults,
3550    but might be tuned.  If you adjust them, be sure to test across a
3551    range of uses of cpplib, including heavy nested function-like macro
3552    expansion.  Also check the change in peak memory usage (NJAMD is a
3553    good tool for this).  */
3554 #define MIN_BUFF_SIZE 8000
3555 #define BUFF_SIZE_UPPER_BOUND(MIN_SIZE) (MIN_BUFF_SIZE + (MIN_SIZE) * 3 / 2)
3556 #define EXTENDED_BUFF_SIZE(BUFF, MIN_EXTRA) \
3557 	(MIN_EXTRA + ((BUFF)->limit - (BUFF)->cur) * 2)
3558 
3559 #if MIN_BUFF_SIZE > BUFF_SIZE_UPPER_BOUND (0)
3560   #error BUFF_SIZE_UPPER_BOUND must be at least as large as MIN_BUFF_SIZE!
3561 #endif
3562 
3563 /* Create a new allocation buffer.  Place the control block at the end
3564    of the buffer, so that buffer overflows will cause immediate chaos.  */
3565 static _cpp_buff *
new_buff(size_t len)3566 new_buff (size_t len)
3567 {
3568   _cpp_buff *result;
3569   unsigned char *base;
3570 
3571   if (len < MIN_BUFF_SIZE)
3572     len = MIN_BUFF_SIZE;
3573   len = CPP_ALIGN (len);
3574 
3575 #ifdef ENABLE_VALGRIND_ANNOTATIONS
3576   /* Valgrind warns about uses of interior pointers, so put _cpp_buff
3577      struct first.  */
3578   size_t slen = CPP_ALIGN2 (sizeof (_cpp_buff), 2 * DEFAULT_ALIGNMENT);
3579   base = XNEWVEC (unsigned char, len + slen);
3580   result = (_cpp_buff *) base;
3581   base += slen;
3582 #else
3583   base = XNEWVEC (unsigned char, len + sizeof (_cpp_buff));
3584   result = (_cpp_buff *) (base + len);
3585 #endif
3586   result->base = base;
3587   result->cur = base;
3588   result->limit = base + len;
3589   result->next = NULL;
3590   return result;
3591 }
3592 
3593 /* Place a chain of unwanted allocation buffers on the free list.  */
3594 void
_cpp_release_buff(cpp_reader * pfile,_cpp_buff * buff)3595 _cpp_release_buff (cpp_reader *pfile, _cpp_buff *buff)
3596 {
3597   _cpp_buff *end = buff;
3598 
3599   while (end->next)
3600     end = end->next;
3601   end->next = pfile->free_buffs;
3602   pfile->free_buffs = buff;
3603 }
3604 
3605 /* Return a free buffer of size at least MIN_SIZE.  */
3606 _cpp_buff *
_cpp_get_buff(cpp_reader * pfile,size_t min_size)3607 _cpp_get_buff (cpp_reader *pfile, size_t min_size)
3608 {
3609   _cpp_buff *result, **p;
3610 
3611   for (p = &pfile->free_buffs;; p = &(*p)->next)
3612     {
3613       size_t size;
3614 
3615       if (*p == NULL)
3616 	return new_buff (min_size);
3617       result = *p;
3618       size = result->limit - result->base;
3619       /* Return a buffer that's big enough, but don't waste one that's
3620          way too big.  */
3621       if (size >= min_size && size <= BUFF_SIZE_UPPER_BOUND (min_size))
3622 	break;
3623     }
3624 
3625   *p = result->next;
3626   result->next = NULL;
3627   result->cur = result->base;
3628   return result;
3629 }
3630 
3631 /* Creates a new buffer with enough space to hold the uncommitted
3632    remaining bytes of BUFF, and at least MIN_EXTRA more bytes.  Copies
3633    the excess bytes to the new buffer.  Chains the new buffer after
3634    BUFF, and returns the new buffer.  */
3635 _cpp_buff *
_cpp_append_extend_buff(cpp_reader * pfile,_cpp_buff * buff,size_t min_extra)3636 _cpp_append_extend_buff (cpp_reader *pfile, _cpp_buff *buff, size_t min_extra)
3637 {
3638   size_t size = EXTENDED_BUFF_SIZE (buff, min_extra);
3639   _cpp_buff *new_buff = _cpp_get_buff (pfile, size);
3640 
3641   buff->next = new_buff;
3642   memcpy (new_buff->base, buff->cur, BUFF_ROOM (buff));
3643   return new_buff;
3644 }
3645 
3646 /* Creates a new buffer with enough space to hold the uncommitted
3647    remaining bytes of the buffer pointed to by BUFF, and at least
3648    MIN_EXTRA more bytes.  Copies the excess bytes to the new buffer.
3649    Chains the new buffer before the buffer pointed to by BUFF, and
3650    updates the pointer to point to the new buffer.  */
3651 void
_cpp_extend_buff(cpp_reader * pfile,_cpp_buff ** pbuff,size_t min_extra)3652 _cpp_extend_buff (cpp_reader *pfile, _cpp_buff **pbuff, size_t min_extra)
3653 {
3654   _cpp_buff *new_buff, *old_buff = *pbuff;
3655   size_t size = EXTENDED_BUFF_SIZE (old_buff, min_extra);
3656 
3657   new_buff = _cpp_get_buff (pfile, size);
3658   memcpy (new_buff->base, old_buff->cur, BUFF_ROOM (old_buff));
3659   new_buff->next = old_buff;
3660   *pbuff = new_buff;
3661 }
3662 
3663 /* Free a chain of buffers starting at BUFF.  */
3664 void
_cpp_free_buff(_cpp_buff * buff)3665 _cpp_free_buff (_cpp_buff *buff)
3666 {
3667   _cpp_buff *next;
3668 
3669   for (; buff; buff = next)
3670     {
3671       next = buff->next;
3672 #ifdef ENABLE_VALGRIND_ANNOTATIONS
3673       free (buff);
3674 #else
3675       free (buff->base);
3676 #endif
3677     }
3678 }
3679 
3680 /* Allocate permanent, unaligned storage of length LEN.  */
3681 unsigned char *
_cpp_unaligned_alloc(cpp_reader * pfile,size_t len)3682 _cpp_unaligned_alloc (cpp_reader *pfile, size_t len)
3683 {
3684   _cpp_buff *buff = pfile->u_buff;
3685   unsigned char *result = buff->cur;
3686 
3687   if (len > (size_t) (buff->limit - result))
3688     {
3689       buff = _cpp_get_buff (pfile, len);
3690       buff->next = pfile->u_buff;
3691       pfile->u_buff = buff;
3692       result = buff->cur;
3693     }
3694 
3695   buff->cur = result + len;
3696   return result;
3697 }
3698 
3699 /* Allocate permanent, unaligned storage of length LEN from a_buff.
3700    That buffer is used for growing allocations when saving macro
3701    replacement lists in a #define, and when parsing an answer to an
3702    assertion in #assert, #unassert or #if (and therefore possibly
3703    whilst expanding macros).  It therefore must not be used by any
3704    code that they might call: specifically the lexer and the guts of
3705    the macro expander.
3706 
3707    All existing other uses clearly fit this restriction: storing
3708    registered pragmas during initialization.  */
3709 unsigned char *
_cpp_aligned_alloc(cpp_reader * pfile,size_t len)3710 _cpp_aligned_alloc (cpp_reader *pfile, size_t len)
3711 {
3712   _cpp_buff *buff = pfile->a_buff;
3713   unsigned char *result = buff->cur;
3714 
3715   if (len > (size_t) (buff->limit - result))
3716     {
3717       buff = _cpp_get_buff (pfile, len);
3718       buff->next = pfile->a_buff;
3719       pfile->a_buff = buff;
3720       result = buff->cur;
3721     }
3722 
3723   buff->cur = result + len;
3724   return result;
3725 }
3726 
3727 /* Say which field of TOK is in use.  */
3728 
3729 enum cpp_token_fld_kind
cpp_token_val_index(const cpp_token * tok)3730 cpp_token_val_index (const cpp_token *tok)
3731 {
3732   switch (TOKEN_SPELL (tok))
3733     {
3734     case SPELL_IDENT:
3735       return CPP_TOKEN_FLD_NODE;
3736     case SPELL_LITERAL:
3737       return CPP_TOKEN_FLD_STR;
3738     case SPELL_OPERATOR:
3739       if (tok->type == CPP_PASTE)
3740 	return CPP_TOKEN_FLD_TOKEN_NO;
3741       else
3742 	return CPP_TOKEN_FLD_NONE;
3743     case SPELL_NONE:
3744       if (tok->type == CPP_MACRO_ARG)
3745 	return CPP_TOKEN_FLD_ARG_NO;
3746       else if (tok->type == CPP_PADDING)
3747 	return CPP_TOKEN_FLD_SOURCE;
3748       else if (tok->type == CPP_PRAGMA)
3749 	return CPP_TOKEN_FLD_PRAGMA;
3750       /* fall through */
3751     default:
3752       return CPP_TOKEN_FLD_NONE;
3753     }
3754 }
3755 
3756 /* All tokens lexed in R after calling this function will be forced to have
3757    their source_location the same as the location referenced by P, until
3758    cpp_stop_forcing_token_locations is called for R.  */
3759 
3760 void
cpp_force_token_locations(cpp_reader * r,source_location * p)3761 cpp_force_token_locations (cpp_reader *r, source_location *p)
3762 {
3763   r->forced_token_location_p = p;
3764 }
3765 
3766 /* Go back to assigning locations naturally for lexed tokens.  */
3767 
3768 void
cpp_stop_forcing_token_locations(cpp_reader * r)3769 cpp_stop_forcing_token_locations (cpp_reader *r)
3770 {
3771   r->forced_token_location_p = NULL;
3772 }
3773