1 /// \file
2 /// Default implementation of CommonTokenStream
3 ///
4
5 // [The "BSD licence"]
6 // Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
7 // http://www.temporal-wave.com
8 // http://www.linkedin.com/in/jimidle
9 //
10 // All rights reserved.
11 //
12 // Redistribution and use in source and binary forms, with or without
13 // modification, are permitted provided that the following conditions
14 // are met:
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 // 3. The name of the author may not be used to endorse or promote products
21 // derived from this software without specific prior written permission.
22 //
23 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26 // IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27 // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28 // NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32 // THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33
34 #include <antlr3tokenstream.h>
35
36 #ifdef ANTLR3_WINDOWS
37 #pragma warning( disable : 4100 )
38 #endif
39
40 // COMMON_TOKEN_STREAM API
41 //
42 static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);
43 static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype);
44 static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard);
45 static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts);
46 static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
47 static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);
48 static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);
49 static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);
50 static void reset (pANTLR3_COMMON_TOKEN_STREAM cts);
51
52 // TOKEN_STREAM API
53 //
54 static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
55 static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
56 static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);
57 static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts);
58 static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);
59 static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts);
60 static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
61 static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);
62 static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);
63
64 // INT STREAM API
65 //
66 static void consume (pANTLR3_INT_STREAM is);
67 static void dbgConsume (pANTLR3_INT_STREAM is);
68 static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
69 static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
70 static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is);
71 static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is);
72 static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);
73 static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is);
74 static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is);
75 static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
76 static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
77 static void rewindLast (pANTLR3_INT_STREAM is);
78 static void dbgRewindLast (pANTLR3_INT_STREAM is);
79 static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
80 static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
81 static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is);
82 static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream);
83 static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream);
84
85 // Helpers
86 //
87 static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream);
88 static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
89 static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
90 static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
91
92 ANTLR3_API pANTLR3_TOKEN_STREAM
antlr3TokenStreamNew()93 antlr3TokenStreamNew()
94 {
95 pANTLR3_TOKEN_STREAM stream;
96
97 // Memory for the interface structure
98 //
99 stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));
100
101 if (stream == NULL)
102 {
103 return NULL;
104 }
105
106 // Install basic API
107 //
108 stream->free = antlr3TokenStreamFree;
109
110
111 return stream;
112 }
113
114 static void
antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)115 antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)
116 {
117 ANTLR3_FREE(stream);
118 }
119
120 static void
antlr3CTSFree(pANTLR3_COMMON_TOKEN_STREAM stream)121 antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream)
122 {
123 // We only free up our subordinate interfaces if they belong
124 // to us, otherwise we let whoever owns them deal with them.
125 //
126 if (stream->tstream->super == stream)
127 {
128 if (stream->tstream->istream->super == stream->tstream)
129 {
130 stream->tstream->istream->free(stream->tstream->istream);
131 stream->tstream->istream = NULL;
132 }
133 stream->tstream->free(stream->tstream);
134 }
135
136 // Now we free our own resources
137 //
138 if (stream->tokens != NULL)
139 {
140 stream->tokens->free(stream->tokens);
141 stream->tokens = NULL;
142 }
143 if (stream->discardSet != NULL)
144 {
145 stream->discardSet->free(stream->discardSet);
146 stream->discardSet = NULL;
147 }
148 if (stream->channelOverrides != NULL)
149 {
150 stream->channelOverrides->free(stream->channelOverrides);
151 stream->channelOverrides = NULL;
152 }
153
154 // Free our memory now
155 //
156 ANTLR3_FREE(stream);
157 }
158
159 // Reset a token stream so it can be used again and can reuse it's
160 // resources.
161 //
162 static void
reset(pANTLR3_COMMON_TOKEN_STREAM cts)163 reset (pANTLR3_COMMON_TOKEN_STREAM cts)
164 {
165
166 // Free any resources that ar most like specifc to the
167 // run we just did.
168 //
169 if (cts->discardSet != NULL)
170 {
171 cts->discardSet->free(cts->discardSet);
172 cts->discardSet = NULL;
173 }
174 if (cts->channelOverrides != NULL)
175 {
176 cts->channelOverrides->free(cts->channelOverrides);
177 cts->channelOverrides = NULL;
178 }
179
180 // Now, if there were any existing tokens in the stream,
181 // then we just reset the vector count so that it starts
182 // again. We must traverse the entries unfortunately as
183 // there may be free pointers for custom token types and
184 // so on. However that is just a quick NULL check on the
185 // vector entries.
186 //
187 if (cts->tokens != NULL)
188 {
189 cts->tokens->clear(cts->tokens);
190 }
191 else
192 {
193 /* Install the token tracking tables
194 */
195 cts->tokens = antlr3VectorNew(0);
196 }
197
198 // Reset to defaults
199 //
200 cts->discardOffChannel = ANTLR3_FALSE;
201 cts->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
202 cts->p = -1;
203 }
204
205 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint,pANTLR3_TOKEN_SOURCE source,pANTLR3_DEBUG_EVENT_LISTENER debugger)206 antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)
207 {
208 pANTLR3_COMMON_TOKEN_STREAM stream;
209
210 // Create a standard token stream
211 //
212 stream = antlr3CommonTokenStreamSourceNew(hint, source);
213
214 // Install the debugger object
215 //
216 stream->tstream->debugger = debugger;
217
218 // Override standard token stream methods with debugging versions
219 //
220 stream->tstream->initialStreamState = ANTLR3_FALSE;
221
222 stream->tstream->_LT = dbgTokLT;
223
224 stream->tstream->istream->consume = dbgConsume;
225 stream->tstream->istream->_LA = dbgLA;
226 stream->tstream->istream->mark = dbgMark;
227 stream->tstream->istream->rewind = dbgRewindStream;
228 stream->tstream->istream->rewindLast = dbgRewindLast;
229 stream->tstream->istream->seek = dbgSeek;
230
231 return stream;
232 }
233
234 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint,pANTLR3_TOKEN_SOURCE source)235 antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)
236 {
237 pANTLR3_COMMON_TOKEN_STREAM stream;
238
239 stream = antlr3CommonTokenStreamNew(hint);
240
241 stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
242
243 stream->channelOverrides = NULL;
244 stream->discardSet = NULL;
245 stream->discardOffChannel = ANTLR3_FALSE;
246
247 stream->tstream->setTokenSource(stream->tstream, source);
248
249 stream->free = antlr3CTSFree;
250 return stream;
251 }
252
253 ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)254 antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)
255 {
256 pANTLR3_COMMON_TOKEN_STREAM stream;
257
258 /* Memory for the interface structure
259 */
260 stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));
261
262 if (stream == NULL)
263 {
264 return NULL;
265 }
266
267 /* Create space for the token stream interface
268 */
269 stream->tstream = antlr3TokenStreamNew();
270 stream->tstream->super = stream;
271
272 /* Create space for the INT_STREAM interfacce
273 */
274 stream->tstream->istream = antlr3IntStreamNew();
275 stream->tstream->istream->super = (stream->tstream);
276 stream->tstream->istream->type = ANTLR3_TOKENSTREAM;
277
278 /* Install the token tracking tables
279 */
280 stream->tokens = antlr3VectorNew(0);
281
282 /* Defaults
283 */
284 stream->p = -1;
285
286 /* Install the common token stream API
287 */
288 stream->setTokenTypeChannel = setTokenTypeChannel;
289 stream->discardTokenType = discardTokenType;
290 stream->discardOffChannelToks = discardOffChannel;
291 stream->getTokens = getTokens;
292 stream->getTokenRange = getTokenRange;
293 stream->getTokensSet = getTokensSet;
294 stream->getTokensList = getTokensList;
295 stream->getTokensType = getTokensType;
296 stream->reset = reset;
297
298 /* Install the token stream API
299 */
300 stream->tstream->_LT = tokLT;
301 stream->tstream->get = get;
302 stream->tstream->getTokenSource = getTokenSource;
303 stream->tstream->setTokenSource = setTokenSource;
304 stream->tstream->toString = toString;
305 stream->tstream->toStringSS = toStringSS;
306 stream->tstream->toStringTT = toStringTT;
307 stream->tstream->setDebugListener = setDebugListener;
308
309 /* Install INT_STREAM interface
310 */
311 stream->tstream->istream->_LA = _LA;
312 stream->tstream->istream->mark = mark;
313 stream->tstream->istream->release = release;
314 stream->tstream->istream->size = size;
315 stream->tstream->istream->index = tindex;
316 stream->tstream->istream->rewind = rewindStream;
317 stream->tstream->istream->rewindLast= rewindLast;
318 stream->tstream->istream->seek = seek;
319 stream->tstream->istream->consume = consume;
320 stream->tstream->istream->getSourceName = getSourceName;
321
322 return stream;
323 }
324
325 // Install a debug listener adn switch to debug mode methods
326 //
327 static void
setDebugListener(pANTLR3_TOKEN_STREAM ts,pANTLR3_DEBUG_EVENT_LISTENER debugger)328 setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)
329 {
330 // Install the debugger object
331 //
332 ts->debugger = debugger;
333
334 // Override standard token stream methods with debugging versions
335 //
336 ts->initialStreamState = ANTLR3_FALSE;
337
338 ts->_LT = dbgTokLT;
339
340 ts->istream->consume = dbgConsume;
341 ts->istream->_LA = dbgLA;
342 ts->istream->mark = dbgMark;
343 ts->istream->rewind = dbgRewindStream;
344 ts->istream->rewindLast = dbgRewindLast;
345 ts->istream->seek = dbgSeek;
346 }
347
348 /** Get the ith token from the current position 1..n where k=1 is the
349 * first symbol of lookahead.
350 */
351 static pANTLR3_COMMON_TOKEN
tokLT(pANTLR3_TOKEN_STREAM ts,ANTLR3_INT32 k)352 tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
353 {
354 ANTLR3_INT32 i;
355 ANTLR3_INT32 n;
356 pANTLR3_COMMON_TOKEN_STREAM cts;
357
358 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
359
360 if (k < 0)
361 {
362 return LB(cts, -k);
363 }
364
365 if (cts->p == -1)
366 {
367 fillBuffer(cts);
368 }
369
370 // Here we used to check for k == 0 and return 0, but this seems
371 // a superfluous check to me. LT(k=0) is therefore just undefined
372 // and we won't waste the clock cycles on the check
373 //
374
375 if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)
376 {
377 pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
378
379 teof->setStartIndex (teof, ts->istream->index (ts->istream));
380 teof->setStopIndex (teof, ts->istream->index (ts->istream));
381 return teof;
382 }
383
384 i = cts->p;
385 n = 1;
386
387 /* Need to find k good tokens, skipping ones that are off channel
388 */
389 while ( n < k)
390 {
391 /* Skip off-channel tokens */
392 i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */
393 n++;
394 }
395 if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize)
396 {
397 pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
398
399 teof->setStartIndex (teof, ts->istream->index(ts->istream));
400 teof->setStopIndex (teof, ts->istream->index(ts->istream));
401 return teof;
402 }
403
404 // Here the token must be in the input vector. Rather then incur
405 // function call penalty, we just return the pointer directly
406 // from the vector
407 //
408 return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
409 //return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);
410 }
411
412 /// Debug only method to flag consumption of initial off-channel
413 /// tokens in the input stream
414 ///
415 static void
consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)416 consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)
417 {
418 ANTLR3_MARKER first;
419 ANTLR3_INT32 i;
420 pANTLR3_TOKEN_STREAM ts;
421
422 ts = (pANTLR3_TOKEN_STREAM) is->super;
423 first = is->index(is);
424
425 for (i=0; i<first; i++)
426 {
427 ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i));
428 }
429
430 ts->initialStreamState = ANTLR3_FALSE;
431
432 }
433
434 /// As per the normal tokLT but sends information to the debugger
435 ///
436 static pANTLR3_COMMON_TOKEN
dbgTokLT(pANTLR3_TOKEN_STREAM ts,ANTLR3_INT32 k)437 dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
438 {
439 if (ts->initialStreamState == ANTLR3_TRUE)
440 {
441 consumeInitialHiddenTokens(ts->istream);
442 }
443 return tokLT(ts, k);
444 }
445
446 #ifdef ANTLR3_WINDOWS
447 /* When fully optimized VC7 complains about non reachable code.
448 * Not yet sure if this is an optimizer bug, or a bug in the flow analysis
449 */
450 #pragma warning( disable : 4702 )
451 #endif
452
453 static pANTLR3_COMMON_TOKEN
LB(pANTLR3_COMMON_TOKEN_STREAM cts,ANTLR3_INT32 k)454 LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k)
455 {
456 ANTLR3_INT32 i;
457 ANTLR3_INT32 n;
458
459 if (cts->p == -1)
460 {
461 fillBuffer(cts);
462 }
463 if (k == 0)
464 {
465 return NULL;
466 }
467 if ((cts->p - k) < 0)
468 {
469 return NULL;
470 }
471
472 i = cts->p;
473 n = 1;
474
475 /* Need to find k good tokens, going backwards, skipping ones that are off channel
476 */
477 while (n <= (ANTLR3_INT32) k)
478 {
479 /* Skip off-channel tokens
480 */
481
482 i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */
483 n++;
484 }
485 if (i < 0)
486 {
487 return NULL;
488 }
489 // Here the token must be in the input vector. Rather then incut
490 // function call penalty, we jsut return the pointer directly
491 // from the vector
492 //
493 return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
494 }
495
496 static pANTLR3_COMMON_TOKEN
get(pANTLR3_TOKEN_STREAM ts,ANTLR3_UINT32 i)497 get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)
498 {
499 pANTLR3_COMMON_TOKEN_STREAM cts;
500
501 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
502
503 return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */
504 }
505
506 static pANTLR3_TOKEN_SOURCE
getTokenSource(pANTLR3_TOKEN_STREAM ts)507 getTokenSource (pANTLR3_TOKEN_STREAM ts)
508 {
509 return ts->tokenSource;
510 }
511
512 static void
setTokenSource(pANTLR3_TOKEN_STREAM ts,pANTLR3_TOKEN_SOURCE tokenSource)513 setTokenSource ( pANTLR3_TOKEN_STREAM ts,
514 pANTLR3_TOKEN_SOURCE tokenSource)
515 {
516 ts->tokenSource = tokenSource;
517 }
518
519 static pANTLR3_STRING
toString(pANTLR3_TOKEN_STREAM ts)520 toString (pANTLR3_TOKEN_STREAM ts)
521 {
522 pANTLR3_COMMON_TOKEN_STREAM cts;
523
524 cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
525
526 if (cts->p == -1)
527 {
528 fillBuffer(cts);
529 }
530
531 return ts->toStringSS(ts, 0, ts->istream->size(ts->istream));
532 }
533
534 static pANTLR3_STRING
toStringSS(pANTLR3_TOKEN_STREAM ts,ANTLR3_UINT32 start,ANTLR3_UINT32 stop)535 toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
536 {
537 pANTLR3_STRING string;
538 pANTLR3_TOKEN_SOURCE tsource;
539 pANTLR3_COMMON_TOKEN tok;
540 ANTLR3_UINT32 i;
541 pANTLR3_COMMON_TOKEN_STREAM cts;
542
543 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
544
545 if (cts->p == -1)
546 {
547 fillBuffer(cts);
548 }
549 if (stop >= ts->istream->size(ts->istream))
550 {
551 stop = ts->istream->size(ts->istream) - 1;
552 }
553
554 /* Who is giving us these tokens?
555 */
556 tsource = ts->getTokenSource(ts);
557
558 if (tsource != NULL && cts->tokens != NULL)
559 {
560 /* Finally, let's get a string
561 */
562 string = tsource->strFactory->newRaw(tsource->strFactory);
563
564 for (i = start; i <= stop; i++)
565 {
566 tok = ts->get(ts, i);
567 if (tok != NULL)
568 {
569 string->appendS(string, tok->getText(tok));
570 }
571 }
572
573 return string;
574 }
575 return NULL;
576
577 }
578
579 static pANTLR3_STRING
toStringTT(pANTLR3_TOKEN_STREAM ts,pANTLR3_COMMON_TOKEN start,pANTLR3_COMMON_TOKEN stop)580 toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
581 {
582 if (start != NULL && stop != NULL)
583 {
584 return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));
585 }
586 else
587 {
588 return NULL;
589 }
590 }
591
592 /** Move the input pointer to the next incoming token. The stream
593 * must become active with LT(1) available. consume() simply
594 * moves the input pointer so that LT(1) points at the next
595 * input symbol. Consume at least one token.
596 *
597 * Walk past any token not on the channel the parser is listening to.
598 */
599 static void
consume(pANTLR3_INT_STREAM is)600 consume (pANTLR3_INT_STREAM is)
601 {
602 pANTLR3_COMMON_TOKEN_STREAM cts;
603 pANTLR3_TOKEN_STREAM ts;
604
605 ts = (pANTLR3_TOKEN_STREAM) is->super;
606 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
607
608 if ((ANTLR3_UINT32)cts->p < cts->tokens->count)
609 {
610 cts->p++;
611 cts->p = skipOffTokenChannels(cts, cts->p);
612 }
613 }
614
615
616 /// As per ordinary consume but notifies the debugger about hidden
617 /// tokens and so on.
618 ///
619 static void
dbgConsume(pANTLR3_INT_STREAM is)620 dbgConsume (pANTLR3_INT_STREAM is)
621 {
622 pANTLR3_TOKEN_STREAM ts;
623 ANTLR3_MARKER a;
624 ANTLR3_MARKER b;
625 pANTLR3_COMMON_TOKEN t;
626
627 ts = (pANTLR3_TOKEN_STREAM) is->super;
628
629 if (ts->initialStreamState == ANTLR3_TRUE)
630 {
631 consumeInitialHiddenTokens(is);
632 }
633
634 a = is->index(is); // Where are we right now?
635 t = ts->_LT(ts, 1); // Current token from stream
636
637 consume(is); // Standard consumer
638
639 b = is->index(is); // Where are we after consuming 1 on channel token?
640
641 ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token
642
643 if (b>a+1)
644 {
645 // The standard consume caused the index to advance by more than 1,
646 // which can only happen if it skipped some off-channel tokens.
647 // we need to tell the debugger about those tokens.
648 //
649 ANTLR3_MARKER i;
650
651 for (i = a+1; i<b; i++)
652 {
653 ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i));
654 }
655
656 }
657 }
658
659 /** A simple filter mechanism whereby you can tell this token stream
660 * to force all tokens of type ttype to be on channel. For example,
661 * when interpreting, we cannot execute actions so we need to tell
662 * the stream to force all WS and NEWLINE to be a different, ignored,
663 * channel.
664 */
665 static void
setTokenTypeChannel(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_UINT32 ttype,ANTLR3_UINT32 channel)666 setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)
667 {
668 if (tokenStream->channelOverrides == NULL)
669 {
670 tokenStream->channelOverrides = antlr3ListNew(10);
671 }
672
673 /* We add one to the channel so we can distinguish NULL as being no entry in the
674 * table for a particular token type.
675 */
676 tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL);
677 }
678
679 static void
discardTokenType(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_INT32 ttype)680 discardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype)
681 {
682 if (tokenStream->discardSet == NULL)
683 {
684 tokenStream->discardSet = antlr3ListNew(31);
685 }
686
687 /* We add one to the channel so we can distinguish NULL as being no entry in the
688 * table for a particular token type. We could use bitsets for this I suppose too.
689 */
690 tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL);
691 }
692
693 static void
discardOffChannel(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_BOOLEAN discard)694 discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard)
695 {
696 tokenStream->discardOffChannel = discard;
697 }
698
699 static pANTLR3_VECTOR
getTokens(pANTLR3_COMMON_TOKEN_STREAM tokenStream)700 getTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream)
701 {
702 if (tokenStream->p == -1)
703 {
704 fillBuffer(tokenStream);
705 }
706
707 return tokenStream->tokens;
708 }
709
710 static pANTLR3_LIST
getTokenRange(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_UINT32 start,ANTLR3_UINT32 stop)711 getTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
712 {
713 return tokenStream->getTokensSet(tokenStream, start, stop, NULL);
714 }
715 /** Given a start and stop index, return a List of all tokens in
716 * the token type BitSet. Return null if no tokens were found. This
717 * method looks at both on and off channel tokens.
718 */
719 static pANTLR3_LIST
getTokensSet(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_UINT32 start,ANTLR3_UINT32 stop,pANTLR3_BITSET types)720 getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)
721 {
722 pANTLR3_LIST filteredList;
723 ANTLR3_UINT32 i;
724 ANTLR3_UINT32 n;
725 pANTLR3_COMMON_TOKEN tok;
726
727 if (tokenStream->p == -1)
728 {
729 fillBuffer(tokenStream);
730 }
731 if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream))
732 {
733 stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream);
734 }
735 if (start > stop)
736 {
737 return NULL;
738 }
739
740 /* We have the range set, now we need to iterate through the
741 * installed tokens and create a new list with just the ones we want
742 * in it. We are just moving pointers about really.
743 */
744 filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream));
745
746 for (i = start, n = 0; i<= stop; i++)
747 {
748 tok = tokenStream->tstream->get(tokenStream->tstream, i);
749
750 if ( types == NULL
751 || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE)
752 )
753 {
754 filteredList->put(filteredList, n++, (void *)tok, NULL);
755 }
756 }
757
758 /* Did we get any then?
759 */
760 if (filteredList->size(filteredList) == 0)
761 {
762 filteredList->free(filteredList);
763 filteredList = NULL;
764 }
765
766 return filteredList;
767 }
768
769 static pANTLR3_LIST
getTokensList(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_UINT32 start,ANTLR3_UINT32 stop,pANTLR3_LIST list)770 getTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)
771 {
772 pANTLR3_BITSET bitSet;
773 pANTLR3_LIST newlist;
774
775 bitSet = antlr3BitsetList(list->table);
776
777 newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
778
779 bitSet->free(bitSet);
780
781 return newlist;
782
783 }
784
785 static pANTLR3_LIST
getTokensType(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_UINT32 start,ANTLR3_UINT32 stop,ANTLR3_UINT32 type)786 getTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)
787 {
788 pANTLR3_BITSET bitSet;
789 pANTLR3_LIST newlist;
790
791 bitSet = antlr3BitsetOf(type, -1);
792 newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
793
794 bitSet->free(bitSet);
795
796 return newlist;
797 }
798
799 static ANTLR3_UINT32
_LA(pANTLR3_INT_STREAM is,ANTLR3_INT32 i)800 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
801 {
802 pANTLR3_TOKEN_STREAM ts;
803 pANTLR3_COMMON_TOKEN tok;
804
805 ts = (pANTLR3_TOKEN_STREAM) is->super;
806
807 tok = ts->_LT(ts, i);
808
809 if (tok != NULL)
810 {
811 return tok->getType(tok);
812 }
813 else
814 {
815 return ANTLR3_TOKEN_INVALID;
816 }
817 }
818
819 /// As per _LA() but for debug mode.
820 ///
821 static ANTLR3_UINT32
dbgLA(pANTLR3_INT_STREAM is,ANTLR3_INT32 i)822 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
823 {
824 pANTLR3_TOKEN_STREAM ts;
825
826 ts = (pANTLR3_TOKEN_STREAM) is->super;
827
828 if (ts->initialStreamState == ANTLR3_TRUE)
829 {
830 consumeInitialHiddenTokens(is);
831 }
832 ts->debugger->LT(ts->debugger, i, tokLT(ts, i));
833 return _LA(is, i);
834 }
835
836 static ANTLR3_MARKER
mark(pANTLR3_INT_STREAM is)837 mark (pANTLR3_INT_STREAM is)
838 {
839 is->lastMarker = is->index(is);
840 return is->lastMarker;
841 }
842
843 /// As per mark() but with a call to tell the debugger we are doing this
844 ///
845 static ANTLR3_MARKER
dbgMark(pANTLR3_INT_STREAM is)846 dbgMark (pANTLR3_INT_STREAM is)
847 {
848 pANTLR3_TOKEN_STREAM ts;
849
850 ts = (pANTLR3_TOKEN_STREAM) is->super;
851
852 is->lastMarker = is->index(is);
853 ts->debugger->mark(ts->debugger, is->lastMarker);
854
855 return is->lastMarker;
856 }
857
858 static void
release(pANTLR3_INT_STREAM is,ANTLR3_MARKER mark)859 release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)
860 {
861 return;
862 }
863
864 static ANTLR3_UINT32
size(pANTLR3_INT_STREAM is)865 size (pANTLR3_INT_STREAM is)
866 {
867 pANTLR3_COMMON_TOKEN_STREAM cts;
868 pANTLR3_TOKEN_STREAM ts;
869
870 if (is->cachedSize > 0)
871 {
872 return is->cachedSize;
873 }
874 ts = (pANTLR3_TOKEN_STREAM) is->super;
875 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
876
877 is->cachedSize = cts->tokens->count;
878 return is->cachedSize;
879 }
880
881 static ANTLR3_MARKER
tindex(pANTLR3_INT_STREAM is)882 tindex (pANTLR3_INT_STREAM is)
883 {
884 pANTLR3_COMMON_TOKEN_STREAM cts;
885 pANTLR3_TOKEN_STREAM ts;
886
887 ts = (pANTLR3_TOKEN_STREAM) is->super;
888 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
889
890 return cts->p;
891 }
892
893 static void
dbgRewindLast(pANTLR3_INT_STREAM is)894 dbgRewindLast (pANTLR3_INT_STREAM is)
895 {
896 pANTLR3_TOKEN_STREAM ts;
897
898 ts = (pANTLR3_TOKEN_STREAM) is->super;
899
900 ts->debugger->rewindLast(ts->debugger);
901
902 is->rewind(is, is->lastMarker);
903 }
904 static void
rewindLast(pANTLR3_INT_STREAM is)905 rewindLast (pANTLR3_INT_STREAM is)
906 {
907 is->rewind(is, is->lastMarker);
908 }
909 static void
rewindStream(pANTLR3_INT_STREAM is,ANTLR3_MARKER marker)910 rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
911 {
912 is->seek(is, (ANTLR3_UINT32)(marker));
913 }
914 static void
dbgRewindStream(pANTLR3_INT_STREAM is,ANTLR3_MARKER marker)915 dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
916 {
917 pANTLR3_TOKEN_STREAM ts;
918
919 ts = (pANTLR3_TOKEN_STREAM) is->super;
920
921 ts->debugger->rewind(ts->debugger, marker);
922
923 is->seek(is, (ANTLR3_UINT32)(marker));
924 }
925
926 static void
seek(pANTLR3_INT_STREAM is,ANTLR3_MARKER index)927 seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
928 {
929 pANTLR3_COMMON_TOKEN_STREAM cts;
930 pANTLR3_TOKEN_STREAM ts;
931
932 ts = (pANTLR3_TOKEN_STREAM) is->super;
933 cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
934
935 cts->p = (ANTLR3_UINT32)index;
936 }
937 static void
dbgSeek(pANTLR3_INT_STREAM is,ANTLR3_MARKER index)938 dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
939 {
940 // TODO: Implement seek in debugger when Ter adds it to Java
941 //
942 seek(is, index);
943 }
944 ANTLR3_API void
fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)945 fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)
946 {
947 fillBuffer(tokenStream);
948 }
949 static void
fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream)950 fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {
951 ANTLR3_UINT32 index;
952 pANTLR3_COMMON_TOKEN tok;
953 ANTLR3_BOOLEAN discard;
954 void * channelI;
955
956 /* Start at index 0 of course
957 */
958 index = 0;
959
960 /* Pick out the next token from the token source
961 * Remember we just get a pointer (reference if you like) here
962 * and so if we store it anywhere, we don't set any pointers to auto free it.
963 */
964 tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
965
966 while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)
967 {
968 discard = ANTLR3_FALSE; /* Assume we are not discarding */
969
970 /* I employ a bit of a trick, or perhaps hack here. Rather than
971 * store a pointer to a structure in the override map and discard set
972 * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
973 * we can distinguish "not being there" from "being channel or type 0"
974 */
975
976 if (tokenStream->discardSet != NULL
977 && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL)
978 {
979 discard = ANTLR3_TRUE;
980 }
981 else if ( tokenStream->discardOffChannel == ANTLR3_TRUE
982 && tok->getChannel(tok) != tokenStream->channel
983 )
984 {
985 discard = ANTLR3_TRUE;
986 }
987 else if (tokenStream->channelOverrides != NULL)
988 {
989 /* See if this type is in the override map
990 */
991 channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1);
992
993 if (channelI != NULL)
994 {
995 /* Override found
996 */
997 tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1);
998 }
999 }
1000
1001 /* If not discarding it, add it to the list at the current index
1002 */
1003 if (discard == ANTLR3_FALSE)
1004 {
1005 /* Add it, indicating that we will delete it and the table should not
1006 */
1007 tok->setTokenIndex(tok, index);
1008 tokenStream->p++;
1009 tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL);
1010 index++;
1011 }
1012
1013 tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
1014 }
1015
1016 /* Cache the size so we don't keep doing indirect method calls. We do this as
1017 * early as possible so that anything after this may utilize the cached value.
1018 */
1019 tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;
1020
1021 /* Set the consume pointer to the first token that is on our channel
1022 */
1023 tokenStream->p = 0;
1024 tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);
1025
1026 }
1027
1028 /// Given a starting index, return the index of the first on-channel
1029 /// token.
1030 ///
1031 static ANTLR3_UINT32
skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_INT32 i)1032 skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) {
1033 ANTLR3_INT32 n;
1034 pANTLR3_COMMON_TOKEN tok;
1035
1036 n = tokenStream->tstream->istream->cachedSize;
1037
1038 while (i < n)
1039 {
1040 tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element;
1041
1042 if (tok->channel!= tokenStream->channel)
1043 {
1044 i++;
1045 }
1046 else
1047 {
1048 return i;
1049 }
1050 }
1051 return i;
1052 }
1053
1054 static ANTLR3_UINT32
skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream,ANTLR3_INT32 x)1055 skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x)
1056 {
1057 pANTLR3_COMMON_TOKEN tok;
1058
1059 while (x >= 0)
1060 {
1061 tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element;
1062
1063 if ((tok->channel != tokenStream->channel))
1064 {
1065 x--;
1066 }
1067 else
1068 {
1069 return x;
1070 }
1071 }
1072 return x;
1073 }
1074
1075 /// Return a string that represents the name assoicated with the input source
1076 ///
1077 /// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.
1078 ///
1079 /// /returns
1080 /// /implements ANTLR3_INT_STREAM_struct::getSourceName()
1081 ///
1082 static pANTLR3_STRING
getSourceName(pANTLR3_INT_STREAM is)1083 getSourceName (pANTLR3_INT_STREAM is)
1084 {
1085 // Slightly convoluted as we must trace back to the lexer's input source
1086 // via the token source. The streamName that is here is not initialized
1087 // because this is a token stream, not a file or string stream, which are the
1088 // only things that have a context for a source name.
1089 //
1090 return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;
1091 }
1092