1 /* ANTLRTokenBuffer.cpp
2 *
3 * SOFTWARE RIGHTS
4 *
5 * We reserve no LEGAL rights to the Purdue Compiler Construction Tool
6 * Set (PCCTS) -- PCCTS is in the public domain. An individual or
7 * company may do whatever they wish with source code distributed with
8 * PCCTS or the code generated by PCCTS, including the incorporation of
9 * PCCTS, or its output, into commerical software.
10 *
11 * We encourage users to develop software with PCCTS. However, we do ask
12 * that credit is given to us for developing PCCTS. By "credit",
13 * we mean that if you incorporate our source code into one of your
14 * programs (commercial product, research project, or otherwise) that you
15 * acknowledge this fact somewhere in the documentation, research report,
16 * etc... If you like PCCTS and have developed a nice tool with the
17 * output, please mention that you developed it using PCCTS. In
18 * addition, we ask that this header remain intact in our source code.
19 * As long as these guidelines are kept, we expect to continue enhancing
20 * this system and expect to make other tools available as they are
21 * completed.
22 *
23 * ANTLR 1.33
24 * Terence Parr
25 * Parr Research Corporation
26 * with Purdue University and AHPCRC, University of Minnesota
27 * 1989-2000
28 */
29
30 typedef int ANTLRTokenType; // fool AToken.h into compiling
31
32 class ANTLRParser; /* MR1 */
33
34 #define ANTLR_SUPPORT_CODE
35
36 #include "pcctscfg.h"
37
38 #include ATOKENBUFFER_H
39 #include APARSER_H // MR23
40
41 typedef ANTLRAbstractToken *_ANTLRTokenPtr;
42
43 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW)
44 static unsigned char test[1000];
45 #endif
46
47 #ifdef DBG_REFCOUNTTOKEN
48 int ANTLRRefCountToken::ctor = 0; /* MR23 */
49 int ANTLRRefCountToken::dtor = 0; /* MR23 */
50 #endif
51
52 ANTLRTokenBuffer::
ANTLRTokenBuffer(ANTLRTokenStream * _input,int _k,int _chunk_size_formal)53 ANTLRTokenBuffer(ANTLRTokenStream *_input, int _k, int _chunk_size_formal) /* MR14 */
54 {
55 this->input = _input;
56 this->k = _k;
57 buffer_size = chunk_size = _chunk_size_formal;
58 buffer = (_ANTLRTokenPtr *)
59 calloc(chunk_size+1,sizeof(_ANTLRTokenPtr ));
60 if ( buffer == NULL ) {
61 panic("cannot alloc token buffer");
62 }
63 buffer++; // leave the first elem empty so tp-1 is valid ptr
64
65 tp = &buffer[0];
66 last = tp-1;
67 next = &buffer[0];
68 num_markers = 0;
69 end_of_buffer = &buffer[buffer_size-1];
70 threshold = &buffer[(int)(buffer_size/2)]; // MR23 - Used to be 1.0/2.0 !
71 _deleteTokens = 1; // assume we delete tokens
72 parser=NULL; // MR5 - uninitialized reference
73 }
74
f()75 static void f() {;}
76 ANTLRTokenBuffer::
~ANTLRTokenBuffer()77 ~ANTLRTokenBuffer()
78 {
79 f();
80 // Delete all remaining tokens (from 0..last inclusive)
81 if ( _deleteTokens )
82 {
83 _ANTLRTokenPtr *z;
84 for (z=buffer; z<=last; z++)
85 {
86 (*z)->deref();
87 // z->deref();
88 #ifdef DBG_REFCOUNTTOKEN
89 /* MR23 */ printMessage(stderr, "##########dtor: deleting token '%s' (ref %d)\n",
90 ((ANTLRCommonToken *)*z)->getText(), (*z)->nref());
91 #endif
92 if ( (*z)->nref()==0 )
93 {
94 delete (*z);
95 }
96 }
97 }
98
99 if ( buffer!=NULL ) free((char *)(buffer-1));
100 }
101
102 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW)
103 #include "pccts_stdio.h"
104 PCCTS_NAMESPACE_STD
105 #endif
106
107 _ANTLRTokenPtr ANTLRTokenBuffer::
getToken()108 getToken()
109 {
110 if ( tp <= last ) // is there any buffered lookahead still to be read?
111 {
112 return *tp++; // read buffered lookahead
113 }
114 // out of buffered lookahead, get some more "real"
115 // input from getANTLRToken()
116 if ( num_markers==0 )
117 {
118 if( next > threshold )
119 {
120 #ifdef DBG_TBUF
121 /* MR23 */ printMessage(stderr,"getToken: next > threshold (high water is %d)\n", threshold-buffer);
122 #endif
123 makeRoom();
124 }
125 }
126 else {
127 if ( next > end_of_buffer )
128 {
129 #ifdef DBG_TBUF
130 /* MR23 */ printMessage(stderr,"getToken: next > end_of_buffer (size is %d)\n", buffer_size);
131 #endif
132 extendBuffer();
133 }
134 }
135 *next = getANTLRToken();
136 (*next)->ref(); // say we have a copy of this pointer in buffer
137 last = next;
138 next++;
139 tp = last;
140 return *tp++;
141 }
142
143 void ANTLRTokenBuffer::
rewind(int pos)144 rewind(int pos)
145 {
146 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW)
147 /* MR23 */ printMessage(stderr, "rewind(%d)[nm=%d,from=%d,%d.n=%d]\n", pos, num_markers, tp-buffer,pos,test[pos]);
148 test[pos]--;
149 #endif
150 tp = &buffer[pos];
151 num_markers--;
152 }
153
154 /*
155 * This function is used to specify that the token pointers read
156 * by the ANTLRTokenBuffer should be buffered up (to be reused later).
157 */
158 int ANTLRTokenBuffer::
mark()159 mark()
160 {
161 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW)
162 test[tp-buffer]++;
163 /* MR23 */ printMessage(stderr,"mark(%d)[nm=%d,%d.n=%d]\n",tp-buffer,num_markers+1,tp-buffer,test[tp-buffer]);
164 #endif
165 num_markers++;
166 return tp - buffer;
167 }
168
169 /*
170 * returns the token pointer n positions ahead.
171 * This implies that bufferedToken(1) gets the NEXT symbol of lookahead.
172 * This is used in conjunction with the ANTLRParser lookahead buffer.
173 *
174 * No markers are set or anything. A bunch of input is buffered--that's all.
175 * The tp pointer is left alone as the lookahead has not been advanced
176 * with getToken(). The next call to getToken() will find a token
177 * in the buffer and won't have to call getANTLRToken().
178 *
179 * If this is called before a consume() is done, how_many_more_i_need is
180 * set to 'n'.
181 */
182 _ANTLRTokenPtr ANTLRTokenBuffer::
bufferedToken(int n)183 bufferedToken(int n)
184 {
185 // int how_many_more_i_need = (last-tp < 0) ? n : n-(last-tp)-1;
186 int how_many_more_i_need = (tp > last) ? n : n-(last-tp)-1;
187 // Make sure that at least n tokens are available in the buffer
188 #ifdef DBG_TBUF
189 /* MR23 */ printMessage(stderr, "bufferedToken(%d)\n", n);
190 #endif
191 for (int i=1; i<=how_many_more_i_need; i++)
192 {
193 if ( next > end_of_buffer ) // buffer overflow?
194 {
195 extendBuffer();
196 }
197 *next = getANTLRToken();
198 (*next)->ref(); // say we have a copy of this pointer in buffer
199 last = next;
200 next++;
201 }
202 return tp[n - 1];
203 }
204
205 /* If no markers are set, the none of the input needs to be saved (except
206 * for the lookahead Token pointers). We save only k-1 token pointers as
207 * we are guaranteed to do a getANTLRToken() right after this because otherwise
208 * we wouldn't have needed to extend the buffer.
209 *
210 * If there are markers in the buffer, we need to save things and so
211 * extendBuffer() is called.
212 */
213 void ANTLRTokenBuffer::
makeRoom()214 makeRoom()
215 {
216 #ifdef DBG_TBUF
217 /* MR23 */ printMessage(stderr, "in makeRoom.................\n");
218 /* MR23 */ printMessage(stderr, "num_markers==%d\n", num_markers);
219 #endif
220 /*
221 if ( num_markers == 0 )
222 {
223 */
224 #ifdef DBG_TBUF
225 /* MR23 */ printMessage(stderr, "moving lookahead and resetting next\n");
226
227 _ANTLRTokenPtr *r;
228 /* MR23 */ printMessage(stderr, "tbuf = [");
229 for (r=buffer; r<=last; r++)
230 {
231 if ( *r==NULL ) /* MR23 */ printMessage(stderr, " xxx");
232 else /* MR23 */ printMessage(stderr, " '%s'", ((ANTLRCommonToken *)*r)->getText());
233 }
234 /* MR23 */ printMessage(stderr, " ]\n");
235
236 /* MR23 */ printMessage(stderr,
237 "before: tp=%d, last=%d, next=%d, threshold=%d\n",tp-buffer,last-buffer,next-buffer,threshold-buffer);
238 #endif
239
240 // Delete all tokens from 0..last-(k-1) inclusive
241 if ( _deleteTokens )
242 {
243 _ANTLRTokenPtr *z;
244 for (z=buffer; z<=last-(k-1); z++)
245 {
246 (*z)->deref();
247 // z->deref();
248 #ifdef DBG_REFCOUNTTOKEN
249 /* MR23 */ printMessage(stderr, "##########makeRoom: deleting token '%s' (ref %d)\n",
250 ((ANTLRCommonToken *)*z)->getText(), (*z)->nref());
251 #endif
252 if ( (*z)->nref()==0 )
253 {
254 delete (*z);
255 }
256 }
257 }
258
259 // reset the buffer to initial conditions, but move k-1 symbols
260 // to the beginning of buffer and put new input symbol at k
261 _ANTLRTokenPtr *p = buffer, *q = last-(k-1)+1;
262 // ANTLRAbstractToken **p = buffer, **q = end_of_buffer-(k-1)+1;
263 #ifdef DBG_TBUF
264 /* MR23 */ printMessage(stderr, "lookahead buffer = [");
265 #endif
266 for (int i=1; i<=(k-1); i++)
267 {
268 *p++ = *q++;
269 #ifdef DBG_TBUF
270 /* MR23 */ printMessage(stderr,
271 " '%s'", ((ANTLRCommonToken *)buffer[i-1])->getText());
272 #endif
273 }
274 #ifdef DBG_TBUF
275 /* MR23 */ printMessage(stderr, " ]\n");
276 #endif
277 next = &buffer[k-1];
278 tp = &buffer[k-1]; // tp points to what will be filled in next
279 last = tp-1;
280 #ifdef DBG_TBUF
281 /* MR23 */ printMessage(stderr,
282 "after: tp=%d, last=%d, next=%d\n",
283 tp-buffer, last-buffer, next-buffer);
284 #endif
285 /*
286 }
287 else {
288 extendBuffer();
289 }
290 */
291 }
292
293 /* This function extends 'buffer' by chunk_size and returns with all
294 * pointers at the same relative positions in the buffer (the buffer base
295 * address could have changed in realloc()) except that 'next' comes
296 * back set to where the next token should be stored. All other pointers
297 * are untouched.
298 */
299 void
300 ANTLRTokenBuffer::
extendBuffer()301 extendBuffer()
302 {
303 int save_last = last-buffer, save_tp = tp-buffer, save_next = next-buffer;
304 #ifdef DBG_TBUF
305 /* MR23 */ printMessage(stderr, "extending physical buffer\n");
306 #endif
307 buffer_size += chunk_size;
308 buffer = (_ANTLRTokenPtr *)
309 realloc((char *)(buffer-1),
310 (buffer_size+1)*sizeof(_ANTLRTokenPtr ));
311 if ( buffer == NULL ) {
312 panic("cannot alloc token buffer");
313 }
314 buffer++; // leave the first elem empty so tp-1 is valid ptr
315
316 tp = buffer + save_tp; // put the pointers back to same relative position
317 last = buffer + save_last;
318 next = buffer + save_next;
319 end_of_buffer = &buffer[buffer_size-1];
320 threshold = &buffer[(int)(buffer_size*(1.0/2.0))];
321
322 /*
323 // zero out new token ptrs so we'll know if something to delete in buffer
324 ANTLRAbstractToken **p = end_of_buffer-chunk_size+1;
325 for (; p<=end_of_buffer; p++) *p = NULL;
326 */
327 }
328
329 ANTLRParser * ANTLRTokenBuffer:: // MR1
setParser(ANTLRParser * p)330 setParser(ANTLRParser *p) { // MR1
331 ANTLRParser *old=parser; // MR1
332 parser=p; // MR1
333 input->setParser(p); // MR1
334 return old; // MR1
335 } // MR1
336 // MR1
337 ANTLRParser * ANTLRTokenBuffer:: // MR1
getParser()338 getParser() { // MR1
339 return parser; // MR1
340 } // MR1
341
panic(const char * msg)342 void ANTLRTokenBuffer::panic(const char *msg) // MR23
343 {
344 if (parser) //MR23
345 parser->panic(msg); //MR23
346 else //MR23
347 exit(PCCTS_EXIT_FAILURE);
348 }
349
350 //MR23
printMessage(FILE * pFile,const char * pFormat,...)351 int ANTLRTokenBuffer::printMessage(FILE* pFile, const char* pFormat, ...)
352 {
353 va_list marker;
354 va_start( marker, pFormat );
355
356 int iRet = 0;
357 if (parser)
358 parser->printMessageV(pFile, pFormat, marker);
359 else
360 iRet = vfprintf(pFile, pFormat, marker);
361
362 va_end( marker );
363 return iRet;
364 }
365
366 /* to avoid having to link in another file just for the smart token ptr
367 * stuff, we include it here. Ugh.
368 *
369 * MR23 This causes nothing but problems for IDEs.
370 * Change from .cpp to .h
371 *
372 */
373
374 #include ATOKPTR_IMPL_H
375