1 /* -*- Mode: c; c-basic-offset: 2 -*-
2  *
3  * n3_lexer.l - Raptor Notation 3 lexer - making tokens for grammar generator
4  *
5  * Copyright (C) 2003-2008, David Beckett http://www.dajobe.org/
6  * Copyright (C) 2003-2005, University of Bristol, UK http://www.bristol.ac.uk/
7  *
8  * This package is Free Software and part of Redland http://librdf.org/
9  *
10  * It is licensed under the following three licenses as alternatives:
11  *   1. GNU Lesser General Public License (LGPL) V2.1 or any newer version
12  *   2. GNU General Public License (GPL) V2 or any newer version
13  *   3. Apache License, V2.0 or any newer version
14  *
15  * You may not use this file except in compliance with at least one of
16  * the above three licenses.
17  *
18  * See LICENSE.html or LICENSE.txt at the top of this package for the
19  * complete terms and further detail along with the license texts for
20  * the licenses in COPYING.LIB, COPYING and LICENSE-2.0.txt respectively.
21  *
22  *
23  * This is an incomplete Notation 3 parser.
24  *
25  * To generate the C files from this source, rather than use the
26  * shipped n3_lexer.c/.h needs a patched version of flex 2.5.31 such
27  * as the one available in Debian GNU/Linux.   Details below
28  * near the %option descriptions.
29  *
30  */
31 
32 
33 /* recognise 8-bits */
34 %option 8bit
35 %option warn nodefault
36 
37 /* all symbols prefixed by this */
38 %option prefix="n3_lexer_"
39 
40 /* This is not needed, flex is invoked -on3_lexer.c */
41 /* %option outfile="n3_lexer.c" */
42 
43 /* Emit a C header file for prototypes
44  * Only available in flex 2.5.13 or newer.
45  * It was renamed to header-file in flex 2.5.19
46  */
47 %option header-file="n3_lexer.h"
48 
49 /* Do not emit #include <unistd.h>
50  * Only available in flex 2.5.7 or newer.
51  * Broken in flex 2.5.31 without patches.
52  */
53 %option nounistd
54 
55 /* Never interactive */
56 /*  No isatty() check */
57 %option never-interactive
58 
59 /* Batch scanner */
60 %option batch
61 
62 /* Never use yyunput */
63 %option nounput
64 
65 /* Supply our own alloc/realloc/free functions */
66 %option noyyalloc noyyrealloc noyyfree
67 
68 /* Re-entrant scanner */
69 %option reentrant
70 
71 
72   /* definitions */
73 
74 %{
75 /* NOTE: These headers are NOT included here but are inserted by
76  * fix-flex since otherwise it appears far too late in the generated C
77  */
78 
79 /*
80 #ifdef HAVE_CONFIG_H
81 #include <raptor_config.h>
82 #endif
83 
84 #ifdef WIN32
85 #include <win32_raptor_config.h>
86 #endif
87 */
88 
89 
90 #include <stdio.h>
91 #include <string.h>
92 #include <ctype.h>
93 #include <stdarg.h>
94 #ifdef HAVE_ERRNO_H
95 #include <errno.h>
96 #endif
97 #ifdef HAVE_STDLIB_H
98 #include <stdlib.h>
99 #endif
100 #ifdef HAVE_SETJMP_H
101 #include <setjmp.h>
102 #endif
103 
104 #include <raptor.h>
105 #include <raptor_internal.h>
106 
107 #include <n3_parser.h>
108 
109 #include <n3_common.h>
110 
111 
112 /* Prototypes */
113 static unsigned char *n3_copy_token(unsigned char *text, size_t len);
114 static unsigned char *n3_copy_string_token(raptor_parser* rdf_parser, unsigned char *text, size_t len, int delim);
115 void n3_lexer_syntax_error(void* rdf_parser, const char *msg, ...) RAPTOR_PRINTF_FORMAT(2, 3);
116 
117 #ifdef RAPTOR_DEBUG
118 const char * n3_token_print(raptor_world* world, int token, YYSTYPE *lval);
119 #endif
120 
121 int n3_lexer_lex (YYSTYPE *n3_parser_lval, yyscan_t yyscanner);
122 #define YY_DECL int n3_lexer_lex (YYSTYPE *n3_parser_lval, yyscan_t yyscanner)
123 
124 #ifdef __cplusplus
125 #define INPUT_FN yyinput
126 #else
127 #define INPUT_FN input
128 #endif
129 
130 
131 /* Missing n3_lexer.c/h prototypes */
132 int n3_lexer_get_column(yyscan_t yyscanner);
133 void n3_lexer_set_column(int  column_no , yyscan_t yyscanner);
134 
135 static void n3_lexer_cleanup(yyscan_t yyscanner);
136 
137 #ifdef HAVE_SETJMP
138 static jmp_buf n3_lexer_fatal_error_longjmp_env;
139 
140 /* fatal error handler declaration */
141 #define YY_FATAL_ERROR(msg) do {		\
142     n3_lexer_fatal_error(msg, yyscanner);   \
143     longjmp(n3_lexer_fatal_error_longjmp_env, 1);        \
144 } while(0)
145 #else
146 #define YY_FATAL_ERROR(msg) do {		\
147     n3_lexer_fatal_error(msg, yyscanner);   \
148     abort();                                    \
149 } while(0)
150 #endif
151 
152 static void n3_lexer_fatal_error(yyconst char *msg, yyscan_t yyscanner);
153 
154 /* Fatal error handler that returns EOF instead of abort()/longjmp()
155  * so that parser can clean up properly */
156 #define YY_FATAL_ERROR_EOF(msg) do { \
157     n3_lexer_fatal_error(msg, yyscanner); \
158     yyterminate(); \
159 } while(0)
160 
161 /* Out-of-memory reporting macro */
162 #define N3_LEXER_OOM() YY_FATAL_ERROR_EOF(n3_lexer_oom_text)
163 static const char n3_lexer_oom_text[]="n3_lexer: Out of memory";
164 
165 %}
166 
167 /* from SPARQL */
168 LANGUAGETOKEN [A-Za-z][-A-Z_a-z0-9]*
169 NCCHAR1 [A-Za-z\\\x80-\xff]
170 NCCHAR {NCCHAR1}|"-"|"_"|[0-9]
171 NCNAME_PREFIX {NCCHAR1}(({NCCHAR}|".")*{NCCHAR})?
172 NCNAME ("_"|{NCCHAR1})(({NCCHAR}|".")*{NCCHAR})?
173 QNAME {NCNAME_PREFIX}?":"{NCNAME}?
174 BNAME "_:"{NCNAME}
175 
176 /* similar to SPARQL but no need for <= check here */
177 QUOTEDURI \<[^\>]*\>
178 
179 DECIMAL [0-9]+"."[0-9]*|"."[0-9]+
180 DOUBLE [0-9]+"."[0-9]*{EXPONENT}|"."([0-9])+{EXPONENT}|([0-9])+{EXPONENT}
181 EXPONENT [eE][+-]?[0-9]+
182 
183 
184 %x PREF LITERAL
185 
186 
187 %%
188   /* rules */
189 
190 %{
191   raptor_parser *rdf_parser=(raptor_parser*)yyextra;
192   raptor_n3_parser* n3_parser=(raptor_n3_parser*)rdf_parser->context;
193 
194 #ifdef HAVE_SETJMP
195   if(setjmp(n3_lexer_fatal_error_longjmp_env))
196     return 1;
197 #endif
198 %}
199 
200 
201 \r\n|\r|\n   { n3_parser->lineno++; }
202 
203 [\ \t\v]+   { /* empty */ }
204 
205 "a" { return A; }
206 
207 "."       { return DOT; }
208 ","       { return COMMA; }
209 ";"       { return SEMICOLON; }
210 "["       { return LEFT_SQUARE; }
211 "]"       { return RIGHT_SQUARE; }
212 "@prefix" { BEGIN(PREF); return PREFIX; }
213 "@"       { return AT; }
214 "^^"      { return HAT; }
215 "("       { return LEFT_ROUND; }
216 ")"       { return RIGHT_ROUND; }
217 
218 
219 '([^'\\\n\r]|\\[^\n\r])*'    { n3_parser_lval->string=n3_copy_string_token(rdf_parser, (unsigned char*)yytext+1, yyleng-2, '\"'); /* ' */
220                                if(!n3_parser_lval->string)
221                                  YY_FATAL_ERROR_EOF("n3_copy_string_token failed");
222                                return STRING_LITERAL; }
223 
224 \"([^"\\\n\r]|\\[^\n\r])*\"   { n3_parser_lval->string=n3_copy_string_token(rdf_parser, (unsigned char*)yytext+1, yyleng-2, '"'); /* ' */
225                                 if(!n3_parser_lval->string)
226                                   YY_FATAL_ERROR_EOF("n3_copy_string_token failed");
227                                 return STRING_LITERAL; }
228 
229 \"\"\"				{ BEGIN(LITERAL);
230                                   n3_parser->sb=raptor_new_stringbuffer();
231                                   if(!n3_parser->sb)
232                                     N3_LEXER_OOM();
233                           }
234 
235 <LITERAL>\"\"\"			{
236 		  size_t len;
237 
238 		  BEGIN(INITIAL);
239                   len=raptor_stringbuffer_length(n3_parser->sb);
240                   n3_parser_lval->string=(unsigned char *)RAPTOR_MALLOC(cstring, len+1);
241                   if(!n3_parser_lval->string)
242                     N3_LEXER_OOM();
243                   raptor_stringbuffer_copy_to_string(n3_parser->sb, (unsigned char*)n3_parser_lval->string, len);
244                   n3_parser_lval->string[len]='\0';
245 
246                   raptor_free_stringbuffer(n3_parser->sb);
247                   n3_parser->sb=NULL;
248                   return STRING_LITERAL; }
249 
250 <LITERAL>\"|(\\[^uU]|\\u....|\\U........|[^\"]|\n)*	{
251 		  if (*yytext == EOF) {
252                     BEGIN(INITIAL);
253                     n3_syntax_error(rdf_parser, "End of file in middle of literal");
254                     raptor_free_stringbuffer(n3_parser->sb);
255                     n3_parser->sb=NULL;
256                     return EOF;
257                   }
258 
259                   if(raptor_stringbuffer_append_turtle_string(n3_parser->sb, (unsigned char*)yytext, yyleng, '"', (raptor_simple_message_handler)n3_lexer_syntax_error, rdf_parser)) { /* " */
260                     BEGIN(INITIAL);
261                     raptor_free_stringbuffer(n3_parser->sb);
262                     n3_parser->sb=NULL;
263                     YY_FATAL_ERROR_EOF("raptor_stringbuffer_append_turtle_string failed");
264                   }
265 
266    }
267 
268 {BNAME}	{ n3_parser_lval->string=n3_copy_token((unsigned char*)yytext+2, yyleng-2);
269           if(!n3_parser_lval->string)
270             YY_FATAL_ERROR_EOF("n3_copy_token failed");
271                           return BLANK_LITERAL; }
272 
273 {QNAME}	{ n3_parser_lval->uri=n3_qname_to_uri(rdf_parser, (unsigned char*)yytext, yyleng);
274           if(!n3_parser_lval->uri)
275             YY_FATAL_ERROR_EOF("n3_qname_to_uri failed");
276                           return QNAME_LITERAL; }
277 
278 [-+]?{DECIMAL}	{ n3_parser_lval->string=n3_copy_token((unsigned char*)yytext, yyleng);
279                   if(!n3_parser_lval->string)
280                     YY_FATAL_ERROR_EOF("n3_copy_token failed");
281                         return DECIMAL_LITERAL;
282 }
283 
284 [-+]?{DOUBLE} {
285                         double d;
286                         int n;
287 
288                         n=sscanf((const char*)yytext, "%lf", &d);
289                         if(n != 1) {
290                           n3_syntax_error(rdf_parser, "N3 syntax error - Illegal floating point constant %s", yytext);
291                           yyterminate();
292                         }
293                         n3_parser_lval->floating=d;
294                         return FLOATING_LITERAL;
295 }
296 
297 [-+]?[0-9]+        { n3_parser_lval->integer=atoi(yytext);
298                           return INTEGER_LITERAL; }
299 
300 <PREF>[\ \t\v]+ { /* eat up leading whitespace */ }
301 <PREF>{NCNAME_PREFIX}":"	{ n3_parser_lval->string=n3_copy_token((unsigned char*)yytext, yyleng);
302                             if(!n3_parser_lval->string)
303                               YY_FATAL_ERROR_EOF("n3_copy_token failed");
304                           BEGIN(INITIAL);
305                           return IDENTIFIER; }
306 <PREF>":"	{ BEGIN(INITIAL);
307 		  n3_parser_lval->string=n3_copy_token((unsigned char*)yytext, 0);
308       if(!n3_parser_lval->string)
309         YY_FATAL_ERROR_EOF("n3_copy_token failed");
310                   return IDENTIFIER; }
311 
312 <PREF>(.|\n)	{ BEGIN(INITIAL);
313 		  if (*yytext == EOF)
314                     return EOF;
315 
316                   n3_syntax_error(rdf_parser, "syntax error at '%c' - @prefix name must end in :", *yytext);
317                   yyterminate();  }
318 
319 
320 {QUOTEDURI}   { if(yyleng == 2)
321                 n3_parser_lval->uri=raptor_uri_copy_v2(rdf_parser->world, rdf_parser->base_uri);
322               else {
323                 yytext[yyleng-1]='\0';
324                 n3_parser_lval->uri=raptor_new_uri_relative_to_base_v2(rdf_parser->world, rdf_parser->base_uri, (const unsigned char*)yytext+1);
325                 if(!n3_parser_lval->uri)
326                   YY_FATAL_ERROR_EOF("raptor_new_uri_relative_to_base failed");
327               }
328               return URI_LITERAL; }
329 
330 {LANGUAGETOKEN}	{ n3_parser_lval->string=n3_copy_token((unsigned char*)yytext, yyleng);
331                   if(!n3_parser_lval->string)
332                     YY_FATAL_ERROR_EOF("n3_copy_token failed");
333                           return IDENTIFIER; }
334 
335 \#[^\r\n]*(\r\n|\r|\n)	{ /* # comment */
336 		n3_parser->lineno++;
337                 }
338 
339 \#[^\r\n]*	{ /* # comment on the last line with no terminating newline */
340                 }
341 
342 .         	{ if (*yytext == EOF)
343                     return EOF;
344 
345                   n3_syntax_error(rdf_parser, "syntax error at '%c'", *yytext);
346                   yyterminate();
347 		}
348 
349 %%
350   /* user code */
351 
352 int
353 yywrap (yyscan_t yyscanner) {
354   return 1;
355 }
356 
357 
358 static unsigned char *
359 n3_copy_token(unsigned char *text, size_t len)
360 {
361   unsigned char *s;
362   if(!len)
363     len=strlen((const char*)text);
364   s=(unsigned char *)RAPTOR_MALLOC(cstring, len+1);
365   if(s) {
366     strncpy((char*)s, (const char*)text, len);
367     s[len] = '\0';
368   }
369   return s;
370 }
371 
372 
373 static unsigned char *
374 n3_copy_string_token(raptor_parser* rdf_parser,
375                      unsigned char *string, size_t len, int delim)
376 {
377   raptor_stringbuffer* sb=NULL;
378   int rc;
379 
380   if(len) {
381     sb=raptor_new_stringbuffer();
382     if(!sb)
383       return NULL;
384 
385     rc=raptor_stringbuffer_append_turtle_string(sb, string, len, delim,
386                                                 (raptor_simple_message_handler)n3_lexer_syntax_error,
387                                                 rdf_parser);
388     if(rc) {
389       raptor_free_stringbuffer(sb);
390       return NULL;
391     }
392 
393     len=raptor_stringbuffer_length(sb);
394   }
395 
396   string=(unsigned char *)RAPTOR_MALLOC(cstring, len+1);
397   if(string) {
398     if(sb) {
399       raptor_stringbuffer_copy_to_string(sb, string, len+1);
400     }
401     string[len]='\0';
402   }
403 
404   if(sb)
405     raptor_free_stringbuffer(sb);
406 
407   return string;
408 }
409 
410 
411 
412 void
413 n3_lexer_syntax_error(void* ctx, const char *message, ...)
414 {
415   raptor_parser* rdf_parser=(raptor_parser *)ctx;
416   raptor_n3_parser* n3_parser=(raptor_n3_parser*)rdf_parser->context;
417   va_list arguments;
418 
419   rdf_parser->locator.line=n3_parser->lineno;
420 #ifdef RAPTOR_N3_USE_ERROR_COLUMNS
421   rdf_parser->locator.column=n3_lexer_get_column(yyscanner);
422 #endif
423 
424   va_start(arguments, message);
425   raptor_parser_error_varargs(((raptor_parser*)rdf_parser), message, arguments);
426 
427   va_end(arguments);
428 }
429 
430 
431 /*
432  * n3_lexer_fatal_error:
433  * @msg:
434  * @yyscanner:
435  *
436  * INTERNAL - replacement for the generated error handler.
437  */
438 static void n3_lexer_fatal_error(yyconst char *msg, yyscan_t yyscanner)
439 {
440   raptor_parser *rdf_parser=NULL;
441 
442   if(yyscanner)
443     rdf_parser=(raptor_parser *)n3_lexer_get_extra(yyscanner);
444 
445   if(rdf_parser)
446     /* avoid "format not a string literal and no format arguments" warning with %s */
447     raptor_parser_fatal_error(rdf_parser, "%s", msg);
448   else {
449     fputs(msg, stderr);
450     fputc('\n', stderr);
451   }
452 }
453 
454 
455 /* Define LEXER_ALLOC_TRACKING to enable allocated memory tracking
456  * - fixes lexer memory leak when ensure_buffer_stack fails
457  */
458 
459 #ifdef LEXER_ALLOC_TRACKING
460 typedef struct {
461   /* Number of void* slots allocated */
462   int lexer_allocs_size;
463   /* Allocted void* slots follow in memory after this header */
464 } lexer_alloc_tracker_header;
465 
466 /* Initial alloc tracker slot array size - 2 seems to be enough for almost all cases */
467 static const int initial_lexer_allocs_size=2;
468 #endif
469 
470 /*
471  * n3_lexer_cleanup:
472  * @yyscanner:
473  *
474  * INTERNAL - Clean up unfreed lexer allocs if LEXER_ALLOC_TRACKING is enabled.
475  */
476 static void n3_lexer_cleanup(yyscan_t yyscanner)
477 {
478 #ifdef LEXER_ALLOC_TRACKING
479   raptor_parser *rdf_parser;
480   lexer_alloc_tracker_header *tracker;
481   void **lexer_allocs;
482   int i;
483 
484   if(!yyscanner)
485     return;
486 
487   rdf_parser=(raptor_parser *)n3_lexer_get_extra(yyscanner);
488   if(!rdf_parser)
489     return;
490 
491   tracker=(lexer_alloc_tracker_header *)rdf_parser->lexer_user_data;
492   if(!tracker)
493     return;
494   lexer_allocs=(void**)&tracker[1];
495 
496   for(i=0; i<tracker->lexer_allocs_size; ++i) {
497     if(lexer_allocs[i])
498       free(lexer_allocs[i]);
499     lexer_allocs[i]=NULL;
500   }
501   free(rdf_parser->lexer_user_data);
502   rdf_parser->lexer_user_data=NULL;
503 #endif
504 }
505 
506 
507 /*
508  * n3_lexer_alloc:
509  * @size
510  * @yyscanner
511  *
512  * INTERNAL - alloc replacement.
513  * Tracks allocated cells if LEXER_ALLOC_TRACKING is enabled.
514  */
515 void *n3_lexer_alloc(yy_size_t size, yyscan_t yyscanner)
516 {
517 #ifdef LEXER_ALLOC_TRACKING
518   raptor_parser *rdf_parser;
519   lexer_alloc_tracker_header *tracker;
520   void **lexer_allocs;
521   int i;
522   void *ptr;
523 
524   /* yyscanner not initialized -> probably initializing yyscanner itself
525    * -> just malloc without tracking
526    */
527   if(!yyscanner)
528     return malloc(size);
529 
530   rdf_parser=(raptor_parser *)n3_lexer_get_extra(yyscanner);
531   if(!rdf_parser)
532     YY_FATAL_ERROR("lexer_alloc: yyscanner extra not initialized");
533 
534   /* try to allocate tracker if it does not exist */
535   tracker=(lexer_alloc_tracker_header *)rdf_parser->lexer_user_data;
536   if(!tracker) {
537     /* allocate tracker header + array of void* slots */
538     tracker=(lexer_alloc_tracker_header*)calloc(1, sizeof(lexer_alloc_tracker_header)+initial_lexer_allocs_size*sizeof(void*));
539     if(!tracker)
540       YY_FATAL_ERROR("lexer_alloc: cannot allocate tracker");
541     tracker->lexer_allocs_size=initial_lexer_allocs_size;
542     rdf_parser->lexer_user_data=(void *)tracker;
543   }
544   lexer_allocs=(void**)&tracker[1];
545 
546   /* allocate memory */
547   ptr=malloc(size);
548 
549   /* find a free slot for ptr */
550   for(i=0; i<tracker->lexer_allocs_size; ++i) {
551     if(!lexer_allocs[i]) {
552       lexer_allocs[i]=ptr;
553       break;
554     }
555   }
556 
557   /* no free slots -> grow tracker slot array */
558   if(i>=tracker->lexer_allocs_size) {
559     int j;
560     void **dest;
561     tracker=(lexer_alloc_tracker_header*)calloc(1, sizeof(lexer_alloc_tracker_header)+i*2*sizeof(void*));
562     if(!tracker) {
563       if(ptr)
564         free(ptr);
565       YY_FATAL_ERROR("lexer_alloc: cannot grow tracker");
566     }
567     tracker->lexer_allocs_size=i*2;
568 
569     /* copy data from old tracker */
570     dest=(void**)&tracker[1];
571     for(j=0; j<i; ++j) {
572       dest[j]=lexer_allocs[j];
573     }
574 
575     /* set new item to first free slot */
576     dest[j]=ptr;
577 
578     /* free old tracker and replace with new one */
579     free(rdf_parser->lexer_user_data);
580     rdf_parser->lexer_user_data=tracker;
581   }
582 
583   return ptr;
584 #else
585   return malloc(size);
586 #endif
587 }
588 
589 
590 /*
591  * n3_lexer_realloc:
592  *
593  * INTERNAL - realloc replacement
594  * Tracks allocated cells if LEXER_ALLOC_TRACKING is enabled.
595  */
596 void *n3_lexer_realloc(void *ptr, yy_size_t size, yyscan_t yyscanner)
597 {
598 #ifdef LEXER_ALLOC_TRACKING
599   raptor_parser *rdf_parser;
600   lexer_alloc_tracker_header *tracker;
601   void **lexer_allocs;
602   int i;
603   void *newptr;
604 
605   if(!yyscanner)
606     YY_FATAL_ERROR("lexer_realloc: yyscanner not initialized");
607 
608   rdf_parser=(raptor_parser *)n3_lexer_get_extra(yyscanner);
609   if(!rdf_parser)
610     YY_FATAL_ERROR("lexer_realloc: yyscanner extra not initialized");
611 
612   tracker=(lexer_alloc_tracker_header *)rdf_parser->lexer_user_data;
613   if(!tracker)
614     YY_FATAL_ERROR("lexer_realloc: no alloc tracker");
615   lexer_allocs=(void**)&tracker[1];
616 
617   /* find the old slot for ptr */
618   for(i=0; i<tracker->lexer_allocs_size; ++i) {
619     if(lexer_allocs[i]==ptr)
620       break;
621   }
622 
623   /* no old slot -> error */
624   if(i>=tracker->lexer_allocs_size)
625     YY_FATAL_ERROR("lexer_realloc: cell not in tracker");
626 
627   /* realloc */
628   newptr=realloc((char*)ptr, size);
629 
630   /* replace entry in tracker */
631   lexer_allocs[i]=newptr;
632 
633   return newptr;
634 #else
635   return realloc((char*)ptr, size);
636 #endif
637 }
638 
639 
640 /*
641  * n3_lexer_free:
642  *
643  * INTERNAL - free replacement.
644  * Checks for NULL pointer to be freed unlike the default lexer free function.
645  * Tracks allocated cells if LEXER_ALLOC_TRACKING is enabled.
646  */
647 void n3_lexer_free(void *ptr, yyscan_t yyscanner)
648 {
649 #ifdef LEXER_ALLOC_TRACKING
650   raptor_parser *rdf_parser;
651   lexer_alloc_tracker_header *tracker;
652   void **lexer_allocs;
653   int i;
654 
655   /* do not free NULL */
656   if(!ptr)
657     return;
658 
659   /* free ptr even if we would encounter an error */
660   free(ptr);
661 
662   /* yyscanner is allocated with n3_lexer_alloc() but it's never stored in the tracker
663    * - we need yyscanner to access the tracker */
664   if(!yyscanner || ptr==yyscanner)
665     return;
666 
667   rdf_parser=(raptor_parser *)n3_lexer_get_extra(yyscanner);
668   if(!rdf_parser)
669     return;
670 
671   tracker=(lexer_alloc_tracker_header *)rdf_parser->lexer_user_data;
672   if(!tracker)
673     return;
674   lexer_allocs=(void**)&tracker[1];
675 
676   /* find the slot for ptr */
677   for(i=0; i<tracker->lexer_allocs_size; ++i) {
678     if(lexer_allocs[i]==ptr)
679       break;
680   }
681 
682   /* no slot -> error */
683   if(i>=tracker->lexer_allocs_size)
684     YY_FATAL_ERROR("lexer_free: cell not in tracker");
685 
686   /* remove entry from tracker */
687   lexer_allocs[i]=NULL;
688 #else
689   if(ptr)
690     free(ptr);
691 #endif
692 }
693 
694 
695 #ifdef RAPTOR_DEBUG
696 
697 const char *
698 n3_token_print(raptor_world* world, int token, YYSTYPE *lval)
699 {
700   static char buffer[2048];
701 
702   if(!token)
703     return "<<EOF>>";
704 
705   switch(token) {
706     case PREFIX:
707       return "PREFIX";
708 
709     case A:
710       return "A";
711 
712     case DOT:
713       return "DOT";
714 
715     case COMMA:
716       return "COMMA";
717 
718     case SEMICOLON:
719       return "SEMICOLON";
720 
721     case LEFT_SQUARE:
722       return "LEFT_SQUARE";
723 
724     case RIGHT_SQUARE:
725       return "RIGHT_SQUARE";
726 
727     case AT:
728       return "AT";
729 
730     case HAT:
731       return "HAT";
732 
733     case STRING_LITERAL:
734       sprintf(buffer, "STRING_LITERAL(%s)", lval->string);
735       return buffer;
736 
737     case URI_LITERAL:
738       sprintf(buffer, "URI_LITERAL(%s)",
739               (lval->uri ? (char*)raptor_uri_as_string_v2(world, lval->uri) : ""));
740       return buffer;
741 
742     case BLANK_LITERAL:
743       sprintf(buffer, "BLANK_LITERAL(%s)", lval->string);
744       return buffer;
745 
746     case QNAME_LITERAL:
747       sprintf(buffer, "QNAME_LITERAL(%s)",
748               (lval->uri ? (char*)raptor_uri_as_string_v2(world, lval->uri) : ""));
749       return buffer;
750 
751     case INTEGER_LITERAL:
752       sprintf(buffer, "INTEGER_LITERAL(%d)", lval->integer);
753       return buffer;
754 
755     case FLOATING_LITERAL:
756       sprintf(buffer, "FLOATING_LITERAL(%1g)", lval->floating);
757       return buffer;
758 
759     case IDENTIFIER:
760       sprintf(buffer, "IDENTIFIER(%s)",
761               (lval->string ? (char*)lval->string : ""));
762       return buffer;
763 
764     case DECIMAL_LITERAL:
765       sprintf(buffer, "DECIMAL_LITERAL(%s)", lval->string);
766       return buffer;
767 
768     case ERROR_TOKEN:
769       return "ERROR";
770 
771    default:
772      RAPTOR_DEBUG2("UNKNOWN token %d - add a new case\n", token);
773      return "(UNKNOWN)";
774   }
775 }
776 #endif
777 
778 
779 
780 #ifdef STANDALONE
781 
782 static void
783 n3_token_free(int token, YYSTYPE *lval)
784 {
785   if(!token)
786     return;
787 
788   switch(token) {
789     case STRING_LITERAL:
790     case BLANK_LITERAL:
791     case IDENTIFIER:
792       if(lval->string)
793         RAPTOR_FREE(cstring, lval->string);
794       break;
795 
796     case URI_LITERAL:
797     case QNAME_LITERAL:
798       if(lval->uri)
799         raptor_free_uri(lval->uri);
800       break;
801     default:
802       break;
803   }
804 }
805 
806 
807 int
808 main(int argc, char *argv[])
809 {
810   raptor_parser rdf_parser;
811   raptor_n3_parser n3_parser;
812   yyscan_t scanner;
813   int token=EOF;
814   FILE *fh;
815   YYSTYPE lval;
816   const unsigned char *uri_string;
817   char *filename=NULL;
818 
819   raptor_init();
820 
821   if(argc > 1) {
822     filename=argv[1];
823     fh=fopen(filename, "r");
824     if(!fh) {
825       fprintf(stderr, "%s: Cannot open file %s - %s\n", argv[0], filename,
826               strerror(errno));
827       exit(1);
828     }
829   } else {
830     filename="<stdin>";
831     fh=stdin;
832   }
833 
834   memset(&rdf_parser, 0, sizeof(raptor_parser));
835   memset(&n3_parser, 0, sizeof(raptor_n3_parser));
836 
837   yylex_init(&n3_parser.scanner);
838   scanner=n3_parser.scanner;
839   n3_lexer_set_in(fh, scanner);
840   n3_lexer_set_extra(&rdf_parser, scanner);
841 
842   /* Initialise enough of the parser and locator to get error messages */
843   rdf_parser.context=&n3_parser;
844   n3_parser.lineno=1;
845   rdf_parser.locator.file=filename;
846   rdf_parser.locator.column= -1;
847 
848   uri_string=raptor_uri_filename_to_uri_string(filename);
849   rdf_parser.base_uri=raptor_new_uri(uri_string);
850   RAPTOR_FREE(cstring, (void*)uri_string);
851 
852   while(1) {
853     memset(&lval, 0, sizeof(YYSTYPE));
854     if(n3_lexer_get_text(scanner) != NULL)
855       printf("yyinput '%s'\n", n3_lexer_get_text(scanner));
856     token=yylex(&lval, scanner);
857 #ifdef RAPTOR_DEBUG
858     printf("token %s\n", n3_token_print(raptor_world_instance(), token, &lval)); /* FIXME */
859 #else
860     printf("token %d\n", token);
861 #endif
862     n3_token_free(token, &lval);
863     if(!token || token == EOF || token == ERROR_TOKEN)
864       break;
865   }
866 
867   yylex_destroy(scanner);
868 
869   raptor_free_uri(rdf_parser.base_uri);
870 
871   raptor_finish();
872 
873 
874   if(token == ERROR_TOKEN)
875     return 1;
876 
877   return 0;
878 }
879 #endif
880