1 /* -*- Mode: c; c-basic-offset: 2 -*- 2 * 3 * n3_lexer.l - Raptor Notation 3 lexer - making tokens for grammar generator 4 * 5 * Copyright (C) 2003-2008, David Beckett http://www.dajobe.org/ 6 * Copyright (C) 2003-2005, University of Bristol, UK http://www.bristol.ac.uk/ 7 * 8 * This package is Free Software and part of Redland http://librdf.org/ 9 * 10 * It is licensed under the following three licenses as alternatives: 11 * 1. GNU Lesser General Public License (LGPL) V2.1 or any newer version 12 * 2. GNU General Public License (GPL) V2 or any newer version 13 * 3. Apache License, V2.0 or any newer version 14 * 15 * You may not use this file except in compliance with at least one of 16 * the above three licenses. 17 * 18 * See LICENSE.html or LICENSE.txt at the top of this package for the 19 * complete terms and further detail along with the license texts for 20 * the licenses in COPYING.LIB, COPYING and LICENSE-2.0.txt respectively. 21 * 22 * 23 * This is an incomplete Notation 3 parser. 24 * 25 * To generate the C files from this source, rather than use the 26 * shipped n3_lexer.c/.h needs a patched version of flex 2.5.31 such 27 * as the one available in Debian GNU/Linux. Details below 28 * near the %option descriptions. 29 * 30 */ 31 32 33 /* recognise 8-bits */ 34 %option 8bit 35 %option warn nodefault 36 37 /* all symbols prefixed by this */ 38 %option prefix="n3_lexer_" 39 40 /* This is not needed, flex is invoked -on3_lexer.c */ 41 /* %option outfile="n3_lexer.c" */ 42 43 /* Emit a C header file for prototypes 44 * Only available in flex 2.5.13 or newer. 45 * It was renamed to header-file in flex 2.5.19 46 */ 47 %option header-file="n3_lexer.h" 48 49 /* Do not emit #include <unistd.h> 50 * Only available in flex 2.5.7 or newer. 51 * Broken in flex 2.5.31 without patches. 52 */ 53 %option nounistd 54 55 /* Never interactive */ 56 /* No isatty() check */ 57 %option never-interactive 58 59 /* Batch scanner */ 60 %option batch 61 62 /* Never use yyunput */ 63 %option nounput 64 65 /* Supply our own alloc/realloc/free functions */ 66 %option noyyalloc noyyrealloc noyyfree 67 68 /* Re-entrant scanner */ 69 %option reentrant 70 71 72 /* definitions */ 73 74 %{ 75 /* NOTE: These headers are NOT included here but are inserted by 76 * fix-flex since otherwise it appears far too late in the generated C 77 */ 78 79 /* 80 #ifdef HAVE_CONFIG_H 81 #include <raptor_config.h> 82 #endif 83 84 #ifdef WIN32 85 #include <win32_raptor_config.h> 86 #endif 87 */ 88 89 90 #include <stdio.h> 91 #include <string.h> 92 #include <ctype.h> 93 #include <stdarg.h> 94 #ifdef HAVE_ERRNO_H 95 #include <errno.h> 96 #endif 97 #ifdef HAVE_STDLIB_H 98 #include <stdlib.h> 99 #endif 100 #ifdef HAVE_SETJMP_H 101 #include <setjmp.h> 102 #endif 103 104 #include <raptor.h> 105 #include <raptor_internal.h> 106 107 #include <n3_parser.h> 108 109 #include <n3_common.h> 110 111 112 /* Prototypes */ 113 static unsigned char *n3_copy_token(unsigned char *text, size_t len); 114 static unsigned char *n3_copy_string_token(raptor_parser* rdf_parser, unsigned char *text, size_t len, int delim); 115 void n3_lexer_syntax_error(void* rdf_parser, const char *msg, ...) RAPTOR_PRINTF_FORMAT(2, 3); 116 117 #ifdef RAPTOR_DEBUG 118 const char * n3_token_print(raptor_world* world, int token, YYSTYPE *lval); 119 #endif 120 121 int n3_lexer_lex (YYSTYPE *n3_parser_lval, yyscan_t yyscanner); 122 #define YY_DECL int n3_lexer_lex (YYSTYPE *n3_parser_lval, yyscan_t yyscanner) 123 124 #ifdef __cplusplus 125 #define INPUT_FN yyinput 126 #else 127 #define INPUT_FN input 128 #endif 129 130 131 /* Missing n3_lexer.c/h prototypes */ 132 int n3_lexer_get_column(yyscan_t yyscanner); 133 void n3_lexer_set_column(int column_no , yyscan_t yyscanner); 134 135 static void n3_lexer_cleanup(yyscan_t yyscanner); 136 137 #ifdef HAVE_SETJMP 138 static jmp_buf n3_lexer_fatal_error_longjmp_env; 139 140 /* fatal error handler declaration */ 141 #define YY_FATAL_ERROR(msg) do { \ 142 n3_lexer_fatal_error(msg, yyscanner); \ 143 longjmp(n3_lexer_fatal_error_longjmp_env, 1); \ 144 } while(0) 145 #else 146 #define YY_FATAL_ERROR(msg) do { \ 147 n3_lexer_fatal_error(msg, yyscanner); \ 148 abort(); \ 149 } while(0) 150 #endif 151 152 static void n3_lexer_fatal_error(yyconst char *msg, yyscan_t yyscanner); 153 154 /* Fatal error handler that returns EOF instead of abort()/longjmp() 155 * so that parser can clean up properly */ 156 #define YY_FATAL_ERROR_EOF(msg) do { \ 157 n3_lexer_fatal_error(msg, yyscanner); \ 158 yyterminate(); \ 159 } while(0) 160 161 /* Out-of-memory reporting macro */ 162 #define N3_LEXER_OOM() YY_FATAL_ERROR_EOF(n3_lexer_oom_text) 163 static const char n3_lexer_oom_text[]="n3_lexer: Out of memory"; 164 165 %} 166 167 /* from SPARQL */ 168 LANGUAGETOKEN [A-Za-z][-A-Z_a-z0-9]* 169 NCCHAR1 [A-Za-z\\\x80-\xff] 170 NCCHAR {NCCHAR1}|"-"|"_"|[0-9] 171 NCNAME_PREFIX {NCCHAR1}(({NCCHAR}|".")*{NCCHAR})? 172 NCNAME ("_"|{NCCHAR1})(({NCCHAR}|".")*{NCCHAR})? 173 QNAME {NCNAME_PREFIX}?":"{NCNAME}? 174 BNAME "_:"{NCNAME} 175 176 /* similar to SPARQL but no need for <= check here */ 177 QUOTEDURI \<[^\>]*\> 178 179 DECIMAL [0-9]+"."[0-9]*|"."[0-9]+ 180 DOUBLE [0-9]+"."[0-9]*{EXPONENT}|"."([0-9])+{EXPONENT}|([0-9])+{EXPONENT} 181 EXPONENT [eE][+-]?[0-9]+ 182 183 184 %x PREF LITERAL 185 186 187 %% 188 /* rules */ 189 190 %{ 191 raptor_parser *rdf_parser=(raptor_parser*)yyextra; 192 raptor_n3_parser* n3_parser=(raptor_n3_parser*)rdf_parser->context; 193 194 #ifdef HAVE_SETJMP 195 if(setjmp(n3_lexer_fatal_error_longjmp_env)) 196 return 1; 197 #endif 198 %} 199 200 201 \r\n|\r|\n { n3_parser->lineno++; } 202 203 [\ \t\v]+ { /* empty */ } 204 205 "a" { return A; } 206 207 "." { return DOT; } 208 "," { return COMMA; } 209 ";" { return SEMICOLON; } 210 "[" { return LEFT_SQUARE; } 211 "]" { return RIGHT_SQUARE; } 212 "@prefix" { BEGIN(PREF); return PREFIX; } 213 "@" { return AT; } 214 "^^" { return HAT; } 215 "(" { return LEFT_ROUND; } 216 ")" { return RIGHT_ROUND; } 217 218 219 '([^'\\\n\r]|\\[^\n\r])*' { n3_parser_lval->string=n3_copy_string_token(rdf_parser, (unsigned char*)yytext+1, yyleng-2, '\"'); /* ' */ 220 if(!n3_parser_lval->string) 221 YY_FATAL_ERROR_EOF("n3_copy_string_token failed"); 222 return STRING_LITERAL; } 223 224 \"([^"\\\n\r]|\\[^\n\r])*\" { n3_parser_lval->string=n3_copy_string_token(rdf_parser, (unsigned char*)yytext+1, yyleng-2, '"'); /* ' */ 225 if(!n3_parser_lval->string) 226 YY_FATAL_ERROR_EOF("n3_copy_string_token failed"); 227 return STRING_LITERAL; } 228 229 \"\"\" { BEGIN(LITERAL); 230 n3_parser->sb=raptor_new_stringbuffer(); 231 if(!n3_parser->sb) 232 N3_LEXER_OOM(); 233 } 234 235 <LITERAL>\"\"\" { 236 size_t len; 237 238 BEGIN(INITIAL); 239 len=raptor_stringbuffer_length(n3_parser->sb); 240 n3_parser_lval->string=(unsigned char *)RAPTOR_MALLOC(cstring, len+1); 241 if(!n3_parser_lval->string) 242 N3_LEXER_OOM(); 243 raptor_stringbuffer_copy_to_string(n3_parser->sb, (unsigned char*)n3_parser_lval->string, len); 244 n3_parser_lval->string[len]='\0'; 245 246 raptor_free_stringbuffer(n3_parser->sb); 247 n3_parser->sb=NULL; 248 return STRING_LITERAL; } 249 250 <LITERAL>\"|(\\[^uU]|\\u....|\\U........|[^\"]|\n)* { 251 if (*yytext == EOF) { 252 BEGIN(INITIAL); 253 n3_syntax_error(rdf_parser, "End of file in middle of literal"); 254 raptor_free_stringbuffer(n3_parser->sb); 255 n3_parser->sb=NULL; 256 return EOF; 257 } 258 259 if(raptor_stringbuffer_append_turtle_string(n3_parser->sb, (unsigned char*)yytext, yyleng, '"', (raptor_simple_message_handler)n3_lexer_syntax_error, rdf_parser)) { /* " */ 260 BEGIN(INITIAL); 261 raptor_free_stringbuffer(n3_parser->sb); 262 n3_parser->sb=NULL; 263 YY_FATAL_ERROR_EOF("raptor_stringbuffer_append_turtle_string failed"); 264 } 265 266 } 267 268 {BNAME} { n3_parser_lval->string=n3_copy_token((unsigned char*)yytext+2, yyleng-2); 269 if(!n3_parser_lval->string) 270 YY_FATAL_ERROR_EOF("n3_copy_token failed"); 271 return BLANK_LITERAL; } 272 273 {QNAME} { n3_parser_lval->uri=n3_qname_to_uri(rdf_parser, (unsigned char*)yytext, yyleng); 274 if(!n3_parser_lval->uri) 275 YY_FATAL_ERROR_EOF("n3_qname_to_uri failed"); 276 return QNAME_LITERAL; } 277 278 [-+]?{DECIMAL} { n3_parser_lval->string=n3_copy_token((unsigned char*)yytext, yyleng); 279 if(!n3_parser_lval->string) 280 YY_FATAL_ERROR_EOF("n3_copy_token failed"); 281 return DECIMAL_LITERAL; 282 } 283 284 [-+]?{DOUBLE} { 285 double d; 286 int n; 287 288 n=sscanf((const char*)yytext, "%lf", &d); 289 if(n != 1) { 290 n3_syntax_error(rdf_parser, "N3 syntax error - Illegal floating point constant %s", yytext); 291 yyterminate(); 292 } 293 n3_parser_lval->floating=d; 294 return FLOATING_LITERAL; 295 } 296 297 [-+]?[0-9]+ { n3_parser_lval->integer=atoi(yytext); 298 return INTEGER_LITERAL; } 299 300 <PREF>[\ \t\v]+ { /* eat up leading whitespace */ } 301 <PREF>{NCNAME_PREFIX}":" { n3_parser_lval->string=n3_copy_token((unsigned char*)yytext, yyleng); 302 if(!n3_parser_lval->string) 303 YY_FATAL_ERROR_EOF("n3_copy_token failed"); 304 BEGIN(INITIAL); 305 return IDENTIFIER; } 306 <PREF>":" { BEGIN(INITIAL); 307 n3_parser_lval->string=n3_copy_token((unsigned char*)yytext, 0); 308 if(!n3_parser_lval->string) 309 YY_FATAL_ERROR_EOF("n3_copy_token failed"); 310 return IDENTIFIER; } 311 312 <PREF>(.|\n) { BEGIN(INITIAL); 313 if (*yytext == EOF) 314 return EOF; 315 316 n3_syntax_error(rdf_parser, "syntax error at '%c' - @prefix name must end in :", *yytext); 317 yyterminate(); } 318 319 320 {QUOTEDURI} { if(yyleng == 2) 321 n3_parser_lval->uri=raptor_uri_copy_v2(rdf_parser->world, rdf_parser->base_uri); 322 else { 323 yytext[yyleng-1]='\0'; 324 n3_parser_lval->uri=raptor_new_uri_relative_to_base_v2(rdf_parser->world, rdf_parser->base_uri, (const unsigned char*)yytext+1); 325 if(!n3_parser_lval->uri) 326 YY_FATAL_ERROR_EOF("raptor_new_uri_relative_to_base failed"); 327 } 328 return URI_LITERAL; } 329 330 {LANGUAGETOKEN} { n3_parser_lval->string=n3_copy_token((unsigned char*)yytext, yyleng); 331 if(!n3_parser_lval->string) 332 YY_FATAL_ERROR_EOF("n3_copy_token failed"); 333 return IDENTIFIER; } 334 335 \#[^\r\n]*(\r\n|\r|\n) { /* # comment */ 336 n3_parser->lineno++; 337 } 338 339 \#[^\r\n]* { /* # comment on the last line with no terminating newline */ 340 } 341 342 . { if (*yytext == EOF) 343 return EOF; 344 345 n3_syntax_error(rdf_parser, "syntax error at '%c'", *yytext); 346 yyterminate(); 347 } 348 349 %% 350 /* user code */ 351 352 int 353 yywrap (yyscan_t yyscanner) { 354 return 1; 355 } 356 357 358 static unsigned char * 359 n3_copy_token(unsigned char *text, size_t len) 360 { 361 unsigned char *s; 362 if(!len) 363 len=strlen((const char*)text); 364 s=(unsigned char *)RAPTOR_MALLOC(cstring, len+1); 365 if(s) { 366 strncpy((char*)s, (const char*)text, len); 367 s[len] = '\0'; 368 } 369 return s; 370 } 371 372 373 static unsigned char * 374 n3_copy_string_token(raptor_parser* rdf_parser, 375 unsigned char *string, size_t len, int delim) 376 { 377 raptor_stringbuffer* sb=NULL; 378 int rc; 379 380 if(len) { 381 sb=raptor_new_stringbuffer(); 382 if(!sb) 383 return NULL; 384 385 rc=raptor_stringbuffer_append_turtle_string(sb, string, len, delim, 386 (raptor_simple_message_handler)n3_lexer_syntax_error, 387 rdf_parser); 388 if(rc) { 389 raptor_free_stringbuffer(sb); 390 return NULL; 391 } 392 393 len=raptor_stringbuffer_length(sb); 394 } 395 396 string=(unsigned char *)RAPTOR_MALLOC(cstring, len+1); 397 if(string) { 398 if(sb) { 399 raptor_stringbuffer_copy_to_string(sb, string, len+1); 400 } 401 string[len]='\0'; 402 } 403 404 if(sb) 405 raptor_free_stringbuffer(sb); 406 407 return string; 408 } 409 410 411 412 void 413 n3_lexer_syntax_error(void* ctx, const char *message, ...) 414 { 415 raptor_parser* rdf_parser=(raptor_parser *)ctx; 416 raptor_n3_parser* n3_parser=(raptor_n3_parser*)rdf_parser->context; 417 va_list arguments; 418 419 rdf_parser->locator.line=n3_parser->lineno; 420 #ifdef RAPTOR_N3_USE_ERROR_COLUMNS 421 rdf_parser->locator.column=n3_lexer_get_column(yyscanner); 422 #endif 423 424 va_start(arguments, message); 425 raptor_parser_error_varargs(((raptor_parser*)rdf_parser), message, arguments); 426 427 va_end(arguments); 428 } 429 430 431 /* 432 * n3_lexer_fatal_error: 433 * @msg: 434 * @yyscanner: 435 * 436 * INTERNAL - replacement for the generated error handler. 437 */ 438 static void n3_lexer_fatal_error(yyconst char *msg, yyscan_t yyscanner) 439 { 440 raptor_parser *rdf_parser=NULL; 441 442 if(yyscanner) 443 rdf_parser=(raptor_parser *)n3_lexer_get_extra(yyscanner); 444 445 if(rdf_parser) 446 /* avoid "format not a string literal and no format arguments" warning with %s */ 447 raptor_parser_fatal_error(rdf_parser, "%s", msg); 448 else { 449 fputs(msg, stderr); 450 fputc('\n', stderr); 451 } 452 } 453 454 455 /* Define LEXER_ALLOC_TRACKING to enable allocated memory tracking 456 * - fixes lexer memory leak when ensure_buffer_stack fails 457 */ 458 459 #ifdef LEXER_ALLOC_TRACKING 460 typedef struct { 461 /* Number of void* slots allocated */ 462 int lexer_allocs_size; 463 /* Allocted void* slots follow in memory after this header */ 464 } lexer_alloc_tracker_header; 465 466 /* Initial alloc tracker slot array size - 2 seems to be enough for almost all cases */ 467 static const int initial_lexer_allocs_size=2; 468 #endif 469 470 /* 471 * n3_lexer_cleanup: 472 * @yyscanner: 473 * 474 * INTERNAL - Clean up unfreed lexer allocs if LEXER_ALLOC_TRACKING is enabled. 475 */ 476 static void n3_lexer_cleanup(yyscan_t yyscanner) 477 { 478 #ifdef LEXER_ALLOC_TRACKING 479 raptor_parser *rdf_parser; 480 lexer_alloc_tracker_header *tracker; 481 void **lexer_allocs; 482 int i; 483 484 if(!yyscanner) 485 return; 486 487 rdf_parser=(raptor_parser *)n3_lexer_get_extra(yyscanner); 488 if(!rdf_parser) 489 return; 490 491 tracker=(lexer_alloc_tracker_header *)rdf_parser->lexer_user_data; 492 if(!tracker) 493 return; 494 lexer_allocs=(void**)&tracker[1]; 495 496 for(i=0; i<tracker->lexer_allocs_size; ++i) { 497 if(lexer_allocs[i]) 498 free(lexer_allocs[i]); 499 lexer_allocs[i]=NULL; 500 } 501 free(rdf_parser->lexer_user_data); 502 rdf_parser->lexer_user_data=NULL; 503 #endif 504 } 505 506 507 /* 508 * n3_lexer_alloc: 509 * @size 510 * @yyscanner 511 * 512 * INTERNAL - alloc replacement. 513 * Tracks allocated cells if LEXER_ALLOC_TRACKING is enabled. 514 */ 515 void *n3_lexer_alloc(yy_size_t size, yyscan_t yyscanner) 516 { 517 #ifdef LEXER_ALLOC_TRACKING 518 raptor_parser *rdf_parser; 519 lexer_alloc_tracker_header *tracker; 520 void **lexer_allocs; 521 int i; 522 void *ptr; 523 524 /* yyscanner not initialized -> probably initializing yyscanner itself 525 * -> just malloc without tracking 526 */ 527 if(!yyscanner) 528 return malloc(size); 529 530 rdf_parser=(raptor_parser *)n3_lexer_get_extra(yyscanner); 531 if(!rdf_parser) 532 YY_FATAL_ERROR("lexer_alloc: yyscanner extra not initialized"); 533 534 /* try to allocate tracker if it does not exist */ 535 tracker=(lexer_alloc_tracker_header *)rdf_parser->lexer_user_data; 536 if(!tracker) { 537 /* allocate tracker header + array of void* slots */ 538 tracker=(lexer_alloc_tracker_header*)calloc(1, sizeof(lexer_alloc_tracker_header)+initial_lexer_allocs_size*sizeof(void*)); 539 if(!tracker) 540 YY_FATAL_ERROR("lexer_alloc: cannot allocate tracker"); 541 tracker->lexer_allocs_size=initial_lexer_allocs_size; 542 rdf_parser->lexer_user_data=(void *)tracker; 543 } 544 lexer_allocs=(void**)&tracker[1]; 545 546 /* allocate memory */ 547 ptr=malloc(size); 548 549 /* find a free slot for ptr */ 550 for(i=0; i<tracker->lexer_allocs_size; ++i) { 551 if(!lexer_allocs[i]) { 552 lexer_allocs[i]=ptr; 553 break; 554 } 555 } 556 557 /* no free slots -> grow tracker slot array */ 558 if(i>=tracker->lexer_allocs_size) { 559 int j; 560 void **dest; 561 tracker=(lexer_alloc_tracker_header*)calloc(1, sizeof(lexer_alloc_tracker_header)+i*2*sizeof(void*)); 562 if(!tracker) { 563 if(ptr) 564 free(ptr); 565 YY_FATAL_ERROR("lexer_alloc: cannot grow tracker"); 566 } 567 tracker->lexer_allocs_size=i*2; 568 569 /* copy data from old tracker */ 570 dest=(void**)&tracker[1]; 571 for(j=0; j<i; ++j) { 572 dest[j]=lexer_allocs[j]; 573 } 574 575 /* set new item to first free slot */ 576 dest[j]=ptr; 577 578 /* free old tracker and replace with new one */ 579 free(rdf_parser->lexer_user_data); 580 rdf_parser->lexer_user_data=tracker; 581 } 582 583 return ptr; 584 #else 585 return malloc(size); 586 #endif 587 } 588 589 590 /* 591 * n3_lexer_realloc: 592 * 593 * INTERNAL - realloc replacement 594 * Tracks allocated cells if LEXER_ALLOC_TRACKING is enabled. 595 */ 596 void *n3_lexer_realloc(void *ptr, yy_size_t size, yyscan_t yyscanner) 597 { 598 #ifdef LEXER_ALLOC_TRACKING 599 raptor_parser *rdf_parser; 600 lexer_alloc_tracker_header *tracker; 601 void **lexer_allocs; 602 int i; 603 void *newptr; 604 605 if(!yyscanner) 606 YY_FATAL_ERROR("lexer_realloc: yyscanner not initialized"); 607 608 rdf_parser=(raptor_parser *)n3_lexer_get_extra(yyscanner); 609 if(!rdf_parser) 610 YY_FATAL_ERROR("lexer_realloc: yyscanner extra not initialized"); 611 612 tracker=(lexer_alloc_tracker_header *)rdf_parser->lexer_user_data; 613 if(!tracker) 614 YY_FATAL_ERROR("lexer_realloc: no alloc tracker"); 615 lexer_allocs=(void**)&tracker[1]; 616 617 /* find the old slot for ptr */ 618 for(i=0; i<tracker->lexer_allocs_size; ++i) { 619 if(lexer_allocs[i]==ptr) 620 break; 621 } 622 623 /* no old slot -> error */ 624 if(i>=tracker->lexer_allocs_size) 625 YY_FATAL_ERROR("lexer_realloc: cell not in tracker"); 626 627 /* realloc */ 628 newptr=realloc((char*)ptr, size); 629 630 /* replace entry in tracker */ 631 lexer_allocs[i]=newptr; 632 633 return newptr; 634 #else 635 return realloc((char*)ptr, size); 636 #endif 637 } 638 639 640 /* 641 * n3_lexer_free: 642 * 643 * INTERNAL - free replacement. 644 * Checks for NULL pointer to be freed unlike the default lexer free function. 645 * Tracks allocated cells if LEXER_ALLOC_TRACKING is enabled. 646 */ 647 void n3_lexer_free(void *ptr, yyscan_t yyscanner) 648 { 649 #ifdef LEXER_ALLOC_TRACKING 650 raptor_parser *rdf_parser; 651 lexer_alloc_tracker_header *tracker; 652 void **lexer_allocs; 653 int i; 654 655 /* do not free NULL */ 656 if(!ptr) 657 return; 658 659 /* free ptr even if we would encounter an error */ 660 free(ptr); 661 662 /* yyscanner is allocated with n3_lexer_alloc() but it's never stored in the tracker 663 * - we need yyscanner to access the tracker */ 664 if(!yyscanner || ptr==yyscanner) 665 return; 666 667 rdf_parser=(raptor_parser *)n3_lexer_get_extra(yyscanner); 668 if(!rdf_parser) 669 return; 670 671 tracker=(lexer_alloc_tracker_header *)rdf_parser->lexer_user_data; 672 if(!tracker) 673 return; 674 lexer_allocs=(void**)&tracker[1]; 675 676 /* find the slot for ptr */ 677 for(i=0; i<tracker->lexer_allocs_size; ++i) { 678 if(lexer_allocs[i]==ptr) 679 break; 680 } 681 682 /* no slot -> error */ 683 if(i>=tracker->lexer_allocs_size) 684 YY_FATAL_ERROR("lexer_free: cell not in tracker"); 685 686 /* remove entry from tracker */ 687 lexer_allocs[i]=NULL; 688 #else 689 if(ptr) 690 free(ptr); 691 #endif 692 } 693 694 695 #ifdef RAPTOR_DEBUG 696 697 const char * 698 n3_token_print(raptor_world* world, int token, YYSTYPE *lval) 699 { 700 static char buffer[2048]; 701 702 if(!token) 703 return "<<EOF>>"; 704 705 switch(token) { 706 case PREFIX: 707 return "PREFIX"; 708 709 case A: 710 return "A"; 711 712 case DOT: 713 return "DOT"; 714 715 case COMMA: 716 return "COMMA"; 717 718 case SEMICOLON: 719 return "SEMICOLON"; 720 721 case LEFT_SQUARE: 722 return "LEFT_SQUARE"; 723 724 case RIGHT_SQUARE: 725 return "RIGHT_SQUARE"; 726 727 case AT: 728 return "AT"; 729 730 case HAT: 731 return "HAT"; 732 733 case STRING_LITERAL: 734 sprintf(buffer, "STRING_LITERAL(%s)", lval->string); 735 return buffer; 736 737 case URI_LITERAL: 738 sprintf(buffer, "URI_LITERAL(%s)", 739 (lval->uri ? (char*)raptor_uri_as_string_v2(world, lval->uri) : "")); 740 return buffer; 741 742 case BLANK_LITERAL: 743 sprintf(buffer, "BLANK_LITERAL(%s)", lval->string); 744 return buffer; 745 746 case QNAME_LITERAL: 747 sprintf(buffer, "QNAME_LITERAL(%s)", 748 (lval->uri ? (char*)raptor_uri_as_string_v2(world, lval->uri) : "")); 749 return buffer; 750 751 case INTEGER_LITERAL: 752 sprintf(buffer, "INTEGER_LITERAL(%d)", lval->integer); 753 return buffer; 754 755 case FLOATING_LITERAL: 756 sprintf(buffer, "FLOATING_LITERAL(%1g)", lval->floating); 757 return buffer; 758 759 case IDENTIFIER: 760 sprintf(buffer, "IDENTIFIER(%s)", 761 (lval->string ? (char*)lval->string : "")); 762 return buffer; 763 764 case DECIMAL_LITERAL: 765 sprintf(buffer, "DECIMAL_LITERAL(%s)", lval->string); 766 return buffer; 767 768 case ERROR_TOKEN: 769 return "ERROR"; 770 771 default: 772 RAPTOR_DEBUG2("UNKNOWN token %d - add a new case\n", token); 773 return "(UNKNOWN)"; 774 } 775 } 776 #endif 777 778 779 780 #ifdef STANDALONE 781 782 static void 783 n3_token_free(int token, YYSTYPE *lval) 784 { 785 if(!token) 786 return; 787 788 switch(token) { 789 case STRING_LITERAL: 790 case BLANK_LITERAL: 791 case IDENTIFIER: 792 if(lval->string) 793 RAPTOR_FREE(cstring, lval->string); 794 break; 795 796 case URI_LITERAL: 797 case QNAME_LITERAL: 798 if(lval->uri) 799 raptor_free_uri(lval->uri); 800 break; 801 default: 802 break; 803 } 804 } 805 806 807 int 808 main(int argc, char *argv[]) 809 { 810 raptor_parser rdf_parser; 811 raptor_n3_parser n3_parser; 812 yyscan_t scanner; 813 int token=EOF; 814 FILE *fh; 815 YYSTYPE lval; 816 const unsigned char *uri_string; 817 char *filename=NULL; 818 819 raptor_init(); 820 821 if(argc > 1) { 822 filename=argv[1]; 823 fh=fopen(filename, "r"); 824 if(!fh) { 825 fprintf(stderr, "%s: Cannot open file %s - %s\n", argv[0], filename, 826 strerror(errno)); 827 exit(1); 828 } 829 } else { 830 filename="<stdin>"; 831 fh=stdin; 832 } 833 834 memset(&rdf_parser, 0, sizeof(raptor_parser)); 835 memset(&n3_parser, 0, sizeof(raptor_n3_parser)); 836 837 yylex_init(&n3_parser.scanner); 838 scanner=n3_parser.scanner; 839 n3_lexer_set_in(fh, scanner); 840 n3_lexer_set_extra(&rdf_parser, scanner); 841 842 /* Initialise enough of the parser and locator to get error messages */ 843 rdf_parser.context=&n3_parser; 844 n3_parser.lineno=1; 845 rdf_parser.locator.file=filename; 846 rdf_parser.locator.column= -1; 847 848 uri_string=raptor_uri_filename_to_uri_string(filename); 849 rdf_parser.base_uri=raptor_new_uri(uri_string); 850 RAPTOR_FREE(cstring, (void*)uri_string); 851 852 while(1) { 853 memset(&lval, 0, sizeof(YYSTYPE)); 854 if(n3_lexer_get_text(scanner) != NULL) 855 printf("yyinput '%s'\n", n3_lexer_get_text(scanner)); 856 token=yylex(&lval, scanner); 857 #ifdef RAPTOR_DEBUG 858 printf("token %s\n", n3_token_print(raptor_world_instance(), token, &lval)); /* FIXME */ 859 #else 860 printf("token %d\n", token); 861 #endif 862 n3_token_free(token, &lval); 863 if(!token || token == EOF || token == ERROR_TOKEN) 864 break; 865 } 866 867 yylex_destroy(scanner); 868 869 raptor_free_uri(rdf_parser.base_uri); 870 871 raptor_finish(); 872 873 874 if(token == ERROR_TOKEN) 875 return 1; 876 877 return 0; 878 } 879 #endif 880