1;;; semantic/lex.el --- Lexical Analyzer builder -*- lexical-binding:t -*- 2 3;; Copyright (C) 1999-2021 Free Software Foundation, Inc. 4 5;; Author: Eric M. Ludlam <zappo@gnu.org> 6 7;; This file is part of GNU Emacs. 8 9;; GNU Emacs is free software: you can redistribute it and/or modify 10;; it under the terms of the GNU General Public License as published by 11;; the Free Software Foundation, either version 3 of the License, or 12;; (at your option) any later version. 13 14;; GNU Emacs is distributed in the hope that it will be useful, 15;; but WITHOUT ANY WARRANTY; without even the implied warranty of 16;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17;; GNU General Public License for more details. 18 19;; You should have received a copy of the GNU General Public License 20;; along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>. 21 22;;; Commentary: 23;; 24;; This file handles the creation of lexical analyzers for different 25;; languages in Emacs Lisp. The purpose of a lexical analyzer is to 26;; convert a buffer into a list of lexical tokens. Each token 27;; contains the token class (such as 'number, 'symbol, 'IF, etc) and 28;; the location in the buffer it was found. Optionally, a token also 29;; contains a string representing what is at the designated buffer 30;; location. 31;; 32;; Tokens are pushed onto a token stream, which is basically a list of 33;; all the lexical tokens from the analyzed region. The token stream 34;; is then handed to the grammar which parsers the file. 35;; 36;;; How it works 37;; 38;; Each analyzer specifies a condition and forms. These conditions 39;; and forms are assembled into a function by `define-lex' that does 40;; the lexical analysis. 41;; 42;; In the lexical analyzer created with `define-lex', each condition 43;; is tested for a given point. When the condition is true, the forms 44;; run. 45;; 46;; The forms can push a lexical token onto the token stream. The 47;; analyzer forms also must move the current analyzer point. If the 48;; analyzer point is moved without pushing a token, then the matched 49;; syntax is effectively ignored, or skipped. 50;; 51;; Thus, starting at the beginning of a region to be analyzed, each 52;; condition is tested. One will match, and a lexical token might be 53;; pushed, and the point is moved to the end of the lexical token 54;; identified. At the new position, the process occurs again until 55;; the end of the specified region is reached. 56;; 57;;; How to use semantic-lex 58;; 59;; To create a lexer for a language, use the `define-lex' macro. 60;; 61;; The `define-lex' macro accepts a list of lexical analyzers. Each 62;; analyzer is created with `define-lex-analyzer', or one of the 63;; derivative macros. A single analyzer defines a regular expression 64;; to match text in a buffer, and a short segment of code to create 65;; one lexical token. 66;; 67;; Each analyzer has a NAME, DOC, a CONDITION, and possibly some 68;; FORMS. The NAME is the name used in `define-lex'. The DOC 69;; describes what the analyzer should do. 70;; 71;; The CONDITION evaluates the text at the current point in the 72;; current buffer. If CONDITION is true, then the FORMS will be 73;; executed. 74;; 75;; The purpose of the FORMS is to push new lexical tokens onto the 76;; list of tokens for the current buffer, and to move point after the 77;; matched text. 78;; 79;; Some macros for creating one analyzer are: 80;; 81;; define-lex-analyzer - A generic analyzer associating any style of 82;; condition to forms. 83;; define-lex-regex-analyzer - Matches a regular expression. 84;; define-lex-simple-regex-analyzer - Matches a regular expressions, 85;; and pushes the match. 86;; define-lex-block-analyzer - Matches list syntax, and defines 87;; handles open/close delimiters. 88;; 89;; These macros are used by the grammar compiler when lexical 90;; information is specified in a grammar: 91;; define-lex- * -type-analyzer - Matches syntax specified in 92;; a grammar, and pushes one token for it. The * would 93;; be `sexp' for things like lists or strings, and 94;; `string' for things that need to match some special 95;; string, such as "\\." where a literal match is needed. 96;; 97;;; Lexical Tables 98;; 99;; There are tables of different symbols managed in semantic-lex.el. 100;; They are: 101;; 102;; Lexical keyword table - A Table of symbols declared in a grammar 103;; file with the %keyword declaration. 104;; Keywords are used by `semantic-lex-symbol-or-keyword' 105;; to create lexical tokens based on the keyword. 106;; 107;; Lexical type table - A table of symbols declared in a grammar 108;; file with the %type declaration. 109;; The grammar compiler uses the type table to create new 110;; lexical analyzers. These analyzers are then used to when 111;; a new lexical analyzer is made for a language. 112;; 113;;; Lexical Types 114;; 115;; A lexical type defines a kind of lexical analyzer that will be 116;; automatically generated from a grammar file based on some 117;; predetermined attributes. For now these two attributes are 118;; recognized : 119;; 120;; * matchdatatype : define the kind of lexical analyzer. That is : 121;; 122;; - regexp : define a regexp analyzer (see 123;; `define-lex-regex-type-analyzer') 124;; 125;; - string : define a string analyzer (see 126;; `define-lex-string-type-analyzer') 127;; 128;; - block : define a block type analyzer (see 129;; `define-lex-block-type-analyzer') 130;; 131;; - sexp : define a sexp analyzer (see 132;; `define-lex-sexp-type-analyzer') 133;; 134;; - keyword : define a keyword analyzer (see 135;; `define-lex-keyword-type-analyzer') 136;; 137;; * syntax : define the syntax that matches a syntactic 138;; expression. When syntax is matched the corresponding type 139;; analyzer is entered and the resulting match data will be 140;; interpreted based on the kind of analyzer (see matchdatatype 141;; above). 142;; 143;; The following lexical types are predefined : 144;; 145;; +-------------+---------------+--------------------------------+ 146;; | type | matchdatatype | syntax | 147;; +-------------+---------------+--------------------------------+ 148;; | punctuation | string | "\\(\\s.\\|\\s$\\|\\s'\\)+" | 149;; | keyword | keyword | "\\(\\sw\\|\\s_\\)+" | 150;; | symbol | regexp | "\\(\\sw\\|\\s_\\)+" | 151;; | string | sexp | "\\s\"" | 152;; | number | regexp | semantic-lex-number-expression | 153;; | block | block | "\\s(\\|\\s)" | 154;; +-------------+---------------+--------------------------------+ 155;; 156;; In a grammar you must use a %type expression to automatically generate 157;; the corresponding analyzers of that type. 158;; 159;; Here is an example to auto-generate punctuation analyzers 160;; with 'matchdatatype and 'syntax predefined (see table above) 161;; 162;; %type <punctuation> ;; will auto-generate this kind of analyzers 163;; 164;; It is equivalent to write : 165;; 166;; %type <punctuation> syntax "\\(\\s.\\|\\s$\\|\\s'\\)+" matchdatatype string 167;; 168;; ;; Some punctuation based on the type defines above 169;; 170;; %token <punctuation> NOT "!" 171;; %token <punctuation> NOTEQ "!=" 172;; %token <punctuation> MOD "%" 173;; %token <punctuation> MODEQ "%=" 174;; 175 176;;; On the Semantic 1.x lexer 177;; 178;; In semantic 1.x, the lexical analyzer was an all purpose routine. 179;; To boost efficiency, the analyzer is now a series of routines that 180;; are constructed at build time into a single routine. This will 181;; eliminate unneeded if statements to speed the lexer. 182 183(require 'semantic/fw) 184 185;;; Code: 186 187;;; Semantic 2.x lexical analysis 188;; 189(defun semantic-lex-map-symbols (fun table &optional property) 190 "Call function FUN on every symbol in TABLE. 191If optional PROPERTY is non-nil, call FUN only on every symbol which 192as a PROPERTY value. FUN receives a symbol as argument." 193 (if (obarrayp table) 194 (mapatoms 195 (lambda (symbol) 196 (if (or (null property) (get symbol property)) 197 (funcall fun symbol))) 198 table))) 199 200;;; Lexical keyword table handling. 201;; 202;; These keywords are keywords defined for using in a grammar with the 203;; %keyword declaration, and are not keywords used in Emacs Lisp. 204 205(defvar-local semantic-flex-keywords-obarray nil 206 "Buffer local keyword obarray for the lexical analyzer. 207These keywords are matched explicitly, and converted into special symbols.") 208 209(defmacro semantic-lex-keyword-invalid (name) 210 "Signal that NAME is an invalid keyword name." 211 `(signal 'wrong-type-argument '(semantic-lex-keyword-p ,name))) 212 213(defsubst semantic-lex-keyword-symbol (name) 214 "Return keyword symbol with NAME or nil if not found." 215 (and (obarrayp semantic-flex-keywords-obarray) 216 (stringp name) 217 (intern-soft name semantic-flex-keywords-obarray))) 218 219(defsubst semantic-lex-keyword-p (name) 220 "Return non-nil if a keyword with NAME exists in the keyword table. 221Return nil otherwise." 222 (and (setq name (semantic-lex-keyword-symbol name)) 223 (symbol-value name))) 224 225(defsubst semantic-lex-keyword-set (name value) 226 "Set value of keyword with NAME to VALUE and return VALUE." 227 (set (intern name semantic-flex-keywords-obarray) value)) 228 229(defsubst semantic-lex-keyword-value (name) 230 "Return value of keyword with NAME. 231Signal an error if a keyword with NAME does not exist." 232 (let ((keyword (semantic-lex-keyword-symbol name))) 233 (if keyword 234 (symbol-value keyword) 235 (semantic-lex-keyword-invalid name)))) 236 237(defsubst semantic-lex-keyword-put (name property value) 238 "For keyword with NAME, set its PROPERTY to VALUE." 239 (let ((keyword (semantic-lex-keyword-symbol name))) 240 (if keyword 241 (put keyword property value) 242 (semantic-lex-keyword-invalid name)))) 243 244(defsubst semantic-lex-keyword-get (name property) 245 "For keyword with NAME, return its PROPERTY value." 246 (let ((keyword (semantic-lex-keyword-symbol name))) 247 (if keyword 248 (get keyword property) 249 (semantic-lex-keyword-invalid name)))) 250 251(defun semantic-lex-make-keyword-table (specs &optional propspecs) 252 "Convert keyword SPECS into an obarray and return it. 253SPECS must be a list of (NAME . TOKSYM) elements, where: 254 255 NAME is the name of the keyword symbol to define. 256 TOKSYM is the lexical token symbol of that keyword. 257 258If optional argument PROPSPECS is non-nil, then interpret it, and 259apply those properties. 260PROPSPECS must be a list of (NAME PROPERTY VALUE) elements." 261 ;; Create the symbol hash table 262 (let ((semantic-flex-keywords-obarray (make-vector 13 0)) 263 spec) 264 ;; fill it with stuff 265 (while specs 266 (setq spec (car specs) 267 specs (cdr specs)) 268 (semantic-lex-keyword-set (car spec) (cdr spec))) 269 ;; Apply all properties 270 (while propspecs 271 (setq spec (car propspecs) 272 propspecs (cdr propspecs)) 273 (semantic-lex-keyword-put (car spec) (nth 1 spec) (nth 2 spec))) 274 semantic-flex-keywords-obarray)) 275 276(defsubst semantic-lex-map-keywords (fun &optional property) 277 "Call function FUN on every lexical keyword. 278If optional PROPERTY is non-nil, call FUN only on every keyword which 279as a PROPERTY value. FUN receives a lexical keyword as argument." 280 (semantic-lex-map-symbols 281 fun semantic-flex-keywords-obarray property)) 282 283(defun semantic-lex-keywords (&optional property) 284 "Return a list of lexical keywords. 285If optional PROPERTY is non-nil, return only keywords which have a 286PROPERTY set." 287 (let (keywords) 288 (semantic-lex-map-keywords 289 (lambda (symbol) (setq keywords (cons symbol keywords))) 290 property) 291 keywords)) 292 293;;; Inline functions: 294 295(defvar semantic-lex-unterminated-syntax-end-function) 296(defvar semantic-lex-analysis-bounds) 297(defvar semantic-lex-end-point) 298 299(defsubst semantic-lex-token-bounds (token) 300 "Fetch the start and end locations of the lexical token TOKEN. 301Return a pair (START . END)." 302 (if (not (numberp (car (cdr token)))) 303 (cdr (cdr token)) 304 (cdr token))) 305 306(defsubst semantic-lex-token-start (token) 307 "Fetch the start position of the lexical token TOKEN. 308See also the function `semantic-lex-token'." 309 (car (semantic-lex-token-bounds token))) 310 311(defsubst semantic-lex-token-end (token) 312 "Fetch the end position of the lexical token TOKEN. 313See also the function `semantic-lex-token'." 314 (cdr (semantic-lex-token-bounds token))) 315 316(defsubst semantic-lex-unterminated-syntax-detected (syntax) 317 "Inside a lexical analyzer, use this when unterminated syntax was found. 318Argument SYNTAX indicates the type of syntax that is unterminated. 319The job of this function is to move (point) to a new logical location 320so that analysis can continue, if possible." 321 (goto-char 322 (funcall semantic-lex-unterminated-syntax-end-function 323 syntax 324 (car semantic-lex-analysis-bounds) 325 (cdr semantic-lex-analysis-bounds) 326 )) 327 (setq semantic-lex-end-point (point))) 328 329;;; Type table handling. 330;; 331;; The lexical type table manages types that occur in a grammar file 332;; with the %type declaration. Types represent different syntaxes. 333;; See code for `semantic-lex-preset-default-types' for the classic 334;; types of syntax. 335(defvar-local semantic-lex-types-obarray nil 336 "Buffer local types obarray for the lexical analyzer.") 337 338(defun semantic-lex-type-invalid (type) 339 "Signal that TYPE is an invalid lexical type name." 340 (signal 'wrong-type-argument `(semantic-lex-type-p ,type))) 341 342(defsubst semantic-lex-type-symbol (type) 343 "Return symbol with TYPE or nil if not found." 344 (and (obarrayp semantic-lex-types-obarray) 345 (stringp type) 346 (intern-soft type semantic-lex-types-obarray))) 347 348(defsubst semantic-lex-type-p (type) 349 "Return non-nil if a symbol with TYPE name exists." 350 (and (setq type (semantic-lex-type-symbol type)) 351 (symbol-value type))) 352 353(defsubst semantic-lex-type-set (type value) 354 "Set value of symbol with TYPE name to VALUE and return VALUE." 355 (set (intern type semantic-lex-types-obarray) value)) 356 357(defsubst semantic-lex-type-value (type &optional noerror) 358 "Return value of symbol with TYPE name. 359If optional argument NOERROR is non-nil return nil if a symbol with 360TYPE name does not exist. Otherwise signal an error." 361 (let ((sym (semantic-lex-type-symbol type))) 362 (if sym 363 (symbol-value sym) 364 (unless noerror 365 (semantic-lex-type-invalid type))))) 366 367(defsubst semantic-lex-type-put (type property value &optional add) 368 "For symbol with TYPE name, set its PROPERTY to VALUE. 369If optional argument ADD is non-nil, create a new symbol with TYPE 370name if it does not already exist. Otherwise signal an error." 371 (let ((sym (semantic-lex-type-symbol type))) 372 (unless sym 373 (or add (semantic-lex-type-invalid type)) 374 (semantic-lex-type-set type nil) 375 (setq sym (semantic-lex-type-symbol type))) 376 (put sym property value))) 377 378(defsubst semantic-lex-type-get (type property &optional noerror) 379 "For symbol with TYPE name, return its PROPERTY value. 380If optional argument NOERROR is non-nil return nil if a symbol with 381TYPE name does not exist. Otherwise signal an error." 382 (let ((sym (semantic-lex-type-symbol type))) 383 (if sym 384 (get sym property) 385 (unless noerror 386 (semantic-lex-type-invalid type))))) 387 388(defun semantic-lex-preset-default-types () 389 "Install useful default properties for well known types." 390 (semantic-lex-type-put "punctuation" 'matchdatatype 'string t) 391 (semantic-lex-type-put "punctuation" 'syntax "\\(\\s.\\|\\s$\\|\\s'\\)+") 392 (semantic-lex-type-put "keyword" 'matchdatatype 'keyword t) 393 (semantic-lex-type-put "keyword" 'syntax "\\(\\sw\\|\\s_\\)+") 394 (semantic-lex-type-put "symbol" 'matchdatatype 'regexp t) 395 (semantic-lex-type-put "symbol" 'syntax "\\(\\sw\\|\\s_\\)+") 396 (semantic-lex-type-put "string" 'matchdatatype 'sexp t) 397 (semantic-lex-type-put "string" 'syntax "\\s\"") 398 (semantic-lex-type-put "number" 'matchdatatype 'regexp t) 399 (semantic-lex-type-put "number" 'syntax 'semantic-lex-number-expression) 400 (semantic-lex-type-put "block" 'matchdatatype 'block t) 401 (semantic-lex-type-put "block" 'syntax "\\s(\\|\\s)") 402 ) 403 404(defun semantic-lex-make-type-table (specs &optional propspecs) 405 "Convert type SPECS into an obarray and return it. 406SPECS must be a list of (TYPE . TOKENS) elements, where: 407 408 TYPE is the name of the type symbol to define. 409 TOKENS is a list of (TOKSYM . MATCHER) elements, where: 410 411 TOKSYM is any lexical token symbol. 412 MATCHER is a string or regexp a text must match to be a such 413 lexical token. 414 415If optional argument PROPSPECS is non-nil, then interpret it, and 416apply those properties. 417PROPSPECS must be a list of (TYPE PROPERTY VALUE)." 418 ;; Create the symbol hash table 419 (let* ((semantic-lex-types-obarray (make-vector 13 0)) 420 spec type tokens token alist default) 421 ;; fill it with stuff 422 (while specs 423 (setq spec (car specs) 424 specs (cdr specs) 425 type (car spec) 426 tokens (cdr spec) 427 default nil 428 alist nil) 429 (while tokens 430 (setq token (car tokens) 431 tokens (cdr tokens)) 432 (if (cdr token) 433 (setq alist (cons token alist)) 434 (setq token (car token)) 435 (if default 436 (message 437 "*Warning* default value of <%s> tokens changed to %S, was %S" 438 type token default)) 439 (setq default token))) 440 ;; Ensure the default matching spec is the first one. 441 (semantic-lex-type-set type (cons default (nreverse alist)))) 442 ;; Install useful default types & properties 443 (semantic-lex-preset-default-types) 444 ;; Apply all properties 445 (while propspecs 446 (setq spec (car propspecs) 447 propspecs (cdr propspecs)) 448 ;; Create the type if necessary. 449 (semantic-lex-type-put (car spec) (nth 1 spec) (nth 2 spec) t)) 450 semantic-lex-types-obarray)) 451 452(defsubst semantic-lex-map-types (fun &optional property) 453 "Call function FUN on every lexical type. 454If optional PROPERTY is non-nil, call FUN only on every type symbol 455which has a PROPERTY value. FUN receives a type symbol as argument." 456 (semantic-lex-map-symbols 457 fun semantic-lex-types-obarray property)) 458 459(defun semantic-lex-types (&optional property) 460 "Return a list of lexical type symbols. 461If optional PROPERTY is non-nil, return only type symbols which have 462PROPERTY set." 463 (let (types) 464 (semantic-lex-map-types 465 (lambda (symbol) (setq types (cons symbol types))) 466 property) 467 types)) 468 469;;; Lexical Analyzer framework settings 470;; 471 472(defvar-local semantic-lex-analyzer #'semantic-lex 473 "The lexical analyzer used for a given buffer. 474See `semantic-lex' for documentation.") 475 476(defvar semantic-lex-tokens 477 '( 478 (bol) 479 (charquote) 480 (close-paren) 481 (comment) 482 (newline) 483 (open-paren) 484 (punctuation) 485 (semantic-list) 486 (string) 487 (symbol) 488 (whitespace) 489 ) 490 "An alist of semantic token types. 491As of December 2001 (semantic 1.4beta13), this variable is not used in 492any code. The only use is to refer to the doc-string from elsewhere. 493 494The key to this alist is the symbol representing token type that 495\\[semantic-flex] returns. These are 496 497 - bol: Empty string matching a beginning of line. 498 This token is produced with 499 `semantic-lex-beginning-of-line'. 500 501 - charquote: String sequences that match `\\s\\+' regexp. 502 This token is produced with `semantic-lex-charquote'. 503 504 - close-paren: Characters that match `\\s)' regexp. 505 These are typically `)', `}', `]', etc. 506 This token is produced with 507 `semantic-lex-close-paren'. 508 509 - comment: A comment chunk. These token types are not 510 produced by default. 511 This token is produced with `semantic-lex-comments'. 512 Comments are ignored with `semantic-lex-ignore-comments'. 513 Comments are treated as whitespace with 514 `semantic-lex-comments-as-whitespace'. 515 516 - newline Characters matching `\\s-*\\(\n\\|\\s>\\)' regexp. 517 This token is produced with `semantic-lex-newline'. 518 519 - open-paren: Characters that match `\\s(' regexp. 520 These are typically `(', `{', `[', etc. 521 If `semantic-lex-paren-or-list' is used, 522 then `open-paren' is not usually generated unless 523 the `depth' argument to \\[semantic-lex] is 524 greater than 0. 525 This token is always produced if the analyzer 526 `semantic-lex-open-paren' is used. 527 528 - punctuation: Characters matching `{\\(\\s.\\|\\s$\\|\\s'\\)' 529 regexp. 530 This token is produced with `semantic-lex-punctuation'. 531 Always specify this analyzer after the comment 532 analyzer. 533 534 - semantic-list: String delimited by matching parenthesis, braces, 535 etc. that the lexer skipped over, because the 536 `depth' parameter to \\[semantic-flex] was not high 537 enough. 538 This token is produced with `semantic-lex-paren-or-list'. 539 540 - string: Quoted strings, i.e., string sequences that start 541 and end with characters matching `\\s\"' 542 regexp. The lexer relies on @code{forward-sexp} to 543 find the matching end. 544 This token is produced with `semantic-lex-string'. 545 546 - symbol: String sequences that match `\\(\\sw\\|\\s_\\)+' 547 regexp. 548 This token is produced with 549 `semantic-lex-symbol-or-keyword'. Always add this analyzer 550 after `semantic-lex-number', or other analyzers that 551 match its regular expression. 552 553 - whitespace: Characters that match `\\s-+' regexp. 554 This token is produced with `semantic-lex-whitespace'.") 555 556(defvar-local semantic-lex-syntax-modifications nil 557 "Changes to the syntax table for this buffer. 558These changes are active only while the buffer is being flexed. 559This is a list where each element has the form: 560 (CHAR CLASS) 561CHAR is the char passed to `modify-syntax-entry', 562and CLASS is the string also passed to `modify-syntax-entry' to define 563what syntax class CHAR has.") 564 565(defvar-local semantic-lex-syntax-table nil 566 "Syntax table used by lexical analysis. 567See also `semantic-lex-syntax-modifications'.") 568 569(defvar-local semantic-lex-comment-regex nil 570 "Regular expression for identifying comment start during lexical analysis. 571This may be automatically set when semantic initializes in a mode, but 572may need to be overridden for some special languages.") 573 574(defvar-local semantic-lex-number-expression 575 ;; This expression was written by David Ponce for Java, and copied 576 ;; here for C and any other similar language. 577 (eval-when-compile 578 (concat "\\(" 579 "\\<[0-9]+[.][0-9]+\\([eE][-+]?[0-9]+\\)?[fFdD]?\\>" 580 "\\|" 581 "\\<[0-9]+[.][eE][-+]?[0-9]+[fFdD]?\\>" 582 "\\|" 583 "\\<[0-9]+[.][fFdD]\\>" 584 "\\|" 585 "\\<[0-9]+[.]" 586 "\\|" 587 "[.][0-9]+\\([eE][-+]?[0-9]+\\)?[fFdD]?\\>" 588 "\\|" 589 "\\<[0-9]+[eE][-+]?[0-9]+[fFdD]?\\>" 590 "\\|" 591 "\\<0[xX][[:xdigit:]]+[lL]?\\>" 592 "\\|" 593 "\\<[0-9]+[lLfFdD]?\\>" 594 "\\)" 595 )) 596 "Regular expression for matching a number. 597If this value is nil, no number extraction is done during lex. 598This expression tries to match C and Java like numbers. 599 600DECIMAL_LITERAL: 601 [1-9][0-9]* 602 ; 603HEX_LITERAL: 604 0[xX][[:xdigit:]]+ 605 ; 606OCTAL_LITERAL: 607 0[0-7]* 608 ; 609INTEGER_LITERAL: 610 <DECIMAL_LITERAL>[lL]? 611 | <HEX_LITERAL>[lL]? 612 | <OCTAL_LITERAL>[lL]? 613 ; 614EXPONENT: 615 [eE][+-]?[09]+ 616 ; 617FLOATING_POINT_LITERAL: 618 [0-9]+[.][0-9]*<EXPONENT>?[fFdD]? 619 | [.][0-9]+<EXPONENT>?[fFdD]? 620 | [0-9]+<EXPONENT>[fFdD]? 621 | [0-9]+<EXPONENT>?[fFdD] 622 ;") 623 624(defvar-local semantic-lex-depth 0 625 "Default lexing depth. 626This specifies how many lists to create tokens in.") 627 628(defvar semantic-lex-unterminated-syntax-end-function 629 (lambda (_syntax _syntax-start lex-end) lex-end) 630 "Function called when unterminated syntax is encountered. 631This should be set to one function. That function should take three 632parameters. The SYNTAX, or type of syntax which is unterminated. 633SYNTAX-START where the broken syntax begins. 634LEX-END is where the lexical analysis was asked to end. 635This function can be used for languages that can intelligently fix up 636broken syntax, or the exit lexical analysis via `throw' or `signal' 637when finding unterminated syntax.") 638 639;;; Interactive testing commands 640 641(declare-function semantic-elapsed-time "semantic") 642 643(defun semantic-lex-test (arg) 644 "Test the semantic lexer in the current buffer. 645If universal argument ARG, then try the whole buffer." 646 (interactive "P") 647 (require 'semantic) 648 (let* ((start (current-time)) 649 (result (semantic-lex 650 (if arg (point-min) (point)) 651 (point-max)))) 652 (message "Elapsed Time: %.2f seconds." 653 (semantic-elapsed-time start nil)) 654 (pop-to-buffer "*Lexer Output*") 655 (require 'pp) 656 (erase-buffer) 657 (insert (pp-to-string result)) 658 (goto-char (point-min)) 659 )) 660 661(defvar semantic-lex-debug nil 662 "When non-nil, debug the local lexical analyzer.") 663 664(defun semantic-lex-debug (arg) 665 "Debug the semantic lexer in the current buffer. 666Argument ARG specifies of the analyze the whole buffer, or start at point. 667While engaged, each token identified by the lexer will be highlighted 668in the target buffer A description of the current token will be 669displayed in the minibuffer. Press SPC to move to the next lexical token." 670 (interactive "P") 671 (require 'semantic/debug) 672 (let ((semantic-lex-debug t)) 673 (semantic-lex-test arg))) 674 675(defun semantic-lex-highlight-token (token) 676 "Highlight the lexical TOKEN. 677TOKEN is a lexical token with a START And END position. 678Return the overlay." 679 (let ((o (make-overlay (semantic-lex-token-start token) 680 (semantic-lex-token-end token)))) 681 (overlay-put o 'face 'highlight) 682 o)) 683 684;;; Lexical analyzer creation 685;; 686;; Code for creating a lex function from lists of analyzers. 687;; 688;; A lexical analyzer is created from a list of individual analyzers. 689;; Each individual analyzer specifies a single match, and code that 690;; goes with it. 691;; 692;; Creation of an analyzer assembles these analyzers into a new function 693;; with the behaviors of all the individual analyzers. 694;; 695(defmacro semantic-lex-one-token (analyzers) 696 "Calculate one token from the current buffer at point. 697Uses locally bound variables from `define-lex'. 698Argument ANALYZERS is the list of analyzers being used." 699 (cons 'cond (mapcar #'symbol-value analyzers))) 700 701(defvar semantic-lex-end-point nil 702 "The end point as tracked through lexical functions.") 703 704(defvar semantic-lex-current-depth nil 705 "The current depth as tracked through lexical functions.") 706 707(defvar semantic-lex-maximum-depth nil 708 "The maximum depth of parenthesis as tracked through lexical functions.") 709 710(defvar semantic-lex-token-stream nil 711 "The current token stream we are collecting.") 712 713(defvar semantic-lex-analysis-bounds nil 714 "The bounds of the current analysis.") 715 716(defvar semantic-lex-block-streams nil 717 "Streams of tokens inside collapsed blocks. 718This is an alist of (ANCHOR . STREAM) elements where ANCHOR is the 719start position of the block, and STREAM is the list of tokens in that 720block.") 721 722(define-obsolete-variable-alias 'semantic-lex-reset-hooks 723 'semantic-lex-reset-functions "24.3") 724(defvar semantic-lex-reset-functions nil 725 "Abnormal hook used by major-modes to reset lexical analyzers. 726Hook functions are called with START and END values for the 727current lexical pass. Should be set with `add-hook', specifying 728a LOCAL option.") 729 730;; Stack of nested blocks. 731(defvar semantic-lex-block-stack nil) 732;;(defcustom semantic-lex-timeout 5 733;; "Number of sections of lexing before giving up." 734;; :type 'integer 735;; :group 'semantic) 736 737(defsubst semantic-lex-debug-break (token) 738 "Break during lexical analysis at TOKEN." 739 (when semantic-lex-debug 740 (let ((o nil)) 741 (unwind-protect 742 (progn 743 (when token 744 (setq o (semantic-lex-highlight-token token))) 745 (read-event 746 (format "%S :: Depth: %d :: SPC - continue" token semantic-lex-current-depth)) 747 ) 748 (when o 749 (delete-overlay o)))))) 750 751(defmacro define-lex (name doc &rest analyzers) 752 "Create a new lexical analyzer with NAME. 753DOC is a documentation string describing this analyzer. 754ANALYZERS are small code snippets of analyzers to use when 755building the new NAMED analyzer. Only use analyzers which 756are written to be used in `define-lex'. 757Each analyzer should be an analyzer created with `define-lex-analyzer'. 758Note: The order in which analyzers are listed is important. 759If two analyzers can match the same text, it is important to order the 760analyzers so that the one you want to match first occurs first. For 761example, it is good to put a number analyzer in front of a symbol 762analyzer which might mistake a number for a symbol." 763 (declare (debug (&define name stringp (&rest symbolp))) (indent 1)) 764 `(defun ,name (start end &optional depth length) 765 ,(concat doc "\nSee `semantic-lex' for more information.") 766 ;; Make sure the state of block parsing starts over. 767 (setq semantic-lex-block-streams nil) 768 ;; Allow specialty reset items. 769 (run-hook-with-args 'semantic-lex-reset-functions start end) 770 ;; Lexing state. 771 (let* (;(starttime (current-time)) 772 (starting-position (point)) 773 (semantic-lex-token-stream nil) 774 (semantic-lex-block-stack nil) 775 (tmp-start start) 776 (semantic-lex-end-point start) 777 (semantic-lex-current-depth 0) 778 ;; Use the default depth when not specified. 779 (semantic-lex-maximum-depth 780 (or depth semantic-lex-depth)) 781 ;; Bounds needed for unterminated syntax 782 (semantic-lex-analysis-bounds (cons start end)) 783 ;; This entry prevents text properties from 784 ;; confusing our lexical analysis. See Emacs 22 (CVS) 785 ;; version of C++ mode with template hack text properties. 786 (parse-sexp-lookup-properties nil) 787 ) 788 ;; Maybe REMOVE THIS LATER. 789 ;; Trying to find incremental parser bug. 790 (when (> end (point-max)) 791 (error ,(format "%s: end (%%d) > point-max (%%d)" name) 792 end (point-max))) 793 (with-syntax-table semantic-lex-syntax-table 794 (goto-char start) 795 (while (and (< (point) end) 796 (or (not length) 797 (<= (length semantic-lex-token-stream) length))) 798 (semantic-lex-one-token ,analyzers) 799 (when (eq semantic-lex-end-point tmp-start) 800 (error ,(format "%s: endless loop at %%d, after %%S" name) 801 tmp-start (car semantic-lex-token-stream))) 802 (setq tmp-start semantic-lex-end-point) 803 (goto-char semantic-lex-end-point) 804 ;;(when (> (semantic-elapsed-time starttime nil) 805 ;; semantic-lex-timeout) 806 ;; (error "Timeout during lex at char %d" (point))) 807 (semantic-throw-on-input 'lex) 808 (semantic-lex-debug-break (car semantic-lex-token-stream)) 809 )) 810 ;; Check that there is no unterminated block. 811 (when semantic-lex-block-stack 812 (let* ((last (pop semantic-lex-block-stack)) 813 (blk last)) 814 (while blk 815 (message 816 ,(format "%s: `%%s' block from %%S is unterminated" name) 817 (car blk) (cadr blk)) 818 (setq blk (pop semantic-lex-block-stack))) 819 (semantic-lex-unterminated-syntax-detected (car last)))) 820 ;; Return to where we started. 821 ;; Do not wrap in protective stuff so that if there is an error 822 ;; thrown, the user knows where. 823 (goto-char starting-position) 824 ;; Return the token stream 825 (nreverse semantic-lex-token-stream)))) 826 827;;; Lexical token API 828;; 829;; Functions for accessing parts of a token. Use these functions 830;; instead of accessing the list structure directly because the 831;; contents of the lexical may change. 832;; 833(defmacro semantic-lex-token (symbol start end &optional str) 834 "Create a lexical token. 835SYMBOL is a symbol representing the class of syntax found. 836START and END define the bounds of the token in the current buffer. 837Optional STR is the string for the token only if the bounds in 838the buffer do not cover the string they represent. (As from 839macro expansion.)" 840 ;; This if statement checks the existence of a STR argument at 841 ;; compile time, where STR is some symbol or constant. If the 842 ;; variable STr (runtime) is nil, this will make an incorrect decision. 843 ;; 844 ;; It is like this to maintain the original speed of the compiled 845 ;; code. 846 (if str 847 `(cons ,symbol (cons ,str (cons ,start ,end))) 848 `(cons ,symbol (cons ,start ,end)))) 849 850(defun semantic-lex-token-p (thing) 851 "Return non-nil if THING is a semantic lex token. 852This is an exhaustively robust check." 853 (and (consp thing) 854 (symbolp (car thing)) 855 (or (and (numberp (nth 1 thing)) 856 (numberp (nthcdr 2 thing))) 857 (and (stringp (nth 1 thing)) 858 (numberp (nth 2 thing)) 859 (numberp (nthcdr 3 thing))) 860 )) 861 ) 862 863(defun semantic-lex-token-with-text-p (thing) 864 "Return non-nil if THING is a semantic lex token. 865This is an exhaustively robust check." 866 (and (consp thing) 867 (symbolp (car thing)) 868 (= (length thing) 4) 869 (stringp (nth 1 thing)) 870 (numberp (nth 2 thing)) 871 (numberp (nth 3 thing))) 872 ) 873 874(defun semantic-lex-token-without-text-p (thing) 875 "Return non-nil if THING is a semantic lex token. 876This is an exhaustively robust check." 877 (and (consp thing) 878 (symbolp (car thing)) 879 (= (length thing) 3) 880 (numberp (nth 1 thing)) 881 (numberp (nth 2 thing))) 882 ) 883 884(eval-and-compile 885 886(defun semantic-lex-expand-block-specs (specs) 887 "Expand block specifications SPECS into a Lisp form. 888SPECS is a list of (BLOCK BEGIN END) elements where BLOCK, BEGIN, and 889END are token class symbols that indicate to produce one collapsed 890BLOCK token from tokens found between BEGIN and END ones. 891BLOCK must be a non-nil symbol, and at least one of the BEGIN or END 892symbols must be non-nil too. 893When BEGIN is non-nil, generate a call to `semantic-lex-start-block' 894when a BEGIN token class is encountered. 895When END is non-nil, generate a call to `semantic-lex-end-block' when 896an END token class is encountered." 897 (let ((class (make-symbol "class")) 898 (form nil)) 899 (dolist (spec specs) 900 (when (car spec) 901 (when (nth 1 spec) 902 (push `((eq ',(nth 1 spec) ,class) 903 (semantic-lex-start-block ',(car spec))) 904 form)) 905 (when (nth 2 spec) 906 (push `((eq ',(nth 2 spec) ,class) 907 (semantic-lex-end-block ',(car spec))) 908 form)))) 909 (when form 910 `((let ((,class (semantic-lex-token-class 911 (car semantic-lex-token-stream)))) 912 (cond ,@(nreverse form)))) 913 ))) 914) 915 916(defmacro semantic-lex-push-token (token &rest blockspecs) 917 "Push TOKEN in the lexical analyzer token stream. 918Return the lexical analysis current end point. 919If optional arguments BLOCKSPECS is non-nil, it specifies to process 920collapsed block tokens. See `semantic-lex-expand-block-specs' for 921more details. 922This macro should only be called within the bounds of 923`define-lex-analyzer'. It changes the values of the lexical analyzer 924variables `token-stream' and `semantic-lex-end-point'. If you need to 925move `semantic-lex-end-point' somewhere else, just modify this 926variable after calling `semantic-lex-push-token'." 927 `(progn 928 (push ,token semantic-lex-token-stream) 929 ,@(semantic-lex-expand-block-specs blockspecs) 930 (setq semantic-lex-end-point 931 (semantic-lex-token-end (car semantic-lex-token-stream))) 932 )) 933 934(defsubst semantic-lex-token-class (token) 935 "Fetch the class of the lexical token TOKEN. 936See also the function `semantic-lex-token'." 937 (car token)) 938 939(defsubst semantic-lex-token-text (token) 940 "Fetch the text associated with the lexical token TOKEN. 941See also the function `semantic-lex-token'." 942 (if (stringp (car (cdr token))) 943 (car (cdr token)) 944 (buffer-substring-no-properties 945 (semantic-lex-token-start token) 946 (semantic-lex-token-end token)))) 947 948(defun semantic-lex-init () 949 "Initialize any lexical state for this buffer." 950 (unless semantic-lex-comment-regex 951 (setq semantic-lex-comment-regex 952 (if comment-start-skip 953 (concat "\\(\\s<\\|" comment-start-skip "\\)") 954 "\\(\\s<\\)"))) 955 ;; Setup the lexer syntax-table 956 (setq semantic-lex-syntax-table (copy-syntax-table (syntax-table))) 957 (dolist (mod semantic-lex-syntax-modifications) 958 (modify-syntax-entry 959 (car mod) (nth 1 mod) semantic-lex-syntax-table))) 960 961;;;###autoload 962(define-overloadable-function semantic-lex (start end &optional depth length) 963 "Lexically analyze text in the current buffer between START and END. 964Optional argument DEPTH indicates at what level to scan over entire 965lists. The last argument, LENGTH specifies that `semantic-lex' 966should only return LENGTH tokens. The return value is a token stream. 967Each element is a list, such of the form 968 (symbol start-expression . end-expression) 969where SYMBOL denotes the token type. 970See `semantic-lex-tokens' variable for details on token types. END 971does not mark the end of the text scanned, only the end of the 972beginning of text scanned. Thus, if a string extends past END, the 973end of the return token will be larger than END. To truly restrict 974scanning, use `narrow-to-region'." 975 (funcall semantic-lex-analyzer start end depth length)) 976 977(defsubst semantic-lex-buffer (&optional depth) 978 "Lex the current buffer. 979Optional argument DEPTH is the depth to scan into lists." 980 (semantic-lex (point-min) (point-max) depth)) 981 982(defsubst semantic-lex-list (semlist depth) 983 "Lex the body of SEMLIST to DEPTH." 984 (semantic-lex (semantic-lex-token-start semlist) 985 (semantic-lex-token-end semlist) 986 depth)) 987 988;;; Collapsed block tokens delimited by any tokens. 989;; 990(defun semantic-lex-start-block (syntax) 991 "Mark the last read token as the beginning of a SYNTAX block." 992 (if (or (not semantic-lex-maximum-depth) 993 (< semantic-lex-current-depth semantic-lex-maximum-depth)) 994 (setq semantic-lex-current-depth (1+ semantic-lex-current-depth)) 995 (push (list syntax (car semantic-lex-token-stream)) 996 semantic-lex-block-stack))) 997 998(defun semantic-lex-end-block (syntax) 999 "Process the end of a previously marked SYNTAX block. 1000That is, collapse the tokens inside that block, including the 1001beginning and end of block tokens, into a high level block token of 1002class SYNTAX. 1003The token at beginning of block is the one marked by a previous call 1004to `semantic-lex-start-block'. The current token is the end of block. 1005The collapsed tokens are saved in `semantic-lex-block-streams'." 1006 (if (null semantic-lex-block-stack) 1007 (setq semantic-lex-current-depth (1- semantic-lex-current-depth)) 1008 (let* ((stream semantic-lex-token-stream) 1009 (blk (pop semantic-lex-block-stack)) 1010 (bstream (cdr blk)) 1011 (first (car bstream)) 1012 (last (pop stream)) ;; The current token mark the EOBLK 1013 tok) 1014 (if (not (eq (car blk) syntax)) 1015 ;; SYNTAX doesn't match the syntax of the current block in 1016 ;; the stack. So we encountered the end of the SYNTAX block 1017 ;; before the end of the current one in the stack which is 1018 ;; signaled unterminated. 1019 (semantic-lex-unterminated-syntax-detected (car blk)) 1020 ;; Move tokens found inside the block from the main stream 1021 ;; into a separate block stream. 1022 (while (and stream (not (eq (setq tok (pop stream)) first))) 1023 (push tok bstream)) 1024 ;; The token marked as beginning of block was not encountered. 1025 ;; This should not happen! 1026 (or (eq tok first) 1027 (error "Token %S not found at beginning of block `%s'" 1028 first syntax)) 1029 ;; Save the block stream for future reuse, to avoid to redo 1030 ;; the lexical analysis of the block content! 1031 ;; Anchor the block stream with its start position, so we can 1032 ;; use: (cdr (assq start semantic-lex-block-streams)) to 1033 ;; quickly retrieve the lexical stream associated to a block. 1034 (setcar blk (semantic-lex-token-start first)) 1035 (setcdr blk (nreverse bstream)) 1036 (push blk semantic-lex-block-streams) 1037 ;; In the main stream, replace the tokens inside the block by 1038 ;; a high level block token of class SYNTAX. 1039 (setq semantic-lex-token-stream stream) 1040 (semantic-lex-push-token 1041 (semantic-lex-token 1042 syntax (car blk) (semantic-lex-token-end last))) 1043 )))) 1044 1045;;; Analyzer creation macros 1046;; 1047;; An individual analyzer is a condition and code that goes with it. 1048;; 1049;; Created analyzers become variables with the code associated with them 1050;; as the symbol value. These analyzers are assembled into a lexer 1051;; to create new lexical analyzers. 1052 1053(defcustom semantic-lex-debug-analyzers nil 1054 "Non-nil means to debug analyzers with syntax protection. 1055Only in effect if `debug-on-error' is also non-nil." 1056 :group 'semantic 1057 :type 'boolean) 1058 1059(defmacro semantic-lex-unterminated-syntax-protection (syntax &rest forms) 1060 "For SYNTAX, execute FORMS with protection for unterminated syntax. 1061If FORMS throws an error, treat this as a syntax problem, and 1062execute the unterminated syntax code. FORMS should return a position. 1063Regardless of an error, the cursor should be moved to the end of 1064the desired syntax, and a position returned. 1065If `debug-on-error' is set, errors are not caught, so that you can 1066debug them. 1067Avoid using a large FORMS since it is duplicated." 1068 (declare (indent 1) (debug t)) 1069 `(if (and debug-on-error semantic-lex-debug-analyzers) 1070 (progn ,@forms) 1071 (condition-case nil 1072 (progn ,@forms) 1073 (error 1074 (semantic-lex-unterminated-syntax-detected ,syntax))))) 1075 1076(defmacro define-lex-analyzer (name doc condition &rest forms) 1077 "Create a single lexical analyzer NAME with DOC. 1078When an analyzer is called, the current buffer and point are 1079positioned in a buffer at the location to be analyzed. 1080CONDITION is an expression which returns t if FORMS should be run. 1081Within the bounds of CONDITION and FORMS, the use of backquote 1082can be used to evaluate expressions at compile time. 1083While forms are running, the following variables will be locally bound: 1084 `semantic-lex-analysis-bounds' - The bounds of the current analysis. 1085 of the form (START . END) 1086 `semantic-lex-maximum-depth' - The maximum depth of semantic-list 1087 for the current analysis. 1088 `semantic-lex-current-depth' - The current depth of `semantic-list' that has 1089 been descended. 1090 `semantic-lex-end-point' - End Point after match. 1091 Analyzers should set this to a buffer location if their 1092 match string does not represent the end of the matched text. 1093 `semantic-lex-token-stream' - The token list being collected. 1094 Add new lexical tokens to this list. 1095Proper action in FORMS is to move the value of `semantic-lex-end-point' to 1096after the location of the analyzed entry, and to add any discovered tokens 1097at the beginning of `semantic-lex-token-stream'. 1098This can be done by using `semantic-lex-push-token'." 1099 (declare (debug (&define name stringp form def-body)) (indent 1)) 1100 `(eval-and-compile 1101 ;; This is the real info used by `define-lex' (via semantic-lex-one-token). 1102 (defconst ,name '(,condition ,@forms) ,doc) 1103 ;; Build a single lexical analyzer function, so the doc for 1104 ;; function help is automatically provided, and perhaps the 1105 ;; function could be useful for testing and debugging one 1106 ;; analyzer. 1107 (defun ,name () 1108 ,doc 1109 (let ((semantic-lex-token-stream nil) 1110 (semantic-lex-end-point (point)) 1111 (semantic-lex-analysis-bounds (cons (point) (point-max))) 1112 (semantic-lex-current-depth 0) 1113 (semantic-lex-maximum-depth semantic-lex-depth)) 1114 (when ,condition ,@forms) 1115 semantic-lex-token-stream)))) 1116 1117(defmacro define-lex-regex-analyzer (name doc regexp &rest forms) 1118 "Create a lexical analyzer with NAME and DOC that will match REGEXP. 1119FORMS are evaluated upon a successful match. 1120See `define-lex-analyzer' for more about analyzers." 1121 (declare (debug (&define name stringp form def-body)) (indent 1)) 1122 `(define-lex-analyzer ,name 1123 ,doc 1124 (looking-at ,regexp) 1125 ,@forms 1126 )) 1127 1128(defmacro define-lex-simple-regex-analyzer (name doc regexp toksym 1129 &optional index 1130 &rest forms) 1131 "Create a lexical analyzer with NAME and DOC that match REGEXP. 1132TOKSYM is the symbol to use when creating a semantic lexical token. 1133INDEX is the index into the match that defines the bounds of the token. 1134Index should be a plain integer, and not specified in the macro as an 1135expression. 1136FORMS are evaluated upon a successful match BEFORE the new token is 1137created. It is valid to ignore FORMS. 1138See `define-lex-analyzer' for more about analyzers." 1139 (declare (debug 1140 (&define name stringp form symbolp [ &optional form ] def-body)) 1141 (indent 1)) 1142 `(define-lex-analyzer ,name 1143 ,doc 1144 (looking-at ,regexp) 1145 ,@forms 1146 (semantic-lex-push-token 1147 (semantic-lex-token ,toksym 1148 (match-beginning ,(or index 0)) 1149 (match-end ,(or index 0)))) 1150 )) 1151 1152(defmacro define-lex-block-analyzer (name doc spec1 &rest specs) 1153 "Create a lexical analyzer NAME for paired delimiters blocks. 1154It detects a paired delimiters block or the corresponding open or 1155close delimiter depending on the value of the variable 1156`semantic-lex-current-depth'. DOC is the documentation string of the lexical 1157analyzer. SPEC1 and SPECS specify the token symbols and open, close 1158delimiters used. Each SPEC has the form: 1159 1160\(BLOCK-SYM (OPEN-DELIM OPEN-SYM) (CLOSE-DELIM CLOSE-SYM)) 1161 1162where BLOCK-SYM is the symbol returned in a block token. OPEN-DELIM 1163and CLOSE-DELIM are respectively the open and close delimiters 1164identifying a block. OPEN-SYM and CLOSE-SYM are respectively the 1165symbols returned in open and close tokens." 1166 (declare (debug (&define name stringp form (&rest form))) 1167 (indent 1)) 1168 (let ((specs (cons spec1 specs)) 1169 spec open olist clist) 1170 (while specs 1171 (setq spec (car specs) 1172 specs (cdr specs) 1173 open (nth 1 spec) 1174 ;; build alist ((OPEN-DELIM OPEN-SYM BLOCK-SYM) ...) 1175 olist (cons (list (car open) (cadr open) (car spec)) olist) 1176 ;; build alist ((CLOSE-DELIM CLOSE-SYM) ...) 1177 clist (cons (nth 2 spec) clist))) 1178 `(define-lex-analyzer ,name 1179 ,doc 1180 (and 1181 (looking-at "\\(\\s(\\|\\s)\\)") 1182 (let ((text (match-string 0)) match) 1183 (cond 1184 ((setq match (assoc text ',olist)) 1185 (if (or (not semantic-lex-maximum-depth) 1186 (< semantic-lex-current-depth semantic-lex-maximum-depth)) 1187 (progn 1188 (setq semantic-lex-current-depth (1+ semantic-lex-current-depth)) 1189 (semantic-lex-push-token 1190 (semantic-lex-token 1191 (nth 1 match) 1192 (match-beginning 0) (match-end 0)))) 1193 (semantic-lex-push-token 1194 (semantic-lex-token 1195 (nth 2 match) 1196 (match-beginning 0) 1197 (save-excursion 1198 (semantic-lex-unterminated-syntax-protection (nth 2 match) 1199 (forward-list 1) 1200 (point))) 1201 )) 1202 )) 1203 ((setq match (assoc text ',clist)) 1204 (if (> semantic-lex-current-depth 0) 1205 (progn 1206 (setq semantic-lex-current-depth (1- semantic-lex-current-depth)) 1207 (semantic-lex-push-token 1208 (semantic-lex-token 1209 (nth 1 match) 1210 (match-beginning 0) (match-end 0))))))))) 1211 ))) 1212 1213;;; Analyzers 1214;; 1215;; Pre-defined common analyzers. 1216;; 1217(define-lex-analyzer semantic-lex-default-action 1218 "The default action when no other lexical actions match text. 1219This action will just throw an error." 1220 t 1221 (error "Unmatched Text during Lexical Analysis")) 1222 1223(define-lex-analyzer semantic-lex-beginning-of-line 1224 "Detect and create a beginning of line token (BOL)." 1225 (and (bolp) 1226 ;; Just insert a (bol N . N) token in the token stream, 1227 ;; without moving the point. N is the point at the 1228 ;; beginning of line. 1229 (semantic-lex-push-token (semantic-lex-token 'bol (point) (point))) 1230 nil) ;; CONTINUE 1231 ;; We identify and add the BOL token onto the stream, but since 1232 ;; semantic-lex-end-point doesn't move, we always fail CONDITION, and have no 1233 ;; FORMS body. 1234 nil) 1235 1236(define-lex-simple-regex-analyzer semantic-lex-newline 1237 "Detect and create newline tokens." 1238 "\\s-*\\(\n\\|\\s>\\)" 'newline 1) 1239 1240(define-lex-regex-analyzer semantic-lex-newline-as-whitespace 1241 "Detect and create newline tokens. 1242Use this ONLY if newlines are not whitespace characters (such as when 1243they are comment end characters) AND when you want whitespace tokens." 1244 "\\s-*\\(\n\\|\\s>\\)" 1245 ;; Language wants whitespaces. Create a token for it. 1246 (if (eq (semantic-lex-token-class (car semantic-lex-token-stream)) 1247 'whitespace) 1248 ;; Merge whitespace tokens together if they are adjacent. Two 1249 ;; whitespace tokens may be separated by a comment which is not in 1250 ;; the token stream. 1251 (setcdr (semantic-lex-token-bounds (car semantic-lex-token-stream)) 1252 (match-end 0)) 1253 (semantic-lex-push-token 1254 (semantic-lex-token 1255 'whitespace (match-beginning 0) (match-end 0))))) 1256 1257(define-lex-regex-analyzer semantic-lex-ignore-newline 1258 "Detect and ignore newline tokens. 1259Use this ONLY if newlines are not whitespace characters (such as when 1260they are comment end characters)." 1261 "\\s-*\\(\n\\|\\s>\\)" 1262 (setq semantic-lex-end-point (match-end 0))) 1263 1264(define-lex-regex-analyzer semantic-lex-whitespace 1265 "Detect and create whitespace tokens." 1266 ;; catch whitespace when needed 1267 "\\s-+" 1268 ;; Language wants whitespaces. Create a token for it. 1269 (if (eq (semantic-lex-token-class (car semantic-lex-token-stream)) 1270 'whitespace) 1271 ;; Merge whitespace tokens together if they are adjacent. Two 1272 ;; whitespace tokens may be separated by a comment which is not in 1273 ;; the token stream. 1274 (progn 1275 (setq semantic-lex-end-point (match-end 0)) 1276 (setcdr (semantic-lex-token-bounds (car semantic-lex-token-stream)) 1277 semantic-lex-end-point)) 1278 (semantic-lex-push-token 1279 (semantic-lex-token 1280 'whitespace (match-beginning 0) (match-end 0))))) 1281 1282(define-lex-regex-analyzer semantic-lex-ignore-whitespace 1283 "Detect and skip over whitespace tokens." 1284 ;; catch whitespace when needed 1285 "\\s-+" 1286 ;; Skip over the detected whitespace, do not create a token for it. 1287 (setq semantic-lex-end-point (match-end 0))) 1288 1289(define-lex-simple-regex-analyzer semantic-lex-number 1290 "Detect and create number tokens. 1291See `semantic-lex-number-expression' for details on matching numbers, 1292and number formats." 1293 semantic-lex-number-expression 'number) 1294 1295(define-lex-regex-analyzer semantic-lex-symbol-or-keyword 1296 "Detect and create symbol and keyword tokens." 1297 "\\(\\sw\\|\\s_\\)+" 1298 (semantic-lex-push-token 1299 (semantic-lex-token 1300 (or (semantic-lex-keyword-p (match-string 0)) 'symbol) 1301 (match-beginning 0) (match-end 0)))) 1302 1303(define-lex-simple-regex-analyzer semantic-lex-charquote 1304 "Detect and create charquote tokens." 1305 ;; Character quoting characters (ie, \n as newline) 1306 "\\s\\+" 'charquote) 1307 1308(define-lex-simple-regex-analyzer semantic-lex-punctuation 1309 "Detect and create punctuation tokens." 1310 "\\(\\s.\\|\\s$\\|\\s'\\)" 'punctuation) 1311 1312(define-lex-analyzer semantic-lex-punctuation-type 1313 "Detect and create a punctuation type token. 1314Recognized punctuation is defined in the current table of lexical 1315types, as the value of the `punctuation' token type." 1316 (and (looking-at "\\(\\s.\\|\\s$\\|\\s'\\)+") 1317 (let* ((key (match-string 0)) 1318 (pos (match-beginning 0)) 1319 (end (match-end 0)) 1320 (len (- end pos)) 1321 (lst (semantic-lex-type-value "punctuation" t)) 1322 (def (car lst)) ;; default lexical symbol or nil 1323 (lst (cdr lst)) ;; alist of (LEX-SYM . PUNCT-STRING) 1324 (elt nil)) 1325 (if lst 1326 ;; Starting with the longest one, search if the 1327 ;; punctuation string is defined for this language. 1328 (while (and (> len 0) (not (setq elt (rassoc key lst)))) 1329 (setq len (1- len) 1330 key (substring key 0 len)))) 1331 (if elt ;; Return the punctuation token found 1332 (semantic-lex-push-token 1333 (semantic-lex-token (car elt) pos (+ pos len))) 1334 (if def ;; Return a default generic token 1335 (semantic-lex-push-token 1336 (semantic-lex-token def pos end)) 1337 ;; Nothing match 1338 ))))) 1339 1340(define-lex-regex-analyzer semantic-lex-paren-or-list 1341 "Detect open parenthesis. 1342Return either a paren token or a semantic list token depending on 1343`semantic-lex-current-depth'." 1344 "\\s(" 1345 (if (or (not semantic-lex-maximum-depth) 1346 (< semantic-lex-current-depth semantic-lex-maximum-depth)) 1347 (progn 1348 (setq semantic-lex-current-depth (1+ semantic-lex-current-depth)) 1349 (semantic-lex-push-token 1350 (semantic-lex-token 1351 'open-paren (match-beginning 0) (match-end 0)))) 1352 (semantic-lex-push-token 1353 (semantic-lex-token 1354 'semantic-list (match-beginning 0) 1355 (save-excursion 1356 (semantic-lex-unterminated-syntax-protection 'semantic-list 1357 (forward-list 1) 1358 (point)) 1359 ))) 1360 )) 1361 1362(define-lex-simple-regex-analyzer semantic-lex-open-paren 1363 "Detect and create an open parenthesis token." 1364 "\\s(" 'open-paren 0 (setq semantic-lex-current-depth (1+ semantic-lex-current-depth))) 1365 1366(define-lex-simple-regex-analyzer semantic-lex-close-paren 1367 "Detect and create a close parenthesis token." 1368 "\\s)" 'close-paren 0 (setq semantic-lex-current-depth (1- semantic-lex-current-depth))) 1369 1370(define-lex-regex-analyzer semantic-lex-string 1371 "Detect and create a string token." 1372 "\\s\"" 1373 ;; Zing to the end of this string. 1374 (semantic-lex-push-token 1375 (semantic-lex-token 1376 'string (point) 1377 (save-excursion 1378 (semantic-lex-unterminated-syntax-protection 'string 1379 (forward-sexp 1) 1380 (point)) 1381 )))) 1382 1383(define-lex-regex-analyzer semantic-lex-comments 1384 "Detect and create a comment token." 1385 semantic-lex-comment-regex 1386 (save-excursion 1387 (forward-comment 1) 1388 ;; Generate newline token if enabled 1389 (if (bolp) (backward-char 1)) 1390 (setq semantic-lex-end-point (point)) 1391 ;; Language wants comments or want them as whitespaces, 1392 ;; link them together. 1393 (if (eq (semantic-lex-token-class (car semantic-lex-token-stream)) 'comment) 1394 (setcdr (semantic-lex-token-bounds (car semantic-lex-token-stream)) 1395 semantic-lex-end-point) 1396 (semantic-lex-push-token 1397 (semantic-lex-token 1398 'comment (match-beginning 0) semantic-lex-end-point))))) 1399 1400(define-lex-regex-analyzer semantic-lex-comments-as-whitespace 1401 "Detect comments and create a whitespace token." 1402 semantic-lex-comment-regex 1403 (save-excursion 1404 (forward-comment 1) 1405 ;; Generate newline token if enabled 1406 (if (bolp) (backward-char 1)) 1407 (setq semantic-lex-end-point (point)) 1408 ;; Language wants comments or want them as whitespaces, 1409 ;; link them together. 1410 (if (eq (semantic-lex-token-class (car semantic-lex-token-stream)) 'whitespace) 1411 (setcdr (semantic-lex-token-bounds (car semantic-lex-token-stream)) 1412 semantic-lex-end-point) 1413 (semantic-lex-push-token 1414 (semantic-lex-token 1415 'whitespace (match-beginning 0) semantic-lex-end-point))))) 1416 1417(define-lex-regex-analyzer semantic-lex-ignore-comments 1418 "Detect and create a comment token." 1419 semantic-lex-comment-regex 1420 (let ((comment-start-point (point))) 1421 (forward-comment 1) 1422 (if (eq (point) comment-start-point) 1423 ;; In this case our start-skip string failed 1424 ;; to work properly. Lets try and move over 1425 ;; whatever white space we matched to begin 1426 ;; with. 1427 (skip-syntax-forward "-.'" (point-at-eol)) 1428 ;; We may need to back up so newlines or whitespace is generated. 1429 (if (bolp) 1430 (backward-char 1))) 1431 (if (eq (point) comment-start-point) 1432 (error "Strange comment syntax prevents lexical analysis")) 1433 (setq semantic-lex-end-point (point)))) 1434 1435;;; Comment lexer 1436;; 1437;; Predefined lexers that could be used instead of creating new 1438;; analyzers. 1439 1440(define-lex semantic-comment-lexer 1441 "A simple lexical analyzer that handles comments. 1442This lexer will only return comment tokens. It is the default lexer 1443used by `semantic-find-doc-snarf-comment' to snarf up the comment at 1444point." 1445 semantic-lex-ignore-whitespace 1446 semantic-lex-ignore-newline 1447 semantic-lex-comments 1448 semantic-lex-default-action) 1449 1450;;; Test Lexer 1451;; 1452(define-lex semantic-simple-lexer 1453 "A simple lexical analyzer that handles simple buffers. 1454This lexer ignores comments and whitespace, and will return 1455syntax as specified by the syntax table." 1456 semantic-lex-ignore-whitespace 1457 semantic-lex-ignore-newline 1458 semantic-lex-number 1459 semantic-lex-symbol-or-keyword 1460 semantic-lex-charquote 1461 semantic-lex-paren-or-list 1462 semantic-lex-close-paren 1463 semantic-lex-string 1464 semantic-lex-ignore-comments 1465 semantic-lex-punctuation 1466 semantic-lex-default-action) 1467 1468;;; Analyzers generated from grammar. 1469;; 1470;; Some analyzers are hand written. Analyzers created with these 1471;; functions are generated from the grammar files. 1472 1473(defmacro define-lex-keyword-type-analyzer (name doc syntax) 1474 "Define a keyword type analyzer NAME with DOC string. 1475SYNTAX is the regexp that matches a keyword syntactic expression." 1476 (declare (indent 1)) 1477 (let ((key (make-symbol "key"))) 1478 `(define-lex-analyzer ,name 1479 ,doc 1480 (and (looking-at ,syntax) 1481 (let ((,key (semantic-lex-keyword-p (match-string 0)))) 1482 (when ,key 1483 (semantic-lex-push-token 1484 (semantic-lex-token 1485 ,key (match-beginning 0) (match-end 0))))))) 1486 )) 1487 1488(defmacro define-lex-sexp-type-analyzer (name doc syntax token) 1489 "Define a sexp type analyzer NAME with DOC string. 1490SYNTAX is the regexp that matches the beginning of the s-expression. 1491TOKEN is the lexical token returned when SYNTAX matches." 1492 (declare (indent 1)) 1493 `(define-lex-regex-analyzer ,name 1494 ,doc 1495 ,syntax 1496 (semantic-lex-push-token 1497 (semantic-lex-token 1498 ,token (point) 1499 (save-excursion 1500 (semantic-lex-unterminated-syntax-protection ,token 1501 (forward-sexp 1) 1502 (point)))))) 1503 ) 1504 1505(defmacro define-lex-regex-type-analyzer (name doc syntax matches default) 1506 "Define a regexp type analyzer NAME with DOC string. 1507SYNTAX is the regexp that matches a syntactic expression. 1508MATCHES is an alist of lexical elements used to refine the syntactic 1509expression. 1510DEFAULT is the default lexical token returned when no MATCHES." 1511 (declare (indent 1)) 1512 (if matches 1513 (let* ((val (make-symbol "val")) 1514 (lst (make-symbol "lst")) 1515 (elt (make-symbol "elt")) 1516 (pos (make-symbol "pos")) 1517 (end (make-symbol "end"))) 1518 `(define-lex-analyzer ,name 1519 ,doc 1520 (and (looking-at ,syntax) 1521 (let* ((,val (match-string 0)) 1522 (,pos (match-beginning 0)) 1523 (,end (match-end 0)) 1524 (,lst ,matches) 1525 ,elt) 1526 (while (and ,lst (not ,elt)) 1527 (if (string-match (cdar ,lst) ,val) 1528 (setq ,elt (caar ,lst)) 1529 (setq ,lst (cdr ,lst)))) 1530 (semantic-lex-push-token 1531 (semantic-lex-token (or ,elt ,default) ,pos ,end)))) 1532 )) 1533 `(define-lex-simple-regex-analyzer ,name 1534 ,doc 1535 ,syntax ,default) 1536 )) 1537 1538(defmacro define-lex-string-type-analyzer (name doc syntax matches default) 1539 "Define a string type analyzer NAME with DOC string. 1540SYNTAX is the regexp that matches a syntactic expression. 1541MATCHES is an alist of lexical elements used to refine the syntactic 1542expression. 1543DEFAULT is the default lexical token returned when no MATCHES." 1544 (declare (indent 1)) 1545 (if matches 1546 (let* ((val (make-symbol "val")) 1547 (lst (make-symbol "lst")) 1548 (elt (make-symbol "elt")) 1549 (pos (make-symbol "pos")) 1550 (end (make-symbol "end")) 1551 (len (make-symbol "len"))) 1552 `(define-lex-analyzer ,name 1553 ,doc 1554 (and (looking-at ,syntax) 1555 (let* ((,val (match-string 0)) 1556 (,pos (match-beginning 0)) 1557 (,end (match-end 0)) 1558 (,len (- ,end ,pos)) 1559 (,lst ,matches) 1560 ,elt) 1561 ;; Starting with the longest one, search if a lexical 1562 ;; value match a token defined for this language. 1563 (while (and (> ,len 0) (not (setq ,elt (rassoc ,val ,lst)))) 1564 (setq ,len (1- ,len) 1565 ,val (substring ,val 0 ,len))) 1566 (when ,elt ;; Adjust token end position. 1567 (setq ,elt (car ,elt) 1568 ,end (+ ,pos ,len))) 1569 (semantic-lex-push-token 1570 (semantic-lex-token (or ,elt ,default) ,pos ,end)))) 1571 )) 1572 `(define-lex-simple-regex-analyzer ,name 1573 ,doc 1574 ,syntax ,default) 1575 )) 1576 1577(defmacro define-lex-block-type-analyzer (name doc syntax matches) 1578 "Define a block type analyzer NAME with DOC string. 1579 1580SYNTAX is the regexp that matches block delimiters, typically the 1581open (`\\\\s(') and close (`\\\\s)') parenthesis syntax classes. 1582 1583MATCHES is a pair (OPEN-SPECS . CLOSE-SPECS) that defines blocks. 1584 1585 OPEN-SPECS is a list of (OPEN-DELIM OPEN-TOKEN BLOCK-TOKEN) elements 1586 where: 1587 1588 OPEN-DELIM is a string: the block open delimiter character. 1589 1590 OPEN-TOKEN is the lexical token class associated to the OPEN-DELIM 1591 delimiter. 1592 1593 BLOCK-TOKEN is the lexical token class associated to the block 1594 that starts at the OPEN-DELIM delimiter. 1595 1596 CLOSE-SPECS is a list of (CLOSE-DELIM CLOSE-TOKEN) elements where: 1597 1598 CLOSE-DELIM is a string: the block end delimiter character. 1599 1600 CLOSE-TOKEN is the lexical token class associated to the 1601 CLOSE-DELIM delimiter. 1602 1603Each element in OPEN-SPECS must have a corresponding element in 1604CLOSE-SPECS. 1605 1606The lexer will return a BLOCK-TOKEN token when the value of 1607`semantic-lex-current-depth' is greater than or equal to the maximum 1608depth of parenthesis tracking (see also the function `semantic-lex'). 1609Otherwise it will return OPEN-TOKEN and CLOSE-TOKEN tokens. 1610 1611TO DO: Put the following in the developer's guide and just put a 1612reference here. 1613 1614In the grammar: 1615 1616The value of a block token must be a string that contains a readable 1617sexp of the form: 1618 1619 \"(OPEN-TOKEN CLOSE-TOKEN)\" 1620 1621OPEN-TOKEN and CLOSE-TOKEN represent the block delimiters, and must be 1622lexical tokens of respectively `open-paren' and `close-paren' types. 1623Their value is the corresponding delimiter character as a string. 1624 1625Here is a small example to analyze a parenthesis block: 1626 1627 %token <block> PAREN_BLOCK \"(LPAREN RPAREN)\" 1628 %token <open-paren> LPAREN \"(\" 1629 %token <close-paren> RPAREN \")\" 1630 1631When the lexer encounters the open-paren delimiter \"(\": 1632 1633 - If the maximum depth of parenthesis tracking is not reached (that 1634 is, current depth < max depth), it returns a (LPAREN start . end) 1635 token, then continue analysis inside the block. Later, when the 1636 corresponding close-paren delimiter \")\" will be encountered, it 1637 will return a (RPAREN start . end) token. 1638 1639 - If the maximum depth of parenthesis tracking is reached (current 1640 depth >= max depth), it returns the whole parenthesis block as 1641 a (PAREN_BLOCK start . end) token." 1642 (declare (indent 1)) 1643 (let* ((val (make-symbol "val")) 1644 (lst (make-symbol "lst")) 1645 (elt (make-symbol "elt"))) 1646 `(define-lex-analyzer ,name 1647 ,doc 1648 (and 1649 (looking-at ,syntax) ;; "\\(\\s(\\|\\s)\\)" 1650 (let ((,val (match-string 0)) 1651 (,lst ,matches) 1652 ,elt) 1653 (cond 1654 ((setq ,elt (assoc ,val (car ,lst))) 1655 (if (or (not semantic-lex-maximum-depth) 1656 (< semantic-lex-current-depth semantic-lex-maximum-depth)) 1657 (progn 1658 (setq semantic-lex-current-depth (1+ semantic-lex-current-depth)) 1659 (semantic-lex-push-token 1660 (semantic-lex-token 1661 (nth 1 ,elt) 1662 (match-beginning 0) (match-end 0)))) 1663 (semantic-lex-push-token 1664 (semantic-lex-token 1665 (nth 2 ,elt) 1666 (match-beginning 0) 1667 (save-excursion 1668 (semantic-lex-unterminated-syntax-protection (nth 2 ,elt) 1669 (forward-list 1) 1670 (point))))))) 1671 ((setq ,elt (assoc ,val (cdr ,lst))) 1672 (setq semantic-lex-current-depth (1- semantic-lex-current-depth)) 1673 (semantic-lex-push-token 1674 (semantic-lex-token 1675 (nth 1 ,elt) 1676 (match-beginning 0) (match-end 0)))) 1677 )))) 1678 )) 1679 1680;;; Lexical Safety 1681;; 1682;; The semantic lexers, unlike other lexers, can throw errors on 1683;; unbalanced syntax. Since editing is all about changing text 1684;; we need to provide a convenient way to protect against syntactic 1685;; inequalities. 1686 1687(defmacro semantic-lex-catch-errors (symbol &rest forms) 1688 "Using SYMBOL, execute FORMS catching lexical errors. 1689If FORMS results in a call to the parser that throws a lexical error, 1690the error will be caught here without the buffer's cache being thrown 1691out of date. 1692If there is an error, the syntax that failed is returned. 1693If there is no error, then the last value of FORMS is returned." 1694 (declare (indent 1) (debug (symbolp def-body))) 1695 (let ((ret (make-symbol "ret")) 1696 (syntax (make-symbol "syntax")) 1697 (start (make-symbol "start")) 1698 (end (make-symbol "end"))) 1699 `(let* ((semantic-lex-unterminated-syntax-end-function 1700 (lambda (,syntax ,start ,end) 1701 (throw ',symbol ,syntax))) 1702 (,ret (catch ',symbol 1703 (save-excursion 1704 ,@forms 1705 nil)))) 1706 ;; Great Sadness. Assume that FORMS execute within the 1707 ;; confines of the current buffer only! Mark this thing 1708 ;; unparsable iff the special symbol was thrown. This 1709 ;; will prevent future calls from parsing, but will allow 1710 ;; then to still return the cache. 1711 (when ,ret 1712 ;; Leave this message off. If an APP using this fcn wants 1713 ;; a message, they can do it themselves. This cleans up 1714 ;; problems with the idle scheduler obscuring useful data. 1715 ;;(message "Buffer not currently parsable (%S)." ,ret) 1716 (semantic-parse-tree-unparseable)) 1717 ,ret))) 1718 1719 1720;;; Compatibility with Semantic 1.x lexical analysis 1721 1722(defvar semantic-flex-tokens semantic-lex-tokens 1723 "An alist of semantic token types. 1724See variable `semantic-lex-tokens'.") 1725(make-obsolete-variable 'semantic-flex-tokens 1726 'semantic-lex-tokens "28.1") 1727 1728(defvar semantic-flex-unterminated-syntax-end-function 1729 (lambda (_syntax _syntax-start flex-end) flex-end) 1730 "Function called when unterminated syntax is encountered. 1731This should be set to one function. That function should take three 1732parameters. The SYNTAX, or type of syntax which is unterminated. 1733SYNTAX-START where the broken syntax begins. 1734FLEX-END is where the lexical analysis was asked to end. 1735This function can be used for languages that can intelligently fix up 1736broken syntax, or the exit lexical analysis via `throw' or `signal' 1737when finding unterminated syntax.") 1738(make-obsolete-variable 'semantic-flex-unterminated-syntax-end-function 1739 nil "28.1") 1740 1741(defvar-local semantic-flex-extensions nil 1742 "Buffer local extensions to the lexical analyzer. 1743This should contain an alist with a key of a regex and a data element of 1744a function. The function should both move point, and return a lexical 1745token of the form: 1746 ( TYPE START . END) 1747nil is also a valid return value. 1748TYPE can be any type of symbol, as long as it doesn't occur as a 1749nonterminal in the language definition.") 1750(make-obsolete-variable 'semantic-flex-extensions nil "28.1") 1751 1752(defvar-local semantic-flex-syntax-modifications nil 1753 "Changes to the syntax table for this buffer. 1754These changes are active only while the buffer is being flexed. 1755This is a list where each element has the form: 1756 (CHAR CLASS) 1757CHAR is the char passed to `modify-syntax-entry', 1758and CLASS is the string also passed to `modify-syntax-entry' to define 1759what syntax class CHAR has.") 1760(make-obsolete-variable 'semantic-flex-syntax-modifications nil "28.1") 1761 1762(defvar-local semantic-ignore-comments t 1763 "Default comment handling. 1764The value t means to strip comments when flexing; nil means 1765to keep comments as part of the token stream.") 1766(make-obsolete-variable 'semantic-ignore-comments nil "28.1") 1767 1768(defvar-local semantic-flex-enable-newlines nil 1769 "When flexing, report newlines as syntactic elements. 1770Useful for languages where the newline is a special case terminator. 1771Only set this on a per mode basis, not globally.") 1772(make-obsolete-variable 'semantic-flex-enable-newlines nil "28.1") 1773 1774(defvar-local semantic-flex-enable-whitespace nil 1775 "When flexing, report whitespace as syntactic elements. 1776Useful for languages where the syntax is whitespace dependent. 1777Only set this on a per mode basis, not globally.") 1778(make-obsolete-variable 'semantic-flex-enable-whitespace nil "28.1") 1779 1780(defvar-local semantic-flex-enable-bol nil 1781 "When flexing, report beginning of lines as syntactic elements. 1782Useful for languages like python which are indentation sensitive. 1783Only set this on a per mode basis, not globally.") 1784(make-obsolete-variable 'semantic-flex-enable-bol nil "28.1") 1785 1786(defvar-local semantic-number-expression semantic-lex-number-expression 1787 "See variable `semantic-lex-number-expression'.") 1788(make-obsolete-variable 'semantic-number-expression 1789 'semantic-lex-number-expression "28.1") 1790 1791(defvar-local semantic-flex-depth 0 1792 "Default flexing depth. 1793This specifies how many lists to create tokens in.") 1794(make-obsolete-variable 'semantic-flex-depth nil "28.1") 1795 1796(provide 'semantic/lex) 1797 1798;; Local variables: 1799;; generated-autoload-file: "loaddefs.el" 1800;; generated-autoload-load-name: "semantic/lex" 1801;; End: 1802 1803;;; semantic/lex.el ends here 1804