1header {
2package antlr;
3}
4
5{
6import java.util.Enumeration;
7import java.io.DataInputStream;
8import java.io.InputStream;
9import java.io.FileInputStream;
10import java.io.IOException;
11}
12
13/* ANTLR Translator Generator
14 * Project led by Terence Parr at http://www.cs.usfca.edu
15 * Software rights: http://www.antlr.org/license.html
16 *
17 * $Id: //depot/code/org.antlr/release/antlr-2.7.7/antlr/antlr.g#2 $
18 */
19
20class ANTLRParser extends Parser;
21options {
22	exportVocab=ANTLR;
23	defaultErrorHandler=false;
24	k=2;
25}
26
27tokens {
28	"tokens";
29}
30
31{
32	private static final boolean DEBUG_PARSER = false;
33
34	ANTLRGrammarParseBehavior behavior;
35	Tool antlrTool;
36	protected int blockNesting= -1;
37
38	public ANTLRParser(
39		TokenBuffer tokenBuf,
40		ANTLRGrammarParseBehavior behavior_,
41		Tool tool_
42	) {
43		super(tokenBuf, 1);
44		tokenNames = _tokenNames;
45		behavior = behavior_;
46		antlrTool = tool_;
47	}
48
49        public void reportError(String s) {
50            antlrTool.error(s, getFilename(), -1, -1);
51        }
52
53        public void reportError(RecognitionException e) {
54            reportError(e, e.getErrorMessage());
55        }
56
57        public void reportError(RecognitionException e, String s) {
58            antlrTool.error(s, e.getFilename(), e.getLine(), e.getColumn());
59        }
60
61        public void reportWarning(String s) {
62            antlrTool.warning(s, getFilename(), -1, -1);
63        }
64
65	private boolean lastInRule() throws TokenStreamException {
66		if ( blockNesting==0 && (LA(1)==SEMI || LA(1)==LITERAL_exception || LA(1)==OR) ) {
67			return true;
68		}
69		return false;
70	}
71
72	private void checkForMissingEndRule(Token label) {
73		if ( label.getColumn()==1 ) {
74			antlrTool.warning("did you forget to terminate previous rule?", getFilename(), label.getLine(), label.getColumn());
75		}
76	}
77}
78
79grammar
80   :
81	( 	{
82			n = null;	// RK: prevent certain orders of header actions
83							// overwriting eachother.
84		}
85	 	"header"
86	 	(n:STRING_LITERAL)?
87	 	h:ACTION
88 		{
89			// store the header action
90			// FIXME: 'n' should be checked for validity
91			behavior.refHeaderAction(n,h);
92		}
93	)*
94	( fileOptionsSpec )?
95	( classDef )*
96	EOF
97	;
98	exception catch [RecognitionException ex] {
99        reportError(ex, "rule grammar trapped:\n"+ex.toString());
100		consumeUntil(EOF);
101	}
102
103classDef
104{String doc=null;}
105	:
106	( a:ACTION { behavior.refPreambleAction(a);} )?
107	( d:DOC_COMMENT {doc=d.getText();} )?
108	(	("lexclass" | "class" id "extends" "Lexer" ) => lexerSpec[doc]
109	|	( "class" id "extends" "TreeParser" ) => treeParserSpec[doc]
110	|	parserSpec[doc]
111	)
112	rules
113	{ behavior.endGrammar(); }
114	;
115	exception catch [RecognitionException ex] {
116		if ( ex instanceof NoViableAltException ) {
117			NoViableAltException e = (NoViableAltException)ex;
118			// RK: These probably generate inconsequent error messages...
119			// have to see how this comes out..
120			if ( e.token.getType()==DOC_COMMENT ) {
121				reportError(ex, "JAVADOC comments may only prefix rules and grammars");
122			}
123			else {
124				reportError(ex, "rule classDef trapped:\n"+ex.toString());
125			}
126		}
127		else {
128			reportError(ex, "rule classDef trapped:\n"+ex.toString());
129		}
130		behavior.abortGrammar();
131		boolean consuming = true;
132		// consume everything until the next class definition or EOF
133		while (consuming) {
134			consume();
135			switch(LA(1)) {
136			case LITERAL_class:
137			case LITERAL_lexclass:
138			case EOF:
139				consuming = false;
140				break;
141			}
142		}
143	}
144
145fileOptionsSpec
146{ Token idTok; Token value; }
147	:	OPTIONS
148		(
149			idTok = id
150			ASSIGN
151			value = optionValue
152			{ behavior.setFileOption(idTok, value,getInputState().filename); }
153			SEMI
154		)*
155		RCURLY
156	;
157
158parserOptionsSpec
159{ Token idTok; Token value; }
160	:	OPTIONS
161		(
162			idTok = id
163			ASSIGN
164			value = optionValue
165			{ behavior.setGrammarOption(idTok, value); }
166			SEMI
167		)*
168		RCURLY
169	;
170
171treeParserOptionsSpec
172{ Token idTok; Token value; }
173	:	OPTIONS
174		(
175			idTok = id
176			ASSIGN
177			value = optionValue
178			{ behavior.setGrammarOption(idTok, value); }
179			SEMI
180		)*
181		RCURLY
182	;
183
184lexerOptionsSpec
185{ Token idTok; Token value; BitSet b; }
186	:
187	OPTIONS
188	(	// Special case for vocabulary option because it has a bit-set
189		"charVocabulary"
190		ASSIGN
191		b = charSet
192		SEMI
193		{ behavior.setCharVocabulary(b); }
194
195	|	idTok = id
196		ASSIGN
197		value = optionValue
198		{ behavior.setGrammarOption(idTok, value); }
199		SEMI
200	)*
201	RCURLY
202	;
203
204subruleOptionsSpec
205{ Token idTok; Token value; }
206	:	OPTIONS
207		(	idTok = id
208			ASSIGN
209			value = optionValue
210			{ behavior.setSubruleOption(idTok, value); }
211			SEMI
212		)*
213		RCURLY
214	;
215
216// optionValue returns a Token which may be one of several things:
217//    STRING_LITERAL -- a quoted string
218//    CHAR_LITERAL -- a single quoted character
219//		INT -- an integer
220//		RULE_REF or TOKEN_REF -- an identifier
221optionValue
222returns [ Token retval ]
223{ retval = null; }
224	:	retval = qualifiedID
225	|	sl:STRING_LITERAL { retval = sl; }
226	|	cl:CHAR_LITERAL { retval = cl; }
227	|	il:INT { retval = il; }
228	;
229
230charSet
231returns [ BitSet b ]
232{
233	b = null;
234	BitSet tmpSet = null;
235}
236	:
237		// TODO: generate a bit set
238		b = setBlockElement
239		(
240			OR
241			tmpSet = setBlockElement
242			{ b.orInPlace(tmpSet); }
243		)*
244	;
245
246setBlockElement
247returns [ BitSet b ]
248{
249	b = null;
250	int rangeMin = 0;
251}
252	:
253	c1:CHAR_LITERAL
254	{
255		rangeMin = ANTLRLexer.tokenTypeForCharLiteral(c1.getText());
256		b = BitSet.of(rangeMin);
257	}
258	(
259		RANGE c2:CHAR_LITERAL
260		{
261			int rangeMax = ANTLRLexer.tokenTypeForCharLiteral(c2.getText());
262			if (rangeMax < rangeMin) {
263				antlrTool.error("Malformed range line ", getFilename(), c1.getLine(), c1.getColumn());
264			}
265			for (int i = rangeMin+1; i <= rangeMax; i++) {
266				b.add(i);
267			}
268		}
269	)?
270	;
271
272tokensSpec
273	:	TOKENS
274			(	(	{s1=null;}
275					t1:TOKEN_REF
276					( ASSIGN s1:STRING_LITERAL )?
277					{behavior.defineToken(t1, s1);}
278					(tokensSpecOptions[t1])?
279				|	s3:STRING_LITERAL
280					{behavior.defineToken(null, s3);}
281					(tokensSpecOptions[s3])?
282				)
283				SEMI
284			)+
285		RCURLY
286	;
287
288tokensSpecOptions[Token t]
289{
290	Token o=null, v=null;
291}
292	:	OPEN_ELEMENT_OPTION
293		o=id ASSIGN v=optionValue
294		{behavior.refTokensSpecElementOption(t,o,v);}
295		(
296			SEMI
297			o=id ASSIGN v=optionValue
298			{behavior.refTokensSpecElementOption(t,o,v);}
299		)*
300		CLOSE_ELEMENT_OPTION
301	;
302
303superClass returns [String sup]
304{sup=null;}
305	:	LPAREN
306			{
307			sup = LT(1).getText();
308			sup = StringUtils.stripFrontBack(sup, "\"", "\"");
309			}
310			(STRING_LITERAL)
311		RPAREN
312	;
313
314parserSpec[String doc]
315{
316	Token idTok;
317	String sup=null;
318}
319	:	"class"
320		idTok = id
321		(	"extends" "Parser"
322			(sup=superClass)?
323		|	{
324			antlrTool.warning("use 'class X extends Parser'", getFilename(), idTok.getLine(), idTok.getColumn());
325//			System.out.println("warning: line " +
326//				idTok.getLine() + ": use 'class X extends Parser'");
327			}
328		)
329		{behavior.startParser(getFilename(), idTok, sup, doc);}
330		SEMI
331		(parserOptionsSpec)?
332		{ behavior.endOptions(); }
333		(tokensSpec)?
334		( a:ACTION {behavior.refMemberAction(a);} )?
335	;
336
337lexerSpec[String doc]
338{
339	Token idTok;
340	String sup=null;
341}
342	:   (	lc:"lexclass"
343			idTok = id
344			{
345				antlrTool.warning("lexclass' is deprecated; use 'class X extends Lexer'",
346								 getFilename(), lc.getLine(), lc.getColumn());
347//				System.out.println("warning: line " + lc.getLine() + ": 'lexclass' is deprecated; use 'class X extends Lexer'");
348			}
349		|	"class"
350			idTok = id
351			"extends"
352			"Lexer"
353			(sup=superClass)?
354		)
355		{behavior.startLexer(getFilename(), idTok,sup,doc);}
356		SEMI
357		(lexerOptionsSpec)?
358		{ behavior.endOptions(); }
359		(tokensSpec)?
360		( a:ACTION {behavior.refMemberAction(a);} )?
361	;
362
363treeParserSpec[String doc]
364{
365	Token idTok;
366	String sup=null;
367}
368	:	"class"
369		idTok = id
370		"extends"
371		"TreeParser"
372		(sup=superClass)?
373		{behavior.startTreeWalker(getFilename(), idTok,sup,doc);}
374		SEMI
375		(treeParserOptionsSpec)?
376		{ behavior.endOptions(); }
377		(tokensSpec)?
378		( a:ACTION {behavior.refMemberAction(a);} )?
379	;
380
381rules
382    :   (
383			options {
384				// limitation of appox LL(k) says ambig upon
385				// DOC_COMMENT TOKEN_REF, but that's an impossible sequence
386				warnWhenFollowAmbig=false;
387			}
388		:	rule
389		)+
390    ;
391
392rule
393{
394	String access="public";
395	Token idTok;
396	String doc=null;
397	boolean ruleAutoGen = true;
398	blockNesting = -1;	// block increments, so -1 to make rule at level 0
399}
400	:
401	(	d:DOC_COMMENT	{doc=d.getText();}
402	)?
403	(	p1:"protected"	{access=p1.getText();}
404	|	p2:"public"		{access=p2.getText();}
405	|	p3:"private"	{access=p3.getText();}
406	)?
407	idTok = id
408	( BANG { ruleAutoGen = false; } )?
409	{
410		behavior.defineRuleName(idTok, access, ruleAutoGen, doc);
411	}
412	( aa:ARG_ACTION { behavior.refArgAction(aa); }  )?
413	( "returns" rt:ARG_ACTION { behavior.refReturnAction(rt); } )?
414	( throwsSpec )?
415	( ruleOptionsSpec )?
416	(a:ACTION {behavior.refInitAction(a);})?
417	COLON block SEMI
418	( exceptionGroup )?
419	{behavior.endRule(idTok.getText());}
420	;
421/*
422	//
423	// for now, syntax error in rule aborts the whole grammar
424	//
425	exception catch [ParserException ex] {
426		behavior.abortRule(idTok);
427		behavior.hasError();
428		// Consume until something that looks like end of a rule
429		consume();
430		while (LA(1) != SEMI && LA(1) != EOF) {
431			consume();
432		}
433		consume();
434	}
435*/
436
437ruleOptionsSpec
438{ Token idTok; Token value; }
439	:	OPTIONS
440		(
441			idTok = id
442			ASSIGN
443			value = optionValue
444			{ behavior.setRuleOption(idTok, value); }
445			SEMI
446		)*
447		RCURLY
448	;
449
450throwsSpec
451{
452	String t=null;
453	Token a,b;
454}
455	:	"throws" a=id {t=a.getText();}
456		( COMMA b=id {t+=","+b.getText();} )*
457		{ behavior.setUserExceptions(t);	}
458	;
459
460block
461    :   {blockNesting++;}
462		alternative ( OR alternative )*
463		{blockNesting--;}
464    ;
465
466alternative
467{ boolean altAutoGen = true; }
468    :
469		(BANG { altAutoGen=false;} )?
470		{behavior.beginAlt(altAutoGen);}
471		( element )* ( exceptionSpecNoLabel )?
472		{behavior.endAlt();}
473    ;
474
475exceptionGroup
476	:	{ behavior.beginExceptionGroup(); }
477		( exceptionSpec )+
478		{ behavior.endExceptionGroup(); }
479   ;
480
481exceptionSpec
482{ Token labelAction = null; }
483   :
484   "exception"
485   ( aa:ARG_ACTION { labelAction = aa; } )?
486   { behavior.beginExceptionSpec(labelAction); }
487   ( exceptionHandler )*
488   { behavior.endExceptionSpec(); }
489   ;
490
491exceptionSpecNoLabel
492   :
493   "exception"
494   { behavior.beginExceptionSpec(null); }
495   ( exceptionHandler )*
496   { behavior.endExceptionSpec(); }
497   ;
498
499exceptionHandler
500{ Token exType; Token exName; }
501   :
502   "catch"
503   a1:ARG_ACTION
504   a2:ACTION
505   { behavior.refExceptionHandler(a1, a2); }
506   ;
507
508element
509	:	elementNoOptionSpec (elementOptionSpec)?
510	;
511
512elementOptionSpec
513{
514	Token o=null, v=null;
515}
516	:	OPEN_ELEMENT_OPTION
517		o=id ASSIGN v=optionValue
518		{behavior.refElementOption(o,v);}
519		(
520			SEMI
521			o=id ASSIGN v=optionValue
522			{behavior.refElementOption(o,v);}
523		)*
524		CLOSE_ELEMENT_OPTION
525	;
526
527elementNoOptionSpec
528{
529	Token label = null;
530	Token assignId = null;
531	Token args = null;
532	int autoGen = GrammarElement.AUTO_GEN_NONE;
533}
534	:	assignId=id
535		ASSIGN
536		( label=id COLON {checkForMissingEndRule(label);} )?
537		(	rr:RULE_REF
538			( aa:ARG_ACTION { args=aa; } )?
539			( BANG { autoGen = GrammarElement.AUTO_GEN_BANG; } )?
540			{ behavior.refRule(assignId, rr, label, args, autoGen); }
541		|	// this syntax only valid for lexer
542			tr:TOKEN_REF
543			( aa2:ARG_ACTION { args=aa2; } )?
544			{ behavior.refToken(assignId, tr, label, args, false, autoGen, lastInRule()); }
545		)
546	|
547		(label=id COLON {checkForMissingEndRule(label);} )?
548		(	r2:RULE_REF
549			( aa3:ARG_ACTION { args=aa3; } )?
550			( BANG { autoGen = GrammarElement.AUTO_GEN_BANG; } )?
551			{ behavior.refRule(assignId, r2, label, args, autoGen); }
552		|
553			range [label]
554		|
555			terminal [label]
556		|
557			NOT_OP
558			(	notTerminal[label]
559			|	ebnf[label,true]
560			)
561		|
562			ebnf[label,false]
563		)
564	|
565		a:ACTION	{ behavior.refAction(a);}
566	|
567		p:SEMPRED	{ behavior.refSemPred(p);}
568	|
569		tree
570	;
571
572tree :
573	lp:TREE_BEGIN
574	{ behavior.beginTree(lp); }
575	rootNode
576	{behavior.beginChildList();}
577	( element )+
578	{behavior.endChildList();}
579	RPAREN
580	{ behavior.endTree(); }
581	;
582
583rootNode
584{ Token label = null; }
585	:
586		(label=id COLON {checkForMissingEndRule(label);} )?
587		terminal[label]
588//	|	range[null]
589	;
590
591ebnf
592[ Token label, boolean not ]
593	:	lp:LPAREN
594		{behavior.beginSubRule(label, lp, not);}
595
596		(
597			// 2nd alt and optional branch ambig due to
598			// linear approx LL(2) issue.  COLON ACTION
599			// matched correctly in 2nd alt.
600			options {
601				warnWhenFollowAmbig = false;
602			}
603		:
604			subruleOptionsSpec
605			( aa:ACTION {behavior.refInitAction(aa);} )?
606			COLON
607		|	ab:ACTION {behavior.refInitAction(ab);}
608			COLON
609		)?
610
611		block
612		RPAREN
613
614		(	(	QUESTION{behavior.optionalSubRule();}
615			|	STAR	{behavior.zeroOrMoreSubRule();}
616			|	PLUS	{behavior.oneOrMoreSubRule();}
617			)?
618			( BANG {behavior.noASTSubRule(); } )?
619		|
620			IMPLIES	{behavior.synPred();}
621		)
622		{behavior.endSubRule();}
623	;
624
625ast_type_spec
626returns [ int autoGen ]
627{ autoGen = GrammarElement.AUTO_GEN_NONE; }
628	:	(	CARET { autoGen = GrammarElement.AUTO_GEN_CARET; }
629		|	BANG { autoGen = GrammarElement.AUTO_GEN_BANG; }
630		)?
631	;
632
633range
634[ Token label ]
635{
636	Token trLeft=null;
637	Token trRight=null;
638	int autoGen=GrammarElement.AUTO_GEN_NONE;
639}
640	:	crLeft:CHAR_LITERAL RANGE crRight:CHAR_LITERAL
641		( BANG { autoGen = GrammarElement.AUTO_GEN_BANG; } )?
642		{ behavior.refCharRange(crLeft, crRight, label, autoGen, lastInRule()); }
643	|
644		(t:TOKEN_REF{trLeft=t;}|u:STRING_LITERAL{trLeft=u;})
645		RANGE
646		(v:TOKEN_REF{trRight=v;}|w:STRING_LITERAL{trRight=w;})
647		autoGen = ast_type_spec
648		{ behavior.refTokenRange(trLeft, trRight, label, autoGen, lastInRule()); }
649	;
650
651terminal
652[ Token label ]
653{
654	int autoGen=GrammarElement.AUTO_GEN_NONE;
655	Token args=null;
656}
657	:
658		cl:CHAR_LITERAL
659		( BANG { autoGen = GrammarElement.AUTO_GEN_BANG; } )?
660		{behavior.refCharLiteral(cl, label, false, autoGen, lastInRule());}
661	|
662		tr:TOKEN_REF
663		autoGen = ast_type_spec
664		// Args are only valid for lexer
665		( aa:ARG_ACTION { args=aa; } )?
666		{ behavior.refToken(null, tr, label, args, false, autoGen, lastInRule()); }
667	|
668		sl:STRING_LITERAL
669		autoGen = ast_type_spec
670		{behavior.refStringLiteral(sl, label, autoGen, lastInRule());}
671	|
672		wi:WILDCARD
673		autoGen = ast_type_spec
674		{behavior.refWildcard(wi, label, autoGen);}
675	;
676
677notTerminal
678[ Token label ]
679{ int autoGen=GrammarElement.AUTO_GEN_NONE; }
680	:
681		cl:CHAR_LITERAL
682		( BANG { autoGen = GrammarElement.AUTO_GEN_BANG; } )?
683		{behavior.refCharLiteral(cl, label, true, autoGen, lastInRule());}
684	|
685		tr:TOKEN_REF
686		autoGen = ast_type_spec
687		{behavior.refToken(null, tr, label, null, true, autoGen, lastInRule());}
688	;
689
690/** Match a.b.c.d qualified ids; WILDCARD here is overloaded as
691 *  id separator; that is, I need a reference to the '.' token.
692 */
693qualifiedID returns [Token qidTok=null]
694{
695	StringBuffer buf = new StringBuffer(30);
696	Token a;
697}
698	:	a=id {buf.append(a.getText());}
699		(	WILDCARD a=id
700			{buf.append('.'); buf.append(a.getText());}
701		)*
702		{
703		 // can use either TOKEN_REF or RULE_REF; should
704		 // really create a QID or something instead.
705		 qidTok = new CommonToken(TOKEN_REF, buf.toString());
706		 qidTok.setLine(a.getLine());
707		}
708	;
709
710id
711returns [ Token idTok ]
712{ idTok = null; }
713	:	a:TOKEN_REF			{idTok = a;}
714	|	b:RULE_REF			{idTok = b;}
715	;
716
717class ANTLRLexer extends Lexer;
718options {
719	k=2;
720	exportVocab=ANTLR;
721	testLiterals=false;
722	interactive=true;
723	charVocabulary='\003'..'\377';
724}
725
726tokens {
727	"options";
728}
729
730{
731	/**Convert 'c' to an integer char value. */
732	public static int escapeCharValue(String cs) {
733		//System.out.println("escapeCharValue("+cs+")");
734		if ( cs.charAt(1)!='\\' ) return 0;
735		switch ( cs.charAt(2) ) {
736		case 'b' : return '\b';
737		case 'r' : return '\r';
738		case 't' : return '\t';
739		case 'n' : return '\n';
740		case 'f' : return '\f';
741		case '"' : return '\"';
742		case '\'' :return '\'';
743		case '\\' :return '\\';
744
745		case 'u' :
746			// Unicode char
747			if (cs.length() != 8) {
748				return 0;
749			}
750			else {
751				return
752					Character.digit(cs.charAt(3), 16) * 16 * 16 * 16 +
753					Character.digit(cs.charAt(4), 16) * 16 * 16 +
754					Character.digit(cs.charAt(5), 16) * 16 +
755					Character.digit(cs.charAt(6), 16);
756			}
757
758		case '0' :
759		case '1' :
760		case '2' :
761		case '3' :
762			if ( cs.length()>5 && Character.isDigit(cs.charAt(4)) ) {
763				return (cs.charAt(2)-'0')*8*8 + (cs.charAt(3)-'0')*8 + (cs.charAt(4)-'0');
764			}
765			if ( cs.length()>4 && Character.isDigit(cs.charAt(3)) ) {
766				return (cs.charAt(2)-'0')*8 + (cs.charAt(3)-'0');
767			}
768			return cs.charAt(2)-'0';
769
770		case '4' :
771		case '5' :
772		case '6' :
773		case '7' :
774			if ( cs.length()>4 && Character.isDigit(cs.charAt(3)) ) {
775				return (cs.charAt(2)-'0')*8 + (cs.charAt(3)-'0');
776			}
777			return cs.charAt(2)-'0';
778
779		default :
780			return 0;
781		}
782	}
783
784	public static int tokenTypeForCharLiteral(String lit) {
785		if ( lit.length()>3 ) {  // does char contain escape?
786			return escapeCharValue(lit);
787		}
788		else {
789			return lit.charAt(1);
790		}
791	}
792}
793
794WS	:	(	/*	'\r' '\n' can be matched in one alternative or by matching
795				'\r' in one iteration and '\n' in another.  I am trying to
796				handle any flavor of newline that comes in, but the language
797				that allows both "\r\n" and "\r" and "\n" to all be valid
798				newline is ambiguous.  Consequently, the resulting grammar
799				must be ambiguous.  I'm shutting this warning off.
800			 */
801			options {
802				generateAmbigWarnings=false;
803			}
804		:	' '
805		|	'\t'
806		|	'\r' '\n'	{newline();}
807		|	'\r'		{newline();}
808		|	'\n'		{newline();}
809		)
810		{ $setType(Token.SKIP); }
811	;
812
813COMMENT :
814	( SL_COMMENT | t:ML_COMMENT {$setType(t.getType());} )
815	{if ( _ttype != DOC_COMMENT ) $setType(Token.SKIP);}
816	;
817
818protected
819SL_COMMENT :
820	"//"
821	( ~('\n'|'\r') )*
822	(
823		/*	'\r' '\n' can be matched in one alternative or by matching
824			'\r' and then in the next token.  The language
825			that allows both "\r\n" and "\r" and "\n" to all be valid
826			newline is ambiguous.  Consequently, the resulting grammar
827			must be ambiguous.  I'm shutting this warning off.
828		 */
829			options {
830				generateAmbigWarnings=false;
831			}
832		:	'\r' '\n'
833		|	'\r'
834		|	'\n'
835	)
836	{ newline(); }
837	;
838
839protected
840ML_COMMENT :
841	"/*"
842	(	{ LA(2)!='/' }? '*' {$setType(DOC_COMMENT);}
843	|
844	)
845	(
846		/*	'\r' '\n' can be matched in one alternative or by matching
847			'\r' and then in the next token.  The language
848			that allows both "\r\n" and "\r" and "\n" to all be valid
849			newline is ambiguous.  Consequently, the resulting grammar
850			must be ambiguous.  I'm shutting this warning off.
851		 */
852		options {
853			greedy=false;  // make it exit upon "*/"
854			generateAmbigWarnings=false; // shut off newline errors
855		}
856	:	'\r' '\n'	{newline();}
857	|	'\r'		{newline();}
858	|	'\n'		{newline();}
859	|	~('\n'|'\r')
860	)*
861	"*/"
862	;
863
864OPEN_ELEMENT_OPTION
865	:	'<'
866	;
867
868CLOSE_ELEMENT_OPTION
869	:	'>'
870	;
871
872COMMA : ',';
873
874QUESTION :	'?' ;
875
876TREE_BEGIN : "#(" ;
877
878LPAREN:	'(' ;
879
880RPAREN:	')' ;
881
882COLON :	':' ;
883
884STAR:	'*' ;
885
886PLUS:	'+' ;
887
888ASSIGN : '=' ;
889
890IMPLIES : "=>" ;
891
892SEMI:	';' ;
893
894CARET : '^' ;
895
896BANG : '!' ;
897
898OR	:	'|' ;
899
900WILDCARD : '.' ;
901
902RANGE : ".." ;
903
904NOT_OP :	'~' ;
905
906RCURLY:	'}'	;
907
908CHAR_LITERAL
909	:	'\'' (ESC|~'\'') '\''
910	;
911
912STRING_LITERAL
913	:	'"' (ESC|~'"')* '"'
914	;
915
916protected
917ESC	:	'\\'
918		(	'n'
919		|	'r'
920		|	't'
921		|	'b'
922		|	'f'
923		|	'w'
924		|	'a'
925		|	'"'
926		|	'\''
927		|	'\\'
928		|	('0'..'3')
929			( options { warnWhenFollowAmbig = false; }: '0'..'7'
930			( options { warnWhenFollowAmbig = false; }: '0'..'7'
931		   )?
932			)?
933		|	('4'..'7')
934			( options { warnWhenFollowAmbig = false; }: '0'..'7' )?
935		|	'u' XDIGIT XDIGIT XDIGIT XDIGIT
936		)
937	;
938
939protected
940DIGIT
941	:	'0'..'9'
942	;
943
944protected
945XDIGIT :
946		'0' .. '9'
947	|	'a' .. 'f'
948	|	'A' .. 'F'
949	;
950
951INT	:	('0'..'9')+
952	;
953
954ARG_ACTION
955   :
956	NESTED_ARG_ACTION
957	{ setText(StringUtils.stripFrontBack(getText(), "[", "]")); }
958	;
959
960protected
961NESTED_ARG_ACTION :
962	'['
963	(
964		/*	'\r' '\n' can be matched in one alternative or by matching
965			'\r' and then '\n' in the next iteration.
966		 */
967		options {
968			generateAmbigWarnings=false; // shut off newline errors
969		}
970	:	NESTED_ARG_ACTION
971	|	'\r' '\n'	{newline();}
972	|	'\r'		{newline();}
973	|	'\n'		{newline();}
974	|	CHAR_LITERAL
975	|	STRING_LITERAL
976	|	~']'
977	)*
978	']'
979	;
980
981ACTION
982{int actionLine=getLine(); int actionColumn = getColumn(); }
983	:	NESTED_ACTION
984		(	'?'	{_ttype = SEMPRED;} )?
985		{
986			if ( _ttype==ACTION ) {
987				setText(StringUtils.stripFrontBack(getText(), "{", "}"));
988			}
989			else {
990				setText(StringUtils.stripFrontBack(getText(), "{", "}?"));
991			}
992			CommonToken t = new CommonToken(_ttype,$getText);
993			t.setLine(actionLine);			// set action line to start
994			t.setColumn(actionColumn);
995			$setToken(t);
996		}
997	;
998
999protected
1000NESTED_ACTION :
1001	'{'
1002	(
1003		options {
1004			greedy = false; // exit upon '}'
1005		}
1006	:
1007		(
1008			options {
1009				generateAmbigWarnings = false; // shut off newline warning
1010			}
1011		:	'\r' '\n'	{newline();}
1012		|	'\r' 		{newline();}
1013		|	'\n'		{newline();}
1014		)
1015	|	NESTED_ACTION
1016	|	CHAR_LITERAL
1017	|	COMMENT
1018	|	STRING_LITERAL
1019	|	.
1020	)*
1021	'}'
1022   ;
1023
1024TOKEN_REF
1025options { testLiterals = true; }
1026	:	'A'..'Z'
1027		(	// scarf as many letters/numbers as you can
1028			options {
1029				warnWhenFollowAmbig=false;
1030			}
1031		:
1032			'a'..'z'|'A'..'Z'|'_'|'0'..'9'
1033		)*
1034	;
1035
1036// we get a warning here when looking for options '{', but it works right
1037RULE_REF
1038{
1039	int t=0;
1040}
1041	:	t=INTERNAL_RULE_REF {_ttype=t;}
1042		(	{t==LITERAL_options}? WS_LOOP ('{' {_ttype = OPTIONS;})?
1043		|	{t==LITERAL_tokens}? WS_LOOP ('{' {_ttype = TOKENS;})?
1044		|
1045		)
1046	;
1047
1048protected
1049WS_LOOP
1050	:	(	// grab as much WS as you can
1051			options {
1052				greedy=true;
1053			}
1054		:
1055			WS
1056		|	COMMENT
1057		)*
1058	;
1059
1060protected
1061INTERNAL_RULE_REF returns [int t]
1062{
1063	t = RULE_REF;
1064}
1065	:	'a'..'z'
1066		(	// scarf as many letters/numbers as you can
1067			options {
1068				warnWhenFollowAmbig=false;
1069			}
1070		:
1071			'a'..'z'|'A'..'Z'|'_'|'0'..'9'
1072		)*
1073		{t = testLiteralsTable(t);}
1074	;
1075
1076protected
1077WS_OPT :
1078	(WS)?
1079	;
1080