1 // 2 // cs-tokenizer.cs: The Tokenizer for the C# compiler 3 // This also implements the preprocessor 4 // 5 // Author: Miguel de Icaza (miguel@gnu.org) 6 // Marek Safar (marek.safar@gmail.com) 7 // 8 // Dual licensed under the terms of the MIT X11 or GNU GPL 9 // 10 // Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com) 11 // Copyright 2004-2008 Novell, Inc 12 // Copyright 2011 Xamarin, Inc (http://www.xamarin.com) 13 // 14 15 using System; 16 using System.Text; 17 using System.Collections.Generic; 18 using System.Globalization; 19 using System.Diagnostics; 20 using System.Collections; 21 22 namespace Mono.CSharp 23 { 24 // 25 // This class has to be used by parser only, it reuses token 26 // details once a file is parsed 27 // 28 public class LocatedToken 29 { 30 public int row, column; 31 public string value; 32 public SourceFile file; 33 LocatedToken()34 public LocatedToken () 35 { 36 } 37 LocatedToken(string value, Location loc)38 public LocatedToken (string value, Location loc) 39 { 40 this.value = value; 41 file = loc.SourceFile; 42 row = loc.Row; 43 column = loc.Column; 44 } 45 ToString()46 public override string ToString () 47 { 48 return string.Format ("Token '{0}' at {1},{2}", Value, row, column); 49 } 50 51 public Location Location 52 { 53 get { return new Location (file, row, column); } 54 } 55 56 public string Value 57 { 58 get { return value; } 59 } 60 } 61 62 /// <summary> 63 /// Tokenizer for C# source code. 64 /// </summary> 65 public class Tokenizer : yyParser.yyInput 66 { 67 class KeywordEntry<T> 68 { 69 public readonly T Token; 70 public KeywordEntry<T> Next; 71 public readonly char[] Value; 72 KeywordEntry(string value, T token)73 public KeywordEntry (string value, T token) 74 { 75 this.Value = value.ToCharArray (); 76 this.Token = token; 77 } 78 } 79 80 sealed class IdentifiersComparer : IEqualityComparer<char[]> 81 { 82 readonly int length; 83 IdentifiersComparer(int length)84 public IdentifiersComparer (int length) 85 { 86 this.length = length; 87 } 88 Equals(char[] x, char[] y)89 public bool Equals (char[] x, char[] y) 90 { 91 for (int i = 0; i < length; ++i) 92 if (x [i] != y [i]) 93 return false; 94 95 return true; 96 } 97 GetHashCode(char[] obj)98 public int GetHashCode (char[] obj) 99 { 100 int h = 0; 101 for (int i = 0; i < length; ++i) 102 h = (h << 5) - h + obj [i]; 103 104 return h; 105 } 106 } 107 108 public class LocatedTokenBuffer 109 { 110 readonly LocatedToken[] buffer; 111 public int pos; 112 LocatedTokenBuffer()113 public LocatedTokenBuffer () 114 { 115 buffer = new LocatedToken[0]; 116 } 117 LocatedTokenBuffer(LocatedToken[] buffer)118 public LocatedTokenBuffer (LocatedToken[] buffer) 119 { 120 this.buffer = buffer ?? new LocatedToken[0]; 121 } 122 Create(SourceFile file, int row, int column)123 public LocatedToken Create (SourceFile file, int row, int column) 124 { 125 return Create (null, file, row, column); 126 } 127 Create(string value, SourceFile file, int row, int column)128 public LocatedToken Create (string value, SourceFile file, int row, int column) 129 { 130 // 131 // TODO: I am not very happy about the logic but it's the best 132 // what I could come up with for now. 133 // Ideally we should be using just tiny buffer (256 elements) which 134 // is enough to hold all details for currect stack and recycle elements 135 // poped from the stack but there is a trick needed to recycle 136 // them properly. 137 // 138 LocatedToken entry; 139 if (pos >= buffer.Length) { 140 entry = new LocatedToken (); 141 } else { 142 entry = buffer[pos]; 143 if (entry == null) { 144 entry = new LocatedToken (); 145 buffer[pos] = entry; 146 } 147 148 ++pos; 149 } 150 entry.value = value; 151 entry.file = file; 152 entry.row = row; 153 entry.column = column; 154 return entry; 155 } 156 157 // 158 // Used for token not required by expression evaluator 159 // 160 [Conditional ("FULL_AST")] CreateOptional(SourceFile file, int row, int col, ref object token)161 public void CreateOptional (SourceFile file, int row, int col, ref object token) 162 { 163 token = Create (file, row, col); 164 } 165 } 166 167 public enum PreprocessorDirective 168 { 169 Invalid = 0, 170 171 Region = 1, 172 Endregion = 2, 173 If = 3 | RequiresArgument, 174 Endif = 4, 175 Elif = 5 | RequiresArgument, 176 Else = 6, 177 Define = 7 | RequiresArgument, 178 Undef = 8 | RequiresArgument, 179 Error = 9, 180 Warning = 10, 181 Pragma = 11 | CustomArgumentsParsing, 182 Line = 12 | CustomArgumentsParsing, 183 184 CustomArgumentsParsing = 1 << 10, 185 RequiresArgument = 1 << 11 186 } 187 188 readonly SeekableStreamReader reader; 189 readonly CompilationSourceFile source_file; 190 readonly CompilerContext context; 191 readonly Report Report; 192 193 194 SourceFile current_source; 195 Location hidden_block_start; 196 int ref_line = 1; 197 int line = 1; 198 int col = 0; 199 int previous_col; 200 int current_token; 201 readonly int tab_size; 202 bool handle_get_set = false; 203 bool handle_remove_add = false; 204 bool handle_where; 205 bool lambda_arguments_parsing; 206 List<Location> escaped_identifiers; 207 int parsing_generic_less_than; 208 readonly bool doc_processing; 209 readonly LocatedTokenBuffer ltb; 210 211 // 212 // Used mainly for parser optimizations. Some expressions for instance 213 // can appear only in block (including initializer, base initializer) 214 // scope only 215 // 216 public int parsing_block; 217 internal bool query_parsing; 218 219 // 220 // When parsing type only, useful for ambiguous nullable types 221 // 222 public int parsing_type; 223 224 // 225 // Set when parsing generic declaration (type or method header) 226 // 227 public bool parsing_generic_declaration; 228 public bool parsing_generic_declaration_doc; 229 230 // 231 // The value indicates that we have not reach any declaration or 232 // namespace yet 233 // 234 public int parsing_declaration; 235 236 public bool parsing_attribute_section; 237 238 public bool parsing_modifiers; 239 240 public bool parsing_catch_when; 241 242 int parsing_string_interpolation; 243 int string_interpolation_section; 244 Stack<bool> parsing_string_interpolation_quoted; 245 246 public bool parsing_interpolation_format; 247 248 // 249 // The special characters to inject on streams to run the unit parser 250 // in the special expression mode. Using private characters from 251 // Plane Sixteen (U+100000 to U+10FFFD) 252 // 253 // This character is only tested just before the tokenizer is about to report 254 // an error; So on the regular operation mode, this addition will have no 255 // impact on the tokenizer's performance. 256 // 257 258 public const int EvalStatementParserCharacter = 0x100000; 259 public const int EvalCompilationUnitParserCharacter = 0x100001; 260 public const int EvalUsingDeclarationsParserCharacter = 0x100002; 261 public const int DocumentationXref = 0x100003; 262 263 const int UnicodeLS = 0x2028; 264 const int UnicodePS = 0x2029; 265 266 // 267 // XML documentation buffer. The save point is used to divide 268 // comments on types and comments on members. 269 // 270 StringBuilder xml_comment_buffer; 271 272 // 273 // See comment on XmlCommentState enumeration. 274 // 275 XmlCommentState xml_doc_state = XmlCommentState.Allowed; 276 277 // 278 // Whether tokens have been seen on this line 279 // 280 bool tokens_seen = false; 281 282 // 283 // Set to true once the GENERATE_COMPLETION token has bee 284 // returned. This helps produce one GENERATE_COMPLETION, 285 // as many COMPLETE_COMPLETION as necessary to complete the 286 // AST tree and one final EOF. 287 // 288 bool generated; 289 290 // 291 // Whether a token has been seen on the file 292 // This is needed because `define' is not allowed to be used 293 // after a token has been seen. 294 // 295 bool any_token_seen; 296 297 // 298 // Class variables 299 // 300 static readonly KeywordEntry<int>[][] keywords; 301 static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor; 302 static readonly HashSet<string> keyword_strings; 303 static readonly NumberStyles styles; 304 static readonly NumberFormatInfo csharp_format_info; 305 306 // Pragma arguments 307 static readonly char[] pragma_warning = "warning".ToCharArray (); 308 static readonly char[] pragma_warning_disable = "disable".ToCharArray (); 309 static readonly char[] pragma_warning_restore = "restore".ToCharArray (); 310 static readonly char[] pragma_checksum = "checksum".ToCharArray (); 311 static readonly char[] line_hidden = "hidden".ToCharArray (); 312 static readonly char[] line_default = "default".ToCharArray (); 313 314 static readonly char[] simple_whitespaces = new char[] { ' ', '\t' }; 315 316 public bool PropertyParsing { 317 get { return handle_get_set; } 318 set { handle_get_set = value; } 319 } 320 321 public bool EventParsing { 322 get { return handle_remove_add; } 323 set { handle_remove_add = value; } 324 } 325 326 public bool ConstraintsParsing { 327 get { return handle_where; } 328 set { handle_where = value; } 329 } 330 331 public XmlCommentState doc_state { 332 get { return xml_doc_state; } 333 set { 334 if (value == XmlCommentState.Allowed) { 335 check_incorrect_doc_comment (); 336 reset_doc_comment (); 337 } 338 xml_doc_state = value; 339 } 340 } 341 342 // 343 // This is used to trigger completion generation on the parser 344 public bool CompleteOnEOF; 345 AddEscapedIdentifier(Location loc)346 void AddEscapedIdentifier (Location loc) 347 { 348 if (escaped_identifiers == null) 349 escaped_identifiers = new List<Location> (); 350 351 escaped_identifiers.Add (loc); 352 } 353 IsEscapedIdentifier(ATypeNameExpression name)354 public bool IsEscapedIdentifier (ATypeNameExpression name) 355 { 356 return escaped_identifiers != null && escaped_identifiers.Contains (name.Location); 357 } 358 359 // 360 // Values for the associated token returned 361 // 362 internal int putback_char; // Used by repl only 363 object val; 364 365 // 366 // Pre-processor 367 // 368 const int TAKING = 1; 369 const int ELSE_SEEN = 4; 370 const int PARENT_TAKING = 8; 371 const int REGION = 16; 372 373 // 374 // pre-processor if stack state: 375 // 376 Stack<int> ifstack; 377 378 public const int MaxIdentifierLength = 512; 379 public const int MaxNumberLength = 512; 380 381 readonly char[] id_builder; 382 readonly Dictionary<char[], string>[] identifiers; 383 readonly char[] number_builder; 384 int number_pos; 385 386 char[] value_builder = new char[64]; 387 388 public int Line { 389 get { 390 return ref_line; 391 } 392 } 393 394 // 395 // This is used when the tokenizer needs to save 396 // the current position as it needs to do some parsing 397 // on its own to deamiguate a token in behalf of the 398 // parser. 399 // 400 Stack<Position> position_stack = new Stack<Position> (2); 401 402 class Position { 403 public int position; 404 public int line; 405 public int ref_line; 406 public int col; 407 public Location hidden; 408 public int putback_char; 409 public int previous_col; 410 public Stack<int> ifstack; 411 public int parsing_generic_less_than; 412 public int current_token; 413 public object val; 414 public int parsing_string_interpolation; 415 public int string_interpolation_section; 416 public Stack<bool> parsing_string_interpolation_quoted; 417 Position(Tokenizer t)418 public Position (Tokenizer t) 419 { 420 position = t.reader.Position; 421 line = t.line; 422 ref_line = t.ref_line; 423 col = t.col; 424 hidden = t.hidden_block_start; 425 putback_char = t.putback_char; 426 previous_col = t.previous_col; 427 if (t.ifstack != null && t.ifstack.Count != 0) { 428 // There is no simple way to clone Stack<T> all 429 // methods reverse the order 430 var clone = t.ifstack.ToArray (); 431 Array.Reverse (clone); 432 ifstack = new Stack<int> (clone); 433 } 434 parsing_generic_less_than = t.parsing_generic_less_than; 435 string_interpolation_section = t.string_interpolation_section; 436 current_token = t.current_token; 437 val = t.val; 438 parsing_string_interpolation = t.parsing_string_interpolation; 439 string_interpolation_section = t.string_interpolation_section; 440 if (t.parsing_string_interpolation_quoted != null && t.parsing_string_interpolation_quoted.Count != 0) { 441 var clone = t.parsing_string_interpolation_quoted.ToArray (); 442 Array.Reverse (clone); 443 parsing_string_interpolation_quoted = new Stack<bool> (clone); 444 } 445 } 446 } 447 Tokenizer(SeekableStreamReader input, CompilationSourceFile file, ParserSession session, Report report)448 public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, ParserSession session, Report report) 449 { 450 this.source_file = file; 451 this.context = file.Compiler; 452 this.current_source = file.SourceFile; 453 this.identifiers = session.Identifiers; 454 this.id_builder = session.IDBuilder; 455 this.number_builder = session.NumberBuilder; 456 this.ltb = new LocatedTokenBuffer (session.LocatedTokens); 457 this.Report = report; 458 459 reader = input; 460 461 putback_char = -1; 462 463 xml_comment_buffer = new StringBuilder (); 464 doc_processing = context.Settings.DocumentationFile != null; 465 466 tab_size = context.Settings.TabSize; 467 } 468 PushPosition()469 public void PushPosition () 470 { 471 position_stack.Push (new Position (this)); 472 } 473 PopPosition()474 public void PopPosition () 475 { 476 Position p = position_stack.Pop (); 477 478 reader.Position = p.position; 479 ref_line = p.ref_line; 480 line = p.line; 481 col = p.col; 482 hidden_block_start = p.hidden; 483 putback_char = p.putback_char; 484 previous_col = p.previous_col; 485 ifstack = p.ifstack; 486 parsing_generic_less_than = p.parsing_generic_less_than; 487 parsing_string_interpolation = p.parsing_string_interpolation; 488 parsing_string_interpolation_quoted = p.parsing_string_interpolation_quoted; 489 current_token = p.current_token; 490 val = p.val; 491 } 492 493 // Do not reset the position, ignore it. DiscardPosition()494 public void DiscardPosition () 495 { 496 position_stack.Pop (); 497 } 498 AddKeyword(string kw, int token)499 static void AddKeyword (string kw, int token) 500 { 501 keyword_strings.Add (kw); 502 503 AddKeyword (keywords, kw, token); 504 } 505 AddPreprocessorKeyword(string kw, PreprocessorDirective directive)506 static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive) 507 { 508 AddKeyword (keywords_preprocessor, kw, directive); 509 } 510 AddKeyword(KeywordEntry<T>[][] keywords, string kw, T token)511 static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token) 512 { 513 int length = kw.Length; 514 if (keywords[length] == null) { 515 keywords[length] = new KeywordEntry<T>['z' - '_' + 1]; 516 } 517 518 int char_index = kw[0] - '_'; 519 var kwe = keywords[length][char_index]; 520 if (kwe == null) { 521 keywords[length][char_index] = new KeywordEntry<T> (kw, token); 522 return; 523 } 524 525 while (kwe.Next != null) { 526 kwe = kwe.Next; 527 } 528 529 kwe.Next = new KeywordEntry<T> (kw, token); 530 } 531 532 // 533 // Class initializer 534 // Tokenizer()535 static Tokenizer () 536 { 537 keyword_strings = new HashSet<string> (); 538 539 // 11 is the length of the longest keyword for now 540 keywords = new KeywordEntry<int>[11][]; 541 542 AddKeyword ("__arglist", Token.ARGLIST); 543 AddKeyword ("__makeref", Token.MAKEREF); 544 AddKeyword ("__reftype", Token.REFTYPE); 545 AddKeyword ("__refvalue", Token.REFVALUE); 546 AddKeyword ("abstract", Token.ABSTRACT); 547 AddKeyword ("as", Token.AS); 548 AddKeyword ("add", Token.ADD); 549 AddKeyword ("base", Token.BASE); 550 AddKeyword ("bool", Token.BOOL); 551 AddKeyword ("break", Token.BREAK); 552 AddKeyword ("byte", Token.BYTE); 553 AddKeyword ("case", Token.CASE); 554 AddKeyword ("catch", Token.CATCH); 555 AddKeyword ("char", Token.CHAR); 556 AddKeyword ("checked", Token.CHECKED); 557 AddKeyword ("class", Token.CLASS); 558 AddKeyword ("const", Token.CONST); 559 AddKeyword ("continue", Token.CONTINUE); 560 AddKeyword ("decimal", Token.DECIMAL); 561 AddKeyword ("default", Token.DEFAULT); 562 AddKeyword ("delegate", Token.DELEGATE); 563 AddKeyword ("do", Token.DO); 564 AddKeyword ("double", Token.DOUBLE); 565 AddKeyword ("else", Token.ELSE); 566 AddKeyword ("enum", Token.ENUM); 567 AddKeyword ("event", Token.EVENT); 568 AddKeyword ("explicit", Token.EXPLICIT); 569 AddKeyword ("extern", Token.EXTERN); 570 AddKeyword ("false", Token.FALSE); 571 AddKeyword ("finally", Token.FINALLY); 572 AddKeyword ("fixed", Token.FIXED); 573 AddKeyword ("float", Token.FLOAT); 574 AddKeyword ("for", Token.FOR); 575 AddKeyword ("foreach", Token.FOREACH); 576 AddKeyword ("goto", Token.GOTO); 577 AddKeyword ("get", Token.GET); 578 AddKeyword ("if", Token.IF); 579 AddKeyword ("implicit", Token.IMPLICIT); 580 AddKeyword ("in", Token.IN); 581 AddKeyword ("int", Token.INT); 582 AddKeyword ("interface", Token.INTERFACE); 583 AddKeyword ("internal", Token.INTERNAL); 584 AddKeyword ("is", Token.IS); 585 AddKeyword ("lock", Token.LOCK); 586 AddKeyword ("long", Token.LONG); 587 AddKeyword ("namespace", Token.NAMESPACE); 588 AddKeyword ("new", Token.NEW); 589 AddKeyword ("null", Token.NULL); 590 AddKeyword ("object", Token.OBJECT); 591 AddKeyword ("operator", Token.OPERATOR); 592 AddKeyword ("out", Token.OUT); 593 AddKeyword ("override", Token.OVERRIDE); 594 AddKeyword ("params", Token.PARAMS); 595 AddKeyword ("private", Token.PRIVATE); 596 AddKeyword ("protected", Token.PROTECTED); 597 AddKeyword ("public", Token.PUBLIC); 598 AddKeyword ("readonly", Token.READONLY); 599 AddKeyword ("ref", Token.REF); 600 AddKeyword ("remove", Token.REMOVE); 601 AddKeyword ("return", Token.RETURN); 602 AddKeyword ("sbyte", Token.SBYTE); 603 AddKeyword ("sealed", Token.SEALED); 604 AddKeyword ("set", Token.SET); 605 AddKeyword ("short", Token.SHORT); 606 AddKeyword ("sizeof", Token.SIZEOF); 607 AddKeyword ("stackalloc", Token.STACKALLOC); 608 AddKeyword ("static", Token.STATIC); 609 AddKeyword ("string", Token.STRING); 610 AddKeyword ("struct", Token.STRUCT); 611 AddKeyword ("switch", Token.SWITCH); 612 AddKeyword ("this", Token.THIS); 613 AddKeyword ("throw", Token.THROW); 614 AddKeyword ("true", Token.TRUE); 615 AddKeyword ("try", Token.TRY); 616 AddKeyword ("typeof", Token.TYPEOF); 617 AddKeyword ("uint", Token.UINT); 618 AddKeyword ("ulong", Token.ULONG); 619 AddKeyword ("unchecked", Token.UNCHECKED); 620 AddKeyword ("unsafe", Token.UNSAFE); 621 AddKeyword ("ushort", Token.USHORT); 622 AddKeyword ("using", Token.USING); 623 AddKeyword ("virtual", Token.VIRTUAL); 624 AddKeyword ("void", Token.VOID); 625 AddKeyword ("volatile", Token.VOLATILE); 626 AddKeyword ("while", Token.WHILE); 627 AddKeyword ("partial", Token.PARTIAL); 628 AddKeyword ("where", Token.WHERE); 629 630 // LINQ keywords 631 AddKeyword ("from", Token.FROM); 632 AddKeyword ("join", Token.JOIN); 633 AddKeyword ("on", Token.ON); 634 AddKeyword ("equals", Token.EQUALS); 635 AddKeyword ("select", Token.SELECT); 636 AddKeyword ("group", Token.GROUP); 637 AddKeyword ("by", Token.BY); 638 AddKeyword ("let", Token.LET); 639 AddKeyword ("orderby", Token.ORDERBY); 640 AddKeyword ("ascending", Token.ASCENDING); 641 AddKeyword ("descending", Token.DESCENDING); 642 AddKeyword ("into", Token.INTO); 643 644 // Contextual async keywords 645 AddKeyword ("async", Token.ASYNC); 646 AddKeyword ("await", Token.AWAIT); 647 648 // Contextual filter catch keyword 649 AddKeyword ("when", Token.WHEN); 650 651 keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][]; 652 653 AddPreprocessorKeyword ("region", PreprocessorDirective.Region); 654 AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion); 655 AddPreprocessorKeyword ("if", PreprocessorDirective.If); 656 AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif); 657 AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif); 658 AddPreprocessorKeyword ("else", PreprocessorDirective.Else); 659 AddPreprocessorKeyword ("define", PreprocessorDirective.Define); 660 AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef); 661 AddPreprocessorKeyword ("error", PreprocessorDirective.Error); 662 AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning); 663 AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma); 664 AddPreprocessorKeyword ("line", PreprocessorDirective.Line); 665 666 csharp_format_info = NumberFormatInfo.InvariantInfo; 667 styles = NumberStyles.Float; 668 } 669 GetKeyword(char[] id, int id_len)670 int GetKeyword (char[] id, int id_len) 671 { 672 // 673 // Keywords are stored in an array of arrays grouped by their 674 // length and then by the first character 675 // 676 if (id_len >= keywords.Length || keywords [id_len] == null) 677 return -1; 678 679 int first_index = id [0] - '_'; 680 if (first_index > 'z' - '_') 681 return -1; 682 683 var kwe = keywords [id_len] [first_index]; 684 if (kwe == null) 685 return -1; 686 687 int res; 688 do { 689 res = kwe.Token; 690 for (int i = 1; i < id_len; ++i) { 691 if (id [i] != kwe.Value [i]) { 692 res = 0; 693 kwe = kwe.Next; 694 break; 695 } 696 } 697 } while (res == 0 && kwe != null); 698 699 if (res == 0) 700 return -1; 701 702 int next_token; 703 switch (res) { 704 case Token.GET: 705 case Token.SET: 706 if (!handle_get_set) 707 res = -1; 708 break; 709 case Token.REMOVE: 710 case Token.ADD: 711 if (!handle_remove_add) 712 res = -1; 713 break; 714 case Token.EXTERN: 715 if (parsing_declaration == 0) 716 res = Token.EXTERN_ALIAS; 717 break; 718 case Token.DEFAULT: 719 switch (peek_token ()) { 720 case Token.COLON: 721 // Special case: foo == null ? default : 1; 722 if (current_token != Token.INTERR) { 723 token (); 724 res = Token.DEFAULT_COLON; 725 } 726 break; 727 case Token.OPEN_PARENS: 728 case Token.OPEN_PARENS_CAST: 729 res = Token.DEFAULT_VALUE; 730 break; 731 } 732 break; 733 case Token.WHEN: 734 if (current_token != Token.CATCH && !parsing_catch_when) 735 res = -1; 736 break; 737 case Token.WHERE: 738 if (!(handle_where && current_token != Token.COLON) && !query_parsing) 739 res = -1; 740 break; 741 case Token.FROM: 742 // 743 // A query expression is any expression that starts with `from identifier' 744 // followed by any token except ; , = 745 // 746 if (!query_parsing) { 747 if (lambda_arguments_parsing || parsing_block == 0) { 748 res = -1; 749 break; 750 } 751 752 PushPosition (); 753 // HACK: to disable generics micro-parser, because PushPosition does not 754 // store identifiers array 755 parsing_generic_less_than = 1; 756 switch (xtoken ()) { 757 case Token.IDENTIFIER: 758 case Token.INT: 759 case Token.BOOL: 760 case Token.BYTE: 761 case Token.CHAR: 762 case Token.DECIMAL: 763 case Token.DOUBLE: 764 case Token.FLOAT: 765 case Token.LONG: 766 case Token.OBJECT: 767 case Token.STRING: 768 case Token.UINT: 769 case Token.ULONG: 770 next_token = xtoken (); 771 if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS || next_token == Token.ASSIGN) 772 goto default; 773 774 res = Token.FROM_FIRST; 775 query_parsing = true; 776 if (context.Settings.Version <= LanguageVersion.ISO_2) 777 Report.FeatureIsNotAvailable (context, Location, "query expressions"); 778 break; 779 case Token.VOID: 780 Expression.Error_VoidInvalidInTheContext (Location, Report); 781 break; 782 default: 783 PopPosition (); 784 // HACK: A token is not a keyword so we need to restore identifiers buffer 785 // which has been overwritten before we grabbed the identifier 786 id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm'; 787 return -1; 788 } 789 PopPosition (); 790 } 791 break; 792 case Token.JOIN: 793 case Token.ON: 794 case Token.EQUALS: 795 case Token.SELECT: 796 case Token.GROUP: 797 case Token.BY: 798 case Token.LET: 799 case Token.ORDERBY: 800 case Token.ASCENDING: 801 case Token.DESCENDING: 802 case Token.INTO: 803 if (!query_parsing) 804 res = -1; 805 break; 806 807 case Token.USING: 808 case Token.NAMESPACE: 809 // TODO: some explanation needed 810 check_incorrect_doc_comment (); 811 parsing_modifiers = false; 812 break; 813 814 case Token.PARTIAL: 815 if (parsing_block > 0) { 816 res = -1; 817 break; 818 } 819 820 // Save current position and parse next token. 821 PushPosition (); 822 823 next_token = token (); 824 bool ok = 825 next_token == Token.CLASS || 826 next_token == Token.STRUCT || 827 next_token == Token.INTERFACE || 828 next_token == Token.VOID || 829 next_token == Token.REF_STRUCT; 830 831 PopPosition (); 832 833 if (ok) { 834 if (next_token == Token.VOID) { 835 if (context.Settings.Version <= LanguageVersion.ISO_2) 836 Report.FeatureIsNotAvailable (context, Location, "partial methods"); 837 } else if (context.Settings.Version == LanguageVersion.ISO_1) 838 Report.FeatureIsNotAvailable (context, Location, "partial types"); 839 840 return res; 841 } 842 843 if (next_token < Token.LAST_KEYWORD) { 844 Report.Error (267, Location, 845 "The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword"); 846 return token (); 847 } 848 849 // HACK: A token is not a keyword so we need to restore identifiers buffer 850 // which has been overwritten before we grabbed the identifier 851 id_builder[0] = 'p'; 852 id_builder[1] = 'a'; 853 id_builder[2] = 'r'; 854 id_builder[3] = 't'; 855 id_builder[4] = 'i'; 856 id_builder[5] = 'a'; 857 id_builder[6] = 'l'; 858 res = -1; 859 break; 860 861 case Token.ASYNC: 862 if (parsing_modifiers) { 863 // 864 // Skip attributes section or constructor called async 865 // 866 if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) { 867 res = -1; 868 } else { 869 // async is keyword 870 } 871 } else if (parsing_block > 0) { 872 switch (peek_token ()) { 873 case Token.DELEGATE: 874 case Token.OPEN_PARENS_LAMBDA: 875 // async is keyword 876 break; 877 case Token.IDENTIFIER: 878 PushPosition (); 879 xtoken (); 880 if (xtoken () != Token.ARROW) { 881 PopPosition (); 882 goto default; 883 } 884 885 PopPosition (); 886 break; 887 default: 888 // peek_token could overwrite id_buffer 889 id_builder [0] = 'a'; id_builder [1] = 's'; id_builder [2] = 'y'; id_builder [3] = 'n'; id_builder [4] = 'c'; 890 res = -1; 891 break; 892 } 893 } else { 894 res = -1; 895 } 896 897 if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) { 898 Report.FeatureIsNotAvailable (context, Location, "asynchronous functions"); 899 } 900 901 break; 902 903 case Token.AWAIT: 904 if (parsing_block == 0) 905 res = -1; 906 907 break; 908 case Token.THROW: 909 switch (current_token) { 910 case Token.ARROW: 911 case Token.OP_COALESCING: 912 case Token.INTERR: 913 res = Token.THROW_EXPR; 914 break; 915 } 916 917 break; 918 case Token.REF: 919 if (peek_token () == Token.STRUCT) { 920 token (); 921 res = Token.REF_STRUCT; 922 } 923 break; 924 } 925 926 927 return res; 928 } 929 GetPreprocessorDirective(char[] id, int id_len)930 static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len) 931 { 932 // 933 // Keywords are stored in an array of arrays grouped by their 934 // length and then by the first character 935 // 936 if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null) 937 return PreprocessorDirective.Invalid; 938 939 int first_index = id[0] - '_'; 940 if (first_index > 'z' - '_') 941 return PreprocessorDirective.Invalid; 942 943 var kwe = keywords_preprocessor[id_len][first_index]; 944 if (kwe == null) 945 return PreprocessorDirective.Invalid; 946 947 PreprocessorDirective res = PreprocessorDirective.Invalid; 948 do { 949 res = kwe.Token; 950 for (int i = 1; i < id_len; ++i) { 951 if (id[i] != kwe.Value[i]) { 952 res = 0; 953 kwe = kwe.Next; 954 break; 955 } 956 } 957 } while (res == PreprocessorDirective.Invalid && kwe != null); 958 959 return res; 960 } 961 962 public Location Location { 963 get { 964 return new Location (current_source, ref_line, col); 965 } 966 } 967 is_identifier_start_character(int c)968 static bool is_identifier_start_character (int c) 969 { 970 if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') 971 return true; 972 973 if (c < 0x80) 974 return false; 975 976 return is_identifier_start_character_slow_part ((char) c); 977 } 978 is_identifier_part_character(char c)979 static bool is_identifier_part_character (char c) 980 { 981 if (c >= 'a' && c <= 'z') 982 return true; 983 984 if (c >= 'A' && c <= 'Z') 985 return true; 986 987 if (c == '_' || (c >= '0' && c <= '9')) 988 return true; 989 990 if (c < 0x80) 991 return false; 992 993 return is_identifier_part_character_slow_part (c); 994 } 995 is_identifier_start_character_slow_part(char c)996 static bool is_identifier_start_character_slow_part (char c) 997 { 998 switch (Char.GetUnicodeCategory (c)) { 999 case UnicodeCategory.LetterNumber: 1000 case UnicodeCategory.UppercaseLetter: 1001 case UnicodeCategory.LowercaseLetter: 1002 case UnicodeCategory.TitlecaseLetter: 1003 case UnicodeCategory.ModifierLetter: 1004 case UnicodeCategory.OtherLetter: 1005 return true; 1006 } 1007 return false; 1008 } 1009 is_identifier_part_character_slow_part(char c)1010 static bool is_identifier_part_character_slow_part (char c) 1011 { 1012 switch (Char.GetUnicodeCategory (c)) { 1013 // connecting-character: A Unicode character of the class Pc 1014 case UnicodeCategory.ConnectorPunctuation: 1015 1016 // combining-character: A Unicode character of classes Mn or Mc 1017 case UnicodeCategory.NonSpacingMark: 1018 case UnicodeCategory.SpacingCombiningMark: 1019 1020 // decimal-digit-character: A Unicode character of the class Nd 1021 case UnicodeCategory.DecimalDigitNumber: 1022 1023 // plus is_identifier_start_character_slow_part 1024 case UnicodeCategory.LetterNumber: 1025 case UnicodeCategory.UppercaseLetter: 1026 case UnicodeCategory.LowercaseLetter: 1027 case UnicodeCategory.TitlecaseLetter: 1028 case UnicodeCategory.ModifierLetter: 1029 case UnicodeCategory.OtherLetter: 1030 return true; 1031 1032 // formatting-character: A Unicode character of the class Cf 1033 case UnicodeCategory.Format: 1034 // csc bug compatibility which recognizes it as a whitespace 1035 return c != 0xFEFF; 1036 } 1037 1038 return false; 1039 } 1040 IsKeyword(string s)1041 public static bool IsKeyword (string s) 1042 { 1043 return keyword_strings.Contains (s); 1044 } 1045 1046 // 1047 // Open parens micro parser 1048 // TokenizeOpenParens()1049 int TokenizeOpenParens () 1050 { 1051 int ptoken; 1052 current_token = -1; 1053 1054 int bracket_level = 0; 1055 bool is_type = false; 1056 bool can_be_type = false; 1057 bool at_least_one_comma = false; 1058 1059 while (true) { 1060 ptoken = current_token; 1061 token (); 1062 1063 switch (current_token) { 1064 case Token.CLOSE_PARENS: 1065 token (); 1066 1067 // 1068 // Expression inside parens is lambda, (int i) => 1069 // 1070 if (current_token == Token.ARROW) 1071 return Token.OPEN_PARENS_LAMBDA; 1072 1073 // 1074 // Expression inside parens is deconstruct expression, (a, x.y) = ... 1075 // 1076 if (current_token == Token.ASSIGN && at_least_one_comma) 1077 return Token.OPEN_PARENS_DECONSTRUCT; 1078 1079 // 1080 // Expression inside parens is single type, (int[]) 1081 // 1082 if (is_type) { 1083 if (current_token == Token.SEMICOLON) 1084 return Token.OPEN_PARENS; 1085 1086 return Token.OPEN_PARENS_CAST; 1087 } 1088 1089 // 1090 // Expression is possible cast, look at next token, (T)null 1091 // 1092 if (can_be_type) { 1093 switch (current_token) { 1094 case Token.OPEN_PARENS: 1095 case Token.BANG: 1096 case Token.TILDE: 1097 case Token.IDENTIFIER: 1098 case Token.LITERAL: 1099 case Token.BASE: 1100 case Token.CHECKED: 1101 case Token.DELEGATE: 1102 case Token.FALSE: 1103 case Token.FIXED: 1104 case Token.NEW: 1105 case Token.NULL: 1106 case Token.SIZEOF: 1107 case Token.THIS: 1108 case Token.THROW: 1109 case Token.TRUE: 1110 case Token.TYPEOF: 1111 case Token.UNCHECKED: 1112 case Token.UNSAFE: 1113 case Token.DEFAULT: 1114 case Token.DEFAULT_VALUE: 1115 case Token.AWAIT: 1116 1117 // 1118 // These can be part of a member access 1119 // 1120 case Token.INT: 1121 case Token.UINT: 1122 case Token.SHORT: 1123 case Token.USHORT: 1124 case Token.LONG: 1125 case Token.ULONG: 1126 case Token.DOUBLE: 1127 case Token.FLOAT: 1128 case Token.CHAR: 1129 case Token.BYTE: 1130 case Token.DECIMAL: 1131 case Token.BOOL: 1132 case Token.STRING: 1133 case Token.SBYTE: 1134 return Token.OPEN_PARENS_CAST; 1135 } 1136 } 1137 return Token.OPEN_PARENS; 1138 1139 case Token.DOT: 1140 case Token.DOUBLE_COLON: 1141 if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT) 1142 goto default; 1143 1144 continue; 1145 1146 case Token.IDENTIFIER: 1147 case Token.AWAIT: 1148 switch (ptoken) { 1149 case Token.DOT: 1150 if (bracket_level == 0) { 1151 is_type = false; 1152 can_be_type = true; 1153 } 1154 1155 continue; 1156 case Token.OP_GENERICS_LT: 1157 case Token.COMMA: 1158 case Token.DOUBLE_COLON: 1159 case -1: 1160 if (bracket_level == 0) 1161 can_be_type = true; 1162 continue; 1163 default: 1164 can_be_type = is_type = false; 1165 continue; 1166 } 1167 1168 case Token.OBJECT: 1169 case Token.STRING: 1170 case Token.BOOL: 1171 case Token.DECIMAL: 1172 case Token.FLOAT: 1173 case Token.DOUBLE: 1174 case Token.SBYTE: 1175 case Token.BYTE: 1176 case Token.SHORT: 1177 case Token.USHORT: 1178 case Token.INT: 1179 case Token.UINT: 1180 case Token.LONG: 1181 case Token.ULONG: 1182 case Token.CHAR: 1183 case Token.VOID: 1184 if (bracket_level == 0) 1185 is_type = true; 1186 continue; 1187 1188 case Token.COMMA: 1189 if (bracket_level == 0) { 1190 bracket_level = 100; 1191 can_be_type = is_type = false; 1192 at_least_one_comma = true; 1193 } 1194 continue; 1195 1196 case Token.OP_GENERICS_LT: 1197 case Token.OPEN_BRACKET: 1198 if (bracket_level++ == 0) 1199 is_type = true; 1200 continue; 1201 1202 case Token.OP_GENERICS_GT: 1203 case Token.CLOSE_BRACKET: 1204 --bracket_level; 1205 continue; 1206 1207 case Token.INTERR_NULLABLE: 1208 case Token.STAR: 1209 if (bracket_level == 0) 1210 is_type = true; 1211 continue; 1212 1213 case Token.REF: 1214 case Token.OUT: 1215 can_be_type = is_type = false; 1216 continue; 1217 1218 default: 1219 return Token.OPEN_PARENS; 1220 } 1221 } 1222 } 1223 IsValidIdentifier(string s)1224 public static bool IsValidIdentifier (string s) 1225 { 1226 if (s == null || s.Length == 0) 1227 return false; 1228 1229 if (!is_identifier_start_character (s [0])) 1230 return false; 1231 1232 for (int i = 1; i < s.Length; i ++) 1233 if (! is_identifier_part_character (s [i])) 1234 return false; 1235 1236 return true; 1237 } 1238 parse_less_than(ref int genericDimension)1239 bool parse_less_than (ref int genericDimension) 1240 { 1241 start: 1242 int the_token = token (); 1243 if (the_token == Token.OPEN_BRACKET) { 1244 while (true) { 1245 the_token = token (); 1246 if (the_token == Token.EOF) 1247 return true; 1248 1249 if (the_token == Token.CLOSE_BRACKET) 1250 break; 1251 } 1252 the_token = token (); 1253 } else if (the_token == Token.IN || the_token == Token.OUT) { 1254 the_token = token (); 1255 } 1256 switch (the_token) { 1257 case Token.IDENTIFIER: 1258 case Token.OBJECT: 1259 case Token.STRING: 1260 case Token.BOOL: 1261 case Token.DECIMAL: 1262 case Token.FLOAT: 1263 case Token.DOUBLE: 1264 case Token.SBYTE: 1265 case Token.BYTE: 1266 case Token.SHORT: 1267 case Token.USHORT: 1268 case Token.INT: 1269 case Token.UINT: 1270 case Token.LONG: 1271 case Token.ULONG: 1272 case Token.CHAR: 1273 case Token.VOID: 1274 break; 1275 case Token.OP_GENERICS_GT: 1276 genericDimension = 1; 1277 return true; 1278 case Token.IN: 1279 case Token.OUT: 1280 return true; 1281 case Token.COMMA: 1282 do { 1283 ++genericDimension; 1284 the_token = token (); 1285 } while (the_token == Token.COMMA); 1286 1287 if (the_token == Token.OP_GENERICS_GT) { 1288 ++genericDimension; 1289 return true; 1290 } 1291 1292 return false; 1293 case Token.OPEN_PARENS: 1294 int parens_count = 1; 1295 while (true) { 1296 switch (token ()) { 1297 case Token.COMMA: 1298 // tuple declaration after < 1299 if (parens_count == 1) 1300 return true; 1301 continue; 1302 case Token.OPEN_PARENS: 1303 ++parens_count; 1304 continue; 1305 case Token.CLOSE_PARENS: 1306 if (--parens_count <= 0) 1307 return false; 1308 continue; 1309 case Token.OP_GENERICS_GT: 1310 case Token.EOF: 1311 return false; 1312 } 1313 } 1314 1315 default: 1316 return false; 1317 } 1318 again: 1319 the_token = token (); 1320 1321 if (the_token == Token.OP_GENERICS_GT) 1322 return true; 1323 else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON) 1324 goto start; 1325 else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR) 1326 goto again; 1327 else if (the_token == Token.OP_GENERICS_LT) { 1328 int unused = 0; 1329 if (!parse_less_than (ref unused)) 1330 return false; 1331 goto again; 1332 } else if (the_token == Token.OPEN_BRACKET) { 1333 rank_specifiers: 1334 the_token = token (); 1335 if (the_token == Token.CLOSE_BRACKET) 1336 goto again; 1337 else if (the_token == Token.COMMA) 1338 goto rank_specifiers; 1339 return false; 1340 } 1341 1342 return false; 1343 } 1344 peek_token()1345 public int peek_token () 1346 { 1347 int the_token; 1348 1349 PushPosition (); 1350 the_token = token (); 1351 PopPosition (); 1352 1353 return the_token; 1354 } 1355 1356 // 1357 // Tonizes `?' using custom disambiguous rules to return one 1358 // of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR 1359 // 1360 // Tricky expression looks like: 1361 // 1362 // Foo ? a = x ? b : c; 1363 // TokenizePossibleNullableType()1364 int TokenizePossibleNullableType () 1365 { 1366 if (parsing_block == 0 || parsing_type > 0) 1367 return Token.INTERR_NULLABLE; 1368 1369 int d = peek_char (); 1370 if (d == '?') { 1371 get_char (); 1372 return Token.OP_COALESCING; 1373 } 1374 1375 if (d == '.') { 1376 d = reader.Peek (); 1377 return d >= '0' && d <= '9' ? Token.INTERR : Token.INTERR_OPERATOR; 1378 } 1379 1380 if (d != ' ') { 1381 if (d == ',' || d == ';' || d == '>') 1382 return Token.INTERR_NULLABLE; 1383 if (d == '*' || (d >= '0' && d <= '9')) 1384 return Token.INTERR; 1385 } 1386 1387 PushPosition (); 1388 current_token = Token.NONE; 1389 int next_token; 1390 int parens = 0; 1391 int generics = 0; 1392 int brackets = 0; 1393 1394 var nt = xtoken (); 1395 switch (nt) { 1396 case Token.DOT: 1397 case Token.OPEN_BRACKET_EXPR: 1398 next_token = Token.INTERR_OPERATOR; 1399 break; 1400 case Token.LITERAL: 1401 case Token.TRUE: 1402 case Token.FALSE: 1403 case Token.NULL: 1404 case Token.THIS: 1405 case Token.NEW: 1406 case Token.INTERPOLATED_STRING: 1407 case Token.THROW: 1408 case Token.DEFAULT_COLON: 1409 next_token = Token.INTERR; 1410 break; 1411 1412 case Token.SEMICOLON: 1413 case Token.COMMA: 1414 case Token.CLOSE_PARENS: 1415 case Token.OPEN_BRACKET: 1416 case Token.OP_GENERICS_GT: 1417 case Token.INTERR: 1418 case Token.OP_COALESCING: 1419 case Token.COLON: 1420 next_token = Token.INTERR_NULLABLE; 1421 break; 1422 1423 case Token.OPEN_PARENS: 1424 case Token.OPEN_PARENS_CAST: 1425 case Token.OPEN_PARENS_LAMBDA: 1426 next_token = -1; 1427 ++parens; 1428 break; 1429 1430 case Token.OP_GENERICS_LT: 1431 case Token.OP_GENERICS_LT_DECL: 1432 case Token.GENERIC_DIMENSION: 1433 next_token = -1; 1434 ++generics; 1435 break; 1436 1437 default: 1438 next_token = -1; 1439 break; 1440 } 1441 1442 if (next_token == -1) { 1443 switch (xtoken ()) { 1444 case Token.COMMA: 1445 case Token.SEMICOLON: 1446 case Token.OPEN_BRACE: 1447 case Token.IN: 1448 next_token = Token.INTERR_NULLABLE; 1449 break; 1450 1451 case Token.COLON: 1452 next_token = Token.INTERR; 1453 break; 1454 1455 case Token.OPEN_PARENS: 1456 case Token.OPEN_PARENS_CAST: 1457 case Token.OPEN_PARENS_LAMBDA: 1458 ++parens; 1459 goto default; 1460 1461 case Token.OPEN_BRACKET: 1462 case Token.OPEN_BRACKET_EXPR: 1463 ++brackets; 1464 goto default; 1465 1466 case Token.CLOSE_PARENS: 1467 --parens; 1468 goto default; 1469 1470 case Token.OP_GENERICS_LT: 1471 case Token.OP_GENERICS_LT_DECL: 1472 case Token.GENERIC_DIMENSION: 1473 ++generics; 1474 goto default; 1475 1476 default: 1477 int ntoken; 1478 int interrs = 1; 1479 int colons = 0; 1480 int braces = 0; 1481 // 1482 // All shorcuts failed, do it hard way 1483 // 1484 while ((ntoken = xtoken ()) != Token.EOF) { 1485 switch (ntoken) { 1486 case Token.OPEN_BRACE: 1487 ++braces; 1488 continue; 1489 case Token.OPEN_PARENS: 1490 case Token.OPEN_PARENS_CAST: 1491 case Token.OPEN_PARENS_LAMBDA: 1492 ++parens; 1493 continue; 1494 case Token.CLOSE_BRACE: 1495 --braces; 1496 continue; 1497 case Token.OP_GENERICS_LT: 1498 case Token.OP_GENERICS_LT_DECL: 1499 case Token.GENERIC_DIMENSION: 1500 ++generics; 1501 continue; 1502 case Token.OPEN_BRACKET: 1503 case Token.OPEN_BRACKET_EXPR: 1504 ++brackets; 1505 continue; 1506 case Token.CLOSE_BRACKET: 1507 --brackets; 1508 continue; 1509 case Token.CLOSE_PARENS: 1510 if (parens > 0) { 1511 --parens; 1512 continue; 1513 } 1514 1515 PopPosition (); 1516 return Token.INTERR_NULLABLE; 1517 1518 case Token.OP_GENERICS_GT: 1519 if (generics > 0) { 1520 --generics; 1521 continue; 1522 } 1523 1524 PopPosition (); 1525 return Token.INTERR_NULLABLE; 1526 } 1527 1528 if (braces != 0) 1529 continue; 1530 1531 if (ntoken == Token.SEMICOLON) 1532 break; 1533 1534 if (parens != 0) 1535 continue; 1536 1537 if (ntoken == Token.COMMA) { 1538 if (generics != 0 || brackets != 0) 1539 continue; 1540 1541 PopPosition (); 1542 return Token.INTERR_NULLABLE; 1543 } 1544 1545 if (ntoken == Token.COLON) { 1546 if (++colons == interrs) 1547 break; 1548 continue; 1549 } 1550 1551 if (ntoken == Token.INTERR) { 1552 ++interrs; 1553 continue; 1554 } 1555 } 1556 1557 next_token = colons != interrs && braces == 0 ? Token.INTERR_NULLABLE : Token.INTERR; 1558 break; 1559 } 1560 } 1561 1562 PopPosition (); 1563 return next_token; 1564 } 1565 decimal_digits(int c)1566 bool decimal_digits (int c) 1567 { 1568 int d; 1569 bool seen_digits = false; 1570 1571 if (c != -1){ 1572 if (number_pos == MaxNumberLength) 1573 Error_NumericConstantTooLong (); 1574 number_builder [number_pos++] = (char) c; 1575 } 1576 1577 // 1578 // We use peek_char2, because decimal_digits needs to do a 1579 // 2-character look-ahead (5.ToString for example). 1580 // 1581 while ((d = peek_char2 ()) != -1){ 1582 if (d >= '0' && d <= '9'){ 1583 if (number_pos == MaxNumberLength) 1584 Error_NumericConstantTooLong (); 1585 number_builder [number_pos++] = (char) d; 1586 get_char (); 1587 seen_digits = true; 1588 } else 1589 break; 1590 } 1591 1592 return seen_digits; 1593 } 1594 is_hex(int e)1595 static bool is_hex (int e) 1596 { 1597 return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f'); 1598 } 1599 real_type_suffix(int c)1600 static TypeCode real_type_suffix (int c) 1601 { 1602 switch (c){ 1603 case 'F': case 'f': 1604 return TypeCode.Single; 1605 case 'D': case 'd': 1606 return TypeCode.Double; 1607 case 'M': case 'm': 1608 return TypeCode.Decimal; 1609 default: 1610 return TypeCode.Empty; 1611 } 1612 } 1613 integer_type_suffix(ulong ul, int c, Location loc)1614 ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc) 1615 { 1616 bool is_unsigned = false; 1617 bool is_long = false; 1618 1619 if (c != -1){ 1620 bool scanning = true; 1621 do { 1622 switch (c){ 1623 case 'U': case 'u': 1624 if (is_unsigned) 1625 scanning = false; 1626 is_unsigned = true; 1627 get_char (); 1628 break; 1629 1630 case 'l': 1631 if (!is_unsigned){ 1632 // 1633 // if we have not seen anything in between 1634 // report this error 1635 // 1636 Report.Warning (78, 4, Location, "The `l' suffix is easily confused with the digit `1' (use `L' for clarity)"); 1637 } 1638 1639 goto case 'L'; 1640 1641 case 'L': 1642 if (is_long) 1643 scanning = false; 1644 is_long = true; 1645 get_char (); 1646 break; 1647 1648 default: 1649 scanning = false; 1650 break; 1651 } 1652 c = peek_char (); 1653 } while (scanning); 1654 } 1655 1656 if (is_long && is_unsigned){ 1657 return new ULongLiteral (context.BuiltinTypes, ul, loc); 1658 } 1659 1660 if (is_unsigned){ 1661 // uint if possible, or ulong else. 1662 1663 if ((ul & 0xffffffff00000000) == 0) 1664 return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc); 1665 else 1666 return new ULongLiteral (context.BuiltinTypes, ul, loc); 1667 } else if (is_long){ 1668 // long if possible, ulong otherwise 1669 if ((ul & 0x8000000000000000) != 0) 1670 return new ULongLiteral (context.BuiltinTypes, ul, loc); 1671 else 1672 return new LongLiteral (context.BuiltinTypes, (long) ul, loc); 1673 } else { 1674 // int, uint, long or ulong in that order 1675 if ((ul & 0xffffffff00000000) == 0){ 1676 uint ui = (uint) ul; 1677 1678 if ((ui & 0x80000000) != 0) 1679 return new UIntLiteral (context.BuiltinTypes, ui, loc); 1680 else 1681 return new IntLiteral (context.BuiltinTypes, (int) ui, loc); 1682 } else { 1683 if ((ul & 0x8000000000000000) != 0) 1684 return new ULongLiteral (context.BuiltinTypes, ul, loc); 1685 else 1686 return new LongLiteral (context.BuiltinTypes, (long) ul, loc); 1687 } 1688 } 1689 } 1690 1691 // 1692 // given `c' as the next char in the input decide whether 1693 // we need to convert to a special type, and then choose 1694 // the best representation for the integer 1695 // adjust_int(int c, Location loc)1696 ILiteralConstant adjust_int (int c, Location loc) 1697 { 1698 try { 1699 if (number_pos > 9){ 1700 ulong ul = (uint) (number_builder [0] - '0'); 1701 1702 for (int i = 1; i < number_pos; i++){ 1703 ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0'))); 1704 } 1705 1706 return integer_type_suffix (ul, c, loc); 1707 } else { 1708 uint ui = (uint) (number_builder [0] - '0'); 1709 1710 for (int i = 1; i < number_pos; i++){ 1711 ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0'))); 1712 } 1713 1714 return integer_type_suffix (ui, c, loc); 1715 } 1716 } catch (OverflowException) { 1717 Error_NumericConstantTooLong (); 1718 return new IntLiteral (context.BuiltinTypes, 0, loc); 1719 } 1720 catch (FormatException) { 1721 Report.Error (1013, Location, "Invalid number"); 1722 return new IntLiteral (context.BuiltinTypes, 0, loc); 1723 } 1724 } 1725 adjust_real(TypeCode t, Location loc)1726 ILiteralConstant adjust_real (TypeCode t, Location loc) 1727 { 1728 string s = new string (number_builder, 0, number_pos); 1729 const string error_details = "Floating-point constant is outside the range of type `{0}'"; 1730 1731 switch (t){ 1732 case TypeCode.Decimal: 1733 try { 1734 return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc); 1735 } catch (OverflowException) { 1736 Report.Error (594, Location, error_details, "decimal"); 1737 return new DecimalLiteral (context.BuiltinTypes, 0, loc); 1738 } 1739 case TypeCode.Single: 1740 try { 1741 return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc); 1742 } catch (OverflowException) { 1743 Report.Error (594, Location, error_details, "float"); 1744 return new FloatLiteral (context.BuiltinTypes, 0, loc); 1745 } 1746 default: 1747 try { 1748 return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc); 1749 } catch (OverflowException) { 1750 Report.Error (594, loc, error_details, "double"); 1751 return new DoubleLiteral (context.BuiltinTypes, 0, loc); 1752 } 1753 } 1754 } 1755 handle_hex(Location loc)1756 ILiteralConstant handle_hex (Location loc) 1757 { 1758 int d; 1759 ulong ul; 1760 1761 get_char (); 1762 while ((d = peek_char ()) != -1){ 1763 if (is_hex (d)){ 1764 number_builder [number_pos++] = (char) d; 1765 get_char (); 1766 } else 1767 break; 1768 } 1769 1770 string s = new String (number_builder, 0, number_pos); 1771 1772 try { 1773 if (number_pos <= 8) 1774 ul = System.UInt32.Parse (s, NumberStyles.HexNumber); 1775 else 1776 ul = System.UInt64.Parse (s, NumberStyles.HexNumber); 1777 1778 return integer_type_suffix (ul, peek_char (), loc); 1779 } catch (OverflowException){ 1780 Error_NumericConstantTooLong (); 1781 return new IntLiteral (context.BuiltinTypes, 0, loc); 1782 } 1783 catch (FormatException) { 1784 Report.Error (1013, Location, "Invalid number"); 1785 return new IntLiteral (context.BuiltinTypes, 0, loc); 1786 } 1787 } 1788 1789 // 1790 // Invoked if we know we have .digits or digits 1791 // is_number(int c, bool dotLead)1792 int is_number (int c, bool dotLead) 1793 { 1794 ILiteralConstant res; 1795 1796 #if FULL_AST 1797 int read_start = reader.Position - 1; 1798 if (dotLead) { 1799 // 1800 // Caller did peek_char 1801 // 1802 --read_start; 1803 } 1804 #endif 1805 number_pos = 0; 1806 var loc = Location; 1807 1808 if (!dotLead){ 1809 if (c == '0'){ 1810 int peek = peek_char (); 1811 1812 if (peek == 'x' || peek == 'X') { 1813 val = res = handle_hex (loc); 1814 #if FULL_AST 1815 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1); 1816 #endif 1817 1818 return Token.LITERAL; 1819 } 1820 } 1821 decimal_digits (c); 1822 c = peek_char (); 1823 } 1824 1825 // 1826 // We need to handle the case of 1827 // "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER) 1828 // 1829 bool is_real = false; 1830 if (c == '.'){ 1831 if (!dotLead) 1832 get_char (); 1833 1834 if (decimal_digits ('.')){ 1835 is_real = true; 1836 c = peek_char (); 1837 } else { 1838 putback ('.'); 1839 number_pos--; 1840 val = res = adjust_int (-1, loc); 1841 1842 #if FULL_AST 1843 res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1); 1844 #endif 1845 return Token.LITERAL; 1846 } 1847 } 1848 1849 if (c == 'e' || c == 'E'){ 1850 is_real = true; 1851 get_char (); 1852 if (number_pos == MaxNumberLength) 1853 Error_NumericConstantTooLong (); 1854 number_builder [number_pos++] = (char) c; 1855 c = get_char (); 1856 1857 if (c == '+'){ 1858 if (number_pos == MaxNumberLength) 1859 Error_NumericConstantTooLong (); 1860 number_builder [number_pos++] = '+'; 1861 c = -1; 1862 } else if (c == '-') { 1863 if (number_pos == MaxNumberLength) 1864 Error_NumericConstantTooLong (); 1865 number_builder [number_pos++] = '-'; 1866 c = -1; 1867 } else { 1868 if (number_pos == MaxNumberLength) 1869 Error_NumericConstantTooLong (); 1870 number_builder [number_pos++] = '+'; 1871 } 1872 1873 decimal_digits (c); 1874 c = peek_char (); 1875 } 1876 1877 var type = real_type_suffix (c); 1878 if (type == TypeCode.Empty && !is_real) { 1879 res = adjust_int (c, loc); 1880 } else { 1881 is_real = true; 1882 1883 if (type != TypeCode.Empty) { 1884 get_char (); 1885 } 1886 1887 res = adjust_real (type, loc); 1888 } 1889 1890 val = res; 1891 1892 #if FULL_AST 1893 var chars = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty && c > 0 ? 1 : 0)); 1894 if (chars[chars.Length - 1] == '\r') 1895 Array.Resize (ref chars, chars.Length - 1); 1896 res.ParsedValue = chars; 1897 #endif 1898 1899 return Token.LITERAL; 1900 } 1901 1902 // 1903 // Accepts exactly count (4 or 8) hex, no more no less 1904 // getHex(int count, out int surrogate, out bool error)1905 int getHex (int count, out int surrogate, out bool error) 1906 { 1907 int i; 1908 int total = 0; 1909 int c; 1910 int top = count != -1 ? count : 4; 1911 1912 get_char (); 1913 error = false; 1914 surrogate = 0; 1915 for (i = 0; i < top; i++){ 1916 c = get_char (); 1917 1918 if (c >= '0' && c <= '9') 1919 c = (int) c - (int) '0'; 1920 else if (c >= 'A' && c <= 'F') 1921 c = (int) c - (int) 'A' + 10; 1922 else if (c >= 'a' && c <= 'f') 1923 c = (int) c - (int) 'a' + 10; 1924 else { 1925 error = true; 1926 return 0; 1927 } 1928 1929 total = (total * 16) + c; 1930 if (count == -1){ 1931 int p = peek_char (); 1932 if (p == -1) 1933 break; 1934 if (!is_hex ((char)p)) 1935 break; 1936 } 1937 } 1938 1939 if (top == 8) { 1940 if (total > 0x0010FFFF) { 1941 error = true; 1942 return 0; 1943 } 1944 1945 if (total >= 0x00010000) { 1946 surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00); 1947 total = ((total - 0x00010000) / 0x0400 + 0xD800); 1948 } 1949 } 1950 1951 return total; 1952 } 1953 escape(int c, out int surrogate)1954 int escape (int c, out int surrogate) 1955 { 1956 bool error; 1957 int d; 1958 int v; 1959 1960 d = peek_char (); 1961 if (c != '\\') { 1962 surrogate = 0; 1963 return c; 1964 } 1965 1966 switch (d){ 1967 case 'a': 1968 v = '\a'; break; 1969 case 'b': 1970 v = '\b'; break; 1971 case 'n': 1972 v = '\n'; break; 1973 case 't': 1974 v = '\t'; break; 1975 case 'v': 1976 v = '\v'; break; 1977 case 'r': 1978 v = '\r'; break; 1979 case '\\': 1980 v = '\\'; break; 1981 case 'f': 1982 v = '\f'; break; 1983 case '0': 1984 v = 0; break; 1985 case '"': 1986 v = '"'; break; 1987 case '\'': 1988 v = '\''; break; 1989 case 'x': 1990 v = getHex (-1, out surrogate, out error); 1991 if (error) 1992 goto default; 1993 return v; 1994 case 'u': 1995 case 'U': 1996 return EscapeUnicode (d, out surrogate); 1997 default: 1998 surrogate = 0; 1999 Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ()); 2000 return d; 2001 } 2002 2003 get_char (); 2004 surrogate = 0; 2005 return v; 2006 } 2007 EscapeUnicode(int ch, out int surrogate)2008 int EscapeUnicode (int ch, out int surrogate) 2009 { 2010 bool error; 2011 if (ch == 'U') { 2012 ch = getHex (8, out surrogate, out error); 2013 } else { 2014 ch = getHex (4, out surrogate, out error); 2015 } 2016 2017 if (error) 2018 Report.Error (1009, Location, "Unrecognized escape sequence"); 2019 2020 return ch; 2021 } 2022 get_char()2023 int get_char () 2024 { 2025 int x; 2026 if (putback_char != -1) { 2027 x = putback_char; 2028 putback_char = -1; 2029 } else { 2030 x = reader.Read (); 2031 } 2032 2033 if (x <= 13) { 2034 if (x == '\r') { 2035 if (peek_char () == '\n') { 2036 putback_char = -1; 2037 } 2038 2039 x = '\n'; 2040 advance_line (); 2041 } else if (x == '\n') { 2042 advance_line (); 2043 } else { 2044 col++; 2045 } 2046 } else if (x >= UnicodeLS && x <= UnicodePS) { 2047 advance_line (); 2048 } else { 2049 col++; 2050 } 2051 2052 return x; 2053 } 2054 advance_line()2055 void advance_line () 2056 { 2057 line++; 2058 ref_line++; 2059 previous_col = col; 2060 col = 0; 2061 } 2062 peek_char()2063 int peek_char () 2064 { 2065 if (putback_char == -1) 2066 putback_char = reader.Read (); 2067 return putback_char; 2068 } 2069 peek_char2()2070 int peek_char2 () 2071 { 2072 if (putback_char != -1) 2073 return putback_char; 2074 return reader.Peek (); 2075 } 2076 putback(int c)2077 public void putback (int c) 2078 { 2079 if (putback_char != -1) { 2080 throw new InternalErrorException (string.Format ("Secondary putback [{0}] putting back [{1}] is not allowed", (char)putback_char, (char) c), Location); 2081 } 2082 2083 if (c == '\n' || col == 0 || (c >= UnicodeLS && c <= UnicodePS)) { 2084 // It won't happen though. 2085 line--; 2086 ref_line--; 2087 col = previous_col; 2088 } 2089 else 2090 col--; 2091 putback_char = c; 2092 } 2093 advance()2094 public bool advance () 2095 { 2096 return peek_char () != -1 || CompleteOnEOF; 2097 } 2098 2099 public Object Value { 2100 get { 2101 return val; 2102 } 2103 } 2104 value()2105 public Object value () 2106 { 2107 return val; 2108 } 2109 token()2110 public int token () 2111 { 2112 current_token = xtoken (); 2113 return current_token; 2114 } 2115 TokenizePreprocessorKeyword(out int c)2116 int TokenizePreprocessorKeyword (out int c) 2117 { 2118 // skip over white space 2119 do { 2120 c = get_char (); 2121 } while (c == ' ' || c == '\t'); 2122 2123 2124 int pos = 0; 2125 while (c != -1 && c >= 'a' && c <= 'z') { 2126 id_builder[pos++] = (char) c; 2127 c = get_char (); 2128 if (c == '\\') { 2129 int peek = peek_char (); 2130 if (peek == 'U' || peek == 'u') { 2131 int surrogate; 2132 c = EscapeUnicode (c, out surrogate); 2133 if (surrogate != 0) { 2134 if (is_identifier_part_character ((char) c)) { 2135 id_builder[pos++] = (char) c; 2136 } 2137 c = surrogate; 2138 } 2139 } 2140 } 2141 } 2142 2143 return pos; 2144 } 2145 get_cmd_arg(out string arg)2146 PreprocessorDirective get_cmd_arg (out string arg) 2147 { 2148 int c; 2149 2150 tokens_seen = false; 2151 arg = ""; 2152 2153 var cmd = GetPreprocessorDirective (id_builder, TokenizePreprocessorKeyword (out c)); 2154 2155 if ((cmd & PreprocessorDirective.CustomArgumentsParsing) != 0) 2156 return cmd; 2157 2158 // skip over white space 2159 while (c == ' ' || c == '\t') 2160 c = get_char (); 2161 2162 int has_identifier_argument = (int)(cmd & PreprocessorDirective.RequiresArgument); 2163 int pos = 0; 2164 2165 while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) { 2166 if (c == '\\' && has_identifier_argument >= 0) { 2167 if (has_identifier_argument != 0) { 2168 has_identifier_argument = 1; 2169 2170 int peek = peek_char (); 2171 if (peek == 'U' || peek == 'u') { 2172 int surrogate; 2173 c = EscapeUnicode (c, out surrogate); 2174 if (surrogate != 0) { 2175 if (is_identifier_part_character ((char) c)) { 2176 if (pos == value_builder.Length) 2177 Array.Resize (ref value_builder, pos * 2); 2178 2179 value_builder[pos++] = (char) c; 2180 } 2181 c = surrogate; 2182 } 2183 } 2184 } else { 2185 has_identifier_argument = -1; 2186 } 2187 } else if (c == '/' && peek_char () == '/') { 2188 // 2189 // Eat single-line comments 2190 // 2191 get_char (); 2192 ReadToEndOfLine (); 2193 break; 2194 } 2195 2196 if (pos == value_builder.Length) 2197 Array.Resize (ref value_builder, pos * 2); 2198 2199 value_builder[pos++] = (char) c; 2200 c = get_char (); 2201 } 2202 2203 if (pos != 0) { 2204 if (pos > MaxIdentifierLength) 2205 arg = new string (value_builder, 0, pos); 2206 else 2207 arg = InternIdentifier (value_builder, pos); 2208 2209 // Eat any trailing whitespaces 2210 arg = arg.Trim (simple_whitespaces); 2211 } 2212 2213 return cmd; 2214 } 2215 2216 // 2217 // Handles the #line directive 2218 // PreProcessLine()2219 bool PreProcessLine () 2220 { 2221 Location loc = Location; 2222 2223 int c; 2224 2225 int length = TokenizePreprocessorKeyword (out c); 2226 if (length == line_default.Length) { 2227 if (!IsTokenIdentifierEqual (line_default)) 2228 return false; 2229 2230 current_source = source_file.SourceFile; 2231 if (!hidden_block_start.IsNull) { 2232 current_source.RegisterHiddenScope (hidden_block_start, loc); 2233 hidden_block_start = Location.Null; 2234 } 2235 2236 ref_line = line; 2237 return true; 2238 } 2239 2240 if (length == line_hidden.Length) { 2241 if (!IsTokenIdentifierEqual (line_hidden)) 2242 return false; 2243 2244 if (hidden_block_start.IsNull) 2245 hidden_block_start = loc; 2246 2247 return true; 2248 } 2249 2250 if (length != 0 || c < '0' || c > '9') { 2251 // 2252 // Eat any remaining characters to continue parsing on next line 2253 // 2254 ReadToEndOfLine (); 2255 return false; 2256 } 2257 2258 int new_line = TokenizeNumber (c); 2259 if (new_line < 1) { 2260 // 2261 // Eat any remaining characters to continue parsing on next line 2262 // 2263 ReadToEndOfLine (); 2264 return new_line != 0; 2265 } 2266 2267 c = get_char (); 2268 if (c == ' ') { 2269 // skip over white space 2270 do { 2271 c = get_char (); 2272 } while (c == ' ' || c == '\t'); 2273 } else if (c == '"') { 2274 c = 0; 2275 } 2276 2277 if (c != '\n' && c != '/' && c != '"' && c != UnicodeLS && c != UnicodePS) { 2278 // 2279 // Eat any remaining characters to continue parsing on next line 2280 // 2281 ReadToEndOfLine (); 2282 2283 Report.Error (1578, loc, "Filename, single-line comment or end-of-line expected"); 2284 return true; 2285 } 2286 2287 string new_file_name = null; 2288 if (c == '"') { 2289 new_file_name = TokenizeFileName (ref c); 2290 2291 // skip over white space 2292 while (c == ' ' || c == '\t') { 2293 c = get_char (); 2294 } 2295 } 2296 2297 if (c == '\n' || c == UnicodeLS || c == UnicodePS) { 2298 2299 } else if (c == '/') { 2300 ReadSingleLineComment (); 2301 } else { 2302 // 2303 // Eat any remaining characters to continue parsing on next line 2304 // 2305 ReadToEndOfLine (); 2306 2307 Error_EndLineExpected (); 2308 return true; 2309 } 2310 2311 if (new_file_name != null) { 2312 current_source = context.LookupFile (source_file, new_file_name); 2313 source_file.AddIncludeFile (current_source); 2314 } 2315 2316 if (!hidden_block_start.IsNull) { 2317 current_source.RegisterHiddenScope (hidden_block_start, loc); 2318 hidden_block_start = Location.Null; 2319 } 2320 2321 ref_line = new_line; 2322 return true; 2323 } 2324 2325 // 2326 // Handles #define and #undef 2327 // PreProcessDefinition(bool is_define, string ident, bool caller_is_taking)2328 void PreProcessDefinition (bool is_define, string ident, bool caller_is_taking) 2329 { 2330 if (ident.Length == 0 || ident == "true" || ident == "false"){ 2331 Report.Error (1001, Location, "Missing identifier to pre-processor directive"); 2332 return; 2333 } 2334 2335 if (ident.IndexOfAny (simple_whitespaces) != -1){ 2336 Error_EndLineExpected (); 2337 return; 2338 } 2339 2340 if (!is_identifier_start_character (ident [0])) 2341 Report.Error (1001, Location, "Identifier expected: {0}", ident); 2342 2343 foreach (char c in ident.Substring (1)){ 2344 if (!is_identifier_part_character (c)){ 2345 Report.Error (1001, Location, "Identifier expected: {0}", ident); 2346 return; 2347 } 2348 } 2349 2350 if (!caller_is_taking) 2351 return; 2352 2353 if (is_define) { 2354 // 2355 // #define ident 2356 // 2357 if (context.Settings.IsConditionalSymbolDefined (ident)) 2358 return; 2359 2360 source_file.AddDefine (ident); 2361 } else { 2362 // 2363 // #undef ident 2364 // 2365 source_file.AddUndefine (ident); 2366 } 2367 } 2368 read_hex(out bool error)2369 byte read_hex (out bool error) 2370 { 2371 int total; 2372 int c = get_char (); 2373 2374 if ((c >= '0') && (c <= '9')) 2375 total = (int) c - (int) '0'; 2376 else if ((c >= 'A') && (c <= 'F')) 2377 total = (int) c - (int) 'A' + 10; 2378 else if ((c >= 'a') && (c <= 'f')) 2379 total = (int) c - (int) 'a' + 10; 2380 else { 2381 error = true; 2382 return 0; 2383 } 2384 2385 total *= 16; 2386 c = get_char (); 2387 2388 if ((c >= '0') && (c <= '9')) 2389 total += (int) c - (int) '0'; 2390 else if ((c >= 'A') && (c <= 'F')) 2391 total += (int) c - (int) 'A' + 10; 2392 else if ((c >= 'a') && (c <= 'f')) 2393 total += (int) c - (int) 'a' + 10; 2394 else { 2395 error = true; 2396 return 0; 2397 } 2398 2399 error = false; 2400 return (byte) total; 2401 } 2402 2403 // 2404 // Parses #pragma checksum 2405 // ParsePragmaChecksum()2406 bool ParsePragmaChecksum () 2407 { 2408 // 2409 // The syntax is ` "foo.txt" "{guid}" "hash"' 2410 // 2411 // guid is predefined hash algorithm guid {406ea660-64cf-4c82-b6f0-42d48172a799} for md5 2412 // 2413 int c = get_char (); 2414 2415 if (c != '"') 2416 return false; 2417 2418 string file_name = TokenizeFileName (ref c); 2419 2420 // TODO: Any white-spaces count 2421 if (c != ' ') 2422 return false; 2423 2424 SourceFile file = context.LookupFile (source_file, file_name); 2425 2426 if (get_char () != '"' || get_char () != '{') 2427 return false; 2428 2429 bool error; 2430 byte[] guid_bytes = new byte [16]; 2431 int i = 0; 2432 2433 for (; i < 4; i++) { 2434 guid_bytes [i] = read_hex (out error); 2435 if (error) 2436 return false; 2437 } 2438 2439 if (get_char () != '-') 2440 return false; 2441 2442 for (; i < 10; i++) { 2443 guid_bytes [i] = read_hex (out error); 2444 if (error) 2445 return false; 2446 2447 guid_bytes [i++] = read_hex (out error); 2448 if (error) 2449 return false; 2450 2451 if (get_char () != '-') 2452 return false; 2453 } 2454 2455 for (; i < 16; i++) { 2456 guid_bytes [i] = read_hex (out error); 2457 if (error) 2458 return false; 2459 } 2460 2461 if (get_char () != '}' || get_char () != '"') 2462 return false; 2463 2464 // TODO: Any white-spaces count 2465 c = get_char (); 2466 if (c != ' ') 2467 return false; 2468 2469 if (get_char () != '"') 2470 return false; 2471 2472 // Any length of checksum 2473 List<byte> checksum_bytes = new List<byte> (16); 2474 2475 var checksum_location = Location; 2476 c = peek_char (); 2477 while (c != '"' && c != -1) { 2478 checksum_bytes.Add (read_hex (out error)); 2479 if (error) 2480 return false; 2481 2482 c = peek_char (); 2483 } 2484 2485 if (c == '/') { 2486 ReadSingleLineComment (); 2487 } else if (get_char () != '"') { 2488 return false; 2489 } 2490 2491 if (context.Settings.GenerateDebugInfo) { 2492 var chsum = checksum_bytes.ToArray (); 2493 2494 if (file.HasChecksum) { 2495 if (!ArrayComparer.IsEqual (file.Checksum, chsum)) { 2496 // TODO: Report.SymbolRelatedToPreviousError 2497 Report.Warning (1697, 1, checksum_location, "Different checksum values specified for file `{0}'", file.Name); 2498 } 2499 } 2500 2501 file.SetChecksum (guid_bytes, chsum); 2502 current_source.AutoGenerated = true; 2503 } 2504 2505 return true; 2506 } 2507 IsTokenIdentifierEqual(char[] identifier)2508 bool IsTokenIdentifierEqual (char[] identifier) 2509 { 2510 for (int i = 0; i < identifier.Length; ++i) { 2511 if (identifier[i] != id_builder[i]) 2512 return false; 2513 } 2514 2515 return true; 2516 } 2517 ScanClosingInterpolationBrace()2518 bool ScanClosingInterpolationBrace () 2519 { 2520 PushPosition (); 2521 2522 bool? res = null; 2523 int str_quote = 0; 2524 do { 2525 var c = reader.Read (); 2526 switch (c) { 2527 case '\"': 2528 ++str_quote; 2529 break; 2530 case '\\': 2531 // Skip escaped " character 2532 c = reader.Read (); 2533 if (c == -1) 2534 res = false; 2535 break; 2536 case -1: 2537 res = false; 2538 break; 2539 case '}': 2540 if (str_quote % 2 == 1) { 2541 res = true; 2542 } 2543 2544 break; 2545 } 2546 } while (res == null); 2547 2548 PopPosition (); 2549 return res.Value; 2550 } 2551 TokenizeNumber(int value)2552 int TokenizeNumber (int value) 2553 { 2554 number_pos = 0; 2555 2556 decimal_digits (value); 2557 uint ui = (uint) (number_builder[0] - '0'); 2558 2559 try { 2560 for (int i = 1; i < number_pos; i++) { 2561 ui = checked ((ui * 10) + ((uint) (number_builder[i] - '0'))); 2562 } 2563 2564 return (int) ui; 2565 } catch (OverflowException) { 2566 Error_NumericConstantTooLong (); 2567 return -1; 2568 } 2569 } 2570 TokenizeFileName(ref int c)2571 string TokenizeFileName (ref int c) 2572 { 2573 var string_builder = new StringBuilder (); 2574 while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS) { 2575 c = get_char (); 2576 if (c == '"') { 2577 c = get_char (); 2578 break; 2579 } 2580 2581 string_builder.Append ((char) c); 2582 } 2583 2584 if (string_builder.Length == 0) { 2585 Report.Warning (1709, 1, Location, "Filename specified for preprocessor directive is empty"); 2586 } 2587 2588 2589 return string_builder.ToString (); 2590 } 2591 TokenizePragmaWarningIdentifier(ref int c, ref bool identifier)2592 int TokenizePragmaWarningIdentifier (ref int c, ref bool identifier) 2593 { 2594 if ((c >= '0' && c <= '9') || is_identifier_start_character (c)) { 2595 int number; 2596 2597 if (c >= '0' && c <= '9') { 2598 number_pos = 0; 2599 number = TokenizeNumber (c); 2600 2601 c = get_char (); 2602 2603 if (c != ' ' && c != '\t' && c != ',' && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS) { 2604 return ReadPragmaWarningComment (c); 2605 } 2606 } else { 2607 // 2608 // LAMESPEC v6: No spec what identifier really is in this context, it seems keywords are allowed too 2609 // 2610 int pos = 0; 2611 number = -1; 2612 id_builder [pos++] = (char)c; 2613 while (c < MaxIdentifierLength) { 2614 c = reader.Read (); 2615 id_builder [pos] = (char)c; 2616 2617 if (c >= '0' && c <= '9') { 2618 if (pos == 5 && id_builder [0] == 'C' && id_builder [1] == 'S') { 2619 // Recognize CSXXXX as C# XXXX warning 2620 number = 0; 2621 int pow = 1000; 2622 for (int i = 0; i < 4; ++i) { 2623 var ch = id_builder [i + 2]; 2624 if (ch < '0' || ch > '9') { 2625 number = -1; 2626 break; 2627 } 2628 2629 number += (ch - '0') * pow; 2630 pow /= 10; 2631 } 2632 } 2633 } else if (c == '\n' || c == UnicodeLS || c == UnicodePS) { 2634 advance_line (); 2635 break; 2636 } else if ((c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && c != '_') { 2637 break; 2638 } 2639 2640 ++pos; 2641 } 2642 2643 if (number < 0) { 2644 identifier = true; 2645 number = pos; 2646 } 2647 } 2648 2649 // skip over white space 2650 while (c == ' ' || c == '\t') 2651 c = get_char (); 2652 2653 if (c == ',') { 2654 c = get_char (); 2655 } 2656 2657 // skip over white space 2658 while (c == ' ' || c == '\t') 2659 c = get_char (); 2660 2661 return number; 2662 } 2663 2664 return ReadPragmaWarningComment (c); 2665 } 2666 ReadPragmaWarningComment(int c)2667 int ReadPragmaWarningComment (int c) 2668 { 2669 if (c == '/') { 2670 ReadSingleLineComment (); 2671 } else { 2672 Report.Warning (1692, 1, Location, "Invalid number"); 2673 2674 // Read everything till the end of the line or file 2675 ReadToEndOfLine (); 2676 } 2677 2678 return -1; 2679 } 2680 ReadToEndOfLine()2681 void ReadToEndOfLine () 2682 { 2683 int c; 2684 do { 2685 c = get_char (); 2686 } while (c != -1 && c != '\n' && c != UnicodeLS && c != UnicodePS); 2687 } 2688 ReadSingleLineComment()2689 void ReadSingleLineComment () 2690 { 2691 if (peek_char () != '/') 2692 Report.Warning (1696, 1, Location, "Single-line comment or end-of-line expected"); 2693 2694 // Read everything till the end of the line or file 2695 ReadToEndOfLine (); 2696 } 2697 2698 /// <summary> 2699 /// Handles #pragma directive 2700 /// </summary> ParsePragmaDirective()2701 void ParsePragmaDirective () 2702 { 2703 int c; 2704 int length = TokenizePreprocessorKeyword (out c); 2705 if (length == pragma_warning.Length && IsTokenIdentifierEqual (pragma_warning)) { 2706 length = TokenizePreprocessorKeyword (out c); 2707 2708 // 2709 // #pragma warning disable 2710 // #pragma warning restore 2711 // 2712 if (length == pragma_warning_disable.Length) { 2713 bool disable = IsTokenIdentifierEqual (pragma_warning_disable); 2714 if (disable || IsTokenIdentifierEqual (pragma_warning_restore)) { 2715 // skip over white space 2716 while (c == ' ' || c == '\t') 2717 c = get_char (); 2718 2719 var loc = Location; 2720 2721 if (c == '\n' || c == '/' || c == UnicodeLS || c == UnicodePS) { 2722 if (c == '/') 2723 ReadSingleLineComment (); 2724 2725 // 2726 // Disable/Restore all warnings 2727 // 2728 if (disable) { 2729 Report.RegisterWarningRegion (loc).WarningDisable (loc.Row); 2730 } else { 2731 Report.RegisterWarningRegion (loc).WarningEnable (loc.Row); 2732 } 2733 } else { 2734 // 2735 // Disable/Restore a warning or group of warnings 2736 // 2737 int code; 2738 do { 2739 bool identifier = false; 2740 code = TokenizePragmaWarningIdentifier (ref c, ref identifier); 2741 if (code > 0) { 2742 if (identifier) { 2743 // no-op, custom warnings cannot occur in mcs 2744 } else if (disable) { 2745 Report.RegisterWarningRegion (loc).WarningDisable (loc, code, context.Report); 2746 } else { 2747 Report.RegisterWarningRegion (loc).WarningEnable (loc, code, context); 2748 } 2749 } 2750 } while (code >= 0 && c != '\n' && c != -1 && c != UnicodeLS && c != UnicodePS); 2751 } 2752 2753 return; 2754 } 2755 } 2756 2757 Report.Warning (1634, 1, Location, "Expected disable or restore"); 2758 2759 // Eat any remaining characters on the line 2760 ReadToEndOfLine (); 2761 2762 return; 2763 } 2764 2765 // 2766 // #pragma checksum 2767 // 2768 if (length == pragma_checksum.Length && IsTokenIdentifierEqual (pragma_checksum)) { 2769 if (c != ' ' || !ParsePragmaChecksum ()) { 2770 Report.Warning (1695, 1, Location, 2771 "Invalid #pragma checksum syntax. Expected \"filename\" \"{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}\" \"XXXX...\""); 2772 } 2773 2774 return; 2775 } 2776 2777 Report.Warning (1633, 1, Location, "Unrecognized #pragma directive"); 2778 2779 // Eat any remaining characters on the line 2780 ReadToEndOfLine (); 2781 } 2782 eval_val(string s)2783 bool eval_val (string s) 2784 { 2785 if (s == "true") 2786 return true; 2787 if (s == "false") 2788 return false; 2789 2790 return source_file.IsConditionalDefined (s); 2791 } 2792 pp_primary(ref string s)2793 bool pp_primary (ref string s) 2794 { 2795 s = s.Trim (); 2796 int len = s.Length; 2797 2798 if (len > 0){ 2799 char c = s [0]; 2800 2801 if (c == '('){ 2802 s = s.Substring (1); 2803 bool val = pp_expr (ref s, false); 2804 if (s.Length > 0 && s [0] == ')'){ 2805 s = s.Substring (1); 2806 return val; 2807 } 2808 Error_InvalidDirective (); 2809 return false; 2810 } 2811 2812 if (is_identifier_start_character (c)){ 2813 int j = 1; 2814 2815 while (j < len){ 2816 c = s [j]; 2817 2818 if (is_identifier_part_character (c)){ 2819 j++; 2820 continue; 2821 } 2822 bool v = eval_val (s.Substring (0, j)); 2823 s = s.Substring (j); 2824 return v; 2825 } 2826 bool vv = eval_val (s); 2827 s = ""; 2828 return vv; 2829 } 2830 } 2831 Error_InvalidDirective (); 2832 return false; 2833 } 2834 pp_unary(ref string s)2835 bool pp_unary (ref string s) 2836 { 2837 s = s.Trim (); 2838 int len = s.Length; 2839 2840 if (len > 0){ 2841 if (s [0] == '!'){ 2842 if (len > 1 && s [1] == '='){ 2843 Error_InvalidDirective (); 2844 return false; 2845 } 2846 s = s.Substring (1); 2847 return ! pp_primary (ref s); 2848 } else 2849 return pp_primary (ref s); 2850 } else { 2851 Error_InvalidDirective (); 2852 return false; 2853 } 2854 } 2855 pp_eq(ref string s)2856 bool pp_eq (ref string s) 2857 { 2858 bool va = pp_unary (ref s); 2859 2860 s = s.Trim (); 2861 int len = s.Length; 2862 if (len > 0){ 2863 if (s [0] == '='){ 2864 if (len > 2 && s [1] == '='){ 2865 s = s.Substring (2); 2866 return va == pp_unary (ref s); 2867 } else { 2868 Error_InvalidDirective (); 2869 return false; 2870 } 2871 } else if (s [0] == '!' && len > 1 && s [1] == '='){ 2872 s = s.Substring (2); 2873 2874 return va != pp_unary (ref s); 2875 2876 } 2877 } 2878 2879 return va; 2880 2881 } 2882 pp_and(ref string s)2883 bool pp_and (ref string s) 2884 { 2885 bool va = pp_eq (ref s); 2886 2887 s = s.Trim (); 2888 int len = s.Length; 2889 if (len > 0){ 2890 if (s [0] == '&'){ 2891 if (len > 2 && s [1] == '&'){ 2892 s = s.Substring (2); 2893 return (va & pp_and (ref s)); 2894 } else { 2895 Error_InvalidDirective (); 2896 return false; 2897 } 2898 } 2899 } 2900 return va; 2901 } 2902 2903 // 2904 // Evaluates an expression for `#if' or `#elif' 2905 // pp_expr(ref string s, bool isTerm)2906 bool pp_expr (ref string s, bool isTerm) 2907 { 2908 bool va = pp_and (ref s); 2909 s = s.Trim (); 2910 int len = s.Length; 2911 if (len > 0){ 2912 char c = s [0]; 2913 2914 if (c == '|'){ 2915 if (len > 2 && s [1] == '|'){ 2916 s = s.Substring (2); 2917 return va | pp_expr (ref s, isTerm); 2918 } else { 2919 Error_InvalidDirective (); 2920 return false; 2921 } 2922 } 2923 if (isTerm) { 2924 Error_EndLineExpected (); 2925 return false; 2926 } 2927 } 2928 2929 return va; 2930 } 2931 eval(string s)2932 bool eval (string s) 2933 { 2934 bool v = pp_expr (ref s, true); 2935 s = s.Trim (); 2936 if (s.Length != 0){ 2937 return false; 2938 } 2939 2940 return v; 2941 } 2942 Error_NumericConstantTooLong()2943 void Error_NumericConstantTooLong () 2944 { 2945 Report.Error (1021, Location, "Integral constant is too large"); 2946 } 2947 Error_InvalidDirective()2948 void Error_InvalidDirective () 2949 { 2950 Report.Error (1517, Location, "Invalid preprocessor directive"); 2951 } 2952 Error_UnexpectedDirective(string extra)2953 void Error_UnexpectedDirective (string extra) 2954 { 2955 Report.Error ( 2956 1028, Location, 2957 "Unexpected processor directive ({0})", extra); 2958 } 2959 Error_TokensSeen()2960 void Error_TokensSeen () 2961 { 2962 Report.Error (1032, Location, 2963 "Cannot define or undefine preprocessor symbols after first token in file"); 2964 } 2965 Eror_WrongPreprocessorLocation()2966 void Eror_WrongPreprocessorLocation () 2967 { 2968 Report.Error (1040, Location, 2969 "Preprocessor directives must appear as the first non-whitespace character on a line"); 2970 } 2971 Error_EndLineExpected()2972 void Error_EndLineExpected () 2973 { 2974 Report.Error (1025, Location, "Single-line comment or end-of-line expected"); 2975 } 2976 2977 // 2978 // Raises a warning when tokenizer found documentation comment 2979 // on unexpected place 2980 // WarningMisplacedComment(Location loc)2981 void WarningMisplacedComment (Location loc) 2982 { 2983 if (doc_state != XmlCommentState.Error) { 2984 doc_state = XmlCommentState.Error; 2985 Report.Warning (1587, 2, loc, "XML comment is not placed on a valid language element"); 2986 } 2987 } 2988 2989 // 2990 // if true, then the code continues processing the code 2991 // if false, the code stays in a loop until another directive is 2992 // reached. 2993 // When caller_is_taking is false we ignore all directives except the ones 2994 // which can help us to identify where the #if block ends ParsePreprocessingDirective(bool caller_is_taking)2995 bool ParsePreprocessingDirective (bool caller_is_taking) 2996 { 2997 string arg; 2998 bool region_directive = false; 2999 3000 var directive = get_cmd_arg (out arg); 3001 3002 // 3003 // The first group of pre-processing instructions is always processed 3004 // 3005 switch (directive) { 3006 case PreprocessorDirective.Region: 3007 region_directive = true; 3008 arg = "true"; 3009 goto case PreprocessorDirective.If; 3010 3011 case PreprocessorDirective.Endregion: 3012 if (ifstack == null || ifstack.Count == 0){ 3013 Error_UnexpectedDirective ("no #region for this #endregion"); 3014 return true; 3015 } 3016 int pop = ifstack.Pop (); 3017 3018 if ((pop & REGION) == 0) 3019 Report.Error (1027, Location, "Expected `#endif' directive"); 3020 3021 return caller_is_taking; 3022 3023 case PreprocessorDirective.If: 3024 if (ifstack == null) 3025 ifstack = new Stack<int> (2); 3026 3027 int flags = region_directive ? REGION : 0; 3028 if (ifstack.Count == 0){ 3029 flags |= PARENT_TAKING; 3030 } else { 3031 int state = ifstack.Peek (); 3032 if ((state & TAKING) != 0) { 3033 flags |= PARENT_TAKING; 3034 } 3035 } 3036 3037 if (eval (arg) && caller_is_taking) { 3038 ifstack.Push (flags | TAKING); 3039 return true; 3040 } 3041 ifstack.Push (flags); 3042 return false; 3043 3044 case PreprocessorDirective.Endif: 3045 if (ifstack == null || ifstack.Count == 0){ 3046 Error_UnexpectedDirective ("no #if for this #endif"); 3047 return true; 3048 } else { 3049 pop = ifstack.Pop (); 3050 3051 if ((pop & REGION) != 0) 3052 Report.Error (1038, Location, "#endregion directive expected"); 3053 3054 if (arg.Length != 0) { 3055 Error_EndLineExpected (); 3056 } 3057 3058 if (ifstack.Count == 0) 3059 return true; 3060 3061 int state = ifstack.Peek (); 3062 return (state & TAKING) != 0; 3063 } 3064 3065 case PreprocessorDirective.Elif: 3066 if (ifstack == null || ifstack.Count == 0){ 3067 Error_UnexpectedDirective ("no #if for this #elif"); 3068 return true; 3069 } else { 3070 int state = ifstack.Pop (); 3071 3072 if ((state & REGION) != 0) { 3073 Report.Error (1038, Location, "#endregion directive expected"); 3074 return true; 3075 } 3076 3077 if ((state & ELSE_SEEN) != 0){ 3078 Error_UnexpectedDirective ("#elif not valid after #else"); 3079 return true; 3080 } 3081 3082 if ((state & TAKING) != 0) { 3083 ifstack.Push (0); 3084 return false; 3085 } 3086 3087 if (eval (arg) && ((state & PARENT_TAKING) != 0)){ 3088 ifstack.Push (state | TAKING); 3089 return true; 3090 } 3091 3092 ifstack.Push (state); 3093 return false; 3094 } 3095 3096 case PreprocessorDirective.Else: 3097 if (ifstack == null || ifstack.Count == 0){ 3098 Error_UnexpectedDirective ("no #if for this #else"); 3099 return true; 3100 } else { 3101 int state = ifstack.Peek (); 3102 3103 if ((state & REGION) != 0) { 3104 Report.Error (1038, Location, "#endregion directive expected"); 3105 return true; 3106 } 3107 3108 if ((state & ELSE_SEEN) != 0){ 3109 Error_UnexpectedDirective ("#else within #else"); 3110 return true; 3111 } 3112 3113 ifstack.Pop (); 3114 3115 if (arg.Length != 0) { 3116 Error_EndLineExpected (); 3117 return true; 3118 } 3119 3120 bool ret = false; 3121 if ((state & PARENT_TAKING) != 0) { 3122 ret = (state & TAKING) == 0; 3123 3124 if (ret) 3125 state |= TAKING; 3126 else 3127 state &= ~TAKING; 3128 } 3129 3130 ifstack.Push (state | ELSE_SEEN); 3131 3132 return ret; 3133 } 3134 case PreprocessorDirective.Define: 3135 if (any_token_seen){ 3136 if (caller_is_taking) 3137 Error_TokensSeen (); 3138 return caller_is_taking; 3139 } 3140 PreProcessDefinition (true, arg, caller_is_taking); 3141 return caller_is_taking; 3142 3143 case PreprocessorDirective.Undef: 3144 if (any_token_seen){ 3145 if (caller_is_taking) 3146 Error_TokensSeen (); 3147 return caller_is_taking; 3148 } 3149 PreProcessDefinition (false, arg, caller_is_taking); 3150 return caller_is_taking; 3151 3152 case PreprocessorDirective.Invalid: 3153 Report.Error (1024, Location, "Wrong preprocessor directive"); 3154 return true; 3155 } 3156 3157 // 3158 // These are only processed if we are in a `taking' block 3159 // 3160 if (!caller_is_taking) 3161 return false; 3162 3163 switch (directive){ 3164 case PreprocessorDirective.Error: 3165 Report.Error (1029, Location, "#error: '{0}'", arg); 3166 return true; 3167 3168 case PreprocessorDirective.Warning: 3169 Report.Warning (1030, 1, Location, "#warning: `{0}'", arg); 3170 return true; 3171 3172 case PreprocessorDirective.Pragma: 3173 if (context.Settings.Version == LanguageVersion.ISO_1) { 3174 Report.FeatureIsNotAvailable (context, Location, "#pragma"); 3175 } 3176 3177 ParsePragmaDirective (); 3178 return true; 3179 3180 case PreprocessorDirective.Line: 3181 Location loc = Location; 3182 if (!PreProcessLine ()) 3183 Report.Error (1576, loc, "The line number specified for #line directive is missing or invalid"); 3184 3185 return caller_is_taking; 3186 } 3187 3188 throw new NotImplementedException (directive.ToString ()); 3189 } 3190 consume_string(bool quoted)3191 int consume_string (bool quoted) 3192 { 3193 int c; 3194 int pos = 0; 3195 Location start_location = Location; 3196 if (quoted) 3197 start_location = start_location - 1; 3198 3199 #if FULL_AST 3200 int reader_pos = reader.Position; 3201 #endif 3202 3203 while (true){ 3204 // Cannot use get_char because of \r in quoted strings 3205 if (putback_char != -1) { 3206 c = putback_char; 3207 putback_char = -1; 3208 } else { 3209 c = reader.Read (); 3210 } 3211 3212 if (c == '"') { 3213 ++col; 3214 3215 if (quoted && peek_char () == '"') { 3216 if (pos == value_builder.Length) 3217 Array.Resize (ref value_builder, pos * 2); 3218 3219 value_builder[pos++] = (char) c; 3220 get_char (); 3221 continue; 3222 } 3223 3224 ILiteralConstant res = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location); 3225 val = res; 3226 #if FULL_AST 3227 res.ParsedValue = quoted ? 3228 reader.ReadChars (reader_pos - 2, reader.Position - 1) : 3229 reader.ReadChars (reader_pos - 1, reader.Position); 3230 #endif 3231 3232 return Token.LITERAL; 3233 } 3234 3235 if (c == '\n' || c == UnicodeLS || c == UnicodePS) { 3236 if (!quoted) { 3237 Report.Error (1010, Location, "Newline in constant"); 3238 3239 advance_line (); 3240 3241 // Don't add \r to string literal 3242 if (pos > 1 && value_builder [pos - 1] == '\r') 3243 --pos; 3244 3245 val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location); 3246 return Token.LITERAL; 3247 } 3248 3249 advance_line (); 3250 } else if (c == '\\' && !quoted) { 3251 ++col; 3252 int surrogate; 3253 c = escape (c, out surrogate); 3254 if (c == -1) 3255 return Token.ERROR; 3256 if (surrogate != 0) { 3257 if (pos == value_builder.Length) 3258 Array.Resize (ref value_builder, pos * 2); 3259 3260 value_builder[pos++] = (char) c; 3261 c = surrogate; 3262 } 3263 } else if (c == -1) { 3264 Report.Error (1039, Location, "Unterminated string literal"); 3265 return Token.EOF; 3266 } else { 3267 ++col; 3268 } 3269 3270 if (pos == value_builder.Length) 3271 Array.Resize (ref value_builder, pos * 2); 3272 3273 value_builder[pos++] = (char) c; 3274 } 3275 } 3276 consume_identifier(int s)3277 private int consume_identifier (int s) 3278 { 3279 int res = consume_identifier (s, false); 3280 3281 if (doc_state == XmlCommentState.Allowed) 3282 doc_state = XmlCommentState.NotAllowed; 3283 3284 return res; 3285 } 3286 consume_identifier(int c, bool quoted)3287 int consume_identifier (int c, bool quoted) 3288 { 3289 // 3290 // This method is very performance sensitive. It accounts 3291 // for approximately 25% of all parser time 3292 // 3293 3294 int pos = 0; 3295 int column = col; 3296 if (quoted) 3297 --column; 3298 3299 if (c == '\\') { 3300 int surrogate; 3301 c = escape (c, out surrogate); 3302 if (quoted || is_identifier_start_character (c)) { 3303 // it's added bellow 3304 } else if (surrogate != 0) { 3305 id_builder [pos++] = (char)c; 3306 c = surrogate; 3307 } else { 3308 Report.Error (1056, Location, "Unexpected character `\\{0}'", c.ToString ("x4")); 3309 return Token.ERROR; 3310 } 3311 } 3312 3313 id_builder [pos++] = (char) c; 3314 3315 try { 3316 while (true) { 3317 c = reader.Read (); 3318 3319 if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= '0' && c <= '9')) { 3320 id_builder [pos++] = (char) c; 3321 continue; 3322 } 3323 3324 if (c < 0x80) { 3325 if (c == '\\') { 3326 int surrogate; 3327 c = escape (c, out surrogate); 3328 if (is_identifier_part_character ((char) c)) 3329 id_builder[pos++] = (char) c; 3330 else if (surrogate != 0) { 3331 c = surrogate; 3332 } else { 3333 switch (c) { 3334 // TODO: Probably need more whitespace characters 3335 case 0xFEFF: 3336 putback_char = c; 3337 break; 3338 default: 3339 Report.Error (1056, Location, "Unexpected character `\\{0}'", c.ToString ("x4")); 3340 return Token.ERROR; 3341 } 3342 } 3343 3344 continue; 3345 } 3346 } else if (is_identifier_part_character_slow_part ((char) c)) { 3347 id_builder [pos++] = (char) c; 3348 continue; 3349 } 3350 3351 putback_char = c; 3352 break; 3353 } 3354 } catch (IndexOutOfRangeException) { 3355 Report.Error (645, Location, "Identifier too long (limit is 512 chars)"); 3356 --pos; 3357 col += pos; 3358 } 3359 3360 col += pos - 1; 3361 3362 // 3363 // Optimization: avoids doing the keyword lookup 3364 // on uppercase letters 3365 // 3366 if (id_builder [0] >= '_' && !quoted) { 3367 int keyword = GetKeyword (id_builder, pos); 3368 if (keyword != -1) { 3369 val = ltb.Create (keyword == Token.AWAIT ? "await" : null, current_source, ref_line, column); 3370 return keyword; 3371 } 3372 } 3373 3374 string s = InternIdentifier (id_builder, pos); 3375 val = ltb.Create (s, current_source, ref_line, column); 3376 if (quoted && parsing_attribute_section) 3377 AddEscapedIdentifier (((LocatedToken) val).Location); 3378 3379 return Token.IDENTIFIER; 3380 } 3381 InternIdentifier(char[] charBuffer, int length)3382 string InternIdentifier (char[] charBuffer, int length) 3383 { 3384 // 3385 // Keep identifiers in an array of hashtables to avoid needless 3386 // allocations 3387 // 3388 var identifiers_group = identifiers[length]; 3389 string s; 3390 if (identifiers_group != null) { 3391 if (identifiers_group.TryGetValue (charBuffer, out s)) { 3392 return s; 3393 } 3394 } else { 3395 // TODO: this should be number of files dependant 3396 // corlib compilation peaks at 1000 and System.Core at 150 3397 int capacity = length > 20 ? 10 : 100; 3398 identifiers_group = new Dictionary<char[], string> (capacity, new IdentifiersComparer (length)); 3399 identifiers[length] = identifiers_group; 3400 } 3401 3402 char[] chars = new char[length]; 3403 Array.Copy (charBuffer, chars, length); 3404 3405 s = new string (charBuffer, 0, length); 3406 identifiers_group.Add (chars, s); 3407 return s; 3408 } 3409 xtoken()3410 public int xtoken () 3411 { 3412 if (parsing_interpolation_format) { 3413 return TokenizeInterpolationFormat (); 3414 } 3415 3416 int d, c; 3417 3418 // Whether we have seen comments on the current line 3419 bool comments_seen = false; 3420 while ((c = get_char ()) != -1) { 3421 switch (c) { 3422 case '\t': 3423 col = ((col - 1 + tab_size) / tab_size) * tab_size; 3424 continue; 3425 3426 case ' ': 3427 case '\f': 3428 case '\v': 3429 case 0xa0: 3430 case 0: 3431 case 0xFEFF: // Ignore BOM anywhere in the file 3432 continue; 3433 3434 /* This is required for compatibility with .NET 3435 case 0xEF: 3436 if (peek_char () == 0xBB) { 3437 PushPosition (); 3438 get_char (); 3439 if (get_char () == 0xBF) 3440 continue; 3441 PopPosition (); 3442 } 3443 break; 3444 */ 3445 case '\\': 3446 tokens_seen = true; 3447 return consume_identifier (c); 3448 3449 case '{': 3450 val = ltb.Create (current_source, ref_line, col); 3451 3452 if (parsing_string_interpolation > 0) 3453 ++string_interpolation_section; 3454 3455 return Token.OPEN_BRACE; 3456 case '}': 3457 if (parsing_string_interpolation > 0) { 3458 if (string_interpolation_section == 0) { 3459 --parsing_string_interpolation; 3460 bool quoted; 3461 if (parsing_string_interpolation_quoted != null && parsing_string_interpolation_quoted.Count > 0) { 3462 quoted = parsing_string_interpolation_quoted.Pop (); 3463 } else { 3464 quoted = false; 3465 } 3466 3467 return TokenizeInterpolatedString (quoted); 3468 } 3469 3470 --string_interpolation_section; 3471 } 3472 3473 val = ltb.Create (current_source, ref_line, col); 3474 return Token.CLOSE_BRACE; 3475 case '[': 3476 // To block doccomment inside attribute declaration. 3477 if (doc_state == XmlCommentState.Allowed) 3478 doc_state = XmlCommentState.NotAllowed; 3479 3480 val = ltb.Create (current_source, ref_line, col); 3481 3482 if (parsing_block == 0 || lambda_arguments_parsing) 3483 return Token.OPEN_BRACKET; 3484 3485 int next = peek_char (); 3486 switch (next) { 3487 case ']': 3488 case ',': 3489 return Token.OPEN_BRACKET; 3490 3491 case ' ': 3492 case '\f': 3493 case '\v': 3494 case '\r': 3495 case '\n': 3496 case UnicodeLS: 3497 case UnicodePS: 3498 case '/': 3499 next = peek_token (); 3500 if (next == Token.COMMA || next == Token.CLOSE_BRACKET) 3501 return Token.OPEN_BRACKET; 3502 3503 return Token.OPEN_BRACKET_EXPR; 3504 default: 3505 return Token.OPEN_BRACKET_EXPR; 3506 } 3507 case ']': 3508 ltb.CreateOptional (current_source, ref_line, col, ref val); 3509 return Token.CLOSE_BRACKET; 3510 case '(': 3511 val = ltb.Create (current_source, ref_line, col); 3512 // 3513 // An expression versions of parens can appear in block context only 3514 // 3515 if (parsing_block != 0 && !lambda_arguments_parsing) { 3516 3517 // 3518 // Optmize most common case where we know that parens 3519 // is not special 3520 // 3521 switch (current_token) { 3522 case Token.IDENTIFIER: 3523 case Token.IF: 3524 case Token.FOR: 3525 case Token.FOREACH: 3526 case Token.TYPEOF: 3527 case Token.WHILE: 3528 case Token.SWITCH: 3529 case Token.USING: 3530 case Token.DEFAULT: 3531 case Token.DEFAULT_VALUE: 3532 case Token.DELEGATE: 3533 case Token.OP_GENERICS_GT: 3534 case Token.REFVALUE: 3535 return Token.OPEN_PARENS; 3536 } 3537 3538 // Optimize using peek 3539 int xx = peek_char (); 3540 switch (xx) { 3541 case '(': 3542 case '\'': 3543 case '"': 3544 case '0': 3545 case '1': 3546 return Token.OPEN_PARENS; 3547 } 3548 3549 lambda_arguments_parsing = true; 3550 PushPosition (); 3551 d = TokenizeOpenParens (); 3552 PopPosition (); 3553 lambda_arguments_parsing = false; 3554 return d; 3555 } 3556 3557 return Token.OPEN_PARENS; 3558 case ')': 3559 ltb.CreateOptional (current_source, ref_line, col, ref val); 3560 return Token.CLOSE_PARENS; 3561 case ',': 3562 ltb.CreateOptional (current_source, ref_line, col, ref val); 3563 return Token.COMMA; 3564 case ';': 3565 ltb.CreateOptional (current_source, ref_line, col, ref val); 3566 return Token.SEMICOLON; 3567 case '~': 3568 val = ltb.Create (current_source, ref_line, col); 3569 return Token.TILDE; 3570 case '?': 3571 val = ltb.Create (current_source, ref_line, col); 3572 return TokenizePossibleNullableType (); 3573 case '<': 3574 val = ltb.Create (current_source, ref_line, col); 3575 if (parsing_generic_less_than++ > 0) 3576 return Token.OP_GENERICS_LT; 3577 3578 return TokenizeLessThan (); 3579 3580 case '>': 3581 val = ltb.Create (current_source, ref_line, col); 3582 d = peek_char (); 3583 3584 if (d == '='){ 3585 get_char (); 3586 return Token.OP_GE; 3587 } 3588 3589 if (parsing_generic_less_than > 1 || (parsing_generic_less_than == 1 && d != '>')) { 3590 parsing_generic_less_than--; 3591 return Token.OP_GENERICS_GT; 3592 } 3593 3594 if (d == '>') { 3595 get_char (); 3596 d = peek_char (); 3597 3598 if (d == '=') { 3599 get_char (); 3600 return Token.OP_SHIFT_RIGHT_ASSIGN; 3601 } 3602 return Token.OP_SHIFT_RIGHT; 3603 } 3604 3605 return Token.OP_GT; 3606 3607 case '+': 3608 val = ltb.Create (current_source, ref_line, col); 3609 d = peek_char (); 3610 if (d == '+') { 3611 d = Token.OP_INC; 3612 } else if (d == '=') { 3613 d = Token.OP_ADD_ASSIGN; 3614 } else { 3615 return Token.PLUS; 3616 } 3617 get_char (); 3618 return d; 3619 3620 case '-': 3621 val = ltb.Create (current_source, ref_line, col); 3622 d = peek_char (); 3623 if (d == '-') { 3624 d = Token.OP_DEC; 3625 } else if (d == '=') 3626 d = Token.OP_SUB_ASSIGN; 3627 else if (d == '>') 3628 d = Token.OP_PTR; 3629 else { 3630 return Token.MINUS; 3631 } 3632 get_char (); 3633 return d; 3634 3635 case '!': 3636 val = ltb.Create (current_source, ref_line, col); 3637 if (peek_char () == '='){ 3638 get_char (); 3639 return Token.OP_NE; 3640 } 3641 return Token.BANG; 3642 3643 case '=': 3644 val = ltb.Create (current_source, ref_line, col); 3645 d = peek_char (); 3646 if (d == '='){ 3647 get_char (); 3648 return Token.OP_EQ; 3649 } 3650 if (d == '>'){ 3651 get_char (); 3652 return Token.ARROW; 3653 } 3654 3655 return Token.ASSIGN; 3656 3657 case '&': 3658 val = ltb.Create (current_source, ref_line, col); 3659 d = peek_char (); 3660 if (d == '&'){ 3661 get_char (); 3662 return Token.OP_AND; 3663 } 3664 if (d == '='){ 3665 get_char (); 3666 return Token.OP_AND_ASSIGN; 3667 } 3668 return Token.BITWISE_AND; 3669 3670 case '|': 3671 val = ltb.Create (current_source, ref_line, col); 3672 d = peek_char (); 3673 if (d == '|'){ 3674 get_char (); 3675 return Token.OP_OR; 3676 } 3677 if (d == '='){ 3678 get_char (); 3679 return Token.OP_OR_ASSIGN; 3680 } 3681 return Token.BITWISE_OR; 3682 3683 case '*': 3684 val = ltb.Create (current_source, ref_line, col); 3685 if (peek_char () == '='){ 3686 get_char (); 3687 return Token.OP_MULT_ASSIGN; 3688 } 3689 return Token.STAR; 3690 3691 case '/': 3692 d = peek_char (); 3693 if (d == '='){ 3694 val = ltb.Create (current_source, ref_line, col); 3695 get_char (); 3696 return Token.OP_DIV_ASSIGN; 3697 } 3698 3699 // Handle double-slash comments. 3700 if (d == '/'){ 3701 if (parsing_string_interpolation > 0) { 3702 Report.Error (8077, Location, "A single-line comment may not be used in an interpolated string"); 3703 goto case '}'; 3704 } 3705 3706 get_char (); 3707 if (doc_processing) { 3708 if (peek_char () == '/') { 3709 get_char (); 3710 // Don't allow ////. 3711 if ((d = peek_char ()) != '/') { 3712 if (doc_state == XmlCommentState.Allowed) 3713 handle_one_line_xml_comment (); 3714 else if (doc_state == XmlCommentState.NotAllowed) 3715 WarningMisplacedComment (Location - 3); 3716 } 3717 } else { 3718 if (xml_comment_buffer.Length > 0) 3719 doc_state = XmlCommentState.NotAllowed; 3720 } 3721 } 3722 3723 ReadToEndOfLine (); 3724 3725 any_token_seen |= tokens_seen; 3726 tokens_seen = false; 3727 comments_seen = false; 3728 continue; 3729 } else if (d == '*'){ 3730 get_char (); 3731 bool docAppend = false; 3732 if (doc_processing && peek_char () == '*') { 3733 get_char (); 3734 // But when it is /**/, just do nothing. 3735 if (peek_char () == '/') { 3736 get_char (); 3737 continue; 3738 } 3739 if (doc_state == XmlCommentState.Allowed) 3740 docAppend = true; 3741 else if (doc_state == XmlCommentState.NotAllowed) { 3742 WarningMisplacedComment (Location - 2); 3743 } 3744 } 3745 3746 int current_comment_start = 0; 3747 if (docAppend) { 3748 current_comment_start = xml_comment_buffer.Length; 3749 xml_comment_buffer.Append (Environment.NewLine); 3750 } 3751 3752 while ((d = get_char ()) != -1){ 3753 if (d == '*' && peek_char () == '/'){ 3754 get_char (); 3755 comments_seen = true; 3756 break; 3757 } 3758 if (docAppend) 3759 xml_comment_buffer.Append ((char) d); 3760 3761 if (d == '\n' || d == UnicodeLS || d == UnicodePS){ 3762 any_token_seen |= tokens_seen; 3763 tokens_seen = false; 3764 // 3765 // Reset 'comments_seen' just to be consistent. 3766 // It doesn't matter either way, here. 3767 // 3768 comments_seen = false; 3769 } 3770 } 3771 if (!comments_seen) 3772 Report.Error (1035, Location, "End-of-file found, '*/' expected"); 3773 3774 if (docAppend) 3775 update_formatted_doc_comment (current_comment_start); 3776 continue; 3777 } 3778 val = ltb.Create (current_source, ref_line, col); 3779 return Token.DIV; 3780 3781 case '%': 3782 val = ltb.Create (current_source, ref_line, col); 3783 if (peek_char () == '='){ 3784 get_char (); 3785 return Token.OP_MOD_ASSIGN; 3786 } 3787 return Token.PERCENT; 3788 3789 case '^': 3790 val = ltb.Create (current_source, ref_line, col); 3791 if (peek_char () == '='){ 3792 get_char (); 3793 return Token.OP_XOR_ASSIGN; 3794 } 3795 return Token.CARRET; 3796 3797 case ':': 3798 val = ltb.Create (current_source, ref_line, col); 3799 if (peek_char () == ':') { 3800 get_char (); 3801 return Token.DOUBLE_COLON; 3802 } 3803 return Token.COLON; 3804 3805 case '0': case '1': case '2': case '3': case '4': 3806 case '5': case '6': case '7': case '8': case '9': 3807 tokens_seen = true; 3808 return is_number (c, false); 3809 3810 case '\n': // white space 3811 case UnicodeLS: 3812 case UnicodePS: 3813 any_token_seen |= tokens_seen; 3814 tokens_seen = false; 3815 comments_seen = false; 3816 continue; 3817 3818 case '.': 3819 tokens_seen = true; 3820 d = peek_char (); 3821 if (d >= '0' && d <= '9') 3822 return is_number (c, true); 3823 3824 ltb.CreateOptional (current_source, ref_line, col, ref val); 3825 return Token.DOT; 3826 3827 case '#': 3828 if (tokens_seen || comments_seen) { 3829 Eror_WrongPreprocessorLocation (); 3830 return Token.ERROR; 3831 } 3832 3833 if (ParsePreprocessingDirective (true)) 3834 continue; 3835 3836 bool directive_expected = false; 3837 while ((c = get_char ()) != -1) { 3838 if (col == 1) { 3839 directive_expected = true; 3840 } else if (!directive_expected) { 3841 // TODO: Implement comment support for disabled code and uncomment this code 3842 // if (c == '#') { 3843 // Eror_WrongPreprocessorLocation (); 3844 // return Token.ERROR; 3845 // } 3846 continue; 3847 } 3848 3849 if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\v' || c == UnicodeLS || c == UnicodePS) 3850 continue; 3851 3852 if (c == '#') { 3853 if (ParsePreprocessingDirective (false)) 3854 break; 3855 } 3856 directive_expected = false; 3857 } 3858 3859 if (c != -1) { 3860 tokens_seen = false; 3861 continue; 3862 } 3863 3864 return Token.EOF; 3865 3866 case '"': 3867 if (parsing_string_interpolation > 0 && !ScanClosingInterpolationBrace ()) { 3868 parsing_string_interpolation = 0; 3869 Report.Error (8076, Location, "Missing close delimiter `}' for interpolated expression"); 3870 val = new StringLiteral (context.BuiltinTypes, "", Location); 3871 return Token.INTERPOLATED_STRING_END; 3872 } 3873 3874 return consume_string (false); 3875 3876 case '\'': 3877 return TokenizeBackslash (); 3878 3879 case '@': 3880 c = get_char (); 3881 if (c == '"') { 3882 tokens_seen = true; 3883 return consume_string (true); 3884 } 3885 3886 if (is_identifier_start_character (c)){ 3887 return consume_identifier (c, true); 3888 } 3889 3890 Report.Error (1646, Location, "Keyword, identifier, or string expected after verbatim specifier: @"); 3891 return Token.ERROR; 3892 3893 case '$': 3894 switch (peek_char ()) { 3895 case '"': 3896 get_char (); 3897 return TokenizeInterpolatedString (false); 3898 case '@': 3899 get_char (); 3900 if (peek_char () == '"') { 3901 get_char (); 3902 return TokenizeInterpolatedString (true); 3903 } 3904 3905 break; 3906 } 3907 3908 break; 3909 case EvalStatementParserCharacter: 3910 return Token.EVAL_STATEMENT_PARSER; 3911 case EvalCompilationUnitParserCharacter: 3912 return Token.EVAL_COMPILATION_UNIT_PARSER; 3913 case EvalUsingDeclarationsParserCharacter: 3914 return Token.EVAL_USING_DECLARATIONS_UNIT_PARSER; 3915 case DocumentationXref: 3916 return Token.DOC_SEE; 3917 } 3918 3919 if (is_identifier_start_character (c)) { 3920 tokens_seen = true; 3921 return consume_identifier (c); 3922 } 3923 3924 if (char.IsWhiteSpace ((char) c)) 3925 continue; 3926 3927 Report.Error (1056, Location, "Unexpected character `{0}'", ((char) c).ToString ()); 3928 } 3929 3930 if (CompleteOnEOF){ 3931 if (generated) 3932 return Token.COMPLETE_COMPLETION; 3933 3934 generated = true; 3935 return Token.GENERATE_COMPLETION; 3936 } 3937 3938 3939 return Token.EOF; 3940 } 3941 TokenizeBackslash()3942 int TokenizeBackslash () 3943 { 3944 #if FULL_AST 3945 int read_start = reader.Position; 3946 #endif 3947 Location start_location = Location; 3948 int c = get_char (); 3949 tokens_seen = true; 3950 if (c == '\'') { 3951 val = new CharLiteral (context.BuiltinTypes, (char) c, start_location); 3952 Report.Error (1011, start_location, "Empty character literal"); 3953 return Token.LITERAL; 3954 } 3955 3956 if (c == '\n' || c == UnicodeLS || c == UnicodePS) { 3957 Report.Error (1010, start_location, "Newline in constant"); 3958 return Token.ERROR; 3959 } 3960 3961 int d; 3962 c = escape (c, out d); 3963 if (c == -1) 3964 return Token.ERROR; 3965 if (d != 0) 3966 throw new NotImplementedException (); 3967 3968 ILiteralConstant res = new CharLiteral (context.BuiltinTypes, (char) c, start_location); 3969 val = res; 3970 c = get_char (); 3971 3972 if (c != '\'') { 3973 Report.Error (1012, start_location, "Too many characters in character literal"); 3974 3975 // Try to recover, read until newline or next "'" 3976 while ((c = get_char ()) != -1) { 3977 if (c == '\n' || c == '\'' || c == UnicodeLS || c == UnicodePS) 3978 break; 3979 } 3980 } 3981 3982 #if FULL_AST 3983 res.ParsedValue = reader.ReadChars (read_start - 1, reader.Position); 3984 #endif 3985 3986 return Token.LITERAL; 3987 } 3988 TokenizeLessThan()3989 int TokenizeLessThan () 3990 { 3991 int d; 3992 3993 if (current_token != Token.OPERATOR) { 3994 // Save current position and parse next token. 3995 PushPosition (); 3996 int generic_dimension = 0; 3997 if (parse_less_than (ref generic_dimension)) { 3998 if (parsing_generic_declaration && (parsing_generic_declaration_doc || token () != Token.DOT)) { 3999 d = Token.OP_GENERICS_LT_DECL; 4000 } else { 4001 if (generic_dimension > 0) { 4002 val = generic_dimension; 4003 DiscardPosition (); 4004 return Token.GENERIC_DIMENSION; 4005 } 4006 4007 d = Token.OP_GENERICS_LT; 4008 } 4009 PopPosition (); 4010 return d; 4011 } 4012 4013 PopPosition (); 4014 } 4015 4016 parsing_generic_less_than = 0; 4017 4018 d = peek_char (); 4019 if (d == '<') { 4020 get_char (); 4021 d = peek_char (); 4022 4023 if (d == '=') { 4024 get_char (); 4025 return Token.OP_SHIFT_LEFT_ASSIGN; 4026 } 4027 return Token.OP_SHIFT_LEFT; 4028 } 4029 4030 if (d == '=') { 4031 get_char (); 4032 return Token.OP_LE; 4033 } 4034 return Token.OP_LT; 4035 } 4036 TokenizeInterpolatedString(bool quoted)4037 int TokenizeInterpolatedString (bool quoted) 4038 { 4039 int pos = 0; 4040 var start_location = Location; 4041 4042 while (true) { 4043 var ch = get_char (); 4044 switch (ch) { 4045 case '"': 4046 if (quoted && peek_char () == '"') { 4047 get_char (); 4048 break; 4049 } 4050 4051 val = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location); 4052 return Token.INTERPOLATED_STRING_END; 4053 case '{': 4054 if (peek_char () == '{') { 4055 value_builder [pos++] = (char)ch; 4056 get_char (); 4057 break; 4058 } 4059 4060 ++parsing_string_interpolation; 4061 if (quoted) { 4062 if (parsing_string_interpolation_quoted == null) 4063 parsing_string_interpolation_quoted = new Stack<bool> (); 4064 } 4065 4066 if (parsing_string_interpolation_quoted != null) { 4067 parsing_string_interpolation_quoted.Push (quoted); 4068 } 4069 4070 val = new StringLiteral (context.BuiltinTypes, CreateStringFromBuilder (pos), start_location); 4071 return Token.INTERPOLATED_STRING; 4072 case '\\': 4073 if (quoted) 4074 break; 4075 4076 ++col; 4077 int surrogate; 4078 ch = escape (ch, out surrogate); 4079 if (ch == -1) 4080 return Token.ERROR; 4081 4082 if (ch == '{' || ch == '}') { 4083 Report.Error (8087, Location, "A `{0}' character may only be escaped by doubling `{0}{0}' in an interpolated string", ((char) ch).ToString ()); 4084 } 4085 4086 if (surrogate != 0) { 4087 if (pos == value_builder.Length) 4088 Array.Resize (ref value_builder, pos * 2); 4089 4090 if (pos == value_builder.Length) 4091 Array.Resize (ref value_builder, pos * 2); 4092 4093 value_builder [pos++] = (char)ch; 4094 ch = surrogate; 4095 } 4096 4097 break; 4098 case -1: 4099 return Token.EOF; 4100 } 4101 4102 ++col; 4103 if (pos == value_builder.Length) 4104 Array.Resize (ref value_builder, pos * 2); 4105 4106 value_builder[pos++] = (char) ch; 4107 } 4108 } 4109 TokenizeInterpolationFormat()4110 int TokenizeInterpolationFormat () 4111 { 4112 int pos = 0; 4113 int braces = 0; 4114 while (true) { 4115 var ch = get_char (); 4116 switch (ch) { 4117 case '{': 4118 ++braces; 4119 break; 4120 case '}': 4121 if (braces == 0) { 4122 putback_char = ch; 4123 if (pos == 0) { 4124 Report.Error (8089, Location, "Empty interpolated expression format specifier"); 4125 } else if (Array.IndexOf (simple_whitespaces, value_builder [pos - 1]) >= 0) { 4126 Report.Error (8088, Location, "A interpolated expression format specifier may not contain trailing whitespace"); 4127 } 4128 4129 val = CreateStringFromBuilder (pos); 4130 return Token.LITERAL; 4131 } 4132 4133 --braces; 4134 break; 4135 case '\\': 4136 if (parsing_string_interpolation_quoted != null && parsing_string_interpolation_quoted.Peek ()) 4137 break; 4138 4139 ++col; 4140 int surrogate; 4141 ch = escape (ch, out surrogate); 4142 if (ch == -1) 4143 return Token.ERROR; 4144 4145 if (ch == '{' || ch == '}') { 4146 Report.Error (8087, Location, "A `{0}' character may only be escaped by doubling `{0}{0}' in an interpolated string", ((char) ch).ToString ()); 4147 } 4148 4149 if (surrogate != 0) { 4150 if (pos == value_builder.Length) 4151 Array.Resize (ref value_builder, pos * 2); 4152 4153 value_builder [pos++] = (char)ch; 4154 ch = surrogate; 4155 } 4156 4157 break; 4158 case -1: 4159 return Token.EOF; 4160 } 4161 4162 ++col; 4163 value_builder[pos++] = (char) ch; 4164 } 4165 } 4166 CreateStringFromBuilder(int pos)4167 string CreateStringFromBuilder (int pos) 4168 { 4169 if (pos == 0) 4170 return string.Empty; 4171 if (pos <= 4) 4172 return InternIdentifier (value_builder, pos); 4173 4174 return new string (value_builder, 0, pos); 4175 } 4176 4177 // 4178 // Handles one line xml comment 4179 // handle_one_line_xml_comment()4180 private void handle_one_line_xml_comment () 4181 { 4182 int c; 4183 while ((c = peek_char ()) != -1 && c != '\n' && c != '\r') { 4184 xml_comment_buffer.Append ((char) get_char ()); 4185 } 4186 if (c == '\r' || c == '\n') 4187 xml_comment_buffer.Append (Environment.NewLine); 4188 } 4189 4190 // 4191 // Remove heading "*" in Javadoc-like xml documentation. 4192 // update_formatted_doc_comment(int current_comment_start)4193 private void update_formatted_doc_comment (int current_comment_start) 4194 { 4195 int length = xml_comment_buffer.Length - current_comment_start; 4196 string [] lines = xml_comment_buffer.ToString ( 4197 current_comment_start, 4198 length).Replace ("\r", "").Split ('\n'); 4199 4200 // The first line starts with /**, thus it is not target 4201 // for the format check. 4202 for (int i = 1; i < lines.Length; i++) { 4203 string s = lines [i]; 4204 int idx = s.IndexOf ('*'); 4205 string head = null; 4206 if (idx < 0) { 4207 if (i < lines.Length - 1) 4208 return; 4209 head = s; 4210 } else 4211 head = s.Substring (0, idx); 4212 foreach (char c in head) 4213 if (c != ' ') 4214 return; 4215 lines [i] = s.Substring (idx + 1); 4216 } 4217 xml_comment_buffer.Remove (current_comment_start, length); 4218 xml_comment_buffer.Insert (current_comment_start, String.Join (Environment.NewLine, lines)); 4219 } 4220 4221 // 4222 // Checks if there was incorrect doc comments and raise 4223 // warnings. 4224 // check_incorrect_doc_comment()4225 public void check_incorrect_doc_comment () 4226 { 4227 if (xml_comment_buffer.Length > 0) 4228 WarningMisplacedComment (Location); 4229 } 4230 4231 // 4232 // Consumes the saved xml comment lines (if any) 4233 // as for current target member or type. 4234 // consume_doc_comment()4235 public string consume_doc_comment () 4236 { 4237 if (xml_comment_buffer.Length > 0) { 4238 string ret = xml_comment_buffer.ToString (); 4239 reset_doc_comment (); 4240 return ret; 4241 } 4242 return null; 4243 } 4244 reset_doc_comment()4245 void reset_doc_comment () 4246 { 4247 xml_comment_buffer.Length = 0; 4248 } 4249 cleanup()4250 public void cleanup () 4251 { 4252 if (ifstack != null && ifstack.Count >= 1) { 4253 int state = ifstack.Pop (); 4254 if ((state & REGION) != 0) 4255 Report.Error (1038, Location, "#endregion directive expected"); 4256 else 4257 Report.Error (1027, Location, "Expected `#endif' directive"); 4258 } 4259 } 4260 } 4261 4262 // 4263 // Indicates whether it accepts XML documentation or not. 4264 // 4265 public enum XmlCommentState { 4266 // comment is allowed in this state. 4267 Allowed, 4268 // comment is not allowed in this state. 4269 NotAllowed, 4270 // once comments appeared when it is NotAllowed, then the 4271 // state is changed to it, until the state is changed to 4272 // .Allowed. 4273 Error 4274 } 4275 } 4276 4277