1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6
7 /* tokenization of CSS style sheets */
8
9 #include "nsCSSScanner.h"
10 #include "nsStyleUtil.h"
11 #include "nsISupportsImpl.h"
12 #include "mozilla/ArrayUtils.h"
13 #include "mozilla/css/ErrorReporter.h"
14 #include "mozilla/Likely.h"
15 #include <algorithm>
16
17 /* Character class tables and related helper functions. */
18
19 static const uint8_t IS_HEX_DIGIT = 0x01;
20 static const uint8_t IS_IDSTART = 0x02;
21 static const uint8_t IS_IDCHAR = 0x04;
22 static const uint8_t IS_URL_CHAR = 0x08;
23 static const uint8_t IS_HSPACE = 0x10;
24 static const uint8_t IS_VSPACE = 0x20;
25 static const uint8_t IS_SPACE = IS_HSPACE|IS_VSPACE;
26 static const uint8_t IS_STRING = 0x40;
27
28 #define H IS_HSPACE
29 #define V IS_VSPACE
30 #define I IS_IDCHAR
31 #define J IS_IDSTART
32 #define U IS_URL_CHAR
33 #define S IS_STRING
34 #define X IS_HEX_DIGIT
35
36 #define SH S|H
37 #define SU S|U
38 #define SUI S|U|I
39 #define SUIJ S|U|I|J
40 #define SUIX S|U|I|X
41 #define SUIJX S|U|I|J|X
42
43 static const uint8_t gLexTable[] = {
44 // 00 01 02 03 04 05 06 07
45 0, S, S, S, S, S, S, S,
46 // 08 TAB LF 0B FF CR 0E 0F
47 S, SH, V, S, V, V, S, S,
48 // 10 11 12 13 14 15 16 17
49 S, S, S, S, S, S, S, S,
50 // 18 19 1A 1B 1C 1D 1E 1F
51 S, S, S, S, S, S, S, S,
52 //SPC ! " # $ % & '
53 SH, SU, 0, SU, SU, SU, SU, 0,
54 // ( ) * + , - . /
55 S, S, SU, SU, SU, SUI, SU, SU,
56 // 0 1 2 3 4 5 6 7
57 SUIX, SUIX, SUIX, SUIX, SUIX, SUIX, SUIX, SUIX,
58 // 8 9 : ; < = > ?
59 SUIX, SUIX, SU, SU, SU, SU, SU, SU,
60 // @ A B C D E F G
61 SU,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX, SUIJ,
62 // H I J K L M N O
63 SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
64 // P Q R S T U V W
65 SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
66 // X Y Z [ \ ] ^ _
67 SUIJ, SUIJ, SUIJ, SU, J, SU, SU, SUIJ,
68 // ` a b c d e f g
69 SU,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX, SUIJ,
70 // h i j k l m n o
71 SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
72 // p q r s t u v w
73 SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
74 // x y z { | } ~ 7F
75 SUIJ, SUIJ, SUIJ, SU, SU, SU, SU, S,
76 };
77
78 static_assert(MOZ_ARRAY_LENGTH(gLexTable) == 128,
79 "gLexTable expected to cover all 128 ASCII characters");
80
81 #undef I
82 #undef J
83 #undef U
84 #undef S
85 #undef X
86 #undef SH
87 #undef SU
88 #undef SUI
89 #undef SUIJ
90 #undef SUIX
91 #undef SUIJX
92
93 /**
94 * True if 'ch' is in character class 'cls', which should be one of
95 * the constants above or some combination of them. All characters
96 * above U+007F are considered to be in 'cls'. EOF is never in 'cls'.
97 */
98 static inline bool
IsOpenCharClass(int32_t ch,uint8_t cls)99 IsOpenCharClass(int32_t ch, uint8_t cls) {
100 return ch >= 0 && (ch >= 128 || (gLexTable[ch] & cls) != 0);
101 }
102
103 /**
104 * True if 'ch' is in character class 'cls', which should be one of
105 * the constants above or some combination of them. No characters
106 * above U+007F are considered to be in 'cls'. EOF is never in 'cls'.
107 */
108 static inline bool
IsClosedCharClass(int32_t ch,uint8_t cls)109 IsClosedCharClass(int32_t ch, uint8_t cls) {
110 return uint32_t(ch) < 128 && (gLexTable[ch] & cls) != 0;
111 }
112
113 /**
114 * True if 'ch' is CSS whitespace, i.e. any of the ASCII characters
115 * TAB, LF, FF, CR, or SPC.
116 */
117 static inline bool
IsWhitespace(int32_t ch)118 IsWhitespace(int32_t ch) {
119 return IsClosedCharClass(ch, IS_SPACE);
120 }
121
122 /**
123 * True if 'ch' is horizontal whitespace, i.e. TAB or SPC.
124 */
125 static inline bool
IsHorzSpace(int32_t ch)126 IsHorzSpace(int32_t ch) {
127 return IsClosedCharClass(ch, IS_HSPACE);
128 }
129
130 /**
131 * True if 'ch' is vertical whitespace, i.e. LF, FF, or CR. Vertical
132 * whitespace requires special handling when consumed, see AdvanceLine.
133 */
134 static inline bool
IsVertSpace(int32_t ch)135 IsVertSpace(int32_t ch) {
136 return IsClosedCharClass(ch, IS_VSPACE);
137 }
138
139 /**
140 * True if 'ch' is a character that can appear in the middle of an identifier.
141 * This includes U+0000 since it is handled as U+FFFD, but for purposes of
142 * GatherText it should not be included in IsOpenCharClass.
143 */
144 static inline bool
IsIdentChar(int32_t ch)145 IsIdentChar(int32_t ch) {
146 return IsOpenCharClass(ch, IS_IDCHAR) || ch == 0;
147 }
148
149 /**
150 * True if 'ch' is a character that by itself begins an identifier.
151 * This includes U+0000 since it is handled as U+FFFD, but for purposes of
152 * GatherText it should not be included in IsOpenCharClass.
153 * (This is a subset of IsIdentChar.)
154 */
155 static inline bool
IsIdentStart(int32_t ch)156 IsIdentStart(int32_t ch) {
157 return IsOpenCharClass(ch, IS_IDSTART) || ch == 0;
158 }
159
160 /**
161 * True if the two-character sequence aFirstChar+aSecondChar begins an
162 * identifier.
163 */
164 static inline bool
StartsIdent(int32_t aFirstChar,int32_t aSecondChar)165 StartsIdent(int32_t aFirstChar, int32_t aSecondChar)
166 {
167 return IsIdentStart(aFirstChar) ||
168 (aFirstChar == '-' && (aSecondChar == '-' || IsIdentStart(aSecondChar)));
169 }
170
171 /**
172 * True if 'ch' is a decimal digit.
173 */
174 static inline bool
IsDigit(int32_t ch)175 IsDigit(int32_t ch) {
176 return (ch >= '0') && (ch <= '9');
177 }
178
179 /**
180 * True if 'ch' is a hexadecimal digit.
181 */
182 static inline bool
IsHexDigit(int32_t ch)183 IsHexDigit(int32_t ch) {
184 return IsClosedCharClass(ch, IS_HEX_DIGIT);
185 }
186
187 /**
188 * Assuming that 'ch' is a decimal digit, return its numeric value.
189 */
190 static inline uint32_t
DecimalDigitValue(int32_t ch)191 DecimalDigitValue(int32_t ch)
192 {
193 return ch - '0';
194 }
195
196 /**
197 * Assuming that 'ch' is a hexadecimal digit, return its numeric value.
198 */
199 static inline uint32_t
HexDigitValue(int32_t ch)200 HexDigitValue(int32_t ch)
201 {
202 if (IsDigit(ch)) {
203 return DecimalDigitValue(ch);
204 } else {
205 // Note: c&7 just keeps the low three bits which causes
206 // upper and lower case alphabetics to both yield their
207 // "relative to 10" value for computing the hex value.
208 return (ch & 0x7) + 9;
209 }
210 }
211
212 /**
213 * If 'ch' can be the first character of a two-character match operator
214 * token, return the token type code for that token, otherwise return
215 * eCSSToken_Symbol to indicate that it can't.
216 */
217 static inline nsCSSTokenType
MatchOperatorType(int32_t ch)218 MatchOperatorType(int32_t ch)
219 {
220 switch (ch) {
221 case '~': return eCSSToken_Includes;
222 case '|': return eCSSToken_Dashmatch;
223 case '^': return eCSSToken_Beginsmatch;
224 case '$': return eCSSToken_Endsmatch;
225 case '*': return eCSSToken_Containsmatch;
226 default: return eCSSToken_Symbol;
227 }
228 }
229
230 /* Out-of-line nsCSSToken methods. */
231
232 /**
233 * Append the textual representation of |this| to |aBuffer|.
234 */
235 void
AppendToString(nsString & aBuffer) const236 nsCSSToken::AppendToString(nsString& aBuffer) const
237 {
238 switch (mType) {
239 case eCSSToken_Ident:
240 nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
241 break;
242
243 case eCSSToken_AtKeyword:
244 aBuffer.Append('@');
245 nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
246 break;
247
248 case eCSSToken_ID:
249 case eCSSToken_Hash:
250 aBuffer.Append('#');
251 nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
252 break;
253
254 case eCSSToken_Function:
255 nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
256 aBuffer.Append('(');
257 break;
258
259 case eCSSToken_URL:
260 case eCSSToken_Bad_URL:
261 aBuffer.AppendLiteral("url(");
262 if (mSymbol != char16_t(0)) {
263 nsStyleUtil::AppendEscapedCSSString(mIdent, aBuffer, mSymbol);
264 } else {
265 aBuffer.Append(mIdent);
266 }
267 if (mType == eCSSToken_URL) {
268 aBuffer.Append(char16_t(')'));
269 }
270 break;
271
272 case eCSSToken_Number:
273 if (mIntegerValid) {
274 aBuffer.AppendInt(mInteger, 10);
275 } else {
276 aBuffer.AppendFloat(mNumber);
277 }
278 break;
279
280 case eCSSToken_Percentage:
281 aBuffer.AppendFloat(mNumber * 100.0f);
282 aBuffer.Append(char16_t('%'));
283 break;
284
285 case eCSSToken_Dimension:
286 if (mIntegerValid) {
287 aBuffer.AppendInt(mInteger, 10);
288 } else {
289 aBuffer.AppendFloat(mNumber);
290 }
291 nsStyleUtil::AppendEscapedCSSIdent(mIdent, aBuffer);
292 break;
293
294 case eCSSToken_Bad_String:
295 nsStyleUtil::AppendEscapedCSSString(mIdent, aBuffer, mSymbol);
296 // remove the trailing quote character
297 aBuffer.Truncate(aBuffer.Length() - 1);
298 break;
299
300 case eCSSToken_String:
301 nsStyleUtil::AppendEscapedCSSString(mIdent, aBuffer, mSymbol);
302 break;
303
304 case eCSSToken_Symbol:
305 aBuffer.Append(mSymbol);
306 break;
307
308 case eCSSToken_Whitespace:
309 aBuffer.Append(' ');
310 break;
311
312 case eCSSToken_HTMLComment:
313 case eCSSToken_URange:
314 aBuffer.Append(mIdent);
315 break;
316
317 case eCSSToken_Includes:
318 aBuffer.AppendLiteral("~=");
319 break;
320 case eCSSToken_Dashmatch:
321 aBuffer.AppendLiteral("|=");
322 break;
323 case eCSSToken_Beginsmatch:
324 aBuffer.AppendLiteral("^=");
325 break;
326 case eCSSToken_Endsmatch:
327 aBuffer.AppendLiteral("$=");
328 break;
329 case eCSSToken_Containsmatch:
330 aBuffer.AppendLiteral("*=");
331 break;
332
333 default:
334 NS_ERROR("invalid token type");
335 break;
336 }
337 }
338
339 /* nsCSSScanner methods. */
340
nsCSSScanner(const nsAString & aBuffer,uint32_t aLineNumber)341 nsCSSScanner::nsCSSScanner(const nsAString& aBuffer, uint32_t aLineNumber)
342 : mBuffer(aBuffer.BeginReading())
343 , mOffset(0)
344 , mCount(aBuffer.Length())
345 , mLineNumber(aLineNumber)
346 , mLineOffset(0)
347 , mTokenLineNumber(aLineNumber)
348 , mTokenLineOffset(0)
349 , mTokenOffset(0)
350 , mRecordStartOffset(0)
351 , mEOFCharacters(eEOFCharacters_None)
352 , mReporter(nullptr)
353 , mSVGMode(false)
354 , mRecording(false)
355 , mSeenBadToken(false)
356 , mSeenVariableReference(false)
357 {
358 MOZ_COUNT_CTOR(nsCSSScanner);
359 }
360
~nsCSSScanner()361 nsCSSScanner::~nsCSSScanner()
362 {
363 MOZ_COUNT_DTOR(nsCSSScanner);
364 }
365
366 void
StartRecording()367 nsCSSScanner::StartRecording()
368 {
369 MOZ_ASSERT(!mRecording, "already started recording");
370 mRecording = true;
371 mRecordStartOffset = mOffset;
372 }
373
374 void
StopRecording()375 nsCSSScanner::StopRecording()
376 {
377 MOZ_ASSERT(mRecording, "haven't started recording");
378 mRecording = false;
379 }
380
381 void
StopRecording(nsString & aBuffer)382 nsCSSScanner::StopRecording(nsString& aBuffer)
383 {
384 MOZ_ASSERT(mRecording, "haven't started recording");
385 mRecording = false;
386 aBuffer.Append(mBuffer + mRecordStartOffset,
387 mOffset - mRecordStartOffset);
388 }
389
390 uint32_t
RecordingLength() const391 nsCSSScanner::RecordingLength() const
392 {
393 MOZ_ASSERT(mRecording, "haven't started recording");
394 return mOffset - mRecordStartOffset;
395 }
396
397 #ifdef DEBUG
398 bool
IsRecording() const399 nsCSSScanner::IsRecording() const
400 {
401 return mRecording;
402 }
403 #endif
404
405 nsDependentSubstring
GetCurrentLine() const406 nsCSSScanner::GetCurrentLine() const
407 {
408 uint32_t end = mTokenOffset;
409 while (end < mCount && !IsVertSpace(mBuffer[end])) {
410 end++;
411 }
412 return nsDependentSubstring(mBuffer + mTokenLineOffset,
413 mBuffer + end);
414 }
415
416 /**
417 * Return the raw UTF-16 code unit at position |mOffset + n| within
418 * the read buffer. If that is beyond the end of the buffer, returns
419 * -1 to indicate end of input.
420 */
421 inline int32_t
Peek(uint32_t n)422 nsCSSScanner::Peek(uint32_t n)
423 {
424 if (mOffset + n >= mCount) {
425 return -1;
426 }
427 return mBuffer[mOffset + n];
428 }
429
430 /**
431 * Advance |mOffset| over |n| code units. Advance(0) is a no-op.
432 * If |n| is greater than the distance to end of input, will silently
433 * stop at the end. May not be used to advance over a line boundary;
434 * AdvanceLine() must be used instead.
435 */
436 inline void
Advance(uint32_t n)437 nsCSSScanner::Advance(uint32_t n)
438 {
439 #ifdef DEBUG
440 while (mOffset < mCount && n > 0) {
441 MOZ_ASSERT(!IsVertSpace(mBuffer[mOffset]),
442 "may not Advance() over a line boundary");
443 mOffset++;
444 n--;
445 }
446 #else
447 if (mOffset + n >= mCount || mOffset + n < mOffset)
448 mOffset = mCount;
449 else
450 mOffset += n;
451 #endif
452 }
453
454 /**
455 * Advance |mOffset| over a line boundary.
456 */
457 void
AdvanceLine()458 nsCSSScanner::AdvanceLine()
459 {
460 MOZ_ASSERT(IsVertSpace(mBuffer[mOffset]),
461 "may not AdvanceLine() over a horizontal character");
462 // Advance over \r\n as a unit.
463 if (mBuffer[mOffset] == '\r' && mOffset + 1 < mCount &&
464 mBuffer[mOffset+1] == '\n')
465 mOffset += 2;
466 else
467 mOffset += 1;
468 // 0 is a magical line number meaning that we don't know (i.e., script)
469 if (mLineNumber != 0)
470 mLineNumber++;
471 mLineOffset = mOffset;
472 }
473
474 /**
475 * Back up |mOffset| over |n| code units. Backup(0) is a no-op.
476 * If |n| is greater than the distance to beginning of input, will
477 * silently stop at the beginning. May not be used to back up over a
478 * line boundary.
479 */
480 void
Backup(uint32_t n)481 nsCSSScanner::Backup(uint32_t n)
482 {
483 #ifdef DEBUG
484 while (mOffset > 0 && n > 0) {
485 MOZ_ASSERT(!IsVertSpace(mBuffer[mOffset-1]),
486 "may not Backup() over a line boundary");
487 mOffset--;
488 n--;
489 }
490 #else
491 if (mOffset < n)
492 mOffset = 0;
493 else
494 mOffset -= n;
495 #endif
496 }
497
498 void
SavePosition(nsCSSScannerPosition & aState)499 nsCSSScanner::SavePosition(nsCSSScannerPosition& aState)
500 {
501 aState.mOffset = mOffset;
502 aState.mLineNumber = mLineNumber;
503 aState.mLineOffset = mLineOffset;
504 aState.mTokenLineNumber = mTokenLineNumber;
505 aState.mTokenLineOffset = mTokenLineOffset;
506 aState.mTokenOffset = mTokenOffset;
507 aState.mInitialized = true;
508 }
509
510 void
RestoreSavedPosition(const nsCSSScannerPosition & aState)511 nsCSSScanner::RestoreSavedPosition(const nsCSSScannerPosition& aState)
512 {
513 MOZ_ASSERT(aState.mInitialized, "have not saved state");
514 if (aState.mInitialized) {
515 mOffset = aState.mOffset;
516 mLineNumber = aState.mLineNumber;
517 mLineOffset = aState.mLineOffset;
518 mTokenLineNumber = aState.mTokenLineNumber;
519 mTokenLineOffset = aState.mTokenLineOffset;
520 mTokenOffset = aState.mTokenOffset;
521 }
522 }
523
524 /**
525 * Skip over a sequence of whitespace characters (vertical or
526 * horizontal) starting at the current read position.
527 */
528 void
SkipWhitespace()529 nsCSSScanner::SkipWhitespace()
530 {
531 for (;;) {
532 int32_t ch = Peek();
533 if (!IsWhitespace(ch)) { // EOF counts as non-whitespace
534 break;
535 }
536 if (IsVertSpace(ch)) {
537 AdvanceLine();
538 } else {
539 Advance();
540 }
541 }
542 }
543
544 /**
545 * Skip over one CSS comment starting at the current read position.
546 */
547 void
SkipComment()548 nsCSSScanner::SkipComment()
549 {
550 MOZ_ASSERT(Peek() == '/' && Peek(1) == '*', "should not have been called");
551 Advance(2);
552 for (;;) {
553 int32_t ch = Peek();
554 if (ch < 0) {
555 if (mReporter)
556 mReporter->ReportUnexpectedEOF("PECommentEOF");
557 SetEOFCharacters(eEOFCharacters_Asterisk | eEOFCharacters_Slash);
558 return;
559 }
560 if (ch == '*') {
561 Advance();
562 ch = Peek();
563 if (ch < 0) {
564 if (mReporter)
565 mReporter->ReportUnexpectedEOF("PECommentEOF");
566 SetEOFCharacters(eEOFCharacters_Slash);
567 return;
568 }
569 if (ch == '/') {
570 Advance();
571 return;
572 }
573 } else if (IsVertSpace(ch)) {
574 AdvanceLine();
575 } else {
576 Advance();
577 }
578 }
579 }
580
581 /**
582 * If there is a valid escape sequence starting at the current read
583 * position, consume it, decode it, append the result to |aOutput|,
584 * and return true. Otherwise, consume nothing, leave |aOutput|
585 * unmodified, and return false. If |aInString| is true, accept the
586 * additional form of escape sequence allowed within string-like tokens.
587 */
588 bool
GatherEscape(nsString & aOutput,bool aInString)589 nsCSSScanner::GatherEscape(nsString& aOutput, bool aInString)
590 {
591 MOZ_ASSERT(Peek() == '\\', "should not have been called");
592 int32_t ch = Peek(1);
593 if (ch < 0) {
594 // If we are in a string (or a url() containing a string), we want to drop
595 // the backslash on the floor. Otherwise, we want to treat it as a U+FFFD
596 // character.
597 Advance();
598 if (aInString) {
599 SetEOFCharacters(eEOFCharacters_DropBackslash);
600 } else {
601 aOutput.Append(UCS2_REPLACEMENT_CHAR);
602 SetEOFCharacters(eEOFCharacters_ReplacementChar);
603 }
604 return true;
605 }
606 if (IsVertSpace(ch)) {
607 if (aInString) {
608 // In strings (and in url() containing a string), escaped
609 // newlines are completely removed, to allow splitting over
610 // multiple lines.
611 Advance();
612 AdvanceLine();
613 return true;
614 }
615 // Outside of strings, backslash followed by a newline is not an escape.
616 return false;
617 }
618
619 if (!IsHexDigit(ch)) {
620 // "Any character (except a hexadecimal digit, linefeed, carriage
621 // return, or form feed) can be escaped with a backslash to remove
622 // its special meaning." -- CSS2.1 section 4.1.3
623 Advance(2);
624 if (ch == 0) {
625 aOutput.Append(UCS2_REPLACEMENT_CHAR);
626 } else {
627 aOutput.Append(ch);
628 }
629 return true;
630 }
631
632 // "[at most six hexadecimal digits following a backslash] stand
633 // for the ISO 10646 character with that number, which must not be
634 // zero. (It is undefined in CSS 2.1 what happens if a style sheet
635 // does contain a character with Unicode codepoint zero.)"
636 // -- CSS2.1 section 4.1.3
637
638 // At this point we know we have \ followed by at least one
639 // hexadecimal digit, therefore the escape sequence is valid and we
640 // can go ahead and consume the backslash.
641 Advance();
642 uint32_t val = 0;
643 int i = 0;
644 do {
645 val = val * 16 + HexDigitValue(ch);
646 i++;
647 Advance();
648 ch = Peek();
649 } while (i < 6 && IsHexDigit(ch));
650
651 // "Interpret the hex digits as a hexadecimal number. If this number is zero,
652 // or is greater than the maximum allowed codepoint, return U+FFFD
653 // REPLACEMENT CHARACTER" -- CSS Syntax Level 3
654 if (MOZ_UNLIKELY(val == 0)) {
655 aOutput.Append(UCS2_REPLACEMENT_CHAR);
656 } else {
657 AppendUCS4ToUTF16(ENSURE_VALID_CHAR(val), aOutput);
658 }
659
660 // Consume exactly one whitespace character after a
661 // hexadecimal escape sequence.
662 if (IsVertSpace(ch)) {
663 AdvanceLine();
664 } else if (IsHorzSpace(ch)) {
665 Advance();
666 }
667 return true;
668 }
669
670 /**
671 * Consume a run of "text" beginning with the current read position,
672 * consisting of characters in the class |aClass| (which must be a
673 * suitable argument to IsOpenCharClass) plus escape sequences.
674 * Append the text to |aText|, after decoding escape sequences.
675 *
676 * Returns true if at least one character was appended to |aText|,
677 * false otherwise.
678 */
679 bool
GatherText(uint8_t aClass,nsString & aText)680 nsCSSScanner::GatherText(uint8_t aClass, nsString& aText)
681 {
682 // This is all of the character classes currently used with
683 // GatherText. If you have a need to use this function with a
684 // different class, go ahead and add it.
685 MOZ_ASSERT(aClass == IS_STRING ||
686 aClass == IS_IDCHAR ||
687 aClass == IS_URL_CHAR,
688 "possibly-inappropriate character class");
689
690 uint32_t start = mOffset;
691 bool inString = aClass == IS_STRING;
692
693 for (;;) {
694 // Consume runs of unescaped characters in one go.
695 uint32_t n = mOffset;
696 while (n < mCount && IsOpenCharClass(mBuffer[n], aClass)) {
697 n++;
698 }
699 if (n > mOffset) {
700 aText.Append(&mBuffer[mOffset], n - mOffset);
701 mOffset = n;
702 }
703 if (n == mCount) {
704 break;
705 }
706
707 int32_t ch = Peek();
708 MOZ_ASSERT(!IsOpenCharClass(ch, aClass),
709 "should not have exited the inner loop");
710 if (ch == 0) {
711 Advance();
712 aText.Append(UCS2_REPLACEMENT_CHAR);
713 continue;
714 }
715
716 if (ch != '\\') {
717 break;
718 }
719 if (!GatherEscape(aText, inString)) {
720 break;
721 }
722 }
723
724 return mOffset > start;
725 }
726
727 /**
728 * Scan an Ident token. This also handles Function and URL tokens,
729 * both of which begin indistinguishably from an identifier. It can
730 * produce a Symbol token when an apparent identifier actually led
731 * into an invalid escape sequence.
732 */
733 bool
ScanIdent(nsCSSToken & aToken)734 nsCSSScanner::ScanIdent(nsCSSToken& aToken)
735 {
736 if (MOZ_UNLIKELY(!GatherText(IS_IDCHAR, aToken.mIdent))) {
737 MOZ_ASSERT(Peek() == '\\',
738 "unexpected IsIdentStart character that did not begin an ident");
739 aToken.mSymbol = Peek();
740 Advance();
741 return true;
742 }
743
744 if (MOZ_LIKELY(Peek() != '(')) {
745 aToken.mType = eCSSToken_Ident;
746 return true;
747 }
748
749 Advance();
750 aToken.mType = eCSSToken_Function;
751 if (aToken.mIdent.LowerCaseEqualsLiteral("url")) {
752 NextURL(aToken);
753 } else if (aToken.mIdent.LowerCaseEqualsLiteral("var")) {
754 mSeenVariableReference = true;
755 }
756 return true;
757 }
758
759 /**
760 * Scan an AtKeyword token. Also handles production of Symbol when
761 * an '@' is not followed by an identifier.
762 */
763 bool
ScanAtKeyword(nsCSSToken & aToken)764 nsCSSScanner::ScanAtKeyword(nsCSSToken& aToken)
765 {
766 MOZ_ASSERT(Peek() == '@', "should not have been called");
767
768 // Fall back for when '@' isn't followed by an identifier.
769 aToken.mSymbol = '@';
770 Advance();
771
772 int32_t ch = Peek();
773 if (StartsIdent(ch, Peek(1))) {
774 if (GatherText(IS_IDCHAR, aToken.mIdent)) {
775 aToken.mType = eCSSToken_AtKeyword;
776 }
777 }
778 return true;
779 }
780
781 /**
782 * Scan a Hash token. Handles the distinction between eCSSToken_ID
783 * and eCSSToken_Hash, and handles production of Symbol when a '#'
784 * is not followed by identifier characters.
785 */
786 bool
ScanHash(nsCSSToken & aToken)787 nsCSSScanner::ScanHash(nsCSSToken& aToken)
788 {
789 MOZ_ASSERT(Peek() == '#', "should not have been called");
790
791 // Fall back for when '#' isn't followed by identifier characters.
792 aToken.mSymbol = '#';
793 Advance();
794
795 int32_t ch = Peek();
796 if (IsIdentChar(ch) || ch == '\\') {
797 nsCSSTokenType type =
798 StartsIdent(ch, Peek(1)) ? eCSSToken_ID : eCSSToken_Hash;
799 aToken.mIdent.SetLength(0);
800 if (GatherText(IS_IDCHAR, aToken.mIdent)) {
801 aToken.mType = type;
802 }
803 }
804
805 return true;
806 }
807
808 /**
809 * Scan a Number, Percentage, or Dimension token (all of which begin
810 * like a Number). Can produce a Symbol when a '.' is not followed by
811 * digits, or when '+' or '-' are not followed by either a digit or a
812 * '.' and then a digit. Can also produce a HTMLComment when it
813 * encounters '-->'.
814 */
815 bool
ScanNumber(nsCSSToken & aToken)816 nsCSSScanner::ScanNumber(nsCSSToken& aToken)
817 {
818 int32_t c = Peek();
819 #ifdef DEBUG
820 {
821 int32_t c2 = Peek(1);
822 int32_t c3 = Peek(2);
823 MOZ_ASSERT(IsDigit(c) ||
824 (IsDigit(c2) && (c == '.' || c == '+' || c == '-')) ||
825 (IsDigit(c3) && (c == '+' || c == '-') && c2 == '.'),
826 "should not have been called");
827 }
828 #endif
829
830 // Sign of the mantissa (-1 or 1).
831 int32_t sign = c == '-' ? -1 : 1;
832 // Absolute value of the integer part of the mantissa. This is a double so
833 // we don't run into overflow issues for consumers that only care about our
834 // floating-point value while still being able to express the full int32_t
835 // range for consumers who want integers.
836 double intPart = 0;
837 // Fractional part of the mantissa. This is a double so that when we convert
838 // to float at the end we'll end up rounding to nearest float instead of
839 // truncating down (as we would if fracPart were a float and we just
840 // effectively lost the last several digits).
841 double fracPart = 0;
842 // Absolute value of the power of 10 that we should multiply by (only
843 // relevant for numbers in scientific notation). Has to be a signed integer,
844 // because multiplication of signed by unsigned converts the unsigned to
845 // signed, so if we plan to actually multiply by expSign...
846 int32_t exponent = 0;
847 // Sign of the exponent.
848 int32_t expSign = 1;
849
850 aToken.mHasSign = (c == '+' || c == '-');
851 if (aToken.mHasSign) {
852 Advance();
853 c = Peek();
854 }
855
856 bool gotDot = (c == '.');
857
858 if (!gotDot) {
859 // Scan the integer part of the mantissa.
860 MOZ_ASSERT(IsDigit(c), "should have been excluded by logic above");
861 do {
862 intPart = 10*intPart + DecimalDigitValue(c);
863 Advance();
864 c = Peek();
865 } while (IsDigit(c));
866
867 gotDot = (c == '.') && IsDigit(Peek(1));
868 }
869
870 if (gotDot) {
871 // Scan the fractional part of the mantissa.
872 Advance();
873 c = Peek();
874 MOZ_ASSERT(IsDigit(c), "should have been excluded by logic above");
875 // Power of ten by which we need to divide our next digit
876 double divisor = 10;
877 do {
878 fracPart += DecimalDigitValue(c) / divisor;
879 divisor *= 10;
880 Advance();
881 c = Peek();
882 } while (IsDigit(c));
883 }
884
885 bool gotE = false;
886 if (c == 'e' || c == 'E') {
887 int32_t expSignChar = Peek(1);
888 int32_t nextChar = Peek(2);
889 if (IsDigit(expSignChar) ||
890 ((expSignChar == '-' || expSignChar == '+') && IsDigit(nextChar))) {
891 gotE = true;
892 if (expSignChar == '-') {
893 expSign = -1;
894 }
895 Advance(); // consumes the E
896 if (expSignChar == '-' || expSignChar == '+') {
897 Advance();
898 c = nextChar;
899 } else {
900 c = expSignChar;
901 }
902 MOZ_ASSERT(IsDigit(c), "should have been excluded by logic above");
903 do {
904 exponent = 10*exponent + DecimalDigitValue(c);
905 Advance();
906 c = Peek();
907 } while (IsDigit(c));
908 }
909 }
910
911 nsCSSTokenType type = eCSSToken_Number;
912
913 // Set mIntegerValid for all cases (except %, below) because we need
914 // it for the "2n" in :nth-child(2n).
915 aToken.mIntegerValid = false;
916
917 // Time to reassemble our number.
918 // Do all the math in double precision so it's truncated only once.
919 double value = sign * (intPart + fracPart);
920 if (gotE) {
921 // Avoid multiplication of 0 by Infinity.
922 if (value != 0.0) {
923 // Explicitly cast expSign*exponent to double to avoid issues with
924 // overloaded pow() on Windows.
925 value *= pow(10.0, double(expSign * exponent));
926 }
927 } else if (!gotDot) {
928 // Clamp values outside of integer range.
929 if (sign > 0) {
930 aToken.mInteger = int32_t(std::min(intPart, double(INT32_MAX)));
931 } else {
932 aToken.mInteger = int32_t(std::max(-intPart, double(INT32_MIN)));
933 }
934 aToken.mIntegerValid = true;
935 }
936
937 nsString& ident = aToken.mIdent;
938
939 // Check for Dimension and Percentage tokens.
940 if (c >= 0) {
941 if (StartsIdent(c, Peek(1))) {
942 if (GatherText(IS_IDCHAR, ident)) {
943 type = eCSSToken_Dimension;
944 }
945 } else if (c == '%') {
946 Advance();
947 type = eCSSToken_Percentage;
948 value = value / 100.0f;
949 aToken.mIntegerValid = false;
950 }
951 }
952 MOZ_ASSERT(!IsNaN(value), "The value should not be NaN");
953 aToken.mNumber = value;
954 aToken.mType = type;
955 return true;
956 }
957
958 /**
959 * Scan a string constant ('foo' or "foo"). Will always produce
960 * either a String or a Bad_String token; the latter occurs when the
961 * close quote is missing. Always returns true (for convenience in Next()).
962 */
963 bool
ScanString(nsCSSToken & aToken)964 nsCSSScanner::ScanString(nsCSSToken& aToken)
965 {
966 int32_t aStop = Peek();
967 MOZ_ASSERT(aStop == '"' || aStop == '\'', "should not have been called");
968 aToken.mType = eCSSToken_String;
969 aToken.mSymbol = char16_t(aStop); // Remember how it's quoted.
970 Advance();
971
972 for (;;) {
973 GatherText(IS_STRING, aToken.mIdent);
974
975 int32_t ch = Peek();
976 if (ch == -1) {
977 AddEOFCharacters(aStop == '"' ? eEOFCharacters_DoubleQuote :
978 eEOFCharacters_SingleQuote);
979 break; // EOF ends a string token with no error.
980 }
981 if (ch == aStop) {
982 Advance();
983 break;
984 }
985 // Both " and ' are excluded from IS_STRING.
986 if (ch == '"' || ch == '\'') {
987 aToken.mIdent.Append(ch);
988 Advance();
989 continue;
990 }
991
992 mSeenBadToken = true;
993 aToken.mType = eCSSToken_Bad_String;
994 if (mReporter)
995 mReporter->ReportUnexpected("SEUnterminatedString", aToken);
996 break;
997 }
998 return true;
999 }
1000
1001 /**
1002 * Scan a unicode-range token. These match the regular expression
1003 *
1004 * u\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?
1005 *
1006 * However, some such tokens are "invalid". There are three valid forms:
1007 *
1008 * u+[0-9a-f]{x} 1 <= x <= 6
1009 * u+[0-9a-f]{x}\?{y} 1 <= x+y <= 6
1010 * u+[0-9a-f]{x}-[0-9a-f]{y} 1 <= x <= 6, 1 <= y <= 6
1011 *
1012 * All unicode-range tokens have their text recorded in mIdent; valid ones
1013 * are also decoded into mInteger and mInteger2, and mIntegerValid is set.
1014 * Note that this does not validate the numeric range, only the syntactic
1015 * form.
1016 */
1017 bool
ScanURange(nsCSSToken & aResult)1018 nsCSSScanner::ScanURange(nsCSSToken& aResult)
1019 {
1020 int32_t intro1 = Peek();
1021 int32_t intro2 = Peek(1);
1022 int32_t ch = Peek(2);
1023
1024 MOZ_ASSERT((intro1 == 'u' || intro1 == 'U') &&
1025 intro2 == '+' &&
1026 (IsHexDigit(ch) || ch == '?'),
1027 "should not have been called");
1028
1029 aResult.mIdent.Append(intro1);
1030 aResult.mIdent.Append(intro2);
1031 Advance(2);
1032
1033 bool valid = true;
1034 bool haveQues = false;
1035 uint32_t low = 0;
1036 uint32_t high = 0;
1037 int i = 0;
1038
1039 do {
1040 aResult.mIdent.Append(ch);
1041 if (IsHexDigit(ch)) {
1042 if (haveQues) {
1043 valid = false; // All question marks should be at the end.
1044 }
1045 low = low*16 + HexDigitValue(ch);
1046 high = high*16 + HexDigitValue(ch);
1047 } else {
1048 haveQues = true;
1049 low = low*16 + 0x0;
1050 high = high*16 + 0xF;
1051 }
1052
1053 i++;
1054 Advance();
1055 ch = Peek();
1056 } while (i < 6 && (IsHexDigit(ch) || ch == '?'));
1057
1058 if (ch == '-' && IsHexDigit(Peek(1))) {
1059 if (haveQues) {
1060 valid = false;
1061 }
1062
1063 aResult.mIdent.Append(ch);
1064 Advance();
1065 ch = Peek();
1066 high = 0;
1067 i = 0;
1068 do {
1069 aResult.mIdent.Append(ch);
1070 high = high*16 + HexDigitValue(ch);
1071
1072 i++;
1073 Advance();
1074 ch = Peek();
1075 } while (i < 6 && IsHexDigit(ch));
1076 }
1077
1078 aResult.mInteger = low;
1079 aResult.mInteger2 = high;
1080 aResult.mIntegerValid = valid;
1081 aResult.mType = eCSSToken_URange;
1082 return true;
1083 }
1084
1085 #ifdef DEBUG
1086 /* static */ void
AssertEOFCharactersValid(uint32_t c)1087 nsCSSScanner::AssertEOFCharactersValid(uint32_t c)
1088 {
1089 MOZ_ASSERT(c == eEOFCharacters_None ||
1090 c == eEOFCharacters_ReplacementChar ||
1091 c == eEOFCharacters_Slash ||
1092 c == (eEOFCharacters_Asterisk |
1093 eEOFCharacters_Slash) ||
1094 c == eEOFCharacters_DoubleQuote ||
1095 c == eEOFCharacters_SingleQuote ||
1096 c == (eEOFCharacters_DropBackslash |
1097 eEOFCharacters_DoubleQuote) ||
1098 c == (eEOFCharacters_DropBackslash |
1099 eEOFCharacters_SingleQuote) ||
1100 c == eEOFCharacters_CloseParen ||
1101 c == (eEOFCharacters_ReplacementChar |
1102 eEOFCharacters_CloseParen) ||
1103 c == (eEOFCharacters_DoubleQuote |
1104 eEOFCharacters_CloseParen) ||
1105 c == (eEOFCharacters_SingleQuote |
1106 eEOFCharacters_CloseParen) ||
1107 c == (eEOFCharacters_DropBackslash |
1108 eEOFCharacters_DoubleQuote |
1109 eEOFCharacters_CloseParen) ||
1110 c == (eEOFCharacters_DropBackslash |
1111 eEOFCharacters_SingleQuote |
1112 eEOFCharacters_CloseParen),
1113 "invalid EOFCharacters value");
1114 }
1115 #endif
1116
1117 void
SetEOFCharacters(uint32_t aEOFCharacters)1118 nsCSSScanner::SetEOFCharacters(uint32_t aEOFCharacters)
1119 {
1120 mEOFCharacters = EOFCharacters(aEOFCharacters);
1121 }
1122
1123 void
AddEOFCharacters(uint32_t aEOFCharacters)1124 nsCSSScanner::AddEOFCharacters(uint32_t aEOFCharacters)
1125 {
1126 mEOFCharacters = EOFCharacters(mEOFCharacters | aEOFCharacters);
1127 }
1128
1129 static const char16_t kImpliedEOFCharacters[] = {
1130 UCS2_REPLACEMENT_CHAR, '*', '/', '"', '\'', ')', 0
1131 };
1132
1133 /* static */ void
AppendImpliedEOFCharacters(EOFCharacters aEOFCharacters,nsAString & aResult)1134 nsCSSScanner::AppendImpliedEOFCharacters(EOFCharacters aEOFCharacters,
1135 nsAString& aResult)
1136 {
1137 // First, ignore eEOFCharacters_DropBackslash.
1138 uint32_t c = aEOFCharacters >> 1;
1139
1140 // All of the remaining EOFCharacters bits represent appended characters,
1141 // and the bits are in the order that they need appending.
1142 for (const char16_t* p = kImpliedEOFCharacters; *p && c; p++, c >>= 1) {
1143 if (c & 1) {
1144 aResult.Append(*p);
1145 }
1146 }
1147
1148 MOZ_ASSERT(c == 0, "too many bits in mEOFCharacters");
1149 }
1150
1151 /**
1152 * Consume the part of an URL token after the initial 'url('. Caller
1153 * is assumed to have consumed 'url(' already. Will always produce
1154 * either an URL or a Bad_URL token.
1155 *
1156 * Exposed for use by nsCSSParser::ParseMozDocumentRule, which applies
1157 * the special lexical rules for URL tokens in a nonstandard context.
1158 */
1159 void
NextURL(nsCSSToken & aToken)1160 nsCSSScanner::NextURL(nsCSSToken& aToken)
1161 {
1162 SkipWhitespace();
1163
1164 // aToken.mIdent may be "url" at this point; clear that out
1165 aToken.mIdent.Truncate();
1166
1167 int32_t ch = Peek();
1168 // Do we have a string?
1169 if (ch == '"' || ch == '\'') {
1170 ScanString(aToken);
1171 if (MOZ_UNLIKELY(aToken.mType == eCSSToken_Bad_String)) {
1172 aToken.mType = eCSSToken_Bad_URL;
1173 return;
1174 }
1175 MOZ_ASSERT(aToken.mType == eCSSToken_String, "unexpected token type");
1176
1177 } else {
1178 // Otherwise, this is the start of a non-quoted url (which may be empty).
1179 aToken.mSymbol = char16_t(0);
1180 GatherText(IS_URL_CHAR, aToken.mIdent);
1181 }
1182
1183 // Consume trailing whitespace and then look for a close parenthesis.
1184 SkipWhitespace();
1185 ch = Peek();
1186 // ch can be less than zero indicating EOF
1187 if (MOZ_LIKELY(ch < 0 || ch == ')')) {
1188 Advance();
1189 aToken.mType = eCSSToken_URL;
1190 if (ch < 0) {
1191 AddEOFCharacters(eEOFCharacters_CloseParen);
1192 }
1193 } else {
1194 mSeenBadToken = true;
1195 aToken.mType = eCSSToken_Bad_URL;
1196 }
1197 }
1198
1199 /**
1200 * Primary scanner entry point. Consume one token and fill in
1201 * |aToken| accordingly. Will skip over any number of comments first,
1202 * and will also skip over rather than return whitespace and comment
1203 * tokens, depending on the value of |aSkip|.
1204 *
1205 * Returns true if it successfully consumed a token, false if EOF has
1206 * been reached. Will always advance the current read position by at
1207 * least one character unless called when already at EOF.
1208 */
1209 bool
Next(nsCSSToken & aToken,nsCSSScannerExclude aSkip)1210 nsCSSScanner::Next(nsCSSToken& aToken, nsCSSScannerExclude aSkip)
1211 {
1212 int32_t ch;
1213
1214 // do this here so we don't have to do it in dozens of other places
1215 aToken.mIdent.Truncate();
1216 aToken.mType = eCSSToken_Symbol;
1217
1218 for (;;) {
1219 // Consume any number of comments, and possibly also whitespace tokens,
1220 // in between other tokens.
1221 mTokenOffset = mOffset;
1222 mTokenLineOffset = mLineOffset;
1223 mTokenLineNumber = mLineNumber;
1224
1225 ch = Peek();
1226 if (IsWhitespace(ch)) {
1227 SkipWhitespace();
1228 if (aSkip != eCSSScannerExclude_WhitespaceAndComments) {
1229 aToken.mType = eCSSToken_Whitespace;
1230 return true;
1231 }
1232 continue; // start again at the beginning
1233 }
1234 if (ch == '/' && !IsSVGMode() && Peek(1) == '*') {
1235 SkipComment();
1236 if (aSkip == eCSSScannerExclude_None) {
1237 aToken.mType = eCSSToken_Comment;
1238 return true;
1239 }
1240 continue; // start again at the beginning
1241 }
1242 break;
1243 }
1244
1245 // EOF
1246 if (ch < 0) {
1247 return false;
1248 }
1249
1250 // 'u' could be UNICODE-RANGE or an identifier-family token
1251 if (ch == 'u' || ch == 'U') {
1252 int32_t c2 = Peek(1);
1253 int32_t c3 = Peek(2);
1254 if (c2 == '+' && (IsHexDigit(c3) || c3 == '?')) {
1255 return ScanURange(aToken);
1256 }
1257 return ScanIdent(aToken);
1258 }
1259
1260 // identifier family
1261 if (IsIdentStart(ch)) {
1262 return ScanIdent(aToken);
1263 }
1264
1265 // number family
1266 if (IsDigit(ch)) {
1267 return ScanNumber(aToken);
1268 }
1269
1270 if (ch == '.' && IsDigit(Peek(1))) {
1271 return ScanNumber(aToken);
1272 }
1273
1274 if (ch == '+') {
1275 int32_t c2 = Peek(1);
1276 if (IsDigit(c2) || (c2 == '.' && IsDigit(Peek(2)))) {
1277 return ScanNumber(aToken);
1278 }
1279 }
1280
1281 // '-' can start an identifier-family token, a number-family token,
1282 // or an HTML-comment
1283 if (ch == '-') {
1284 int32_t c2 = Peek(1);
1285 int32_t c3 = Peek(2);
1286 if (IsIdentStart(c2) || (c2 == '-' && c3 != '>')) {
1287 return ScanIdent(aToken);
1288 }
1289 if (IsDigit(c2) || (c2 == '.' && IsDigit(c3))) {
1290 return ScanNumber(aToken);
1291 }
1292 if (c2 == '-' && c3 == '>') {
1293 Advance(3);
1294 aToken.mType = eCSSToken_HTMLComment;
1295 aToken.mIdent.AssignLiteral("-->");
1296 return true;
1297 }
1298 }
1299
1300 // the other HTML-comment token
1301 if (ch == '<' && Peek(1) == '!' && Peek(2) == '-' && Peek(3) == '-') {
1302 Advance(4);
1303 aToken.mType = eCSSToken_HTMLComment;
1304 aToken.mIdent.AssignLiteral("<!--");
1305 return true;
1306 }
1307
1308 // AT_KEYWORD
1309 if (ch == '@') {
1310 return ScanAtKeyword(aToken);
1311 }
1312
1313 // HASH
1314 if (ch == '#') {
1315 return ScanHash(aToken);
1316 }
1317
1318 // STRING
1319 if (ch == '"' || ch == '\'') {
1320 return ScanString(aToken);
1321 }
1322
1323 // Match operators: ~= |= ^= $= *=
1324 nsCSSTokenType opType = MatchOperatorType(ch);
1325 if (opType != eCSSToken_Symbol && Peek(1) == '=') {
1326 aToken.mType = opType;
1327 Advance(2);
1328 return true;
1329 }
1330
1331 // Otherwise, a symbol (DELIM).
1332 aToken.mSymbol = ch;
1333 Advance();
1334 return true;
1335 }
1336
1337 /* nsCSSGridTemplateAreaScanner methods. */
1338
nsCSSGridTemplateAreaScanner(const nsAString & aBuffer)1339 nsCSSGridTemplateAreaScanner::nsCSSGridTemplateAreaScanner(const nsAString& aBuffer)
1340 : mBuffer(aBuffer.BeginReading())
1341 , mOffset(0)
1342 , mCount(aBuffer.Length())
1343 {
1344 }
1345
1346 bool
Next(nsCSSGridTemplateAreaToken & aTokenResult)1347 nsCSSGridTemplateAreaScanner::Next(nsCSSGridTemplateAreaToken& aTokenResult)
1348 {
1349 int32_t ch;
1350 // Skip whitespace
1351 do {
1352 if (mOffset >= mCount) {
1353 return false;
1354 }
1355 ch = mBuffer[mOffset];
1356 mOffset++;
1357 } while (IsWhitespace(ch));
1358
1359 if (IsOpenCharClass(ch, IS_IDCHAR)) {
1360 // Named cell token
1361 uint32_t start = mOffset - 1; // offset of |ch|
1362 while (mOffset < mCount && IsOpenCharClass(mBuffer[mOffset], IS_IDCHAR)) {
1363 mOffset++;
1364 }
1365 aTokenResult.mName.Assign(&mBuffer[start], mOffset - start);
1366 aTokenResult.isTrash = false;
1367 } else if (ch == '.') {
1368 // Null cell token
1369 // Skip any other '.'
1370 while (mOffset < mCount && mBuffer[mOffset] == '.') {
1371 mOffset++;
1372 }
1373 aTokenResult.mName.Truncate();
1374 aTokenResult.isTrash = false;
1375 } else {
1376 // Trash token
1377 aTokenResult.isTrash = true;
1378 }
1379 return true;
1380 }
1381