1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set sw=2 ts=2 et tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "nsHtml5StreamParser.h"
8
9 #include <stdlib.h>
10 #include <string.h>
11 #include <algorithm>
12 #include <new>
13 #include <type_traits>
14 #include <utility>
15 #include "GeckoProfiler.h"
16 #include "js/GCAPI.h"
17 #include "mozilla/ArrayIterator.h"
18 #include "mozilla/Buffer.h"
19 #include "mozilla/CheckedInt.h"
20 #include "mozilla/DebugOnly.h"
21 #include "mozilla/Encoding.h"
22 #include "mozilla/EncodingDetector.h"
23 #include "mozilla/Likely.h"
24 #include "mozilla/Maybe.h"
25 #include "mozilla/SchedulerGroup.h"
26 #include "mozilla/ScopeExit.h"
27 #include "mozilla/Services.h"
28 #include "mozilla/StaticPrefs_html5.h"
29 #include "mozilla/StaticPrefs_intl.h"
30 #include "mozilla/TaskCategory.h"
31 #include "mozilla/TextUtils.h"
32 #include "mozilla/Tuple.h"
33 #include "mozilla/UniquePtrExtensions.h"
34 #include "mozilla/Unused.h"
35 #include "mozilla/dom/BindingDeclarations.h"
36 #include "mozilla/dom/BrowsingContext.h"
37 #include "mozilla/dom/DebuggerUtilsBinding.h"
38 #include "mozilla/dom/DocGroup.h"
39 #include "mozilla/dom/Document.h"
40 #include "mozilla/mozalloc.h"
41 #include "mozilla/Vector.h"
42 #include "nsContentSink.h"
43 #include "nsContentUtils.h"
44 #include "nsCycleCollectionTraversalCallback.h"
45 #include "nsHtml5AtomTable.h"
46 #include "nsHtml5ByteReadable.h"
47 #include "nsHtml5Highlighter.h"
48 #include "nsHtml5Module.h"
49 #include "nsHtml5OwningUTF16Buffer.h"
50 #include "nsHtml5Parser.h"
51 #include "nsHtml5Speculation.h"
52 #include "nsHtml5StreamParserPtr.h"
53 #include "nsHtml5Tokenizer.h"
54 #include "nsHtml5TreeBuilder.h"
55 #include "nsHtml5TreeOpExecutor.h"
56 #include "nsHtml5TreeOpStage.h"
57 #include "nsIChannel.h"
58 #include "nsIContentSink.h"
59 #include "nsID.h"
60 #include "nsIDTD.h"
61 #include "nsIDocShell.h"
62 #include "nsIEventTarget.h"
63 #include "nsIHttpChannel.h"
64 #include "nsIInputStream.h"
65 #include "nsINestedURI.h"
66 #include "nsIObserverService.h"
67 #include "nsIRequest.h"
68 #include "nsIRunnable.h"
69 #include "nsIScriptError.h"
70 #include "nsIThread.h"
71 #include "nsIThreadRetargetableRequest.h"
72 #include "nsIThreadRetargetableStreamListener.h"
73 #include "nsITimer.h"
74 #include "nsIURI.h"
75 #include "nsJSEnvironment.h"
76 #include "nsLiteralString.h"
77 #include "nsNetUtil.h"
78 #include "nsString.h"
79 #include "nsTPromiseFlatString.h"
80 #include "nsThreadUtils.h"
81 #include "nsXULAppAPI.h"
82
83 extern "C" {
84 // Defined in intl/encoding_glue/src/lib.rs
85 const mozilla::Encoding* xmldecl_parse(const uint8_t* buf, size_t buf_len);
86 };
87
88 using namespace mozilla;
89 using namespace mozilla::dom;
90
91 /*
92 * Note that nsHtml5StreamParser implements cycle collecting AddRef and
93 * Release. Therefore, nsHtml5StreamParser must never be refcounted from
94 * the parser thread!
95 *
96 * To work around this limitation, runnables posted by the main thread to the
97 * parser thread hold their reference to the stream parser in an
98 * nsHtml5StreamParserPtr. Upon creation, nsHtml5StreamParserPtr addrefs the
99 * object it holds
100 * just like a regular nsRefPtr. This is OK, since the creation of the
101 * runnable and the nsHtml5StreamParserPtr happens on the main thread.
102 *
103 * When the runnable is done on the parser thread, the destructor of
104 * nsHtml5StreamParserPtr runs there. It doesn't call Release on the held object
105 * directly. Instead, it posts another runnable back to the main thread where
106 * that runnable calls Release on the wrapped object.
107 *
108 * When posting runnables in the other direction, the runnables have to be
109 * created on the main thread when nsHtml5StreamParser is instantiated and
110 * held for the lifetime of the nsHtml5StreamParser. This works, because the
111 * same runnabled can be dispatched multiple times and currently runnables
112 * posted from the parser thread to main thread don't need to wrap any
113 * runnable-specific data. (In the other direction, the runnables most notably
114 * wrap the byte data of the stream.)
115 */
116 NS_IMPL_CYCLE_COLLECTING_ADDREF(nsHtml5StreamParser)
117 NS_IMPL_CYCLE_COLLECTING_RELEASE(nsHtml5StreamParser)
118
119 NS_INTERFACE_TABLE_HEAD(nsHtml5StreamParser)
120 NS_INTERFACE_TABLE(nsHtml5StreamParser, nsISupports)
121 NS_INTERFACE_TABLE_TO_MAP_SEGUE_CYCLE_COLLECTION(nsHtml5StreamParser)
122 NS_INTERFACE_MAP_END
123
124 NS_IMPL_CYCLE_COLLECTION_CLASS(nsHtml5StreamParser)
125
126 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(nsHtml5StreamParser)
127 tmp->DropTimer();
128 NS_IMPL_CYCLE_COLLECTION_UNLINK(mRequest)
129 NS_IMPL_CYCLE_COLLECTION_UNLINK(mOwner)
130 tmp->mExecutorFlusher = nullptr;
131 tmp->mLoadFlusher = nullptr;
132 tmp->mExecutor = nullptr;
133 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
134
135 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(nsHtml5StreamParser)
136 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mRequest)
137 NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOwner)
138 // hack: count the strongly owned edge wrapped in the runnable
139 if (tmp->mExecutorFlusher) {
140 NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb, "mExecutorFlusher->mExecutor");
141 cb.NoteXPCOMChild(static_cast<nsIContentSink*>(tmp->mExecutor));
142 }
143 // hack: count the strongly owned edge wrapped in the runnable
144 if (tmp->mLoadFlusher) {
145 NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb, "mLoadFlusher->mExecutor");
146 cb.NoteXPCOMChild(static_cast<nsIContentSink*>(tmp->mExecutor));
147 }
148 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
149
150 class nsHtml5ExecutorFlusher : public Runnable {
151 private:
152 RefPtr<nsHtml5TreeOpExecutor> mExecutor;
153
154 public:
nsHtml5ExecutorFlusher(nsHtml5TreeOpExecutor * aExecutor)155 explicit nsHtml5ExecutorFlusher(nsHtml5TreeOpExecutor* aExecutor)
156 : Runnable("nsHtml5ExecutorFlusher"), mExecutor(aExecutor) {}
Run()157 NS_IMETHOD Run() override {
158 if (!mExecutor->isInList()) {
159 Document* doc = mExecutor->GetDocument();
160 if (XRE_IsContentProcess() &&
161 nsContentUtils::
162 HighPriorityEventPendingForTopLevelDocumentBeforeContentfulPaint(
163 doc)) {
164 // Possible early paint pending, reuse the runnable and try to
165 // call RunFlushLoop later.
166 nsCOMPtr<nsIRunnable> flusher = this;
167 if (NS_SUCCEEDED(
168 doc->Dispatch(TaskCategory::Network, flusher.forget()))) {
169 PROFILER_MARKER_UNTYPED("HighPrio blocking parser flushing(1)", DOM);
170 return NS_OK;
171 }
172 }
173 mExecutor->RunFlushLoop();
174 }
175 return NS_OK;
176 }
177 };
178
179 class nsHtml5LoadFlusher : public Runnable {
180 private:
181 RefPtr<nsHtml5TreeOpExecutor> mExecutor;
182
183 public:
nsHtml5LoadFlusher(nsHtml5TreeOpExecutor * aExecutor)184 explicit nsHtml5LoadFlusher(nsHtml5TreeOpExecutor* aExecutor)
185 : Runnable("nsHtml5LoadFlusher"), mExecutor(aExecutor) {}
Run()186 NS_IMETHOD Run() override {
187 mExecutor->FlushSpeculativeLoads();
188 return NS_OK;
189 }
190 };
191
nsHtml5StreamParser(nsHtml5TreeOpExecutor * aExecutor,nsHtml5Parser * aOwner,eParserMode aMode)192 nsHtml5StreamParser::nsHtml5StreamParser(nsHtml5TreeOpExecutor* aExecutor,
193 nsHtml5Parser* aOwner,
194 eParserMode aMode)
195 : mBomState(eBomState::BOM_SNIFFING_NOT_STARTED),
196 mCharsetSource(kCharsetUninitialized),
197 mEncodingSwitchSource(kCharsetUninitialized),
198 mEncoding(X_USER_DEFINED_ENCODING), // Obviously bogus value to notice if
199 // not updated
200 mNeedsEncodingSwitchTo(nullptr),
201 mSeenEligibleMetaCharset(false),
202 mChardetEof(false),
203 #ifdef DEBUG
204 mStartedFeedingDetector(false),
205 mStartedFeedingDevTools(false),
206 #endif
207 mReparseForbidden(false),
208 mForceAutoDetection(false),
209 mChannelHadCharset(false),
210 mLookingForMetaCharset(false),
211 mStartsWithLtQuestion(false),
212 mLookingForXmlDeclarationForXmlViewSource(false),
213 mTemplatePushedOrHeadPopped(false),
214 mGtBuffer(nullptr),
215 mGtPos(0),
216 mLastBuffer(nullptr), // Will be filled when starting
217 mExecutor(aExecutor),
218 mTreeBuilder(new nsHtml5TreeBuilder(
219 (aMode == VIEW_SOURCE_HTML || aMode == VIEW_SOURCE_XML)
220 ? nullptr
221 : mExecutor->GetStage(),
222 mExecutor->GetStage(), aMode == NORMAL)),
223 mTokenizer(
224 new nsHtml5Tokenizer(mTreeBuilder.get(), aMode == VIEW_SOURCE_XML)),
225 mTokenizerMutex("nsHtml5StreamParser mTokenizerMutex"),
226 mOwner(aOwner),
227 mLastWasCR(false),
228 mStreamState(eHtml5StreamState::STREAM_NOT_STARTED),
229 mSpeculating(false),
230 mAtEOF(false),
231 mSpeculationMutex("nsHtml5StreamParser mSpeculationMutex"),
232 mSpeculationFailureCount(0),
233 mNumBytesBuffered(0),
234 mTerminated(false),
235 mInterrupted(false),
236 mEventTarget(nsHtml5Module::GetStreamParserThread()->SerialEventTarget()),
237 mExecutorFlusher(new nsHtml5ExecutorFlusher(aExecutor)),
238 mLoadFlusher(new nsHtml5LoadFlusher(aExecutor)),
239 mInitialEncodingWasFromParentFrame(false),
240 mHasHadErrors(false),
241 mDetectorHasSeenNonAscii(false),
242 mDetectorHadOnlySeenAsciiWhenFirstGuessing(false),
243 mDecodingLocalFileWithoutTokenizing(false),
244 mBufferingBytes(false),
245 mFlushTimer(NS_NewTimer(mEventTarget)),
246 mFlushTimerMutex("nsHtml5StreamParser mFlushTimerMutex"),
247 mFlushTimerArmed(false),
248 mFlushTimerEverFired(false),
249 mMode(aMode) {
250 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
251 #ifdef DEBUG
252 mAtomTable.SetPermittedLookupEventTarget(mEventTarget);
253 #endif
254 mTokenizer->setInterner(&mAtomTable);
255 mTokenizer->setEncodingDeclarationHandler(this);
256
257 if (aMode == VIEW_SOURCE_HTML || aMode == VIEW_SOURCE_XML) {
258 nsHtml5Highlighter* highlighter =
259 new nsHtml5Highlighter(mExecutor->GetStage());
260 mTokenizer->EnableViewSource(highlighter); // takes ownership
261 mTreeBuilder->EnableViewSource(highlighter); // doesn't own
262 }
263
264 // There's a zeroing operator new for everything else
265 }
266
~nsHtml5StreamParser()267 nsHtml5StreamParser::~nsHtml5StreamParser() {
268 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
269 mTokenizer->end();
270 #ifdef DEBUG
271 {
272 mozilla::MutexAutoLock flushTimerLock(mFlushTimerMutex);
273 MOZ_ASSERT(!mFlushTimer, "Flush timer was not dropped before dtor!");
274 }
275 mRequest = nullptr;
276 mUnicodeDecoder = nullptr;
277 mFirstBuffer = nullptr;
278 mExecutor = nullptr;
279 mTreeBuilder = nullptr;
280 mTokenizer = nullptr;
281 mOwner = nullptr;
282 #endif
283 }
284
GetChannel(nsIChannel ** aChannel)285 nsresult nsHtml5StreamParser::GetChannel(nsIChannel** aChannel) {
286 NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
287 return mRequest ? CallQueryInterface(mRequest, aChannel)
288 : NS_ERROR_NOT_AVAILABLE;
289 }
290
291 std::tuple<NotNull<const Encoding*>, nsCharsetSource>
GuessEncoding(bool aInitial)292 nsHtml5StreamParser::GuessEncoding(bool aInitial) {
293 if (aInitial) {
294 if (!mDetectorHasSeenNonAscii) {
295 mDetectorHadOnlySeenAsciiWhenFirstGuessing = true;
296 }
297 }
298 MOZ_ASSERT(
299 mCharsetSource != kCharsetFromFinalUserForcedAutoDetection &&
300 mCharsetSource != kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8 &&
301 mCharsetSource !=
302 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic &&
303 mCharsetSource !=
304 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content &&
305 mCharsetSource !=
306 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD &&
307 mCharsetSource != kCharsetFromFinalAutoDetectionFile);
308 auto ifHadBeenForced = mDetector->Guess(EmptyCString(), true);
309 auto encoding =
310 mForceAutoDetection
311 ? ifHadBeenForced
312 : mDetector->Guess(mTLD, mDecodingLocalFileWithoutTokenizing);
313 nsCharsetSource source =
314 aInitial
315 ? (mForceAutoDetection
316 ? kCharsetFromInitialUserForcedAutoDetection
317 : (mDecodingLocalFileWithoutTokenizing
318 ? kCharsetFromFinalAutoDetectionFile
319 : kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Generic))
320 : (mForceAutoDetection
321 ? kCharsetFromFinalUserForcedAutoDetection
322 : (mDecodingLocalFileWithoutTokenizing
323 ? kCharsetFromFinalAutoDetectionFile
324 : kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic));
325 if (source == kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic) {
326 if (encoding == ISO_2022_JP_ENCODING) {
327 if (EncodingDetector::TldMayAffectGuess(mTLD)) {
328 source = kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content;
329 }
330 } else if (!mDetectorHasSeenNonAscii) {
331 source = kCharsetFromInitialAutoDetectionASCII; // deliberately Initial
332 } else if (ifHadBeenForced == UTF_8_ENCODING) {
333 // XXX subdivide by mDetectorHadOnlySeenAsciiWhenFirstGuessing in
334 // follow-up Not doing now to scope down the telemetry data review.
335 source = kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8;
336 } else if (encoding != ifHadBeenForced) {
337 source = kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD;
338 } else if (EncodingDetector::TldMayAffectGuess(mTLD)) {
339 source = kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content;
340 }
341 } else if (source ==
342 kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Generic) {
343 if (encoding == ISO_2022_JP_ENCODING) {
344 if (EncodingDetector::TldMayAffectGuess(mTLD)) {
345 source = kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Content;
346 }
347 } else if (!mDetectorHasSeenNonAscii) {
348 source = kCharsetFromInitialAutoDetectionASCII;
349 } else if (ifHadBeenForced == UTF_8_ENCODING) {
350 source = kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8;
351 } else if (encoding != ifHadBeenForced) {
352 source =
353 kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD;
354 } else if (EncodingDetector::TldMayAffectGuess(mTLD)) {
355 source = kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Content;
356 }
357 }
358 return {encoding, source};
359 }
360
FeedDetector(Span<const uint8_t> aBuffer)361 void nsHtml5StreamParser::FeedDetector(Span<const uint8_t> aBuffer) {
362 #ifdef DEBUG
363 mStartedFeedingDetector = true;
364 #endif
365 MOZ_ASSERT(!mChardetEof);
366 mDetectorHasSeenNonAscii = mDetector->Feed(aBuffer, false);
367 }
368
DetectorEof()369 void nsHtml5StreamParser::DetectorEof() {
370 #ifdef DEBUG
371 mStartedFeedingDetector = true;
372 #endif
373 if (mChardetEof) {
374 return;
375 }
376 mChardetEof = true;
377 mDetectorHasSeenNonAscii = mDetector->Feed(Span<const uint8_t>(), true);
378 }
379
SetViewSourceTitle(nsIURI * aURL)380 void nsHtml5StreamParser::SetViewSourceTitle(nsIURI* aURL) {
381 MOZ_ASSERT(NS_IsMainThread());
382
383 BrowsingContext* browsingContext =
384 mExecutor->GetDocument()->GetBrowsingContext();
385 if (browsingContext && browsingContext->WatchedByDevTools()) {
386 mURIToSendToDevtools = aURL;
387
388 nsID uuid;
389 nsresult rv = nsID::GenerateUUIDInPlace(uuid);
390 if (!NS_FAILED(rv)) {
391 char buffer[NSID_LENGTH];
392 uuid.ToProvidedString(buffer);
393 mUUIDForDevtools = NS_ConvertASCIItoUTF16(buffer);
394 }
395 }
396
397 if (aURL) {
398 nsCOMPtr<nsIURI> temp;
399 if (aURL->SchemeIs("view-source")) {
400 nsCOMPtr<nsINestedURI> nested = do_QueryInterface(aURL);
401 nested->GetInnerURI(getter_AddRefs(temp));
402 } else {
403 temp = aURL;
404 }
405 if (temp->SchemeIs("data")) {
406 // Avoid showing potentially huge data: URLs. The three last bytes are
407 // UTF-8 for an ellipsis.
408 mViewSourceTitle.AssignLiteral("data:\xE2\x80\xA6");
409 } else {
410 nsresult rv = temp->GetSpec(mViewSourceTitle);
411 if (NS_FAILED(rv)) {
412 mViewSourceTitle.AssignLiteral("\xE2\x80\xA6");
413 }
414 }
415 }
416 }
417
418 nsresult
SetupDecodingAndWriteSniffingBufferAndCurrentSegment(Span<const uint8_t> aPrefix,Span<const uint8_t> aFromSegment)419 nsHtml5StreamParser::SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
420 Span<const uint8_t> aPrefix, Span<const uint8_t> aFromSegment) {
421 NS_ASSERTION(IsParserThread(), "Wrong thread!");
422 mUnicodeDecoder = mEncoding->NewDecoderWithBOMRemoval();
423 nsresult rv = WriteStreamBytes(aPrefix);
424 NS_ENSURE_SUCCESS(rv, rv);
425 return WriteStreamBytes(aFromSegment);
426 }
427
SetupDecodingFromBom(NotNull<const Encoding * > aEncoding)428 void nsHtml5StreamParser::SetupDecodingFromBom(
429 NotNull<const Encoding*> aEncoding) {
430 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
431 mEncoding = aEncoding;
432 mDecodingLocalFileWithoutTokenizing = false;
433 mLookingForMetaCharset = false;
434 mBufferingBytes = false;
435 mUnicodeDecoder = mEncoding->NewDecoderWithoutBOMHandling();
436 mCharsetSource = kCharsetFromByteOrderMark;
437 mForceAutoDetection = false;
438 mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, false);
439 mBomState = BOM_SNIFFING_OVER;
440 if (mMode == VIEW_SOURCE_HTML) {
441 mTokenizer->StartViewSourceCharacters();
442 }
443 }
444
SetupDecodingFromUtf16BogoXml(NotNull<const Encoding * > aEncoding)445 void nsHtml5StreamParser::SetupDecodingFromUtf16BogoXml(
446 NotNull<const Encoding*> aEncoding) {
447 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
448 mEncoding = aEncoding;
449 mDecodingLocalFileWithoutTokenizing = false;
450 mLookingForMetaCharset = false;
451 mBufferingBytes = false;
452 mUnicodeDecoder = mEncoding->NewDecoderWithoutBOMHandling();
453 mCharsetSource = kCharsetFromXmlDeclarationUtf16;
454 mForceAutoDetection = false;
455 mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, false);
456 mBomState = BOM_SNIFFING_OVER;
457 if (mMode == VIEW_SOURCE_HTML) {
458 mTokenizer->StartViewSourceCharacters();
459 }
460 auto dst = mLastBuffer->TailAsSpan(READ_BUFFER_SIZE);
461 dst[0] = '<';
462 dst[1] = '?';
463 dst[2] = 'x';
464 mLastBuffer->AdvanceEnd(3);
465 MOZ_ASSERT(!mStartedFeedingDevTools);
466 OnNewContent(dst.To(3));
467 }
468
LengthOfLtContainingPrefixInSecondBuffer()469 size_t nsHtml5StreamParser::LengthOfLtContainingPrefixInSecondBuffer() {
470 MOZ_ASSERT(mBufferedBytes.Length() <= 2);
471 if (mBufferedBytes.Length() < 2) {
472 return 0;
473 }
474 Buffer<uint8_t>& second = mBufferedBytes[1];
475 const uint8_t* elements = second.Elements();
476 const uint8_t* lt = (const uint8_t*)memchr(elements, '>', second.Length());
477 if (lt) {
478 return (lt - elements) + 1;
479 }
480 return 0;
481 }
482
SniffStreamBytes(Span<const uint8_t> aFromSegment,bool aEof)483 nsresult nsHtml5StreamParser::SniffStreamBytes(Span<const uint8_t> aFromSegment,
484 bool aEof) {
485 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
486 MOZ_ASSERT_IF(aEof, aFromSegment.IsEmpty());
487
488 if (mCharsetSource >= kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8 &&
489 mCharsetSource <= kCharsetFromFinalUserForcedAutoDetection) {
490 if (mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN) {
491 mTreeBuilder->MaybeComplainAboutCharset("EncDetectorReloadPlain", true,
492 0);
493 } else {
494 mTreeBuilder->MaybeComplainAboutCharset("EncDetectorReload", true, 0);
495 }
496 }
497
498 // mEncoding and mCharsetSource potentially have come from channel or higher
499 // by now. If we find a BOM, SetupDecodingFromBom() will overwrite them.
500 // If we don't find a BOM, the previously set values of mEncoding and
501 // mCharsetSource are not modified by the BOM sniffing here.
502 static uint8_t utf8[] = {0xEF, 0xBB};
503 static uint8_t utf16le[] = {0xFF};
504 static uint8_t utf16be[] = {0xFE};
505 static uint8_t utf16leXml[] = {'<', 0x00, '?', 0x00, 'x'};
506 static uint8_t utf16beXml[] = {0x00, '<', 0x00, '?', 0x00};
507 // Buffer for replaying past bytes based on state machine state. If
508 // writing this from scratch, probably wouldn't do it this way, but
509 // let's keep the changes to a minimum.
510 const uint8_t* prefix = utf8;
511 size_t prefixLength = 0;
512 if (aEof && mBomState == BOM_SNIFFING_NOT_STARTED) {
513 // Avoid handling aEof in the BOM_SNIFFING_NOT_STARTED state below.
514 mBomState = BOM_SNIFFING_OVER;
515 }
516 for (size_t i = 0;
517 (i < aFromSegment.Length() && mBomState != BOM_SNIFFING_OVER) || aEof;
518 i++) {
519 switch (mBomState) {
520 case BOM_SNIFFING_NOT_STARTED:
521 MOZ_ASSERT(i == 0, "Bad BOM sniffing state.");
522 MOZ_ASSERT(!aEof, "Should have checked for aEof above!");
523 switch (aFromSegment[0]) {
524 case 0xEF:
525 mBomState = SEEN_UTF_8_FIRST_BYTE;
526 break;
527 case 0xFF:
528 mBomState = SEEN_UTF_16_LE_FIRST_BYTE;
529 break;
530 case 0xFE:
531 mBomState = SEEN_UTF_16_BE_FIRST_BYTE;
532 break;
533 case 0x00:
534 if (mCharsetSource < kCharsetFromXmlDeclarationUtf16 &&
535 mCharsetSource != kCharsetFromChannel) {
536 mBomState = SEEN_UTF_16_BE_XML_FIRST;
537 } else {
538 mBomState = BOM_SNIFFING_OVER;
539 }
540 break;
541 case '<':
542 if (mCharsetSource < kCharsetFromXmlDeclarationUtf16 &&
543 mCharsetSource != kCharsetFromChannel) {
544 mBomState = SEEN_UTF_16_LE_XML_FIRST;
545 } else {
546 mBomState = BOM_SNIFFING_OVER;
547 }
548 break;
549 default:
550 mBomState = BOM_SNIFFING_OVER;
551 break;
552 }
553 break;
554 case SEEN_UTF_16_LE_FIRST_BYTE:
555 if (!aEof && aFromSegment[i] == 0xFE) {
556 SetupDecodingFromBom(UTF_16LE_ENCODING);
557 return WriteStreamBytes(aFromSegment.From(i + 1));
558 }
559 prefix = utf16le;
560 prefixLength = 1 - i;
561 mBomState = BOM_SNIFFING_OVER;
562 break;
563 case SEEN_UTF_16_BE_FIRST_BYTE:
564 if (!aEof && aFromSegment[i] == 0xFF) {
565 SetupDecodingFromBom(UTF_16BE_ENCODING);
566 return WriteStreamBytes(aFromSegment.From(i + 1));
567 }
568 prefix = utf16be;
569 prefixLength = 1 - i;
570 mBomState = BOM_SNIFFING_OVER;
571 break;
572 case SEEN_UTF_8_FIRST_BYTE:
573 if (!aEof && aFromSegment[i] == 0xBB) {
574 mBomState = SEEN_UTF_8_SECOND_BYTE;
575 } else {
576 prefixLength = 1 - i;
577 mBomState = BOM_SNIFFING_OVER;
578 }
579 break;
580 case SEEN_UTF_8_SECOND_BYTE:
581 if (!aEof && aFromSegment[i] == 0xBF) {
582 SetupDecodingFromBom(UTF_8_ENCODING);
583 return WriteStreamBytes(aFromSegment.From(i + 1));
584 }
585 prefixLength = 2 - i;
586 mBomState = BOM_SNIFFING_OVER;
587 break;
588 case SEEN_UTF_16_BE_XML_FIRST:
589 if (!aEof && aFromSegment[i] == '<') {
590 mBomState = SEEN_UTF_16_BE_XML_SECOND;
591 } else {
592 prefix = utf16beXml;
593 prefixLength = 1 - i;
594 mBomState = BOM_SNIFFING_OVER;
595 }
596 break;
597 case SEEN_UTF_16_BE_XML_SECOND:
598 if (!aEof && aFromSegment[i] == 0x00) {
599 mBomState = SEEN_UTF_16_BE_XML_THIRD;
600 } else {
601 prefix = utf16beXml;
602 prefixLength = 2 - i;
603 mBomState = BOM_SNIFFING_OVER;
604 }
605 break;
606 case SEEN_UTF_16_BE_XML_THIRD:
607 if (!aEof && aFromSegment[i] == '?') {
608 mBomState = SEEN_UTF_16_BE_XML_FOURTH;
609 } else {
610 prefix = utf16beXml;
611 prefixLength = 3 - i;
612 mBomState = BOM_SNIFFING_OVER;
613 }
614 break;
615 case SEEN_UTF_16_BE_XML_FOURTH:
616 if (!aEof && aFromSegment[i] == 0x00) {
617 mBomState = SEEN_UTF_16_BE_XML_FIFTH;
618 } else {
619 prefix = utf16beXml;
620 prefixLength = 4 - i;
621 mBomState = BOM_SNIFFING_OVER;
622 }
623 break;
624 case SEEN_UTF_16_BE_XML_FIFTH:
625 if (!aEof && aFromSegment[i] == 'x') {
626 SetupDecodingFromUtf16BogoXml(UTF_16BE_ENCODING);
627 return WriteStreamBytes(aFromSegment.From(i + 1));
628 }
629 prefix = utf16beXml;
630 prefixLength = 5 - i;
631 mBomState = BOM_SNIFFING_OVER;
632 break;
633 case SEEN_UTF_16_LE_XML_FIRST:
634 if (!aEof && aFromSegment[i] == 0x00) {
635 mBomState = SEEN_UTF_16_LE_XML_SECOND;
636 } else {
637 if (!aEof && aFromSegment[i] == '?' &&
638 !(mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN)) {
639 mStartsWithLtQuestion = true;
640 }
641 prefix = utf16leXml;
642 prefixLength = 1 - i;
643 mBomState = BOM_SNIFFING_OVER;
644 }
645 break;
646 case SEEN_UTF_16_LE_XML_SECOND:
647 if (!aEof && aFromSegment[i] == '?') {
648 mBomState = SEEN_UTF_16_LE_XML_THIRD;
649 } else {
650 prefix = utf16leXml;
651 prefixLength = 2 - i;
652 mBomState = BOM_SNIFFING_OVER;
653 }
654 break;
655 case SEEN_UTF_16_LE_XML_THIRD:
656 if (!aEof && aFromSegment[i] == 0x00) {
657 mBomState = SEEN_UTF_16_LE_XML_FOURTH;
658 } else {
659 prefix = utf16leXml;
660 prefixLength = 3 - i;
661 mBomState = BOM_SNIFFING_OVER;
662 }
663 break;
664 case SEEN_UTF_16_LE_XML_FOURTH:
665 if (!aEof && aFromSegment[i] == 'x') {
666 mBomState = SEEN_UTF_16_LE_XML_FIFTH;
667 } else {
668 prefix = utf16leXml;
669 prefixLength = 4 - i;
670 mBomState = BOM_SNIFFING_OVER;
671 }
672 break;
673 case SEEN_UTF_16_LE_XML_FIFTH:
674 if (!aEof && aFromSegment[i] == 0x00) {
675 SetupDecodingFromUtf16BogoXml(UTF_16LE_ENCODING);
676 return WriteStreamBytes(aFromSegment.From(i + 1));
677 }
678 prefix = utf16leXml;
679 prefixLength = 5 - i;
680 mBomState = BOM_SNIFFING_OVER;
681 break;
682 default:
683 mBomState = BOM_SNIFFING_OVER;
684 break;
685 }
686 if (aEof) {
687 break;
688 }
689 }
690 // if we get here, there either was no BOM or the BOM sniffing isn't complete
691 // yet
692
693 MOZ_ASSERT(mCharsetSource != kCharsetFromByteOrderMark,
694 "Should not come here if BOM was found.");
695 MOZ_ASSERT(mCharsetSource != kCharsetFromXmlDeclarationUtf16,
696 "Should not come here if UTF-16 bogo-XML declaration was found.");
697 MOZ_ASSERT(mCharsetSource != kCharsetFromOtherComponent,
698 "kCharsetFromOtherComponent is for XSLT.");
699
700 if (mBomState == BOM_SNIFFING_OVER) {
701 if (mMode == VIEW_SOURCE_XML && mStartsWithLtQuestion &&
702 mCharsetSource < kCharsetFromChannel) {
703 // Sniff for XML declaration only.
704 MOZ_ASSERT(!mLookingForXmlDeclarationForXmlViewSource);
705 MOZ_ASSERT(!aEof);
706 MOZ_ASSERT(!mLookingForMetaCharset);
707 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing);
708 // Maybe we've already buffered a '>'.
709 MOZ_ASSERT(!mBufferedBytes.IsEmpty(),
710 "How did at least <? not get buffered?");
711 Buffer<uint8_t>& first = mBufferedBytes[0];
712 const Encoding* encoding =
713 xmldecl_parse(first.Elements(), first.Length());
714 if (encoding) {
715 mEncoding = WrapNotNull(encoding);
716 mCharsetSource = kCharsetFromXmlDeclaration;
717 } else if (memchr(first.Elements(), '>', first.Length())) {
718 // There was a '>', but an encoding still wasn't found.
719 ; // fall through to commit to the UTF-8 default.
720 } else if (size_t lengthOfPrefix =
721 LengthOfLtContainingPrefixInSecondBuffer()) {
722 // This can only happen if the first buffer was a lone '<', because
723 // we come here upon seeing the second byte '?' if the first two bytes
724 // were "<?". That is, the only way how we aren't dealing with the first
725 // buffer is if the first buffer only contained a single '<' and we are
726 // dealing with the second buffer that starts with '?'.
727 MOZ_ASSERT(first.Length() == 1);
728 MOZ_ASSERT(mBufferedBytes[1][0] == '?');
729 // Our scanner for XML declaration-like syntax wants to see a contiguous
730 // buffer, so let's linearize the data. (Ideally, the XML declaration
731 // scanner would be incremental, but this is the rare path anyway.)
732 Vector<uint8_t> contiguous;
733 if (!contiguous.append(first.Elements(), first.Length())) {
734 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
735 return NS_ERROR_OUT_OF_MEMORY;
736 }
737 if (!contiguous.append(mBufferedBytes[1].Elements(), lengthOfPrefix)) {
738 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
739 return NS_ERROR_OUT_OF_MEMORY;
740 }
741 encoding = xmldecl_parse(contiguous.begin(), contiguous.length());
742 if (encoding) {
743 mEncoding = WrapNotNull(encoding);
744 mCharsetSource = kCharsetFromXmlDeclaration;
745 }
746 // else no XML decl, commit to the UTF-8 default.
747 } else {
748 MOZ_ASSERT(mBufferingBytes);
749 mLookingForXmlDeclarationForXmlViewSource = true;
750 return NS_OK;
751 }
752 } else if (mMode != VIEW_SOURCE_XML &&
753 (mForceAutoDetection || mCharsetSource < kCharsetFromChannel)) {
754 // In order to use the buffering logic for meta with mForceAutoDetection,
755 // we set mLookingForMetaCharset but still actually potentially ignore the
756 // meta.
757 mFirstBufferOfMetaScan = mFirstBuffer;
758 MOZ_ASSERT(mLookingForMetaCharset);
759
760 if (mMode == VIEW_SOURCE_HTML) {
761 mTokenizer->FlushViewSource();
762 }
763 mTreeBuilder->Flush();
764 // Encoding committer flushes the ops on the main thread.
765
766 mozilla::MutexAutoLock speculationAutoLock(mSpeculationMutex);
767 nsHtml5Speculation* speculation = new nsHtml5Speculation(
768 mFirstBuffer, mFirstBuffer->getStart(), mTokenizer->getLineNumber(),
769 mTreeBuilder->newSnapshot());
770 MOZ_ASSERT(!mFlushTimerArmed, "How did we end up arming the timer?");
771 if (mMode == VIEW_SOURCE_HTML) {
772 mTokenizer->SetViewSourceOpSink(speculation);
773 mTokenizer->StartViewSourceCharacters();
774 } else {
775 MOZ_ASSERT(mMode != VIEW_SOURCE_XML);
776 mTreeBuilder->SetOpSink(speculation);
777 }
778 mSpeculations.AppendElement(speculation); // adopts the pointer
779 mSpeculating = true;
780 } else {
781 mLookingForMetaCharset = false;
782 mBufferingBytes = false;
783 mDecodingLocalFileWithoutTokenizing = false;
784 if (mMode == VIEW_SOURCE_HTML) {
785 mTokenizer->StartViewSourceCharacters();
786 }
787 }
788 mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, false);
789 return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
790 Span(prefix, prefixLength), aFromSegment);
791 }
792
793 return NS_OK;
794 }
795
796 class AddContentRunnable : public Runnable {
797 public:
AddContentRunnable(const nsAString & aParserID,nsIURI * aURI,Span<const char16_t> aData,bool aComplete)798 AddContentRunnable(const nsAString& aParserID, nsIURI* aURI,
799 Span<const char16_t> aData, bool aComplete)
800 : Runnable("AddContent") {
801 nsAutoCString spec;
802 aURI->GetSpec(spec);
803 mData.mUri.Construct(NS_ConvertUTF8toUTF16(spec));
804 mData.mParserID.Construct(aParserID);
805 mData.mContents.Construct(aData.Elements(), aData.Length());
806 mData.mComplete.Construct(aComplete);
807 }
808
Run()809 NS_IMETHOD Run() override {
810 nsAutoString json;
811 if (!mData.ToJSON(json)) {
812 return NS_ERROR_FAILURE;
813 }
814
815 nsCOMPtr<nsIObserverService> obsService = services::GetObserverService();
816 if (obsService) {
817 obsService->NotifyObservers(nullptr, "devtools-html-content",
818 PromiseFlatString(json).get());
819 }
820
821 return NS_OK;
822 }
823
824 HTMLContent mData;
825 };
826
OnNewContent(Span<const char16_t> aData)827 inline void nsHtml5StreamParser::OnNewContent(Span<const char16_t> aData) {
828 #ifdef DEBUG
829 mStartedFeedingDevTools = true;
830 #endif
831 if (mURIToSendToDevtools) {
832 if (aData.IsEmpty()) {
833 // Optimize out the runnable.
834 return;
835 }
836 NS_DispatchToMainThread(new AddContentRunnable(mUUIDForDevtools,
837 mURIToSendToDevtools, aData,
838 /* aComplete */ false));
839 }
840 }
841
OnContentComplete()842 inline void nsHtml5StreamParser::OnContentComplete() {
843 #ifdef DEBUG
844 mStartedFeedingDevTools = true;
845 #endif
846 if (mURIToSendToDevtools) {
847 NS_DispatchToMainThread(new AddContentRunnable(
848 mUUIDForDevtools, mURIToSendToDevtools, Span<const char16_t>(),
849 /* aComplete */ true));
850 mURIToSendToDevtools = nullptr;
851 }
852 }
853
WriteStreamBytes(Span<const uint8_t> aFromSegment)854 nsresult nsHtml5StreamParser::WriteStreamBytes(
855 Span<const uint8_t> aFromSegment) {
856 NS_ASSERTION(IsParserThread(), "Wrong thread!");
857 // mLastBuffer should always point to a buffer of the size
858 // READ_BUFFER_SIZE.
859 if (!mLastBuffer) {
860 NS_WARNING("mLastBuffer should not be null!");
861 MarkAsBroken(NS_ERROR_NULL_POINTER);
862 return NS_ERROR_NULL_POINTER;
863 }
864 size_t totalRead = 0;
865 auto src = aFromSegment;
866 for (;;) {
867 auto dst = mLastBuffer->TailAsSpan(READ_BUFFER_SIZE);
868 auto [result, read, written, hadErrors] =
869 mUnicodeDecoder->DecodeToUTF16(src, dst, false);
870 if (!(mLookingForMetaCharset || mDecodingLocalFileWithoutTokenizing)) {
871 OnNewContent(dst.To(written));
872 }
873 if (hadErrors && !mHasHadErrors) {
874 mHasHadErrors = true;
875 if (mEncoding == UTF_8_ENCODING) {
876 mTreeBuilder->TryToEnableEncodingMenu();
877 }
878 }
879 src = src.From(read);
880 totalRead += read;
881 mLastBuffer->AdvanceEnd(written);
882 if (result == kOutputFull) {
883 RefPtr<nsHtml5OwningUTF16Buffer> newBuf =
884 nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE);
885 if (!newBuf) {
886 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
887 return NS_ERROR_OUT_OF_MEMORY;
888 }
889 mLastBuffer = (mLastBuffer->next = std::move(newBuf));
890 } else {
891 MOZ_ASSERT(totalRead == aFromSegment.Length(),
892 "The Unicode decoder consumed the wrong number of bytes.");
893 (void)totalRead;
894 if (!mLookingForMetaCharset && mDecodingLocalFileWithoutTokenizing &&
895 mNumBytesBuffered == LOCAL_FILE_UTF_8_BUFFER_SIZE) {
896 MOZ_ASSERT(!mStartedFeedingDetector);
897 for (auto&& buffer : mBufferedBytes) {
898 FeedDetector(buffer);
899 }
900 // If the file is exactly LOCAL_FILE_UTF_8_BUFFER_SIZE bytes long
901 // we end up not considering the EOF. That's not fatal, since we
902 // don't consider the EOF if the file is
903 // LOCAL_FILE_UTF_8_BUFFER_SIZE + 1 bytes long.
904 auto [encoding, source] = GuessEncoding(true);
905 mCharsetSource = source;
906 if (encoding != mEncoding) {
907 mEncoding = encoding;
908 ReDecodeLocalFile();
909 } else {
910 MOZ_ASSERT(mEncoding == UTF_8_ENCODING);
911 CommitLocalFileToEncoding();
912 }
913 }
914 return NS_OK;
915 }
916 }
917 }
918
ReDecodeLocalFile()919 void nsHtml5StreamParser::ReDecodeLocalFile() {
920 MOZ_ASSERT(mDecodingLocalFileWithoutTokenizing && !mLookingForMetaCharset);
921 MOZ_ASSERT(mFirstBufferOfMetaScan);
922 MOZ_ASSERT(mCharsetSource == kCharsetFromFinalAutoDetectionFile ||
923 (mForceAutoDetection &&
924 mCharsetSource == kCharsetFromInitialUserForcedAutoDetection));
925
926 DiscardMetaSpeculation();
927
928 MOZ_ASSERT(mEncoding != UTF_8_ENCODING);
929
930 mDecodingLocalFileWithoutTokenizing = false;
931
932 mEncoding->NewDecoderWithBOMRemovalInto(*mUnicodeDecoder);
933 mHasHadErrors = false;
934
935 // Throw away previous decoded data
936 mLastBuffer = mFirstBuffer;
937 mLastBuffer->next = nullptr;
938 mLastBuffer->setStart(0);
939 mLastBuffer->setEnd(0);
940
941 mBufferingBytes = false;
942 mForceAutoDetection = false; // To stop feeding the detector
943 mFirstBufferOfMetaScan = nullptr;
944
945 mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, true);
946
947 // Decode again
948 for (auto&& buffer : mBufferedBytes) {
949 DoDataAvailable(buffer);
950 }
951
952 if (mMode == VIEW_SOURCE_HTML) {
953 mTokenizer->FlushViewSource();
954 }
955 mTreeBuilder->Flush();
956 }
957
CommitLocalFileToEncoding()958 void nsHtml5StreamParser::CommitLocalFileToEncoding() {
959 MOZ_ASSERT(mDecodingLocalFileWithoutTokenizing && !mLookingForMetaCharset);
960 MOZ_ASSERT(mFirstBufferOfMetaScan);
961 mDecodingLocalFileWithoutTokenizing = false;
962 MOZ_ASSERT(mCharsetSource == kCharsetFromFinalAutoDetectionFile ||
963 (mForceAutoDetection &&
964 mCharsetSource == kCharsetFromInitialUserForcedAutoDetection));
965 MOZ_ASSERT(mEncoding == UTF_8_ENCODING);
966
967 MOZ_ASSERT(!mStartedFeedingDevTools);
968 if (mURIToSendToDevtools) {
969 nsHtml5OwningUTF16Buffer* buffer = mFirstBufferOfMetaScan;
970 while (buffer) {
971 Span<const char16_t> data(buffer->getBuffer() + buffer->getStart(),
972 buffer->getLength());
973 OnNewContent(data);
974 buffer = buffer->next;
975 }
976 }
977
978 mFirstBufferOfMetaScan = nullptr;
979
980 mBufferingBytes = false;
981 mForceAutoDetection = false; // To stop feeding the detector
982 mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, true);
983 if (mMode == VIEW_SOURCE_HTML) {
984 mTokenizer->FlushViewSource();
985 }
986 mTreeBuilder->Flush();
987 }
988
989 class MaybeRunCollector : public Runnable {
990 public:
MaybeRunCollector(nsIDocShell * aDocShell)991 explicit MaybeRunCollector(nsIDocShell* aDocShell)
992 : Runnable("MaybeRunCollector"), mDocShell(aDocShell) {}
993
Run()994 NS_IMETHOD Run() override {
995 nsJSContext::MaybeRunNextCollectorSlice(mDocShell,
996 JS::GCReason::HTML_PARSER);
997 return NS_OK;
998 }
999
1000 nsCOMPtr<nsIDocShell> mDocShell;
1001 };
1002
OnStartRequest(nsIRequest * aRequest)1003 nsresult nsHtml5StreamParser::OnStartRequest(nsIRequest* aRequest) {
1004 MOZ_RELEASE_ASSERT(STREAM_NOT_STARTED == mStreamState,
1005 "Got OnStartRequest when the stream had already started.");
1006 MOZ_ASSERT(
1007 !mExecutor->HasStarted(),
1008 "Got OnStartRequest at the wrong stage in the executor life cycle.");
1009 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
1010
1011 // To avoid the cost of instantiating the detector when it's not needed,
1012 // let's instantiate only if we make it out of this method with the
1013 // intent to use it.
1014 auto detectorCreator = MakeScopeExit([&] {
1015 if ((mForceAutoDetection || mCharsetSource < kCharsetFromParentFrame) ||
1016 !(mMode == LOAD_AS_DATA || mMode == VIEW_SOURCE_XML)) {
1017 mDetector = mozilla::EncodingDetector::Create();
1018 }
1019 });
1020
1021 mRequest = aRequest;
1022
1023 mStreamState = STREAM_BEING_READ;
1024
1025 // For View Source, the parser should run with scripts "enabled" if a normal
1026 // load would have scripts enabled.
1027 bool scriptingEnabled =
1028 mMode == LOAD_AS_DATA ? false : mExecutor->IsScriptEnabled();
1029 mOwner->StartTokenizer(scriptingEnabled);
1030
1031 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing);
1032 bool isSrcdoc = false;
1033 nsCOMPtr<nsIChannel> channel;
1034 nsresult rv = GetChannel(getter_AddRefs(channel));
1035 if (NS_SUCCEEDED(rv)) {
1036 isSrcdoc = NS_IsSrcdocChannel(channel);
1037 if (!isSrcdoc && mCharsetSource <= kCharsetFromFallback) {
1038 nsCOMPtr<nsIURI> originalURI;
1039 rv = channel->GetOriginalURI(getter_AddRefs(originalURI));
1040 if (NS_SUCCEEDED(rv)) {
1041 if (originalURI->SchemeIs("resource")) {
1042 mCharsetSource = kCharsetFromBuiltIn;
1043 mEncoding = UTF_8_ENCODING;
1044 mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, false);
1045 } else {
1046 nsCOMPtr<nsIURI> currentURI;
1047 rv = channel->GetURI(getter_AddRefs(currentURI));
1048 if (NS_SUCCEEDED(rv)) {
1049 nsCOMPtr<nsIURI> innermost = NS_GetInnermostURI(currentURI);
1050 if (innermost->SchemeIs("file")) {
1051 MOZ_ASSERT(mEncoding == UTF_8_ENCODING);
1052 if (!(mMode == LOAD_AS_DATA || mMode == VIEW_SOURCE_XML)) {
1053 mDecodingLocalFileWithoutTokenizing = true;
1054 }
1055 } else {
1056 nsAutoCString host;
1057 innermost->GetAsciiHost(host);
1058 if (!host.IsEmpty()) {
1059 // First let's see if the host is DNS-absolute and ends with a
1060 // dot and get rid of that one.
1061 if (host.Last() == '.') {
1062 host.SetLength(host.Length() - 1);
1063 }
1064 int32_t index = host.RFindChar('.');
1065 if (index != kNotFound) {
1066 // We tolerate an IPv4 component as generic "TLD", so don't
1067 // bother checking.
1068 ToLowerCase(
1069 Substring(host, index + 1, host.Length() - (index + 1)),
1070 mTLD);
1071 }
1072 }
1073 }
1074 }
1075 }
1076 }
1077 }
1078 }
1079 mTreeBuilder->setIsSrcdocDocument(isSrcdoc);
1080 mTreeBuilder->setScriptingEnabled(scriptingEnabled);
1081 mTreeBuilder->SetPreventScriptExecution(
1082 !((mMode == NORMAL) && scriptingEnabled));
1083 mTokenizer->start();
1084 mExecutor->Start();
1085 mExecutor->StartReadingFromStage();
1086
1087 if (mMode == PLAIN_TEXT) {
1088 mTreeBuilder->StartPlainText();
1089 mTokenizer->StartPlainText();
1090 MOZ_ASSERT(
1091 mTemplatePushedOrHeadPopped); // Needed to force 1024-byte sniffing
1092 // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
1093 // can find them.
1094 mTreeBuilder->Flush();
1095 } else if (mMode == VIEW_SOURCE_PLAIN) {
1096 nsAutoString viewSourceTitle;
1097 CopyUTF8toUTF16(mViewSourceTitle, viewSourceTitle);
1098 mTreeBuilder->EnsureBufferSpace(viewSourceTitle.Length());
1099 mTreeBuilder->StartPlainTextViewSource(viewSourceTitle);
1100 mTokenizer->StartPlainText();
1101 MOZ_ASSERT(
1102 mTemplatePushedOrHeadPopped); // Needed to force 1024-byte sniffing
1103 // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
1104 // can find them.
1105 mTreeBuilder->Flush();
1106 } else if (mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML) {
1107 // Generate and flush the View Source document up to and including the
1108 // pre element start.
1109 mTokenizer->StartViewSource(NS_ConvertUTF8toUTF16(mViewSourceTitle));
1110 if (mMode == VIEW_SOURCE_XML) {
1111 mTokenizer->StartViewSourceCharacters();
1112 }
1113 // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
1114 // can find them.
1115 mTokenizer->FlushViewSource();
1116 }
1117
1118 /*
1119 * If you move the following line, be very careful not to cause
1120 * WillBuildModel to be called before the document has had its
1121 * script global object set.
1122 */
1123 rv = mExecutor->WillBuildModel();
1124 NS_ENSURE_SUCCESS(rv, rv);
1125
1126 RefPtr<nsHtml5OwningUTF16Buffer> newBuf =
1127 nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE);
1128 if (!newBuf) {
1129 // marks this stream parser as terminated,
1130 // which prevents entry to code paths that
1131 // would use mFirstBuffer or mLastBuffer.
1132 return mExecutor->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1133 }
1134 MOZ_ASSERT(!mFirstBuffer, "How come we have the first buffer set?");
1135 MOZ_ASSERT(!mLastBuffer, "How come we have the last buffer set?");
1136 mFirstBuffer = mLastBuffer = newBuf;
1137
1138 rv = NS_OK;
1139
1140 mNetworkEventTarget =
1141 mExecutor->GetDocument()->EventTargetFor(TaskCategory::Network);
1142
1143 nsCOMPtr<nsIHttpChannel> httpChannel(do_QueryInterface(mRequest, &rv));
1144 if (NS_SUCCEEDED(rv)) {
1145 // Non-HTTP channels are bogus enough that we let them work with unlabeled
1146 // runnables for now. Asserting for HTTP channels only.
1147 MOZ_ASSERT(mNetworkEventTarget || mMode == LOAD_AS_DATA,
1148 "How come the network event target is still null?");
1149
1150 nsAutoCString method;
1151 Unused << httpChannel->GetRequestMethod(method);
1152 // XXX does Necko have a way to renavigate POST, etc. without hitting
1153 // the network?
1154 if (!method.EqualsLiteral("GET")) {
1155 // This is the old Gecko behavior but the HTML5 spec disagrees.
1156 // Don't reparse on POST.
1157 mReparseForbidden = true;
1158 }
1159 }
1160
1161 // Attempt to retarget delivery of data (via OnDataAvailable) to the parser
1162 // thread, rather than through the main thread.
1163 nsCOMPtr<nsIThreadRetargetableRequest> threadRetargetableRequest =
1164 do_QueryInterface(mRequest, &rv);
1165 if (threadRetargetableRequest) {
1166 rv = threadRetargetableRequest->RetargetDeliveryTo(mEventTarget);
1167 if (NS_SUCCEEDED(rv)) {
1168 // Parser thread should be now ready to get data from necko and parse it
1169 // and main thread might have a chance to process a collector slice.
1170 // We need to do this asynchronously so that necko may continue processing
1171 // the request.
1172 nsCOMPtr<nsIRunnable> runnable =
1173 new MaybeRunCollector(mExecutor->GetDocument()->GetDocShell());
1174 mozilla::SchedulerGroup::Dispatch(
1175 mozilla::TaskCategory::GarbageCollection, runnable.forget());
1176 }
1177 }
1178
1179 if (NS_FAILED(rv)) {
1180 NS_WARNING("Failed to retarget HTML data delivery to the parser thread.");
1181 }
1182
1183 if (mCharsetSource == kCharsetFromParentFrame) {
1184 // Remember this for error reporting.
1185 mInitialEncodingWasFromParentFrame = true;
1186 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing);
1187 }
1188
1189 if (mForceAutoDetection || mCharsetSource < kCharsetFromChannel) {
1190 mBufferingBytes = true;
1191 if (mMode != VIEW_SOURCE_XML) {
1192 // We need to set mLookingForMetaCharset to true here in case the first
1193 // buffer to arrive is larger than 1024. We need the code that splits
1194 // the buffers at 1024 bytes to work even in that case.
1195 mLookingForMetaCharset = true;
1196 }
1197 }
1198
1199 if (mCharsetSource < kCharsetFromUtf8OnlyMime) {
1200 // we aren't ready to commit to an encoding yet
1201 // leave converter uninstantiated for now
1202 return NS_OK;
1203 }
1204
1205 MOZ_ASSERT(!(mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML));
1206
1207 MOZ_ASSERT(mEncoding == UTF_8_ENCODING,
1208 "How come UTF-8-only MIME type didn't set encoding to UTF-8?");
1209
1210 // We are loading JSON/WebVTT/etc. into a browsing context.
1211 // There's no need to remove the BOM manually here, because
1212 // the UTF-8 decoder removes it.
1213 mReparseForbidden = true;
1214 mForceAutoDetection = false;
1215
1216 // Instantiate the converter here to avoid BOM sniffing.
1217 mDecodingLocalFileWithoutTokenizing = false;
1218 mUnicodeDecoder = mEncoding->NewDecoderWithBOMRemoval();
1219 return NS_OK;
1220 }
1221
DoStopRequest()1222 void nsHtml5StreamParser::DoStopRequest() {
1223 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1224 MOZ_RELEASE_ASSERT(STREAM_BEING_READ == mStreamState,
1225 "Stream ended without being open.");
1226 mTokenizerMutex.AssertCurrentThreadOwns();
1227
1228 auto guard = MakeScopeExit([&] { OnContentComplete(); });
1229
1230 if (IsTerminated()) {
1231 return;
1232 }
1233
1234 if (MOZ_UNLIKELY(mLookingForXmlDeclarationForXmlViewSource)) {
1235 mLookingForXmlDeclarationForXmlViewSource = false;
1236 mBufferingBytes = false;
1237 mUnicodeDecoder = mEncoding->NewDecoderWithoutBOMHandling();
1238 mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, false);
1239
1240 for (auto&& buffer : mBufferedBytes) {
1241 Unused << WriteStreamBytes(buffer);
1242 }
1243 } else if (!mUnicodeDecoder) {
1244 nsresult rv;
1245 if (NS_FAILED(rv = SniffStreamBytes(Span<const uint8_t>(), true))) {
1246 MarkAsBroken(rv);
1247 return;
1248 }
1249 }
1250
1251 MOZ_ASSERT(mUnicodeDecoder,
1252 "Should have a decoder after finalizing sniffing.");
1253
1254 // mLastBuffer should always point to a buffer of the size
1255 // READ_BUFFER_SIZE.
1256 if (!mLastBuffer) {
1257 NS_WARNING("mLastBuffer should not be null!");
1258 MarkAsBroken(NS_ERROR_NULL_POINTER);
1259 return;
1260 }
1261
1262 Span<uint8_t> src; // empty span
1263 for (;;) {
1264 auto dst = mLastBuffer->TailAsSpan(READ_BUFFER_SIZE);
1265 uint32_t result;
1266 size_t read;
1267 size_t written;
1268 bool hadErrors;
1269 // Do not use structured binding lest deal with [-Werror=unused-variable]
1270 std::tie(result, read, written, hadErrors) =
1271 mUnicodeDecoder->DecodeToUTF16(src, dst, true);
1272 if (!(mLookingForMetaCharset || mDecodingLocalFileWithoutTokenizing)) {
1273 OnNewContent(dst.To(written));
1274 }
1275 if (hadErrors) {
1276 mHasHadErrors = true;
1277 }
1278 MOZ_ASSERT(read == 0, "How come an empty span was read form?");
1279 mLastBuffer->AdvanceEnd(written);
1280 if (result == kOutputFull) {
1281 RefPtr<nsHtml5OwningUTF16Buffer> newBuf =
1282 nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE);
1283 if (!newBuf) {
1284 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1285 return;
1286 }
1287 mLastBuffer = (mLastBuffer->next = std::move(newBuf));
1288 } else {
1289 if (!mLookingForMetaCharset && mDecodingLocalFileWithoutTokenizing) {
1290 MOZ_ASSERT(mNumBytesBuffered < LOCAL_FILE_UTF_8_BUFFER_SIZE);
1291 MOZ_ASSERT(!mStartedFeedingDetector);
1292 for (auto&& buffer : mBufferedBytes) {
1293 FeedDetector(buffer);
1294 }
1295 MOZ_ASSERT(!mChardetEof);
1296 DetectorEof();
1297 auto [encoding, source] = GuessEncoding(true);
1298 mCharsetSource = source;
1299 if (encoding != mEncoding) {
1300 mEncoding = encoding;
1301 ReDecodeLocalFile();
1302 DoStopRequest();
1303 return;
1304 }
1305 MOZ_ASSERT(mEncoding == UTF_8_ENCODING);
1306 CommitLocalFileToEncoding();
1307 }
1308 break;
1309 }
1310 }
1311
1312 mStreamState = STREAM_ENDED;
1313
1314 if (IsTerminatedOrInterrupted()) {
1315 return;
1316 }
1317
1318 ParseAvailableData();
1319 }
1320
1321 class nsHtml5RequestStopper : public Runnable {
1322 private:
1323 nsHtml5StreamParserPtr mStreamParser;
1324
1325 public:
nsHtml5RequestStopper(nsHtml5StreamParser * aStreamParser)1326 explicit nsHtml5RequestStopper(nsHtml5StreamParser* aStreamParser)
1327 : Runnable("nsHtml5RequestStopper"), mStreamParser(aStreamParser) {}
Run()1328 NS_IMETHOD Run() override {
1329 mozilla::MutexAutoLock autoLock(mStreamParser->mTokenizerMutex);
1330 mStreamParser->DoStopRequest();
1331 mStreamParser->PostLoadFlusher();
1332 return NS_OK;
1333 }
1334 };
1335
OnStopRequest(nsIRequest * aRequest,nsresult status)1336 nsresult nsHtml5StreamParser::OnStopRequest(nsIRequest* aRequest,
1337 nsresult status) {
1338 MOZ_ASSERT(mRequest == aRequest, "Got Stop on wrong stream.");
1339 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
1340 nsCOMPtr<nsIRunnable> stopper = new nsHtml5RequestStopper(this);
1341 if (NS_FAILED(mEventTarget->Dispatch(stopper, nsIThread::DISPATCH_NORMAL))) {
1342 NS_WARNING("Dispatching StopRequest event failed.");
1343 }
1344 return NS_OK;
1345 }
1346
DoDataAvailableBuffer(mozilla::Buffer<uint8_t> && aBuffer)1347 void nsHtml5StreamParser::DoDataAvailableBuffer(
1348 mozilla::Buffer<uint8_t>&& aBuffer) {
1349 if (MOZ_UNLIKELY(!mBufferingBytes)) {
1350 DoDataAvailable(aBuffer);
1351 return;
1352 }
1353 if (MOZ_UNLIKELY(mLookingForXmlDeclarationForXmlViewSource)) {
1354 const uint8_t* elements = aBuffer.Elements();
1355 size_t length = aBuffer.Length();
1356 const uint8_t* lt = (const uint8_t*)memchr(elements, '>', length);
1357 if (!lt) {
1358 mBufferedBytes.AppendElement(std::move(aBuffer));
1359 return;
1360 }
1361
1362 // We found an '>'. Now there either is or isn't an XML decl.
1363 length = (lt - elements) + 1;
1364 Vector<uint8_t> contiguous;
1365 for (auto&& buffer : mBufferedBytes) {
1366 if (!contiguous.append(buffer.Elements(), buffer.Length())) {
1367 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1368 return;
1369 }
1370 }
1371 if (!contiguous.append(elements, length)) {
1372 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1373 return;
1374 }
1375
1376 const Encoding* encoding =
1377 xmldecl_parse(contiguous.begin(), contiguous.length());
1378 if (encoding) {
1379 mEncoding = WrapNotNull(encoding);
1380 mCharsetSource = kCharsetFromXmlDeclaration;
1381 }
1382
1383 mLookingForXmlDeclarationForXmlViewSource = false;
1384 mBufferingBytes = false;
1385 mUnicodeDecoder = mEncoding->NewDecoderWithoutBOMHandling();
1386 mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, false);
1387
1388 for (auto&& buffer : mBufferedBytes) {
1389 DoDataAvailable(buffer);
1390 }
1391 DoDataAvailable(aBuffer);
1392 mBufferedBytes.Clear();
1393 return;
1394 }
1395 CheckedInt<size_t> bufferedPlusLength(aBuffer.Length());
1396 bufferedPlusLength += mNumBytesBuffered;
1397 if (!bufferedPlusLength.isValid()) {
1398 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1399 return;
1400 }
1401 // Ensure that WriteStreamBytes() sees buffers ending
1402 // exactly at the two special boundaries.
1403 bool metaBoundaryWithinBuffer =
1404 mLookingForMetaCharset &&
1405 mNumBytesBuffered < UNCONDITIONAL_META_SCAN_BOUNDARY &&
1406 bufferedPlusLength.value() > UNCONDITIONAL_META_SCAN_BOUNDARY;
1407 bool localFileLimitWithinBuffer =
1408 mDecodingLocalFileWithoutTokenizing &&
1409 mNumBytesBuffered < LOCAL_FILE_UTF_8_BUFFER_SIZE &&
1410 bufferedPlusLength.value() > LOCAL_FILE_UTF_8_BUFFER_SIZE;
1411 if (!metaBoundaryWithinBuffer && !localFileLimitWithinBuffer) {
1412 // Truncation OK, because we just checked the range.
1413 mNumBytesBuffered = bufferedPlusLength.value();
1414 mBufferedBytes.AppendElement(std::move(aBuffer));
1415 DoDataAvailable(mBufferedBytes.LastElement());
1416 } else {
1417 MOZ_RELEASE_ASSERT(
1418 !(metaBoundaryWithinBuffer && localFileLimitWithinBuffer),
1419 "How can Necko give us a buffer this large?");
1420 size_t boundary = metaBoundaryWithinBuffer
1421 ? UNCONDITIONAL_META_SCAN_BOUNDARY
1422 : LOCAL_FILE_UTF_8_BUFFER_SIZE;
1423 // Truncation OK, because the constant is small enough.
1424 size_t overBoundary = bufferedPlusLength.value() - boundary;
1425 MOZ_RELEASE_ASSERT(overBoundary < aBuffer.Length());
1426 size_t untilBoundary = aBuffer.Length() - overBoundary;
1427 auto span = aBuffer.AsSpan();
1428 auto head = span.To(untilBoundary);
1429 auto tail = span.From(untilBoundary);
1430 MOZ_RELEASE_ASSERT(mNumBytesBuffered + untilBoundary == boundary);
1431 // The following copies may end up being useless, but optimizing
1432 // them away would add complexity.
1433 Maybe<Buffer<uint8_t>> maybeHead = Buffer<uint8_t>::CopyFrom(head);
1434 if (maybeHead.isNothing()) {
1435 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1436 return;
1437 }
1438 mNumBytesBuffered = boundary;
1439 mBufferedBytes.AppendElement(std::move(*maybeHead));
1440 DoDataAvailable(mBufferedBytes.LastElement());
1441 // Re-decode may have happened here.
1442
1443 Maybe<Buffer<uint8_t>> maybeTail = Buffer<uint8_t>::CopyFrom(tail);
1444 if (maybeTail.isNothing()) {
1445 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1446 return;
1447 }
1448 mNumBytesBuffered += tail.Length();
1449 mBufferedBytes.AppendElement(std::move(*maybeTail));
1450 DoDataAvailable(mBufferedBytes.LastElement());
1451 }
1452 // Do this clean-up here to avoid use-after-free when
1453 // DoDataAvailable is passed a span pointing into an
1454 // element of mBufferedBytes.
1455 if (!mBufferingBytes) {
1456 mBufferedBytes.Clear();
1457 }
1458 }
1459
DoDataAvailable(Span<const uint8_t> aBuffer)1460 void nsHtml5StreamParser::DoDataAvailable(Span<const uint8_t> aBuffer) {
1461 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1462 MOZ_RELEASE_ASSERT(STREAM_BEING_READ == mStreamState,
1463 "DoDataAvailable called when stream not open.");
1464 mTokenizerMutex.AssertCurrentThreadOwns();
1465
1466 if (IsTerminated()) {
1467 return;
1468 }
1469
1470 nsresult rv;
1471 if (HasDecoder()) {
1472 if ((mForceAutoDetection || mCharsetSource < kCharsetFromParentFrame) &&
1473 !mBufferingBytes && !mReparseForbidden &&
1474 !(mMode == LOAD_AS_DATA || mMode == VIEW_SOURCE_XML)) {
1475 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing,
1476 "How is mBufferingBytes false if "
1477 "mDecodingLocalFileWithoutTokenizing is true?");
1478 FeedDetector(aBuffer);
1479 }
1480 rv = WriteStreamBytes(aBuffer);
1481 } else {
1482 rv = SniffStreamBytes(aBuffer, false);
1483 }
1484 if (NS_FAILED(rv)) {
1485 MarkAsBroken(rv);
1486 return;
1487 }
1488
1489 if (IsTerminatedOrInterrupted()) {
1490 return;
1491 }
1492
1493 if (!mLookingForMetaCharset && mDecodingLocalFileWithoutTokenizing) {
1494 return;
1495 }
1496
1497 ParseAvailableData();
1498
1499 if (mBomState != BOM_SNIFFING_OVER || mFlushTimerArmed || mSpeculating) {
1500 return;
1501 }
1502
1503 {
1504 mozilla::MutexAutoLock flushTimerLock(mFlushTimerMutex);
1505 mFlushTimer->InitWithNamedFuncCallback(
1506 nsHtml5StreamParser::TimerCallback, static_cast<void*>(this),
1507 mFlushTimerEverFired ? StaticPrefs::html5_flushtimer_initialdelay()
1508 : StaticPrefs::html5_flushtimer_subsequentdelay(),
1509 nsITimer::TYPE_ONE_SHOT, "nsHtml5StreamParser::DoDataAvailable");
1510 }
1511 mFlushTimerArmed = true;
1512 }
1513
1514 class nsHtml5DataAvailable : public Runnable {
1515 private:
1516 nsHtml5StreamParserPtr mStreamParser;
1517 Buffer<uint8_t> mData;
1518
1519 public:
nsHtml5DataAvailable(nsHtml5StreamParser * aStreamParser,Buffer<uint8_t> && aData)1520 nsHtml5DataAvailable(nsHtml5StreamParser* aStreamParser,
1521 Buffer<uint8_t>&& aData)
1522 : Runnable("nsHtml5DataAvailable"),
1523 mStreamParser(aStreamParser),
1524 mData(std::move(aData)) {}
Run()1525 NS_IMETHOD Run() override {
1526 mozilla::MutexAutoLock autoLock(mStreamParser->mTokenizerMutex);
1527 mStreamParser->DoDataAvailableBuffer(std::move(mData));
1528 mStreamParser->PostLoadFlusher();
1529 return NS_OK;
1530 }
1531 };
1532
OnDataAvailable(nsIRequest * aRequest,nsIInputStream * aInStream,uint64_t aSourceOffset,uint32_t aLength)1533 nsresult nsHtml5StreamParser::OnDataAvailable(nsIRequest* aRequest,
1534 nsIInputStream* aInStream,
1535 uint64_t aSourceOffset,
1536 uint32_t aLength) {
1537 nsresult rv;
1538
1539 MOZ_ASSERT(mRequest == aRequest, "Got data on wrong stream.");
1540 uint32_t totalRead;
1541 // Main thread to parser thread dispatch requires copying to buffer first.
1542 if (MOZ_UNLIKELY(NS_IsMainThread())) {
1543 if (NS_FAILED(rv = mExecutor->IsBroken())) {
1544 return rv;
1545 }
1546 Maybe<Buffer<uint8_t>> maybe = Buffer<uint8_t>::Alloc(aLength);
1547 if (maybe.isNothing()) {
1548 return mExecutor->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1549 }
1550 Buffer<uint8_t> data(std::move(*maybe));
1551 rv = aInStream->Read(reinterpret_cast<char*>(data.Elements()),
1552 data.Length(), &totalRead);
1553 NS_ENSURE_SUCCESS(rv, rv);
1554 MOZ_ASSERT(totalRead == aLength);
1555
1556 nsCOMPtr<nsIRunnable> dataAvailable =
1557 new nsHtml5DataAvailable(this, std::move(data));
1558 if (NS_FAILED(mEventTarget->Dispatch(dataAvailable,
1559 nsIThread::DISPATCH_NORMAL))) {
1560 NS_WARNING("Dispatching DataAvailable event failed.");
1561 }
1562 return rv;
1563 }
1564
1565 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1566 mozilla::MutexAutoLock autoLock(mTokenizerMutex);
1567
1568 if (NS_FAILED(rv = mTreeBuilder->IsBroken())) {
1569 return rv;
1570 }
1571
1572 // Since we're getting OnDataAvailable directly on the parser thread,
1573 // there is no nsHtml5DataAvailable that would call PostLoadFlusher.
1574 // Hence, we need to call PostLoadFlusher() before this method returns.
1575 // Braces for RAII clarity relative to the mutex despite not being
1576 // strictly necessary.
1577 {
1578 auto speculationFlusher = MakeScopeExit([&] { PostLoadFlusher(); });
1579
1580 if (mBufferingBytes) {
1581 Maybe<Buffer<uint8_t>> maybe = Buffer<uint8_t>::Alloc(aLength);
1582 if (maybe.isNothing()) {
1583 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
1584 return NS_ERROR_OUT_OF_MEMORY;
1585 }
1586 Buffer<uint8_t> data(std::move(*maybe));
1587 rv = aInStream->Read(reinterpret_cast<char*>(data.Elements()),
1588 data.Length(), &totalRead);
1589 NS_ENSURE_SUCCESS(rv, rv);
1590 MOZ_ASSERT(totalRead == aLength);
1591 DoDataAvailableBuffer(std::move(data));
1592 return rv;
1593 }
1594 // Read directly from response buffer.
1595 rv = aInStream->ReadSegments(CopySegmentsToParser, this, aLength,
1596 &totalRead);
1597 NS_ENSURE_SUCCESS(rv, rv);
1598 MOZ_ASSERT(totalRead == aLength);
1599 return rv;
1600 }
1601 }
1602
1603 // Called under lock by function ptr
1604 /* static */
CopySegmentsToParser(nsIInputStream * aInStream,void * aClosure,const char * aFromSegment,uint32_t aToOffset,uint32_t aCount,uint32_t * aWriteCount)1605 nsresult nsHtml5StreamParser::CopySegmentsToParser(
1606 nsIInputStream* aInStream, void* aClosure, const char* aFromSegment,
1607 uint32_t aToOffset, uint32_t aCount, uint32_t* aWriteCount) {
1608 nsHtml5StreamParser* parser = static_cast<nsHtml5StreamParser*>(aClosure);
1609
1610 parser->DoDataAvailable(AsBytes(Span(aFromSegment, aCount)));
1611 // Assume DoDataAvailable consumed all available bytes.
1612 *aWriteCount = aCount;
1613 return NS_OK;
1614 }
1615
PreferredForInternalEncodingDecl(const nsAString & aEncoding)1616 const Encoding* nsHtml5StreamParser::PreferredForInternalEncodingDecl(
1617 const nsAString& aEncoding) {
1618 const Encoding* newEncoding = Encoding::ForLabel(aEncoding);
1619 if (!newEncoding) {
1620 // the encoding name is bogus
1621 mTreeBuilder->MaybeComplainAboutCharset("EncMetaUnsupported", true,
1622 mTokenizer->getLineNumber());
1623 return nullptr;
1624 }
1625
1626 if (newEncoding == UTF_16BE_ENCODING || newEncoding == UTF_16LE_ENCODING) {
1627 mTreeBuilder->MaybeComplainAboutCharset("EncMetaUtf16", true,
1628 mTokenizer->getLineNumber());
1629 newEncoding = UTF_8_ENCODING;
1630 }
1631
1632 if (newEncoding == X_USER_DEFINED_ENCODING) {
1633 // WebKit/Blink hack for Indian and Armenian legacy sites
1634 mTreeBuilder->MaybeComplainAboutCharset("EncMetaUserDefined", true,
1635 mTokenizer->getLineNumber());
1636 newEncoding = WINDOWS_1252_ENCODING;
1637 }
1638
1639 if (newEncoding == REPLACEMENT_ENCODING) {
1640 // No line number, because the replacement encoding doesn't allow
1641 // showing the lines.
1642 mTreeBuilder->MaybeComplainAboutCharset("EncMetaReplacement", true, 0);
1643 }
1644
1645 return newEncoding;
1646 }
1647
internalEncodingDeclaration(nsHtml5String aEncoding)1648 bool nsHtml5StreamParser::internalEncodingDeclaration(nsHtml5String aEncoding) {
1649 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1650 if ((mCharsetSource >= kCharsetFromMetaTag &&
1651 mCharsetSource != kCharsetFromFinalAutoDetectionFile) ||
1652 mSeenEligibleMetaCharset) {
1653 return false;
1654 }
1655
1656 nsString newEncoding; // Not Auto, because using it to hold nsStringBuffer*
1657 aEncoding.ToString(newEncoding);
1658 auto encoding = PreferredForInternalEncodingDecl(newEncoding);
1659 if (!encoding) {
1660 return false;
1661 }
1662
1663 mSeenEligibleMetaCharset = true;
1664
1665 if (!mLookingForMetaCharset) {
1666 if (mInitialEncodingWasFromParentFrame) {
1667 mTreeBuilder->MaybeComplainAboutCharset("EncMetaTooLateFrame", true,
1668 mTokenizer->getLineNumber());
1669 } else {
1670 mTreeBuilder->MaybeComplainAboutCharset("EncMetaTooLate", true,
1671 mTokenizer->getLineNumber());
1672 }
1673 return false;
1674 }
1675 if (mTemplatePushedOrHeadPopped) {
1676 mTreeBuilder->MaybeComplainAboutCharset("EncMetaAfterHeadInKilobyte", false,
1677 mTokenizer->getLineNumber());
1678 }
1679
1680 if (mForceAutoDetection &&
1681 (encoding->IsAsciiCompatible() || encoding == ISO_2022_JP_ENCODING)) {
1682 return false;
1683 }
1684
1685 mNeedsEncodingSwitchTo = encoding;
1686 mEncodingSwitchSource = kCharsetFromMetaTag;
1687 return true;
1688 }
1689
TemplatePushedOrHeadPopped()1690 bool nsHtml5StreamParser::TemplatePushedOrHeadPopped() {
1691 MOZ_ASSERT(
1692 IsParserThread() || mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN,
1693 "Wrong thread!");
1694 mTemplatePushedOrHeadPopped = true;
1695 return mNumBytesBuffered >= UNCONDITIONAL_META_SCAN_BOUNDARY;
1696 }
1697
RememberGt(int32_t aPos)1698 void nsHtml5StreamParser::RememberGt(int32_t aPos) {
1699 if (mLookingForMetaCharset) {
1700 mGtBuffer = mFirstBuffer;
1701 mGtPos = aPos;
1702 }
1703 }
1704
PostLoadFlusher()1705 void nsHtml5StreamParser::PostLoadFlusher() {
1706 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1707 mTokenizerMutex.AssertCurrentThreadOwns();
1708
1709 mTreeBuilder->FlushLoads();
1710 // Dispatch this runnable unconditionally, because the loads
1711 // that need flushing may have been flushed earlier even if the
1712 // flush right above here did nothing. (Is this still true?)
1713 nsCOMPtr<nsIRunnable> runnable(mLoadFlusher);
1714 if (NS_FAILED(DispatchToMain(runnable.forget()))) {
1715 NS_WARNING("failed to dispatch load flush event");
1716 }
1717 }
1718
FlushTreeOpsAndDisarmTimer()1719 void nsHtml5StreamParser::FlushTreeOpsAndDisarmTimer() {
1720 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
1721 if (mFlushTimerArmed) {
1722 // avoid calling Cancel if the flush timer isn't armed to avoid acquiring
1723 // a mutex
1724 {
1725 mozilla::MutexAutoLock flushTimerLock(mFlushTimerMutex);
1726 mFlushTimer->Cancel();
1727 }
1728 mFlushTimerArmed = false;
1729 }
1730 if (mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML) {
1731 mTokenizer->FlushViewSource();
1732 }
1733 mTreeBuilder->Flush();
1734 nsCOMPtr<nsIRunnable> runnable(mExecutorFlusher);
1735 if (NS_FAILED(DispatchToMain(runnable.forget()))) {
1736 NS_WARNING("failed to dispatch executor flush event");
1737 }
1738 }
1739
SwitchDecoderIfAsciiSoFar(NotNull<const Encoding * > aEncoding)1740 void nsHtml5StreamParser::SwitchDecoderIfAsciiSoFar(
1741 NotNull<const Encoding*> aEncoding) {
1742 if (mEncoding == aEncoding) {
1743 MOZ_ASSERT(!mStartedFeedingDevTools);
1744 // Report all already-decoded buffers to the dev tools if needed.
1745 if (mURIToSendToDevtools) {
1746 nsHtml5OwningUTF16Buffer* buffer = mFirstBufferOfMetaScan;
1747 while (buffer) {
1748 auto s = Span(buffer->getBuffer(), buffer->getEnd());
1749 OnNewContent(s);
1750 buffer = buffer->next;
1751 }
1752 }
1753 return;
1754 }
1755 if (!mEncoding->IsAsciiCompatible() || !aEncoding->IsAsciiCompatible()) {
1756 return;
1757 }
1758 size_t numAscii = 0;
1759 MOZ_ASSERT(mFirstBufferOfMetaScan,
1760 "Why did we come here without starting meta scan?");
1761 nsHtml5OwningUTF16Buffer* buffer = mFirstBufferOfMetaScan;
1762 while (buffer != mFirstBuffer) {
1763 MOZ_ASSERT(buffer, "mFirstBuffer should have acted as sentinel!");
1764 MOZ_ASSERT(buffer->getStart() == buffer->getEnd(),
1765 "Why wasn't an early buffer fully consumed?");
1766 auto s = Span(buffer->getBuffer(), buffer->getStart());
1767 if (!IsAscii(s)) {
1768 return;
1769 }
1770 numAscii += s.Length();
1771 buffer = buffer->next;
1772 }
1773 auto s = Span(mFirstBuffer->getBuffer(), mFirstBuffer->getStart());
1774 if (!IsAscii(s)) {
1775 return;
1776 }
1777 numAscii += s.Length();
1778
1779 MOZ_ASSERT(!mStartedFeedingDevTools);
1780 // Report the ASCII prefix to dev tools if needed
1781 if (mURIToSendToDevtools) {
1782 buffer = mFirstBufferOfMetaScan;
1783 while (buffer != mFirstBuffer) {
1784 MOZ_ASSERT(buffer, "mFirstBuffer should have acted as sentinel!");
1785 MOZ_ASSERT(buffer->getStart() == buffer->getEnd(),
1786 "Why wasn't an early buffer fully consumed?");
1787 auto s = Span(buffer->getBuffer(), buffer->getStart());
1788 OnNewContent(s);
1789 buffer = buffer->next;
1790 }
1791 auto s = Span(mFirstBuffer->getBuffer(), mFirstBuffer->getStart());
1792 OnNewContent(s);
1793 }
1794
1795 // Success! Now let's get rid of the already-decoded but not tokenized data:
1796 mFirstBuffer->setEnd(mFirstBuffer->getStart());
1797 mLastBuffer = mFirstBuffer;
1798 mFirstBuffer->next = nullptr;
1799
1800 // Note: We could have scanned further for ASCII, which could avoid some
1801 // buffer deallocation and reallocation. However, chances are that if we got
1802 // until meta without non-ASCII before, there's going to be a title with
1803 // non-ASCII soon after anyway, so let's avoid the complexity of finding out.
1804
1805 MOZ_ASSERT(mUnicodeDecoder, "How come we scanned meta without a decoder?");
1806 mEncoding = aEncoding;
1807 mEncoding->NewDecoderWithoutBOMHandlingInto(*mUnicodeDecoder);
1808 mHasHadErrors = false;
1809
1810 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing,
1811 "Must have set mDecodingLocalFileWithoutTokenizing to false to "
1812 "report data to dev tools below");
1813 MOZ_ASSERT(!mLookingForMetaCharset,
1814 "Must have set mLookingForMetaCharset to false to report data to "
1815 "dev tools below");
1816
1817 // Now skip over as many bytes and redecode the tail of the
1818 // buffered bytes.
1819 size_t skipped = 0;
1820 for (auto&& buffer : mBufferedBytes) {
1821 size_t nextSkipped = skipped + buffer.Length();
1822 if (nextSkipped <= numAscii) {
1823 skipped = nextSkipped;
1824 continue;
1825 }
1826 if (skipped >= numAscii) {
1827 WriteStreamBytes(buffer);
1828 skipped = nextSkipped;
1829 continue;
1830 }
1831 size_t tailLength = nextSkipped - numAscii;
1832 WriteStreamBytes(Span<uint8_t>(buffer).From(buffer.Length() - tailLength));
1833 skipped = nextSkipped;
1834 }
1835 }
1836
CountGts()1837 size_t nsHtml5StreamParser::CountGts() {
1838 if (!mGtBuffer) {
1839 return 0;
1840 }
1841 size_t gts = 0;
1842 nsHtml5OwningUTF16Buffer* buffer = mFirstBufferOfMetaScan;
1843 for (;;) {
1844 MOZ_ASSERT(buffer, "How did we walk past mGtBuffer?");
1845 char16_t* buf = buffer->getBuffer();
1846 if (buffer == mGtBuffer) {
1847 for (int32_t i = 0; i <= mGtPos; ++i) {
1848 if (buf[i] == u'>') {
1849 ++gts;
1850 }
1851 }
1852 break;
1853 }
1854 for (int32_t i = 0; i < buffer->getEnd(); ++i) {
1855 if (buf[i] == u'>') {
1856 ++gts;
1857 }
1858 }
1859 buffer = buffer->next;
1860 }
1861 return gts;
1862 }
1863
DiscardMetaSpeculation()1864 void nsHtml5StreamParser::DiscardMetaSpeculation() {
1865 mozilla::MutexAutoLock speculationAutoLock(mSpeculationMutex);
1866 // Rewind the stream
1867 MOZ_ASSERT(!mAtEOF, "How did we end up setting this?");
1868 mTokenizer->resetToDataState();
1869 mTokenizer->setLineNumber(1);
1870 mLastWasCR = false;
1871
1872 if (mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN) {
1873 // resetToDataState() above logically rewinds to the state before
1874 // the plain text start, so we need to start plain text again to
1875 // put the tokenizer into the plain text state.
1876 mTokenizer->StartPlainText();
1877 }
1878
1879 mFirstBuffer = mLastBuffer;
1880 mFirstBuffer->setStart(0);
1881 mFirstBuffer->setEnd(0);
1882 mFirstBuffer->next = nullptr;
1883
1884 mTreeBuilder->flushCharacters(); // empty the pending buffer
1885 mTreeBuilder->ClearOps(); // now get rid of the failed ops
1886
1887 if (mMode == VIEW_SOURCE_HTML) {
1888 mTokenizer->RewindViewSource();
1889 }
1890
1891 {
1892 // We know that this resets the tree builder back to the start state.
1893 // This must happen _after_ the flushCharacters() call above!
1894 const auto& speculation = mSpeculations.ElementAt(0);
1895 mTreeBuilder->loadState(speculation->GetSnapshot());
1896 }
1897
1898 // Experimentation suggests that we don't need to do anything special
1899 // for ignoring the leading LF in View Source here.
1900
1901 mSpeculations.Clear(); // potentially a huge number of destructors
1902 // run here synchronously...
1903
1904 // Now set up a new speculation for the main thread to find.
1905 // Note that we stay in the speculating state, because the main thread
1906 // knows how to come out of that state and this thread does not.
1907
1908 nsHtml5Speculation* speculation = new nsHtml5Speculation(
1909 mFirstBuffer, mFirstBuffer->getStart(), mTokenizer->getLineNumber(),
1910 mTreeBuilder->newSnapshot());
1911 MOZ_ASSERT(!mFlushTimerArmed, "How did we end up arming the timer?");
1912 if (mMode == VIEW_SOURCE_HTML) {
1913 mTokenizer->SetViewSourceOpSink(speculation);
1914 mTokenizer->StartViewSourceCharacters();
1915 } else {
1916 MOZ_ASSERT(mMode != VIEW_SOURCE_XML);
1917 mTreeBuilder->SetOpSink(speculation);
1918 }
1919 mSpeculations.AppendElement(speculation); // adopts the pointer
1920 MOZ_ASSERT(mSpeculating, "How did we end speculating?");
1921 }
1922
1923 /*
1924 * The general idea is to match WebKit and Blink exactly for meta
1925 * scan except:
1926 *
1927 * 1. WebKit and Blink look for meta as if scripting was disabled
1928 * for `noscript` purposes. This implementation matches the
1929 * `noscript` treatment of the observable DOM building (in order
1930 * to be able to use the same tree builder run).
1931 * 2. WebKit and Blink look for meta as if the foreign content
1932 * feedback from the tree builder to the tokenizer didn't exist.
1933 * This implementation considers the foreign content rules in
1934 * order to be able to use the same tree builder run for meta
1935 * and the observable DOM building. Note that since <svg> and
1936 * <math> imply the end of head, this only matters for meta after
1937 * head but starting within the 1024-byte zone.
1938 *
1939 * Template is treated specially, because that WebKit/Blink behavior
1940 * is easy to emulate unlike the above two exceptions. In general,
1941 * the meta scan token handler in WebKit and Blink behaves as if there
1942 * was a scripting-disabled tree builder predating the introduction
1943 * of foreign content and template.
1944 *
1945 * Meta is honored if it _starts_ within the first 1024 kilobytes or,
1946 * if by the 1024-byte boundary head hasn't ended and a template
1947 * element hasn't started, a meta occurs before the first of the head
1948 * ending or a template element starting.
1949 *
1950 * If a meta isn't honored according to the above definition, and
1951 * we aren't dealing with plain text, the buffered bytes, which by
1952 * now have to contain `>` character unless we encountered EOF, are
1953 * scanned for syntax resembling an XML declaration.
1954 *
1955 * If neither a meta nor syntax resembling an XML declaration has
1956 * been honored and we aren't inheriting the encoding from a
1957 * same-origin parent or parsing for XHR, chardetng is used.
1958 * chardetng runs first for the part of the document that was searched
1959 * for meta and then at EOF. The part searched for meta is defined as
1960 * follows in order to avoid network buffer boundary-dependent
1961 * behavior:
1962 *
1963 * 1. At least the first 1024 bytes. (This is what happens for plain
1964 * text.)
1965 * 2. If the 1024-byte boundary is within a tag, comment, doctype,
1966 * or CDATA section, at least up to the end of that token or CDATA
1967 * section. (Exception: If the 1024-byte boundary is in an RCDATA
1968 * end tag that hasn't yet been decided to be an end tag, the
1969 * token is not considered.)
1970 * 3. If at the 1024-byte boundary, head hasn't ended and there hasn't
1971 * been a template tag, up to the end of the first template tag
1972 * or token ending the head, whichever comes first.
1973 * 4. Except if head is ended by a text token, only to the end of the
1974 * most recent tag, comment, or doctype token. (Because text is
1975 * coalesced, so it would be harder to correlate the text to the
1976 * bytes.)
1977 *
1978 * An encoding-related reload is still possible if chardetng's guess
1979 * at EOF differs from its initial guess.
1980 */
ProcessLookingForMetaCharset(bool aEof)1981 bool nsHtml5StreamParser::ProcessLookingForMetaCharset(bool aEof) {
1982 MOZ_ASSERT(mBomState == BOM_SNIFFING_OVER);
1983 MOZ_ASSERT(mMode != VIEW_SOURCE_XML);
1984 bool rewound = false;
1985 MOZ_ASSERT(mForceAutoDetection ||
1986 mCharsetSource < kCharsetFromInitialAutoDetectionASCII ||
1987 mCharsetSource == kCharsetFromParentFrame,
1988 "Why are we looking for meta charset if we've seen it?");
1989 // NOTE! We may come here multiple times with
1990 // mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY
1991 // if the tokenizer suspends multiple times after decoding has reached
1992 // mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY. That's why
1993 // we need to also check whether the we are at the end of the last
1994 // decoded buffer.
1995 // Note that DoDataAvailableBuffer() ensures that the code here has
1996 // the opportunity to run at the exact UNCONDITIONAL_META_SCAN_BOUNDARY
1997 // even if there isn't a network buffer boundary there.
1998 bool atKilobyte = false;
1999 if ((mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY &&
2000 mFirstBuffer == mLastBuffer && !mFirstBuffer->hasMore())) {
2001 atKilobyte = true;
2002 mTokenizer->AtKilobyteBoundary();
2003 }
2004 if (!mNeedsEncodingSwitchTo &&
2005 (aEof || (mTemplatePushedOrHeadPopped &&
2006 !mTokenizer->IsInTokenStartedAtKilobyteBoundary() &&
2007 (atKilobyte ||
2008 mNumBytesBuffered > UNCONDITIONAL_META_SCAN_BOUNDARY)))) {
2009 // meta charset was not found
2010 mLookingForMetaCharset = false;
2011 if (mStartsWithLtQuestion && mCharsetSource < kCharsetFromXmlDeclaration) {
2012 // Look for bogo XML declaration.
2013 // Search the first buffer in the hope that '>' is within it.
2014 MOZ_ASSERT(!mBufferedBytes.IsEmpty(),
2015 "How did at least <? not get buffered?");
2016 Buffer<uint8_t>& first = mBufferedBytes[0];
2017 const Encoding* encoding =
2018 xmldecl_parse(first.Elements(), first.Length());
2019 if (!encoding) {
2020 // Our bogo XML declaration scanner wants to see a contiguous buffer, so
2021 // let's linearize the data. (Ideally, the XML declaration scanner would
2022 // be incremental, but this is the rare path anyway.)
2023 Vector<uint8_t> contiguous;
2024 if (!contiguous.append(first.Elements(), first.Length())) {
2025 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
2026 return false;
2027 }
2028 for (size_t i = 1; i < mBufferedBytes.Length(); ++i) {
2029 Buffer<uint8_t>& buffer = mBufferedBytes[i];
2030 const uint8_t* elements = buffer.Elements();
2031 size_t length = buffer.Length();
2032 const uint8_t* lt = (const uint8_t*)memchr(elements, '>', length);
2033 bool stop = false;
2034 if (lt) {
2035 length = (lt - elements) + 1;
2036 stop = true;
2037 }
2038 if (!contiguous.append(elements, length)) {
2039 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
2040 return false;
2041 }
2042 if (stop) {
2043 // Avoid linearizing all buffered bytes unnecessarily.
2044 break;
2045 }
2046 }
2047 encoding = xmldecl_parse(contiguous.begin(), contiguous.length());
2048 }
2049 if (encoding) {
2050 if (!(mForceAutoDetection && (encoding->IsAsciiCompatible() ||
2051 encoding == ISO_2022_JP_ENCODING))) {
2052 mForceAutoDetection = false;
2053 mNeedsEncodingSwitchTo = encoding;
2054 mEncodingSwitchSource = kCharsetFromXmlDeclaration;
2055 }
2056 }
2057 }
2058 // Check again in case we found an encoding in the bogo XML declaration.
2059 if (!mNeedsEncodingSwitchTo &&
2060 (mForceAutoDetection ||
2061 mCharsetSource < kCharsetFromInitialAutoDetectionASCII) &&
2062 !(mMode == LOAD_AS_DATA || mMode == VIEW_SOURCE_XML) &&
2063 !(mDecodingLocalFileWithoutTokenizing && !aEof &&
2064 mNumBytesBuffered <= LOCAL_FILE_UTF_8_BUFFER_SIZE)) {
2065 MOZ_ASSERT(!mStartedFeedingDetector);
2066 if (mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY || aEof) {
2067 // We know that all the buffered bytes have been tokenized, so feed
2068 // them all to chardetng.
2069 for (auto&& buffer : mBufferedBytes) {
2070 FeedDetector(buffer);
2071 }
2072 if (aEof) {
2073 MOZ_ASSERT(!mChardetEof);
2074 DetectorEof();
2075 }
2076 auto [encoding, source] = GuessEncoding(true);
2077 mNeedsEncodingSwitchTo = encoding;
2078 mEncodingSwitchSource = source;
2079 } else if (mNumBytesBuffered > UNCONDITIONAL_META_SCAN_BOUNDARY) {
2080 size_t gtsLeftToFind = CountGts();
2081 size_t bytesSeen = 0;
2082 // We sync the bytes to the UTF-16 code units seen to avoid depending
2083 // on network buffer boundaries. We do the syncing by counting '>'
2084 // bytes / code units. However, we always scan at least 1024 bytes.
2085 // The 1024-byte boundary is guaranteed to be between buffers.
2086 // The guarantee is implemented in DoDataAvailableBuffer().
2087 for (auto&& buffer : mBufferedBytes) {
2088 if (!mNeedsEncodingSwitchTo) {
2089 if (gtsLeftToFind) {
2090 auto span = buffer.AsSpan();
2091 bool feed = true;
2092 for (size_t i = 0; i < span.Length(); ++i) {
2093 if (span[i] == uint8_t('>')) {
2094 --gtsLeftToFind;
2095 if (!gtsLeftToFind) {
2096 if (bytesSeen < UNCONDITIONAL_META_SCAN_BOUNDARY) {
2097 break;
2098 }
2099 ++i; // Skip the gt
2100 FeedDetector(span.To(i));
2101 auto [encoding, source] = GuessEncoding(true);
2102 mNeedsEncodingSwitchTo = encoding;
2103 mEncodingSwitchSource = source;
2104 FeedDetector(span.From(i));
2105 bytesSeen += buffer.Length();
2106 // No need to update bytesSeen anymore, but let's do it for
2107 // debugging.
2108 // We should do `continue outer;` but C++ can't.
2109 feed = false;
2110 break;
2111 }
2112 }
2113 }
2114 if (feed) {
2115 FeedDetector(buffer);
2116 bytesSeen += buffer.Length();
2117 }
2118 continue;
2119 }
2120 if (bytesSeen == UNCONDITIONAL_META_SCAN_BOUNDARY) {
2121 auto [encoding, source] = GuessEncoding(true);
2122 mNeedsEncodingSwitchTo = encoding;
2123 mEncodingSwitchSource = source;
2124 }
2125 }
2126 FeedDetector(buffer);
2127 bytesSeen += buffer.Length();
2128 }
2129 }
2130 MOZ_ASSERT(mNeedsEncodingSwitchTo,
2131 "How come we didn't call GuessEncoding()?");
2132 }
2133 }
2134 if (mNeedsEncodingSwitchTo) {
2135 mDecodingLocalFileWithoutTokenizing = false;
2136 mLookingForMetaCharset = false;
2137
2138 auto needsEncodingSwitchTo = WrapNotNull(mNeedsEncodingSwitchTo);
2139 mNeedsEncodingSwitchTo = nullptr;
2140
2141 SwitchDecoderIfAsciiSoFar(needsEncodingSwitchTo);
2142 // The above line may have changed mEncoding so that mEncoding equals
2143 // needsEncodingSwitchTo.
2144
2145 mCharsetSource = mEncodingSwitchSource;
2146
2147 if (mMode == VIEW_SOURCE_HTML) {
2148 mTokenizer->FlushViewSource();
2149 }
2150 mTreeBuilder->Flush();
2151
2152 if (mEncoding != needsEncodingSwitchTo) {
2153 // Speculation failed
2154 rewound = true;
2155
2156 if (mEncoding == ISO_2022_JP_ENCODING ||
2157 needsEncodingSwitchTo == ISO_2022_JP_ENCODING) {
2158 // Chances are no Web author will fix anything due to this message, so
2159 // this is here to help understanding issues when debugging sites made
2160 // by someone else.
2161 mTreeBuilder->MaybeComplainAboutCharset("EncSpeculationFail2022", false,
2162 mTokenizer->getLineNumber());
2163 } else {
2164 if (mCharsetSource == kCharsetFromMetaTag) {
2165 mTreeBuilder->MaybeComplainAboutCharset(
2166 "EncSpeculationFailMeta", false, mTokenizer->getLineNumber());
2167 } else if (mCharsetSource == kCharsetFromXmlDeclaration) {
2168 // This intentionally refers to the line number of how far ahead
2169 // the document was parsed even though the bogo XML decl is always
2170 // on line 1.
2171 mTreeBuilder->MaybeComplainAboutCharset(
2172 "EncSpeculationFailXml", false, mTokenizer->getLineNumber());
2173 }
2174 }
2175
2176 DiscardMetaSpeculation();
2177 // Redecode the stream.
2178 mEncoding = needsEncodingSwitchTo;
2179 mUnicodeDecoder = mEncoding->NewDecoderWithBOMRemoval();
2180 mHasHadErrors = false;
2181
2182 MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing,
2183 "Must have set mDecodingLocalFileWithoutTokenizing to false "
2184 "to report data to dev tools below");
2185 MOZ_ASSERT(!mLookingForMetaCharset,
2186 "Must have set mLookingForMetaCharset to false to report data "
2187 "to dev tools below");
2188 for (auto&& buffer : mBufferedBytes) {
2189 WriteStreamBytes(buffer);
2190 }
2191 }
2192 } else if (!mLookingForMetaCharset && !mDecodingLocalFileWithoutTokenizing) {
2193 MOZ_ASSERT(!mStartedFeedingDevTools);
2194 // Report all already-decoded buffers to the dev tools if needed.
2195 if (mURIToSendToDevtools) {
2196 nsHtml5OwningUTF16Buffer* buffer = mFirstBufferOfMetaScan;
2197 while (buffer) {
2198 auto s = Span(buffer->getBuffer(), buffer->getEnd());
2199 OnNewContent(s);
2200 buffer = buffer->next;
2201 }
2202 }
2203 }
2204 if (!mLookingForMetaCharset) {
2205 mGtBuffer = nullptr;
2206 mGtPos = 0;
2207
2208 if (!mDecodingLocalFileWithoutTokenizing) {
2209 mFirstBufferOfMetaScan = nullptr;
2210 mBufferingBytes = false;
2211 mBufferedBytes.Clear();
2212 mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, true);
2213 if (mMode == VIEW_SOURCE_HTML) {
2214 mTokenizer->FlushViewSource();
2215 }
2216 mTreeBuilder->Flush();
2217 }
2218 }
2219 return rewound;
2220 }
2221
ParseAvailableData()2222 void nsHtml5StreamParser::ParseAvailableData() {
2223 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
2224 mTokenizerMutex.AssertCurrentThreadOwns();
2225 MOZ_ASSERT(!(mDecodingLocalFileWithoutTokenizing && !mLookingForMetaCharset));
2226
2227 if (IsTerminatedOrInterrupted()) {
2228 return;
2229 }
2230
2231 if (mSpeculating && !IsSpeculationEnabled()) {
2232 return;
2233 }
2234
2235 bool requestedReload = false;
2236 for (;;) {
2237 if (!mFirstBuffer->hasMore()) {
2238 if (mFirstBuffer == mLastBuffer) {
2239 switch (mStreamState) {
2240 case STREAM_BEING_READ:
2241 // never release the last buffer.
2242 if (!mSpeculating) {
2243 // reuse buffer space if not speculating
2244 mFirstBuffer->setStart(0);
2245 mFirstBuffer->setEnd(0);
2246 }
2247 return; // no more data for now but expecting more
2248 case STREAM_ENDED:
2249 if (mAtEOF) {
2250 return;
2251 }
2252 if (mLookingForMetaCharset) {
2253 // When called with aEof=true, ProcessLookingForMetaCharset()
2254 // is guaranteed to set mLookingForMetaCharset to false so
2255 // that we can't come here twice.
2256 if (ProcessLookingForMetaCharset(true)) {
2257 if (IsTerminatedOrInterrupted()) {
2258 return;
2259 }
2260 continue;
2261 }
2262 } else if ((mForceAutoDetection ||
2263 mCharsetSource < kCharsetFromParentFrame) &&
2264 !(mMode == LOAD_AS_DATA || mMode == VIEW_SOURCE_XML) &&
2265 !mReparseForbidden) {
2266 // An earlier DetectorEof() call is possible in which case
2267 // the one here is a no-op.
2268 DetectorEof();
2269 auto [encoding, source] = GuessEncoding(false);
2270 if (encoding != mEncoding) {
2271 // Request a reload from the docshell.
2272 MOZ_ASSERT(
2273 (source >=
2274 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8 &&
2275 source <=
2276 kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD) ||
2277 source == kCharsetFromFinalUserForcedAutoDetection);
2278 mTreeBuilder->NeedsCharsetSwitchTo(encoding, source, 0);
2279 requestedReload = true;
2280 } else if (mCharsetSource ==
2281 kCharsetFromInitialAutoDetectionASCII &&
2282 mDetectorHasSeenNonAscii) {
2283 mCharsetSource = source;
2284 mTreeBuilder->UpdateCharsetSource(mCharsetSource);
2285 }
2286 }
2287
2288 mAtEOF = true;
2289 if (!mForceAutoDetection && !requestedReload) {
2290 if (mCharsetSource == kCharsetFromParentFrame) {
2291 mTreeBuilder->MaybeComplainAboutCharset("EncNoDeclarationFrame",
2292 false, 0);
2293 } else if (mCharsetSource == kCharsetFromXmlDeclaration) {
2294 // We know the bogo XML decl is always on the first line.
2295 mTreeBuilder->MaybeComplainAboutCharset("EncXmlDecl", false, 1);
2296 } else if (
2297 mCharsetSource >=
2298 kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8 &&
2299 mCharsetSource <=
2300 kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD) {
2301 if (mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN) {
2302 mTreeBuilder->MaybeComplainAboutCharset("EncNoDeclPlain",
2303 true, 0);
2304 } else {
2305 mTreeBuilder->MaybeComplainAboutCharset("EncNoDecl", true, 0);
2306 }
2307 }
2308
2309 if (mHasHadErrors && mEncoding != REPLACEMENT_ENCODING) {
2310 if (mEncoding == UTF_8_ENCODING) {
2311 mTreeBuilder->TryToEnableEncodingMenu();
2312 }
2313 if (mCharsetSource == kCharsetFromParentFrame) {
2314 if (mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN) {
2315 mTreeBuilder->MaybeComplainAboutCharset(
2316 "EncErrorFramePlain", true, 0);
2317 } else {
2318 mTreeBuilder->MaybeComplainAboutCharset("EncErrorFrame",
2319 true, 0);
2320 }
2321 } else if (
2322 mCharsetSource >= kCharsetFromXmlDeclaration &&
2323 !(mCharsetSource >=
2324 kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8 &&
2325 mCharsetSource <=
2326 kCharsetFromFinalUserForcedAutoDetection)) {
2327 mTreeBuilder->MaybeComplainAboutCharset("EncError", true, 0);
2328 }
2329 }
2330 }
2331 if (NS_SUCCEEDED(mTreeBuilder->IsBroken())) {
2332 mTokenizer->eof();
2333 nsresult rv;
2334 if (NS_FAILED((rv = mTreeBuilder->IsBroken()))) {
2335 MarkAsBroken(rv);
2336 } else {
2337 mTreeBuilder->StreamEnded();
2338 if (mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML) {
2339 mTokenizer->EndViewSource();
2340 }
2341 }
2342 }
2343 FlushTreeOpsAndDisarmTimer();
2344 return; // no more data and not expecting more
2345 default:
2346 MOZ_ASSERT_UNREACHABLE("It should be impossible to reach this.");
2347 return;
2348 }
2349 }
2350 mFirstBuffer = mFirstBuffer->next;
2351 continue;
2352 }
2353
2354 // now we have a non-empty buffer
2355 mFirstBuffer->adjust(mLastWasCR);
2356 mLastWasCR = false;
2357 if (mFirstBuffer->hasMore()) {
2358 if (!mTokenizer->EnsureBufferSpace(mFirstBuffer->getLength())) {
2359 MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
2360 return;
2361 }
2362 mLastWasCR = mTokenizer->tokenizeBuffer(mFirstBuffer);
2363 nsresult rv;
2364 if (NS_FAILED((rv = mTreeBuilder->IsBroken()))) {
2365 MarkAsBroken(rv);
2366 return;
2367 }
2368 if (mTreeBuilder->HasScript()) {
2369 // HasScript() cannot return true if the tree builder is preventing
2370 // script execution.
2371 MOZ_ASSERT(mMode == NORMAL);
2372 mozilla::MutexAutoLock speculationAutoLock(mSpeculationMutex);
2373 nsHtml5Speculation* speculation = new nsHtml5Speculation(
2374 mFirstBuffer, mFirstBuffer->getStart(), mTokenizer->getLineNumber(),
2375 mTreeBuilder->newSnapshot());
2376 mTreeBuilder->AddSnapshotToScript(speculation->GetSnapshot(),
2377 speculation->GetStartLineNumber());
2378 if (mLookingForMetaCharset) {
2379 if (mMode == VIEW_SOURCE_HTML) {
2380 mTokenizer->FlushViewSource();
2381 }
2382 mTreeBuilder->Flush();
2383 } else {
2384 FlushTreeOpsAndDisarmTimer();
2385 }
2386 mTreeBuilder->SetOpSink(speculation);
2387 mSpeculations.AppendElement(speculation); // adopts the pointer
2388 mSpeculating = true;
2389 }
2390 if (IsTerminatedOrInterrupted()) {
2391 return;
2392 }
2393 }
2394 if (mLookingForMetaCharset) {
2395 Unused << ProcessLookingForMetaCharset(false);
2396 }
2397 }
2398 }
2399
2400 class nsHtml5StreamParserContinuation : public Runnable {
2401 private:
2402 nsHtml5StreamParserPtr mStreamParser;
2403
2404 public:
nsHtml5StreamParserContinuation(nsHtml5StreamParser * aStreamParser)2405 explicit nsHtml5StreamParserContinuation(nsHtml5StreamParser* aStreamParser)
2406 : Runnable("nsHtml5StreamParserContinuation"),
2407 mStreamParser(aStreamParser) {}
Run()2408 NS_IMETHOD Run() override {
2409 mozilla::MutexAutoLock autoLock(mStreamParser->mTokenizerMutex);
2410 mStreamParser->Uninterrupt();
2411 mStreamParser->ParseAvailableData();
2412 return NS_OK;
2413 }
2414 };
2415
ContinueAfterScriptsOrEncodingCommitment(nsHtml5Tokenizer * aTokenizer,nsHtml5TreeBuilder * aTreeBuilder,bool aLastWasCR)2416 void nsHtml5StreamParser::ContinueAfterScriptsOrEncodingCommitment(
2417 nsHtml5Tokenizer* aTokenizer, nsHtml5TreeBuilder* aTreeBuilder,
2418 bool aLastWasCR) {
2419 // nullptr for aTokenizer means encoding commitment as opposed to the "after
2420 // scripts" case.
2421
2422 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
2423 MOZ_ASSERT(mMode != VIEW_SOURCE_XML,
2424 "ContinueAfterScriptsOrEncodingCommitment called in XML view "
2425 "source mode!");
2426 MOZ_ASSERT(!(aTokenizer && mMode == VIEW_SOURCE_HTML),
2427 "ContinueAfterScriptsOrEncodingCommitment called with non-null "
2428 "tokenizer in HTML view "
2429 "source mode.");
2430 if (NS_FAILED(mExecutor->IsBroken())) {
2431 return;
2432 }
2433 MOZ_ASSERT(!(aTokenizer && mMode != NORMAL),
2434 "We should only be executing scripts in the normal mode.");
2435 if (!aTokenizer && (mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN ||
2436 mMode == VIEW_SOURCE_HTML)) {
2437 // Take the ops that were generated from OnStartRequest for the synthetic
2438 // head section of the document for plain text and HTML View Source.
2439 // XML View Source never needs this kind of encoding commitment.
2440 // We need to take the ops here so that they end up in the queue before
2441 // the ops that we take from a speculation later in this method.
2442 mExecutor->TakeOpsFromStage();
2443 } else {
2444 #ifdef DEBUG
2445 mExecutor->AssertStageEmpty();
2446 #endif
2447 }
2448 bool speculationFailed = false;
2449 {
2450 mozilla::MutexAutoLock speculationAutoLock(mSpeculationMutex);
2451 if (mSpeculations.IsEmpty()) {
2452 MOZ_ASSERT_UNREACHABLE(
2453 "ContinueAfterScriptsOrEncodingCommitment called without "
2454 "speculations.");
2455 return;
2456 }
2457
2458 const auto& speculation = mSpeculations.ElementAt(0);
2459 if (aTokenizer &&
2460 (aLastWasCR || !aTokenizer->isInDataState() ||
2461 !aTreeBuilder->snapshotMatches(speculation->GetSnapshot()))) {
2462 speculationFailed = true;
2463 // We've got a failed speculation :-(
2464 MaybeDisableFutureSpeculation();
2465 Interrupt(); // Make the parser thread release the tokenizer mutex sooner
2466 // Note that the interrupted state continues across possible intervening
2467 // Necko events until the nsHtml5StreamParserContinuation posted at the
2468 // end of this method runs. Therefore, this thread is guaranteed to
2469 // acquire mTokenizerMutex soon even if an intervening Necko event grabbed
2470 // it between now and the acquisition below.
2471
2472 // now fall out of the speculationAutoLock into the tokenizerAutoLock
2473 // block
2474 } else {
2475 // We've got a successful speculation!
2476 if (mSpeculations.Length() > 1) {
2477 // the first speculation isn't the current speculation, so there's
2478 // no need to bother the parser thread.
2479 speculation->FlushToSink(mExecutor);
2480 MOZ_ASSERT(!mExecutor->IsScriptExecuting(),
2481 "ParseUntilBlocked() was supposed to ensure we don't come "
2482 "here when scripts are executing.");
2483 MOZ_ASSERT(!aTokenizer || mExecutor->IsInFlushLoop(),
2484 "How are we here if "
2485 "RunFlushLoop() didn't call ParseUntilBlocked() or we're "
2486 "not committing to an encoding?");
2487 mSpeculations.RemoveElementAt(0);
2488 return;
2489 }
2490 // else
2491 Interrupt(); // Make the parser thread release the tokenizer mutex sooner
2492 // Note that the interrupted state continues across possible intervening
2493 // Necko events until the nsHtml5StreamParserContinuation posted at the
2494 // end of this method runs. Therefore, this thread is guaranteed to
2495 // acquire mTokenizerMutex soon even if an intervening Necko event grabbed
2496 // it between now and the acquisition below.
2497
2498 // now fall through
2499 // the first speculation is the current speculation. Need to
2500 // release the the speculation mutex and acquire the tokenizer
2501 // mutex. (Just acquiring the other mutex here would deadlock)
2502 }
2503 }
2504 {
2505 mozilla::MutexAutoLock tokenizerAutoLock(mTokenizerMutex);
2506 #ifdef DEBUG
2507 {
2508 mAtomTable.SetPermittedLookupEventTarget(
2509 GetMainThreadSerialEventTarget());
2510 }
2511 #endif
2512 // In principle, the speculation mutex should be acquired here,
2513 // but there's no point, because the parser thread only acquires it
2514 // when it has also acquired the tokenizer mutex and we are already
2515 // holding the tokenizer mutex.
2516 if (speculationFailed) {
2517 MOZ_ASSERT(mMode == NORMAL);
2518 // Rewind the stream
2519 mAtEOF = false;
2520 const auto& speculation = mSpeculations.ElementAt(0);
2521 mFirstBuffer = speculation->GetBuffer();
2522 mFirstBuffer->setStart(speculation->GetStart());
2523 mTokenizer->setLineNumber(speculation->GetStartLineNumber());
2524
2525 nsContentUtils::ReportToConsole(
2526 nsIScriptError::warningFlag, "DOM Events"_ns,
2527 mExecutor->GetDocument(), nsContentUtils::eDOM_PROPERTIES,
2528 "SpeculationFailed2", nsTArray<nsString>(), nullptr, u""_ns,
2529 speculation->GetStartLineNumber());
2530
2531 nsHtml5OwningUTF16Buffer* buffer = mFirstBuffer->next;
2532 while (buffer) {
2533 buffer->setStart(0);
2534 buffer = buffer->next;
2535 }
2536
2537 mSpeculations.Clear(); // potentially a huge number of destructors
2538 // run here synchronously on the main thread...
2539
2540 mTreeBuilder->flushCharacters(); // empty the pending buffer
2541 mTreeBuilder->ClearOps(); // now get rid of the failed ops
2542
2543 mTreeBuilder->SetOpSink(mExecutor->GetStage());
2544 mExecutor->StartReadingFromStage();
2545 mSpeculating = false;
2546
2547 // Copy state over
2548 mLastWasCR = aLastWasCR;
2549 mTokenizer->loadState(aTokenizer);
2550 mTreeBuilder->loadState(aTreeBuilder);
2551 } else {
2552 // We've got a successful speculation and at least a moment ago it was
2553 // the current speculation
2554 mSpeculations.ElementAt(0)->FlushToSink(mExecutor);
2555 MOZ_ASSERT(!mExecutor->IsScriptExecuting(),
2556 "ParseUntilBlocked() was supposed to ensure we don't come "
2557 "here when scripts are executing.");
2558 MOZ_ASSERT(!aTokenizer || mExecutor->IsInFlushLoop(),
2559 "How are we here if "
2560 "RunFlushLoop() didn't call ParseUntilBlocked() or we're not "
2561 "committing to an encoding?");
2562 mSpeculations.RemoveElementAt(0);
2563 if (mSpeculations.IsEmpty()) {
2564 if (mMode == VIEW_SOURCE_HTML) {
2565 // If we looked for meta charset in the HTML View Source case.
2566 mTokenizer->SetViewSourceOpSink(mExecutor->GetStage());
2567 } else {
2568 // yes, it was still the only speculation. Now stop speculating
2569 // However, before telling the executor to read from stage, flush
2570 // any pending ops straight to the executor, because otherwise
2571 // they remain unflushed until we get more data from the network.
2572 mTreeBuilder->SetOpSink(mExecutor);
2573 mTreeBuilder->Flush(true);
2574 mTreeBuilder->SetOpSink(mExecutor->GetStage());
2575 }
2576 mExecutor->StartReadingFromStage();
2577 mSpeculating = false;
2578 }
2579 }
2580 nsCOMPtr<nsIRunnable> event = new nsHtml5StreamParserContinuation(this);
2581 if (NS_FAILED(mEventTarget->Dispatch(event, nsIThread::DISPATCH_NORMAL))) {
2582 NS_WARNING("Failed to dispatch nsHtml5StreamParserContinuation");
2583 }
2584 // A stream event might run before this event runs, but that's harmless.
2585 #ifdef DEBUG
2586 mAtomTable.SetPermittedLookupEventTarget(mEventTarget);
2587 #endif
2588 }
2589 }
2590
ContinueAfterFailedCharsetSwitch()2591 void nsHtml5StreamParser::ContinueAfterFailedCharsetSwitch() {
2592 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
2593 nsCOMPtr<nsIRunnable> event = new nsHtml5StreamParserContinuation(this);
2594 if (NS_FAILED(mEventTarget->Dispatch(event, nsIThread::DISPATCH_NORMAL))) {
2595 NS_WARNING("Failed to dispatch nsHtml5StreamParserContinuation");
2596 }
2597 }
2598
2599 class nsHtml5TimerKungFu : public Runnable {
2600 private:
2601 nsHtml5StreamParserPtr mStreamParser;
2602
2603 public:
nsHtml5TimerKungFu(nsHtml5StreamParser * aStreamParser)2604 explicit nsHtml5TimerKungFu(nsHtml5StreamParser* aStreamParser)
2605 : Runnable("nsHtml5TimerKungFu"), mStreamParser(aStreamParser) {}
Run()2606 NS_IMETHOD Run() override {
2607 mozilla::MutexAutoLock flushTimerLock(mStreamParser->mFlushTimerMutex);
2608 if (mStreamParser->mFlushTimer) {
2609 mStreamParser->mFlushTimer->Cancel();
2610 mStreamParser->mFlushTimer = nullptr;
2611 }
2612 return NS_OK;
2613 }
2614 };
2615
DropTimer()2616 void nsHtml5StreamParser::DropTimer() {
2617 MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
2618 /*
2619 * Simply nulling out the timer wouldn't work, because if the timer is
2620 * armed, it needs to be canceled first. Simply canceling it first wouldn't
2621 * work, because nsTimerImpl::Cancel is not safe for calling from outside
2622 * the thread where nsTimerImpl::Fire would run. It's not safe to
2623 * dispatch a runnable to cancel the timer from the destructor of this
2624 * class, because the timer has a weak (void*) pointer back to this instance
2625 * of the stream parser and having the timer fire before the runnable
2626 * cancels it would make the timer access a deleted object.
2627 *
2628 * This DropTimer method addresses these issues. This method must be called
2629 * on the main thread before the destructor of this class is reached.
2630 * The nsHtml5TimerKungFu object has an nsHtml5StreamParserPtr that addrefs
2631 * this
2632 * stream parser object to keep it alive until the runnable is done.
2633 * The runnable cancels the timer on the parser thread, drops the timer
2634 * and lets nsHtml5StreamParserPtr send a runnable back to the main thread to
2635 * release the stream parser.
2636 */
2637 mozilla::MutexAutoLock flushTimerLock(mFlushTimerMutex);
2638 if (mFlushTimer) {
2639 nsCOMPtr<nsIRunnable> event = new nsHtml5TimerKungFu(this);
2640 if (NS_FAILED(mEventTarget->Dispatch(event, nsIThread::DISPATCH_NORMAL))) {
2641 NS_WARNING("Failed to dispatch TimerKungFu event");
2642 }
2643 }
2644 }
2645
2646 // Using a static, because the method name Notify is taken by the chardet
2647 // callback.
TimerCallback(nsITimer * aTimer,void * aClosure)2648 void nsHtml5StreamParser::TimerCallback(nsITimer* aTimer, void* aClosure) {
2649 (static_cast<nsHtml5StreamParser*>(aClosure))->TimerFlush();
2650 }
2651
TimerFlush()2652 void nsHtml5StreamParser::TimerFlush() {
2653 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
2654 mozilla::MutexAutoLock autoLock(mTokenizerMutex);
2655
2656 MOZ_ASSERT(!mSpeculating, "Flush timer fired while speculating.");
2657
2658 // The timer fired if we got here. No need to cancel it. Mark it as
2659 // not armed, though.
2660 mFlushTimerArmed = false;
2661
2662 mFlushTimerEverFired = true;
2663
2664 if (IsTerminatedOrInterrupted()) {
2665 return;
2666 }
2667
2668 if (mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML) {
2669 mTreeBuilder->Flush(); // delete useless ops
2670 if (mTokenizer->FlushViewSource()) {
2671 nsCOMPtr<nsIRunnable> runnable(mExecutorFlusher);
2672 if (NS_FAILED(DispatchToMain(runnable.forget()))) {
2673 NS_WARNING("failed to dispatch executor flush event");
2674 }
2675 }
2676 } else {
2677 // we aren't speculating and we don't know when new data is
2678 // going to arrive. Send data to the main thread.
2679 if (mTreeBuilder->Flush(true)) {
2680 nsCOMPtr<nsIRunnable> runnable(mExecutorFlusher);
2681 if (NS_FAILED(DispatchToMain(runnable.forget()))) {
2682 NS_WARNING("failed to dispatch executor flush event");
2683 }
2684 }
2685 }
2686 }
2687
MarkAsBroken(nsresult aRv)2688 void nsHtml5StreamParser::MarkAsBroken(nsresult aRv) {
2689 MOZ_ASSERT(IsParserThread(), "Wrong thread!");
2690 mTokenizerMutex.AssertCurrentThreadOwns();
2691
2692 Terminate();
2693 mTreeBuilder->MarkAsBroken(aRv);
2694 mozilla::DebugOnly<bool> hadOps = mTreeBuilder->Flush(false);
2695 MOZ_ASSERT(hadOps, "Should have had the markAsBroken op!");
2696 nsCOMPtr<nsIRunnable> runnable(mExecutorFlusher);
2697 if (NS_FAILED(DispatchToMain(runnable.forget()))) {
2698 NS_WARNING("failed to dispatch executor flush event");
2699 }
2700 }
2701
DispatchToMain(already_AddRefed<nsIRunnable> && aRunnable)2702 nsresult nsHtml5StreamParser::DispatchToMain(
2703 already_AddRefed<nsIRunnable>&& aRunnable) {
2704 if (mNetworkEventTarget) {
2705 return mNetworkEventTarget->Dispatch(std::move(aRunnable));
2706 }
2707 return SchedulerGroup::UnlabeledDispatch(TaskCategory::Network,
2708 std::move(aRunnable));
2709 }
2710