1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6 #include "AnimationSurfaceProvider.h"
7
8 #include "mozilla/StaticPrefs_image.h"
9 #include "mozilla/gfx/gfxVars.h"
10 #include "mozilla/layers/SharedSurfacesChild.h"
11 #include "mozilla/layers/SourceSurfaceSharedData.h"
12 #include "nsProxyRelease.h"
13
14 #include "DecodePool.h"
15 #include "Decoder.h"
16
17 using namespace mozilla::gfx;
18 using namespace mozilla::layers;
19
20 namespace mozilla {
21 namespace image {
22
AnimationSurfaceProvider(NotNull<RasterImage * > aImage,const SurfaceKey & aSurfaceKey,NotNull<Decoder * > aDecoder,size_t aCurrentFrame)23 AnimationSurfaceProvider::AnimationSurfaceProvider(
24 NotNull<RasterImage*> aImage, const SurfaceKey& aSurfaceKey,
25 NotNull<Decoder*> aDecoder, size_t aCurrentFrame)
26 : ISurfaceProvider(ImageKey(aImage.get()), aSurfaceKey,
27 AvailabilityState::StartAsPlaceholder()),
28 mImage(aImage.get()),
29 mDecodingMutex("AnimationSurfaceProvider::mDecoder"),
30 mDecoder(aDecoder.get()),
31 mFramesMutex("AnimationSurfaceProvider::mFrames"),
32 mCompositedFrameRequested(false),
33 mSharedAnimation(MakeRefPtr<SharedSurfacesAnimation>()) {
34 MOZ_ASSERT(!mDecoder->IsMetadataDecode(),
35 "Use MetadataDecodingTask for metadata decodes");
36 MOZ_ASSERT(!mDecoder->IsFirstFrameDecode(),
37 "Use DecodedSurfaceProvider for single-frame image decodes");
38
39 // Calculate how many frames we need to decode in this animation before we
40 // enter decode-on-demand mode.
41 IntSize frameSize = aSurfaceKey.Size();
42 size_t threshold =
43 (size_t(StaticPrefs::image_animated_decode_on_demand_threshold_kb()) *
44 1024) /
45 (sizeof(uint32_t) * frameSize.width * frameSize.height);
46 size_t batch = StaticPrefs::image_animated_decode_on_demand_batch_size();
47
48 mFrames.reset(
49 new AnimationFrameRetainedBuffer(threshold, batch, aCurrentFrame));
50 }
51
~AnimationSurfaceProvider()52 AnimationSurfaceProvider::~AnimationSurfaceProvider() {
53 DropImageReference();
54
55 mSharedAnimation->Destroy();
56 if (mDecoder) {
57 mDecoder->SetFrameRecycler(nullptr);
58 }
59 }
60
DropImageReference()61 void AnimationSurfaceProvider::DropImageReference() {
62 if (!mImage) {
63 return; // Nothing to do.
64 }
65
66 // RasterImage objects need to be destroyed on the main thread.
67 SurfaceCache::ReleaseImageOnMainThread(mImage.forget());
68 }
69
Reset()70 void AnimationSurfaceProvider::Reset() {
71 // We want to go back to the beginning.
72 bool mayDiscard;
73 bool restartDecoder = false;
74
75 {
76 MutexAutoLock lock(mFramesMutex);
77
78 // If we have not crossed the threshold, we know we haven't discarded any
79 // frames, and thus we know it is safe move our display index back to the
80 // very beginning. It would be cleaner to let the frame buffer make this
81 // decision inside the AnimationFrameBuffer::Reset method, but if we have
82 // crossed the threshold, we need to hold onto the decoding mutex too. We
83 // should avoid blocking the main thread on the decoder threads.
84 mayDiscard = mFrames->MayDiscard();
85 if (!mayDiscard) {
86 restartDecoder = mFrames->Reset();
87 }
88 }
89
90 if (mayDiscard) {
91 // We are over the threshold and have started discarding old frames. In
92 // that case we need to seize the decoding mutex. Thankfully we know that
93 // we are in the process of decoding at most the batch size frames, so
94 // this should not take too long to acquire.
95 MutexAutoLock lock(mDecodingMutex);
96
97 // We may have hit an error while redecoding. Because FrameAnimator is
98 // tightly coupled to our own state, that means we would need to go through
99 // some heroics to resume animating in those cases. The typical reason for
100 // a redecode to fail is out of memory, and recycling should prevent most of
101 // those errors. When image.animated.generate-full-frames has shipped
102 // enabled on a release or two, we can simply remove the old FrameAnimator
103 // blending code and simplify this quite a bit -- just always pop the next
104 // full frame and timeout off the stack.
105 if (mDecoder) {
106 mDecoder = DecoderFactory::CloneAnimationDecoder(mDecoder);
107 MOZ_ASSERT(mDecoder);
108
109 MutexAutoLock lock2(mFramesMutex);
110 restartDecoder = mFrames->Reset();
111 } else {
112 MOZ_ASSERT(mFrames->HasRedecodeError());
113 }
114 }
115
116 if (restartDecoder) {
117 DecodePool::Singleton()->AsyncRun(this);
118 }
119 }
120
Advance(size_t aFrame)121 void AnimationSurfaceProvider::Advance(size_t aFrame) {
122 bool restartDecoder;
123
124 RefPtr<SourceSurface> surface;
125 IntRect dirtyRect;
126 {
127 // Typical advancement of a frame.
128 MutexAutoLock lock(mFramesMutex);
129 restartDecoder = mFrames->AdvanceTo(aFrame);
130
131 imgFrame* frame = mFrames->Get(aFrame, /* aForDisplay */ true);
132 MOZ_ASSERT(frame);
133 if (aFrame != 0) {
134 dirtyRect = frame->GetDirtyRect();
135 } else {
136 MOZ_ASSERT(mFrames->SizeKnown());
137 dirtyRect = mFrames->FirstFrameRefreshArea();
138 }
139 surface = frame->GetSourceSurface();
140 MOZ_ASSERT(surface);
141 }
142
143 if (restartDecoder) {
144 DecodePool::Singleton()->AsyncRun(this);
145 }
146
147 mCompositedFrameRequested = false;
148 auto* sharedSurface = static_cast<SourceSurfaceSharedData*>(surface.get());
149 mSharedAnimation->SetCurrentFrame(sharedSurface, dirtyRect);
150 }
151
DrawableRef(size_t aFrame)152 DrawableFrameRef AnimationSurfaceProvider::DrawableRef(size_t aFrame) {
153 MutexAutoLock lock(mFramesMutex);
154
155 if (Availability().IsPlaceholder()) {
156 MOZ_ASSERT_UNREACHABLE("Calling DrawableRef() on a placeholder");
157 return DrawableFrameRef();
158 }
159
160 imgFrame* frame = mFrames->Get(aFrame, /* aForDisplay */ true);
161 if (!frame) {
162 return DrawableFrameRef();
163 }
164
165 return frame->DrawableRef();
166 }
167
GetFrame(size_t aFrame)168 already_AddRefed<imgFrame> AnimationSurfaceProvider::GetFrame(size_t aFrame) {
169 MutexAutoLock lock(mFramesMutex);
170
171 if (Availability().IsPlaceholder()) {
172 MOZ_ASSERT_UNREACHABLE("Calling GetFrame() on a placeholder");
173 return nullptr;
174 }
175
176 RefPtr<imgFrame> frame = mFrames->Get(aFrame, /* aForDisplay */ false);
177 MOZ_ASSERT_IF(frame, frame->IsFinished());
178 return frame.forget();
179 }
180
IsFinished() const181 bool AnimationSurfaceProvider::IsFinished() const {
182 MutexAutoLock lock(mFramesMutex);
183
184 if (Availability().IsPlaceholder()) {
185 MOZ_ASSERT_UNREACHABLE("Calling IsFinished() on a placeholder");
186 return false;
187 }
188
189 return mFrames->IsFirstFrameFinished();
190 }
191
IsFullyDecoded() const192 bool AnimationSurfaceProvider::IsFullyDecoded() const {
193 MutexAutoLock lock(mFramesMutex);
194 return mFrames->SizeKnown() && !mFrames->MayDiscard();
195 }
196
LogicalSizeInBytes() const197 size_t AnimationSurfaceProvider::LogicalSizeInBytes() const {
198 // When decoding animated images, we need at most three live surfaces: the
199 // composited surface, the previous composited surface for
200 // DisposalMethod::RESTORE_PREVIOUS, and the surface we're currently decoding
201 // into. The composited surfaces are always BGRA. Although the surface we're
202 // decoding into may be paletted, and may be smaller than the real size of the
203 // image, we assume the worst case here.
204 // XXX(seth): Note that this is actually not accurate yet; we're storing the
205 // full sequence of frames, not just the three live surfaces mentioned above.
206 // Unfortunately there's no way to know in advance how many frames an
207 // animation has, so we really can't do better here. This will become correct
208 // once bug 1289954 is complete.
209 IntSize size = GetSurfaceKey().Size();
210 return 3 * size.width * size.height * sizeof(uint32_t);
211 }
212
AddSizeOfExcludingThis(MallocSizeOf aMallocSizeOf,const AddSizeOfCb & aCallback)213 void AnimationSurfaceProvider::AddSizeOfExcludingThis(
214 MallocSizeOf aMallocSizeOf, const AddSizeOfCb& aCallback) {
215 // Note that the surface cache lock is already held here, and then we acquire
216 // mFramesMutex. For this method, this ordering is unavoidable, which means
217 // that we must be careful to always use the same ordering elsewhere.
218 MutexAutoLock lock(mFramesMutex);
219 mFrames->AddSizeOfExcludingThis(aMallocSizeOf, aCallback);
220 }
221
Run()222 void AnimationSurfaceProvider::Run() {
223 MutexAutoLock lock(mDecodingMutex);
224
225 if (!mDecoder) {
226 MOZ_ASSERT_UNREACHABLE("Running after decoding finished?");
227 return;
228 }
229
230 while (true) {
231 // Run the decoder.
232 LexerResult result = mDecoder->Decode(WrapNotNull(this));
233
234 if (result.is<TerminalState>()) {
235 // We may have a new frame now, but it's not guaranteed - a decoding
236 // failure or truncated data may mean that no new frame got produced.
237 // Since we're not sure, rather than call CheckForNewFrameAtYield() here
238 // we call CheckForNewFrameAtTerminalState(), which handles both of these
239 // possibilities.
240 bool continueDecoding = CheckForNewFrameAtTerminalState();
241 FinishDecoding();
242
243 // Even if it is the last frame, we may not have enough frames buffered
244 // ahead of the current. If we are shutting down, we want to ensure we
245 // release the thread as soon as possible. The animation may advance even
246 // during shutdown, which keeps us decoding, and thus blocking the decode
247 // pool during teardown.
248 if (!mDecoder || !continueDecoding ||
249 DecodePool::Singleton()->IsShuttingDown()) {
250 return;
251 }
252
253 // Restart from the very beginning because the decoder was recreated.
254 continue;
255 }
256
257 // If there is output available we want to change the entry in the surface
258 // cache from a placeholder to an actual surface now before NotifyProgress
259 // call below so that when consumers get the frame complete notification
260 // from the NotifyProgress they can actually get a surface from the surface
261 // cache.
262 bool checkForNewFrameAtYieldResult = false;
263 if (result == LexerResult(Yield::OUTPUT_AVAILABLE)) {
264 checkForNewFrameAtYieldResult = CheckForNewFrameAtYield();
265 }
266
267 // Notify for the progress we've made so far.
268 if (mImage && mDecoder->HasProgress()) {
269 NotifyProgress(WrapNotNull(mImage), WrapNotNull(mDecoder));
270 }
271
272 if (result == LexerResult(Yield::NEED_MORE_DATA)) {
273 // We can't make any more progress right now. The decoder itself will
274 // ensure that we get reenqueued when more data is available; just return
275 // for now.
276 return;
277 }
278
279 // There's new output available - a new frame! Grab it. If we don't need any
280 // more for the moment we can break out of the loop. If we are shutting
281 // down, we want to ensure we release the thread as soon as possible. The
282 // animation may advance even during shutdown, which keeps us decoding, and
283 // thus blocking the decode pool during teardown.
284 MOZ_ASSERT(result == LexerResult(Yield::OUTPUT_AVAILABLE));
285 if (!checkForNewFrameAtYieldResult ||
286 DecodePool::Singleton()->IsShuttingDown()) {
287 return;
288 }
289 }
290 }
291
CheckForNewFrameAtYield()292 bool AnimationSurfaceProvider::CheckForNewFrameAtYield() {
293 mDecodingMutex.AssertCurrentThreadOwns();
294 MOZ_ASSERT(mDecoder);
295
296 bool justGotFirstFrame = false;
297 bool continueDecoding = false;
298
299 {
300 MutexAutoLock lock(mFramesMutex);
301
302 // Try to get the new frame from the decoder.
303 RefPtr<imgFrame> frame = mDecoder->GetCurrentFrame();
304 MOZ_ASSERT(mDecoder->HasFrameToTake());
305 mDecoder->ClearHasFrameToTake();
306
307 if (!frame) {
308 MOZ_ASSERT_UNREACHABLE("Decoder yielded but didn't produce a frame?");
309 return true;
310 }
311
312 // We should've gotten a different frame than last time.
313 MOZ_ASSERT(!mFrames->IsLastInsertedFrame(frame));
314
315 // Append the new frame to the list.
316 AnimationFrameBuffer::InsertStatus status =
317 mFrames->Insert(std::move(frame));
318
319 // If we hit a redecode error, then we actually want to stop. This happens
320 // when we tried to insert more frames than we originally had (e.g. the
321 // original decoder attempt hit an OOM error sooner than we did). Better to
322 // stop the animation than to get out of sync with FrameAnimator.
323 if (mFrames->HasRedecodeError()) {
324 mDecoder = nullptr;
325 return false;
326 }
327
328 switch (status) {
329 case AnimationFrameBuffer::InsertStatus::DISCARD_CONTINUE:
330 continueDecoding = true;
331 [[fallthrough]];
332 case AnimationFrameBuffer::InsertStatus::DISCARD_YIELD:
333 RequestFrameDiscarding();
334 break;
335 case AnimationFrameBuffer::InsertStatus::CONTINUE:
336 continueDecoding = true;
337 break;
338 case AnimationFrameBuffer::InsertStatus::YIELD:
339 break;
340 default:
341 MOZ_ASSERT_UNREACHABLE("Unhandled insert status!");
342 break;
343 }
344
345 // We only want to handle the first frame if it is the first pass for the
346 // animation decoder. The owning image will be cleared after that.
347 size_t frameCount = mFrames->Size();
348 if (frameCount == 1 && mImage) {
349 justGotFirstFrame = true;
350 }
351 }
352
353 if (justGotFirstFrame) {
354 AnnounceSurfaceAvailable();
355 }
356
357 return continueDecoding;
358 }
359
CheckForNewFrameAtTerminalState()360 bool AnimationSurfaceProvider::CheckForNewFrameAtTerminalState() {
361 mDecodingMutex.AssertCurrentThreadOwns();
362 MOZ_ASSERT(mDecoder);
363
364 bool justGotFirstFrame = false;
365 bool continueDecoding;
366
367 {
368 MutexAutoLock lock(mFramesMutex);
369
370 // The decoder may or may not have a new frame for us at this point. Avoid
371 // reinserting the same frame again.
372 RefPtr<imgFrame> frame = mDecoder->GetCurrentFrame();
373
374 // If the decoder didn't finish a new frame (ie if, after starting the
375 // frame, it got an error and aborted the frame and the rest of the decode)
376 // that means it won't be reporting it to the image or FrameAnimator so we
377 // should ignore it too, that's what HasFrameToTake tracks basically.
378 if (!mDecoder->HasFrameToTake()) {
379 frame = nullptr;
380 } else {
381 MOZ_ASSERT(frame);
382 mDecoder->ClearHasFrameToTake();
383 }
384
385 if (!frame || mFrames->IsLastInsertedFrame(frame)) {
386 return mFrames->MarkComplete(mDecoder->GetFirstFrameRefreshArea());
387 }
388
389 // Append the new frame to the list.
390 AnimationFrameBuffer::InsertStatus status =
391 mFrames->Insert(std::move(frame));
392
393 // If we hit a redecode error, then we actually want to stop. This will be
394 // fully handled in FinishDecoding.
395 if (mFrames->HasRedecodeError()) {
396 return false;
397 }
398
399 switch (status) {
400 case AnimationFrameBuffer::InsertStatus::DISCARD_CONTINUE:
401 case AnimationFrameBuffer::InsertStatus::DISCARD_YIELD:
402 RequestFrameDiscarding();
403 break;
404 case AnimationFrameBuffer::InsertStatus::CONTINUE:
405 case AnimationFrameBuffer::InsertStatus::YIELD:
406 break;
407 default:
408 MOZ_ASSERT_UNREACHABLE("Unhandled insert status!");
409 break;
410 }
411
412 continueDecoding =
413 mFrames->MarkComplete(mDecoder->GetFirstFrameRefreshArea());
414
415 // We only want to handle the first frame if it is the first pass for the
416 // animation decoder. The owning image will be cleared after that.
417 if (mFrames->Size() == 1 && mImage) {
418 justGotFirstFrame = true;
419 }
420 }
421
422 if (justGotFirstFrame) {
423 AnnounceSurfaceAvailable();
424 }
425
426 return continueDecoding;
427 }
428
RequestFrameDiscarding()429 void AnimationSurfaceProvider::RequestFrameDiscarding() {
430 mDecodingMutex.AssertCurrentThreadOwns();
431 mFramesMutex.AssertCurrentThreadOwns();
432 MOZ_ASSERT(mDecoder);
433
434 if (mFrames->MayDiscard() || mFrames->IsRecycling()) {
435 MOZ_ASSERT_UNREACHABLE("Already replaced frame queue!");
436 return;
437 }
438
439 auto oldFrameQueue =
440 static_cast<AnimationFrameRetainedBuffer*>(mFrames.get());
441
442 MOZ_ASSERT(!mDecoder->GetFrameRecycler());
443 if (StaticPrefs::image_animated_decode_on_demand_recycle_AtStartup()) {
444 mFrames.reset(new AnimationFrameRecyclingQueue(std::move(*oldFrameQueue)));
445 mDecoder->SetFrameRecycler(this);
446 } else {
447 mFrames.reset(new AnimationFrameDiscardingQueue(std::move(*oldFrameQueue)));
448 }
449 }
450
AnnounceSurfaceAvailable()451 void AnimationSurfaceProvider::AnnounceSurfaceAvailable() {
452 mFramesMutex.AssertNotCurrentThreadOwns();
453 MOZ_ASSERT(mImage);
454
455 // We just got the first frame; let the surface cache know. We deliberately do
456 // this outside of mFramesMutex to avoid a potential deadlock with
457 // AddSizeOfExcludingThis(), since otherwise we'd be acquiring mFramesMutex
458 // and then the surface cache lock, while the memory reporting code would
459 // acquire the surface cache lock and then mFramesMutex.
460 SurfaceCache::SurfaceAvailable(WrapNotNull(this));
461 }
462
FinishDecoding()463 void AnimationSurfaceProvider::FinishDecoding() {
464 mDecodingMutex.AssertCurrentThreadOwns();
465 MOZ_ASSERT(mDecoder);
466
467 if (mImage) {
468 // Send notifications.
469 NotifyDecodeComplete(WrapNotNull(mImage), WrapNotNull(mDecoder));
470 }
471
472 // Determine if we need to recreate the decoder, in case we are discarding
473 // frames and need to loop back to the beginning.
474 bool recreateDecoder;
475 {
476 MutexAutoLock lock(mFramesMutex);
477 recreateDecoder = !mFrames->HasRedecodeError() && mFrames->MayDiscard();
478 }
479
480 if (recreateDecoder) {
481 mDecoder = DecoderFactory::CloneAnimationDecoder(mDecoder);
482 MOZ_ASSERT(mDecoder);
483 } else {
484 mDecoder = nullptr;
485 }
486
487 // We don't need a reference to our image anymore, either, and we don't want
488 // one. We may be stored in the surface cache for a long time after decoding
489 // finishes. If we don't drop our reference to the image, we'll end up
490 // keeping it alive as long as we remain in the surface cache, which could
491 // greatly extend the image's lifetime - in fact, if the image isn't
492 // discardable, it'd result in a leak!
493 DropImageReference();
494 }
495
ShouldPreferSyncRun() const496 bool AnimationSurfaceProvider::ShouldPreferSyncRun() const {
497 MutexAutoLock lock(mDecodingMutex);
498 MOZ_ASSERT(mDecoder);
499
500 return mDecoder->ShouldSyncDecode(
501 StaticPrefs::image_mem_decode_bytes_at_a_time_AtStartup());
502 }
503
RecycleFrame(gfx::IntRect & aRecycleRect)504 RawAccessFrameRef AnimationSurfaceProvider::RecycleFrame(
505 gfx::IntRect& aRecycleRect) {
506 MutexAutoLock lock(mFramesMutex);
507 MOZ_ASSERT(mFrames->IsRecycling());
508 return mFrames->RecycleFrame(aRecycleRect);
509 }
510
UpdateKey(layers::RenderRootStateManager * aManager,wr::IpcResourceUpdateQueue & aResources,wr::ImageKey & aKey)511 nsresult AnimationSurfaceProvider::UpdateKey(
512 layers::RenderRootStateManager* aManager,
513 wr::IpcResourceUpdateQueue& aResources, wr::ImageKey& aKey) {
514 MOZ_ASSERT(NS_IsMainThread());
515
516 RefPtr<SourceSurface> surface;
517 {
518 MutexAutoLock lock(mFramesMutex);
519 imgFrame* frame =
520 mFrames->Get(mFrames->Displayed(), /* aForDisplay */ true);
521 if (!frame) {
522 return NS_ERROR_NOT_AVAILABLE;
523 }
524
525 surface = frame->GetSourceSurface();
526 }
527
528 mCompositedFrameRequested = true;
529 auto* sharedSurface = static_cast<SourceSurfaceSharedData*>(surface.get());
530 return mSharedAnimation->UpdateKey(sharedSurface, aManager, aResources, aKey);
531 }
532
533 } // namespace image
534 } // namespace mozilla
535