1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6 #include "Image.h"
7
8 #include "imgRequest.h"
9 #include "Layers.h" // for LayerManager
10 #include "nsIObserverService.h"
11 #include "nsRefreshDriver.h"
12 #include "nsContentUtils.h"
13 #include "mozilla/gfx/Point.h"
14 #include "mozilla/gfx/Rect.h"
15 #include "mozilla/gfx/SourceSurfaceRawData.h"
16 #include "mozilla/Services.h"
17 #include "mozilla/SizeOfState.h"
18 #include "mozilla/TimeStamp.h"
19 #include "mozilla/Tuple.h" // for Tie
20 #include "mozilla/layers/SharedSurfacesChild.h"
21 #include "SourceSurfaceBlobImage.h"
22
23 namespace mozilla {
24 namespace image {
25
26 ///////////////////////////////////////////////////////////////////////////////
27 // Memory Reporting
28 ///////////////////////////////////////////////////////////////////////////////
29
ImageMemoryCounter(imgRequest * aRequest,SizeOfState & aState,bool aIsUsed)30 ImageMemoryCounter::ImageMemoryCounter(imgRequest* aRequest,
31 SizeOfState& aState, bool aIsUsed)
32 : mProgress(UINT32_MAX),
33 mType(UINT16_MAX),
34 mIsUsed(aIsUsed),
35 mHasError(false),
36 mValidating(false) {
37 MOZ_ASSERT(aRequest);
38
39 // We don't have the image object yet, but we can get some information.
40 nsCOMPtr<nsIURI> imageURL;
41 nsresult rv = aRequest->GetURI(getter_AddRefs(imageURL));
42 if (NS_SUCCEEDED(rv) && imageURL) {
43 imageURL->GetSpec(mURI);
44 }
45
46 mType = imgIContainer::TYPE_REQUEST;
47 mHasError = NS_FAILED(aRequest->GetImageErrorCode());
48 mValidating = !!aRequest->GetValidator();
49
50 RefPtr<ProgressTracker> tracker = aRequest->GetProgressTracker();
51 if (tracker) {
52 mProgress = tracker->GetProgress();
53 }
54 }
55
ImageMemoryCounter(imgRequest * aRequest,Image * aImage,SizeOfState & aState,bool aIsUsed)56 ImageMemoryCounter::ImageMemoryCounter(imgRequest* aRequest, Image* aImage,
57 SizeOfState& aState, bool aIsUsed)
58 : mProgress(UINT32_MAX),
59 mType(UINT16_MAX),
60 mIsUsed(aIsUsed),
61 mHasError(false),
62 mValidating(false) {
63 MOZ_ASSERT(aRequest);
64 MOZ_ASSERT(aImage);
65
66 // Extract metadata about the image.
67 nsCOMPtr<nsIURI> imageURL(aImage->GetURI());
68 if (imageURL) {
69 imageURL->GetSpec(mURI);
70 }
71
72 int32_t width = 0;
73 int32_t height = 0;
74 aImage->GetWidth(&width);
75 aImage->GetHeight(&height);
76 mIntrinsicSize.SizeTo(width, height);
77
78 mType = aImage->GetType();
79 mHasError = aImage->HasError();
80 mValidating = !!aRequest->GetValidator();
81
82 RefPtr<ProgressTracker> tracker = aImage->GetProgressTracker();
83 if (tracker) {
84 mProgress = tracker->GetProgress();
85 }
86
87 // Populate memory counters for source and decoded data.
88 mValues.SetSource(aImage->SizeOfSourceWithComputedFallback(aState));
89 aImage->CollectSizeOfSurfaces(mSurfaces, aState.mMallocSizeOf);
90
91 // Compute totals.
92 for (const SurfaceMemoryCounter& surfaceCounter : mSurfaces) {
93 mValues += surfaceCounter.Values();
94 }
95 }
96
97 ///////////////////////////////////////////////////////////////////////////////
98 // Image Base Types
99 ///////////////////////////////////////////////////////////////////////////////
100
GetSpecTruncatedTo1k(nsCString & aSpec) const101 bool ImageResource::GetSpecTruncatedTo1k(nsCString& aSpec) const {
102 static const size_t sMaxTruncatedLength = 1024;
103
104 mURI->GetSpec(aSpec);
105 if (sMaxTruncatedLength >= aSpec.Length()) {
106 return true;
107 }
108
109 aSpec.Truncate(sMaxTruncatedLength);
110 return false;
111 }
112
SetCurrentImage(layers::ImageContainer * aContainer,gfx::SourceSurface * aSurface,const Maybe<gfx::IntRect> & aDirtyRect)113 void ImageResource::SetCurrentImage(layers::ImageContainer* aContainer,
114 gfx::SourceSurface* aSurface,
115 const Maybe<gfx::IntRect>& aDirtyRect) {
116 MOZ_ASSERT(NS_IsMainThread());
117 MOZ_ASSERT(aContainer);
118
119 if (!aSurface) {
120 // The OS threw out some or all of our buffer. We'll need to wait for the
121 // redecode (which was automatically triggered by GetFrame) to complete.
122 return;
123 }
124
125 // |image| holds a reference to a SourceSurface which in turn holds a lock on
126 // the current frame's data buffer, ensuring that it doesn't get freed as
127 // long as the layer system keeps this ImageContainer alive.
128 RefPtr<layers::Image> image = new layers::SourceSurfaceImage(aSurface);
129
130 // We can share the producer ID with other containers because it is only
131 // used internally to validate the frames given to a particular container
132 // so that another object cannot add its own. Similarly the frame ID is
133 // only used internally to ensure it is always increasing, and skipping
134 // IDs from an individual container's perspective is acceptable.
135 AutoTArray<layers::ImageContainer::NonOwningImage, 1> imageList;
136 imageList.AppendElement(layers::ImageContainer::NonOwningImage(
137 image, TimeStamp(), mLastFrameID++, mImageProducerID));
138
139 if (aDirtyRect) {
140 aContainer->SetCurrentImagesInTransaction(imageList);
141 } else {
142 aContainer->SetCurrentImages(imageList);
143 }
144
145 // If we are animated, then we should request that the image container be
146 // treated as such, to avoid display list rebuilding to update frames for
147 // WebRender.
148 if (mProgressTracker->GetProgress() & FLAG_IS_ANIMATED) {
149 if (aDirtyRect) {
150 layers::SharedSurfacesChild::UpdateAnimation(aContainer, aSurface,
151 aDirtyRect.ref());
152 } else {
153 gfx::IntRect dirtyRect(gfx::IntPoint(0, 0), aSurface->GetSize());
154 layers::SharedSurfacesChild::UpdateAnimation(aContainer, aSurface,
155 dirtyRect);
156 }
157 }
158 }
159
GetImageContainerImpl(layers::LayerManager * aManager,const gfx::IntSize & aSize,const Maybe<SVGImageContext> & aSVGContext,const Maybe<ImageIntRegion> & aRegion,uint32_t aFlags,layers::ImageContainer ** aOutContainer)160 ImgDrawResult ImageResource::GetImageContainerImpl(
161 layers::LayerManager* aManager, const gfx::IntSize& aSize,
162 const Maybe<SVGImageContext>& aSVGContext,
163 const Maybe<ImageIntRegion>& aRegion, uint32_t aFlags,
164 layers::ImageContainer** aOutContainer) {
165 MOZ_ASSERT(NS_IsMainThread());
166 MOZ_ASSERT(aManager);
167 MOZ_ASSERT((aFlags &
168 ~(FLAG_SYNC_DECODE | FLAG_SYNC_DECODE_IF_FAST | FLAG_RECORD_BLOB |
169 FLAG_ASYNC_NOTIFY | FLAG_HIGH_QUALITY_SCALING)) == FLAG_NONE,
170 "Unsupported flag passed to GetImageContainer");
171
172 ImgDrawResult drawResult;
173 gfx::IntSize size;
174 Tie(drawResult, size) = GetImageContainerSize(aManager, aSize, aFlags);
175 if (drawResult != ImgDrawResult::SUCCESS) {
176 return drawResult;
177 }
178
179 MOZ_ASSERT(!size.IsEmpty());
180
181 if (mAnimationConsumers == 0) {
182 SendOnUnlockedDraw(aFlags);
183 }
184
185 uint32_t flags = (aFlags & ~(FLAG_SYNC_DECODE | FLAG_SYNC_DECODE_IF_FAST)) |
186 FLAG_ASYNC_NOTIFY;
187 RefPtr<layers::ImageContainer> container;
188 ImageContainerEntry* entry = nullptr;
189 int i = mImageContainers.Length() - 1;
190 for (; i >= 0; --i) {
191 entry = &mImageContainers[i];
192 if (size == entry->mSize && flags == entry->mFlags &&
193 aSVGContext == entry->mSVGContext && aRegion == entry->mRegion) {
194 // Lack of a container is handled below.
195 container = RefPtr<layers::ImageContainer>(entry->mContainer);
196 break;
197 } else if (entry->mContainer.IsDead()) {
198 // Stop tracking if our weak pointer to the image container was freed.
199 mImageContainers.RemoveElementAt(i);
200 }
201 }
202
203 if (container) {
204 switch (entry->mLastDrawResult) {
205 case ImgDrawResult::SUCCESS:
206 case ImgDrawResult::BAD_IMAGE:
207 case ImgDrawResult::BAD_ARGS:
208 case ImgDrawResult::NOT_SUPPORTED:
209 container.forget(aOutContainer);
210 return entry->mLastDrawResult;
211 case ImgDrawResult::NOT_READY:
212 case ImgDrawResult::INCOMPLETE:
213 case ImgDrawResult::TEMPORARY_ERROR:
214 // Temporary conditions where we need to rerequest the frame to recover.
215 break;
216 case ImgDrawResult::WRONG_SIZE:
217 // Unused by GetFrameInternal
218 default:
219 MOZ_ASSERT_UNREACHABLE("Unhandled ImgDrawResult type!");
220 container.forget(aOutContainer);
221 return entry->mLastDrawResult;
222 }
223 }
224
225 AutoProfilerImagePaintMarker PROFILER_RAII(this);
226 #ifdef DEBUG
227 NotifyDrawingObservers();
228 #endif
229
230 gfx::IntSize bestSize;
231 RefPtr<gfx::SourceSurface> surface;
232 Tie(drawResult, bestSize, surface) = GetFrameInternal(
233 size, aSVGContext, aRegion, FRAME_CURRENT, aFlags | FLAG_ASYNC_NOTIFY);
234
235 // The requested size might be refused by the surface cache (i.e. due to
236 // factor-of-2 mode). In that case we don't want to create an entry for this
237 // specific size, but rather re-use the entry for the substituted size.
238 if (bestSize != size) {
239 MOZ_ASSERT(!bestSize.IsEmpty());
240
241 // We can only remove the entry if we no longer have a container, because if
242 // there are strong references to it remaining, we need to still update it
243 // in UpdateImageContainer.
244 if (i >= 0 && !container) {
245 mImageContainers.RemoveElementAt(i);
246 }
247
248 // Forget about the stale container, if any. This lets the entry creation
249 // logic do its job below, if it turns out there is no existing best entry
250 // or the best entry doesn't have a container.
251 container = nullptr;
252
253 // We need to do the entry search again for the new size. We skip pruning
254 // because we did this above once already, but ImageContainer is threadsafe,
255 // so there is a remote possibility it got freed.
256 i = mImageContainers.Length() - 1;
257 for (; i >= 0; --i) {
258 entry = &mImageContainers[i];
259 if (bestSize == entry->mSize && flags == entry->mFlags &&
260 aSVGContext == entry->mSVGContext && aRegion == entry->mRegion) {
261 container = RefPtr<layers::ImageContainer>(entry->mContainer);
262 if (container) {
263 switch (entry->mLastDrawResult) {
264 case ImgDrawResult::SUCCESS:
265 case ImgDrawResult::BAD_IMAGE:
266 case ImgDrawResult::BAD_ARGS:
267 case ImgDrawResult::NOT_SUPPORTED:
268 container.forget(aOutContainer);
269 return entry->mLastDrawResult;
270 case ImgDrawResult::NOT_READY:
271 case ImgDrawResult::INCOMPLETE:
272 case ImgDrawResult::TEMPORARY_ERROR:
273 // Temporary conditions where we need to rerequest the frame to
274 // recover. We have already done so!
275 break;
276 case ImgDrawResult::WRONG_SIZE:
277 // Unused by GetFrameInternal
278 default:
279 MOZ_ASSERT_UNREACHABLE("Unhandled DrawResult type!");
280 container.forget(aOutContainer);
281 return entry->mLastDrawResult;
282 }
283 }
284 break;
285 }
286 }
287 }
288
289 if (!container) {
290 // We need a new ImageContainer, so create one.
291 container = layers::LayerManager::CreateImageContainer();
292
293 if (i >= 0) {
294 entry->mContainer = container;
295 } else {
296 entry = mImageContainers.AppendElement(ImageContainerEntry(
297 bestSize, aSVGContext, aRegion, container.get(), flags));
298 }
299 }
300
301 SetCurrentImage(container, surface, Nothing());
302 entry->mLastDrawResult = drawResult;
303 container.forget(aOutContainer);
304 return drawResult;
305 }
306
UpdateImageContainer(const Maybe<gfx::IntRect> & aDirtyRect)307 bool ImageResource::UpdateImageContainer(
308 const Maybe<gfx::IntRect>& aDirtyRect) {
309 MOZ_ASSERT(NS_IsMainThread());
310
311 for (int i = mImageContainers.Length() - 1; i >= 0; --i) {
312 ImageContainerEntry& entry = mImageContainers[i];
313 RefPtr<layers::ImageContainer> container(entry.mContainer);
314 if (container) {
315 // Blob recordings should just be marked as dirty. We will regenerate the
316 // recording when the display list update comes around.
317 if (entry.mFlags & FLAG_RECORD_BLOB) {
318 AutoTArray<layers::ImageContainer::OwningImage, 1> images;
319 container->GetCurrentImages(&images);
320 if (images.IsEmpty()) {
321 MOZ_ASSERT_UNREACHABLE("Empty container!");
322 continue;
323 }
324
325 RefPtr<gfx::SourceSurface> surface =
326 images[0].mImage->GetAsSourceSurface();
327 if (!surface || surface->GetType() != gfx::SurfaceType::BLOB_IMAGE) {
328 MOZ_ASSERT_UNREACHABLE("No/wrong surface in container!");
329 continue;
330 }
331
332 static_cast<SourceSurfaceBlobImage*>(surface.get())->MarkDirty();
333 continue;
334 }
335
336 gfx::IntSize bestSize;
337 RefPtr<gfx::SourceSurface> surface;
338 Tie(entry.mLastDrawResult, bestSize, surface) =
339 GetFrameInternal(entry.mSize, entry.mSVGContext, entry.mRegion,
340 FRAME_CURRENT, entry.mFlags);
341
342 // It is possible that this is a factor-of-2 substitution. Since we
343 // managed to convert the weak reference into a strong reference, that
344 // means that an imagelib user still is holding onto the container. thus
345 // we cannot consolidate and must keep updating the duplicate container.
346 if (aDirtyRect) {
347 SetCurrentImage(container, surface, aDirtyRect);
348 } else {
349 gfx::IntRect dirtyRect(gfx::IntPoint(0, 0), bestSize);
350 SetCurrentImage(container, surface, Some(dirtyRect));
351 }
352 } else {
353 // Stop tracking if our weak pointer to the image container was freed.
354 mImageContainers.RemoveElementAt(i);
355 }
356 }
357
358 return !mImageContainers.IsEmpty();
359 }
360
CollectSizeOfSurfaces(nsTArray<SurfaceMemoryCounter> & aCounters,MallocSizeOf aMallocSizeOf) const361 void ImageResource::CollectSizeOfSurfaces(
362 nsTArray<SurfaceMemoryCounter>& aCounters,
363 MallocSizeOf aMallocSizeOf) const {
364 MOZ_ASSERT(NS_IsMainThread());
365
366 for (const auto& entry : mImageContainers) {
367 RefPtr<layers::ImageContainer> container(entry.mContainer);
368 if (!container) {
369 continue;
370 }
371
372 AutoTArray<layers::ImageContainer::OwningImage, 1> images;
373 container->GetCurrentImages(&images);
374 if (images.IsEmpty()) {
375 continue;
376 }
377
378 RefPtr<gfx::SourceSurface> surface = images[0].mImage->GetAsSourceSurface();
379 if (!surface) {
380 MOZ_ASSERT_UNREACHABLE("No surface in container!");
381 continue;
382 }
383
384 // The surface might be wrapping another.
385 bool isMappedSurface = surface->GetType() == gfx::SurfaceType::DATA_MAPPED;
386 const gfx::SourceSurface* actualSurface =
387 isMappedSurface
388 ? static_cast<gfx::SourceSurfaceMappedData*>(surface.get())
389 ->GetScopedSurface()
390 : surface.get();
391
392 // Check if the surface is already in the report. Ignore if so.
393 bool found = false;
394 for (const auto& counter : aCounters) {
395 if (counter.Surface() == actualSurface) {
396 found = true;
397 break;
398 }
399 }
400 if (found) {
401 continue;
402 }
403
404 // The surface isn't in the report, so it isn't stored in SurfaceCache. We
405 // need to add our own entry here so that it will be included in the memory
406 // report.
407 gfx::SourceSurface::SizeOfInfo info;
408 surface->SizeOfExcludingThis(aMallocSizeOf, info);
409
410 uint32_t heapBytes = aMallocSizeOf(actualSurface);
411 if (isMappedSurface) {
412 heapBytes += aMallocSizeOf(surface.get());
413 }
414
415 SurfaceKey key = ContainerSurfaceKey(surface->GetSize(), entry.mSVGContext,
416 ToSurfaceFlags(entry.mFlags));
417 SurfaceMemoryCounter counter(key, actualSurface, /* aIsLocked */ false,
418 /* aCannotSubstitute */ false,
419 /* aIsFactor2 */ false, /* aFinished */ true,
420 SurfaceMemoryCounterType::CONTAINER);
421
422 counter.Values().SetDecodedHeap(info.mHeapBytes + heapBytes);
423 counter.Values().SetDecodedNonHeap(info.mNonHeapBytes);
424 counter.Values().SetDecodedUnknown(info.mUnknownBytes);
425 counter.Values().SetExternalHandles(info.mExternalHandles);
426 counter.Values().SetExternalId(info.mExternalId);
427 counter.Values().SetSurfaceTypes(info.mTypes);
428
429 aCounters.AppendElement(counter);
430 }
431 }
432
ReleaseImageContainer()433 void ImageResource::ReleaseImageContainer() {
434 MOZ_ASSERT(NS_IsMainThread());
435 mImageContainers.Clear();
436 }
437
438 // Constructor
ImageResource(nsIURI * aURI)439 ImageResource::ImageResource(nsIURI* aURI)
440 : mURI(aURI),
441 mInnerWindowId(0),
442 mAnimationConsumers(0),
443 mAnimationMode(kNormalAnimMode),
444 mInitialized(false),
445 mAnimating(false),
446 mError(false),
447 mImageProducerID(layers::ImageContainer::AllocateProducerID()),
448 mLastFrameID(0) {}
449
~ImageResource()450 ImageResource::~ImageResource() {
451 // Ask our ProgressTracker to drop its weak reference to us.
452 mProgressTracker->ResetImage();
453 }
454
IncrementAnimationConsumers()455 void ImageResource::IncrementAnimationConsumers() {
456 MOZ_ASSERT(NS_IsMainThread(),
457 "Main thread only to encourage serialization "
458 "with DecrementAnimationConsumers");
459 mAnimationConsumers++;
460 }
461
DecrementAnimationConsumers()462 void ImageResource::DecrementAnimationConsumers() {
463 MOZ_ASSERT(NS_IsMainThread(),
464 "Main thread only to encourage serialization "
465 "with IncrementAnimationConsumers");
466 MOZ_ASSERT(mAnimationConsumers >= 1, "Invalid no. of animation consumers!");
467 mAnimationConsumers--;
468 }
469
GetAnimationModeInternal(uint16_t * aAnimationMode)470 nsresult ImageResource::GetAnimationModeInternal(uint16_t* aAnimationMode) {
471 if (mError) {
472 return NS_ERROR_FAILURE;
473 }
474
475 NS_ENSURE_ARG_POINTER(aAnimationMode);
476
477 *aAnimationMode = mAnimationMode;
478 return NS_OK;
479 }
480
SetAnimationModeInternal(uint16_t aAnimationMode)481 nsresult ImageResource::SetAnimationModeInternal(uint16_t aAnimationMode) {
482 if (mError) {
483 return NS_ERROR_FAILURE;
484 }
485
486 NS_ASSERTION(aAnimationMode == kNormalAnimMode ||
487 aAnimationMode == kDontAnimMode ||
488 aAnimationMode == kLoopOnceAnimMode,
489 "Wrong Animation Mode is being set!");
490
491 mAnimationMode = aAnimationMode;
492
493 return NS_OK;
494 }
495
HadRecentRefresh(const TimeStamp & aTime)496 bool ImageResource::HadRecentRefresh(const TimeStamp& aTime) {
497 // Our threshold for "recent" is 1/2 of the default refresh-driver interval.
498 // This ensures that we allow for frame rates at least as fast as the
499 // refresh driver's default rate.
500 static TimeDuration recentThreshold =
501 TimeDuration::FromMilliseconds(nsRefreshDriver::DefaultInterval() / 2.0);
502
503 if (!mLastRefreshTime.IsNull() &&
504 aTime - mLastRefreshTime < recentThreshold) {
505 return true;
506 }
507
508 // else, we can proceed with a refresh.
509 // But first, update our last refresh time:
510 mLastRefreshTime = aTime;
511 return false;
512 }
513
EvaluateAnimation()514 void ImageResource::EvaluateAnimation() {
515 if (!mAnimating && ShouldAnimate()) {
516 nsresult rv = StartAnimation();
517 mAnimating = NS_SUCCEEDED(rv);
518 } else if (mAnimating && !ShouldAnimate()) {
519 StopAnimation();
520 }
521 }
522
SendOnUnlockedDraw(uint32_t aFlags)523 void ImageResource::SendOnUnlockedDraw(uint32_t aFlags) {
524 if (!mProgressTracker) {
525 return;
526 }
527
528 if (!(aFlags & FLAG_ASYNC_NOTIFY)) {
529 mProgressTracker->OnUnlockedDraw();
530 } else {
531 NotNull<RefPtr<ImageResource>> image = WrapNotNull(this);
532 nsCOMPtr<nsIEventTarget> eventTarget = mProgressTracker->GetEventTarget();
533 nsCOMPtr<nsIRunnable> ev = NS_NewRunnableFunction(
534 "image::ImageResource::SendOnUnlockedDraw", [=]() -> void {
535 RefPtr<ProgressTracker> tracker = image->GetProgressTracker();
536 if (tracker) {
537 tracker->OnUnlockedDraw();
538 }
539 });
540 eventTarget->Dispatch(CreateMediumHighRunnable(ev.forget()),
541 NS_DISPATCH_NORMAL);
542 }
543 }
544
545 #ifdef DEBUG
NotifyDrawingObservers()546 void ImageResource::NotifyDrawingObservers() {
547 if (!mURI || !NS_IsMainThread()) {
548 return;
549 }
550
551 if (!mURI->SchemeIs("resource") && !mURI->SchemeIs("chrome")) {
552 return;
553 }
554
555 // Record the image drawing for startup performance testing.
556 nsCOMPtr<nsIURI> uri = mURI;
557 nsContentUtils::AddScriptRunner(NS_NewRunnableFunction(
558 "image::ImageResource::NotifyDrawingObservers", [uri]() {
559 nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
560 NS_WARNING_ASSERTION(obs, "Can't get an observer service handle");
561 if (obs) {
562 nsAutoCString spec;
563 uri->GetSpec(spec);
564 obs->NotifyObservers(nullptr, "image-drawing",
565 NS_ConvertUTF8toUTF16(spec).get());
566 }
567 }));
568 }
569 #endif
570
571 } // namespace image
572 } // namespace mozilla
573