1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* This Source Code Form is subject to the terms of the Mozilla Public 3 * License, v. 2.0. If a copy of the MPL was not distributed with this 4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 5 6 #ifndef mozilla_image_Image_h 7 #define mozilla_image_Image_h 8 9 #include "mozilla/Attributes.h" 10 #include "mozilla/Maybe.h" 11 #include "mozilla/MemoryReporting.h" 12 #include "mozilla/ProfilerMarkers.h" 13 #include "mozilla/SizeOfState.h" 14 #include "mozilla/ThreadSafeWeakPtr.h" 15 #include "mozilla/TimeStamp.h" 16 #include "mozilla/Tuple.h" 17 #include "gfx2DGlue.h" 18 #include "imgIContainer.h" 19 #include "ImageContainer.h" 20 #include "ImageRegion.h" 21 #include "LookupResult.h" 22 #include "nsStringFwd.h" 23 #include "ProgressTracker.h" 24 #include "SurfaceCache.h" 25 26 class imgRequest; 27 class nsIRequest; 28 class nsIInputStream; 29 30 namespace mozilla { 31 namespace image { 32 33 class Image; 34 35 /////////////////////////////////////////////////////////////////////////////// 36 // Memory Reporting 37 /////////////////////////////////////////////////////////////////////////////// 38 39 struct MemoryCounter { MemoryCounterMemoryCounter40 MemoryCounter() 41 : mSource(0), 42 mDecodedHeap(0), 43 mDecodedNonHeap(0), 44 mDecodedUnknown(0), 45 mExternalHandles(0), 46 mFrameIndex(0), 47 mExternalId(0), 48 mSurfaceTypes(0) {} 49 SetSourceMemoryCounter50 void SetSource(size_t aCount) { mSource = aCount; } SourceMemoryCounter51 size_t Source() const { return mSource; } SetDecodedHeapMemoryCounter52 void SetDecodedHeap(size_t aCount) { mDecodedHeap = aCount; } DecodedHeapMemoryCounter53 size_t DecodedHeap() const { return mDecodedHeap; } SetDecodedNonHeapMemoryCounter54 void SetDecodedNonHeap(size_t aCount) { mDecodedNonHeap = aCount; } DecodedNonHeapMemoryCounter55 size_t DecodedNonHeap() const { return mDecodedNonHeap; } SetDecodedUnknownMemoryCounter56 void SetDecodedUnknown(size_t aCount) { mDecodedUnknown = aCount; } DecodedUnknownMemoryCounter57 size_t DecodedUnknown() const { return mDecodedUnknown; } SetExternalHandlesMemoryCounter58 void SetExternalHandles(size_t aCount) { mExternalHandles = aCount; } ExternalHandlesMemoryCounter59 size_t ExternalHandles() const { return mExternalHandles; } SetFrameIndexMemoryCounter60 void SetFrameIndex(size_t aIndex) { mFrameIndex = aIndex; } FrameIndexMemoryCounter61 size_t FrameIndex() const { return mFrameIndex; } SetExternalIdMemoryCounter62 void SetExternalId(uint64_t aId) { mExternalId = aId; } ExternalIdMemoryCounter63 uint64_t ExternalId() const { return mExternalId; } SetSurfaceTypesMemoryCounter64 void SetSurfaceTypes(uint32_t aTypes) { mSurfaceTypes = aTypes; } SurfaceTypesMemoryCounter65 uint32_t SurfaceTypes() const { return mSurfaceTypes; } 66 67 MemoryCounter& operator+=(const MemoryCounter& aOther) { 68 mSource += aOther.mSource; 69 mDecodedHeap += aOther.mDecodedHeap; 70 mDecodedNonHeap += aOther.mDecodedNonHeap; 71 mDecodedUnknown += aOther.mDecodedUnknown; 72 mExternalHandles += aOther.mExternalHandles; 73 mSurfaceTypes |= aOther.mSurfaceTypes; 74 return *this; 75 } 76 77 private: 78 size_t mSource; 79 size_t mDecodedHeap; 80 size_t mDecodedNonHeap; 81 size_t mDecodedUnknown; 82 size_t mExternalHandles; 83 size_t mFrameIndex; 84 uint64_t mExternalId; 85 uint32_t mSurfaceTypes; 86 }; 87 88 enum class SurfaceMemoryCounterType { NORMAL, CONTAINER }; 89 90 struct SurfaceMemoryCounter { 91 SurfaceMemoryCounter( 92 const SurfaceKey& aKey, const gfx::SourceSurface* aSurface, 93 bool aIsLocked, bool aCannotSubstitute, bool aIsFactor2, bool aFinished, 94 SurfaceMemoryCounterType aType = SurfaceMemoryCounterType::NORMAL) mKeySurfaceMemoryCounter95 : mKey(aKey), 96 mSurface(aSurface), 97 mType(aType), 98 mIsLocked(aIsLocked), 99 mCannotSubstitute(aCannotSubstitute), 100 mIsFactor2(aIsFactor2), 101 mFinished(aFinished) {} 102 KeySurfaceMemoryCounter103 const SurfaceKey& Key() const { return mKey; } SurfaceSurfaceMemoryCounter104 const gfx::SourceSurface* Surface() const { return mSurface; } ValuesSurfaceMemoryCounter105 MemoryCounter& Values() { return mValues; } ValuesSurfaceMemoryCounter106 const MemoryCounter& Values() const { return mValues; } TypeSurfaceMemoryCounter107 SurfaceMemoryCounterType Type() const { return mType; } IsLockedSurfaceMemoryCounter108 bool IsLocked() const { return mIsLocked; } CannotSubstituteSurfaceMemoryCounter109 bool CannotSubstitute() const { return mCannotSubstitute; } IsFactor2SurfaceMemoryCounter110 bool IsFactor2() const { return mIsFactor2; } IsFinishedSurfaceMemoryCounter111 bool IsFinished() const { return mFinished; } 112 113 private: 114 const SurfaceKey mKey; 115 const gfx::SourceSurface* MOZ_NON_OWNING_REF mSurface; 116 MemoryCounter mValues; 117 const SurfaceMemoryCounterType mType; 118 const bool mIsLocked; 119 const bool mCannotSubstitute; 120 const bool mIsFactor2; 121 const bool mFinished; 122 }; 123 124 struct ImageMemoryCounter { 125 ImageMemoryCounter(imgRequest* aRequest, SizeOfState& aState, bool aIsUsed); 126 ImageMemoryCounter(imgRequest* aRequest, Image* aImage, SizeOfState& aState, 127 bool aIsUsed); 128 URIImageMemoryCounter129 nsCString& URI() { return mURI; } URIImageMemoryCounter130 const nsCString& URI() const { return mURI; } SurfacesImageMemoryCounter131 const nsTArray<SurfaceMemoryCounter>& Surfaces() const { return mSurfaces; } IntrinsicSizeImageMemoryCounter132 const gfx::IntSize IntrinsicSize() const { return mIntrinsicSize; } ValuesImageMemoryCounter133 const MemoryCounter& Values() const { return mValues; } ProgressImageMemoryCounter134 uint32_t Progress() const { return mProgress; } TypeImageMemoryCounter135 uint16_t Type() const { return mType; } IsUsedImageMemoryCounter136 bool IsUsed() const { return mIsUsed; } HasErrorImageMemoryCounter137 bool HasError() const { return mHasError; } IsValidatingImageMemoryCounter138 bool IsValidating() const { return mValidating; } 139 IsNotableImageMemoryCounter140 bool IsNotable() const { 141 // Errors or requests without images are always notable. 142 if (mHasError || mValidating || mProgress == UINT32_MAX || 143 mProgress & FLAG_HAS_ERROR || mType == imgIContainer::TYPE_REQUEST) { 144 return true; 145 } 146 147 // Sufficiently large images are notable. 148 const size_t NotableThreshold = 16 * 1024; 149 size_t total = mValues.Source() + mValues.DecodedHeap() + 150 mValues.DecodedNonHeap() + mValues.DecodedUnknown(); 151 if (total >= NotableThreshold) { 152 return true; 153 } 154 155 // Incomplete images are always notable as well; the odds of capturing 156 // mid-decode should be fairly low. 157 for (const auto& surface : mSurfaces) { 158 if (!surface.IsFinished()) { 159 return true; 160 } 161 } 162 163 return false; 164 } 165 166 private: 167 nsCString mURI; 168 nsTArray<SurfaceMemoryCounter> mSurfaces; 169 gfx::IntSize mIntrinsicSize; 170 MemoryCounter mValues; 171 uint32_t mProgress; 172 uint16_t mType; 173 const bool mIsUsed; 174 bool mHasError; 175 bool mValidating; 176 }; 177 178 /////////////////////////////////////////////////////////////////////////////// 179 // Image Base Types 180 /////////////////////////////////////////////////////////////////////////////// 181 182 class Image : public imgIContainer { 183 public: 184 /** 185 * Flags for Image initialization. 186 * 187 * Meanings: 188 * 189 * INIT_FLAG_NONE: Lack of flags 190 * 191 * INIT_FLAG_DISCARDABLE: The container should be discardable 192 * 193 * INIT_FLAG_DECODE_IMMEDIATELY: The container should decode as soon as 194 * possible, regardless of what our heuristics say. 195 * 196 * INIT_FLAG_TRANSIENT: The container is likely to exist for only a short time 197 * before being destroyed. (For example, containers for 198 * multipart/x-mixed-replace image parts fall into this category.) If this 199 * flag is set, INIT_FLAG_DISCARDABLE and INIT_FLAG_DECODE_ONLY_ON_DRAW must 200 * not be set. 201 * 202 * INIT_FLAG_SYNC_LOAD: The container is being loaded synchronously, so 203 * it should avoid relying on async workers to get the container ready. 204 */ 205 static const uint32_t INIT_FLAG_NONE = 0x0; 206 static const uint32_t INIT_FLAG_DISCARDABLE = 0x1; 207 static const uint32_t INIT_FLAG_DECODE_IMMEDIATELY = 0x2; 208 static const uint32_t INIT_FLAG_TRANSIENT = 0x4; 209 static const uint32_t INIT_FLAG_SYNC_LOAD = 0x8; 210 211 virtual already_AddRefed<ProgressTracker> GetProgressTracker() = 0; SetProgressTracker(ProgressTracker * aProgressTracker)212 virtual void SetProgressTracker(ProgressTracker* aProgressTracker) {} 213 214 /** 215 * The size, in bytes, occupied by the compressed source data of the image. 216 * If MallocSizeOf does not work on this platform, uses a fallback approach to 217 * ensure that something reasonable is always returned. 218 */ 219 virtual size_t SizeOfSourceWithComputedFallback( 220 SizeOfState& aState) const = 0; 221 222 /** 223 * Collect an accounting of the memory occupied by the image's surfaces (which 224 * together make up its decoded data). Each surface is recorded as a separate 225 * SurfaceMemoryCounter, stored in @aCounters. 226 */ 227 virtual void CollectSizeOfSurfaces(nsTArray<SurfaceMemoryCounter>& aCounters, 228 MallocSizeOf aMallocSizeOf) const = 0; 229 230 virtual void IncrementAnimationConsumers() = 0; 231 virtual void DecrementAnimationConsumers() = 0; 232 #ifdef DEBUG 233 virtual uint32_t GetAnimationConsumers() = 0; 234 #endif 235 236 /** 237 * Called from OnDataAvailable when the stream associated with the image has 238 * received new image data. The arguments are the same as OnDataAvailable's, 239 * but by separating this functionality into a different method we don't 240 * interfere with subclasses which wish to implement nsIStreamListener. 241 * 242 * Images should not do anything that could send out notifications until they 243 * have received their first OnImageDataAvailable notification; in 244 * particular, this means that instantiating decoders should be deferred 245 * until OnImageDataAvailable is called. 246 */ 247 virtual nsresult OnImageDataAvailable(nsIRequest* aRequest, 248 nsISupports* aContext, 249 nsIInputStream* aInStr, 250 uint64_t aSourceOffset, 251 uint32_t aCount) = 0; 252 253 /** 254 * Called from OnStopRequest when the image's underlying request completes. 255 * 256 * @param aRequest The completed request. 257 * @param aContext Context from Necko's OnStopRequest. 258 * @param aStatus A success or failure code. 259 * @param aLastPart Whether this is the final part of the underlying request. 260 */ 261 virtual nsresult OnImageDataComplete(nsIRequest* aRequest, 262 nsISupports* aContext, nsresult aStatus, 263 bool aLastPart) = 0; 264 265 /** 266 * Called when the SurfaceCache discards a surface belonging to this image. 267 */ 268 virtual void OnSurfaceDiscarded(const SurfaceKey& aSurfaceKey) = 0; 269 270 virtual void SetInnerWindowID(uint64_t aInnerWindowId) = 0; 271 virtual uint64_t InnerWindowID() const = 0; 272 273 virtual bool HasError() = 0; 274 virtual void SetHasError() = 0; 275 276 virtual nsIURI* GetURI() const = 0; 277 GetHotspotX(int32_t * aX)278 NS_IMETHOD GetHotspotX(int32_t* aX) override { 279 *aX = 0; 280 return NS_OK; 281 } GetHotspotY(int32_t * aY)282 NS_IMETHOD GetHotspotY(int32_t* aY) override { 283 *aY = 0; 284 return NS_OK; 285 } 286 }; 287 288 class ImageResource : public Image { 289 public: GetProgressTracker()290 already_AddRefed<ProgressTracker> GetProgressTracker() override { 291 RefPtr<ProgressTracker> progressTracker = mProgressTracker; 292 MOZ_ASSERT(progressTracker); 293 return progressTracker.forget(); 294 } 295 SetProgressTracker(ProgressTracker * aProgressTracker)296 void SetProgressTracker(ProgressTracker* aProgressTracker) final { 297 MOZ_ASSERT(aProgressTracker); 298 MOZ_ASSERT(!mProgressTracker); 299 mProgressTracker = aProgressTracker; 300 } 301 302 virtual void IncrementAnimationConsumers() override; 303 virtual void DecrementAnimationConsumers() override; 304 #ifdef DEBUG GetAnimationConsumers()305 virtual uint32_t GetAnimationConsumers() override { 306 return mAnimationConsumers; 307 } 308 #endif 309 OnSurfaceDiscarded(const SurfaceKey & aSurfaceKey)310 virtual void OnSurfaceDiscarded(const SurfaceKey& aSurfaceKey) override {} 311 SetInnerWindowID(uint64_t aInnerWindowId)312 virtual void SetInnerWindowID(uint64_t aInnerWindowId) override { 313 mInnerWindowId = aInnerWindowId; 314 } InnerWindowID()315 virtual uint64_t InnerWindowID() const override { return mInnerWindowId; } 316 HasError()317 virtual bool HasError() override { return mError; } SetHasError()318 virtual void SetHasError() override { mError = true; } 319 320 /* 321 * Returns a non-AddRefed pointer to the URI associated with this image. 322 * Illegal to use off-main-thread. 323 */ GetURI()324 nsIURI* GetURI() const override { return mURI; } 325 326 /* 327 * Should be called by its subclasses after they populate @aCounters so that 328 * we can cross reference against any of our ImageContainers that contain 329 * surfaces not in the cache. 330 */ 331 void CollectSizeOfSurfaces(nsTArray<SurfaceMemoryCounter>& aCounters, 332 MallocSizeOf aMallocSizeOf) const override; 333 334 protected: 335 explicit ImageResource(nsIURI* aURI); 336 ~ImageResource(); 337 GetImageProducerId()338 layers::ContainerProducerID GetImageProducerId() const { 339 return mImageProducerID; 340 } 341 342 bool GetSpecTruncatedTo1k(nsCString& aSpec) const; 343 344 // Shared functionality for implementors of imgIContainer. Every 345 // implementation of attribute animationMode should forward here. 346 nsresult GetAnimationModeInternal(uint16_t* aAnimationMode); 347 nsresult SetAnimationModeInternal(uint16_t aAnimationMode); 348 349 /** 350 * Helper for RequestRefresh. 351 * 352 * If we've had a "recent" refresh (i.e. if this image is being used in 353 * multiple documents & some other document *just* called RequestRefresh() on 354 * this image with a timestamp close to aTime), this method returns true. 355 * 356 * Otherwise, this method updates mLastRefreshTime to aTime & returns false. 357 */ 358 bool HadRecentRefresh(const TimeStamp& aTime); 359 360 /** 361 * Decides whether animation should or should not be happening, 362 * and makes sure the right thing is being done. 363 */ 364 virtual void EvaluateAnimation(); 365 366 /** 367 * Extended by child classes, if they have additional 368 * conditions for being able to animate. 369 */ ShouldAnimate()370 virtual bool ShouldAnimate() { 371 return mAnimationConsumers > 0 && mAnimationMode != kDontAnimMode; 372 } 373 374 virtual nsresult StartAnimation() = 0; 375 virtual nsresult StopAnimation() = 0; 376 377 void SendOnUnlockedDraw(uint32_t aFlags); 378 379 #ifdef DEBUG 380 // Records the image drawing for startup performance testing. 381 void NotifyDrawingObservers(); 382 #endif 383 384 // Member data shared by all implementations of this abstract class 385 RefPtr<ProgressTracker> mProgressTracker; 386 nsCOMPtr<nsIURI> mURI; 387 TimeStamp mLastRefreshTime; 388 uint64_t mInnerWindowId; 389 uint32_t mAnimationConsumers; 390 uint16_t mAnimationMode; // Enum values in imgIContainer 391 bool mInitialized : 1; // Have we been initialized? 392 bool mAnimating : 1; // Are we currently animating? 393 bool mError : 1; // Error handling 394 395 /** 396 * Attempt to find a matching cached surface in the SurfaceCache, and if not 397 * available, request the production of such a surface (either synchronously 398 * or asynchronously). 399 * 400 * If the draw result is BAD_IMAGE, BAD_ARGS or NOT_READY, the size will be 401 * the same as aSize. If it is TEMPORARY_ERROR, INCOMPLETE, or SUCCESS, the 402 * size is a hint as to what we expect the surface size to be, once the best 403 * fitting size is available. It may or may not match the size of the surface 404 * returned at this moment. This is useful for choosing how to store the final 405 * result (e.g. if going into an ImageContainer, ideally we would share the 406 * same container for many requested sizes, if they all end up with the same 407 * best fit size in the end). 408 * 409 * A valid surface should only be returned for SUCCESS and INCOMPLETE. 410 * 411 * Any other draw result is invalid. 412 */ 413 virtual Tuple<ImgDrawResult, gfx::IntSize, RefPtr<gfx::SourceSurface>> GetFrameInternal(const gfx::IntSize & aSize,const Maybe<SVGImageContext> & aSVGContext,const Maybe<ImageIntRegion> & aRegion,uint32_t aWhichFrame,uint32_t aFlags)414 GetFrameInternal(const gfx::IntSize& aSize, 415 const Maybe<SVGImageContext>& aSVGContext, 416 const Maybe<ImageIntRegion>& aRegion, uint32_t aWhichFrame, 417 uint32_t aFlags) { 418 return MakeTuple(ImgDrawResult::BAD_IMAGE, aSize, 419 RefPtr<gfx::SourceSurface>()); 420 } 421 422 /** 423 * Calculate the estimated size to use for an image container with the given 424 * parameters. It may not be the same as the given size, and it may not be 425 * the same as the size of the surface in the image container, but it is the 426 * best effort estimate. 427 */ GetImageContainerSize(layers::LayerManager * aManager,const gfx::IntSize & aSize,uint32_t aFlags)428 virtual Tuple<ImgDrawResult, gfx::IntSize> GetImageContainerSize( 429 layers::LayerManager* aManager, const gfx::IntSize& aSize, 430 uint32_t aFlags) { 431 return MakeTuple(ImgDrawResult::NOT_SUPPORTED, gfx::IntSize(0, 0)); 432 } 433 434 ImgDrawResult GetImageContainerImpl(layers::LayerManager* aManager, 435 const gfx::IntSize& aSize, 436 const Maybe<SVGImageContext>& aSVGContext, 437 const Maybe<ImageIntRegion>& aRegion, 438 uint32_t aFlags, 439 layers::ImageContainer** aContainer); 440 441 /** 442 * Re-requests the appropriate frames for each image container using 443 * GetFrameInternal. 444 * @returns True if any image containers were updated, else false. 445 */ 446 bool UpdateImageContainer(const Maybe<gfx::IntRect>& aDirtyRect); 447 448 void ReleaseImageContainer(); 449 450 class MOZ_RAII AutoProfilerImagePaintMarker { 451 public: AutoProfilerImagePaintMarker(ImageResource * self)452 explicit AutoProfilerImagePaintMarker(ImageResource* self) 453 : mStartTime(TimeStamp::NowUnfuzzed()) { 454 nsAutoCString spec; 455 if (self->mURI && profiler_can_accept_markers()) { 456 static const size_t sMaxTruncatedLength = 1024; 457 self->mURI->GetSpec(mSpec); 458 if (mSpec.Length() >= sMaxTruncatedLength) { 459 mSpec.Truncate(sMaxTruncatedLength); 460 } 461 } 462 } 463 ~AutoProfilerImagePaintMarker()464 ~AutoProfilerImagePaintMarker() { 465 if (!mSpec.IsEmpty()) { 466 PROFILER_MARKER_TEXT("Image Paint", GRAPHICS, 467 MarkerTiming::IntervalUntilNowFrom(mStartTime), 468 mSpec); 469 } 470 } 471 472 protected: 473 TimeStamp mStartTime; 474 nsAutoCString mSpec; 475 }; 476 477 private: 478 void SetCurrentImage(layers::ImageContainer* aContainer, 479 gfx::SourceSurface* aSurface, 480 const Maybe<gfx::IntRect>& aDirtyRect); 481 482 struct ImageContainerEntry { ImageContainerEntryImageContainerEntry483 ImageContainerEntry(const gfx::IntSize& aSize, 484 const Maybe<SVGImageContext>& aSVGContext, 485 const Maybe<ImageIntRegion>& aRegion, 486 layers::ImageContainer* aContainer, uint32_t aFlags) 487 : mSize(aSize), 488 mSVGContext(aSVGContext), 489 mRegion(aRegion), 490 mContainer(aContainer), 491 mLastDrawResult(ImgDrawResult::NOT_READY), 492 mFlags(aFlags) {} 493 494 gfx::IntSize mSize; 495 Maybe<SVGImageContext> mSVGContext; 496 Maybe<ImageIntRegion> mRegion; 497 // A weak pointer to our ImageContainer, which stays alive only as long as 498 // the layer system needs it. 499 ThreadSafeWeakPtr<layers::ImageContainer> mContainer; 500 // If mContainer is non-null, this contains the ImgDrawResult we obtained 501 // the last time we updated it. 502 ImgDrawResult mLastDrawResult; 503 // Cached flags to use for decoding. FLAG_ASYNC_NOTIFY should always be set 504 // but FLAG_HIGH_QUALITY_SCALING may vary. 505 uint32_t mFlags; 506 }; 507 508 AutoTArray<ImageContainerEntry, 1> mImageContainers; 509 layers::ImageContainer::ProducerID mImageProducerID; 510 layers::ImageContainer::FrameID mLastFrameID; 511 }; 512 513 } // namespace image 514 } // namespace mozilla 515 516 #endif // mozilla_image_Image_h 517