1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3  * License, v. 2.0. If a copy of the MPL was not distributed with this
4  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 
6 #ifndef mozilla_image_Image_h
7 #define mozilla_image_Image_h
8 
9 #include "mozilla/Attributes.h"
10 #include "mozilla/Maybe.h"
11 #include "mozilla/MemoryReporting.h"
12 #include "mozilla/ProfilerMarkers.h"
13 #include "mozilla/SizeOfState.h"
14 #include "mozilla/ThreadSafeWeakPtr.h"
15 #include "mozilla/TimeStamp.h"
16 #include "mozilla/Tuple.h"
17 #include "gfx2DGlue.h"
18 #include "imgIContainer.h"
19 #include "ImageContainer.h"
20 #include "ImageRegion.h"
21 #include "LookupResult.h"
22 #include "nsStringFwd.h"
23 #include "ProgressTracker.h"
24 #include "SurfaceCache.h"
25 
26 class imgRequest;
27 class nsIRequest;
28 class nsIInputStream;
29 
30 namespace mozilla {
31 namespace image {
32 
33 class Image;
34 
35 ///////////////////////////////////////////////////////////////////////////////
36 // Memory Reporting
37 ///////////////////////////////////////////////////////////////////////////////
38 
39 struct MemoryCounter {
MemoryCounterMemoryCounter40   MemoryCounter()
41       : mSource(0),
42         mDecodedHeap(0),
43         mDecodedNonHeap(0),
44         mDecodedUnknown(0),
45         mExternalHandles(0),
46         mFrameIndex(0),
47         mExternalId(0),
48         mSurfaceTypes(0) {}
49 
SetSourceMemoryCounter50   void SetSource(size_t aCount) { mSource = aCount; }
SourceMemoryCounter51   size_t Source() const { return mSource; }
SetDecodedHeapMemoryCounter52   void SetDecodedHeap(size_t aCount) { mDecodedHeap = aCount; }
DecodedHeapMemoryCounter53   size_t DecodedHeap() const { return mDecodedHeap; }
SetDecodedNonHeapMemoryCounter54   void SetDecodedNonHeap(size_t aCount) { mDecodedNonHeap = aCount; }
DecodedNonHeapMemoryCounter55   size_t DecodedNonHeap() const { return mDecodedNonHeap; }
SetDecodedUnknownMemoryCounter56   void SetDecodedUnknown(size_t aCount) { mDecodedUnknown = aCount; }
DecodedUnknownMemoryCounter57   size_t DecodedUnknown() const { return mDecodedUnknown; }
SetExternalHandlesMemoryCounter58   void SetExternalHandles(size_t aCount) { mExternalHandles = aCount; }
ExternalHandlesMemoryCounter59   size_t ExternalHandles() const { return mExternalHandles; }
SetFrameIndexMemoryCounter60   void SetFrameIndex(size_t aIndex) { mFrameIndex = aIndex; }
FrameIndexMemoryCounter61   size_t FrameIndex() const { return mFrameIndex; }
SetExternalIdMemoryCounter62   void SetExternalId(uint64_t aId) { mExternalId = aId; }
ExternalIdMemoryCounter63   uint64_t ExternalId() const { return mExternalId; }
SetSurfaceTypesMemoryCounter64   void SetSurfaceTypes(uint32_t aTypes) { mSurfaceTypes = aTypes; }
SurfaceTypesMemoryCounter65   uint32_t SurfaceTypes() const { return mSurfaceTypes; }
66 
67   MemoryCounter& operator+=(const MemoryCounter& aOther) {
68     mSource += aOther.mSource;
69     mDecodedHeap += aOther.mDecodedHeap;
70     mDecodedNonHeap += aOther.mDecodedNonHeap;
71     mDecodedUnknown += aOther.mDecodedUnknown;
72     mExternalHandles += aOther.mExternalHandles;
73     mSurfaceTypes |= aOther.mSurfaceTypes;
74     return *this;
75   }
76 
77  private:
78   size_t mSource;
79   size_t mDecodedHeap;
80   size_t mDecodedNonHeap;
81   size_t mDecodedUnknown;
82   size_t mExternalHandles;
83   size_t mFrameIndex;
84   uint64_t mExternalId;
85   uint32_t mSurfaceTypes;
86 };
87 
88 enum class SurfaceMemoryCounterType { NORMAL, CONTAINER };
89 
90 struct SurfaceMemoryCounter {
91   SurfaceMemoryCounter(
92       const SurfaceKey& aKey, const gfx::SourceSurface* aSurface,
93       bool aIsLocked, bool aCannotSubstitute, bool aIsFactor2, bool aFinished,
94       SurfaceMemoryCounterType aType = SurfaceMemoryCounterType::NORMAL)
mKeySurfaceMemoryCounter95       : mKey(aKey),
96         mSurface(aSurface),
97         mType(aType),
98         mIsLocked(aIsLocked),
99         mCannotSubstitute(aCannotSubstitute),
100         mIsFactor2(aIsFactor2),
101         mFinished(aFinished) {}
102 
KeySurfaceMemoryCounter103   const SurfaceKey& Key() const { return mKey; }
SurfaceSurfaceMemoryCounter104   const gfx::SourceSurface* Surface() const { return mSurface; }
ValuesSurfaceMemoryCounter105   MemoryCounter& Values() { return mValues; }
ValuesSurfaceMemoryCounter106   const MemoryCounter& Values() const { return mValues; }
TypeSurfaceMemoryCounter107   SurfaceMemoryCounterType Type() const { return mType; }
IsLockedSurfaceMemoryCounter108   bool IsLocked() const { return mIsLocked; }
CannotSubstituteSurfaceMemoryCounter109   bool CannotSubstitute() const { return mCannotSubstitute; }
IsFactor2SurfaceMemoryCounter110   bool IsFactor2() const { return mIsFactor2; }
IsFinishedSurfaceMemoryCounter111   bool IsFinished() const { return mFinished; }
112 
113  private:
114   const SurfaceKey mKey;
115   const gfx::SourceSurface* MOZ_NON_OWNING_REF mSurface;
116   MemoryCounter mValues;
117   const SurfaceMemoryCounterType mType;
118   const bool mIsLocked;
119   const bool mCannotSubstitute;
120   const bool mIsFactor2;
121   const bool mFinished;
122 };
123 
124 struct ImageMemoryCounter {
125   ImageMemoryCounter(imgRequest* aRequest, SizeOfState& aState, bool aIsUsed);
126   ImageMemoryCounter(imgRequest* aRequest, Image* aImage, SizeOfState& aState,
127                      bool aIsUsed);
128 
URIImageMemoryCounter129   nsCString& URI() { return mURI; }
URIImageMemoryCounter130   const nsCString& URI() const { return mURI; }
SurfacesImageMemoryCounter131   const nsTArray<SurfaceMemoryCounter>& Surfaces() const { return mSurfaces; }
IntrinsicSizeImageMemoryCounter132   const gfx::IntSize IntrinsicSize() const { return mIntrinsicSize; }
ValuesImageMemoryCounter133   const MemoryCounter& Values() const { return mValues; }
ProgressImageMemoryCounter134   uint32_t Progress() const { return mProgress; }
TypeImageMemoryCounter135   uint16_t Type() const { return mType; }
IsUsedImageMemoryCounter136   bool IsUsed() const { return mIsUsed; }
HasErrorImageMemoryCounter137   bool HasError() const { return mHasError; }
IsValidatingImageMemoryCounter138   bool IsValidating() const { return mValidating; }
139 
IsNotableImageMemoryCounter140   bool IsNotable() const {
141     // Errors or requests without images are always notable.
142     if (mHasError || mValidating || mProgress == UINT32_MAX ||
143         mProgress & FLAG_HAS_ERROR || mType == imgIContainer::TYPE_REQUEST) {
144       return true;
145     }
146 
147     // Sufficiently large images are notable.
148     const size_t NotableThreshold = 16 * 1024;
149     size_t total = mValues.Source() + mValues.DecodedHeap() +
150                    mValues.DecodedNonHeap() + mValues.DecodedUnknown();
151     if (total >= NotableThreshold) {
152       return true;
153     }
154 
155     // Incomplete images are always notable as well; the odds of capturing
156     // mid-decode should be fairly low.
157     for (const auto& surface : mSurfaces) {
158       if (!surface.IsFinished()) {
159         return true;
160       }
161     }
162 
163     return false;
164   }
165 
166  private:
167   nsCString mURI;
168   nsTArray<SurfaceMemoryCounter> mSurfaces;
169   gfx::IntSize mIntrinsicSize;
170   MemoryCounter mValues;
171   uint32_t mProgress;
172   uint16_t mType;
173   const bool mIsUsed;
174   bool mHasError;
175   bool mValidating;
176 };
177 
178 ///////////////////////////////////////////////////////////////////////////////
179 // Image Base Types
180 ///////////////////////////////////////////////////////////////////////////////
181 
182 class Image : public imgIContainer {
183  public:
184   /**
185    * Flags for Image initialization.
186    *
187    * Meanings:
188    *
189    * INIT_FLAG_NONE: Lack of flags
190    *
191    * INIT_FLAG_DISCARDABLE: The container should be discardable
192    *
193    * INIT_FLAG_DECODE_IMMEDIATELY: The container should decode as soon as
194    * possible, regardless of what our heuristics say.
195    *
196    * INIT_FLAG_TRANSIENT: The container is likely to exist for only a short time
197    * before being destroyed. (For example, containers for
198    * multipart/x-mixed-replace image parts fall into this category.) If this
199    * flag is set, INIT_FLAG_DISCARDABLE and INIT_FLAG_DECODE_ONLY_ON_DRAW must
200    * not be set.
201    *
202    * INIT_FLAG_SYNC_LOAD: The container is being loaded synchronously, so
203    * it should avoid relying on async workers to get the container ready.
204    */
205   static const uint32_t INIT_FLAG_NONE = 0x0;
206   static const uint32_t INIT_FLAG_DISCARDABLE = 0x1;
207   static const uint32_t INIT_FLAG_DECODE_IMMEDIATELY = 0x2;
208   static const uint32_t INIT_FLAG_TRANSIENT = 0x4;
209   static const uint32_t INIT_FLAG_SYNC_LOAD = 0x8;
210 
211   virtual already_AddRefed<ProgressTracker> GetProgressTracker() = 0;
SetProgressTracker(ProgressTracker * aProgressTracker)212   virtual void SetProgressTracker(ProgressTracker* aProgressTracker) {}
213 
214   /**
215    * The size, in bytes, occupied by the compressed source data of the image.
216    * If MallocSizeOf does not work on this platform, uses a fallback approach to
217    * ensure that something reasonable is always returned.
218    */
219   virtual size_t SizeOfSourceWithComputedFallback(
220       SizeOfState& aState) const = 0;
221 
222   /**
223    * Collect an accounting of the memory occupied by the image's surfaces (which
224    * together make up its decoded data). Each surface is recorded as a separate
225    * SurfaceMemoryCounter, stored in @aCounters.
226    */
227   virtual void CollectSizeOfSurfaces(nsTArray<SurfaceMemoryCounter>& aCounters,
228                                      MallocSizeOf aMallocSizeOf) const = 0;
229 
230   virtual void IncrementAnimationConsumers() = 0;
231   virtual void DecrementAnimationConsumers() = 0;
232 #ifdef DEBUG
233   virtual uint32_t GetAnimationConsumers() = 0;
234 #endif
235 
236   /**
237    * Called from OnDataAvailable when the stream associated with the image has
238    * received new image data. The arguments are the same as OnDataAvailable's,
239    * but by separating this functionality into a different method we don't
240    * interfere with subclasses which wish to implement nsIStreamListener.
241    *
242    * Images should not do anything that could send out notifications until they
243    * have received their first OnImageDataAvailable notification; in
244    * particular, this means that instantiating decoders should be deferred
245    * until OnImageDataAvailable is called.
246    */
247   virtual nsresult OnImageDataAvailable(nsIRequest* aRequest,
248                                         nsISupports* aContext,
249                                         nsIInputStream* aInStr,
250                                         uint64_t aSourceOffset,
251                                         uint32_t aCount) = 0;
252 
253   /**
254    * Called from OnStopRequest when the image's underlying request completes.
255    *
256    * @param aRequest  The completed request.
257    * @param aContext  Context from Necko's OnStopRequest.
258    * @param aStatus   A success or failure code.
259    * @param aLastPart Whether this is the final part of the underlying request.
260    */
261   virtual nsresult OnImageDataComplete(nsIRequest* aRequest,
262                                        nsISupports* aContext, nsresult aStatus,
263                                        bool aLastPart) = 0;
264 
265   /**
266    * Called when the SurfaceCache discards a surface belonging to this image.
267    */
268   virtual void OnSurfaceDiscarded(const SurfaceKey& aSurfaceKey) = 0;
269 
270   virtual void SetInnerWindowID(uint64_t aInnerWindowId) = 0;
271   virtual uint64_t InnerWindowID() const = 0;
272 
273   virtual bool HasError() = 0;
274   virtual void SetHasError() = 0;
275 
276   virtual nsIURI* GetURI() const = 0;
277 
GetHotspotX(int32_t * aX)278   NS_IMETHOD GetHotspotX(int32_t* aX) override {
279     *aX = 0;
280     return NS_OK;
281   }
GetHotspotY(int32_t * aY)282   NS_IMETHOD GetHotspotY(int32_t* aY) override {
283     *aY = 0;
284     return NS_OK;
285   }
286 };
287 
288 class ImageResource : public Image {
289  public:
GetProgressTracker()290   already_AddRefed<ProgressTracker> GetProgressTracker() override {
291     RefPtr<ProgressTracker> progressTracker = mProgressTracker;
292     MOZ_ASSERT(progressTracker);
293     return progressTracker.forget();
294   }
295 
SetProgressTracker(ProgressTracker * aProgressTracker)296   void SetProgressTracker(ProgressTracker* aProgressTracker) final {
297     MOZ_ASSERT(aProgressTracker);
298     MOZ_ASSERT(!mProgressTracker);
299     mProgressTracker = aProgressTracker;
300   }
301 
302   virtual void IncrementAnimationConsumers() override;
303   virtual void DecrementAnimationConsumers() override;
304 #ifdef DEBUG
GetAnimationConsumers()305   virtual uint32_t GetAnimationConsumers() override {
306     return mAnimationConsumers;
307   }
308 #endif
309 
OnSurfaceDiscarded(const SurfaceKey & aSurfaceKey)310   virtual void OnSurfaceDiscarded(const SurfaceKey& aSurfaceKey) override {}
311 
SetInnerWindowID(uint64_t aInnerWindowId)312   virtual void SetInnerWindowID(uint64_t aInnerWindowId) override {
313     mInnerWindowId = aInnerWindowId;
314   }
InnerWindowID()315   virtual uint64_t InnerWindowID() const override { return mInnerWindowId; }
316 
HasError()317   virtual bool HasError() override { return mError; }
SetHasError()318   virtual void SetHasError() override { mError = true; }
319 
320   /*
321    * Returns a non-AddRefed pointer to the URI associated with this image.
322    * Illegal to use off-main-thread.
323    */
GetURI()324   nsIURI* GetURI() const override { return mURI; }
325 
326   /*
327    * Should be called by its subclasses after they populate @aCounters so that
328    * we can cross reference against any of our ImageContainers that contain
329    * surfaces not in the cache.
330    */
331   void CollectSizeOfSurfaces(nsTArray<SurfaceMemoryCounter>& aCounters,
332                              MallocSizeOf aMallocSizeOf) const override;
333 
334  protected:
335   explicit ImageResource(nsIURI* aURI);
336   ~ImageResource();
337 
GetImageProducerId()338   layers::ContainerProducerID GetImageProducerId() const {
339     return mImageProducerID;
340   }
341 
342   bool GetSpecTruncatedTo1k(nsCString& aSpec) const;
343 
344   // Shared functionality for implementors of imgIContainer. Every
345   // implementation of attribute animationMode should forward here.
346   nsresult GetAnimationModeInternal(uint16_t* aAnimationMode);
347   nsresult SetAnimationModeInternal(uint16_t aAnimationMode);
348 
349   /**
350    * Helper for RequestRefresh.
351    *
352    * If we've had a "recent" refresh (i.e. if this image is being used in
353    * multiple documents & some other document *just* called RequestRefresh() on
354    * this image with a timestamp close to aTime), this method returns true.
355    *
356    * Otherwise, this method updates mLastRefreshTime to aTime & returns false.
357    */
358   bool HadRecentRefresh(const TimeStamp& aTime);
359 
360   /**
361    * Decides whether animation should or should not be happening,
362    * and makes sure the right thing is being done.
363    */
364   virtual void EvaluateAnimation();
365 
366   /**
367    * Extended by child classes, if they have additional
368    * conditions for being able to animate.
369    */
ShouldAnimate()370   virtual bool ShouldAnimate() {
371     return mAnimationConsumers > 0 && mAnimationMode != kDontAnimMode;
372   }
373 
374   virtual nsresult StartAnimation() = 0;
375   virtual nsresult StopAnimation() = 0;
376 
377   void SendOnUnlockedDraw(uint32_t aFlags);
378 
379 #ifdef DEBUG
380   // Records the image drawing for startup performance testing.
381   void NotifyDrawingObservers();
382 #endif
383 
384   // Member data shared by all implementations of this abstract class
385   RefPtr<ProgressTracker> mProgressTracker;
386   nsCOMPtr<nsIURI> mURI;
387   TimeStamp mLastRefreshTime;
388   uint64_t mInnerWindowId;
389   uint32_t mAnimationConsumers;
390   uint16_t mAnimationMode;  // Enum values in imgIContainer
391   bool mInitialized : 1;    // Have we been initialized?
392   bool mAnimating : 1;      // Are we currently animating?
393   bool mError : 1;          // Error handling
394 
395   /**
396    * Attempt to find a matching cached surface in the SurfaceCache, and if not
397    * available, request the production of such a surface (either synchronously
398    * or asynchronously).
399    *
400    * If the draw result is BAD_IMAGE, BAD_ARGS or NOT_READY, the size will be
401    * the same as aSize. If it is TEMPORARY_ERROR, INCOMPLETE, or SUCCESS, the
402    * size is a hint as to what we expect the surface size to be, once the best
403    * fitting size is available. It may or may not match the size of the surface
404    * returned at this moment. This is useful for choosing how to store the final
405    * result (e.g. if going into an ImageContainer, ideally we would share the
406    * same container for many requested sizes, if they all end up with the same
407    * best fit size in the end).
408    *
409    * A valid surface should only be returned for SUCCESS and INCOMPLETE.
410    *
411    * Any other draw result is invalid.
412    */
413   virtual Tuple<ImgDrawResult, gfx::IntSize, RefPtr<gfx::SourceSurface>>
GetFrameInternal(const gfx::IntSize & aSize,const Maybe<SVGImageContext> & aSVGContext,const Maybe<ImageIntRegion> & aRegion,uint32_t aWhichFrame,uint32_t aFlags)414   GetFrameInternal(const gfx::IntSize& aSize,
415                    const Maybe<SVGImageContext>& aSVGContext,
416                    const Maybe<ImageIntRegion>& aRegion, uint32_t aWhichFrame,
417                    uint32_t aFlags) {
418     return MakeTuple(ImgDrawResult::BAD_IMAGE, aSize,
419                      RefPtr<gfx::SourceSurface>());
420   }
421 
422   /**
423    * Calculate the estimated size to use for an image container with the given
424    * parameters. It may not be the same as the given size, and it may not be
425    * the same as the size of the surface in the image container, but it is the
426    * best effort estimate.
427    */
GetImageContainerSize(layers::LayerManager * aManager,const gfx::IntSize & aSize,uint32_t aFlags)428   virtual Tuple<ImgDrawResult, gfx::IntSize> GetImageContainerSize(
429       layers::LayerManager* aManager, const gfx::IntSize& aSize,
430       uint32_t aFlags) {
431     return MakeTuple(ImgDrawResult::NOT_SUPPORTED, gfx::IntSize(0, 0));
432   }
433 
434   ImgDrawResult GetImageContainerImpl(layers::LayerManager* aManager,
435                                       const gfx::IntSize& aSize,
436                                       const Maybe<SVGImageContext>& aSVGContext,
437                                       const Maybe<ImageIntRegion>& aRegion,
438                                       uint32_t aFlags,
439                                       layers::ImageContainer** aContainer);
440 
441   /**
442    * Re-requests the appropriate frames for each image container using
443    * GetFrameInternal.
444    * @returns True if any image containers were updated, else false.
445    */
446   bool UpdateImageContainer(const Maybe<gfx::IntRect>& aDirtyRect);
447 
448   void ReleaseImageContainer();
449 
450   class MOZ_RAII AutoProfilerImagePaintMarker {
451    public:
AutoProfilerImagePaintMarker(ImageResource * self)452     explicit AutoProfilerImagePaintMarker(ImageResource* self)
453         : mStartTime(TimeStamp::NowUnfuzzed()) {
454       nsAutoCString spec;
455       if (self->mURI && profiler_can_accept_markers()) {
456         static const size_t sMaxTruncatedLength = 1024;
457         self->mURI->GetSpec(mSpec);
458         if (mSpec.Length() >= sMaxTruncatedLength) {
459           mSpec.Truncate(sMaxTruncatedLength);
460         }
461       }
462     }
463 
~AutoProfilerImagePaintMarker()464     ~AutoProfilerImagePaintMarker() {
465       if (!mSpec.IsEmpty()) {
466         PROFILER_MARKER_TEXT("Image Paint", GRAPHICS,
467                              MarkerTiming::IntervalUntilNowFrom(mStartTime),
468                              mSpec);
469       }
470     }
471 
472    protected:
473     TimeStamp mStartTime;
474     nsAutoCString mSpec;
475   };
476 
477  private:
478   void SetCurrentImage(layers::ImageContainer* aContainer,
479                        gfx::SourceSurface* aSurface,
480                        const Maybe<gfx::IntRect>& aDirtyRect);
481 
482   struct ImageContainerEntry {
ImageContainerEntryImageContainerEntry483     ImageContainerEntry(const gfx::IntSize& aSize,
484                         const Maybe<SVGImageContext>& aSVGContext,
485                         const Maybe<ImageIntRegion>& aRegion,
486                         layers::ImageContainer* aContainer, uint32_t aFlags)
487         : mSize(aSize),
488           mSVGContext(aSVGContext),
489           mRegion(aRegion),
490           mContainer(aContainer),
491           mLastDrawResult(ImgDrawResult::NOT_READY),
492           mFlags(aFlags) {}
493 
494     gfx::IntSize mSize;
495     Maybe<SVGImageContext> mSVGContext;
496     Maybe<ImageIntRegion> mRegion;
497     // A weak pointer to our ImageContainer, which stays alive only as long as
498     // the layer system needs it.
499     ThreadSafeWeakPtr<layers::ImageContainer> mContainer;
500     // If mContainer is non-null, this contains the ImgDrawResult we obtained
501     // the last time we updated it.
502     ImgDrawResult mLastDrawResult;
503     // Cached flags to use for decoding. FLAG_ASYNC_NOTIFY should always be set
504     // but FLAG_HIGH_QUALITY_SCALING may vary.
505     uint32_t mFlags;
506   };
507 
508   AutoTArray<ImageContainerEntry, 1> mImageContainers;
509   layers::ImageContainer::ProducerID mImageProducerID;
510   layers::ImageContainer::FrameID mLastFrameID;
511 };
512 
513 }  // namespace image
514 }  // namespace mozilla
515 
516 #endif  // mozilla_image_Image_h
517