1 /* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * This Source Code Form is subject to the terms of the Mozilla Public 3 * License, v. 2.0. If a copy of the MPL was not distributed with this 4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 5 6 #ifndef GFX_IMAGECONTAINER_H 7 #define GFX_IMAGECONTAINER_H 8 9 #include <stdint.h> // for uint32_t, uint8_t, uint64_t 10 #include <sys/types.h> // for int32_t 11 #include "gfxTypes.h" 12 #include "ImageTypes.h" // for ImageFormat, etc 13 #include "mozilla/Assertions.h" // for MOZ_ASSERT_HELPER2 14 #include "mozilla/Mutex.h" // for Mutex 15 #include "mozilla/ReentrantMonitor.h" // for ReentrantMonitorAutoEnter, etc 16 #include "mozilla/TimeStamp.h" // for TimeStamp 17 #include "mozilla/gfx/Point.h" // For IntSize 18 #include "mozilla/layers/GonkNativeHandle.h" 19 #include "mozilla/layers/LayersTypes.h" // for LayersBackend, etc 20 #include "mozilla/layers/CompositorTypes.h" 21 #include "mozilla/mozalloc.h" // for operator delete, etc 22 #include "nsAutoPtr.h" // for nsRefPtr, nsAutoArrayPtr, etc 23 #include "nsAutoRef.h" // for nsCountedRef 24 #include "nsCOMPtr.h" // for already_AddRefed 25 #include "nsDebug.h" // for NS_ASSERTION 26 #include "nsISupportsImpl.h" // for Image::Release, etc 27 #include "nsRect.h" // for mozilla::gfx::IntRect 28 #include "nsTArray.h" // for nsTArray 29 #include "mozilla/Atomics.h" 30 #include "mozilla/WeakPtr.h" 31 #include "nsThreadUtils.h" 32 #include "mozilla/gfx/2D.h" 33 #include "nsDataHashtable.h" 34 #include "mozilla/EnumeratedArray.h" 35 #include "mozilla/UniquePtr.h" 36 37 #ifndef XPCOM_GLUE_AVOID_NSPR 38 /** 39 * We need to be able to hold a reference to a Moz2D SourceSurface from Image 40 * subclasses. This is potentially a problem since Images can be addrefed 41 * or released off the main thread. We can ensure that we never AddRef 42 * a SourceSurface off the main thread, but we might want to Release due 43 * to an Image being destroyed off the main thread. 44 * 45 * We use nsCountedRef<nsMainThreadSourceSurfaceRef> to reference the 46 * SourceSurface. When AddRefing, we assert that we're on the main thread. 47 * When Releasing, if we're not on the main thread, we post an event to 48 * the main thread to do the actual release. 49 */ 50 class nsMainThreadSourceSurfaceRef; 51 52 template <> 53 class nsAutoRefTraits<nsMainThreadSourceSurfaceRef> { 54 public: 55 typedef mozilla::gfx::SourceSurface* RawRef; 56 57 /** 58 * The XPCOM event that will do the actual release on the main thread. 59 */ 60 class SurfaceReleaser : public mozilla::Runnable { 61 public: SurfaceReleaser(RawRef aRef)62 explicit SurfaceReleaser(RawRef aRef) : mRef(aRef) {} Run()63 NS_IMETHOD Run() override { 64 mRef->Release(); 65 return NS_OK; 66 } 67 RawRef mRef; 68 }; 69 Void()70 static RawRef Void() { return nullptr; } Release(RawRef aRawRef)71 static void Release(RawRef aRawRef) 72 { 73 if (NS_IsMainThread()) { 74 aRawRef->Release(); 75 return; 76 } 77 nsCOMPtr<nsIRunnable> runnable = new SurfaceReleaser(aRawRef); 78 NS_DispatchToMainThread(runnable); 79 } AddRef(RawRef aRawRef)80 static void AddRef(RawRef aRawRef) 81 { 82 NS_ASSERTION(NS_IsMainThread(), 83 "Can only add a reference on the main thread"); 84 aRawRef->AddRef(); 85 } 86 }; 87 88 class nsOwningThreadSourceSurfaceRef; 89 90 template <> 91 class nsAutoRefTraits<nsOwningThreadSourceSurfaceRef> { 92 public: 93 typedef mozilla::gfx::SourceSurface* RawRef; 94 95 /** 96 * The XPCOM event that will do the actual release on the creation thread. 97 */ 98 class SurfaceReleaser : public mozilla::Runnable { 99 public: SurfaceReleaser(RawRef aRef)100 explicit SurfaceReleaser(RawRef aRef) : mRef(aRef) {} Run()101 NS_IMETHOD Run() override { 102 mRef->Release(); 103 return NS_OK; 104 } 105 RawRef mRef; 106 }; 107 Void()108 static RawRef Void() { return nullptr; } Release(RawRef aRawRef)109 void Release(RawRef aRawRef) 110 { 111 MOZ_ASSERT(mOwningThread); 112 bool current; 113 mOwningThread->IsOnCurrentThread(¤t); 114 if (current) { 115 aRawRef->Release(); 116 return; 117 } 118 nsCOMPtr<nsIRunnable> runnable = new SurfaceReleaser(aRawRef); 119 mOwningThread->Dispatch(runnable, nsIThread::DISPATCH_NORMAL); 120 } AddRef(RawRef aRawRef)121 void AddRef(RawRef aRawRef) 122 { 123 MOZ_ASSERT(!mOwningThread); 124 NS_GetCurrentThread(getter_AddRefs(mOwningThread)); 125 aRawRef->AddRef(); 126 } 127 128 private: 129 nsCOMPtr<nsIThread> mOwningThread; 130 }; 131 132 #endif 133 134 #ifdef XP_WIN 135 struct ID3D10Texture2D; 136 struct ID3D10Device; 137 struct ID3D10ShaderResourceView; 138 #endif 139 140 typedef void* HANDLE; 141 142 namespace mozilla { 143 144 145 namespace layers { 146 147 class ImageClient; 148 class ImageCompositeNotification; 149 class ImageContainerChild; 150 class PImageContainerChild; 151 class SharedPlanarYCbCrImage; 152 class PlanarYCbCrImage; 153 class TextureClient; 154 class KnowsCompositor; 155 class NVImage; 156 157 struct ImageBackendData 158 { ~ImageBackendDataImageBackendData159 virtual ~ImageBackendData() {} 160 161 protected: ImageBackendDataImageBackendData162 ImageBackendData() {} 163 }; 164 165 /* Forward declarations for Image derivatives. */ 166 class GLImage; 167 class EGLImageImage; 168 class SharedRGBImage; 169 #ifdef MOZ_WIDGET_ANDROID 170 class SurfaceTextureImage; 171 #elif defined(XP_MACOSX) 172 class MacIOSurfaceImage; 173 #endif 174 175 /** 176 * A class representing a buffer of pixel data. The data can be in one 177 * of various formats including YCbCr. 178 * 179 * Create an image using an ImageContainer. Fill the image with data, and 180 * then call ImageContainer::SetImage to display it. An image must not be 181 * modified after calling SetImage. Image implementations do not need to 182 * perform locking; when filling an Image, the Image client is responsible 183 * for ensuring only one thread accesses the Image at a time, and after 184 * SetImage the image is immutable. 185 * 186 * When resampling an Image, only pixels within the buffer should be 187 * sampled. For example, cairo images should be sampled in EXTEND_PAD mode. 188 */ 189 class Image { NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Image)190 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Image) 191 192 public: 193 ImageFormat GetFormat() { return mFormat; } GetImplData()194 void* GetImplData() { return mImplData; } 195 196 virtual gfx::IntSize GetSize() = 0; GetOrigin()197 virtual gfx::IntPoint GetOrigin() 198 { 199 return gfx::IntPoint(0, 0); 200 } GetPictureRect()201 virtual gfx::IntRect GetPictureRect() 202 { 203 return gfx::IntRect(GetOrigin().x, GetOrigin().y, GetSize().width, GetSize().height); 204 } 205 GetBackendData(LayersBackend aBackend)206 ImageBackendData* GetBackendData(LayersBackend aBackend) 207 { return mBackendData[aBackend]; } SetBackendData(LayersBackend aBackend,ImageBackendData * aData)208 void SetBackendData(LayersBackend aBackend, ImageBackendData* aData) 209 { mBackendData[aBackend] = aData; } 210 GetSerial()211 int32_t GetSerial() { return mSerial; } 212 213 virtual already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() = 0; 214 IsValid()215 virtual bool IsValid() { return true; } 216 GetBuffer()217 virtual uint8_t* GetBuffer() { return nullptr; } 218 219 /** 220 * For use with the TextureForwarder only (so that the later can 221 * synchronize the TextureClient with the TextureHost). 222 */ GetTextureClient(KnowsCompositor * aForwarder)223 virtual TextureClient* GetTextureClient(KnowsCompositor* aForwarder) { return nullptr; } 224 225 /* Access to derived classes. */ AsEGLImageImage()226 virtual EGLImageImage* AsEGLImageImage() { return nullptr; } AsGLImage()227 virtual GLImage* AsGLImage() { return nullptr; } 228 #ifdef MOZ_WIDGET_ANDROID AsSurfaceTextureImage()229 virtual SurfaceTextureImage* AsSurfaceTextureImage() { return nullptr; } 230 #endif 231 #ifdef XP_MACOSX AsMacIOSurfaceImage()232 virtual MacIOSurfaceImage* AsMacIOSurfaceImage() { return nullptr; } 233 #endif AsPlanarYCbCrImage()234 virtual PlanarYCbCrImage* AsPlanarYCbCrImage() { return nullptr; } 235 AsNVImage()236 virtual NVImage* AsNVImage() { return nullptr; } 237 238 protected: Image(void * aImplData,ImageFormat aFormat)239 Image(void* aImplData, ImageFormat aFormat) : 240 mImplData(aImplData), 241 mSerial(++sSerialCounter), 242 mFormat(aFormat) 243 {} 244 245 // Protected destructor, to discourage deletion outside of Release(): ~Image()246 virtual ~Image() {} 247 248 mozilla::EnumeratedArray<mozilla::layers::LayersBackend, 249 mozilla::layers::LayersBackend::LAYERS_LAST, 250 nsAutoPtr<ImageBackendData>> 251 mBackendData; 252 253 void* mImplData; 254 int32_t mSerial; 255 ImageFormat mFormat; 256 257 static mozilla::Atomic<int32_t> sSerialCounter; 258 }; 259 260 /** 261 * A RecycleBin is owned by an ImageContainer. We store buffers in it that we 262 * want to recycle from one image to the next.It's a separate object from 263 * ImageContainer because images need to store a strong ref to their RecycleBin 264 * and we must avoid creating a reference loop between an ImageContainer and 265 * its active image. 266 */ 267 class BufferRecycleBin final { 268 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(BufferRecycleBin) 269 270 //typedef mozilla::gl::GLContext GLContext; 271 272 public: 273 BufferRecycleBin(); 274 275 void RecycleBuffer(mozilla::UniquePtr<uint8_t[]> aBuffer, uint32_t aSize); 276 // Returns a recycled buffer of the right size, or allocates a new buffer. 277 mozilla::UniquePtr<uint8_t[]> GetBuffer(uint32_t aSize); 278 virtual void ClearRecycledBuffers(); 279 private: 280 typedef mozilla::Mutex Mutex; 281 282 // Private destructor, to discourage deletion outside of Release(): ~BufferRecycleBin()283 ~BufferRecycleBin() 284 { 285 } 286 287 // This protects mRecycledBuffers, mRecycledBufferSize, mRecycledTextures 288 // and mRecycledTextureSizes 289 Mutex mLock; 290 291 // We should probably do something to prune this list on a timer so we don't 292 // eat excess memory while video is paused... 293 nsTArray<mozilla::UniquePtr<uint8_t[]>> mRecycledBuffers; 294 // This is only valid if mRecycledBuffers is non-empty 295 uint32_t mRecycledBufferSize; 296 }; 297 298 /** 299 * A class that manages Image creation for a LayerManager. The only reason 300 * we need a separate class here is that LayerManagers aren't threadsafe 301 * (because layers can only be used on the main thread) and we want to 302 * be able to create images from any thread, to facilitate video playback 303 * without involving the main thread, for example. 304 * Different layer managers can implement child classes of this making it 305 * possible to create layer manager specific images. 306 * This class is not meant to be used directly but rather can be set on an 307 * image container. This is usually done by the layer system internally and 308 * not explicitly by users. For PlanarYCbCr or Cairo images the default 309 * implementation will creates images whose data lives in system memory, for 310 * MacIOSurfaces the default implementation will be a simple MacIOSurface 311 * wrapper. 312 */ 313 314 class ImageFactory 315 { NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ImageFactory)316 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ImageFactory) 317 protected: 318 friend class ImageContainer; 319 320 ImageFactory() {} ~ImageFactory()321 virtual ~ImageFactory() {} 322 323 virtual RefPtr<PlanarYCbCrImage> CreatePlanarYCbCrImage( 324 const gfx::IntSize& aScaleHint, 325 BufferRecycleBin *aRecycleBin); 326 }; 327 328 /** 329 * A class that manages Images for an ImageLayer. The only reason 330 * we need a separate class here is that ImageLayers aren't threadsafe 331 * (because layers can only be used on the main thread) and we want to 332 * be able to set the current Image from any thread, to facilitate 333 * video playback without involving the main thread, for example. 334 * 335 * An ImageContainer can operate in one of these modes: 336 * 1) Normal. Triggered by constructing the ImageContainer with 337 * DISABLE_ASYNC or when compositing is happening on the main thread. 338 * SetCurrentImages changes ImageContainer state but nothing is sent to the 339 * compositor until the next layer transaction. 340 * 2) Asynchronous. Initiated by constructing the ImageContainer with 341 * ENABLE_ASYNC when compositing is happening on the main thread. 342 * SetCurrentImages sends a message through the ImageBridge to the compositor 343 * thread to update the image, without going through the main thread or 344 * a layer transaction. 345 * The ImageContainer uses a shared memory block containing a cross-process mutex 346 * to communicate with the compositor thread. SetCurrentImage synchronously 347 * updates the shared state to point to the new image and the old image 348 * is immediately released (not true in Normal or Asynchronous modes). 349 */ 350 class ImageContainer final : public SupportsWeakPtr<ImageContainer> 351 { 352 friend class ImageContainerChild; 353 354 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ImageContainer) 355 356 public: 357 MOZ_DECLARE_WEAKREFERENCE_TYPENAME(ImageContainer) 358 359 enum Mode { SYNCHRONOUS = 0x0, ASYNCHRONOUS = 0x01 }; 360 361 static const uint64_t sInvalidAsyncContainerId = 0; 362 363 explicit ImageContainer(ImageContainer::Mode flag = SYNCHRONOUS); 364 365 /** 366 * Create ImageContainer just to hold another ASYNCHRONOUS ImageContainer's 367 * async container ID. 368 * @param aAsyncContainerID async container ID for which we are a proxy 369 */ 370 explicit ImageContainer(uint64_t aAsyncContainerID); 371 372 typedef uint32_t FrameID; 373 typedef uint32_t ProducerID; 374 375 RefPtr<PlanarYCbCrImage> CreatePlanarYCbCrImage(); 376 377 // Factory methods for shared image types. 378 RefPtr<SharedRGBImage> CreateSharedRGBImage(); 379 380 struct NonOwningImage { 381 explicit NonOwningImage(Image* aImage = nullptr, 382 TimeStamp aTimeStamp = TimeStamp(), 383 FrameID aFrameID = 0, 384 ProducerID aProducerID = 0) mImageNonOwningImage385 : mImage(aImage), mTimeStamp(aTimeStamp), mFrameID(aFrameID), 386 mProducerID(aProducerID) {} 387 Image* mImage; 388 TimeStamp mTimeStamp; 389 FrameID mFrameID; 390 ProducerID mProducerID; 391 }; 392 /** 393 * Set aImages as the list of timestamped to display. The Images must have 394 * been created by this ImageContainer. 395 * Can be called on any thread. This method takes mReentrantMonitor 396 * when accessing thread-shared state. 397 * aImages must be non-empty. The first timestamp in the list may be 398 * null but the others must not be, and the timestamps must increase. 399 * Every element of aImages must have non-null mImage. 400 * mFrameID can be zero, in which case you won't get meaningful 401 * painted/dropped frame counts. Otherwise you should use a unique and 402 * increasing ID for each decoded and submitted frame (but it's OK to 403 * pass the same frame to SetCurrentImages). 404 * mProducerID is a unique ID for the stream of images. A change in the 405 * mProducerID means changing to a new mFrameID namespace. All frames in 406 * aImages must have the same mProducerID. 407 * 408 * The Image data must not be modified after this method is called! 409 * Note that this must not be called if ENABLE_ASYNC has not been set. 410 * 411 * The implementation calls CurrentImageChanged() while holding 412 * mReentrantMonitor. 413 * 414 * If this ImageContainer has an ImageClient for async video: 415 * Schedule a task to send the image to the compositor using the 416 * PImageBridge protcol without using the main thread. 417 */ 418 void SetCurrentImages(const nsTArray<NonOwningImage>& aImages); 419 420 /** 421 * Clear all images. Let ImageClient release all TextureClients. 422 */ 423 void ClearAllImages(); 424 425 /** 426 * Clear any resources that are not immediately necessary. This may be called 427 * in low-memory conditions. 428 */ 429 void ClearCachedResources(); 430 431 /** 432 * Clear the current images. 433 * This function is expect to be called only from a CompositableClient 434 * that belongs to ImageBridgeChild. Created to prevent dead lock. 435 * See Bug 901224. 436 */ 437 void ClearImagesFromImageBridge(); 438 439 /** 440 * Set an Image as the current image to display. The Image must have 441 * been created by this ImageContainer. 442 * Must be called on the main thread, within a layers transaction. 443 * 444 * This method takes mReentrantMonitor 445 * when accessing thread-shared state. 446 * aImage can be null. While it's null, nothing will be painted. 447 * 448 * The Image data must not be modified after this method is called! 449 * Note that this must not be called if ENABLE_ASYNC been set. 450 * 451 * You won't get meaningful painted/dropped counts when using this method. 452 */ 453 void SetCurrentImageInTransaction(Image* aImage); 454 void SetCurrentImagesInTransaction(const nsTArray<NonOwningImage>& aImages); 455 456 /** 457 * Returns true if this ImageContainer uses the ImageBridge IPDL protocol. 458 * 459 * Can be called from any thread. 460 */ 461 bool IsAsync() const; 462 463 /** 464 * If this ImageContainer uses ImageBridge, returns the ID associated to 465 * this container, for use in the ImageBridge protocol. 466 * Returns 0 if this ImageContainer does not use ImageBridge. Note that 467 * 0 is always an invalid ID for asynchronous image containers. 468 * 469 * Can be called from any thread. 470 */ 471 uint64_t GetAsyncContainerID(); 472 473 /** 474 * Returns if the container currently has an image. 475 * Can be called on any thread. This method takes mReentrantMonitor 476 * when accessing thread-shared state. 477 */ 478 bool HasCurrentImage(); 479 480 struct OwningImage { OwningImageOwningImage481 OwningImage() : mFrameID(0), mProducerID(0), mComposited(false) {} 482 RefPtr<Image> mImage; 483 TimeStamp mTimeStamp; 484 FrameID mFrameID; 485 ProducerID mProducerID; 486 bool mComposited; 487 }; 488 /** 489 * Copy the current Image list to aImages. 490 * This has to add references since otherwise there are race conditions 491 * where the current image is destroyed before the caller can add 492 * a reference. 493 * Can be called on any thread. 494 * May return an empty list to indicate there is no current image. 495 * If aGenerationCounter is non-null, sets *aGenerationCounter to a value 496 * that's unique for this ImageContainer state. 497 */ 498 void GetCurrentImages(nsTArray<OwningImage>* aImages, 499 uint32_t* aGenerationCounter = nullptr); 500 501 /** 502 * Returns the size of the image in pixels. 503 * Can be called on any thread. This method takes mReentrantMonitor when accessing 504 * thread-shared state. 505 */ 506 gfx::IntSize GetCurrentSize(); 507 508 /** 509 * Sets a size that the image is expected to be rendered at. 510 * This is a hint for image backends to optimize scaling. 511 * Default implementation in this class is to ignore the hint. 512 * Can be called on any thread. This method takes mReentrantMonitor 513 * when accessing thread-shared state. 514 */ SetScaleHint(const gfx::IntSize & aScaleHint)515 void SetScaleHint(const gfx::IntSize& aScaleHint) 516 { mScaleHint = aScaleHint; } 517 SetImageFactory(ImageFactory * aFactory)518 void SetImageFactory(ImageFactory *aFactory) 519 { 520 ReentrantMonitorAutoEnter mon(mReentrantMonitor); 521 mImageFactory = aFactory ? aFactory : new ImageFactory(); 522 } 523 GetImageFactory()524 ImageFactory* GetImageFactory() const 525 { 526 return mImageFactory; 527 } 528 529 /** 530 * Returns the delay between the last composited image's presentation 531 * timestamp and when it was first composited. It's possible for the delay 532 * to be negative if the first image in the list passed to SetCurrentImages 533 * has a presentation timestamp greater than "now". 534 * Returns 0 if the composited image had a null timestamp, or if no 535 * image has been composited yet. 536 */ GetPaintDelay()537 TimeDuration GetPaintDelay() 538 { 539 ReentrantMonitorAutoEnter mon(mReentrantMonitor); 540 return mPaintDelay; 541 } 542 543 /** 544 * Returns the number of images which have been contained in this container 545 * and painted at least once. Can be called from any thread. 546 */ GetPaintCount()547 uint32_t GetPaintCount() { 548 ReentrantMonitorAutoEnter mon(mReentrantMonitor); 549 return mPaintCount; 550 } 551 552 /** 553 * An entry in the current image list "expires" when the entry has an 554 * non-null timestamp, and in a SetCurrentImages call the new image list is 555 * non-empty, the timestamp of the first new image is non-null and greater 556 * than the timestamp associated with the image, and the first new image's 557 * frameID is not the same as the entry's. 558 * Every expired image that is never composited is counted as dropped. 559 */ GetDroppedImageCount()560 uint32_t GetDroppedImageCount() 561 { 562 ReentrantMonitorAutoEnter mon(mReentrantMonitor); 563 return mDroppedImageCount; 564 } 565 566 PImageContainerChild* GetPImageContainerChild(); 567 568 /** 569 * Main thread only. 570 */ 571 static ProducerID AllocateProducerID(); 572 573 private: 574 typedef mozilla::ReentrantMonitor ReentrantMonitor; 575 576 // Private destructor, to discourage deletion outside of Release(): 577 ~ImageContainer(); 578 579 void SetCurrentImageInternal(const nsTArray<NonOwningImage>& aImages); 580 581 // This is called to ensure we have an active image, this may not be true 582 // when we're storing image information in a RemoteImageData structure. 583 // NOTE: If we have remote data mRemoteDataMutex should be locked when 584 // calling this function! 585 void EnsureActiveImage(); 586 587 void EnsureImageClient(bool aCreate); 588 589 void NotifyCompositeInternal(const ImageCompositeNotification& aNotification); 590 591 // ReentrantMonitor to protect thread safe access to the "current 592 // image", and any other state which is shared between threads. 593 ReentrantMonitor mReentrantMonitor; 594 595 nsTArray<OwningImage> mCurrentImages; 596 597 // Updates every time mActiveImage changes 598 uint32_t mGenerationCounter; 599 600 // Number of contained images that have been painted at least once. It's up 601 // to the ImageContainer implementation to ensure accesses to this are 602 // threadsafe. 603 uint32_t mPaintCount; 604 605 // See GetPaintDelay. Accessed only with mReentrantMonitor held. 606 TimeDuration mPaintDelay; 607 608 // See GetDroppedImageCount. Accessed only with mReentrantMonitor held. 609 uint32_t mDroppedImageCount; 610 611 // This is the image factory used by this container, layer managers using 612 // this container can set an alternative image factory that will be used to 613 // create images for this container. 614 RefPtr<ImageFactory> mImageFactory; 615 616 gfx::IntSize mScaleHint; 617 618 RefPtr<BufferRecycleBin> mRecycleBin; 619 620 // This member points to an ImageClient if this ImageContainer was 621 // sucessfully created with ENABLE_ASYNC, or points to null otherwise. 622 // 'unsuccessful' in this case only means that the ImageClient could not 623 // be created, most likely because off-main-thread compositing is not enabled. 624 // In this case the ImageContainer is perfectly usable, but it will forward 625 // frames to the compositor through transactions in the main thread rather than 626 // asynchronusly using the ImageBridge IPDL protocol. 627 RefPtr<ImageClient> mImageClient; 628 629 uint64_t mAsyncContainerID; 630 631 nsTArray<FrameID> mFrameIDsNotYetComposited; 632 // ProducerID for last current image(s), including the frames in 633 // mFrameIDsNotYetComposited 634 ProducerID mCurrentProducerID; 635 636 // Object must be released on the ImageBridge thread. Field is immutable 637 // after creation of the ImageContainer. 638 RefPtr<ImageContainerChild> mIPDLChild; 639 640 static mozilla::Atomic<uint32_t> sGenerationCounter; 641 }; 642 643 class AutoLockImage 644 { 645 public: AutoLockImage(ImageContainer * aContainer)646 explicit AutoLockImage(ImageContainer *aContainer) 647 { 648 aContainer->GetCurrentImages(&mImages); 649 } 650 HasImage()651 bool HasImage() const { return !mImages.IsEmpty(); } GetImage()652 Image* GetImage() const 653 { 654 return mImages.IsEmpty() ? nullptr : mImages[0].mImage.get(); 655 } 656 657 private: 658 AutoTArray<ImageContainer::OwningImage,4> mImages; 659 }; 660 661 struct PlanarYCbCrData { 662 // Luminance buffer 663 uint8_t* mYChannel; 664 int32_t mYStride; 665 gfx::IntSize mYSize; 666 int32_t mYSkip; 667 // Chroma buffers 668 uint8_t* mCbChannel; 669 uint8_t* mCrChannel; 670 int32_t mCbCrStride; 671 gfx::IntSize mCbCrSize; 672 int32_t mCbSkip; 673 int32_t mCrSkip; 674 // Picture region 675 uint32_t mPicX; 676 uint32_t mPicY; 677 gfx::IntSize mPicSize; 678 StereoMode mStereoMode; 679 YUVColorSpace mYUVColorSpace; 680 GetPictureRectPlanarYCbCrData681 gfx::IntRect GetPictureRect() const { 682 return gfx::IntRect(mPicX, mPicY, 683 mPicSize.width, 684 mPicSize.height); 685 } 686 PlanarYCbCrDataPlanarYCbCrData687 PlanarYCbCrData() 688 : mYChannel(nullptr), mYStride(0), mYSize(0, 0), mYSkip(0) 689 , mCbChannel(nullptr), mCrChannel(nullptr) 690 , mCbCrStride(0), mCbCrSize(0, 0) , mCbSkip(0), mCrSkip(0) 691 , mPicX(0), mPicY(0), mPicSize(0, 0), mStereoMode(StereoMode::MONO) 692 , mYUVColorSpace(YUVColorSpace::BT601) 693 {} 694 }; 695 696 /****** Image subtypes for the different formats ******/ 697 698 /** 699 * We assume that the image data is in the REC 470M color space (see 700 * Theora specification, section 4.3.1). 701 * 702 * The YCbCr format can be: 703 * 704 * 4:4:4 - CbCr width/height are the same as Y. 705 * 4:2:2 - CbCr width is half that of Y. Height is the same. 706 * 4:2:0 - CbCr width and height is half that of Y. 707 * 708 * The color format is detected based on the height/width ratios 709 * defined above. 710 * 711 * The Image that is rendered is the picture region defined by 712 * mPicX, mPicY and mPicSize. The size of the rendered image is 713 * mPicSize, not mYSize or mCbCrSize. 714 * 715 * mYSkip, mCbSkip, mCrSkip are added to support various output 716 * formats from hardware decoder. They are per-pixel skips in the 717 * source image. 718 * 719 * For example when image width is 640, mYStride is 670, mYSkip is 3, 720 * the mYChannel buffer looks like: 721 * 722 * |<----------------------- mYStride ----------------------------->| 723 * |<----------------- mYSize.width --------------->| 724 * 0 3 6 9 12 15 18 21 659 669 725 * |----------------------------------------------------------------| 726 * |Y___Y___Y___Y___Y___Y___Y___Y... |%%%%%%%%%| 727 * |Y___Y___Y___Y___Y___Y___Y___Y... |%%%%%%%%%| 728 * |Y___Y___Y___Y___Y___Y___Y___Y... |%%%%%%%%%| 729 * | |<->| 730 * mYSkip 731 */ 732 class PlanarYCbCrImage : public Image { 733 public: 734 typedef PlanarYCbCrData Data; 735 736 enum { 737 MAX_DIMENSION = 16384 738 }; 739 ~PlanarYCbCrImage()740 virtual ~PlanarYCbCrImage() {} 741 742 /** 743 * This makes a copy of the data buffers, in order to support functioning 744 * in all different layer managers. 745 */ 746 virtual bool CopyData(const Data& aData) = 0; 747 748 /** 749 * This doesn't make a copy of the data buffers. Can be used when mBuffer is 750 * pre allocated with AllocateAndGetNewBuffer(size) and then AdoptData is 751 * called to only update the picture size, planes etc. fields in mData. 752 * The GStreamer media backend uses this to decode into PlanarYCbCrImage(s) 753 * directly. 754 */ 755 virtual bool AdoptData(const Data &aData); 756 757 /** 758 * This allocates and returns a new buffer 759 */ 760 virtual uint8_t* AllocateAndGetNewBuffer(uint32_t aSize) = 0; 761 762 /** 763 * Ask this Image to not convert YUV to RGB during SetData, and make 764 * the original data available through GetData. This is optional, 765 * and not all PlanarYCbCrImages will support it. 766 */ SetDelayedConversion(bool aDelayed)767 virtual void SetDelayedConversion(bool aDelayed) { } 768 769 /** 770 * Grab the original YUV data. This is optional. 771 */ GetData()772 virtual const Data* GetData() { return &mData; } 773 774 /** 775 * Return the number of bytes of heap memory used to store this image. 776 */ GetDataSize()777 virtual uint32_t GetDataSize() { return mBufferSize; } 778 IsValid()779 virtual bool IsValid() { return !!mBufferSize; } 780 GetSize()781 virtual gfx::IntSize GetSize() { return mSize; } 782 GetOrigin()783 virtual gfx::IntPoint GetOrigin() { return mOrigin; } 784 785 explicit PlanarYCbCrImage(); 786 AsSharedPlanarYCbCrImage()787 virtual SharedPlanarYCbCrImage *AsSharedPlanarYCbCrImage() { return nullptr; } 788 SizeOfIncludingThis(MallocSizeOf aMallocSizeOf)789 virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { 790 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); 791 } 792 793 virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const = 0; 794 AsPlanarYCbCrImage()795 PlanarYCbCrImage* AsPlanarYCbCrImage() { return this; } 796 797 protected: 798 already_AddRefed<gfx::SourceSurface> GetAsSourceSurface(); 799 SetOffscreenFormat(gfxImageFormat aFormat)800 void SetOffscreenFormat(gfxImageFormat aFormat) { mOffscreenFormat = aFormat; } 801 gfxImageFormat GetOffscreenFormat(); 802 803 Data mData; 804 gfx::IntPoint mOrigin; 805 gfx::IntSize mSize; 806 gfxImageFormat mOffscreenFormat; 807 nsCountedRef<nsMainThreadSourceSurfaceRef> mSourceSurface; 808 uint32_t mBufferSize; 809 }; 810 811 class RecyclingPlanarYCbCrImage: public PlanarYCbCrImage { 812 public: RecyclingPlanarYCbCrImage(BufferRecycleBin * aRecycleBin)813 explicit RecyclingPlanarYCbCrImage(BufferRecycleBin *aRecycleBin) : mRecycleBin(aRecycleBin) {} 814 virtual ~RecyclingPlanarYCbCrImage() override; 815 virtual bool CopyData(const Data& aData) override; 816 virtual uint8_t* AllocateAndGetNewBuffer(uint32_t aSize) override; 817 virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override; 818 protected: 819 820 /** 821 * Return a buffer to store image data in. 822 */ 823 mozilla::UniquePtr<uint8_t[]> AllocateBuffer(uint32_t aSize); 824 825 RefPtr<BufferRecycleBin> mRecycleBin; 826 mozilla::UniquePtr<uint8_t[]> mBuffer; 827 }; 828 829 /** 830 * NVImage is used to store YUV420SP_NV12 and YUV420SP_NV21 data natively, which 831 * are not supported by PlanarYCbCrImage. (PlanarYCbCrImage only stores YUV444P, 832 * YUV422P and YUV420P, it converts YUV420SP_NV12 and YUV420SP_NV21 data into 833 * YUV420P in its PlanarYCbCrImage::SetData() method.) 834 * 835 * PlanarYCbCrData is able to express all the YUV family and so we keep use it 836 * in NVImage. 837 */ 838 class NVImage: public Image { 839 typedef PlanarYCbCrData Data; 840 841 public: 842 explicit NVImage(); 843 virtual ~NVImage() override; 844 845 // Methods inherited from layers::Image. 846 virtual gfx::IntSize GetSize() override; 847 virtual gfx::IntRect GetPictureRect() override; 848 virtual already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override; 849 virtual bool IsValid() override; 850 virtual NVImage* AsNVImage() override; 851 852 // Methods mimic layers::PlanarYCbCrImage. 853 virtual bool SetData(const Data& aData); 854 virtual const Data* GetData() const; 855 virtual uint32_t GetBufferSize() const; 856 857 protected: 858 859 /** 860 * Return a buffer to store image data in. 861 */ 862 mozilla::UniquePtr<uint8_t> AllocateBuffer(uint32_t aSize); 863 864 mozilla::UniquePtr<uint8_t> mBuffer; 865 uint32_t mBufferSize; 866 gfx::IntSize mSize; 867 Data mData; 868 nsCountedRef<nsMainThreadSourceSurfaceRef> mSourceSurface; 869 }; 870 871 /** 872 * Currently, the data in a SourceSurfaceImage surface is treated as being in the 873 * device output color space. This class is very simple as all backends 874 * have to know about how to deal with drawing a cairo image. 875 */ 876 class SourceSurfaceImage final : public Image { 877 public: GetAsSourceSurface()878 virtual already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override 879 { 880 RefPtr<gfx::SourceSurface> surface(mSourceSurface); 881 return surface.forget(); 882 } 883 SetTextureFlags(TextureFlags aTextureFlags)884 void SetTextureFlags(TextureFlags aTextureFlags) { mTextureFlags = aTextureFlags; } 885 virtual TextureClient* GetTextureClient(KnowsCompositor* aForwarder) override; 886 GetSize()887 virtual gfx::IntSize GetSize() override { return mSize; } 888 889 SourceSurfaceImage(const gfx::IntSize& aSize, gfx::SourceSurface* aSourceSurface); 890 explicit SourceSurfaceImage(gfx::SourceSurface* aSourceSurface); 891 ~SourceSurfaceImage(); 892 893 private: 894 gfx::IntSize mSize; 895 nsCountedRef<nsOwningThreadSourceSurfaceRef> mSourceSurface; 896 nsDataHashtable<nsUint32HashKey, RefPtr<TextureClient> > mTextureClients; 897 TextureFlags mTextureFlags; 898 }; 899 900 } // namespace layers 901 } // namespace mozilla 902 903 #endif 904