1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "ImageHost.h"
8
9 #include <utility>
10
11 #include "composite/CompositableHost.h" // for CompositableHost, etc
12 #include "ipc/IPCMessageUtils.h" // for null_t
13 #include "mozilla/layers/Compositor.h" // for Compositor
14 #include "mozilla/layers/Effects.h" // for TexturedEffect, Effect, etc
15 #include "mozilla/layers/LayerManagerComposite.h" // for TexturedEffect, Effect, etc
16 #include "nsAString.h"
17 #include "nsDebug.h" // for NS_WARNING, NS_ASSERTION
18 #include "nsPrintfCString.h" // for nsPrintfCString
19 #include "nsString.h" // for nsAutoCString
20
21 namespace mozilla {
22
23 using namespace gfx;
24
25 namespace layers {
26
27 class ISurfaceAllocator;
28
ImageHost(const TextureInfo & aTextureInfo)29 ImageHost::ImageHost(const TextureInfo& aTextureInfo)
30 : CompositableHost(aTextureInfo), ImageComposite(), mLocked(false) {}
31
32 ImageHost::~ImageHost() = default;
33
UseTextureHost(const nsTArray<TimedTexture> & aTextures)34 void ImageHost::UseTextureHost(const nsTArray<TimedTexture>& aTextures) {
35 MOZ_ASSERT(!mLocked);
36
37 CompositableHost::UseTextureHost(aTextures);
38 MOZ_ASSERT(aTextures.Length() >= 1);
39
40 nsTArray<TimedImage> newImages;
41
42 for (uint32_t i = 0; i < aTextures.Length(); ++i) {
43 const TimedTexture& t = aTextures[i];
44 MOZ_ASSERT(t.mTexture);
45 if (i + 1 < aTextures.Length() && t.mProducerID == mLastProducerID &&
46 t.mFrameID < mLastFrameID) {
47 // Ignore frames before a frame that we already composited. We don't
48 // ever want to display these frames. This could be important if
49 // the frame producer adjusts timestamps (e.g. to track the audio clock)
50 // and the new frame times are earlier.
51 continue;
52 }
53 TimedImage& img = *newImages.AppendElement();
54 img.mTextureHost = t.mTexture;
55 img.mTimeStamp = t.mTimeStamp;
56 img.mPictureRect = t.mPictureRect;
57 img.mFrameID = t.mFrameID;
58 img.mProducerID = t.mProducerID;
59 img.mTextureHost->SetCropRect(img.mPictureRect);
60 img.mTextureHost->Updated();
61 }
62
63 SetImages(std::move(newImages));
64
65 // If we only have one image we can upload it right away, otherwise we'll
66 // upload on-demand during composition after we have picked the proper
67 // timestamp.
68 if (ImagesCount() == 1) {
69 SetCurrentTextureHost(GetImage(0)->mTextureHost);
70 }
71
72 HostLayerManager* lm = GetLayerManager();
73
74 // Video producers generally send replacement images with the same frameID but
75 // slightly different timestamps in order to sync with the audio clock. This
76 // means that any CompositeUntil() call we made in Composite() may no longer
77 // guarantee that we'll composite until the next frame is ready. Fix that
78 // here.
79 if (lm && mLastFrameID >= 0) {
80 for (const auto& img : Images()) {
81 bool frameComesAfter =
82 img.mFrameID > mLastFrameID || img.mProducerID != mLastProducerID;
83 if (frameComesAfter && !img.mTimeStamp.IsNull()) {
84 lm->CompositeUntil(img.mTimeStamp +
85 TimeDuration::FromMilliseconds(BIAS_TIME_MS));
86 break;
87 }
88 }
89 }
90 }
91
SetCurrentTextureHost(TextureHost * aTexture)92 void ImageHost::SetCurrentTextureHost(TextureHost* aTexture) {
93 if (aTexture == mCurrentTextureHost.get()) {
94 return;
95 }
96
97 bool swapTextureSources = !!mCurrentTextureHost && !!mCurrentTextureSource &&
98 mCurrentTextureHost->IsValid() &&
99 mCurrentTextureHost->HasIntermediateBuffer();
100
101 if (swapTextureSources) {
102 auto dataSource = mCurrentTextureSource->AsDataTextureSource();
103 if (dataSource) {
104 // The current textureHost has an internal buffer in the form of the
105 // DataTextureSource. Removing the ownership of the texture source
106 // will enable the next texture host we bind to the texture source to
107 // acquire it instead of creating a new one. This is desirable in
108 // ImageHost because the current texture won't be used again with the
109 // same content. It wouldn't be desirable with ContentHost for instance,
110 // because the latter reuses the texture's valid regions.
111 dataSource->SetOwner(nullptr);
112 }
113
114 RefPtr<TextureSource> tmp = mExtraTextureSource;
115 mExtraTextureSource = mCurrentTextureSource.get();
116 mCurrentTextureSource = tmp;
117 } else {
118 mExtraTextureSource = nullptr;
119 }
120
121 mCurrentTextureHost = aTexture;
122 mCurrentTextureHost->PrepareTextureSource(mCurrentTextureSource);
123 }
124
CleanupResources()125 void ImageHost::CleanupResources() {
126 mExtraTextureSource = nullptr;
127 mCurrentTextureSource = nullptr;
128 mCurrentTextureHost = nullptr;
129 }
130
RemoveTextureHost(TextureHost * aTexture)131 void ImageHost::RemoveTextureHost(TextureHost* aTexture) {
132 MOZ_ASSERT(!mLocked);
133
134 CompositableHost::RemoveTextureHost(aTexture);
135 RemoveImagesWithTextureHost(aTexture);
136 }
137
GetCompositionTime() const138 TimeStamp ImageHost::GetCompositionTime() const {
139 TimeStamp time;
140 if (HostLayerManager* lm = GetLayerManager()) {
141 time = lm->GetCompositionTime();
142 }
143 return time;
144 }
145
GetCompositionOpportunityId() const146 CompositionOpportunityId ImageHost::GetCompositionOpportunityId() const {
147 CompositionOpportunityId id;
148 if (HostLayerManager* lm = GetLayerManager()) {
149 id = lm->GetCompositionOpportunityId();
150 }
151 return id;
152 }
153
AppendImageCompositeNotification(const ImageCompositeNotificationInfo & aInfo) const154 void ImageHost::AppendImageCompositeNotification(
155 const ImageCompositeNotificationInfo& aInfo) const {
156 if (HostLayerManager* lm = GetLayerManager()) {
157 lm->AppendImageCompositeNotification(aInfo);
158 }
159 }
160
GetAsTextureHost(IntRect * aPictureRect)161 TextureHost* ImageHost::GetAsTextureHost(IntRect* aPictureRect) {
162 const TimedImage* img = ChooseImage();
163 if (!img) {
164 return nullptr;
165 }
166 SetCurrentTextureHost(img->mTextureHost);
167 if (aPictureRect) {
168 *aPictureRect = img->mPictureRect;
169 }
170 return img->mTextureHost;
171 }
172
Attach(Layer * aLayer,TextureSourceProvider * aProvider,AttachFlags aFlags)173 void ImageHost::Attach(Layer* aLayer, TextureSourceProvider* aProvider,
174 AttachFlags aFlags) {
175 CompositableHost::Attach(aLayer, aProvider, aFlags);
176 for (const auto& img : Images()) {
177 img.mTextureHost->SetTextureSourceProvider(aProvider);
178 img.mTextureHost->Updated();
179 }
180 }
181
Composite(Compositor * aCompositor,LayerComposite * aLayer,EffectChain & aEffectChain,float aOpacity,const gfx::Matrix4x4 & aTransform,const gfx::SamplingFilter aSamplingFilter,const gfx::IntRect & aClipRect,const nsIntRegion * aVisibleRegion,const Maybe<gfx::Polygon> & aGeometry)182 void ImageHost::Composite(Compositor* aCompositor, LayerComposite* aLayer,
183 EffectChain& aEffectChain, float aOpacity,
184 const gfx::Matrix4x4& aTransform,
185 const gfx::SamplingFilter aSamplingFilter,
186 const gfx::IntRect& aClipRect,
187 const nsIntRegion* aVisibleRegion,
188 const Maybe<gfx::Polygon>& aGeometry) {
189 RenderInfo info;
190 if (!PrepareToRender(aCompositor, &info)) {
191 return;
192 }
193
194 const TimedImage* img = info.img;
195
196 {
197 AutoLockCompositableHost autoLock(this);
198 if (autoLock.Failed()) {
199 NS_WARNING("failed to lock front buffer");
200 return;
201 }
202
203 if (!mCurrentTextureHost->BindTextureSource(mCurrentTextureSource)) {
204 return;
205 }
206
207 if (!mCurrentTextureSource) {
208 // BindTextureSource above should have returned false!
209 MOZ_ASSERT(false);
210 return;
211 }
212
213 bool isAlphaPremultiplied =
214 !(mCurrentTextureHost->GetFlags() & TextureFlags::NON_PREMULTIPLIED);
215 RefPtr<TexturedEffect> effect =
216 CreateTexturedEffect(mCurrentTextureHost, mCurrentTextureSource.get(),
217 aSamplingFilter, isAlphaPremultiplied);
218 if (!effect) {
219 return;
220 }
221
222 if (!aCompositor->SupportsEffect(effect->mType)) {
223 return;
224 }
225
226 DiagnosticFlags diagnosticFlags = DiagnosticFlags::IMAGE;
227 if (effect->mType == EffectTypes::NV12) {
228 diagnosticFlags |= DiagnosticFlags::NV12;
229 } else if (effect->mType == EffectTypes::YCBCR) {
230 diagnosticFlags |= DiagnosticFlags::YCBCR;
231 }
232
233 aEffectChain.mPrimaryEffect = effect;
234 gfx::Rect pictureRect(0, 0, img->mPictureRect.Width(),
235 img->mPictureRect.Height());
236 BigImageIterator* it = mCurrentTextureSource->AsBigImageIterator();
237 if (it) {
238 // This iteration does not work if we have multiple texture sources here
239 // (e.g. 3 YCbCr textures). There's nothing preventing the different
240 // planes from having different resolutions or tile sizes. For example, a
241 // YCbCr frame could have Cb and Cr planes that are half the resolution of
242 // the Y plane, in such a way that the Y plane overflows the maximum
243 // texture size and the Cb and Cr planes do not. Then the Y plane would be
244 // split into multiple tiles and the Cb and Cr planes would just be one
245 // tile each.
246 // To handle the general case correctly, we'd have to create a grid of
247 // intersected tiles over all planes, and then draw each grid tile using
248 // the corresponding source tiles from all planes, with appropriate
249 // per-plane per-tile texture coords.
250 // DrawQuad currently assumes that all planes use the same texture coords.
251 MOZ_ASSERT(
252 it->GetTileCount() == 1 || !mCurrentTextureSource->GetNextSibling(),
253 "Can't handle multi-plane BigImages");
254
255 it->BeginBigImageIteration();
256 do {
257 IntRect tileRect = it->GetTileRect();
258 gfx::Rect rect(tileRect.X(), tileRect.Y(), tileRect.Width(),
259 tileRect.Height());
260 rect = rect.Intersect(pictureRect);
261 effect->mTextureCoords =
262 Rect(Float(rect.X() - tileRect.X()) / tileRect.Width(),
263 Float(rect.Y() - tileRect.Y()) / tileRect.Height(),
264 Float(rect.Width()) / tileRect.Width(),
265 Float(rect.Height()) / tileRect.Height());
266 if (img->mTextureHost->GetFlags() & TextureFlags::ORIGIN_BOTTOM_LEFT) {
267 effect->mTextureCoords.SetRectY(effect->mTextureCoords.YMost(),
268 -effect->mTextureCoords.Height());
269 }
270 aCompositor->DrawGeometry(rect, aClipRect, aEffectChain, aOpacity,
271 aTransform, aGeometry);
272 aCompositor->DrawDiagnostics(
273 diagnosticFlags | DiagnosticFlags::BIGIMAGE, rect, aClipRect,
274 aTransform, mFlashCounter);
275 } while (it->NextTile());
276 it->EndBigImageIteration();
277 // layer border
278 aCompositor->DrawDiagnostics(diagnosticFlags, pictureRect, aClipRect,
279 aTransform, mFlashCounter);
280 } else {
281 IntSize textureSize = mCurrentTextureSource->GetSize();
282 effect->mTextureCoords =
283 Rect(Float(img->mPictureRect.X()) / textureSize.width,
284 Float(img->mPictureRect.Y()) / textureSize.height,
285 Float(img->mPictureRect.Width()) / textureSize.width,
286 Float(img->mPictureRect.Height()) / textureSize.height);
287
288 if (img->mTextureHost->GetFlags() & TextureFlags::ORIGIN_BOTTOM_LEFT) {
289 effect->mTextureCoords.SetRectY(effect->mTextureCoords.YMost(),
290 -effect->mTextureCoords.Height());
291 }
292
293 aCompositor->DrawGeometry(pictureRect, aClipRect, aEffectChain, aOpacity,
294 aTransform, aGeometry);
295 aCompositor->DrawDiagnostics(diagnosticFlags, pictureRect, aClipRect,
296 aTransform, mFlashCounter);
297 }
298 }
299
300 FinishRendering(info);
301 }
302
PrepareToRender(TextureSourceProvider * aProvider,RenderInfo * aOutInfo)303 bool ImageHost::PrepareToRender(TextureSourceProvider* aProvider,
304 RenderInfo* aOutInfo) {
305 HostLayerManager* lm = GetLayerManager();
306 if (!lm) {
307 return false;
308 }
309
310 int imageIndex = ChooseImageIndex();
311 if (imageIndex < 0) {
312 return false;
313 }
314
315 if (uint32_t(imageIndex) + 1 < ImagesCount()) {
316 lm->CompositeUntil(GetImage(imageIndex + 1)->mTimeStamp +
317 TimeDuration::FromMilliseconds(BIAS_TIME_MS));
318 }
319
320 const TimedImage* img = GetImage(imageIndex);
321 img->mTextureHost->SetTextureSourceProvider(aProvider);
322 SetCurrentTextureHost(img->mTextureHost);
323
324 aOutInfo->imageIndex = imageIndex;
325 aOutInfo->img = img;
326 aOutInfo->host = mCurrentTextureHost;
327 return true;
328 }
329
AcquireTextureSource(const RenderInfo & aInfo)330 RefPtr<TextureSource> ImageHost::AcquireTextureSource(const RenderInfo& aInfo) {
331 MOZ_ASSERT(aInfo.host == mCurrentTextureHost);
332 if (!aInfo.host->AcquireTextureSource(mCurrentTextureSource)) {
333 return nullptr;
334 }
335 return mCurrentTextureSource.get();
336 }
337
FinishRendering(const RenderInfo & aInfo)338 void ImageHost::FinishRendering(const RenderInfo& aInfo) {
339 OnFinishRendering(aInfo.imageIndex, aInfo.img, mAsyncRef.mProcessId,
340 mAsyncRef.mHandle);
341 }
342
SetTextureSourceProvider(TextureSourceProvider * aProvider)343 void ImageHost::SetTextureSourceProvider(TextureSourceProvider* aProvider) {
344 if (mTextureSourceProvider != aProvider) {
345 for (const auto& img : Images()) {
346 img.mTextureHost->SetTextureSourceProvider(aProvider);
347 }
348 }
349 CompositableHost::SetTextureSourceProvider(aProvider);
350 }
351
PrintInfo(std::stringstream & aStream,const char * aPrefix)352 void ImageHost::PrintInfo(std::stringstream& aStream, const char* aPrefix) {
353 aStream << aPrefix;
354 aStream << nsPrintfCString("ImageHost (0x%p)", this).get();
355
356 nsAutoCString pfx(aPrefix);
357 pfx += " ";
358 for (const auto& img : Images()) {
359 aStream << "\n";
360 img.mTextureHost->PrintInfo(aStream, pfx.get());
361 aStream << " [picture-rect=" << img.mPictureRect << "]";
362 }
363 }
364
Dump(std::stringstream & aStream,const char * aPrefix,bool aDumpHtml)365 void ImageHost::Dump(std::stringstream& aStream, const char* aPrefix,
366 bool aDumpHtml) {
367 for (const auto& img : Images()) {
368 aStream << aPrefix;
369 aStream << (aDumpHtml ? "<ul><li>TextureHost: " : "TextureHost: ");
370 DumpTextureHost(aStream, img.mTextureHost);
371 aStream << (aDumpHtml ? " </li></ul> " : " ");
372 }
373 }
374
GetAsSurface()375 already_AddRefed<gfx::DataSourceSurface> ImageHost::GetAsSurface() {
376 const TimedImage* img = ChooseImage();
377 if (img) {
378 return img->mTextureHost->GetAsSurface();
379 }
380 return nullptr;
381 }
382
Lock()383 bool ImageHost::Lock() {
384 MOZ_ASSERT(!mLocked);
385 const TimedImage* img = ChooseImage();
386 if (!img) {
387 return false;
388 }
389
390 SetCurrentTextureHost(img->mTextureHost);
391
392 if (!mCurrentTextureHost->Lock()) {
393 return false;
394 }
395 mLocked = true;
396 return true;
397 }
398
Unlock()399 void ImageHost::Unlock() {
400 MOZ_ASSERT(mLocked);
401
402 if (mCurrentTextureHost) {
403 mCurrentTextureHost->Unlock();
404 }
405 mLocked = false;
406 }
407
GetImageSize()408 IntSize ImageHost::GetImageSize() {
409 const TimedImage* img = ChooseImage();
410 if (img) {
411 return IntSize(img->mPictureRect.Width(), img->mPictureRect.Height());
412 }
413 return IntSize();
414 }
415
IsOpaque()416 bool ImageHost::IsOpaque() {
417 const TimedImage* img = ChooseImage();
418 if (!img) {
419 return false;
420 }
421
422 if (img->mPictureRect.Width() == 0 || img->mPictureRect.Height() == 0 ||
423 !img->mTextureHost) {
424 return false;
425 }
426
427 gfx::SurfaceFormat format = img->mTextureHost->GetFormat();
428 if (gfx::IsOpaque(format)) {
429 return true;
430 }
431 return false;
432 }
433
GenEffect(const gfx::SamplingFilter aSamplingFilter)434 already_AddRefed<TexturedEffect> ImageHost::GenEffect(
435 const gfx::SamplingFilter aSamplingFilter) {
436 const TimedImage* img = ChooseImage();
437 if (!img) {
438 return nullptr;
439 }
440 SetCurrentTextureHost(img->mTextureHost);
441 if (!mCurrentTextureHost->BindTextureSource(mCurrentTextureSource)) {
442 return nullptr;
443 }
444 bool isAlphaPremultiplied = true;
445 if (mCurrentTextureHost->GetFlags() & TextureFlags::NON_PREMULTIPLIED) {
446 isAlphaPremultiplied = false;
447 }
448
449 return CreateTexturedEffect(mCurrentTextureHost, mCurrentTextureSource,
450 aSamplingFilter, isAlphaPremultiplied);
451 }
452
453 } // namespace layers
454 } // namespace mozilla
455