1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 #include "RemoteVideoDecoder.h"
7
8 #include "mozilla/layers/ImageDataSerializer.h"
9
10 #ifdef MOZ_AV1
11 # include "AOMDecoder.h"
12 # include "DAV1DDecoder.h"
13 #endif
14 #ifdef XP_WIN
15 # include "WMFDecoderModule.h"
16 #endif
17 #include "GPUVideoImage.h"
18 #include "ImageContainer.h" // for PlanarYCbCrData and BufferRecycleBin
19 #include "MediaDataDecoderProxy.h"
20 #include "MediaInfo.h"
21 #include "PDMFactory.h"
22 #include "RemoteDecoderManagerParent.h"
23 #include "RemoteImageHolder.h"
24 #include "mozilla/StaticPrefs_media.h"
25 #include "mozilla/Telemetry.h"
26 #include "mozilla/layers/ImageClient.h"
27 #include "mozilla/layers/TextureClient.h"
28 #include "mozilla/layers/VideoBridgeChild.h"
29
30 namespace mozilla {
31
32 using namespace layers; // for PlanarYCbCrData and BufferRecycleBin
33 using namespace ipc;
34 using namespace gfx;
35
GetTextureForwarder()36 layers::TextureForwarder* KnowsCompositorVideo::GetTextureForwarder() {
37 auto* vbc = VideoBridgeChild::GetSingleton();
38 return (vbc && vbc->CanSend()) ? vbc : nullptr;
39 }
GetLayersIPCActor()40 layers::LayersIPCActor* KnowsCompositorVideo::GetLayersIPCActor() {
41 return GetTextureForwarder();
42 }
43
44 /* static */ already_AddRefed<KnowsCompositorVideo>
TryCreateForIdentifier(const layers::TextureFactoryIdentifier & aIdentifier)45 KnowsCompositorVideo::TryCreateForIdentifier(
46 const layers::TextureFactoryIdentifier& aIdentifier) {
47 VideoBridgeChild* child = VideoBridgeChild::GetSingleton();
48 if (!child) {
49 return nullptr;
50 }
51
52 RefPtr<KnowsCompositorVideo> knowsCompositor = new KnowsCompositorVideo();
53 knowsCompositor->IdentifyTextureHost(aIdentifier);
54 return knowsCompositor.forget();
55 }
56
RemoteVideoDecoderChild(RemoteDecodeIn aLocation)57 RemoteVideoDecoderChild::RemoteVideoDecoderChild(RemoteDecodeIn aLocation)
58 : RemoteDecoderChild(aLocation), mBufferRecycleBin(new BufferRecycleBin) {}
59
ProcessOutput(DecodedOutputIPDL && aDecodedData)60 MediaResult RemoteVideoDecoderChild::ProcessOutput(
61 DecodedOutputIPDL&& aDecodedData) {
62 AssertOnManagerThread();
63 MOZ_ASSERT(aDecodedData.type() == DecodedOutputIPDL::TArrayOfRemoteVideoData);
64
65 nsTArray<RemoteVideoData>& arrayData =
66 aDecodedData.get_ArrayOfRemoteVideoData()->Array();
67
68 for (auto&& data : arrayData) {
69 if (data.image().IsEmpty()) {
70 // This is a NullData object.
71 mDecodedData.AppendElement(MakeRefPtr<NullData>(
72 data.base().offset(), data.base().time(), data.base().duration()));
73 continue;
74 }
75 RefPtr<Image> image = data.image().TransferToImage(mBufferRecycleBin);
76
77 RefPtr<VideoData> video = VideoData::CreateFromImage(
78 data.display(), data.base().offset(), data.base().time(),
79 data.base().duration(), image, data.base().keyframe(),
80 data.base().timecode());
81
82 if (!video) {
83 // OOM
84 return MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__);
85 }
86 mDecodedData.AppendElement(std::move(video));
87 }
88 return NS_OK;
89 }
90
InitIPDL(const VideoInfo & aVideoInfo,float aFramerate,const CreateDecoderParams::OptionSet & aOptions,Maybe<layers::TextureFactoryIdentifier> aIdentifier)91 MediaResult RemoteVideoDecoderChild::InitIPDL(
92 const VideoInfo& aVideoInfo, float aFramerate,
93 const CreateDecoderParams::OptionSet& aOptions,
94 Maybe<layers::TextureFactoryIdentifier> aIdentifier) {
95 MOZ_ASSERT_IF(mLocation == RemoteDecodeIn::GpuProcess, aIdentifier);
96
97 RefPtr<RemoteDecoderManagerChild> manager =
98 RemoteDecoderManagerChild::GetSingleton(mLocation);
99
100 // The manager isn't available because RemoteDecoderManagerChild has been
101 // initialized with null end points and we don't want to decode video on RDD
102 // process anymore. Return false here so that we can fallback to other PDMs.
103 if (!manager) {
104 return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
105 RESULT_DETAIL("RemoteDecoderManager is not available."));
106 }
107
108 if (!manager->CanSend()) {
109 if (mLocation == RemoteDecodeIn::GpuProcess) {
110 // The manager doesn't support sending messages because we've just crashed
111 // and are working on reinitialization. Don't initialize mIPDLSelfRef and
112 // leave us in an error state. We'll then immediately reject the promise
113 // when Init() is called and the caller can try again. Hopefully by then
114 // the new manager is ready, or we've notified the caller of it being no
115 // longer available. If not, then the cycle repeats until we're ready.
116 return NS_OK;
117 }
118
119 return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
120 RESULT_DETAIL("RemoteDecoderManager unable to send."));
121 }
122
123 mIPDLSelfRef = this;
124 VideoDecoderInfoIPDL decoderInfo(aVideoInfo, aFramerate);
125 Unused << manager->SendPRemoteDecoderConstructor(this, decoderInfo, aOptions,
126 aIdentifier);
127
128 return NS_OK;
129 }
130
RemoteVideoDecoderParent(RemoteDecoderManagerParent * aParent,const VideoInfo & aVideoInfo,float aFramerate,const CreateDecoderParams::OptionSet & aOptions,const Maybe<layers::TextureFactoryIdentifier> & aIdentifier,nsISerialEventTarget * aManagerThread,TaskQueue * aDecodeTaskQueue)131 RemoteVideoDecoderParent::RemoteVideoDecoderParent(
132 RemoteDecoderManagerParent* aParent, const VideoInfo& aVideoInfo,
133 float aFramerate, const CreateDecoderParams::OptionSet& aOptions,
134 const Maybe<layers::TextureFactoryIdentifier>& aIdentifier,
135 nsISerialEventTarget* aManagerThread, TaskQueue* aDecodeTaskQueue)
136 : RemoteDecoderParent(aParent, aOptions, aManagerThread, aDecodeTaskQueue),
137 mVideoInfo(aVideoInfo),
138 mFramerate(aFramerate) {
139 if (aIdentifier) {
140 // Check to see if we have a direct PVideoBridge connection to the
141 // destination process specified in aIdentifier, and create a
142 // KnowsCompositor representing that connection if so. If this fails, then
143 // we fall back to returning the decoded frames directly via Output().
144 mKnowsCompositor =
145 KnowsCompositorVideo::TryCreateForIdentifier(*aIdentifier);
146 }
147 }
148
RecvConstruct(ConstructResolver && aResolver)149 IPCResult RemoteVideoDecoderParent::RecvConstruct(
150 ConstructResolver&& aResolver) {
151 auto imageContainer = MakeRefPtr<layers::ImageContainer>();
152 if (mKnowsCompositor && XRE_IsRDDProcess()) {
153 // Ensure to allocate recycle allocator
154 imageContainer->EnsureRecycleAllocatorForRDD(mKnowsCompositor);
155 }
156 auto params = CreateDecoderParams{
157 mVideoInfo, mKnowsCompositor,
158 imageContainer, CreateDecoderParams::VideoFrameRate(mFramerate),
159 mOptions, CreateDecoderParams::NoWrapper(true),
160 };
161
162 mParent->EnsurePDMFactory().CreateDecoder(params)->Then(
163 GetCurrentSerialEventTarget(), __func__,
164 [resolver = std::move(aResolver), self = RefPtr{this}](
165 PlatformDecoderModule::CreateDecoderPromise::ResolveOrRejectValue&&
166 aValue) {
167 if (aValue.IsReject()) {
168 resolver(aValue.RejectValue());
169 return;
170 }
171 MOZ_ASSERT(aValue.ResolveValue());
172 self->mDecoder =
173 new MediaDataDecoderProxy(aValue.ResolveValue().forget(),
174 do_AddRef(self->mDecodeTaskQueue.get()));
175 resolver(NS_OK);
176 });
177 return IPC_OK();
178 }
179
ProcessDecodedData(MediaDataDecoder::DecodedData && aData,DecodedOutputIPDL & aDecodedData)180 MediaResult RemoteVideoDecoderParent::ProcessDecodedData(
181 MediaDataDecoder::DecodedData&& aData, DecodedOutputIPDL& aDecodedData) {
182 MOZ_ASSERT(OnManagerThread());
183
184 // If the video decoder bridge has shut down, stop.
185 if (mKnowsCompositor && !mKnowsCompositor->GetTextureForwarder()) {
186 aDecodedData = MakeRefPtr<ArrayOfRemoteVideoData>();
187 return NS_OK;
188 }
189
190 nsTArray<RemoteVideoData> array;
191
192 for (const auto& data : aData) {
193 MOZ_ASSERT(data->mType == MediaData::Type::VIDEO_DATA ||
194 data->mType == MediaData::Type::NULL_DATA,
195 "Can only decode videos using RemoteDecoderParent!");
196 if (data->mType == MediaData::Type::NULL_DATA) {
197 RemoteVideoData output(
198 MediaDataIPDL(data->mOffset, data->mTime, data->mTimecode,
199 data->mDuration, data->mKeyframe),
200 IntSize(), RemoteImageHolder(), -1);
201
202 array.AppendElement(std::move(output));
203 continue;
204 }
205 VideoData* video = static_cast<VideoData*>(data.get());
206
207 MOZ_ASSERT(video->mImage,
208 "Decoded video must output a layer::Image to "
209 "be used with RemoteDecoderParent");
210
211 RefPtr<TextureClient> texture;
212 SurfaceDescriptor sd;
213 IntSize size;
214 bool needStorage = false;
215
216 if (mKnowsCompositor) {
217 texture = video->mImage->GetTextureClient(mKnowsCompositor);
218
219 if (!texture) {
220 texture = ImageClient::CreateTextureClientForImage(video->mImage,
221 mKnowsCompositor);
222 }
223
224 if (texture) {
225 if (!texture->IsAddedToCompositableClient()) {
226 texture->InitIPDLActor(mKnowsCompositor);
227 texture->SetAddedToCompositableClient();
228 }
229 needStorage = true;
230 SurfaceDescriptorRemoteDecoder remoteSD;
231 texture->GetSurfaceDescriptorRemoteDecoder(&remoteSD);
232 sd = remoteSD;
233 size = texture->GetSize();
234 }
235 }
236
237 // If failed to create a GPU accelerated surface descriptor, fall back to
238 // copying frames via shmem.
239 if (!IsSurfaceDescriptorValid(sd)) {
240 needStorage = false;
241 PlanarYCbCrImage* image = video->mImage->AsPlanarYCbCrImage();
242 if (!image) {
243 return MediaResult(NS_ERROR_UNEXPECTED,
244 "Expected Planar YCbCr image in "
245 "RemoteVideoDecoderParent::ProcessDecodedData");
246 }
247
248 SurfaceDescriptorBuffer sdBuffer;
249 nsresult rv = image->BuildSurfaceDescriptorBuffer(
250 sdBuffer, [&](uint32_t aBufferSize) {
251 ShmemBuffer buffer = AllocateBuffer(aBufferSize);
252 if (buffer.Valid()) {
253 return MemoryOrShmem(std::move(buffer.Get()));
254 }
255 return MemoryOrShmem();
256 });
257 NS_ENSURE_SUCCESS(rv, rv);
258
259 sd = sdBuffer;
260 size = image->GetSize();
261 }
262
263 if (needStorage) {
264 MOZ_ASSERT(sd.type() != SurfaceDescriptor::TSurfaceDescriptorBuffer);
265 mParent->StoreImage(static_cast<const SurfaceDescriptorGPUVideo&>(sd),
266 video->mImage, texture);
267 }
268
269 RemoteVideoData output(
270 MediaDataIPDL(data->mOffset, data->mTime, data->mTimecode,
271 data->mDuration, data->mKeyframe),
272 video->mDisplay,
273 RemoteImageHolder(mParent,
274 XRE_IsGPUProcess() ? VideoBridgeSource::GpuProcess
275 : VideoBridgeSource::RddProcess,
276 size, sd),
277 video->mFrameID);
278
279 array.AppendElement(std::move(output));
280 }
281
282 aDecodedData = MakeRefPtr<ArrayOfRemoteVideoData>(std::move(array));
283
284 return NS_OK;
285 }
286
287 } // namespace mozilla
288