1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 *
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "ImageLogging.h" // Must appear first
8
9 #include "nsAVIFDecoder.h"
10
11 #include "aom/aomdx.h"
12
13 #include "mozilla/gfx/Types.h"
14 #include "YCbCrUtils.h"
15 #include "libyuv.h"
16
17 #include "SurfacePipeFactory.h"
18
19 #include "mozilla/Telemetry.h"
20 #include "mozilla/TelemetryComms.h"
21
22 using namespace mozilla::gfx;
23
24 namespace mozilla {
25
26 Maybe<gfx::YUVColorSpace> GetColorSpace(const Dav1dPicture&, LazyLogModule&);
27
28 namespace image {
29
30 using Telemetry::LABELS_AVIF_AOM_DECODE_ERROR;
31 using Telemetry::LABELS_AVIF_BIT_DEPTH;
32 using Telemetry::LABELS_AVIF_DECODE_RESULT;
33 using Telemetry::LABELS_AVIF_DECODER;
34 using Telemetry::LABELS_AVIF_YUV_COLOR_SPACE;
35
36 static LazyLogModule sAVIFLog("AVIFDecoder");
37
38 static const LABELS_AVIF_BIT_DEPTH gColorDepthLabel[] = {
39 LABELS_AVIF_BIT_DEPTH::color_8, LABELS_AVIF_BIT_DEPTH::color_10,
40 LABELS_AVIF_BIT_DEPTH::color_12, LABELS_AVIF_BIT_DEPTH::color_16};
41
42 static const LABELS_AVIF_YUV_COLOR_SPACE gColorSpaceLabel[] = {
43 LABELS_AVIF_YUV_COLOR_SPACE::BT601, LABELS_AVIF_YUV_COLOR_SPACE::BT709,
44 LABELS_AVIF_YUV_COLOR_SPACE::BT2020, LABELS_AVIF_YUV_COLOR_SPACE::identity};
45
GetImageSize(const Mp4parseAvifImage & image)46 static MaybeIntSize GetImageSize(const Mp4parseAvifImage& image) {
47 // Note this does not take cropping via CleanAperture (clap) into account
48 const struct Mp4parseImageSpatialExtents* ispe = image.spatial_extents;
49 // Decoder::PostSize takes int32_t, but ispe contains uint32_t
50 CheckedInt<int32_t> width = ispe->image_width;
51 CheckedInt<int32_t> height = ispe->image_height;
52
53 if (width.isValid() && height.isValid()) {
54 return Some(IntSize{width.value(), height.value()});
55 }
56
57 return Nothing();
58 }
59
60 // Translate the MIAF/HEIF-based orientation transforms (imir, irot) into
61 // ImageLib's representation. Note that the interpretation of imir was reversed
62 // Between HEIF (ISO 23008-12:2017) and ISO/IEC 23008-12:2017/DAmd 2. This is
63 // handled by mp4parse. See mp4parse::read_imir for details.
GetImageOrientation(const Mp4parseAvifImage & image)64 Orientation GetImageOrientation(const Mp4parseAvifImage& image) {
65 // Per MIAF (ISO/IEC 23000-22:2019) § 7.3.6.7
66 // These properties, if used, shall be indicated to be applied in the
67 // following order: clean aperture first, then rotation, then mirror.
68 // The Orientation type does the same order, but opposite rotation direction
69
70 const Mp4parseIrot heifRot = image.image_rotation;
71 const Mp4parseImir* heifMir = image.image_mirror;
72 Angle mozRot;
73 Flip mozFlip;
74
75 if (!heifMir) { // No mirroring
76 mozFlip = Flip::Unflipped;
77
78 switch (heifRot) {
79 case MP4PARSE_IROT_D0:
80 // ⥠ UPWARDS HARPOON WITH BARB LEFT FROM BAR
81 mozRot = Angle::D0;
82 break;
83 case MP4PARSE_IROT_D90:
84 // ⥞ LEFTWARDS HARPOON WITH BARB DOWN FROM BAR
85 mozRot = Angle::D270;
86 break;
87 case MP4PARSE_IROT_D180:
88 // ⥝ DOWNWARDS HARPOON WITH BARB RIGHT FROM BAR
89 mozRot = Angle::D180;
90 break;
91 case MP4PARSE_IROT_D270:
92 // ⥛ RIGHTWARDS HARPOON WITH BARB UP FROM BAR
93 mozRot = Angle::D90;
94 break;
95 default:
96 MOZ_ASSERT_UNREACHABLE();
97 }
98 } else {
99 MOZ_ASSERT(heifMir);
100 mozFlip = Flip::Horizontal;
101
102 enum class HeifFlippedOrientation : uint8_t {
103 IROT_D0_IMIR_V = (MP4PARSE_IROT_D0 << 1) | MP4PARSE_IMIR_LEFT_RIGHT,
104 IROT_D0_IMIR_H = (MP4PARSE_IROT_D0 << 1) | MP4PARSE_IMIR_TOP_BOTTOM,
105 IROT_D90_IMIR_V = (MP4PARSE_IROT_D90 << 1) | MP4PARSE_IMIR_LEFT_RIGHT,
106 IROT_D90_IMIR_H = (MP4PARSE_IROT_D90 << 1) | MP4PARSE_IMIR_TOP_BOTTOM,
107 IROT_D180_IMIR_V = (MP4PARSE_IROT_D180 << 1) | MP4PARSE_IMIR_LEFT_RIGHT,
108 IROT_D180_IMIR_H = (MP4PARSE_IROT_D180 << 1) | MP4PARSE_IMIR_TOP_BOTTOM,
109 IROT_D270_IMIR_V = (MP4PARSE_IROT_D270 << 1) | MP4PARSE_IMIR_LEFT_RIGHT,
110 IROT_D270_IMIR_H = (MP4PARSE_IROT_D270 << 1) | MP4PARSE_IMIR_TOP_BOTTOM,
111 };
112
113 HeifFlippedOrientation heifO =
114 HeifFlippedOrientation((heifRot << 1) | *heifMir);
115
116 switch (heifO) {
117 case HeifFlippedOrientation::IROT_D0_IMIR_V:
118 case HeifFlippedOrientation::IROT_D180_IMIR_H:
119 // ⥜ UPWARDS HARPOON WITH BARB RIGHT FROM BAR
120 mozRot = Angle::D0;
121 break;
122 case HeifFlippedOrientation::IROT_D270_IMIR_V:
123 case HeifFlippedOrientation::IROT_D90_IMIR_H:
124 // ⥚ LEFTWARDS HARPOON WITH BARB UP FROM BAR
125 mozRot = Angle::D90;
126 break;
127 case HeifFlippedOrientation::IROT_D180_IMIR_V:
128 case HeifFlippedOrientation::IROT_D0_IMIR_H:
129 // ⥡ DOWNWARDS HARPOON WITH BARB LEFT FROM BAR
130 mozRot = Angle::D180;
131 break;
132 case HeifFlippedOrientation::IROT_D90_IMIR_V:
133 case HeifFlippedOrientation::IROT_D270_IMIR_H:
134 // ⥟ RIGHTWARDS HARPOON WITH BARB DOWN FROM BAR
135 mozRot = Angle::D270;
136 break;
137 default:
138 MOZ_ASSERT_UNREACHABLE();
139 }
140 }
141
142 MOZ_LOG(sAVIFLog, LogLevel::Debug,
143 ("GetImageOrientation: (rot%d, imir(%s)) -> (Angle%d, "
144 "Flip%d)",
145 static_cast<int>(heifRot),
146 heifMir ? (*heifMir == MP4PARSE_IMIR_LEFT_RIGHT ? "left-right"
147 : "top-bottom")
148 : "none",
149 static_cast<int>(mozRot), static_cast<int>(mozFlip)));
150 return Orientation{mozRot, mozFlip};
151 }
152
153 class AVIFParser {
154 public:
Create(const Mp4parseIo * aIo)155 static AVIFParser* Create(const Mp4parseIo* aIo) {
156 MOZ_ASSERT(aIo);
157
158 UniquePtr<AVIFParser> p(new AVIFParser(aIo));
159 if (!p->Init()) {
160 return nullptr;
161 }
162 MOZ_ASSERT(p->mParser);
163 return p.release();
164 }
165
~AVIFParser()166 ~AVIFParser() {
167 MOZ_LOG(sAVIFLog, LogLevel::Debug, ("Destroy AVIFParser=%p", this));
168 }
169
GetImage()170 Mp4parseAvifImage* GetImage() {
171 MOZ_ASSERT(mParser);
172
173 if (mAvifImage.isNothing()) {
174 mAvifImage.emplace();
175 Mp4parseStatus status =
176 mp4parse_avif_get_image(mParser.get(), mAvifImage.ptr());
177 MOZ_LOG(sAVIFLog, LogLevel::Debug,
178 ("[this=%p] mp4parse_avif_get_image -> %d; primary_item length: "
179 "%zu, alpha_item length: %zu",
180 this, status, mAvifImage->primary_item.length,
181 mAvifImage->alpha_item.length));
182 if (status != MP4PARSE_STATUS_OK) {
183 mAvifImage.reset();
184 return nullptr;
185 }
186 }
187 return mAvifImage.ptr();
188 }
189
190 private:
AVIFParser(const Mp4parseIo * aIo)191 explicit AVIFParser(const Mp4parseIo* aIo) : mIo(aIo) {
192 MOZ_ASSERT(mIo);
193 MOZ_LOG(sAVIFLog, LogLevel::Debug,
194 ("Create AVIFParser=%p, image.avif.compliance_strictness: %d", this,
195 StaticPrefs::image_avif_compliance_strictness()));
196 }
197
Init()198 bool Init() {
199 MOZ_ASSERT(!mParser);
200
201 Mp4parseAvifParser* parser = nullptr;
202 Mp4parseStatus status =
203 mp4parse_avif_new(mIo,
204 static_cast<enum Mp4parseStrictness>(
205 StaticPrefs::image_avif_compliance_strictness()),
206 &parser);
207 MOZ_LOG(sAVIFLog, LogLevel::Debug,
208 ("[this=%p] mp4parse_avif_new status: %d", this, status));
209 if (status != MP4PARSE_STATUS_OK) {
210 return false;
211 }
212 mParser.reset(parser);
213 return true;
214 }
215
216 struct FreeAvifParser {
operator ()mozilla::image::AVIFParser::FreeAvifParser217 void operator()(Mp4parseAvifParser* aPtr) { mp4parse_avif_free(aPtr); }
218 };
219
220 const Mp4parseIo* mIo;
221 UniquePtr<Mp4parseAvifParser, FreeAvifParser> mParser;
222 Maybe<Mp4parseAvifImage> mAvifImage;
223 };
224
225 // An interface to do decode and get the decoded data
226 class AVIFDecoderInterface {
227 public:
228 using Dav1dResult = nsAVIFDecoder::Dav1dResult;
229 using NonAOMCodecError = nsAVIFDecoder::NonAOMCodecError;
230 using AOMResult = nsAVIFDecoder::AOMResult;
231 using NonDecoderResult = nsAVIFDecoder::NonDecoderResult;
232 using DecodeResult = nsAVIFDecoder::DecodeResult;
233
234 virtual ~AVIFDecoderInterface() = default;
235
236 // Set the mDecodedData if Decode() succeeds
237 virtual DecodeResult Decode(bool aIsMetadataDecode,
238 const Mp4parseAvifImage& parsedImg) = 0;
239 // Must be called after Decode() succeeds
GetDecodedData()240 layers::PlanarYCbCrAData& GetDecodedData() {
241 MOZ_ASSERT(mDecodedData.isSome());
242 return mDecodedData.ref();
243 }
244
245 protected:
AVIFDecoderInterface(UniquePtr<AVIFParser> && aParser)246 explicit AVIFDecoderInterface(UniquePtr<AVIFParser>&& aParser)
247 : mParser(std::move(aParser)) {
248 MOZ_ASSERT(mParser);
249 }
250
IsDecodeSuccess(const DecodeResult & aResult)251 inline static bool IsDecodeSuccess(const DecodeResult& aResult) {
252 return nsAVIFDecoder::IsDecodeSuccess(aResult);
253 }
254
255 UniquePtr<AVIFParser> mParser;
256
257 // The mDecodedData is valid after Decode() succeeds
258 Maybe<layers::PlanarYCbCrAData> mDecodedData;
259 };
260
261 class Dav1dDecoder final : AVIFDecoderInterface {
262 public:
~Dav1dDecoder()263 ~Dav1dDecoder() {
264 MOZ_LOG(sAVIFLog, LogLevel::Verbose, ("Destroy Dav1dDecoder=%p", this));
265
266 if (mPicture) {
267 dav1d_picture_unref(mPicture.take().ptr());
268 }
269
270 if (mAlphaPlane) {
271 dav1d_picture_unref(mAlphaPlane.take().ptr());
272 }
273
274 if (mContext) {
275 dav1d_close(&mContext);
276 MOZ_ASSERT(!mContext);
277 }
278 }
279
Create(UniquePtr<AVIFParser> && aParser,UniquePtr<AVIFDecoderInterface> & aDecoder)280 static DecodeResult Create(UniquePtr<AVIFParser>&& aParser,
281 UniquePtr<AVIFDecoderInterface>& aDecoder) {
282 UniquePtr<Dav1dDecoder> d(new Dav1dDecoder(std::move(aParser)));
283 Dav1dResult r = d->Init();
284 if (r == 0) {
285 MOZ_ASSERT(d->mContext);
286 aDecoder.reset(d.release());
287 }
288 return AsVariant(r);
289 }
290
Decode(bool aIsMetadataDecode,const Mp4parseAvifImage & parsedImg)291 DecodeResult Decode(bool aIsMetadataDecode,
292 const Mp4parseAvifImage& parsedImg) override {
293 MOZ_ASSERT(mParser);
294 MOZ_ASSERT(mContext);
295 MOZ_ASSERT(mPicture.isNothing());
296 MOZ_ASSERT(mDecodedData.isNothing());
297
298 MOZ_LOG(sAVIFLog, LogLevel::Verbose, ("[this=%p] Beginning Decode", this));
299
300 if (!parsedImg.primary_item.data || !parsedImg.primary_item.length) {
301 return AsVariant(NonDecoderResult::NoPrimaryItem);
302 }
303
304 mPicture.emplace();
305 Dav1dResult r =
306 GetPicture(parsedImg.primary_item, mPicture.ptr(), aIsMetadataDecode);
307 if (r != 0) {
308 mPicture.reset();
309 return AsVariant(r);
310 }
311
312 if (parsedImg.alpha_item.data && parsedImg.alpha_item.length) {
313 mAlphaPlane.emplace();
314 Dav1dResult r = GetPicture(parsedImg.alpha_item, mAlphaPlane.ptr(),
315 aIsMetadataDecode);
316 if (r != 0) {
317 mAlphaPlane.reset();
318 return AsVariant(r);
319 }
320
321 // Per § 4 of the AVIF spec
322 // https://aomediacodec.github.io/av1-avif/#auxiliary-images: An AV1 Alpha
323 // Image Item […] shall be encoded with the same bit depth as the
324 // associated master AV1 Image Item
325 if (mPicture->p.bpc != mAlphaPlane->p.bpc) {
326 return AsVariant(NonDecoderResult::AlphaYColorDepthMismatch);
327 }
328 }
329
330 MOZ_ASSERT_IF(mAlphaPlane.isNothing(), !parsedImg.premultiplied_alpha);
331 mDecodedData.emplace(
332 Dav1dPictureToYCbCrAData(mPicture.ptr(), mAlphaPlane.ptrOr(nullptr),
333 parsedImg.premultiplied_alpha));
334
335 return AsVariant(r);
336 }
337
338 private:
Dav1dDecoder(UniquePtr<AVIFParser> && aParser)339 explicit Dav1dDecoder(UniquePtr<AVIFParser>&& aParser)
340 : AVIFDecoderInterface(std::move(aParser)) {
341 MOZ_LOG(sAVIFLog, LogLevel::Verbose, ("Create Dav1dDecoder=%p", this));
342 }
343
Init()344 Dav1dResult Init() {
345 MOZ_ASSERT(!mContext);
346
347 Dav1dSettings settings;
348 dav1d_default_settings(&settings);
349 settings.all_layers = 0;
350 settings.max_frame_delay = 1;
351 // TODO: tune settings a la DAV1DDecoder for AV1 (Bug 1681816)
352
353 return dav1d_open(&mContext, &settings);
354 }
355
GetPicture(const Mp4parseByteData & aBytes,Dav1dPicture * aPicture,bool aIsMetadataDecode)356 Dav1dResult GetPicture(const Mp4parseByteData& aBytes, Dav1dPicture* aPicture,
357 bool aIsMetadataDecode) {
358 MOZ_ASSERT(mContext);
359 MOZ_ASSERT(aPicture);
360
361 Dav1dData dav1dData;
362 Dav1dResult r = dav1d_data_wrap(&dav1dData, aBytes.data, aBytes.length,
363 Dav1dFreeCallback_s, nullptr);
364
365 MOZ_LOG(sAVIFLog, r == 0 ? LogLevel::Verbose : LogLevel::Error,
366 ("[this=%p] dav1d_data_wrap(%p, %zu) -> %d", this, dav1dData.data,
367 dav1dData.sz, r));
368
369 if (r != 0) {
370 return r;
371 }
372
373 r = dav1d_send_data(mContext, &dav1dData);
374
375 MOZ_LOG(sAVIFLog, r == 0 ? LogLevel::Debug : LogLevel::Error,
376 ("[this=%p] dav1d_send_data -> %d", this, r));
377
378 if (r != 0) {
379 return r;
380 }
381
382 r = dav1d_get_picture(mContext, aPicture);
383
384 MOZ_LOG(sAVIFLog, r == 0 ? LogLevel::Debug : LogLevel::Error,
385 ("[this=%p] dav1d_get_picture -> %d", this, r));
386
387 // When bug 1682662 is fixed, revise this assert and subsequent condition
388 MOZ_ASSERT(aIsMetadataDecode || r == 0);
389
390 // We already have the AVIF_DECODE_RESULT histogram to record all the
391 // successful calls, so only bother recording what type of errors we see
392 // via events. Unlike AOM, dav1d returns an int, not an enum, so this is the
393 // easiest way to see if we're getting unexpected behavior to investigate.
394 if (aIsMetadataDecode && r != 0) {
395 // Uncomment once bug 1691156 is fixed
396 // mozilla::Telemetry::SetEventRecordingEnabled("avif"_ns, true);
397
398 mozilla::Telemetry::RecordEvent(
399 mozilla::Telemetry::EventID::Avif_Dav1dGetPicture_ReturnValue,
400 Some(nsPrintfCString("%d", r)), Nothing());
401 }
402
403 return r;
404 }
405
406 // A dummy callback for dav1d_data_wrap
Dav1dFreeCallback_s(const uint8_t * aBuf,void * aCookie)407 static void Dav1dFreeCallback_s(const uint8_t* aBuf, void* aCookie) {
408 // The buf is managed by the mParser inside Dav1dDecoder itself. Do nothing
409 // here.
410 }
411
412 static layers::PlanarYCbCrAData Dav1dPictureToYCbCrAData(
413 Dav1dPicture* aPicture, Dav1dPicture* aAlphaPlane,
414 bool aPremultipliedAlpha);
415
416 Dav1dContext* mContext = nullptr;
417
418 // The pictures are allocated once Decode() succeeds and will be deallocated
419 // when Dav1dDecoder is destroyed
420 Maybe<Dav1dPicture> mPicture;
421 Maybe<Dav1dPicture> mAlphaPlane;
422 };
423
424 class AOMDecoder final : AVIFDecoderInterface {
425 public:
~AOMDecoder()426 ~AOMDecoder() {
427 MOZ_LOG(sAVIFLog, LogLevel::Verbose, ("Destroy AOMDecoder=%p", this));
428
429 if (mContext.isSome()) {
430 aom_codec_err_t r = aom_codec_destroy(mContext.ptr());
431 MOZ_LOG(sAVIFLog, LogLevel::Debug,
432 ("[this=%p] aom_codec_destroy -> %d", this, r));
433 }
434 }
435
Create(UniquePtr<AVIFParser> && aParser,UniquePtr<AVIFDecoderInterface> & aDecoder)436 static DecodeResult Create(UniquePtr<AVIFParser>&& aParser,
437 UniquePtr<AVIFDecoderInterface>& aDecoder) {
438 UniquePtr<AOMDecoder> d(new AOMDecoder(std::move(aParser)));
439 aom_codec_err_t e = d->Init();
440 if (e == AOM_CODEC_OK) {
441 MOZ_ASSERT(d->mContext);
442 aDecoder.reset(d.release());
443 }
444 return AsVariant(AOMResult(e));
445 }
446
Decode(bool aIsMetadataDecode,const Mp4parseAvifImage & parsedImg)447 DecodeResult Decode(bool aIsMetadataDecode,
448 const Mp4parseAvifImage& parsedImg) override {
449 MOZ_ASSERT(mParser);
450 MOZ_ASSERT(mContext.isSome());
451 MOZ_ASSERT(mDecodedData.isNothing());
452
453 if (!parsedImg.primary_item.data || !parsedImg.primary_item.length) {
454 return AsVariant(NonDecoderResult::NoPrimaryItem);
455 }
456
457 aom_image_t* aomImg = nullptr;
458 DecodeResult r =
459 GetImage(parsedImg.primary_item, &aomImg, aIsMetadataDecode);
460 if (!IsDecodeSuccess(r)) {
461 return r;
462 }
463 MOZ_ASSERT(aomImg);
464
465 // The aomImg will be released in next GetImage call (aom_codec_decode
466 // actually). The GetImage could be called again immediately if parsedImg
467 // contains alpha data. Therefore, we need to copy the image and manage it
468 // by AOMDecoder itself.
469 OwnedAOMImage* clonedImg = OwnedAOMImage::CopyFrom(aomImg, false);
470 if (!clonedImg) {
471 return AsVariant(NonDecoderResult::OutOfMemory);
472 }
473 mOwnedImage.reset(clonedImg);
474
475 if (parsedImg.alpha_item.data && parsedImg.alpha_item.length) {
476 aom_image_t* alphaImg = nullptr;
477 DecodeResult r =
478 GetImage(parsedImg.alpha_item, &alphaImg, aIsMetadataDecode);
479 if (!IsDecodeSuccess(r)) {
480 return r;
481 }
482 MOZ_ASSERT(alphaImg);
483
484 OwnedAOMImage* clonedAlphaImg = OwnedAOMImage::CopyFrom(alphaImg, true);
485 if (!clonedAlphaImg) {
486 return AsVariant(NonDecoderResult::OutOfMemory);
487 }
488 mOwnedAlphaPlane.reset(clonedAlphaImg);
489
490 // Per § 4 of the AVIF spec
491 // https://aomediacodec.github.io/av1-avif/#auxiliary-images: An AV1 Alpha
492 // Image Item […] shall be encoded with the same bit depth as the
493 // associated master AV1 Image Item
494 MOZ_ASSERT(mOwnedImage->GetImage() && mOwnedAlphaPlane->GetImage());
495 if (mOwnedImage->GetImage()->bit_depth !=
496 mOwnedAlphaPlane->GetImage()->bit_depth) {
497 return AsVariant(NonDecoderResult::AlphaYColorDepthMismatch);
498 }
499 }
500
501 MOZ_ASSERT_IF(!mOwnedAlphaPlane, !parsedImg.premultiplied_alpha);
502 mDecodedData.emplace(AOMImageToYCbCrAData(
503 mOwnedImage->GetImage(),
504 mOwnedAlphaPlane ? mOwnedAlphaPlane->GetImage() : nullptr,
505 parsedImg.premultiplied_alpha));
506
507 return r;
508 }
509
510 private:
AOMDecoder(UniquePtr<AVIFParser> && aParser)511 explicit AOMDecoder(UniquePtr<AVIFParser>&& aParser)
512 : AVIFDecoderInterface(std::move(aParser)) {
513 MOZ_LOG(sAVIFLog, LogLevel::Verbose, ("Create AOMDecoder=%p", this));
514 }
515
Init()516 aom_codec_err_t Init() {
517 MOZ_ASSERT(mContext.isNothing());
518
519 aom_codec_iface_t* iface = aom_codec_av1_dx();
520 mContext.emplace();
521 aom_codec_err_t r = aom_codec_dec_init(
522 mContext.ptr(), iface, /* cfg = */ nullptr, /* flags = */ 0);
523
524 MOZ_LOG(sAVIFLog, r == AOM_CODEC_OK ? LogLevel::Verbose : LogLevel::Error,
525 ("[this=%p] aom_codec_dec_init -> %d, name = %s", this, r,
526 mContext->name));
527
528 if (r != AOM_CODEC_OK) {
529 mContext.reset();
530 }
531
532 return r;
533 }
534
GetImage(const Mp4parseByteData & aData,aom_image_t ** aImage,bool aIsMetadataDecode)535 DecodeResult GetImage(const Mp4parseByteData& aData, aom_image_t** aImage,
536 bool aIsMetadataDecode) {
537 MOZ_ASSERT(mContext.isSome());
538
539 aom_codec_err_t r =
540 aom_codec_decode(mContext.ptr(), aData.data, aData.length, nullptr);
541
542 MOZ_LOG(sAVIFLog, r == AOM_CODEC_OK ? LogLevel::Verbose : LogLevel::Error,
543 ("[this=%p] aom_codec_decode -> %d", this, r));
544
545 if (aIsMetadataDecode) {
546 switch (r) {
547 case AOM_CODEC_OK:
548 // No need to record any telemetry for the common case
549 break;
550 case AOM_CODEC_ERROR:
551 AccumulateCategorical(LABELS_AVIF_AOM_DECODE_ERROR::error);
552 break;
553 case AOM_CODEC_MEM_ERROR:
554 AccumulateCategorical(LABELS_AVIF_AOM_DECODE_ERROR::mem_error);
555 break;
556 case AOM_CODEC_ABI_MISMATCH:
557 AccumulateCategorical(LABELS_AVIF_AOM_DECODE_ERROR::abi_mismatch);
558 break;
559 case AOM_CODEC_INCAPABLE:
560 AccumulateCategorical(LABELS_AVIF_AOM_DECODE_ERROR::incapable);
561 break;
562 case AOM_CODEC_UNSUP_BITSTREAM:
563 AccumulateCategorical(LABELS_AVIF_AOM_DECODE_ERROR::unsup_bitstream);
564 break;
565 case AOM_CODEC_UNSUP_FEATURE:
566 AccumulateCategorical(LABELS_AVIF_AOM_DECODE_ERROR::unsup_feature);
567 break;
568 case AOM_CODEC_CORRUPT_FRAME:
569 AccumulateCategorical(LABELS_AVIF_AOM_DECODE_ERROR::corrupt_frame);
570 break;
571 case AOM_CODEC_INVALID_PARAM:
572 AccumulateCategorical(LABELS_AVIF_AOM_DECODE_ERROR::invalid_param);
573 break;
574 default:
575 MOZ_ASSERT_UNREACHABLE(
576 "Unknown aom_codec_err_t value from aom_codec_decode");
577 }
578 }
579
580 if (r != AOM_CODEC_OK) {
581 return AsVariant(AOMResult(r));
582 }
583
584 aom_codec_iter_t iter = nullptr;
585 aom_image_t* img = aom_codec_get_frame(mContext.ptr(), &iter);
586
587 MOZ_LOG(sAVIFLog, img == nullptr ? LogLevel::Error : LogLevel::Verbose,
588 ("[this=%p] aom_codec_get_frame -> %p", this, img));
589
590 if (img == nullptr) {
591 return AsVariant(AOMResult(NonAOMCodecError::NoFrame));
592 }
593
594 const CheckedInt<int> decoded_width = img->d_w;
595 const CheckedInt<int> decoded_height = img->d_h;
596
597 if (!decoded_height.isValid() || !decoded_width.isValid()) {
598 MOZ_LOG(sAVIFLog, LogLevel::Debug,
599 ("[this=%p] image dimensions can't be stored in int: d_w: %u, "
600 "d_h: %u",
601 this, img->d_w, img->d_h));
602 return AsVariant(AOMResult(NonAOMCodecError::SizeOverflow));
603 }
604
605 *aImage = img;
606 return AsVariant(AOMResult(r));
607 }
608
609 class OwnedAOMImage {
610 public:
~OwnedAOMImage()611 ~OwnedAOMImage() {
612 MOZ_LOG(sAVIFLog, LogLevel::Verbose, ("Destroy OwnedAOMImage=%p", this));
613 };
614
CopyFrom(aom_image_t * aImage,bool aIsAlpha)615 static OwnedAOMImage* CopyFrom(aom_image_t* aImage, bool aIsAlpha) {
616 MOZ_ASSERT(aImage);
617 UniquePtr<OwnedAOMImage> img(new OwnedAOMImage());
618 if (!img->CloneFrom(aImage, aIsAlpha)) {
619 return nullptr;
620 }
621 return img.release();
622 }
623
GetImage()624 aom_image_t* GetImage() { return mImage.isSome() ? mImage.ptr() : nullptr; }
625
626 private:
OwnedAOMImage()627 OwnedAOMImage() {
628 MOZ_LOG(sAVIFLog, LogLevel::Verbose, ("Create OwnedAOMImage=%p", this));
629 };
630
CloneFrom(aom_image_t * aImage,bool aIsAlpha)631 bool CloneFrom(aom_image_t* aImage, bool aIsAlpha) {
632 MOZ_ASSERT(aImage);
633 MOZ_ASSERT(!mImage);
634 MOZ_ASSERT(!mBuffer);
635
636 uint8_t* srcY = aImage->planes[AOM_PLANE_Y];
637 int yStride = aImage->stride[AOM_PLANE_Y];
638 int yHeight = aom_img_plane_height(aImage, AOM_PLANE_Y);
639 size_t yBufSize = yStride * yHeight;
640
641 // If aImage is alpha plane. The data is located in Y channel.
642 if (aIsAlpha) {
643 mBuffer = MakeUnique<uint8_t[]>(yBufSize);
644 if (!mBuffer) {
645 return false;
646 }
647 uint8_t* destY = mBuffer.get();
648 memcpy(destY, srcY, yBufSize);
649 mImage.emplace(*aImage);
650 mImage->planes[AOM_PLANE_Y] = destY;
651
652 return true;
653 }
654
655 uint8_t* srcCb = aImage->planes[AOM_PLANE_U];
656 int cbStride = aImage->stride[AOM_PLANE_U];
657 int cbHeight = aom_img_plane_height(aImage, AOM_PLANE_U);
658 size_t cbBufSize = cbStride * cbHeight;
659
660 uint8_t* srcCr = aImage->planes[AOM_PLANE_V];
661 int crStride = aImage->stride[AOM_PLANE_V];
662 int crHeight = aom_img_plane_height(aImage, AOM_PLANE_V);
663 size_t crBufSize = crStride * crHeight;
664
665 mBuffer = MakeUnique<uint8_t[]>(yBufSize + cbBufSize + crBufSize);
666 if (!mBuffer) {
667 return false;
668 }
669
670 uint8_t* destY = mBuffer.get();
671 uint8_t* destCb = destY + yBufSize;
672 uint8_t* destCr = destCb + cbBufSize;
673
674 memcpy(destY, srcY, yBufSize);
675 memcpy(destCb, srcCb, cbBufSize);
676 memcpy(destCr, srcCr, crBufSize);
677
678 mImage.emplace(*aImage);
679 mImage->planes[AOM_PLANE_Y] = destY;
680 mImage->planes[AOM_PLANE_U] = destCb;
681 mImage->planes[AOM_PLANE_V] = destCr;
682
683 return true;
684 }
685
686 // The mImage's planes are referenced to mBuffer
687 Maybe<aom_image_t> mImage;
688 UniquePtr<uint8_t[]> mBuffer;
689 };
690
691 static layers::PlanarYCbCrAData AOMImageToYCbCrAData(
692 aom_image_t* aImage, aom_image_t* aAlphaPlane, bool aPremultipliedAlpha);
693
694 Maybe<aom_codec_ctx_t> mContext;
695 UniquePtr<OwnedAOMImage> mOwnedImage;
696 UniquePtr<OwnedAOMImage> mOwnedAlphaPlane;
697 };
698
699 /* static */
Dav1dPictureToYCbCrAData(Dav1dPicture * aPicture,Dav1dPicture * aAlphaPlane,bool aPremultipliedAlpha)700 layers::PlanarYCbCrAData Dav1dDecoder::Dav1dPictureToYCbCrAData(
701 Dav1dPicture* aPicture, Dav1dPicture* aAlphaPlane,
702 bool aPremultipliedAlpha) {
703 MOZ_ASSERT(aPicture);
704
705 static_assert(std::is_same<int, decltype(aPicture->p.w)>::value);
706 static_assert(std::is_same<int, decltype(aPicture->p.h)>::value);
707
708 layers::PlanarYCbCrAData data;
709
710 data.mYChannel = static_cast<uint8_t*>(aPicture->data[0]);
711 data.mYStride = aPicture->stride[0];
712 data.mYSize = gfx::IntSize(aPicture->p.w, aPicture->p.h);
713 data.mYSkip = aPicture->stride[0] - aPicture->p.w;
714 data.mCbChannel = static_cast<uint8_t*>(aPicture->data[1]);
715 data.mCrChannel = static_cast<uint8_t*>(aPicture->data[2]);
716 data.mCbCrStride = aPicture->stride[1];
717
718 switch (aPicture->p.layout) {
719 case DAV1D_PIXEL_LAYOUT_I400: // Monochrome, so no Cb or Cr channels
720 data.mCbCrSize = gfx::IntSize(0, 0);
721 break;
722 case DAV1D_PIXEL_LAYOUT_I420:
723 data.mCbCrSize =
724 gfx::IntSize((aPicture->p.w + 1) / 2, (aPicture->p.h + 1) / 2);
725 break;
726 case DAV1D_PIXEL_LAYOUT_I422:
727 data.mCbCrSize = gfx::IntSize((aPicture->p.w + 1) / 2, aPicture->p.h);
728 break;
729 case DAV1D_PIXEL_LAYOUT_I444:
730 data.mCbCrSize = gfx::IntSize(aPicture->p.w, aPicture->p.h);
731 break;
732 default:
733 MOZ_ASSERT_UNREACHABLE("Unknown pixel layout");
734 }
735
736 data.mCbSkip = aPicture->stride[1] - aPicture->p.w;
737 data.mCrSkip = aPicture->stride[1] - aPicture->p.w;
738 data.mPicX = 0;
739 data.mPicY = 0;
740 data.mPicSize = data.mYSize;
741 data.mStereoMode = StereoMode::MONO;
742 data.mColorDepth = ColorDepthForBitDepth(aPicture->p.bpc);
743
744 auto colorSpace = GetColorSpace(*aPicture, sAVIFLog);
745 if (!colorSpace) {
746 // MIAF specific: UNKNOWN color space should be treated as BT601
747 colorSpace = Some(gfx::YUVColorSpace::BT601);
748 }
749 data.mYUVColorSpace = *colorSpace;
750
751 data.mColorRange = aPicture->seq_hdr->color_range ? gfx::ColorRange::FULL
752 : gfx::ColorRange::LIMITED;
753
754 if (aAlphaPlane) {
755 MOZ_ASSERT(aAlphaPlane->stride[0] == data.mYStride);
756 data.mAlphaChannel = static_cast<uint8_t*>(aAlphaPlane->data[0]);
757 data.mAlphaSize = gfx::IntSize(aAlphaPlane->p.w, aAlphaPlane->p.h);
758 data.mPremultipliedAlpha = aPremultipliedAlpha;
759 }
760
761 return data;
762 }
763
764 /* static */
AOMImageToYCbCrAData(aom_image_t * aImage,aom_image_t * aAlphaPlane,bool aPremultipliedAlpha)765 layers::PlanarYCbCrAData AOMDecoder::AOMImageToYCbCrAData(
766 aom_image_t* aImage, aom_image_t* aAlphaPlane, bool aPremultipliedAlpha) {
767 MOZ_ASSERT(aImage);
768 MOZ_ASSERT(aImage->stride[AOM_PLANE_Y] == aImage->stride[AOM_PLANE_ALPHA]);
769 MOZ_ASSERT(aImage->stride[AOM_PLANE_Y] >=
770 aom_img_plane_width(aImage, AOM_PLANE_Y));
771 MOZ_ASSERT(aImage->stride[AOM_PLANE_U] == aImage->stride[AOM_PLANE_V]);
772 MOZ_ASSERT(aImage->stride[AOM_PLANE_U] >=
773 aom_img_plane_width(aImage, AOM_PLANE_U));
774 MOZ_ASSERT(aImage->stride[AOM_PLANE_V] >=
775 aom_img_plane_width(aImage, AOM_PLANE_V));
776 MOZ_ASSERT(aom_img_plane_width(aImage, AOM_PLANE_U) ==
777 aom_img_plane_width(aImage, AOM_PLANE_V));
778 MOZ_ASSERT(aom_img_plane_height(aImage, AOM_PLANE_U) ==
779 aom_img_plane_height(aImage, AOM_PLANE_V));
780
781 layers::PlanarYCbCrAData data;
782
783 data.mYChannel = aImage->planes[AOM_PLANE_Y];
784 data.mYStride = aImage->stride[AOM_PLANE_Y];
785 data.mYSize = gfx::IntSize(aom_img_plane_width(aImage, AOM_PLANE_Y),
786 aom_img_plane_height(aImage, AOM_PLANE_Y));
787 data.mYSkip =
788 aImage->stride[AOM_PLANE_Y] - aom_img_plane_width(aImage, AOM_PLANE_Y);
789 data.mCbChannel = aImage->planes[AOM_PLANE_U];
790 data.mCrChannel = aImage->planes[AOM_PLANE_V];
791 data.mCbCrStride = aImage->stride[AOM_PLANE_U];
792 data.mCbCrSize = gfx::IntSize(aom_img_plane_width(aImage, AOM_PLANE_U),
793 aom_img_plane_height(aImage, AOM_PLANE_U));
794 data.mCbSkip =
795 aImage->stride[AOM_PLANE_U] - aom_img_plane_width(aImage, AOM_PLANE_U);
796 data.mCrSkip =
797 aImage->stride[AOM_PLANE_V] - aom_img_plane_width(aImage, AOM_PLANE_V);
798 data.mPicX = 0;
799 data.mPicY = 0;
800 data.mPicSize = gfx::IntSize(aImage->d_w, aImage->d_h);
801 data.mStereoMode = StereoMode::MONO;
802 data.mColorDepth = ColorDepthForBitDepth(aImage->bit_depth);
803
804 Maybe<gfx::YUVColorSpace> colorSpace;
805 switch (aImage->mc) {
806 case AOM_CICP_MC_BT_601:
807 colorSpace = Some(gfx::YUVColorSpace::BT601);
808 break;
809 case AOM_CICP_MC_BT_709:
810 colorSpace = Some(gfx::YUVColorSpace::BT709);
811 break;
812 case AOM_CICP_MC_BT_2020_NCL:
813 colorSpace = Some(gfx::YUVColorSpace::BT2020);
814 break;
815 case AOM_CICP_MC_BT_2020_CL:
816 colorSpace = Some(gfx::YUVColorSpace::BT2020);
817 break;
818 case AOM_CICP_MC_IDENTITY:
819 colorSpace = Some(gfx::YUVColorSpace::Identity);
820 break;
821 case AOM_CICP_MC_CHROMAT_NCL:
822 case AOM_CICP_MC_CHROMAT_CL:
823 case AOM_CICP_MC_UNSPECIFIED: // MIAF specific
824 switch (aImage->cp) {
825 case AOM_CICP_CP_BT_601:
826 colorSpace = Some(gfx::YUVColorSpace::BT601);
827 break;
828 case AOM_CICP_CP_BT_709:
829 colorSpace = Some(gfx::YUVColorSpace::BT709);
830 break;
831 case AOM_CICP_CP_BT_2020:
832 colorSpace = Some(gfx::YUVColorSpace::BT2020);
833 break;
834 default:
835 break;
836 }
837 break;
838 default:
839 MOZ_LOG(sAVIFLog, LogLevel::Debug,
840 ("unsupported aom_matrix_coefficients value: %u", aImage->mc));
841 break;
842 }
843 if (!colorSpace) {
844 // MIAF specific: UNKNOWN color space should be treated as BT601
845 colorSpace = Some(gfx::YUVColorSpace::BT601);
846 }
847 data.mYUVColorSpace = *colorSpace;
848
849 switch (aImage->range) {
850 case AOM_CR_STUDIO_RANGE:
851 data.mColorRange = gfx::ColorRange::LIMITED;
852 break;
853 case AOM_CR_FULL_RANGE:
854 data.mColorRange = gfx::ColorRange::FULL;
855 break;
856 default:
857 MOZ_ASSERT_UNREACHABLE("unknown color range");
858 }
859
860 if (aAlphaPlane) {
861 MOZ_ASSERT(aAlphaPlane->stride[AOM_PLANE_Y] == data.mYStride);
862 data.mAlphaChannel = aAlphaPlane->planes[AOM_PLANE_Y];
863 data.mAlphaSize = gfx::IntSize(aAlphaPlane->d_w, aAlphaPlane->d_h);
864 data.mPremultipliedAlpha = aPremultipliedAlpha;
865 }
866
867 return data;
868 }
869
870 // Wrapper to allow rust to call our read adaptor.
ReadSource(uint8_t * aDestBuf,uintptr_t aDestBufSize,void * aUserData)871 intptr_t nsAVIFDecoder::ReadSource(uint8_t* aDestBuf, uintptr_t aDestBufSize,
872 void* aUserData) {
873 MOZ_ASSERT(aDestBuf);
874 MOZ_ASSERT(aUserData);
875
876 MOZ_LOG(sAVIFLog, LogLevel::Verbose,
877 ("AVIF ReadSource, aDestBufSize: %zu", aDestBufSize));
878
879 auto* decoder = reinterpret_cast<nsAVIFDecoder*>(aUserData);
880
881 MOZ_ASSERT(decoder->mReadCursor);
882
883 size_t bufferLength = decoder->mBufferedData.end() - decoder->mReadCursor;
884 size_t n_bytes = std::min(aDestBufSize, bufferLength);
885
886 MOZ_LOG(
887 sAVIFLog, LogLevel::Verbose,
888 ("AVIF ReadSource, %zu bytes ready, copying %zu", bufferLength, n_bytes));
889
890 memcpy(aDestBuf, decoder->mReadCursor, n_bytes);
891 decoder->mReadCursor += n_bytes;
892
893 return n_bytes;
894 }
895
nsAVIFDecoder(RasterImage * aImage)896 nsAVIFDecoder::nsAVIFDecoder(RasterImage* aImage) : Decoder(aImage) {
897 MOZ_LOG(sAVIFLog, LogLevel::Debug,
898 ("[this=%p] nsAVIFDecoder::nsAVIFDecoder", this));
899 }
900
~nsAVIFDecoder()901 nsAVIFDecoder::~nsAVIFDecoder() {
902 MOZ_LOG(sAVIFLog, LogLevel::Debug,
903 ("[this=%p] nsAVIFDecoder::~nsAVIFDecoder", this));
904 }
905
DoDecode(SourceBufferIterator & aIterator,IResumable * aOnResume)906 LexerResult nsAVIFDecoder::DoDecode(SourceBufferIterator& aIterator,
907 IResumable* aOnResume) {
908 DecodeResult result = Decode(aIterator, aOnResume);
909
910 RecordDecodeResultTelemetry(result);
911
912 if (result.is<NonDecoderResult>()) {
913 NonDecoderResult r = result.as<NonDecoderResult>();
914 if (r == NonDecoderResult::NeedMoreData) {
915 return LexerResult(Yield::NEED_MORE_DATA);
916 }
917 return r == NonDecoderResult::MetadataOk
918 ? LexerResult(TerminalState::SUCCESS)
919 : LexerResult(TerminalState::FAILURE);
920 }
921
922 MOZ_ASSERT(result.is<Dav1dResult>() || result.is<AOMResult>());
923 return LexerResult(IsDecodeSuccess(result) ? TerminalState::SUCCESS
924 : TerminalState::FAILURE);
925 }
926
Decode(SourceBufferIterator & aIterator,IResumable * aOnResume)927 nsAVIFDecoder::DecodeResult nsAVIFDecoder::Decode(
928 SourceBufferIterator& aIterator, IResumable* aOnResume) {
929 MOZ_LOG(sAVIFLog, LogLevel::Debug,
930 ("[this=%p] nsAVIFDecoder::DoDecode", this));
931
932 // Since the SourceBufferIterator doesn't guarantee a contiguous buffer,
933 // but the current mp4parse-rust implementation requires it, always buffer
934 // locally. This keeps the code simpler at the cost of some performance, but
935 // this implementation is only experimental, so we don't want to spend time
936 // optimizing it prematurely.
937 while (!mReadCursor) {
938 SourceBufferIterator::State state =
939 aIterator.AdvanceOrScheduleResume(SIZE_MAX, aOnResume);
940
941 MOZ_LOG(sAVIFLog, LogLevel::Debug,
942 ("[this=%p] After advance, iterator state is %d", this, state));
943
944 switch (state) {
945 case SourceBufferIterator::WAITING:
946 return AsVariant(NonDecoderResult::NeedMoreData);
947
948 case SourceBufferIterator::COMPLETE:
949 mReadCursor = mBufferedData.begin();
950 break;
951
952 case SourceBufferIterator::READY: { // copy new data to buffer
953 MOZ_LOG(sAVIFLog, LogLevel::Debug,
954 ("[this=%p] SourceBufferIterator ready, %zu bytes available",
955 this, aIterator.Length()));
956
957 bool appendSuccess =
958 mBufferedData.append(aIterator.Data(), aIterator.Length());
959
960 if (!appendSuccess) {
961 MOZ_LOG(sAVIFLog, LogLevel::Error,
962 ("[this=%p] Failed to append %zu bytes to buffer", this,
963 aIterator.Length()));
964 }
965
966 break;
967 }
968
969 default:
970 MOZ_ASSERT_UNREACHABLE("unexpected SourceBufferIterator state");
971 }
972 }
973
974 Mp4parseIo io = {nsAVIFDecoder::ReadSource, this};
975 UniquePtr<AVIFParser> parser(AVIFParser::Create(&io));
976 if (!parser) {
977 return AsVariant(NonDecoderResult::ParseError);
978 }
979
980 const Mp4parseAvifImage* parsedImagePtr = parser->GetImage();
981 if (!parsedImagePtr) {
982 return AsVariant(NonDecoderResult::NoPrimaryItem);
983 }
984 const Mp4parseAvifImage& parsedImg = *parsedImagePtr;
985
986 if (parsedImg.alpha_item.data) {
987 PostHasTransparency();
988 }
989
990 Orientation orientation = StaticPrefs::image_avif_apply_transforms()
991 ? GetImageOrientation(parsedImg)
992 : Orientation{};
993 MaybeIntSize parsedImageSize = GetImageSize(parsedImg);
994
995 if (parsedImageSize.isSome()) {
996 MOZ_LOG(sAVIFLog, LogLevel::Debug,
997 ("[this=%p] Parser returned image size %d x %d", this,
998 parsedImageSize->width, parsedImageSize->height));
999 PostSize(parsedImageSize->width, parsedImageSize->height, orientation);
1000 if (IsMetadataDecode()) {
1001 MOZ_LOG(
1002 sAVIFLog, LogLevel::Debug,
1003 ("[this=%p] Finishing metadata decode without image decode", this));
1004 return AsVariant(NonDecoderResult::MetadataOk);
1005 }
1006 } else {
1007 MOZ_LOG(sAVIFLog, LogLevel::Error,
1008 ("[this=%p] Parser returned no image size, decoding...", this));
1009 }
1010
1011 UniquePtr<AVIFDecoderInterface> decoder;
1012 DecodeResult r = StaticPrefs::image_avif_use_dav1d()
1013 ? Dav1dDecoder::Create(std::move(parser), decoder)
1014 : AOMDecoder::Create(std::move(parser), decoder);
1015
1016 MOZ_LOG(sAVIFLog, LogLevel::Debug,
1017 ("[this=%p] Create %sDecoder %ssuccessfully", this,
1018 StaticPrefs::image_avif_use_dav1d() ? "Dav1d" : "AOM",
1019 IsDecodeSuccess(r) ? "" : "un"));
1020
1021 if (!IsDecodeSuccess(r)) {
1022 return r;
1023 }
1024
1025 MOZ_ASSERT(decoder);
1026 r = decoder->Decode(IsMetadataDecode(), parsedImg);
1027 MOZ_LOG(sAVIFLog, LogLevel::Debug,
1028 ("[this=%p] Decoder%s->Decode() %s", this,
1029 StaticPrefs::image_avif_use_dav1d() ? "Dav1d" : "AOM",
1030 IsDecodeSuccess(r) ? "succeeds" : "fails"));
1031
1032 if (!IsDecodeSuccess(r)) {
1033 return r;
1034 }
1035
1036 layers::PlanarYCbCrAData& decodedData = decoder->GetDecodedData();
1037
1038 // Technically it's valid but we don't handle it now (Bug 1682318).
1039 if (decodedData.hasAlpha() && decodedData.mAlphaSize != decodedData.mYSize) {
1040 return AsVariant(NonDecoderResult::AlphaYSizeMismatch);
1041 }
1042
1043 if (parsedImageSize.isNothing()) {
1044 // Bug 1696045: TODO add telemetry for missing ispe
1045 MOZ_LOG(sAVIFLog, LogLevel::Error,
1046 ("[this=%p] Using decoded image size: %d x %d", this,
1047 decodedData.mPicSize.width, decodedData.mPicSize.height));
1048 PostSize(decodedData.mPicSize.width, decodedData.mPicSize.height,
1049 orientation);
1050 } else if (decodedData.mPicSize.width != parsedImageSize->width ||
1051 decodedData.mPicSize.height != parsedImageSize->height) {
1052 MOZ_LOG(sAVIFLog, LogLevel::Error,
1053 ("[this=%p] Metadata image size doesn't match decoded image size: "
1054 "(%d x %d) != (%d x %d)",
1055 this, parsedImageSize->width, parsedImageSize->height,
1056 decodedData.mPicSize.width, decodedData.mPicSize.height));
1057 // Bug 1696045: TODO need new error type
1058 return AsVariant(NonDecoderResult::ParseError);
1059 }
1060
1061 const bool hasAlpha = decodedData.hasAlpha();
1062 if (hasAlpha) {
1063 PostHasTransparency();
1064 }
1065
1066 if (IsMetadataDecode()) {
1067 return AsVariant(NonDecoderResult::MetadataOk);
1068 }
1069
1070 // These data must be recorded after metadata has been decoded
1071 // (IsMetadataDecode()=false) or else they would be double-counted.
1072 AccumulateCategorical(
1073 gColorSpaceLabel[static_cast<size_t>(decodedData.mYUVColorSpace)]);
1074 AccumulateCategorical(
1075 gColorDepthLabel[static_cast<size_t>(decodedData.mColorDepth)]);
1076
1077 IntSize rgbSize = Size();
1078 MOZ_ASSERT(rgbSize == decodedData.mPicSize);
1079
1080 // Get suggested format and size. Note that GetYCbCrToRGBDestFormatAndSize
1081 // force format to be B8G8R8X8 if it's not.
1082 gfx::SurfaceFormat format = SurfaceFormat::OS_RGBX;
1083 gfx::GetYCbCrToRGBDestFormatAndSize(decodedData, format, rgbSize);
1084 if (hasAlpha) {
1085 // We would use libyuv to do the YCbCrA -> ARGB convertion, which only works
1086 // for B8G8R8A8.
1087 format = SurfaceFormat::B8G8R8A8;
1088 }
1089
1090 const int bytesPerPixel = BytesPerPixel(format);
1091
1092 const CheckedInt rgbStride = CheckedInt<int>(rgbSize.width) * bytesPerPixel;
1093 const CheckedInt rgbBufLength = rgbStride * rgbSize.height;
1094
1095 if (!rgbStride.isValid() || !rgbBufLength.isValid()) {
1096 MOZ_LOG(sAVIFLog, LogLevel::Debug,
1097 ("[this=%p] overflow calculating rgbBufLength: rbgSize.width: %d, "
1098 "rgbSize.height: %d, "
1099 "bytesPerPixel: %u",
1100 this, rgbSize.width, rgbSize.height, bytesPerPixel));
1101 return AsVariant(NonDecoderResult::SizeOverflow);
1102 }
1103
1104 UniquePtr<uint8_t[]> rgbBuf = MakeUnique<uint8_t[]>(rgbBufLength.value());
1105 const uint8_t* endOfRgbBuf = {rgbBuf.get() + rgbBufLength.value()};
1106
1107 if (!rgbBuf) {
1108 MOZ_LOG(sAVIFLog, LogLevel::Debug,
1109 ("[this=%p] allocation of %u-byte rgbBuf failed", this,
1110 rgbBufLength.value()));
1111 return AsVariant(NonDecoderResult::OutOfMemory);
1112 }
1113
1114 if (hasAlpha) {
1115 const auto wantPremultiply =
1116 !bool(GetSurfaceFlags() & SurfaceFlags::NO_PREMULTIPLY_ALPHA);
1117 const bool& hasPremultiply = decodedData.mPremultipliedAlpha;
1118
1119 PremultFunc premultOp = nullptr;
1120 if (wantPremultiply && !hasPremultiply) {
1121 premultOp = libyuv::ARGBAttenuate;
1122 } else if (!wantPremultiply && hasPremultiply) {
1123 premultOp = libyuv::ARGBUnattenuate;
1124 }
1125
1126 MOZ_LOG(sAVIFLog, LogLevel::Debug,
1127 ("[this=%p] calling gfx::ConvertYCbCrAToARGB premultOp: %p", this,
1128 premultOp));
1129 gfx::ConvertYCbCrAToARGB(decodedData, format, rgbSize, rgbBuf.get(),
1130 rgbStride.value(), premultOp);
1131 } else {
1132 MOZ_LOG(sAVIFLog, LogLevel::Debug,
1133 ("[this=%p] calling gfx::ConvertYCbCrToRGB", this));
1134 gfx::ConvertYCbCrToRGB(decodedData, format, rgbSize, rgbBuf.get(),
1135 rgbStride.value());
1136 }
1137
1138 MOZ_LOG(sAVIFLog, LogLevel::Debug,
1139 ("[this=%p] calling SurfacePipeFactory::CreateSurfacePipe", this));
1140 Maybe<SurfacePipe> pipe = SurfacePipeFactory::CreateSurfacePipe(
1141 this, rgbSize, OutputSize(), FullFrame(), format, format, Nothing(),
1142 nullptr, SurfacePipeFlags());
1143
1144 if (!pipe) {
1145 MOZ_LOG(sAVIFLog, LogLevel::Debug,
1146 ("[this=%p] could not initialize surface pipe", this));
1147 return AsVariant(NonDecoderResult::PipeInitError);
1148 }
1149
1150 MOZ_LOG(sAVIFLog, LogLevel::Debug, ("[this=%p] writing to surface", this));
1151 WriteState writeBufferResult = WriteState::NEED_MORE_DATA;
1152 for (uint8_t* rowPtr = rgbBuf.get(); rowPtr < endOfRgbBuf;
1153 rowPtr += rgbStride.value()) {
1154 writeBufferResult = pipe->WriteBuffer(reinterpret_cast<uint32_t*>(rowPtr));
1155
1156 Maybe<SurfaceInvalidRect> invalidRect = pipe->TakeInvalidRect();
1157 if (invalidRect) {
1158 PostInvalidation(invalidRect->mInputSpaceRect,
1159 Some(invalidRect->mOutputSpaceRect));
1160 }
1161
1162 if (writeBufferResult == WriteState::FAILURE) {
1163 MOZ_LOG(sAVIFLog, LogLevel::Debug,
1164 ("[this=%p] error writing rowPtr to surface pipe", this));
1165
1166 } else if (writeBufferResult == WriteState::FINISHED) {
1167 MOZ_ASSERT(rowPtr + rgbStride.value() == endOfRgbBuf);
1168 }
1169 }
1170
1171 MOZ_LOG(sAVIFLog, LogLevel::Debug,
1172 ("[this=%p] writing to surface complete", this));
1173
1174 if (writeBufferResult == WriteState::FINISHED) {
1175 PostFrameStop(hasAlpha ? Opacity::SOME_TRANSPARENCY
1176 : Opacity::FULLY_OPAQUE);
1177 PostDecodeDone();
1178 return r;
1179 }
1180
1181 return AsVariant(NonDecoderResult::WriteBufferError);
1182 }
1183
1184 /* static */
IsDecodeSuccess(const DecodeResult & aResult)1185 bool nsAVIFDecoder::IsDecodeSuccess(const DecodeResult& aResult) {
1186 if (aResult.is<Dav1dResult>() || aResult.is<AOMResult>()) {
1187 return aResult == DecodeResult(Dav1dResult(0)) ||
1188 aResult == DecodeResult(AOMResult(AOM_CODEC_OK));
1189 }
1190 return false;
1191 }
1192
RecordDecodeResultTelemetry(const nsAVIFDecoder::DecodeResult & aResult)1193 void nsAVIFDecoder::RecordDecodeResultTelemetry(
1194 const nsAVIFDecoder::DecodeResult& aResult) {
1195 if (aResult.is<NonDecoderResult>()) {
1196 switch (aResult.as<NonDecoderResult>()) {
1197 case NonDecoderResult::NeedMoreData:
1198 break;
1199 case NonDecoderResult::MetadataOk:
1200 break;
1201 case NonDecoderResult::ParseError:
1202 AccumulateCategorical(LABELS_AVIF_DECODE_RESULT::parse_error);
1203 break;
1204 case NonDecoderResult::NoPrimaryItem:
1205 AccumulateCategorical(LABELS_AVIF_DECODE_RESULT::no_primary_item);
1206 break;
1207 case NonDecoderResult::SizeOverflow:
1208 AccumulateCategorical(LABELS_AVIF_DECODE_RESULT::size_overflow);
1209 break;
1210 case NonDecoderResult::OutOfMemory:
1211 AccumulateCategorical(LABELS_AVIF_DECODE_RESULT::out_of_memory);
1212 break;
1213 case NonDecoderResult::PipeInitError:
1214 AccumulateCategorical(LABELS_AVIF_DECODE_RESULT::pipe_init_error);
1215 break;
1216 case NonDecoderResult::WriteBufferError:
1217 AccumulateCategorical(LABELS_AVIF_DECODE_RESULT::write_buffer_error);
1218 break;
1219 case NonDecoderResult::AlphaYSizeMismatch:
1220 AccumulateCategorical(LABELS_AVIF_DECODE_RESULT::alpha_y_sz_mismatch);
1221 break;
1222 case NonDecoderResult::AlphaYColorDepthMismatch:
1223 AccumulateCategorical(LABELS_AVIF_DECODE_RESULT::alpha_y_bpc_mismatch);
1224 break;
1225 default:
1226 MOZ_ASSERT_UNREACHABLE("unknown result");
1227 break;
1228 }
1229 } else {
1230 MOZ_ASSERT(aResult.is<Dav1dResult>() || aResult.is<AOMResult>());
1231 AccumulateCategorical(aResult.is<Dav1dResult>() ? LABELS_AVIF_DECODER::dav1d
1232 : LABELS_AVIF_DECODER::aom);
1233 AccumulateCategorical(IsDecodeSuccess(aResult)
1234 ? LABELS_AVIF_DECODE_RESULT::success
1235 : LABELS_AVIF_DECODE_RESULT::decode_error);
1236 }
1237 }
1238
1239 } // namespace image
1240 } // namespace mozilla
1241