1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #ifndef VideoUtils_h
8 #define VideoUtils_h
9
10 #include "AudioSampleFormat.h"
11 #include "MediaInfo.h"
12 #include "TimeUnits.h"
13 #include "VideoLimits.h"
14 #include "mozilla/gfx/Point.h" // for gfx::IntSize
15 #include "mozilla/gfx/Types.h"
16 #include "mozilla/AbstractThread.h"
17 #include "mozilla/Attributes.h"
18 #include "mozilla/CheckedInt.h"
19 #include "mozilla/MozPromise.h"
20 #include "mozilla/ReentrantMonitor.h"
21 #include "mozilla/RefPtr.h"
22 #include "mozilla/SharedThreadPool.h"
23 #include "mozilla/UniquePtr.h"
24 #include "nsCOMPtr.h"
25 #include "nsINamed.h"
26 #include "nsIThread.h"
27 #include "nsITimer.h"
28
29 #include "nsThreadUtils.h"
30 #include "prtime.h"
31
32 using mozilla::CheckedInt32;
33 using mozilla::CheckedInt64;
34 using mozilla::CheckedUint32;
35 using mozilla::CheckedUint64;
36
37 // This file contains stuff we'd rather put elsewhere, but which is
38 // dependent on other changes which we don't want to wait for. We plan to
39 // remove this file in the near future.
40
41 // This belongs in xpcom/monitor/Monitor.h, once we've made
42 // mozilla::Monitor non-reentrant.
43 namespace mozilla {
44
45 class MediaContainerType;
46
47 // EME Key System String.
48 #define EME_KEY_SYSTEM_CLEARKEY "org.w3.clearkey"
49 #define EME_KEY_SYSTEM_WIDEVINE "com.widevine.alpha"
50
51 /**
52 * ReentrantMonitorConditionallyEnter
53 *
54 * Enters the supplied monitor only if the conditional value |aEnter| is true.
55 * E.g. Used to allow unmonitored read access on the decode thread,
56 * and monitored access on all other threads.
57 */
58 class MOZ_STACK_CLASS ReentrantMonitorConditionallyEnter {
59 public:
ReentrantMonitorConditionallyEnter(bool aEnter,ReentrantMonitor & aReentrantMonitor)60 ReentrantMonitorConditionallyEnter(bool aEnter,
61 ReentrantMonitor& aReentrantMonitor)
62 : mReentrantMonitor(nullptr) {
63 MOZ_COUNT_CTOR(ReentrantMonitorConditionallyEnter);
64 if (aEnter) {
65 mReentrantMonitor = &aReentrantMonitor;
66 NS_ASSERTION(mReentrantMonitor, "null monitor");
67 mReentrantMonitor->Enter();
68 }
69 }
~ReentrantMonitorConditionallyEnter(void)70 ~ReentrantMonitorConditionallyEnter(void) {
71 if (mReentrantMonitor) {
72 mReentrantMonitor->Exit();
73 }
74 MOZ_COUNT_DTOR(ReentrantMonitorConditionallyEnter);
75 }
76
77 private:
78 // Restrict to constructor and destructor defined above.
79 ReentrantMonitorConditionallyEnter();
80 ReentrantMonitorConditionallyEnter(const ReentrantMonitorConditionallyEnter&);
81 ReentrantMonitorConditionallyEnter& operator=(
82 const ReentrantMonitorConditionallyEnter&);
83 static void* operator new(size_t) noexcept(true);
84 static void operator delete(void*);
85
86 ReentrantMonitor* mReentrantMonitor;
87 };
88
89 // Shuts down a thread asynchronously.
90 class ShutdownThreadEvent : public Runnable {
91 public:
ShutdownThreadEvent(nsIThread * aThread)92 explicit ShutdownThreadEvent(nsIThread* aThread)
93 : Runnable("ShutdownThreadEvent"), mThread(aThread) {}
94 ~ShutdownThreadEvent() = default;
Run()95 NS_IMETHOD Run() override {
96 mThread->Shutdown();
97 mThread = nullptr;
98 return NS_OK;
99 }
100
101 private:
102 nsCOMPtr<nsIThread> mThread;
103 };
104
105 class MediaResource;
106
107 // Estimates the buffered ranges of a MediaResource using a simple
108 // (byteOffset/length)*duration method. Probably inaccurate, but won't
109 // do file I/O, and can be used when we don't have detailed knowledge
110 // of the byte->time mapping of a resource. aDurationUsecs is the duration
111 // of the media in microseconds. Estimated buffered ranges are stored in
112 // aOutBuffered. Ranges are 0-normalized, i.e. in the range of (0,duration].
113 media::TimeIntervals GetEstimatedBufferedTimeRanges(
114 mozilla::MediaResource* aStream, int64_t aDurationUsecs);
115
116 // Converts from number of audio frames (aFrames) to microseconds, given
117 // the specified audio rate (aRate).
118 CheckedInt64 FramesToUsecs(int64_t aFrames, uint32_t aRate);
119 // Converts from number of audio frames (aFrames) TimeUnit, given
120 // the specified audio rate (aRate).
121 media::TimeUnit FramesToTimeUnit(int64_t aFrames, uint32_t aRate);
122 // Perform aValue * aMul / aDiv, reducing the possibility of overflow due to
123 // aValue * aMul overflowing.
124 CheckedInt64 SaferMultDiv(int64_t aValue, uint64_t aMul, uint64_t aDiv);
125
126 // Converts from microseconds (aUsecs) to number of audio frames, given the
127 // specified audio rate (aRate). Stores the result in aOutFrames. Returns
128 // true if the operation succeeded, or false if there was an integer
129 // overflow while calulating the conversion.
130 CheckedInt64 UsecsToFrames(int64_t aUsecs, uint32_t aRate);
131
132 // Format TimeUnit as number of frames at given rate.
133 CheckedInt64 TimeUnitToFrames(const media::TimeUnit& aTime, uint32_t aRate);
134
135 // Converts milliseconds to seconds.
136 #define MS_TO_SECONDS(ms) ((double)(ms) / (PR_MSEC_PER_SEC))
137
138 // Converts seconds to milliseconds.
139 #define SECONDS_TO_MS(s) ((int)((s) * (PR_MSEC_PER_SEC)))
140
141 // Converts from seconds to microseconds. Returns failure if the resulting
142 // integer is too big to fit in an int64_t.
143 nsresult SecondsToUsecs(double aSeconds, int64_t& aOutUsecs);
144
145 // Scales the display rect aDisplay by aspect ratio aAspectRatio.
146 // Note that aDisplay must be validated by IsValidVideoRegion()
147 // before being used!
148 void ScaleDisplayByAspectRatio(gfx::IntSize& aDisplay, float aAspectRatio);
149
150 // Downmix Stereo audio samples to Mono.
151 // Input are the buffer contains stereo data and the number of frames.
152 void DownmixStereoToMono(mozilla::AudioDataValue* aBuffer, uint32_t aFrames);
153
154 // Decide the number of playback channels according to the
155 // given AudioInfo and the prefs that are being set.
156 uint32_t DecideAudioPlaybackChannels(const AudioInfo& info);
157
158 bool IsDefaultPlaybackDeviceMono();
159
160 bool IsVideoContentType(const nsCString& aContentType);
161
162 // Returns true if it's safe to use aPicture as the picture to be
163 // extracted inside a frame of size aFrame, and scaled up to and displayed
164 // at a size of aDisplay. You should validate the frame, picture, and
165 // display regions before using them to display video frames.
166 bool IsValidVideoRegion(const gfx::IntSize& aFrame,
167 const gfx::IntRect& aPicture,
168 const gfx::IntSize& aDisplay);
169
170 // Template to automatically set a variable to a value on scope exit.
171 // Useful for unsetting flags, etc.
172 template <typename T>
173 class AutoSetOnScopeExit {
174 public:
AutoSetOnScopeExit(T & aVar,T aValue)175 AutoSetOnScopeExit(T& aVar, T aValue) : mVar(aVar), mValue(aValue) {}
~AutoSetOnScopeExit()176 ~AutoSetOnScopeExit() { mVar = mValue; }
177
178 private:
179 T& mVar;
180 const T mValue;
181 };
182
183 enum class MediaThreadType {
184 PLAYBACK, // MediaDecoderStateMachine and MediaFormatReader
185 PLATFORM_DECODER, // MediaDataDecoder
186 PLATFORM_ENCODER, // MediaDataEncoder
187 MTG_CONTROL,
188 WEBRTC_DECODER,
189 MDSM,
190 };
191 // Returns the thread pool that is shared amongst all decoder state machines
192 // for decoding streams.
193 already_AddRefed<SharedThreadPool> GetMediaThreadPool(MediaThreadType aType);
194
195 enum H264_PROFILE {
196 H264_PROFILE_UNKNOWN = 0,
197 H264_PROFILE_BASE = 0x42,
198 H264_PROFILE_MAIN = 0x4D,
199 H264_PROFILE_EXTENDED = 0x58,
200 H264_PROFILE_HIGH = 0x64,
201 };
202
203 enum H264_LEVEL {
204 H264_LEVEL_1 = 10,
205 H264_LEVEL_1_b = 11,
206 H264_LEVEL_1_1 = 11,
207 H264_LEVEL_1_2 = 12,
208 H264_LEVEL_1_3 = 13,
209 H264_LEVEL_2 = 20,
210 H264_LEVEL_2_1 = 21,
211 H264_LEVEL_2_2 = 22,
212 H264_LEVEL_3 = 30,
213 H264_LEVEL_3_1 = 31,
214 H264_LEVEL_3_2 = 32,
215 H264_LEVEL_4 = 40,
216 H264_LEVEL_4_1 = 41,
217 H264_LEVEL_4_2 = 42,
218 H264_LEVEL_5 = 50,
219 H264_LEVEL_5_1 = 51,
220 H264_LEVEL_5_2 = 52
221 };
222
223 // Extracts the H.264/AVC profile and level from an H.264 codecs string.
224 // H.264 codecs parameters have a type defined as avc1.PPCCLL, where
225 // PP = profile_idc, CC = constraint_set flags, LL = level_idc.
226 // See
227 // http://blog.pearce.org.nz/2013/11/what-does-h264avc1-codecs-parameters.html
228 // for more details.
229 // Returns false on failure.
230 bool ExtractH264CodecDetails(const nsAString& aCodecs, uint8_t& aProfile,
231 uint8_t& aConstraint, uint8_t& aLevel);
232
233 struct VideoColorSpace {
234 // TODO: Define the value type as strong type enum
235 // to better know the exact meaning corresponding to ISO/IEC 23001-8:2016.
236 // Default value is listed
237 // https://www.webmproject.org/vp9/mp4/#optional-fields
238 uint8_t mPrimaryId = 1; // Table 2
239 uint8_t mTransferId = 1; // Table 3
240 uint8_t mMatrixId = 1; // Table 4
241 uint8_t mRangeId = 0;
242 };
243
244 // Extracts the VPX codecs parameter string.
245 // See https://www.webmproject.org/vp9/mp4/#codecs-parameter-string
246 // for more details.
247 // Returns false on failure.
248 bool ExtractVPXCodecDetails(const nsAString& aCodec, uint8_t& aProfile,
249 uint8_t& aLevel, uint8_t& aBitDepth);
250 bool ExtractVPXCodecDetails(const nsAString& aCodec, uint8_t& aProfile,
251 uint8_t& aLevel, uint8_t& aBitDepth,
252 uint8_t& aChromaSubsampling,
253 VideoColorSpace& aColorSpace);
254
255 // Use a cryptographic quality PRNG to generate raw random bytes
256 // and convert that to a base64 string.
257 nsresult GenerateRandomName(nsCString& aOutSalt, uint32_t aLength);
258
259 // This version returns a string suitable for use as a file or URL
260 // path. This is based on code from nsExternalAppHandler::SetUpTempFile.
261 nsresult GenerateRandomPathName(nsCString& aOutSalt, uint32_t aLength);
262
263 already_AddRefed<TaskQueue> CreateMediaDecodeTaskQueue(const char* aName);
264
265 // Iteratively invokes aWork until aCondition returns true, or aWork returns
266 // false. Use this rather than a while loop to avoid bogarting the task queue.
267 template <class Work, class Condition>
InvokeUntil(Work aWork,Condition aCondition)268 RefPtr<GenericPromise> InvokeUntil(Work aWork, Condition aCondition) {
269 RefPtr<GenericPromise::Private> p = new GenericPromise::Private(__func__);
270
271 if (aCondition()) {
272 p->Resolve(true, __func__);
273 }
274
275 struct Helper {
276 static void Iteration(const RefPtr<GenericPromise::Private>& aPromise,
277 Work aLocalWork, Condition aLocalCondition) {
278 if (!aLocalWork()) {
279 aPromise->Reject(NS_ERROR_FAILURE, __func__);
280 } else if (aLocalCondition()) {
281 aPromise->Resolve(true, __func__);
282 } else {
283 nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction(
284 "InvokeUntil::Helper::Iteration",
285 [aPromise, aLocalWork, aLocalCondition]() {
286 Iteration(aPromise, aLocalWork, aLocalCondition);
287 });
288 AbstractThread::GetCurrent()->Dispatch(r.forget());
289 }
290 }
291 };
292
293 Helper::Iteration(p, aWork, aCondition);
294 return p;
295 }
296
297 // Simple timer to run a runnable after a timeout.
298 class SimpleTimer : public nsITimerCallback, public nsINamed {
299 public:
300 NS_DECL_ISUPPORTS
301 NS_DECL_NSINAMED
302
303 // Create a new timer to run aTask after aTimeoutMs milliseconds
304 // on thread aTarget. If aTarget is null, task is run on the main thread.
305 static already_AddRefed<SimpleTimer> Create(
306 nsIRunnable* aTask, uint32_t aTimeoutMs,
307 nsIEventTarget* aTarget = nullptr);
308 void Cancel();
309
310 NS_IMETHOD Notify(nsITimer* timer) override;
311
312 private:
313 virtual ~SimpleTimer() = default;
314 nsresult Init(nsIRunnable* aTask, uint32_t aTimeoutMs,
315 nsIEventTarget* aTarget);
316
317 RefPtr<nsIRunnable> mTask;
318 nsCOMPtr<nsITimer> mTimer;
319 };
320
321 void LogToBrowserConsole(const nsAString& aMsg);
322
323 bool ParseMIMETypeString(const nsAString& aMIMEType,
324 nsString& aOutContainerType,
325 nsTArray<nsString>& aOutCodecs);
326
327 bool ParseCodecsString(const nsAString& aCodecs,
328 nsTArray<nsString>& aOutCodecs);
329
330 bool IsH264CodecString(const nsAString& aCodec);
331
332 bool IsAACCodecString(const nsAString& aCodec);
333
334 bool IsVP8CodecString(const nsAString& aCodec);
335
336 bool IsVP9CodecString(const nsAString& aCodec);
337
338 bool IsAV1CodecString(const nsAString& aCodec);
339
340 // Try and create a TrackInfo with a given codec MIME type.
341 UniquePtr<TrackInfo> CreateTrackInfoWithMIMEType(
342 const nsACString& aCodecMIMEType);
343
344 // Try and create a TrackInfo with a given codec MIME type, and optional extra
345 // parameters from a container type (its MIME type and codecs are ignored).
346 UniquePtr<TrackInfo> CreateTrackInfoWithMIMETypeAndContainerTypeExtraParameters(
347 const nsACString& aCodecMIMEType, const MediaContainerType& aContainerType);
348
349 namespace detail {
350
351 // aString should start with aMajor + '/'.
StartsWithMIMETypeMajor(const char * aString,const char * aMajor,size_t aMajorRemaining)352 constexpr bool StartsWithMIMETypeMajor(const char* aString, const char* aMajor,
353 size_t aMajorRemaining) {
354 return (aMajorRemaining == 0 && *aString == '/') ||
355 (*aString == *aMajor &&
356 StartsWithMIMETypeMajor(aString + 1, aMajor + 1,
357 aMajorRemaining - 1));
358 }
359
360 // aString should only contain [a-z0-9\-\.] and a final '\0'.
EndsWithMIMESubtype(const char * aString,size_t aRemaining)361 constexpr bool EndsWithMIMESubtype(const char* aString, size_t aRemaining) {
362 return aRemaining == 0 || (((*aString >= 'a' && *aString <= 'z') ||
363 (*aString >= '0' && *aString <= '9') ||
364 *aString == '-' || *aString == '.') &&
365 EndsWithMIMESubtype(aString + 1, aRemaining - 1));
366 }
367
368 // Simple MIME-type literal string checker with a given (major) type.
369 // Only accepts "{aMajor}/[a-z0-9\-\.]+".
370 template <size_t MajorLengthPlus1>
IsMIMETypeWithMajor(const char * aString,size_t aLength,const char (& aMajor)[MajorLengthPlus1])371 constexpr bool IsMIMETypeWithMajor(const char* aString, size_t aLength,
372 const char (&aMajor)[MajorLengthPlus1]) {
373 return aLength > MajorLengthPlus1 && // Major + '/' + at least 1 char
374 StartsWithMIMETypeMajor(aString, aMajor, MajorLengthPlus1 - 1) &&
375 EndsWithMIMESubtype(aString + MajorLengthPlus1,
376 aLength - MajorLengthPlus1);
377 }
378
379 } // namespace detail
380
381 // Simple MIME-type string checker.
382 // Only accepts lowercase "{application,audio,video}/[a-z0-9\-\.]+".
383 // Add more if necessary.
IsMediaMIMEType(const char * aString,size_t aLength)384 constexpr bool IsMediaMIMEType(const char* aString, size_t aLength) {
385 return detail::IsMIMETypeWithMajor(aString, aLength, "application") ||
386 detail::IsMIMETypeWithMajor(aString, aLength, "audio") ||
387 detail::IsMIMETypeWithMajor(aString, aLength, "video");
388 }
389
390 // Simple MIME-type string literal checker.
391 // Only accepts lowercase "{application,audio,video}/[a-z0-9\-\.]+".
392 // Add more if necessary.
393 template <size_t LengthPlus1>
IsMediaMIMEType(const char (& aString)[LengthPlus1])394 constexpr bool IsMediaMIMEType(const char (&aString)[LengthPlus1]) {
395 return IsMediaMIMEType(aString, LengthPlus1 - 1);
396 }
397
398 // Simple MIME-type string checker.
399 // Only accepts lowercase "{application,audio,video}/[a-z0-9\-\.]+".
400 // Add more if necessary.
IsMediaMIMEType(const nsACString & aString)401 inline bool IsMediaMIMEType(const nsACString& aString) {
402 return IsMediaMIMEType(aString.Data(), aString.Length());
403 }
404
405 enum class StringListRangeEmptyItems {
406 // Skip all empty items (empty string will process nothing)
407 // E.g.: "a,,b" -> ["a", "b"], "" -> nothing
408 Skip,
409 // Process all, except if string is empty
410 // E.g.: "a,,b" -> ["a", "", "b"], "" -> nothing
411 ProcessEmptyItems,
412 // Process all, including 1 empty item in an empty string
413 // E.g.: "a,,b" -> ["a", "", "b"], "" -> [""]
414 ProcessAll
415 };
416
417 template <typename String,
418 StringListRangeEmptyItems empties = StringListRangeEmptyItems::Skip>
419 class StringListRange {
420 typedef typename String::char_type CharType;
421 typedef const CharType* Pointer;
422
423 public:
424 // Iterator into range, trims items and optionally skips empty items.
425 class Iterator {
426 public:
427 bool operator!=(const Iterator& a) const {
428 return mStart != a.mStart || mEnd != a.mEnd;
429 }
430 Iterator& operator++() {
431 SearchItemAt(mComma + 1);
432 return *this;
433 }
434 // DereferencedType should be 'const nsDependent[C]String' pointing into
435 // mList (which is 'const ns[C]String&').
436 typedef decltype(Substring(Pointer(), Pointer())) DereferencedType;
437 DereferencedType operator*() { return Substring(mStart, mEnd); }
438
439 private:
440 friend class StringListRange;
Iterator(const CharType * aRangeStart,uint32_t aLength)441 Iterator(const CharType* aRangeStart, uint32_t aLength)
442 : mRangeEnd(aRangeStart + aLength),
443 mStart(nullptr),
444 mEnd(nullptr),
445 mComma(nullptr) {
446 SearchItemAt(aRangeStart);
447 }
SearchItemAt(Pointer start)448 void SearchItemAt(Pointer start) {
449 // First, skip leading whitespace.
450 for (Pointer p = start;; ++p) {
451 if (p >= mRangeEnd) {
452 if (p > mRangeEnd +
453 (empties != StringListRangeEmptyItems::Skip ? 1 : 0)) {
454 p = mRangeEnd +
455 (empties != StringListRangeEmptyItems::Skip ? 1 : 0);
456 }
457 mStart = mEnd = mComma = p;
458 return;
459 }
460 auto c = *p;
461 if (c == CharType(',')) {
462 // Comma -> Empty item -> Skip or process?
463 if (empties != StringListRangeEmptyItems::Skip) {
464 mStart = mEnd = mComma = p;
465 return;
466 }
467 } else if (c != CharType(' ')) {
468 mStart = p;
469 break;
470 }
471 }
472 // Find comma, recording start of trailing space.
473 Pointer trailingWhitespace = nullptr;
474 for (Pointer p = mStart + 1;; ++p) {
475 if (p >= mRangeEnd) {
476 mEnd = trailingWhitespace ? trailingWhitespace : p;
477 mComma = p;
478 return;
479 }
480 auto c = *p;
481 if (c == CharType(',')) {
482 mEnd = trailingWhitespace ? trailingWhitespace : p;
483 mComma = p;
484 return;
485 }
486 if (c == CharType(' ')) {
487 // Found a whitespace -> Record as trailing if not first one.
488 if (!trailingWhitespace) {
489 trailingWhitespace = p;
490 }
491 } else {
492 // Found a non-whitespace -> Reset trailing whitespace if needed.
493 if (trailingWhitespace) {
494 trailingWhitespace = nullptr;
495 }
496 }
497 }
498 }
499 const Pointer mRangeEnd;
500 Pointer mStart;
501 Pointer mEnd;
502 Pointer mComma;
503 };
504
StringListRange(const String & aList)505 explicit StringListRange(const String& aList) : mList(aList) {}
begin()506 Iterator begin() const {
507 return Iterator(
508 mList.Data() +
509 ((empties == StringListRangeEmptyItems::ProcessEmptyItems &&
510 mList.Length() == 0)
511 ? 1
512 : 0),
513 mList.Length());
514 }
end()515 Iterator end() const {
516 return Iterator(mList.Data() + mList.Length() +
517 (empties != StringListRangeEmptyItems::Skip ? 1 : 0),
518 0);
519 }
520
521 private:
522 const String& mList;
523 };
524
525 template <StringListRangeEmptyItems empties = StringListRangeEmptyItems::Skip,
526 typename String>
MakeStringListRange(const String & aList)527 StringListRange<String, empties> MakeStringListRange(const String& aList) {
528 return StringListRange<String, empties>(aList);
529 }
530
531 template <StringListRangeEmptyItems empties = StringListRangeEmptyItems::Skip,
532 typename ListString, typename ItemString>
StringListContains(const ListString & aList,const ItemString & aItem)533 static bool StringListContains(const ListString& aList,
534 const ItemString& aItem) {
535 for (const auto& listItem : MakeStringListRange<empties>(aList)) {
536 if (listItem.Equals(aItem)) {
537 return true;
538 }
539 }
540 return false;
541 }
542
AppendStringIfNotEmpty(nsACString & aDest,nsACString && aSrc)543 inline void AppendStringIfNotEmpty(nsACString& aDest, nsACString&& aSrc) {
544 if (!aSrc.IsEmpty()) {
545 aDest.Append(NS_LITERAL_CSTRING("\n"));
546 aDest.Append(aSrc);
547 }
548 }
549
550 // Returns true if we're running on a cellular connection; 2G, 3G, etc.
551 // Main thread only.
552 bool OnCellularConnection();
553
DefaultColorSpace(const gfx::IntSize & aSize)554 inline gfx::YUVColorSpace DefaultColorSpace(const gfx::IntSize& aSize) {
555 return aSize.height < 720 ? gfx::YUVColorSpace::BT601
556 : gfx::YUVColorSpace::BT709;
557 }
558
559 } // end namespace mozilla
560
561 #endif
562