1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6 #ifndef VideoOutput_h
7 #define VideoOutput_h
8
9 #include "MediaTrackListener.h"
10 #include "VideoFrameContainer.h"
11
12 namespace mozilla {
13
14 using layers::Image;
15 using layers::ImageContainer;
16 using layers::PlanarYCbCrData;
17 using layers::PlanarYCbCrImage;
18
SetImageToBlackPixel(PlanarYCbCrImage * aImage)19 static bool SetImageToBlackPixel(PlanarYCbCrImage* aImage) {
20 uint8_t blackPixel[] = {0x10, 0x80, 0x80};
21
22 PlanarYCbCrData data;
23 data.mYChannel = blackPixel;
24 data.mCbChannel = blackPixel + 1;
25 data.mCrChannel = blackPixel + 2;
26 data.mYStride = data.mCbCrStride = 1;
27 data.mPicSize = data.mYSize = data.mCbCrSize = gfx::IntSize(1, 1);
28 data.mYUVColorSpace = gfx::YUVColorSpace::BT601;
29 // This could be made FULL once bug 1568745 is complete. A black pixel being
30 // 0x00, 0x80, 0x80
31 data.mColorRange = gfx::ColorRange::LIMITED;
32
33 return aImage->CopyData(data);
34 }
35
36 class VideoOutput : public DirectMediaTrackListener {
37 protected:
38 virtual ~VideoOutput() = default;
39
DropPastFrames()40 void DropPastFrames() {
41 TimeStamp now = TimeStamp::Now();
42 size_t nrChunksInPast = 0;
43 for (const auto& idChunkPair : mFrames) {
44 const VideoChunk& chunk = idChunkPair.second;
45 if (chunk.mTimeStamp > now) {
46 break;
47 }
48 ++nrChunksInPast;
49 }
50 if (nrChunksInPast > 1) {
51 // We need to keep one frame that starts in the past, because it only ends
52 // when the next frame starts (which also needs to be in the past for it
53 // to drop).
54 mFrames.RemoveElementsAt(0, nrChunksInPast - 1);
55 }
56 }
57
SendFramesEnsureLocked()58 void SendFramesEnsureLocked() {
59 mMutex.AssertCurrentThreadOwns();
60 SendFrames();
61 }
62
SendFrames()63 void SendFrames() {
64 DropPastFrames();
65
66 if (mFrames.IsEmpty()) {
67 return;
68 }
69
70 // Collect any new frames produced in this iteration.
71 AutoTArray<ImageContainer::NonOwningImage, 16> images;
72 PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;
73
74 for (const auto& idChunkPair : mFrames) {
75 ImageContainer::FrameID frameId = idChunkPair.first;
76 const VideoChunk& chunk = idChunkPair.second;
77 const VideoFrame& frame = chunk.mFrame;
78 Image* image = frame.GetImage();
79 if (frame.GetForceBlack() || !mEnabled) {
80 if (!mBlackImage) {
81 RefPtr<Image> blackImage = mVideoFrameContainer->GetImageContainer()
82 ->CreatePlanarYCbCrImage();
83 if (blackImage) {
84 // Sets the image to a single black pixel, which will be scaled to
85 // fill the rendered size.
86 if (SetImageToBlackPixel(blackImage->AsPlanarYCbCrImage())) {
87 mBlackImage = blackImage;
88 }
89 }
90 }
91 if (mBlackImage) {
92 image = mBlackImage;
93 }
94 }
95 if (!image) {
96 // We ignore null images.
97 continue;
98 }
99 images.AppendElement(ImageContainer::NonOwningImage(
100 image, chunk.mTimeStamp, frameId, mProducerID));
101
102 lastPrincipalHandle = chunk.GetPrincipalHandle();
103 }
104
105 if (images.IsEmpty()) {
106 // This could happen if the only images in mFrames are null. We leave the
107 // container at the current frame in this case.
108 mVideoFrameContainer->ClearFutureFrames();
109 return;
110 }
111
112 bool principalHandleChanged =
113 lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
114 lastPrincipalHandle != mVideoFrameContainer->GetLastPrincipalHandle();
115
116 if (principalHandleChanged) {
117 mVideoFrameContainer->UpdatePrincipalHandleForFrameID(
118 lastPrincipalHandle, images.LastElement().mFrameID);
119 }
120
121 mVideoFrameContainer->SetCurrentFrames(
122 mFrames[0].second.mFrame.GetIntrinsicSize(), images);
123 mMainThread->Dispatch(NewRunnableMethod("VideoFrameContainer::Invalidate",
124 mVideoFrameContainer,
125 &VideoFrameContainer::Invalidate));
126 }
127
128 public:
VideoOutput(VideoFrameContainer * aContainer,AbstractThread * aMainThread)129 VideoOutput(VideoFrameContainer* aContainer, AbstractThread* aMainThread)
130 : mMutex("VideoOutput::mMutex"),
131 mVideoFrameContainer(aContainer),
132 mMainThread(aMainThread) {}
NotifyRealtimeTrackData(MediaTrackGraph * aGraph,TrackTime aTrackOffset,const MediaSegment & aMedia)133 void NotifyRealtimeTrackData(MediaTrackGraph* aGraph, TrackTime aTrackOffset,
134 const MediaSegment& aMedia) override {
135 MOZ_ASSERT(aMedia.GetType() == MediaSegment::VIDEO);
136 const VideoSegment& video = static_cast<const VideoSegment&>(aMedia);
137 MutexAutoLock lock(mMutex);
138 for (VideoSegment::ConstChunkIterator i(video); !i.IsEnded(); i.Next()) {
139 if (!mLastFrameTime.IsNull() && i->mTimeStamp < mLastFrameTime) {
140 // Time can go backwards if the source is a captured MediaDecoder and
141 // it seeks, as the previously buffered frames would stretch into the
142 // future. If this happens, we clear the buffered frames and start over.
143 mFrames.ClearAndRetainStorage();
144 }
145 mFrames.AppendElement(
146 std::make_pair(mVideoFrameContainer->NewFrameID(), *i));
147 mLastFrameTime = i->mTimeStamp;
148 }
149
150 SendFramesEnsureLocked();
151 }
NotifyRemoved(MediaTrackGraph * aGraph)152 void NotifyRemoved(MediaTrackGraph* aGraph) override {
153 // Doesn't need locking by mMutex, since the direct listener is removed from
154 // the track before we get notified.
155 if (mFrames.Length() <= 1) {
156 // The compositor has already received the last frame.
157 mFrames.ClearAndRetainStorage();
158 mVideoFrameContainer->ClearFutureFrames();
159 return;
160 }
161
162 // The compositor has multiple frames. ClearFutureFrames() would only retain
163 // the first as that's normally the current one. We however stop doing
164 // SetCurrentFrames() once we've received the last frame in a track, so
165 // there might be old frames lingering. We'll find the current one and
166 // re-send that.
167 DropPastFrames();
168 mFrames.RemoveElementsAt(1, mFrames.Length() - 1);
169 SendFrames();
170 mFrames.ClearAndRetainStorage();
171 }
NotifyEnded(MediaTrackGraph * aGraph)172 void NotifyEnded(MediaTrackGraph* aGraph) override {
173 // Doesn't need locking by mMutex, since for the track to end, it must have
174 // been ended by the source, meaning that the source won't append more data.
175 if (mFrames.IsEmpty()) {
176 return;
177 }
178
179 // Re-send only the last one to the compositor.
180 mFrames.RemoveElementsAt(0, mFrames.Length() - 1);
181 SendFrames();
182 mFrames.ClearAndRetainStorage();
183 }
NotifyEnabledStateChanged(MediaTrackGraph * aGraph,bool aEnabled)184 void NotifyEnabledStateChanged(MediaTrackGraph* aGraph,
185 bool aEnabled) override {
186 MutexAutoLock lock(mMutex);
187 mEnabled = aEnabled;
188 // Since mEnabled will affect whether frames are real, or black, we assign
189 // new FrameIDs whenever this changes.
190 for (auto& idChunkPair : mFrames) {
191 idChunkPair.first = mVideoFrameContainer->NewFrameID();
192 }
193 SendFramesEnsureLocked();
194 }
195
196 Mutex mMutex;
197 TimeStamp mLastFrameTime;
198 // Once the frame is forced to black, we initialize mBlackImage for use in any
199 // following forced-black frames.
200 RefPtr<Image> mBlackImage;
201 bool mEnabled = true;
202 // This array is accessed from both the direct video thread, and the graph
203 // thread. Protected by mMutex.
204 nsTArray<std::pair<ImageContainer::FrameID, VideoChunk>> mFrames;
205 const RefPtr<VideoFrameContainer> mVideoFrameContainer;
206 const RefPtr<AbstractThread> mMainThread;
207 const layers::ImageContainer::ProducerID mProducerID =
208 layers::ImageContainer::AllocateProducerID();
209 };
210
211 } // namespace mozilla
212
213 #endif // VideoOutput_h
214