1 /*!
2 * \copy
3 * Copyright (c) 2009-2014, Cisco Systems
4 * Copyright (c) 2014, Mozilla
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 *
14 * * Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
17 * distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
22 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
23 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
27 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
29 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 *
32 *
33 *************************************************************************************
34 */
35
36 #include <stdint.h>
37 #include <time.h>
38 #include <cstdio>
39 #include <cstring>
40 #include <iostream>
41 #include <string>
42 #include <memory>
43 #include <assert.h>
44 #include <limits.h>
45
46 #include "gmp-platform.h"
47 #include "gmp-video-host.h"
48 #include "gmp-video-encode.h"
49 #include "gmp-video-decode.h"
50 #include "gmp-video-frame-i420.h"
51 #include "gmp-video-frame-encoded.h"
52
53 #include "mozilla/PodOperations.h"
54
55 #if defined(_MSC_VER)
56 # define PUBLIC_FUNC __declspec(dllexport)
57 #else
58 # define PUBLIC_FUNC
59 #endif
60
61 #define BIG_FRAME 10000
62
63 #define GL_CRIT 0
64 #define GL_ERROR 1
65 #define GL_INFO 2
66 #define GL_DEBUG 3
67
68 const char* kLogStrings[] = {"Critical", "Error", "Info", "Debug"};
69
70 static int g_log_level = GL_CRIT;
71
72 #define GMPLOG(l, x) \
73 do { \
74 if (l <= g_log_level) { \
75 const char* log_string = "unknown"; \
76 if ((l >= 0) && (l <= 3)) { \
77 log_string = kLogStrings[l]; \
78 } \
79 std::cerr << log_string << ": " << x << std::endl; \
80 } \
81 } while (0)
82
83 class FakeVideoEncoder;
84 class FakeVideoDecoder;
85
86 // Turn off padding for this structure. Having extra bytes can result in
87 // bitstream parsing problems.
88 #pragma pack(push, 1)
89 struct EncodedFrame {
90 struct SPSNalu {
91 uint32_t size_;
92 uint8_t payload[14];
93 } sps_nalu;
94 struct PPSNalu {
95 uint32_t size_;
96 uint8_t payload[4];
97 } pps_nalu;
98 struct IDRNalu {
99 uint32_t size_;
100 uint8_t h264_compat_;
101 uint32_t magic_;
102 uint32_t width_;
103 uint32_t height_;
104 uint8_t y_;
105 uint8_t u_;
106 uint8_t v_;
107 uint32_t timestamp_;
108 } idr_nalu;
109 };
110 #pragma pack(pop)
111
112 #define ENCODED_FRAME_MAGIC 0x004000b8
113
114 template <typename T>
115 class SelfDestruct {
116 public:
SelfDestruct(T * t)117 explicit SelfDestruct(T* t) : t_(t) {}
~SelfDestruct()118 ~SelfDestruct() {
119 if (t_) {
120 t_->Destroy();
121 }
122 }
123
124 private:
125 T* t_;
126 };
127
128 class FakeVideoEncoder : public GMPVideoEncoder {
129 public:
FakeVideoEncoder(GMPVideoHost * hostAPI)130 explicit FakeVideoEncoder(GMPVideoHost* hostAPI) : host_(hostAPI) {}
131
InitEncode(const GMPVideoCodec & codecSettings,const uint8_t * aCodecSpecific,uint32_t aCodecSpecificSize,GMPVideoEncoderCallback * callback,int32_t numberOfCores,uint32_t maxPayloadSize)132 void InitEncode(const GMPVideoCodec& codecSettings,
133 const uint8_t* aCodecSpecific, uint32_t aCodecSpecificSize,
134 GMPVideoEncoderCallback* callback, int32_t numberOfCores,
135 uint32_t maxPayloadSize) override {
136 callback_ = callback;
137 frame_size_ = (maxPayloadSize > 0 && maxPayloadSize < BIG_FRAME)
138 ? maxPayloadSize
139 : BIG_FRAME;
140 frame_size_ -= 24 + 40;
141 // default header+extension size is 24, but let's leave extra room if
142 // we enable more extensions.
143 // XXX -- why isn't the size passed in based on the size minus extensions?
144
145 const char* env = getenv("GMP_LOGGING");
146 if (env) {
147 g_log_level = atoi(env);
148 }
149 GMPLOG(GL_INFO, "Initialized encoder");
150 }
151
SendFrame(GMPVideoi420Frame * inputImage,GMPVideoFrameType frame_type,int nal_type)152 void SendFrame(GMPVideoi420Frame* inputImage, GMPVideoFrameType frame_type,
153 int nal_type) {
154 // Encode this in a frame that looks a little bit like H.264.
155 // Send SPS/PPS/IDR to avoid confusing people
156 // Copy the data. This really should convert this to network byte order.
157 EncodedFrame eframe;
158
159 // These values were chosen to force a SPS id of 0
160 eframe.sps_nalu = {sizeof(EncodedFrame::SPSNalu) - sizeof(uint32_t),
161 {0x67, 0x42, 0xc0, 0xd, 0x8c, 0x8d, 0x40, 0xa0, 0xf9,
162 0x0, 0xf0, 0x88, 0x46, 0xa0}};
163
164 // These values were chosen to force a PPS id of 0
165 eframe.pps_nalu = {sizeof(EncodedFrame::PPSNalu) - sizeof(uint32_t),
166 {0x68, 0xce, 0x3c, 0x80}};
167
168 eframe.idr_nalu.size_ = sizeof(EncodedFrame::IDRNalu) - sizeof(uint32_t);
169 // We force IFrame here - if we send PFrames, the webrtc.org code gets
170 // tripped up attempting to find non-existent previous IFrames.
171 eframe.idr_nalu.h264_compat_ =
172 nal_type; // 5 = IFrame/IDR slice, 1=PFrame/slice
173 eframe.idr_nalu.magic_ = ENCODED_FRAME_MAGIC;
174 eframe.idr_nalu.width_ = inputImage->Width();
175 eframe.idr_nalu.height_ = inputImage->Height();
176 eframe.idr_nalu.y_ = AveragePlane(inputImage->Buffer(kGMPYPlane),
177 inputImage->AllocatedSize(kGMPYPlane));
178 eframe.idr_nalu.u_ = AveragePlane(inputImage->Buffer(kGMPUPlane),
179 inputImage->AllocatedSize(kGMPUPlane));
180 eframe.idr_nalu.v_ = AveragePlane(inputImage->Buffer(kGMPVPlane),
181 inputImage->AllocatedSize(kGMPVPlane));
182
183 eframe.idr_nalu.timestamp_ = inputImage->Timestamp();
184
185 // Now return the encoded data back to the parent.
186 GMPVideoFrame* ftmp;
187 GMPErr err = host_->CreateFrame(kGMPEncodedVideoFrame, &ftmp);
188 if (err != GMPNoErr) {
189 GMPLOG(GL_ERROR, "Error creating encoded frame");
190 return;
191 }
192
193 GMPVideoEncodedFrame* f = static_cast<GMPVideoEncodedFrame*>(ftmp);
194
195 err = f->CreateEmptyFrame(sizeof(eframe));
196 if (err != GMPNoErr) {
197 GMPLOG(GL_ERROR, "Error allocating frame data");
198 f->Destroy();
199 return;
200 }
201 memcpy(f->Buffer(), &eframe, sizeof(eframe));
202 f->SetEncodedWidth(eframe.idr_nalu.width_);
203 f->SetEncodedHeight(eframe.idr_nalu.height_);
204 f->SetTimeStamp(eframe.idr_nalu.timestamp_);
205 f->SetFrameType(frame_type);
206 f->SetCompleteFrame(true);
207 f->SetBufferType(GMP_BufferLength32);
208
209 GMPLOG(GL_DEBUG, "Encoding complete. type= "
210 << f->FrameType()
211 << " NAL_type=" << (int)eframe.idr_nalu.h264_compat_
212 << " length=" << f->Size()
213 << " timestamp=" << f->TimeStamp()
214 << " width/height=" << eframe.idr_nalu.width_ << "x"
215 << eframe.idr_nalu.height_);
216
217 // Return the encoded frame.
218 GMPCodecSpecificInfo info;
219 mozilla::PodZero(&info);
220 info.mCodecType = kGMPVideoCodecH264;
221 info.mBufferType = GMP_BufferLength32;
222 info.mCodecSpecific.mH264.mSimulcastIdx = 0;
223 GMPLOG(GL_DEBUG, "Calling callback");
224 callback_->Encoded(f, reinterpret_cast<uint8_t*>(&info), sizeof(info));
225 GMPLOG(GL_DEBUG, "Callback called");
226 }
227
Encode(GMPVideoi420Frame * inputImage,const uint8_t * aCodecSpecificInfo,uint32_t aCodecSpecificInfoLength,const GMPVideoFrameType * aFrameTypes,uint32_t aFrameTypesLength)228 void Encode(GMPVideoi420Frame* inputImage, const uint8_t* aCodecSpecificInfo,
229 uint32_t aCodecSpecificInfoLength,
230 const GMPVideoFrameType* aFrameTypes,
231 uint32_t aFrameTypesLength) override {
232 GMPLOG(GL_DEBUG, __FUNCTION__ << " size=" << inputImage->Width() << "x"
233 << inputImage->Height());
234
235 assert(aFrameTypesLength != 0);
236 GMPVideoFrameType frame_type = aFrameTypes[0];
237
238 SelfDestruct<GMPVideoi420Frame> ifd(inputImage);
239
240 if (frame_type == kGMPKeyFrame) {
241 if (!inputImage) return;
242 }
243 if (!inputImage) {
244 GMPLOG(GL_ERROR, "no input image");
245 return;
246 }
247
248 if (frame_type == kGMPKeyFrame ||
249 frames_encoded_++ % 10 == 0) { // periodically send iframes anyways
250 // 5 = IFrame/IDR slice
251 SendFrame(inputImage, kGMPKeyFrame, 5);
252 } else {
253 // 1 = PFrame/slice
254 SendFrame(inputImage, frame_type, 1);
255 }
256 }
257
SetChannelParameters(uint32_t aPacketLoss,uint32_t aRTT)258 void SetChannelParameters(uint32_t aPacketLoss, uint32_t aRTT) override {}
259
SetRates(uint32_t aNewBitRate,uint32_t aFrameRate)260 void SetRates(uint32_t aNewBitRate, uint32_t aFrameRate) override {}
261
SetPeriodicKeyFrames(bool aEnable)262 void SetPeriodicKeyFrames(bool aEnable) override {}
263
EncodingComplete()264 void EncodingComplete() override { delete this; }
265
266 private:
AveragePlane(uint8_t * ptr,size_t len)267 uint8_t AveragePlane(uint8_t* ptr, size_t len) {
268 uint64_t val = 0;
269
270 for (size_t i = 0; i < len; ++i) {
271 val += ptr[i];
272 }
273
274 return (val / len) % 0xff;
275 }
276
277 GMPVideoHost* host_;
278 GMPVideoEncoderCallback* callback_ = nullptr;
279 uint32_t frame_size_ = BIG_FRAME;
280 uint32_t frames_encoded_ = 0;
281 };
282
283 class FakeVideoDecoder : public GMPVideoDecoder {
284 public:
FakeVideoDecoder(GMPVideoHost * hostAPI)285 explicit FakeVideoDecoder(GMPVideoHost* hostAPI)
286 : host_(hostAPI), callback_(nullptr) {}
287
288 ~FakeVideoDecoder() override = default;
289
InitDecode(const GMPVideoCodec & codecSettings,const uint8_t * aCodecSpecific,uint32_t aCodecSpecificSize,GMPVideoDecoderCallback * callback,int32_t coreCount)290 void InitDecode(const GMPVideoCodec& codecSettings,
291 const uint8_t* aCodecSpecific, uint32_t aCodecSpecificSize,
292 GMPVideoDecoderCallback* callback,
293 int32_t coreCount) override {
294 GMPLOG(GL_INFO, "InitDecode");
295
296 const char* env = getenv("GMP_LOGGING");
297 if (env) {
298 g_log_level = atoi(env);
299 }
300 callback_ = callback;
301 }
302
Decode(GMPVideoEncodedFrame * inputFrame,bool missingFrames,const uint8_t * aCodecSpecificInfo,uint32_t aCodecSpecificInfoLength,int64_t renderTimeMs=-1)303 void Decode(GMPVideoEncodedFrame* inputFrame, bool missingFrames,
304 const uint8_t* aCodecSpecificInfo,
305 uint32_t aCodecSpecificInfoLength,
306 int64_t renderTimeMs = -1) override {
307 GMPLOG(GL_DEBUG, __FUNCTION__
308 << "Decoding frame size=" << inputFrame->Size()
309 << " timestamp=" << inputFrame->TimeStamp());
310
311 // Attach a self-destructor so that the input frame is destroyed on return.
312 SelfDestruct<GMPVideoEncodedFrame> ifd(inputFrame);
313
314 EncodedFrame* eframe;
315 eframe = reinterpret_cast<EncodedFrame*>(inputFrame->Buffer());
316 GMPLOG(GL_DEBUG, "magic=" << eframe->idr_nalu.magic_ << " h264_compat="
317 << (int)eframe->idr_nalu.h264_compat_
318 << " width=" << eframe->idr_nalu.width_
319 << " height=" << eframe->idr_nalu.height_
320 << " timestamp=" << inputFrame->TimeStamp()
321 << " y/u/v=" << (int)eframe->idr_nalu.y_ << ":"
322 << (int)eframe->idr_nalu.u_ << ":"
323 << (int)eframe->idr_nalu.v_);
324 if (inputFrame->Size() != (sizeof(*eframe))) {
325 GMPLOG(GL_ERROR, "Couldn't decode frame. Size=" << inputFrame->Size());
326 return;
327 }
328
329 if (eframe->idr_nalu.magic_ != ENCODED_FRAME_MAGIC) {
330 GMPLOG(GL_ERROR,
331 "Couldn't decode frame. Magic=" << eframe->idr_nalu.magic_);
332 return;
333 }
334 if (eframe->idr_nalu.h264_compat_ != 5 &&
335 eframe->idr_nalu.h264_compat_ != 1) {
336 // only return video for iframes or pframes
337 GMPLOG(GL_DEBUG, "Not a video frame: NAL type "
338 << (int)eframe->idr_nalu.h264_compat_);
339 return;
340 }
341
342 int width = eframe->idr_nalu.width_;
343 int height = eframe->idr_nalu.height_;
344 int ystride = eframe->idr_nalu.width_;
345 int uvstride = eframe->idr_nalu.width_ / 2;
346
347 GMPLOG(GL_DEBUG, "Video frame ready for display "
348 << width << "x" << height
349 << " timestamp=" << inputFrame->TimeStamp());
350
351 GMPVideoFrame* ftmp = nullptr;
352
353 // Translate the image.
354 GMPErr err = host_->CreateFrame(kGMPI420VideoFrame, &ftmp);
355 if (err != GMPNoErr) {
356 GMPLOG(GL_ERROR, "Couldn't allocate empty I420 frame");
357 return;
358 }
359
360 GMPVideoi420Frame* frame = static_cast<GMPVideoi420Frame*>(ftmp);
361 err = frame->CreateEmptyFrame(width, height, ystride, uvstride, uvstride);
362 if (err != GMPNoErr) {
363 GMPLOG(GL_ERROR, "Couldn't make decoded frame");
364 return;
365 }
366
367 memset(frame->Buffer(kGMPYPlane), eframe->idr_nalu.y_,
368 frame->AllocatedSize(kGMPYPlane));
369 memset(frame->Buffer(kGMPUPlane), eframe->idr_nalu.u_,
370 frame->AllocatedSize(kGMPUPlane));
371 memset(frame->Buffer(kGMPVPlane), eframe->idr_nalu.v_,
372 frame->AllocatedSize(kGMPVPlane));
373
374 GMPLOG(GL_DEBUG, "Allocated size = " << frame->AllocatedSize(kGMPYPlane));
375 frame->SetTimestamp(inputFrame->TimeStamp());
376 frame->SetDuration(inputFrame->Duration());
377 callback_->Decoded(frame);
378 }
379
Reset()380 void Reset() override {}
381
Drain()382 void Drain() override {}
383
DecodingComplete()384 void DecodingComplete() override { delete this; }
385
386 GMPVideoHost* host_;
387 GMPVideoDecoderCallback* callback_;
388 };
389
390 extern "C" {
391
GMPInit(GMPPlatformAPI * aPlatformAPI)392 PUBLIC_FUNC GMPErr GMPInit(GMPPlatformAPI* aPlatformAPI) { return GMPNoErr; }
393
GMPGetAPI(const char * aApiName,void * aHostAPI,void ** aPluginApi)394 PUBLIC_FUNC GMPErr GMPGetAPI(const char* aApiName, void* aHostAPI,
395 void** aPluginApi) {
396 if (!strcmp(aApiName, GMP_API_VIDEO_DECODER)) {
397 *aPluginApi = new FakeVideoDecoder(static_cast<GMPVideoHost*>(aHostAPI));
398 return GMPNoErr;
399 }
400 if (!strcmp(aApiName, GMP_API_VIDEO_ENCODER)) {
401 *aPluginApi = new FakeVideoEncoder(static_cast<GMPVideoHost*>(aHostAPI));
402 return GMPNoErr;
403 }
404 return GMPGenericErr;
405 }
406
GMPShutdown(void)407 PUBLIC_FUNC void GMPShutdown(void) {}
408
409 } // extern "C"
410