1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include <GLES2/gl2.h>
6 #include <GLES2/gl2ext.h>
7 #include <stdint.h>
8 #include <string.h>
9
10 #include <vector>
11
12 #include "ppapi/c/pp_errors.h"
13 #include "ppapi/c/ppb_opengles2.h"
14 #include "ppapi/cpp/completion_callback.h"
15 #include "ppapi/cpp/graphics_3d.h"
16 #include "ppapi/cpp/graphics_3d_client.h"
17 #include "ppapi/cpp/instance.h"
18 #include "ppapi/cpp/media_stream_video_track.h"
19 #include "ppapi/cpp/module.h"
20 #include "ppapi/cpp/rect.h"
21 #include "ppapi/cpp/var.h"
22 #include "ppapi/cpp/var_dictionary.h"
23 #include "ppapi/cpp/video_frame.h"
24 #include "ppapi/lib/gl/gles2/gl2ext_ppapi.h"
25 #include "ppapi/utility/completion_callback_factory.h"
26
27 // When compiling natively on Windows, PostMessage can be #define-d to
28 // something else.
29 #ifdef PostMessage
30 #undef PostMessage
31 #endif
32
33 // Assert |context_| isn't holding any GL Errors. Done as a macro instead of a
34 // function to preserve line number information in the failure message.
35 #define AssertNoGLError() \
36 PP_DCHECK(!glGetError());
37
38 namespace {
39
40 // This object is the global object representing this plugin library as long
41 // as it is loaded.
42 class MediaStreamVideoModule : public pp::Module {
43 public:
MediaStreamVideoModule()44 MediaStreamVideoModule() : pp::Module() {}
~MediaStreamVideoModule()45 virtual ~MediaStreamVideoModule() {}
46
47 virtual pp::Instance* CreateInstance(PP_Instance instance);
48 };
49
50 class MediaStreamVideoDemoInstance : public pp::Instance,
51 public pp::Graphics3DClient {
52 public:
53 MediaStreamVideoDemoInstance(PP_Instance instance, pp::Module* module);
54 virtual ~MediaStreamVideoDemoInstance();
55
56 // pp::Instance implementation (see PPP_Instance).
57 virtual void DidChangeView(const pp::Rect& position,
58 const pp::Rect& clip_ignored);
59 virtual void HandleMessage(const pp::Var& message_data);
60
61 // pp::Graphics3DClient implementation.
Graphics3DContextLost()62 virtual void Graphics3DContextLost() {
63 InitGL();
64 CreateTextures();
65 Render();
66 }
67
68 private:
69 void DrawYUV();
70 void DrawRGB();
71 void Render();
72
73 // GL-related functions.
74 void InitGL();
75 GLuint CreateTexture(int32_t width, int32_t height, int unit, bool rgba);
76 void CreateGLObjects();
77 void CreateShader(GLuint program, GLenum type, const char* source);
78 void PaintFinished(int32_t result);
79 void CreateTextures();
80 void ConfigureTrack();
81
82
83 // MediaStreamVideoTrack callbacks.
84 void OnConfigure(int32_t result);
85 void OnGetFrame(int32_t result, pp::VideoFrame frame);
86
87 pp::Size position_size_;
88 bool is_painting_;
89 bool needs_paint_;
90 bool is_bgra_;
91 GLuint program_yuv_;
92 GLuint program_rgb_;
93 GLuint buffer_;
94 GLuint texture_y_;
95 GLuint texture_u_;
96 GLuint texture_v_;
97 GLuint texture_rgb_;
98 pp::MediaStreamVideoTrack video_track_;
99 pp::CompletionCallbackFactory<MediaStreamVideoDemoInstance> callback_factory_;
100 std::vector<int32_t> attrib_list_;
101
102 // MediaStreamVideoTrack attributes:
103 bool need_config_;
104 PP_VideoFrame_Format attrib_format_;
105 int32_t attrib_width_;
106 int32_t attrib_height_;
107
108 // Owned data.
109 pp::Graphics3D* context_;
110
111 pp::Size frame_size_;
112 };
113
MediaStreamVideoDemoInstance(PP_Instance instance,pp::Module * module)114 MediaStreamVideoDemoInstance::MediaStreamVideoDemoInstance(
115 PP_Instance instance, pp::Module* module)
116 : pp::Instance(instance),
117 pp::Graphics3DClient(this),
118 is_painting_(false),
119 needs_paint_(false),
120 is_bgra_(false),
121 texture_y_(0),
122 texture_u_(0),
123 texture_v_(0),
124 texture_rgb_(0),
125 callback_factory_(this),
126 need_config_(false),
127 attrib_format_(PP_VIDEOFRAME_FORMAT_I420),
128 attrib_width_(0),
129 attrib_height_(0),
130 context_(NULL) {
131 if (!glInitializePPAPI(pp::Module::Get()->get_browser_interface())) {
132 LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Unable to initialize GL PPAPI!"));
133 assert(false);
134 }
135 }
136
~MediaStreamVideoDemoInstance()137 MediaStreamVideoDemoInstance::~MediaStreamVideoDemoInstance() {
138 delete context_;
139 }
140
DidChangeView(const pp::Rect & position,const pp::Rect & clip_ignored)141 void MediaStreamVideoDemoInstance::DidChangeView(
142 const pp::Rect& position, const pp::Rect& clip_ignored) {
143 if (position.width() == 0 || position.height() == 0)
144 return;
145 if (position.size() == position_size_)
146 return;
147
148 position_size_ = position.size();
149
150 // Initialize graphics.
151 InitGL();
152 Render();
153 }
154
HandleMessage(const pp::Var & var_message)155 void MediaStreamVideoDemoInstance::HandleMessage(const pp::Var& var_message) {
156 if (!var_message.is_dictionary()) {
157 LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid message!"));
158 return;
159 }
160
161 pp::VarDictionary var_dictionary_message(var_message);
162 std::string command = var_dictionary_message.Get("command").AsString();
163
164 if (command == "init") {
165 pp::Var var_track = var_dictionary_message.Get("track");
166 if (!var_track.is_resource())
167 return;
168 pp::Resource resource_track = var_track.AsResource();
169 video_track_ = pp::MediaStreamVideoTrack(resource_track);
170 ConfigureTrack();
171 } else if (command == "format") {
172 std::string str_format = var_dictionary_message.Get("format").AsString();
173 if (str_format == "YV12") {
174 attrib_format_ = PP_VIDEOFRAME_FORMAT_YV12;
175 } else if (str_format == "I420") {
176 attrib_format_ = PP_VIDEOFRAME_FORMAT_I420;
177 } else if (str_format == "BGRA") {
178 attrib_format_ = PP_VIDEOFRAME_FORMAT_BGRA;
179 } else {
180 attrib_format_ = PP_VIDEOFRAME_FORMAT_UNKNOWN;
181 }
182 need_config_ = true;
183 } else if (command == "size") {
184 attrib_width_ = var_dictionary_message.Get("width").AsInt();
185 attrib_height_ = var_dictionary_message.Get("height").AsInt();
186 need_config_ = true;
187 } else {
188 LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid command!"));
189 }
190 }
191
InitGL()192 void MediaStreamVideoDemoInstance::InitGL() {
193 PP_DCHECK(position_size_.width() && position_size_.height());
194 is_painting_ = false;
195
196 delete context_;
197 int32_t attributes[] = {
198 PP_GRAPHICS3DATTRIB_ALPHA_SIZE, 0,
199 PP_GRAPHICS3DATTRIB_BLUE_SIZE, 8,
200 PP_GRAPHICS3DATTRIB_GREEN_SIZE, 8,
201 PP_GRAPHICS3DATTRIB_RED_SIZE, 8,
202 PP_GRAPHICS3DATTRIB_DEPTH_SIZE, 0,
203 PP_GRAPHICS3DATTRIB_STENCIL_SIZE, 0,
204 PP_GRAPHICS3DATTRIB_SAMPLES, 0,
205 PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
206 PP_GRAPHICS3DATTRIB_WIDTH, position_size_.width(),
207 PP_GRAPHICS3DATTRIB_HEIGHT, position_size_.height(),
208 PP_GRAPHICS3DATTRIB_NONE,
209 };
210 context_ = new pp::Graphics3D(this, attributes);
211 PP_DCHECK(!context_->is_null());
212
213 glSetCurrentContextPPAPI(context_->pp_resource());
214
215 // Set viewport window size and clear color bit.
216 glClearColor(1, 0, 0, 1);
217 glClear(GL_COLOR_BUFFER_BIT);
218 glViewport(0, 0, position_size_.width(), position_size_.height());
219
220 BindGraphics(*context_);
221 AssertNoGLError();
222
223 CreateGLObjects();
224 }
225
DrawYUV()226 void MediaStreamVideoDemoInstance::DrawYUV() {
227 static const float kColorMatrix[9] = {
228 1.1643828125f, 1.1643828125f, 1.1643828125f,
229 0.0f, -0.39176171875f, 2.017234375f,
230 1.59602734375f, -0.81296875f, 0.0f
231 };
232
233 glUseProgram(program_yuv_);
234 glUniform1i(glGetUniformLocation(program_yuv_, "y_texture"), 0);
235 glUniform1i(glGetUniformLocation(program_yuv_, "u_texture"), 1);
236 glUniform1i(glGetUniformLocation(program_yuv_, "v_texture"), 2);
237 glUniformMatrix3fv(glGetUniformLocation(program_yuv_, "color_matrix"),
238 1, GL_FALSE, kColorMatrix);
239 AssertNoGLError();
240
241 GLint pos_location = glGetAttribLocation(program_yuv_, "a_position");
242 GLint tc_location = glGetAttribLocation(program_yuv_, "a_texCoord");
243 AssertNoGLError();
244 glEnableVertexAttribArray(pos_location);
245 glVertexAttribPointer(pos_location, 2, GL_FLOAT, GL_FALSE, 0, 0);
246 glEnableVertexAttribArray(tc_location);
247 glVertexAttribPointer(
248 tc_location, 2, GL_FLOAT, GL_FALSE, 0,
249 reinterpret_cast<void*>(16 *
250 sizeof(GLfloat))); // Skip position coordinates.
251 AssertNoGLError();
252
253 glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
254 AssertNoGLError();
255 }
256
DrawRGB()257 void MediaStreamVideoDemoInstance::DrawRGB() {
258 glUseProgram(program_rgb_);
259 glUniform1i(glGetUniformLocation(program_rgb_, "rgb_texture"), 3);
260 AssertNoGLError();
261
262 GLint pos_location = glGetAttribLocation(program_rgb_, "a_position");
263 GLint tc_location = glGetAttribLocation(program_rgb_, "a_texCoord");
264 AssertNoGLError();
265 glEnableVertexAttribArray(pos_location);
266 glVertexAttribPointer(pos_location, 2, GL_FLOAT, GL_FALSE, 0, 0);
267 glEnableVertexAttribArray(tc_location);
268 glVertexAttribPointer(
269 tc_location, 2, GL_FLOAT, GL_FALSE, 0,
270 reinterpret_cast<void*>(16 *
271 sizeof(GLfloat))); // Skip position coordinates.
272 AssertNoGLError();
273
274 glDrawArrays(GL_TRIANGLE_STRIP, 4, 4);
275 }
276
Render()277 void MediaStreamVideoDemoInstance::Render() {
278 PP_DCHECK(!is_painting_);
279 is_painting_ = true;
280 needs_paint_ = false;
281
282 if (texture_y_) {
283 DrawRGB();
284 DrawYUV();
285 } else {
286 glClear(GL_COLOR_BUFFER_BIT);
287 }
288 pp::CompletionCallback cb = callback_factory_.NewCallback(
289 &MediaStreamVideoDemoInstance::PaintFinished);
290 context_->SwapBuffers(cb);
291 }
292
PaintFinished(int32_t result)293 void MediaStreamVideoDemoInstance::PaintFinished(int32_t result) {
294 is_painting_ = false;
295 if (needs_paint_)
296 Render();
297 }
298
CreateTexture(int32_t width,int32_t height,int unit,bool rgba)299 GLuint MediaStreamVideoDemoInstance::CreateTexture(
300 int32_t width, int32_t height, int unit, bool rgba) {
301 GLuint texture_id;
302 glGenTextures(1, &texture_id);
303 AssertNoGLError();
304
305 // Assign parameters.
306 glActiveTexture(GL_TEXTURE0 + unit);
307 glBindTexture(GL_TEXTURE_2D, texture_id);
308 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
309 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
310 glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
311 glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
312 // Allocate texture.
313 glTexImage2D(GL_TEXTURE_2D, 0,
314 rgba ? GL_BGRA_EXT : GL_LUMINANCE,
315 width, height, 0,
316 rgba ? GL_BGRA_EXT : GL_LUMINANCE, GL_UNSIGNED_BYTE, NULL);
317 AssertNoGLError();
318 return texture_id;
319 }
320
CreateGLObjects()321 void MediaStreamVideoDemoInstance::CreateGLObjects() {
322 // Code and constants for shader.
323 static const char kVertexShader[] =
324 "varying vec2 v_texCoord; \n"
325 "attribute vec4 a_position; \n"
326 "attribute vec2 a_texCoord; \n"
327 "void main() \n"
328 "{ \n"
329 " v_texCoord = a_texCoord; \n"
330 " gl_Position = a_position; \n"
331 "}";
332
333 static const char kFragmentShaderYUV[] =
334 "precision mediump float; \n"
335 "varying vec2 v_texCoord; \n"
336 "uniform sampler2D y_texture; \n"
337 "uniform sampler2D u_texture; \n"
338 "uniform sampler2D v_texture; \n"
339 "uniform mat3 color_matrix; \n"
340 "void main() \n"
341 "{ \n"
342 " vec3 yuv; \n"
343 " yuv.x = texture2D(y_texture, v_texCoord).r; \n"
344 " yuv.y = texture2D(u_texture, v_texCoord).r; \n"
345 " yuv.z = texture2D(v_texture, v_texCoord).r; \n"
346 " vec3 rgb = color_matrix * (yuv - vec3(0.0625, 0.5, 0.5));\n"
347 " gl_FragColor = vec4(rgb, 1.0); \n"
348 "}";
349
350 static const char kFragmentShaderRGB[] =
351 "precision mediump float; \n"
352 "varying vec2 v_texCoord; \n"
353 "uniform sampler2D rgb_texture; \n"
354 "void main() \n"
355 "{ \n"
356 " gl_FragColor = texture2D(rgb_texture, v_texCoord); \n"
357 "}";
358
359 // Create shader programs.
360 program_yuv_ = glCreateProgram();
361 CreateShader(program_yuv_, GL_VERTEX_SHADER, kVertexShader);
362 CreateShader(program_yuv_, GL_FRAGMENT_SHADER, kFragmentShaderYUV);
363 glLinkProgram(program_yuv_);
364 AssertNoGLError();
365
366 program_rgb_ = glCreateProgram();
367 CreateShader(program_rgb_, GL_VERTEX_SHADER, kVertexShader);
368 CreateShader(program_rgb_, GL_FRAGMENT_SHADER, kFragmentShaderRGB);
369 glLinkProgram(program_rgb_);
370 AssertNoGLError();
371
372 // Assign vertex positions and texture coordinates to buffers for use in
373 // shader program.
374 static const float kVertices[] = {
375 -1, 1, -1, -1, 0, 1, 0, -1, // Position coordinates.
376 0, 1, 0, -1, 1, 1, 1, -1, // Position coordinates.
377 0, 0, 0, 1, 1, 0, 1, 1, // Texture coordinates.
378 0, 0, 0, 1, 1, 0, 1, 1, // Texture coordinates.
379 };
380
381 glGenBuffers(1, &buffer_);
382 glBindBuffer(GL_ARRAY_BUFFER, buffer_);
383 glBufferData(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices, GL_STATIC_DRAW);
384 AssertNoGLError();
385 }
386
CreateShader(GLuint program,GLenum type,const char * source)387 void MediaStreamVideoDemoInstance::CreateShader(
388 GLuint program, GLenum type, const char* source) {
389 GLuint shader = glCreateShader(type);
390 GLint length = static_cast<GLint>(strlen(source) + 1);
391 glShaderSource(shader, 1, &source, &length);
392 glCompileShader(shader);
393 glAttachShader(program, shader);
394 glDeleteShader(shader);
395 }
396
CreateTextures()397 void MediaStreamVideoDemoInstance::CreateTextures() {
398 int32_t width = frame_size_.width();
399 int32_t height = frame_size_.height();
400 if (width == 0 || height == 0)
401 return;
402 if (texture_y_)
403 glDeleteTextures(1, &texture_y_);
404 if (texture_u_)
405 glDeleteTextures(1, &texture_u_);
406 if (texture_v_)
407 glDeleteTextures(1, &texture_v_);
408 if (texture_rgb_)
409 glDeleteTextures(1, &texture_rgb_);
410 texture_y_ = CreateTexture(width, height, 0, false);
411
412 texture_u_ = CreateTexture(width / 2, height / 2, 1, false);
413 texture_v_ = CreateTexture(width / 2, height / 2, 2, false);
414 texture_rgb_ = CreateTexture(width, height, 3, true);
415 }
416
ConfigureTrack()417 void MediaStreamVideoDemoInstance::ConfigureTrack() {
418 const int32_t attrib_list[] = {
419 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_FORMAT, attrib_format_,
420 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_WIDTH, attrib_width_,
421 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_HEIGHT, attrib_height_,
422 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_NONE
423 };
424 video_track_.Configure(attrib_list, callback_factory_.NewCallback(
425 &MediaStreamVideoDemoInstance::OnConfigure));
426 }
427
OnConfigure(int32_t result)428 void MediaStreamVideoDemoInstance::OnConfigure(int32_t result) {
429 video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
430 &MediaStreamVideoDemoInstance::OnGetFrame));
431 }
432
OnGetFrame(int32_t result,pp::VideoFrame frame)433 void MediaStreamVideoDemoInstance::OnGetFrame(
434 int32_t result, pp::VideoFrame frame) {
435 if (result != PP_OK)
436 return;
437 const char* data = static_cast<const char*>(frame.GetDataBuffer());
438 pp::Size size;
439 frame.GetSize(&size);
440
441 if (size != frame_size_) {
442 frame_size_ = size;
443 CreateTextures();
444 }
445
446 is_bgra_ = (frame.GetFormat() == PP_VIDEOFRAME_FORMAT_BGRA);
447
448 int32_t width = frame_size_.width();
449 int32_t height = frame_size_.height();
450 if (!is_bgra_) {
451 glActiveTexture(GL_TEXTURE0);
452 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
453 GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
454
455 data += width * height;
456 width /= 2;
457 height /= 2;
458
459 glActiveTexture(GL_TEXTURE1);
460 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
461 GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
462
463 data += width * height;
464 glActiveTexture(GL_TEXTURE2);
465 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
466 GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
467 } else {
468 glActiveTexture(GL_TEXTURE3);
469 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
470 GL_BGRA_EXT, GL_UNSIGNED_BYTE, data);
471 }
472
473 if (is_painting_)
474 needs_paint_ = true;
475 else
476 Render();
477
478 video_track_.RecycleFrame(frame);
479 if (need_config_) {
480 ConfigureTrack();
481 need_config_ = false;
482 } else {
483 video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
484 &MediaStreamVideoDemoInstance::OnGetFrame));
485 }
486 }
487
CreateInstance(PP_Instance instance)488 pp::Instance* MediaStreamVideoModule::CreateInstance(PP_Instance instance) {
489 return new MediaStreamVideoDemoInstance(instance, this);
490 }
491
492 } // anonymous namespace
493
494 namespace pp {
495 // Factory function for your specialization of the Module object.
CreateModule()496 Module* CreateModule() {
497 return new MediaStreamVideoModule();
498 }
499 } // namespace pp
500