1/* 2 * GStreamer 3 * Copyright (C) 2015 Matthew Waters <matthew@centricular.com> 4 * 5 * This library is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU Library General Public 7 * License as published by the Free Software Foundation; either 8 * version 2 of the License, or (at your option) any later version. 9 * 10 * This library is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * Library General Public License for more details. 14 * 15 * You should have received a copy of the GNU Library General Public 16 * License along with this library; if not, write to the 17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, 18 * Boston, MA 02110-1301, USA. 19 */ 20 21/** 22 * SECTION:element-caopengllayersink 23 * 24 * caopengllayersink renders incoming video frames to CAOpenGLLayer that 25 * can be retrieved through the layer property and placed in the Core 26 * Animation render tree. 27 */ 28 29#ifdef HAVE_CONFIG_H 30#include "config.h" 31#endif 32 33#include "caopengllayersink.h" 34#include "gstglsinkbin.h" 35#include <QuartzCore/QuartzCore.h> 36 37GST_DEBUG_CATEGORY (gst_debug_ca_sink); 38#define GST_CAT_DEFAULT gst_debug_ca_sink 39 40typedef GstGLSinkBin GstCAOpenGLLayerSinkBin; 41typedef GstGLSinkBinClass GstCAOpenGLLayerSinkBinClass; 42 43G_DEFINE_TYPE (GstCAOpenGLLayerSinkBin, gst_ca_opengl_layer_sink_bin, 44 GST_TYPE_GL_SINK_BIN); 45 46enum 47{ 48 PROP_BIN_0, 49 PROP_BIN_QOS, 50 PROP_BIN_FORCE_ASPECT_RATIO, 51 PROP_BIN_LAST_SAMPLE, 52 PROP_BIN_LAYER, 53}; 54 55static void 56_on_notify_layer (GObject * object, GParamSpec *pspec, gpointer user_data) 57{ 58 GstCAOpenGLLayerSinkBin *self = user_data; 59 60 g_object_notify (G_OBJECT (self), "layer"); 61} 62 63static void 64gst_ca_opengl_layer_sink_bin_set_property (GObject * object, guint prop_id, 65 const GValue * value, GParamSpec * param_spec) 66{ 67 g_object_set_property (G_OBJECT (GST_GL_SINK_BIN (object)->sink), 68 param_spec->name, value); 69} 70 71static void 72gst_ca_opengl_layer_sink_bin_get_property (GObject * object, guint prop_id, 73 GValue * value, GParamSpec * param_spec) 74{ 75 g_object_get_property (G_OBJECT (GST_GL_SINK_BIN (object)->sink), 76 param_spec->name, value); 77} 78 79static void 80gst_ca_opengl_layer_sink_bin_init (GstCAOpenGLLayerSinkBin * self) 81{ 82 gpointer *sink = g_object_new (GST_TYPE_CA_OPENGL_LAYER_SINK, NULL); 83 84 g_signal_connect (sink, "notify::layer", G_CALLBACK (_on_notify_layer), self); 85 86 gst_gl_sink_bin_finish_init_with_element (GST_GL_SINK_BIN (self), 87 GST_ELEMENT (sink)); 88} 89 90static void 91gst_ca_opengl_layer_sink_bin_class_init (GstCAOpenGLLayerSinkBinClass * klass) 92{ 93 GObjectClass *gobject_class = G_OBJECT_CLASS (klass); 94 95 gobject_class->get_property = gst_ca_opengl_layer_sink_bin_get_property; 96 gobject_class->set_property = gst_ca_opengl_layer_sink_bin_set_property; 97 98 g_object_class_install_property (gobject_class, PROP_BIN_FORCE_ASPECT_RATIO, 99 g_param_spec_boolean ("force-aspect-ratio", 100 "Force aspect ratio", 101 "When enabled, scaling will respect original aspect ratio", TRUE, 102 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); 103 104 g_object_class_install_property (gobject_class, PROP_BIN_LAST_SAMPLE, 105 g_param_spec_boxed ("last-sample", "Last Sample", 106 "The last sample received in the sink", GST_TYPE_SAMPLE, 107 G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); 108 109 g_object_class_install_property (gobject_class, PROP_BIN_LAYER, 110 g_param_spec_pointer ("layer", "CAOpenGLLayer", 111 "OpenGL Core Animation layer", 112 G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); 113 114 g_object_class_install_property (gobject_class, PROP_BIN_QOS, 115 g_param_spec_boolean ("qos", "Quality of Service", 116 "Generate Quality-of-Service events upstream", TRUE, 117 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); 118} 119 120#define GST_CA_OPENGL_LAYER_SINK_GET_LOCK(glsink) \ 121 (GST_CA_OPENGL_LAYER_SINK(glsink)->drawing_lock) 122#define GST_CA_OPENGL_LAYER_SINK_LOCK(glsink) \ 123 (g_mutex_lock(&GST_CA_OPENGL_LAYER_SINK_GET_LOCK (glsink))) 124#define GST_CA_OPENGL_LAYER_SINK_UNLOCK(glsink) \ 125 (g_mutex_unlock(&GST_CA_OPENGL_LAYER_SINK_GET_LOCK (glsink))) 126 127#define USING_OPENGL(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL, 1, 0)) 128#define USING_OPENGL3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL3, 3, 1)) 129#define USING_GLES(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES, 1, 0)) 130#define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0)) 131#define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0)) 132 133#define SUPPORTED_GL_APIS GST_GL_API_OPENGL | GST_GL_API_GLES2 | GST_GL_API_OPENGL3 134 135static void gst_ca_opengl_layer_sink_thread_init_redisplay (GstCAOpenGLLayerSink * ca_sink); 136static void gst_ca_opengl_layer_sink_cleanup_glthread (GstCAOpenGLLayerSink * ca_sink); 137static void gst_ca_opengl_layer_sink_on_resize (GstCAOpenGLLayerSink * ca_sink, 138 gint width, gint height); 139static void gst_ca_opengl_layer_sink_on_draw (GstCAOpenGLLayerSink * ca_sink); 140 141static void gst_ca_opengl_layer_sink_finalize (GObject * object); 142static void gst_ca_opengl_layer_sink_set_property (GObject * object, guint prop_id, 143 const GValue * value, GParamSpec * param_spec); 144static void gst_ca_opengl_layer_sink_get_property (GObject * object, guint prop_id, 145 GValue * value, GParamSpec * param_spec); 146 147static gboolean gst_ca_opengl_layer_sink_stop (GstBaseSink * bsink); 148 149static gboolean gst_ca_opengl_layer_sink_query (GstBaseSink * bsink, GstQuery * query); 150static void gst_ca_opengl_layer_sink_set_context (GstElement * element, 151 GstContext * context); 152 153static GstStateChangeReturn gst_ca_opengl_layer_sink_change_state (GstElement * 154 element, GstStateChange transition); 155 156static void gst_ca_opengl_layer_sink_get_times (GstBaseSink * bsink, GstBuffer * buf, 157 GstClockTime * start, GstClockTime * end); 158static gboolean gst_ca_opengl_layer_sink_set_caps (GstBaseSink * bsink, GstCaps * caps); 159static GstFlowReturn gst_ca_opengl_layer_sink_prepare (GstBaseSink * bsink, 160 GstBuffer * buf); 161static GstFlowReturn gst_ca_opengl_layer_sink_show_frame (GstVideoSink * bsink, 162 GstBuffer * buf); 163static gboolean gst_ca_opengl_layer_sink_propose_allocation (GstBaseSink * bsink, 164 GstQuery * query); 165 166static GstStaticPadTemplate gst_ca_opengl_layer_sink_template = 167 GST_STATIC_PAD_TEMPLATE ("sink", 168 GST_PAD_SINK, 169 GST_PAD_ALWAYS, 170 GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), " 171 "format = (string) RGBA, " 172 "width = " GST_VIDEO_SIZE_RANGE ", " 173 "height = " GST_VIDEO_SIZE_RANGE ", " 174 "framerate = " GST_VIDEO_FPS_RANGE "," 175 "texture-target = (string) 2D") 176 ); 177 178enum 179{ 180 PROP_0, 181 PROP_FORCE_ASPECT_RATIO, 182 PROP_CONTEXT, 183 PROP_LAYER, 184}; 185 186#define gst_ca_opengl_layer_sink_parent_class parent_class 187G_DEFINE_TYPE_WITH_CODE (GstCAOpenGLLayerSink, gst_ca_opengl_layer_sink, 188 GST_TYPE_VIDEO_SINK, GST_DEBUG_CATEGORY_INIT (gst_debug_ca_sink, 189 "caopengllayersink", 0, "CAOpenGLLayer Video Sink")); 190 191static void 192gst_ca_opengl_layer_sink_class_init (GstCAOpenGLLayerSinkClass * klass) 193{ 194 GObjectClass *gobject_class; 195 GstElementClass *gstelement_class; 196 GstBaseSinkClass *gstbasesink_class; 197 GstVideoSinkClass *gstvideosink_class; 198 GstElementClass *element_class; 199 200 gobject_class = (GObjectClass *) klass; 201 gstelement_class = (GstElementClass *) klass; 202 gstbasesink_class = (GstBaseSinkClass *) klass; 203 gstvideosink_class = (GstVideoSinkClass *) klass; 204 element_class = GST_ELEMENT_CLASS (klass); 205 206 gobject_class->set_property = gst_ca_opengl_layer_sink_set_property; 207 gobject_class->get_property = gst_ca_opengl_layer_sink_get_property; 208 209 g_object_class_install_property (gobject_class, PROP_FORCE_ASPECT_RATIO, 210 g_param_spec_boolean ("force-aspect-ratio", 211 "Force aspect ratio", 212 "When enabled, scaling will respect original aspect ratio", TRUE, 213 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); 214 215 g_object_class_install_property (gobject_class, PROP_CONTEXT, 216 g_param_spec_object ("context", 217 "OpenGL context", 218 "Get OpenGL context", 219 GST_TYPE_GL_CONTEXT, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); 220 221 g_object_class_install_property (gobject_class, PROP_LAYER, 222 g_param_spec_pointer ("layer", "CAOpenGLLayer", 223 "OpenGL Core Animation layer", 224 G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); 225 226 gst_element_class_set_metadata (element_class, "CAOpenGLLayer video sink", 227 "Sink/Video", "A video sink based on CAOpenGLLayer", 228 "Matthew Waters <matthew@centricular.com>"); 229 230 gst_element_class_add_static_pad_template (element_class, &gst_ca_opengl_layer_sink_template); 231 232 gobject_class->finalize = gst_ca_opengl_layer_sink_finalize; 233 234 gstelement_class->change_state = gst_ca_opengl_layer_sink_change_state; 235 gstelement_class->set_context = gst_ca_opengl_layer_sink_set_context; 236 gstbasesink_class->query = GST_DEBUG_FUNCPTR (gst_ca_opengl_layer_sink_query); 237 gstbasesink_class->set_caps = gst_ca_opengl_layer_sink_set_caps; 238 gstbasesink_class->get_times = gst_ca_opengl_layer_sink_get_times; 239 gstbasesink_class->prepare = gst_ca_opengl_layer_sink_prepare; 240 gstbasesink_class->propose_allocation = gst_ca_opengl_layer_sink_propose_allocation; 241 gstbasesink_class->stop = gst_ca_opengl_layer_sink_stop; 242 243 gstvideosink_class->show_frame = 244 GST_DEBUG_FUNCPTR (gst_ca_opengl_layer_sink_show_frame); 245} 246 247static void 248gst_ca_opengl_layer_sink_init (GstCAOpenGLLayerSink * ca_sink) 249{ 250 ca_sink->display = NULL; 251 ca_sink->keep_aspect_ratio = TRUE; 252 ca_sink->stored_buffer = NULL; 253 ca_sink->redisplay_texture = 0; 254 255 g_mutex_init (&ca_sink->drawing_lock); 256} 257 258static void 259gst_ca_opengl_layer_sink_finalize (GObject * object) 260{ 261 GstCAOpenGLLayerSink *ca_sink; 262 263 g_return_if_fail (GST_IS_CA_OPENGL_LAYER_SINK (object)); 264 265 ca_sink = GST_CA_OPENGL_LAYER_SINK (object); 266 267 g_mutex_clear (&ca_sink->drawing_lock); 268 269 if (ca_sink->layer) { 270 CFRelease(ca_sink->layer); 271 ca_sink->layer = NULL; 272 } 273 274 GST_DEBUG ("finalized"); 275 G_OBJECT_CLASS (parent_class)->finalize (object); 276} 277 278static void 279gst_ca_opengl_layer_sink_set_property (GObject * object, guint prop_id, 280 const GValue * value, GParamSpec * pspec) 281{ 282 GstCAOpenGLLayerSink *ca_sink; 283 284 g_return_if_fail (GST_IS_CA_OPENGL_LAYER_SINK (object)); 285 286 ca_sink = GST_CA_OPENGL_LAYER_SINK (object); 287 288 switch (prop_id) { 289 case PROP_FORCE_ASPECT_RATIO: 290 { 291 ca_sink->keep_aspect_ratio = g_value_get_boolean (value); 292 break; 293 } 294 default: 295 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); 296 break; 297 } 298} 299 300static void 301gst_ca_opengl_layer_sink_get_property (GObject * object, guint prop_id, 302 GValue * value, GParamSpec * pspec) 303{ 304 GstCAOpenGLLayerSink *ca_sink; 305 306 g_return_if_fail (GST_IS_CA_OPENGL_LAYER_SINK (object)); 307 308 ca_sink = GST_CA_OPENGL_LAYER_SINK (object); 309 310 switch (prop_id) { 311 case PROP_FORCE_ASPECT_RATIO: 312 g_value_set_boolean (value, ca_sink->keep_aspect_ratio); 313 break; 314 case PROP_CONTEXT: 315 g_value_set_object (value, ca_sink->context); 316 break; 317 case PROP_LAYER: 318 g_value_set_pointer (value, ca_sink->layer); 319 break; 320 default: 321 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); 322 break; 323 } 324} 325 326static void 327_create_layer (gpointer data) 328{ 329 GstCAOpenGLLayerSink *ca_sink = data; 330 id layer; 331 332 if (!ca_sink->layer) { 333 layer = [[NSClassFromString(@"GstGLCAOpenGLLayer") alloc] 334 initWithGstGLContext:ca_sink->context]; 335 336 ca_sink->layer = (__bridge_retained gpointer)layer; 337 [layer setDrawCallback:(GstGLWindowCB)gst_ca_opengl_layer_sink_on_draw 338 data:ca_sink notify:NULL]; 339 [layer setResizeCallback:(GstGLWindowResizeCB)gst_ca_opengl_layer_sink_on_resize 340 data:ca_sink notify:NULL]; 341 g_object_notify (G_OBJECT (ca_sink), "layer"); 342 } 343} 344 345static void 346_invoke_on_main (GstGLWindowCB func, gpointer data) 347{ 348 if ([NSThread isMainThread]) { 349 func (data); 350 } else { 351 dispatch_sync (dispatch_get_main_queue (), ^{ 352 func (data); 353 }); 354 } 355} 356 357static gboolean 358_ensure_gl_setup (GstCAOpenGLLayerSink * ca_sink) 359{ 360 GError *error = NULL; 361 362 if (!gst_gl_ensure_element_data (ca_sink, &ca_sink->display, 363 &ca_sink->other_context)) 364 return FALSE; 365 366 gst_gl_display_filter_gl_api (ca_sink->display, SUPPORTED_GL_APIS); 367 368 if (!ca_sink->context) { 369 if (!gst_gl_display_create_context (ca_sink->display, 370 ca_sink->other_context, &ca_sink->context, &error)) { 371 goto context_error; 372 } 373 } 374 375 if (!ca_sink->layer) 376 _invoke_on_main ((GstGLWindowCB) _create_layer, ca_sink); 377 378 return TRUE; 379 380context_error: 381 { 382 GST_ELEMENT_ERROR (ca_sink, RESOURCE, NOT_FOUND, ("%s", error->message), 383 (NULL)); 384 gst_object_unref (ca_sink->context); 385 ca_sink->context = NULL; 386 return FALSE; 387 } 388} 389 390static gboolean 391gst_ca_opengl_layer_sink_query (GstBaseSink * bsink, GstQuery * query) 392{ 393 GstCAOpenGLLayerSink *ca_sink = GST_CA_OPENGL_LAYER_SINK (bsink); 394 395 switch (GST_QUERY_TYPE (query)) { 396 case GST_QUERY_CONTEXT: 397 { 398 if (gst_gl_handle_context_query ((GstElement *) ca_sink, query, 399 ca_sink->display, ca_sink->context, ca_sink->other_context)) 400 return TRUE; 401 break; 402 } 403 case GST_QUERY_DRAIN: 404 { 405 GstBuffer *buf = NULL; 406 407 GST_CA_OPENGL_LAYER_SINK_LOCK (ca_sink); 408 ca_sink->redisplay_texture = 0; 409 buf = ca_sink->stored_buffer; 410 ca_sink->stored_buffer = NULL; 411 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 412 413 if (buf) 414 gst_buffer_unref (buf); 415 416 gst_buffer_replace (&ca_sink->next_buffer, NULL); 417 gst_buffer_replace (&ca_sink->next_sync, NULL); 418 419 break; 420 } 421 default: 422 break; 423 } 424 425 return GST_BASE_SINK_CLASS (parent_class)->query (bsink, query); 426} 427 428static gboolean 429gst_ca_opengl_layer_sink_stop (GstBaseSink * bsink) 430{ 431 GstCAOpenGLLayerSink *ca_sink = GST_CA_OPENGL_LAYER_SINK (bsink); 432 433 if (ca_sink->gl_caps) { 434 gst_caps_unref (ca_sink->gl_caps); 435 ca_sink->gl_caps = NULL; 436 } 437 438 return TRUE; 439} 440 441static void 442gst_ca_opengl_layer_sink_set_context (GstElement * element, GstContext * context) 443{ 444 GstCAOpenGLLayerSink *ca_sink = GST_CA_OPENGL_LAYER_SINK (element); 445 446 gst_gl_handle_set_context (element, context, &ca_sink->display, 447 &ca_sink->other_context); 448 449 if (ca_sink->display) 450 gst_gl_display_filter_gl_api (ca_sink->display, SUPPORTED_GL_APIS); 451 452 GST_ELEMENT_CLASS (parent_class)->set_context (element, context); 453} 454 455static GstStateChangeReturn 456gst_ca_opengl_layer_sink_change_state (GstElement * element, GstStateChange transition) 457{ 458 GstCAOpenGLLayerSink *ca_sink; 459 GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; 460 461 GST_DEBUG ("changing state: %s => %s", 462 gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)), 463 gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition))); 464 465 ca_sink = GST_CA_OPENGL_LAYER_SINK (element); 466 467 switch (transition) { 468 case GST_STATE_CHANGE_NULL_TO_READY: 469 _ensure_gl_setup (ca_sink); 470 break; 471 case GST_STATE_CHANGE_READY_TO_PAUSED: 472 break; 473 case GST_STATE_CHANGE_PAUSED_TO_PLAYING: 474 break; 475 default: 476 break; 477 } 478 479 ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); 480 if (ret == GST_STATE_CHANGE_FAILURE) 481 return ret; 482 483 switch (transition) { 484 case GST_STATE_CHANGE_PLAYING_TO_PAUSED: 485 break; 486 case GST_STATE_CHANGE_PAUSED_TO_READY: 487 { 488 /* mark the redisplay_texture as unavailable (=0) 489 * to avoid drawing 490 */ 491 GST_CA_OPENGL_LAYER_SINK_LOCK (ca_sink); 492 ca_sink->redisplay_texture = 0; 493 494 gst_buffer_replace (&ca_sink->stored_sync, NULL); 495 496 if (ca_sink->stored_buffer) { 497 gst_buffer_unref (ca_sink->stored_buffer); 498 ca_sink->stored_buffer = NULL; 499 } 500 gst_buffer_replace (&ca_sink->next_buffer, NULL); 501 gst_buffer_replace (&ca_sink->next_sync, NULL); 502 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 503 504 GST_VIDEO_SINK_WIDTH (ca_sink) = 1; 505 GST_VIDEO_SINK_HEIGHT (ca_sink) = 1; 506 if (ca_sink->context) { 507 gst_object_unref (ca_sink->context); 508 ca_sink->context = NULL; 509 } 510 511 if (ca_sink->display) { 512 gst_object_unref (ca_sink->display); 513 ca_sink->display = NULL; 514 } 515 break; 516 } 517 case GST_STATE_CHANGE_READY_TO_NULL: 518 if (ca_sink->layer) { 519 CFRelease(ca_sink->layer); 520 ca_sink->layer = NULL; 521 } 522 break; 523 default: 524 break; 525 } 526 527 return ret; 528} 529 530static void 531gst_ca_opengl_layer_sink_get_times (GstBaseSink * bsink, GstBuffer * buf, 532 GstClockTime * start, GstClockTime * end) 533{ 534 GstCAOpenGLLayerSink *ca_sink; 535 536 ca_sink = GST_CA_OPENGL_LAYER_SINK (bsink); 537 538 if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) { 539 *start = GST_BUFFER_TIMESTAMP (buf); 540 if (GST_BUFFER_DURATION_IS_VALID (buf)) 541 *end = *start + GST_BUFFER_DURATION (buf); 542 else { 543 if (GST_VIDEO_INFO_FPS_N (&ca_sink->info) > 0) { 544 *end = *start + 545 gst_util_uint64_scale_int (GST_SECOND, 546 GST_VIDEO_INFO_FPS_D (&ca_sink->info), 547 GST_VIDEO_INFO_FPS_N (&ca_sink->info)); 548 } 549 } 550 } 551} 552 553static gboolean 554gst_ca_opengl_layer_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) 555{ 556 GstCAOpenGLLayerSink *ca_sink; 557 gint width; 558 gint height; 559 gboolean ok; 560 gint par_n, par_d; 561 gint display_par_n, display_par_d; 562 guint display_ratio_num, display_ratio_den; 563 GstVideoInfo vinfo; 564 565 GST_DEBUG ("set caps with %" GST_PTR_FORMAT, caps); 566 567 ca_sink = GST_CA_OPENGL_LAYER_SINK (bsink); 568 569 ok = gst_video_info_from_caps (&vinfo, caps); 570 if (!ok) 571 return FALSE; 572 573 width = GST_VIDEO_INFO_WIDTH (&vinfo); 574 height = GST_VIDEO_INFO_HEIGHT (&vinfo); 575 576 par_n = GST_VIDEO_INFO_PAR_N (&vinfo); 577 par_d = GST_VIDEO_INFO_PAR_D (&vinfo); 578 579 if (!par_n) 580 par_n = 1; 581 582 display_par_n = 1; 583 display_par_d = 1; 584 585 ok = gst_video_calculate_display_ratio (&display_ratio_num, 586 &display_ratio_den, width, height, par_n, par_d, display_par_n, 587 display_par_d); 588 589 if (!ok) 590 return FALSE; 591 592 GST_TRACE ("PAR: %u/%u DAR:%u/%u", par_n, par_d, display_par_n, 593 display_par_d); 594 595 if (height % display_ratio_den == 0) { 596 GST_DEBUG ("keeping video height"); 597 GST_VIDEO_SINK_WIDTH (ca_sink) = (guint) 598 gst_util_uint64_scale_int (height, display_ratio_num, 599 display_ratio_den); 600 GST_VIDEO_SINK_HEIGHT (ca_sink) = height; 601 } else if (width % display_ratio_num == 0) { 602 GST_DEBUG ("keeping video width"); 603 GST_VIDEO_SINK_WIDTH (ca_sink) = width; 604 GST_VIDEO_SINK_HEIGHT (ca_sink) = (guint) 605 gst_util_uint64_scale_int (width, display_ratio_den, display_ratio_num); 606 } else { 607 GST_DEBUG ("approximating while keeping video height"); 608 GST_VIDEO_SINK_WIDTH (ca_sink) = (guint) 609 gst_util_uint64_scale_int (height, display_ratio_num, 610 display_ratio_den); 611 GST_VIDEO_SINK_HEIGHT (ca_sink) = height; 612 } 613 GST_DEBUG ("scaling to %dx%d", GST_VIDEO_SINK_WIDTH (ca_sink), 614 GST_VIDEO_SINK_HEIGHT (ca_sink)); 615 616 ca_sink->info = vinfo; 617 if (!_ensure_gl_setup (ca_sink)) 618 return FALSE; 619 620 ca_sink->caps_change = TRUE; 621 622 return TRUE; 623} 624 625static GstFlowReturn 626gst_ca_opengl_layer_sink_prepare (GstBaseSink * bsink, GstBuffer * buf) 627{ 628 GstCAOpenGLLayerSink *ca_sink; 629 GstBuffer *next_sync, *old_sync, *old_buffer; 630 GstVideoFrame gl_frame; 631 GstGLSyncMeta *sync_meta; 632 633 ca_sink = GST_CA_OPENGL_LAYER_SINK (bsink); 634 635 GST_TRACE ("preparing buffer:%p", buf); 636 637 if (GST_VIDEO_SINK_WIDTH (ca_sink) < 1 || 638 GST_VIDEO_SINK_HEIGHT (ca_sink) < 1) { 639 return GST_FLOW_NOT_NEGOTIATED; 640 } 641 642 if (!_ensure_gl_setup (ca_sink)) 643 return GST_FLOW_NOT_NEGOTIATED; 644 645 if (!gst_video_frame_map (&gl_frame, &ca_sink->info, buf, 646 GST_MAP_READ | GST_MAP_GL)) { 647 goto upload_failed; 648 } 649 650 ca_sink->next_tex = *(guint *) gl_frame.data[0]; 651 652 next_sync = gst_buffer_new (); 653 sync_meta = gst_buffer_add_gl_sync_meta (ca_sink->context, next_sync); 654 gst_gl_sync_meta_set_sync_point (sync_meta, ca_sink->context); 655 656 GST_CA_OPENGL_LAYER_SINK_LOCK (ca_sink); 657 ca_sink->next_tex = *(guint *) gl_frame.data[0]; 658 659 old_buffer = ca_sink->next_buffer; 660 ca_sink->next_buffer = gst_buffer_ref (buf); 661 662 old_sync = ca_sink->next_sync; 663 ca_sink->next_sync = next_sync; 664 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 665 666 if (old_buffer) 667 gst_buffer_unref (old_buffer); 668 if (old_sync) 669 gst_buffer_unref (old_sync); 670 671 gst_video_frame_unmap (&gl_frame); 672 673 return GST_FLOW_OK; 674 675upload_failed: 676 { 677 GST_ELEMENT_ERROR (ca_sink, RESOURCE, NOT_FOUND, 678 ("%s", "Failed to upload buffer"), (NULL)); 679 return GST_FLOW_ERROR; 680 } 681} 682 683static GstFlowReturn 684gst_ca_opengl_layer_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf) 685{ 686 GstCAOpenGLLayerSink *ca_sink; 687 GstBuffer *stored_buffer, *old_sync; 688 689 GST_TRACE ("rendering buffer:%p", buf); 690 691 ca_sink = GST_CA_OPENGL_LAYER_SINK (vsink); 692 693 GST_TRACE ("redisplay texture:%u of size:%ux%u, window size:%ux%u", 694 ca_sink->next_tex, GST_VIDEO_INFO_WIDTH (&ca_sink->info), 695 GST_VIDEO_INFO_HEIGHT (&ca_sink->info), 696 GST_VIDEO_SINK_WIDTH (ca_sink), 697 GST_VIDEO_SINK_HEIGHT (ca_sink)); 698 699 /* Avoid to release the texture while drawing */ 700 GST_CA_OPENGL_LAYER_SINK_LOCK (ca_sink); 701 ca_sink->redisplay_texture = ca_sink->next_tex; 702 703 stored_buffer = ca_sink->stored_buffer; 704 ca_sink->stored_buffer = gst_buffer_ref (ca_sink->next_buffer); 705 706 old_sync = ca_sink->stored_sync; 707 ca_sink->stored_sync = gst_buffer_ref (ca_sink->next_sync); 708 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 709 710 /* The layer will automatically call the draw callback to draw the new 711 * content */ 712 [CATransaction begin]; 713 [(__bridge GstGLCAOpenGLLayer *)(ca_sink->layer) setNeedsDisplay]; 714 [CATransaction commit]; 715 716 GST_TRACE ("post redisplay"); 717 718 if (stored_buffer) 719 gst_buffer_unref (stored_buffer); 720 if (old_sync) 721 gst_buffer_unref (old_sync); 722 723 return GST_FLOW_OK; 724} 725 726static gboolean 727gst_ca_opengl_layer_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) 728{ 729 GstCAOpenGLLayerSink *ca_sink = GST_CA_OPENGL_LAYER_SINK (bsink); 730 GstBufferPool *pool = NULL; 731 GstStructure *config; 732 GstCaps *caps; 733 GstVideoInfo info; 734 guint size; 735 gboolean need_pool; 736 737 if (!_ensure_gl_setup (ca_sink)) 738 return FALSE; 739 740 gst_query_parse_allocation (query, &caps, &need_pool); 741 742 if (caps == NULL) 743 goto no_caps; 744 745 if (!gst_video_info_from_caps (&info, caps)) 746 goto invalid_caps; 747 748 /* the normal size of a frame */ 749 size = info.size; 750 751 if (need_pool) { 752 GST_DEBUG_OBJECT (ca_sink, "create new pool"); 753 754 pool = gst_gl_buffer_pool_new (ca_sink->context); 755 config = gst_buffer_pool_get_config (pool); 756 gst_buffer_pool_config_set_params (config, caps, size, 0, 0); 757 758 if (!gst_buffer_pool_set_config (pool, config)) 759 goto config_failed; 760 } 761 762 /* we need at least 2 buffer because we hold on to the last one */ 763 gst_query_add_allocation_pool (query, pool, size, 2, 0); 764 if (pool) 765 gst_object_unref (pool); 766 767 if (ca_sink->context->gl_vtable->FenceSync) 768 gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0); 769 770 return TRUE; 771 772 /* ERRORS */ 773no_caps: 774 { 775 GST_DEBUG_OBJECT (bsink, "no caps specified"); 776 return FALSE; 777 } 778invalid_caps: 779 { 780 GST_DEBUG_OBJECT (bsink, "invalid caps specified"); 781 return FALSE; 782 } 783config_failed: 784 { 785 GST_DEBUG_OBJECT (bsink, "failed setting config"); 786 return FALSE; 787 } 788} 789 790/* *INDENT-OFF* */ 791static const GLfloat vertices[] = { 792 1.0f, 1.0f, 0.0f, 1.0f, 0.0f, 793 -1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 794 -1.0f, -1.0f, 0.0f, 0.0f, 1.0f, 795 1.0f, -1.0f, 0.0f, 1.0f, 1.0f 796}; 797 798static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 }; 799/* *INDENT-ON* */ 800 801static void 802_bind_buffer (GstCAOpenGLLayerSink * ca_sink) 803{ 804 const GstGLFuncs *gl = ca_sink->context->gl_vtable; 805 806 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, ca_sink->vbo_indices); 807 gl->BindBuffer (GL_ARRAY_BUFFER, ca_sink->vertex_buffer); 808 809 /* Load the vertex position */ 810 gl->VertexAttribPointer (ca_sink->attr_position, 3, GL_FLOAT, GL_FALSE, 811 5 * sizeof (GLfloat), (void *) 0); 812 813 /* Load the texture coordinate */ 814 gl->VertexAttribPointer (ca_sink->attr_texture, 2, GL_FLOAT, GL_FALSE, 815 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat))); 816 817 gl->EnableVertexAttribArray (ca_sink->attr_position); 818 gl->EnableVertexAttribArray (ca_sink->attr_texture); 819} 820 821static void 822_unbind_buffer (GstCAOpenGLLayerSink * ca_sink) 823{ 824 const GstGLFuncs *gl = ca_sink->context->gl_vtable; 825 826 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0); 827 gl->BindBuffer (GL_ARRAY_BUFFER, 0); 828 829 gl->DisableVertexAttribArray (ca_sink->attr_position); 830 gl->DisableVertexAttribArray (ca_sink->attr_texture); 831} 832 833/* Called in the gl thread */ 834static void 835gst_ca_opengl_layer_sink_thread_init_redisplay (GstCAOpenGLLayerSink * ca_sink) 836{ 837 const GstGLFuncs *gl = ca_sink->context->gl_vtable; 838 GError *error = NULL; 839 840 if (!(ca_sink->redisplay_shader = gst_gl_shader_new_default (ca_sink->context, &error))) { 841 GST_ERROR_OBJECT (ca_sink, "Failed to link shader: %s", error->message); 842 gst_ca_opengl_layer_sink_cleanup_glthread (ca_sink); 843 return; 844 } 845 846 ca_sink->attr_position = 847 gst_gl_shader_get_attribute_location (ca_sink->redisplay_shader, 848 "a_position"); 849 ca_sink->attr_texture = 850 gst_gl_shader_get_attribute_location (ca_sink->redisplay_shader, 851 "a_texcoord"); 852 853 if (gl->GenVertexArrays) { 854 gl->GenVertexArrays (1, &ca_sink->vao); 855 gl->BindVertexArray (ca_sink->vao); 856 } 857 858 if (!ca_sink->vertex_buffer) { 859 gl->GenBuffers (1, &ca_sink->vertex_buffer); 860 gl->BindBuffer (GL_ARRAY_BUFFER, ca_sink->vertex_buffer); 861 gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), vertices, 862 GL_STATIC_DRAW); 863 } 864 865 if (!ca_sink->vbo_indices) { 866 gl->GenBuffers (1, &ca_sink->vbo_indices); 867 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, ca_sink->vbo_indices); 868 gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices, 869 GL_STATIC_DRAW); 870 } 871 872 if (gl->GenVertexArrays) { 873 _bind_buffer (ca_sink); 874 gl->BindVertexArray (0); 875 } 876 877 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0); 878 gl->BindBuffer (GL_ARRAY_BUFFER, 0); 879} 880 881static void 882gst_ca_opengl_layer_sink_cleanup_glthread (GstCAOpenGLLayerSink * ca_sink) 883{ 884 const GstGLFuncs *gl = ca_sink->context->gl_vtable; 885 886 if (ca_sink->redisplay_shader) { 887 gst_object_unref (ca_sink->redisplay_shader); 888 ca_sink->redisplay_shader = NULL; 889 } 890 891 if (ca_sink->vao) { 892 gl->DeleteVertexArrays (1, &ca_sink->vao); 893 ca_sink->vao = 0; 894 } 895 896 if (ca_sink->vbo_indices) { 897 gl->DeleteBuffers (1, &ca_sink->vbo_indices); 898 ca_sink->vbo_indices = 0; 899 } 900 901 if (ca_sink->vertex_buffer) { 902 gl->DeleteBuffers (1, &ca_sink->vertex_buffer); 903 ca_sink->vertex_buffer = 0; 904 } 905} 906 907static void 908gst_ca_opengl_layer_sink_on_resize (GstCAOpenGLLayerSink * ca_sink, gint width, gint height) 909{ 910 /* Here ca_sink members (ex:ca_sink->info) have a life time of set_caps. 911 * It means that they cannot not change between two set_caps 912 */ 913 const GstGLFuncs *gl = ca_sink->context->gl_vtable; 914 915 GST_TRACE ("GL Window resized to %ux%u", width, height); 916 917 width = MAX (1, width); 918 height = MAX (1, height); 919 920 ca_sink->window_width = width; 921 ca_sink->window_height = height; 922 923 /* default reshape */ 924 if (ca_sink->keep_aspect_ratio) { 925 GstVideoRectangle src, dst, result; 926 927 src.x = 0; 928 src.y = 0; 929 src.w = GST_VIDEO_SINK_WIDTH (ca_sink); 930 src.h = GST_VIDEO_SINK_HEIGHT (ca_sink); 931 932 dst.x = 0; 933 dst.y = 0; 934 dst.w = width; 935 dst.h = height; 936 937 gst_video_sink_center_rect (src, dst, &result, TRUE); 938 gl->Viewport (result.x, result.y, result.w, result.h); 939 } else { 940 gl->Viewport (0, 0, width, height); 941 } 942} 943 944static void 945gst_ca_opengl_layer_sink_on_draw (GstCAOpenGLLayerSink * ca_sink) 946{ 947 /* Here ca_sink members (ex:ca_sink->info) have a life time of set_caps. 948 * It means that they cannot not change between two set_caps as well as 949 * for the redisplay_texture size. 950 * Whereas redisplay_texture id changes every sink_render 951 */ 952 953 const GstGLFuncs *gl = NULL; 954 GstGLSyncMeta *sync_meta; 955 956 g_return_if_fail (GST_IS_CA_OPENGL_LAYER_SINK (ca_sink)); 957 958 gl = ca_sink->context->gl_vtable; 959 960 GST_CA_OPENGL_LAYER_SINK_LOCK (ca_sink); 961 962 if (G_UNLIKELY (!ca_sink->redisplay_shader)) { 963 gst_ca_opengl_layer_sink_thread_init_redisplay (ca_sink); 964 } 965 966 /* check if texture is ready for being drawn */ 967 if (!ca_sink->redisplay_texture) { 968 gl->ClearColor (0.0f, 0.0f, 0.0f, 1.0f); 969 gl->Clear (GL_COLOR_BUFFER_BIT); 970 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 971 return; 972 } 973 974 /* opengl scene */ 975 GST_TRACE ("redrawing texture:%u", ca_sink->redisplay_texture); 976 977 if (ca_sink->caps_change) { 978 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 979 gst_ca_opengl_layer_sink_on_resize (ca_sink, ca_sink->window_width, 980 ca_sink->window_height); 981 GST_CA_OPENGL_LAYER_SINK_LOCK (ca_sink); 982 ca_sink->caps_change = FALSE; 983 } 984 985 sync_meta = gst_buffer_get_gl_sync_meta (ca_sink->stored_sync); 986 if (sync_meta) 987 gst_gl_sync_meta_wait (sync_meta, gst_gl_context_get_current ()); 988 989 gl->BindTexture (GL_TEXTURE_2D, 0); 990 991 gl->ClearColor (0.0, 0.0, 0.0, 0.0); 992 gl->Clear (GL_COLOR_BUFFER_BIT); 993 994 gst_gl_shader_use (ca_sink->redisplay_shader); 995 996 if (gl->GenVertexArrays) 997 gl->BindVertexArray (ca_sink->vao); 998 _bind_buffer (ca_sink); 999 1000 gl->ActiveTexture (GL_TEXTURE0); 1001 gl->BindTexture (GL_TEXTURE_2D, ca_sink->redisplay_texture); 1002 gst_gl_shader_set_uniform_1i (ca_sink->redisplay_shader, "tex", 0); 1003 1004 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0); 1005 1006 if (gl->GenVertexArrays) 1007 gl->BindVertexArray (0); 1008 else 1009 _unbind_buffer (ca_sink); 1010 1011 /* end default opengl scene */ 1012 GST_CA_OPENGL_LAYER_SINK_UNLOCK (ca_sink); 1013} 1014