1 /* SPDX-License-Identifier: GPL-3.0-or-later
2 * Copyright © 2016-2018 The TokTok team.
3 * Copyright © 2013-2015 Tox project.
4 */
5 #ifdef HAVE_CONFIG_H
6 #include "config.h"
7 #endif /* HAVE_CONFIG_H */
8
9 #include "video.h"
10
11 #include <assert.h>
12 #include <stdlib.h>
13 #include <string.h>
14
15 #include "msi.h"
16 #include "ring_buffer.h"
17 #include "rtp.h"
18
19 #include "../toxcore/logger.h"
20 #include "../toxcore/mono_time.h"
21 #include "../toxcore/network.h"
22
23 /**
24 * Soft deadline the decoder should attempt to meet, in "us" (microseconds).
25 * Set to zero for unlimited.
26 *
27 * By convention, the value 1 is used to mean "return as fast as possible."
28 */
29 // TODO(zoff99): don't hardcode this, let the application choose it
30 #define WANTED_MAX_DECODER_FPS 40
31
32 /**
33 * VPX_DL_REALTIME (1)
34 * deadline parameter analogous to VPx REALTIME mode.
35 *
36 * VPX_DL_GOOD_QUALITY (1000000)
37 * deadline parameter analogous to VPx GOOD QUALITY mode.
38 *
39 * VPX_DL_BEST_QUALITY (0)
40 * deadline parameter analogous to VPx BEST QUALITY mode.
41 */
42 #define MAX_DECODE_TIME_US (1000000 / WANTED_MAX_DECODER_FPS) // to allow x fps
43
44 /**
45 * Codec control function to set encoder internal speed settings. Changes in
46 * this value influences, among others, the encoder's selection of motion
47 * estimation methods. Values greater than 0 will increase encoder speed at the
48 * expense of quality.
49 *
50 * Note Valid range for VP8: -16..16
51 */
52 #define VP8E_SET_CPUUSED_VALUE 16
53
54 /**
55 * Initialize encoder with this value. Target bandwidth to use for this stream, in kilobits per second.
56 */
57 #define VIDEO_BITRATE_INITIAL_VALUE 5000
58 #define VIDEO_DECODE_BUFFER_SIZE 5 // this buffer has normally max. 1 entry
59
video_codec_decoder_interface(void)60 static vpx_codec_iface_t *video_codec_decoder_interface(void)
61 {
62 return vpx_codec_vp8_dx();
63 }
video_codec_encoder_interface(void)64 static vpx_codec_iface_t *video_codec_encoder_interface(void)
65 {
66 return vpx_codec_vp8_cx();
67 }
68
69 #define VIDEO_CODEC_DECODER_MAX_WIDTH 800 // its a dummy value, because the struct needs a value there
70 #define VIDEO_CODEC_DECODER_MAX_HEIGHT 600 // its a dummy value, because the struct needs a value there
71
72 #define VPX_MAX_DIST_START 40
73
74 #define VPX_MAX_ENCODER_THREADS 4
75 #define VPX_MAX_DECODER_THREADS 4
76 #define VIDEO_VP8_DECODER_POST_PROCESSING_ENABLED 0
77
vc_init_encoder_cfg(const Logger * log,vpx_codec_enc_cfg_t * cfg,int16_t kf_max_dist)78 static void vc_init_encoder_cfg(const Logger *log, vpx_codec_enc_cfg_t *cfg, int16_t kf_max_dist)
79 {
80 vpx_codec_err_t rc = vpx_codec_enc_config_default(video_codec_encoder_interface(), cfg, 0);
81
82 if (rc != VPX_CODEC_OK) {
83 LOGGER_ERROR(log, "vc_init_encoder_cfg:Failed to get config: %s", vpx_codec_err_to_string(rc));
84 }
85
86 /* Target bandwidth to use for this stream, in kilobits per second */
87 cfg->rc_target_bitrate = VIDEO_BITRATE_INITIAL_VALUE;
88 cfg->g_w = VIDEO_CODEC_DECODER_MAX_WIDTH;
89 cfg->g_h = VIDEO_CODEC_DECODER_MAX_HEIGHT;
90 cfg->g_pass = VPX_RC_ONE_PASS;
91 cfg->g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS;
92 cfg->g_lag_in_frames = 0;
93
94 /* Allow lagged encoding
95 *
96 * If set, this value allows the encoder to consume a number of input
97 * frames before producing output frames. This allows the encoder to
98 * base decisions for the current frame on future frames. This does
99 * increase the latency of the encoding pipeline, so it is not appropriate
100 * in all situations (ex: realtime encoding).
101 *
102 * Note that this is a maximum value -- the encoder may produce frames
103 * sooner than the given limit. Set this value to 0 to disable this
104 * feature.
105 */
106 cfg->kf_min_dist = 0;
107 cfg->kf_mode = VPX_KF_AUTO; // Encoder determines optimal placement automatically
108 cfg->rc_end_usage = VPX_VBR; // what quality mode?
109
110 /*
111 * VPX_VBR Variable Bit Rate (VBR) mode
112 * VPX_CBR Constant Bit Rate (CBR) mode
113 * VPX_CQ Constrained Quality (CQ) mode -> give codec a hint that we may be on low bandwidth connection
114 * VPX_Q Constant Quality (Q) mode
115 */
116 if (kf_max_dist > 1) {
117 cfg->kf_max_dist = kf_max_dist; // a full frame every x frames minimum (can be more often, codec decides automatically)
118 LOGGER_DEBUG(log, "kf_max_dist=%d (1)", cfg->kf_max_dist);
119 } else {
120 cfg->kf_max_dist = VPX_MAX_DIST_START;
121 LOGGER_DEBUG(log, "kf_max_dist=%d (2)", cfg->kf_max_dist);
122 }
123
124 cfg->g_threads = VPX_MAX_ENCODER_THREADS; // Maximum number of threads to use
125 /* TODO: set these to something reasonable */
126 // cfg->g_timebase.num = 1;
127 // cfg->g_timebase.den = 60; // 60 fps
128 cfg->rc_resize_allowed = 1; // allow encoder to resize to smaller resolution
129 cfg->rc_resize_up_thresh = 40;
130 cfg->rc_resize_down_thresh = 5;
131
132 /* TODO: make quality setting an API call, but start with normal quality */
133 #if 0
134 /* Highest-resolution encoder settings */
135 cfg->rc_dropframe_thresh = 0;
136 cfg->rc_resize_allowed = 0;
137 cfg->rc_min_quantizer = 2;
138 cfg->rc_max_quantizer = 56;
139 cfg->rc_undershoot_pct = 100;
140 cfg->rc_overshoot_pct = 15;
141 cfg->rc_buf_initial_sz = 500;
142 cfg->rc_buf_optimal_sz = 600;
143 cfg->rc_buf_sz = 1000;
144 #endif
145 }
146
vc_new(Mono_Time * mono_time,const Logger * log,ToxAV * av,uint32_t friend_number,toxav_video_receive_frame_cb * cb,void * cb_data)147 VCSession *vc_new(Mono_Time *mono_time, const Logger *log, ToxAV *av, uint32_t friend_number,
148 toxav_video_receive_frame_cb *cb, void *cb_data)
149 {
150 VCSession *vc = (VCSession *)calloc(sizeof(VCSession), 1);
151 vpx_codec_err_t rc;
152
153 if (!vc) {
154 LOGGER_WARNING(log, "Allocation failed! Application might misbehave!");
155 return nullptr;
156 }
157
158 if (create_recursive_mutex(vc->queue_mutex) != 0) {
159 LOGGER_WARNING(log, "Failed to create recursive mutex!");
160 free(vc);
161 return nullptr;
162 }
163
164 int cpu_used_value = VP8E_SET_CPUUSED_VALUE;
165
166 vc->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE);
167
168 if (!vc->vbuf_raw) {
169 goto BASE_CLEANUP;
170 }
171
172 /*
173 * VPX_CODEC_USE_FRAME_THREADING
174 * Enable frame-based multi-threading
175 *
176 * VPX_CODEC_USE_ERROR_CONCEALMENT
177 * Conceal errors in decoded frames
178 */
179 vpx_codec_dec_cfg_t dec_cfg;
180 dec_cfg.threads = VPX_MAX_DECODER_THREADS; // Maximum number of threads to use
181 dec_cfg.w = VIDEO_CODEC_DECODER_MAX_WIDTH;
182 dec_cfg.h = VIDEO_CODEC_DECODER_MAX_HEIGHT;
183
184 LOGGER_DEBUG(log, "Using VP8 codec for decoder (0)");
185 rc = vpx_codec_dec_init(vc->decoder, video_codec_decoder_interface(), &dec_cfg,
186 VPX_CODEC_USE_FRAME_THREADING | VPX_CODEC_USE_POSTPROC);
187
188 if (rc == VPX_CODEC_INCAPABLE) {
189 LOGGER_WARNING(log, "Postproc not supported by this decoder (0)");
190 rc = vpx_codec_dec_init(vc->decoder, video_codec_decoder_interface(), &dec_cfg, VPX_CODEC_USE_FRAME_THREADING);
191 }
192
193 if (rc != VPX_CODEC_OK) {
194 LOGGER_ERROR(log, "Init video_decoder failed: %s", vpx_codec_err_to_string(rc));
195 goto BASE_CLEANUP;
196 }
197
198 if (VIDEO_VP8_DECODER_POST_PROCESSING_ENABLED == 1) {
199 vp8_postproc_cfg_t pp = {VP8_DEBLOCK, 1, 0};
200 vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp);
201
202 if (cc_res != VPX_CODEC_OK) {
203 LOGGER_WARNING(log, "Failed to turn on postproc");
204 } else {
205 LOGGER_DEBUG(log, "turn on postproc: OK");
206 }
207 } else {
208 vp8_postproc_cfg_t pp = {0, 0, 0};
209 vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp);
210
211 if (cc_res != VPX_CODEC_OK) {
212 LOGGER_WARNING(log, "Failed to turn OFF postproc");
213 } else {
214 LOGGER_DEBUG(log, "Disable postproc: OK");
215 }
216 }
217
218 /* Set encoder to some initial values
219 */
220 vpx_codec_enc_cfg_t cfg;
221 vc_init_encoder_cfg(log, &cfg, 1);
222
223 LOGGER_DEBUG(log, "Using VP8 codec for encoder (0.1)");
224 rc = vpx_codec_enc_init(vc->encoder, video_codec_encoder_interface(), &cfg, VPX_CODEC_USE_FRAME_THREADING);
225
226 if (rc != VPX_CODEC_OK) {
227 LOGGER_ERROR(log, "Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
228 goto BASE_CLEANUP_1;
229 }
230
231 rc = vpx_codec_control(vc->encoder, VP8E_SET_CPUUSED, cpu_used_value);
232
233 if (rc != VPX_CODEC_OK) {
234 LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
235 vpx_codec_destroy(vc->encoder);
236 goto BASE_CLEANUP_1;
237 }
238
239 /*
240 * VPX_CTRL_USE_TYPE(VP8E_SET_NOISE_SENSITIVITY, unsigned int)
241 * control function to set noise sensitivity
242 * 0: off, 1: OnYOnly, 2: OnYUV, 3: OnYUVAggressive, 4: Adaptive
243 */
244 #if 0
245 rc = vpx_codec_control(vc->encoder, VP8E_SET_NOISE_SENSITIVITY, 2);
246
247 if (rc != VPX_CODEC_OK) {
248 LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
249 vpx_codec_destroy(vc->encoder);
250 goto BASE_CLEANUP_1;
251 }
252
253 #endif
254 vc->linfts = current_time_monotonic(mono_time);
255 vc->lcfd = 60;
256 vc->vcb = cb;
257 vc->vcb_user_data = cb_data;
258 vc->friend_number = friend_number;
259 vc->av = av;
260 vc->log = log;
261 return vc;
262 BASE_CLEANUP_1:
263 vpx_codec_destroy(vc->decoder);
264 BASE_CLEANUP:
265 pthread_mutex_destroy(vc->queue_mutex);
266 rb_kill(vc->vbuf_raw);
267 free(vc);
268 return nullptr;
269 }
270
vc_kill(VCSession * vc)271 void vc_kill(VCSession *vc)
272 {
273 if (!vc) {
274 return;
275 }
276
277 vpx_codec_destroy(vc->encoder);
278 vpx_codec_destroy(vc->decoder);
279 void *p;
280
281 while (rb_read(vc->vbuf_raw, &p)) {
282 free(p);
283 }
284
285 rb_kill(vc->vbuf_raw);
286 pthread_mutex_destroy(vc->queue_mutex);
287 LOGGER_DEBUG(vc->log, "Terminated video handler: %p", (void *)vc);
288 free(vc);
289 }
290
vc_iterate(VCSession * vc)291 void vc_iterate(VCSession *vc)
292 {
293 if (!vc) {
294 return;
295 }
296
297 pthread_mutex_lock(vc->queue_mutex);
298
299 struct RTPMessage *p;
300
301 if (!rb_read(vc->vbuf_raw, (void **)&p)) {
302 LOGGER_TRACE(vc->log, "no Video frame data available");
303 pthread_mutex_unlock(vc->queue_mutex);
304 return;
305 }
306
307 uint16_t log_rb_size = rb_size(vc->vbuf_raw);
308 pthread_mutex_unlock(vc->queue_mutex);
309 const struct RTPHeader *const header = &p->header;
310
311 uint32_t full_data_len;
312
313 if (header->flags & RTP_LARGE_FRAME) {
314 full_data_len = header->data_length_full;
315 LOGGER_DEBUG(vc->log, "vc_iterate:001:full_data_len=%d", (int)full_data_len);
316 } else {
317 full_data_len = p->len;
318 LOGGER_DEBUG(vc->log, "vc_iterate:002");
319 }
320
321 LOGGER_DEBUG(vc->log, "vc_iterate: rb_read p->len=%d p->header.xe=%d", (int)full_data_len, p->header.xe);
322 LOGGER_DEBUG(vc->log, "vc_iterate: rb_read rb size=%d", (int)log_rb_size);
323 const vpx_codec_err_t rc = vpx_codec_decode(vc->decoder, p->data, full_data_len, nullptr, MAX_DECODE_TIME_US);
324 free(p);
325
326 if (rc != VPX_CODEC_OK) {
327 LOGGER_ERROR(vc->log, "Error decoding video: %d %s", (int)rc, vpx_codec_err_to_string(rc));
328 return;
329 }
330
331 /* Play decoded images */
332 vpx_codec_iter_t iter = nullptr;
333
334 for (vpx_image_t *dest = vpx_codec_get_frame(vc->decoder, &iter);
335 dest != nullptr;
336 dest = vpx_codec_get_frame(vc->decoder, &iter)) {
337 if (vc->vcb) {
338 vc->vcb(vc->av, vc->friend_number, dest->d_w, dest->d_h,
339 (const uint8_t *)dest->planes[0], (const uint8_t *)dest->planes[1], (const uint8_t *)dest->planes[2],
340 dest->stride[0], dest->stride[1], dest->stride[2], vc->vcb_user_data);
341 }
342
343 vpx_img_free(dest); // is this needed? none of the VPx examples show that
344 }
345 }
346
vc_queue_message(Mono_Time * mono_time,void * vcp,struct RTPMessage * msg)347 int vc_queue_message(Mono_Time *mono_time, void *vcp, struct RTPMessage *msg)
348 {
349 /* This function is called with complete messages
350 * they have already been assembled.
351 * this function gets called from handle_rtp_packet() and handle_rtp_packet_v3()
352 */
353 if (!vcp || !msg) {
354 if (msg) {
355 free(msg);
356 }
357
358 return -1;
359 }
360
361 VCSession *vc = (VCSession *)vcp;
362 const struct RTPHeader *const header = &msg->header;
363
364 if (msg->header.pt == (RTP_TYPE_VIDEO + 2) % 128) {
365 LOGGER_WARNING(vc->log, "Got dummy!");
366 free(msg);
367 return 0;
368 }
369
370 if (msg->header.pt != RTP_TYPE_VIDEO % 128) {
371 LOGGER_WARNING(vc->log, "Invalid payload type! pt=%d", (int)msg->header.pt);
372 free(msg);
373 return -1;
374 }
375
376 pthread_mutex_lock(vc->queue_mutex);
377
378 if ((header->flags & RTP_LARGE_FRAME) && header->pt == RTP_TYPE_VIDEO % 128) {
379 LOGGER_DEBUG(vc->log, "rb_write msg->len=%d b0=%d b1=%d", (int)msg->len, (int)msg->data[0], (int)msg->data[1]);
380 }
381
382 free(rb_write(vc->vbuf_raw, msg));
383
384 /* Calculate time it took for peer to send us this frame */
385 uint32_t t_lcfd = current_time_monotonic(mono_time) - vc->linfts;
386 vc->lcfd = t_lcfd > 100 ? vc->lcfd : t_lcfd;
387 vc->linfts = current_time_monotonic(mono_time);
388 pthread_mutex_unlock(vc->queue_mutex);
389 return 0;
390 }
391
vc_reconfigure_encoder(VCSession * vc,uint32_t bit_rate,uint16_t width,uint16_t height,int16_t kf_max_dist)392 int vc_reconfigure_encoder(VCSession *vc, uint32_t bit_rate, uint16_t width, uint16_t height, int16_t kf_max_dist)
393 {
394 if (!vc) {
395 return -1;
396 }
397
398 vpx_codec_enc_cfg_t cfg2 = *vc->encoder->config.enc;
399 vpx_codec_err_t rc;
400
401 if (cfg2.rc_target_bitrate == bit_rate && cfg2.g_w == width && cfg2.g_h == height && kf_max_dist == -1) {
402 return 0; /* Nothing changed */
403 }
404
405 if (cfg2.g_w == width && cfg2.g_h == height && kf_max_dist == -1) {
406 /* Only bit rate changed */
407 LOGGER_INFO(vc->log, "bitrate change from: %u to: %u", (uint32_t)cfg2.rc_target_bitrate, (uint32_t)bit_rate);
408 cfg2.rc_target_bitrate = bit_rate;
409 rc = vpx_codec_enc_config_set(vc->encoder, &cfg2);
410
411 if (rc != VPX_CODEC_OK) {
412 LOGGER_ERROR(vc->log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
413 return -1;
414 }
415 } else {
416 /* Resolution is changed, must reinitialize encoder since libvpx v1.4 doesn't support
417 * reconfiguring encoder to use resolutions greater than initially set.
418 */
419 LOGGER_DEBUG(vc->log, "Have to reinitialize vpx encoder on session %p", (void *)vc);
420 vpx_codec_ctx_t new_c;
421 vpx_codec_enc_cfg_t cfg;
422 vc_init_encoder_cfg(vc->log, &cfg, kf_max_dist);
423 cfg.rc_target_bitrate = bit_rate;
424 cfg.g_w = width;
425 cfg.g_h = height;
426
427 LOGGER_DEBUG(vc->log, "Using VP8 codec for encoder");
428 rc = vpx_codec_enc_init(&new_c, video_codec_encoder_interface(), &cfg, VPX_CODEC_USE_FRAME_THREADING);
429
430 if (rc != VPX_CODEC_OK) {
431 LOGGER_ERROR(vc->log, "Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
432 return -1;
433 }
434
435 int cpu_used_value = VP8E_SET_CPUUSED_VALUE;
436
437 rc = vpx_codec_control(&new_c, VP8E_SET_CPUUSED, cpu_used_value);
438
439 if (rc != VPX_CODEC_OK) {
440 LOGGER_ERROR(vc->log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
441 vpx_codec_destroy(&new_c);
442 return -1;
443 }
444
445 vpx_codec_destroy(vc->encoder);
446 memcpy(vc->encoder, &new_c, sizeof(new_c));
447 }
448
449 return 0;
450 }
451