1 /*** 2 3 Olive - Non-Linear Video Editor 4 Copyright (C) 2019 Olive Team 5 6 This program is free software: you can redistribute it and/or modify 7 it under the terms of the GNU General Public License as published by 8 the Free Software Foundation, either version 3 of the License, or 9 (at your option) any later version. 10 11 This program is distributed in the hope that it will be useful, 12 but WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 GNU General Public License for more details. 15 16 You should have received a copy of the GNU General Public License 17 along with this program. If not, see <http://www.gnu.org/licenses/>. 18 19 ***/ 20 21 #ifndef CACHER_H 22 #define CACHER_H 23 24 #define __STDC_FORMAT_MACROS 1 25 #include <inttypes.h> 26 27 extern "C" { 28 #include <libavformat/avformat.h> 29 #include <libavcodec/avcodec.h> 30 #include <libswscale/swscale.h> 31 #include <libswresample/swresample.h> 32 #include <libavfilter/avfilter.h> 33 #include <libavfilter/buffersrc.h> 34 #include <libavfilter/buffersink.h> 35 #include <libavutil/opt.h> 36 #include <libavutil/pixdesc.h> 37 } 38 39 #include <memory> 40 #include <QThread> 41 #include <QVector> 42 #include <QWaitCondition> 43 #include <QMutex> 44 45 #include "rendering/clipqueue.h" 46 47 class Clip; 48 49 /** 50 * @brief The Cacher class 51 * 52 * For footage clips - usually the majority of clips - decoding can be strenuous on CPU and inconsistent in timing. As 53 * a result, we keep a memory cache of upcoming frames that we fill in a background thread so they can be retrieved from 54 * a rendering thread later. This class is the background thread filling up a clip's frame cache (also called a "queue" 55 * since video files are usually stored with frames in linear chronological order). It involves decoding routines to 56 * retrieve raw frames from the file (using libavformat/libavcodec), conversion routines to conform the raw frames to 57 * RGBA/S16LE for the rest of the workflow (using libavfilter/libswscale/libswresample), and memory handling routines 58 * for keeping the cache within limits defined by the user (see Config::upcoming_queue_type). 59 * 60 * Generally the Cacher workflow starts by calling Open() which will start the thread, open a file handle, and create a 61 * decoding instance. Open() is usually called directly from the parent Clip's Clip::Open() and thus expects the 62 * Clip::state_change_lock to be locked. It will unlock it when it's finished opening and is ready to start caching, 63 * meaning Clip::state_change_lock can be used to synchronize threads. 64 * 65 * --- 66 * 67 * **For video:** 68 * 69 * After the Cacher has finished opening, request a frame by calling Cache(). Cache() will tell the Cacher 70 * information about the current playback state, most importantly the current place in time according to the Sequence's 71 * playhead. Cache() determines whether the requested frame is already in the queue, and then signals the Cacher thread 72 * to cache ahead if there's room in the queue (and also remove old frames that are no longer necessary). To retrieve 73 * the requested frame, call Retrieve(). 74 * 75 * If Cache() found the frame already in the queue, Retrieve() will return immediately with this frame. Otherwise 76 * Retrieve() may block while the cacher retrieves it. Therefore it is recommended never to call Retrieve() from 77 * the main thread. Retrieve() may also return `nullptr` if there was an issue, e.g. the cacher failed to retrieve 78 * the frame. 79 * 80 * **For audio:** 81 * 82 * After the Cacher has finished opening, calling Cache() will handle most of the work. It will decode the audio, 83 * convert to the correct sample rate and format, reverse or adjust speed if necessary, and send it to the audio 84 * buffer ready to be played by the output device. It is important to continually call Cache() as it doesn't get 85 * signalled when more samples are available in the audio buffer. Instead, it'll check every time it's called and 86 * fill as much of the buffer as it can. 87 * 88 * If the user seeks, ResetAudio() must be called to signal the Cacher to interrupt the current audio stream and 89 * move somewhere else before continuing. 90 * 91 * --- 92 * 93 * Finally, when the Cacher/parent Clip are no longer in use, call Close() to free all memory and file handling 94 * allocated for the cacher. You can choose whether to wait for Close() and all of its child processes to complete - 95 * e.g. if you need to change something with the Clip or attached Footage that changes how it opens and want to be 96 * thread-safe - or let the Close thread finish up on its own. 97 * 98 * Cacher expects to be multithreaded and all of its public functions are thread-safe. 99 */ 100 class Cacher : public QThread 101 { 102 Q_OBJECT 103 public: 104 /** 105 * @brief Cacher Constructor 106 * 107 * Create Cacher object. The thread is not started here. To start it, call Open(). 108 * 109 * @param c 110 */ 111 Cacher(Clip* c); 112 113 /** 114 * @brief The main QThread loop 115 * 116 * Once the thread has started, all Cacher functions will be called from here until the Cacher closes at which point 117 * it will close and exit gracefully. 118 */ 119 void run(); 120 121 /** 122 * @brief Open the cacher 123 * 124 * Starts the thread and all file/decode handlers. Really just sets some default values and starts the thread, which 125 * will in turn call OpenWorker() at the start of its functions. 126 * 127 * Make sure Clip::state_change_lock is LOCKED before calling this function as the opening process will try to unlock 128 * it when it's finished (leading to a crash if it's not already locked). 129 */ 130 void Open(); 131 132 /** 133 * @brief Request a frame to be cached 134 * 135 * For video, this function is part 1 of the Cache()/Retrieve() workflow. It signals the thread to start caching and 136 * provides a few other details about the playback state. For optimization it'll also check the frame queue if it 137 * already contains the requested frame and use it if so, potentially speeding up Retrieve() later on. Otherwise 138 * it'll interrupt any currently caching operation and signal it to start again. 139 * While Retrieve() will block until the correct frame is retrieved, this function will return fairly quickly (either 140 * immediately if the frame was found in the queue, or once the cacher has restarted caching if not). This means 141 * Cache() can be called from another thread and then that other thread can do other work while the cacher is 142 * retrieving the frame, finally calling Retrieve() once the frame is absolutely necessary. 143 * 144 * For audio, this function will do all the work of signalling the thread to start caching and sending samples to 145 * the output audio buffer. It's used in tandem with ResetAudio() when the Timeline header is changed abruptly. 146 * 147 * @param playhead 148 * 149 * The current Timeline played position in frames 150 * 151 * @param scrubbing 152 * 153 * **TRUE** if the user is currently scrubbing. **FALSE** if not. 154 * 155 * @param nests 156 * 157 * A hierarchy of nested sequences, if the playback traversed any to get to this clip. 158 * 159 * @param playback_speed 160 * 161 * The current playback speed (controlled by Shuttle Left/Stop/Right) 162 */ 163 void Cache(long playhead, bool scrubbing, QVector<Clip*>& nests, int playback_speed); 164 165 /** 166 * @brief Retrieve frame requested by Cache() 167 * 168 * Part 2 of the Cache()/Retrieve() workflow, only used for video. Whichever frame was requested by Cache(), this 169 * function will try to retrieve it. In most cases, this function will be pretty quick as the frame will be available 170 * immediately from Cache()'s optimization or the cacher thread will be close to retrieving the correct frame anyway. 171 * However it does block for however long it takes to retrieve the correct frame (if the cacher is running) so it's 172 * not recommended to call this from any main/GUI thread. 173 * 174 * @return 175 * 176 * The frame requested by Cache(), or `nullptr` if there was an error (e.g. the cacher wasn't running and no frame was 177 * available). 178 */ 179 AVFrame* Retrieve(); 180 181 /** 182 * @brief Close the cacher and free any allocated memory 183 * 184 * When the Cacher thread is no longer needed, Close() should be called in order to free system resources. This will 185 * signal the thread to exit gracefully, but will not delete the thread object since the cacher may need to be 186 * re-opened later by Open(). 187 * 188 * @param wait_for_finish 189 * 190 * **TRUE** if this function should block the calling thread until the Clip has finished closing. Often necessary if 191 * the Clip is being closed specifically to make changes to it. 192 */ 193 void Close(bool wait_for_finish); 194 195 /** 196 * @brief Interrupt and reset audio state 197 * 198 * Used in tandem with Cache(), only for audio clips. Cache() will decode and send audio continually as it's 199 * repeatedly called. If the audio stream needs to be interrupted and moved somewhere else for any reason 200 * (e.g. the user seeked somewhere else), then it's necessary to call ResetAudio() to signal the cacher to 201 * seek to the next place indicated by Cache(). 202 */ 203 void ResetAudio(); 204 205 /** 206 * @brief Retrieve current media width 207 * 208 * In some situations, the actual media we're using may be a different resolution to how we're treating it (e.g. 209 * lower resolution proxies). While most functions will happily treat the media as its original resolution, some 210 * processes will need the absolute resolution from the file which can be acquired here. 211 * 212 * Only call after the thread has been opened by Open(). 213 * 214 * @return 215 * 216 * The true width of the current video file. 217 */ 218 int media_width(); 219 220 /** 221 * @brief Retrieve current media height 222 * 223 * See media_width(). 224 * 225 * Only call after the thread has been opened by Open(). 226 * 227 * @return 228 * 229 * The true height of the current video file. 230 */ 231 int media_height(); 232 233 /** 234 * @brief Retrieve media time base 235 * 236 * For some timing operations, it's necessary to use the source media's timebase. Similar to media_width() and 237 * media_height(), we need the accurate timebase from the file as a proxy's timebase may or may not be the same 238 * as the source file. 239 * 240 * Only call after the thread has been opened by Open(). 241 * 242 * @return 243 * 244 * The timebase of the file. 245 */ 246 AVRational media_time_base(); 247 248 /** 249 * @brief Get cacher queue object 250 * 251 * @return 252 * 253 * A pointer to the cacher's internal frame queue 254 */ 255 ClipQueue* queue(); 256 257 private: 258 /** 259 * @brief Reference to the parent clip. Set in the constructor and never changed during this object's lifetime. 260 */ 261 Clip* clip; 262 263 /** 264 * @brief Frame queue 265 * 266 * Valid fames are cached into this, which also does memory handling when necessary. 267 */ 268 ClipQueue queue_; 269 270 /** 271 * @brief Main wait condition 272 * 273 * Used with Clip::cache_lock as the main block while the the Cacher thread isn't running. Wake this condition 274 * to start caching. 275 */ 276 QWaitCondition wait_cond_; 277 278 /** 279 * @brief Main thread wait condition 280 * 281 * Used with main_thread_lock_ to block Cache() while waiting for a response from the cacher thread. 282 */ 283 QWaitCondition main_thread_wait_; 284 285 /** 286 * @brief Main thread mutex 287 * 288 * Used with main_thread_wait_ to block Cache() while waiting for a response from the cacher thread. 289 */ 290 QMutex main_thread_lock_; 291 292 /** 293 * @brief Retrieve() wait condition 294 * 295 * Used with retrieve_lock_ to block Retrieve() if the cacher hasn't retrieved the correct frame yet. 296 */ 297 QWaitCondition retrieve_wait_; 298 299 /** 300 * @brief Retrieve() mutex 301 * 302 * Used with retrieve_wait_ to block Retrieve() if the cacher hasn't retrieved the correct frame yet. 303 */ 304 QMutex retrieve_lock_; 305 306 /** 307 * @brief Set and used by CacheAudioWorker if the decoder receives an EOF. 308 * 309 * Deprecated. CacheAudioWorker() is functional but probably should be rewritten. 310 */ 311 bool reached_end; 312 313 /** 314 * @brief Current Sequence playhead set by Cache() 315 */ 316 long playhead_; 317 318 /** 319 * @brief Current Sequence scrubbing state set by Cache() 320 */ 321 bool scrubbing_; 322 323 /** 324 * @brief Current Sequence playback speed set by Cache() 325 */ 326 int playback_speed_; 327 328 /** 329 * @brief Current nested Sequence hierarchy set by Cache() 330 */ 331 QVector<Clip*> nests_; 332 333 /** 334 * @brief Signal cache to continue operation after one cycle rather than wait for another signal 335 * 336 * Each cycle of the cacher thread (see run()) will set this to false in the beginning. Each call of Cache() will set 337 * this to **TRUE**. If this variable is **TRUE**, the cacher won't wait for another signal before starting the next 338 * cache cycle, and will instead just start it. 339 * 340 * Used if Cache() is called and interrupts the cacher while it's already running so that the cacher will restart 341 * itself automatically rather than wait for the next cache signal. 342 */ 343 bool queued_; 344 345 /** 346 * @brief Interrupt the current cache cycle 347 * 348 * A cache cycle will cache several frames at a time. Since decoding can be strenuous and time consuming, the 349 * cycle can be interrupted if it needs to abruptly start caching somewhere else. Best used in tandem with 350 * queued_ to automatically start the next cache cycle. 351 */ 352 bool interrupt_; 353 354 // ffmpeg media handling 355 /** 356 * @brief FFmpeg format/file context - used for media decoding 357 */ 358 AVFormatContext* formatCtx; 359 360 /** 361 * @brief FFmpeg decoder context - used for media decoding 362 */ 363 AVCodecContext* codecCtx; 364 365 /** 366 * @brief FFmpeg stream - used for media decoding 367 */ 368 AVStream* stream; 369 370 /** 371 * @brief FFmpeg packet - used for media decoding 372 */ 373 AVPacket* pkt; 374 375 /** 376 * @brief FFmpeg frame - used for media decoding 377 * 378 * This is usually used as a raw decoded frame before the RGBA conversion/AVFilter stack. Converted/filtered frames go 379 * into Cacher::queue. 380 */ 381 AVFrame* frame_; 382 383 /** 384 * @brief Retrieved frame reference for Retrieve() 385 * 386 * If a frame was found by either Cache() or CacheVideoWorker(), it's set here. If no frame is ready yet, this is set 387 * to `nullptr`. 388 */ 389 AVFrame* retrieved_frame = nullptr; 390 391 // converters/filters 392 /** 393 * @brief FFmpeg filter stack 394 * 395 * Used for conversion from the media's pixel format to RGBA for OpenGL. Also any other FFmpeg filters are implemented 396 * here if necessary (e.g. yadif for deinterlacing). GLSL effects are preferred when available since FFmpeg filters 397 * aren't always fast enough for realtime playback. 398 */ 399 AVFilterGraph* filter_graph; 400 401 /** 402 * @brief FFmpeg buffer source 403 * 404 * Raw decoded frames are added to this for conversion/filtering 405 */ 406 AVFilterContext* buffersrc_ctx; 407 408 /** 409 * @brief FFmpeg buffer sink 410 * 411 * Converted/filtered frames are retrieved from here and sent to Cacher::queue. 412 */ 413 AVFilterContext* buffersink_ctx; 414 415 /** 416 * @brief FFmpeg codec reference 417 */ 418 AVCodec* codec; 419 420 /** 421 * @brief Options set by the cacher for FFmpeg's decoders (settings like multithreading or other optimizations) 422 */ 423 AVDictionary* opts; 424 425 // audio playback variables 426 /** 427 * @brief Internal audio reset variable 428 * 429 * Set by AudioReset() and read by CacheAudioWorker() when the audio state needs to be interrupted and reset. 430 */ 431 bool audio_reset_; 432 433 /** 434 * @brief Internal reverse target variable 435 * 436 * Used by CacheAudioWorker() to stitch audio frames together when reversing. Stores the current frame's timestamp 437 * so it knows how much to decode up to when it backtracks and decodes the next samples. 438 */ 439 int64_t reverse_target_; 440 441 /** 442 * @brief Internal frame sample index variable 443 * 444 * Used by CacheAudioWorker() to mark which part of the audio frame to read from 445 */ 446 int frame_sample_index_; 447 448 /** 449 * @brief Internal audio buffer write variable 450 * 451 * Used by CacheAudioWorker() to mark which part of the audio buffer to write to 452 */ 453 qint64 audio_buffer_write; 454 455 /** 456 * @brief Internal variable that holds the playhead the last time the audio state was reset 457 */ 458 long audio_target_frame; 459 460 /** 461 * @brief Main while loop condition to determine whether thread should continue looping 462 * 463 * Open() sets this to **TRUE**, Close() sets this to **FALSE**. If it's false, the main loop in run() will exit and 464 * the thread will exit cleanly. It's not recommended to set this variable directly, use Open() and Close() instead. 465 */ 466 bool caching_; 467 468 /** 469 * @brief Internal variable for whether the current Cacher state is valid or not 470 * 471 * If there was an error opening the Cacher for any reason, this will be false. 472 */ 473 bool is_valid_state_; 474 475 /** 476 * @brief Internal function for opening the file handles and decoder 477 * 478 * After the thread has started, it'll call this function to start all resources necessary for caching. Any 479 * FFmpeg decoding variables and filters are set up here. 480 * 481 * This is 482 * fundamentally different from Open(), this is only meant to be called within the cacher thread and never from 483 * outside and doesn't start the thread like Open() does. 484 */ 485 void OpenWorker(); 486 487 /** 488 * @brief Internal function for starting a cache cycle 489 * 490 * This used to have more function, but now just differentiates between CacheVideoWorker() for video clips and 491 * CacheAudioWorker() for audio clips. 492 */ 493 void CacheWorker(); 494 495 /** 496 * @brief Internal function for closing cacher 497 * 498 * Called if the main thread loop in run() exits by setting caching_ to **FALSE**. Free's up handles and memory 499 * allocated by OpenWorker(). 500 */ 501 void CloseWorker(); 502 503 /** 504 * @brief Internal function for resetting audio state 505 * 506 * This used to be a common function, but is now simply a legacy function for CacheAudioWorker(). Resets and 507 * flushes decoders and seeks to the correct timestamp. 508 */ 509 void Reset(); 510 511 /** 512 * @brief Internal function for setting retrieved_frame and waking up any threads waiting for it 513 * 514 * @param f 515 * 516 * The frame to set as the retrieved frame. 517 */ 518 void SetRetrievedFrame(AVFrame* f); 519 520 /** 521 * @brief Internal function to wake an external calling thread 522 * 523 * In some situations, Cache() may wait for the cacher to respond before returning. This is to assist in thread 524 * synchronization, making sure the cacher has started working and has locked any resources it needs before any 525 * other threads can access them (e.g. with a function like Retrieve() ). This must be called at the start of 526 * any CacheVideoWorker() or CacheAudioWorker() control paths to ensure the render thread doesn't get stuck. 527 */ 528 void WakeMainThread(); 529 530 /** 531 * @brief Retrieve frame from decoder 532 * 533 * Retrieves the next decoded frame from the decoder. Depending on the source media, this frame may or may not be 534 * suitable for usage later in the pipeline as it may or may not be the correct pixel/sample format. For a suitable 535 * frame for the pipeline, use RetrieveFrameAndProcess() instead (which in turn uses this function anyway). 536 * 537 * @param f 538 * 539 * Frame buffer to decode frame into 540 * 541 * @return 542 * 543 * FFmpeg error code (>= 0 on success, a negative error code on failure) 544 */ 545 int RetrieveFrameFromDecoder(AVFrame* f); 546 547 /** 548 * @brief Retrieve frame from decoder and run it through filter stack 549 * 550 * Retrieves the next decoded frame and runs it through the AVFilter stack to create an RGBA frame compatible with 551 * the rest of the pipeline and OpenGL. Use this function if you need a ready-made frame. 552 * 553 * @param f 554 * 555 * A pointer to an AVFrame object. It does not need to be allocated, as this function allocates an AVFrame itself. 556 * You'll also need to free it later with av_frame_free() (though ClipQueue will do this automatically if the frame is 557 * added to it). 558 * 559 * @return 560 * 561 * FFmpeg error code (>= 0 on success, a negative error code on failure) 562 */ 563 int RetrieveFrameAndProcess(AVFrame **f); 564 565 /** 566 * @brief Internal video caching function 567 * 568 * Performs one video cache cycle. Seeks the media and cleans old frames from the queue if necessary. Decodes frames 569 * and adds them to the queue (after calculating whether they're necessary). 570 */ 571 void CacheVideoWorker(); 572 573 /** 574 * @brief Internal audio caching function 575 * 576 * Perform one audio cache cycle. Retrieves audio from decoder, reverses and changes speed if necessary, and sends 577 * audio to the audio buffer which will later be sent to the audio output device. 578 */ 579 void CacheAudioWorker(); 580 581 /** 582 * @brief Internal function using the Cacher's known information to determine whether this media is playing in reverse 583 */ 584 bool IsReversed(); 585 }; 586 587 #endif // CACHER_H 588