1 /* 2 Copyright (c) 2012 Stephen Baker 3 4 Permission is hereby granted, free of charge, to any person obtaining a copy of 5 this software and associated documentation files (the "Software"), to deal in 6 the Software without restriction, including without limitation the rights to 7 use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 8 of the Software, and to permit persons to whom the Software is furnished to do 9 so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be included in all 12 copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 SOFTWARE. 21 */ 22 23 #ifndef TH_VIDEO_H 24 #define TH_VIDEO_H 25 26 #include "config.h" 27 28 #include <atomic> 29 #include <condition_variable> 30 #include <mutex> 31 #include <queue> 32 #include <string> 33 #include <thread> 34 35 #include "SDL.h" 36 37 #if (defined(CORSIX_TH_USE_FFMPEG) || defined(CORSIX_TH_USE_LIBAV)) && \ 38 defined(CORSIX_TH_USE_SDL_MIXER) 39 #include "SDL_mixer.h" 40 41 extern "C" { 42 #ifndef INT64_C 43 #define INT64_C(c) (c##LL) 44 #define UINT64_C(c) (c##ULL) 45 #endif 46 #include <libavformat/avformat.h> 47 #include <libavutil/avutil.h> 48 #include <libswscale/swscale.h> 49 #ifdef CORSIX_TH_USE_FFMPEG 50 #include <libswresample/swresample.h> 51 #elif defined(CORSIX_TH_USE_LIBAV) 52 #include <libavresample/avresample.h> 53 #endif 54 } 55 56 #if (defined(CORSIX_TH_USE_FFMEPG) && \ 57 LIBAVUTIL_VERSION_INT < AV_VERSION_INT(51, 74, 100)) || \ 58 (defined(CORSIX_TH_USE_LIBAV) && \ 59 LIBAVUTIL_VERSION_INT < AV_VERSION_INT(51, 42, 0)) 60 #define AVPixelFormat PixelFormat 61 #define AV_PIX_FMT_RBG24 PIX_FMT_RGB24 62 #endif 63 64 #if (defined(CORSIX_TH_USE_LIBAV) && \ 65 LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 16, 0)) || \ 66 (defined(CORSIX_TH_USE_FFMPEG) && \ 67 LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 37, 100)) 68 #define CORSIX_TH_MOVIE_USE_SEND_PACKET_API 69 #endif 70 71 //! \brief Drop in replacement for AVPacketList 72 //! 73 //! AVPacketList which was deprecated with FFMpeg 4.4. 74 struct th_packet_list { 75 AVPacket pkt; 76 th_packet_list* next; 77 }; 78 79 //! \brief A picture in movie_picture_buffer 80 //! 81 //! Stores the picture from a frame in the movie from the time that it is 82 //! processed until it should be drawn. 83 class movie_picture { 84 public: 85 movie_picture(); 86 ~movie_picture(); 87 88 //! Allocate the buffer to hold a picture of the given size 89 void allocate(int iWidth, int iHeight); 90 91 //! Delete the buffer 92 void deallocate(); 93 94 uint8_t* buffer; ///< Pixel data in #m_pixelFormat 95 const AVPixelFormat pixel_format; ///< The format of pixels to output 96 int width; ///< Picture width 97 int height; ///< Picture height 98 double pts; ///< Presentation time stamp 99 std::mutex mutex; ///< Mutex protecting this picture 100 }; 101 102 //! A buffer for holding movie pictures and drawing them to the renderer 103 class movie_picture_buffer { 104 public: 105 movie_picture_buffer(); 106 ~movie_picture_buffer(); 107 108 // NB: The following functions are called by the main program thread 109 110 //! Indicate that processing should stop and the movie aborted 111 void abort(); 112 113 //! Resume after having aborted 114 void reset(); 115 116 //! Ready the picture buffer for a new renderer or new picture dimensions 117 //! by allocating each movie_picture in the queue, resetting the read 118 //! index and allocating a new texture. 119 //! 120 //! \remark Must be run on the program's graphics thread 121 void allocate(SDL_Renderer* pRenderer, int iWidth, int iHeight); 122 123 //! Destroy the associated texture and deallocate each of the 124 //! movie_pictures in the queue so that the program can release 125 //! the renderer 126 //! 127 //! \remark Must be run on the program's graphics thread 128 void deallocate(); 129 130 //! Advance the read index 131 bool advance(); 132 133 //! Draw the movie_picture at the current read index 134 //! 135 //! \param pRenderer The renderer to draw the picture to 136 //! \param dstrect The rectangle on the renderer to draw to 137 //! 138 //! \remark Must be run on the program's graphics thread 139 void draw(SDL_Renderer* pRenderer, const SDL_Rect& dstrect); 140 141 //! Get the next presentation time stamp 142 double get_next_pts(); 143 144 //! Return whether there are any pictures left to draw in the picture queue 145 //! 146 //! \remark If the movie_picture_buffer is not allocated it cannot be read 147 //! from or written to. Consequently it is both full and empty. 148 bool empty(); 149 150 // NB: These functions are called by a second thread 151 152 //! Return whether there is space to add any more frame data to the queue 153 //! 154 //! \remark If the movie_picture_buffer is not allocated it cannot be read 155 //! from or written to. Consequently it is both full and empty. 156 bool full(); 157 158 //! Write the given frame (and presentation time stamp) to the picture 159 //! queue 160 //! 161 //! \retval 0 Success 162 //! \retval -1 Abort is in progress 163 //! \retval 1 An error writing the frame 164 int write(AVFrame* pFrame, double dPts); 165 166 private: 167 //! Return whether there is space to add any more frame data to the queue 168 //! 169 //! \remark Requires external locking 170 bool unsafe_full(); 171 172 static constexpr size_t picture_buffer_size = 173 4; ///< The number of elements to allocate in the picture queue 174 std::atomic<bool> aborting; ///< Whether we are in the process of aborting 175 bool allocated; ///< Whether the picture buffer has been allocated (and 176 ///< hasn't since been deallocated) 177 int picture_count; ///< The number of elements currently written to the 178 ///< picture queue 179 int read_index; ///< The position in the picture queue to be read next 180 int write_index; ///< The position in the picture queue to be written to 181 ///< next 182 SwsContext* sws_context; ///< The context for software scaling and pixel 183 ///< conversion when writing to the picture queue 184 SDL_Texture* texture; ///< The (potentially hardware) texture to draw the 185 ///< picture to. In OpenGL this should only be 186 ///< accessed on the main thread 187 std::mutex mutex; ///< A mutex for restricting access to the picture buffer 188 ///< to a single thread 189 std::condition_variable 190 cond; ///< A condition for indicating access to the picture buffer 191 movie_picture picture_queue[picture_buffer_size]; ///< The picture queue, a 192 ///< looping FIFO queue 193 ///< of movie_pictures 194 }; 195 196 //! The AVPacketQueue is a thread safe queue of movie packets 197 class av_packet_queue { 198 public: 199 //! Construct a new empty packet queue 200 av_packet_queue(); 201 202 //! Destroy the packet queue. 203 //! 204 //! \remarks Does not free the included packets. The packet queue should be 205 //! flushed before it is destroyed. 206 ~av_packet_queue() = default; 207 208 //! Push a new packet on the back of the queue 209 void push(AVPacket* packet); 210 211 //! Pull the packet from the front of the queue 212 //! 213 //! \param block Whether to block if the queue is empty or immediately 214 //! return a nullptr 215 AVPacket* pull(bool block); 216 217 //! Return the number of packets in the queue 218 int get_count() const; 219 220 //! Release a blocking pull without writing a new packet to the queue. 221 void release(); 222 223 private: 224 th_packet_list* first_packet; ///< The packet at the front of the queue 225 th_packet_list* last_packet; ///< The packet at the end of the queue 226 int count; ///< The number of packets in the queue 227 std::mutex mutex; ///< A mutex restricting access to the packet queue to a 228 ///< single thread 229 std::condition_variable 230 cond; ///< A condition to wait on for signaling the packet queue 231 }; 232 #endif // CORSIX_TH_USE_FFMPEG || CORSIX_TH_USE_LIBAV 233 234 //! Movie player for CorsixTH 235 //! 236 //! The movie player is designed to be preinitialized and used for multiple 237 //! movies. After initializing the movie player, call movie_player::set_renderer 238 //! to assign the current SDL renderer to the movie player. Then 239 //! movie_player::load the desired movie and finally movie_player::play it. 240 class movie_player { 241 public: 242 //! Construct a new movie_player 243 movie_player(); 244 245 //! Destroy the movie_player 246 ~movie_player(); 247 248 //! Assign the renderer on which to draw the movie 249 void set_renderer(SDL_Renderer* pRenderer); 250 251 //! Return whether movies were compiled into CorsixTH 252 bool movies_enabled() const; 253 254 //! Load the movie with the given file name 255 bool load(const char* szFilepath); 256 257 //! Unload and free the currently loaded movie. 258 //! 259 //! \remark This is called by load before loading a new movie so it is 260 //! unnecessary to explicitly call this method. There is no harm either. 261 void unload(); 262 263 //! Play the currently loaded movie 264 //! 265 //! \param iChannel The audio channel to use 266 void play(int iChannel); 267 268 //! Stop the currently playing movie 269 void stop(); 270 271 //! Return the original height of the movie 272 int get_native_height() const; 273 274 //! Return the original width of the movie 275 int get_native_width() const; 276 277 //! Return whether the movie has an audio stream 278 bool has_audio_track() const; 279 280 //! Return a text description of the last error encountered 281 const char* get_last_error() const; 282 283 //! Clear the last error so that if there is no more errors before the next 284 //! call to movie_player::get_last_error() it will return an empty string. 285 void clear_last_error(); 286 287 //! Draw the next frame if it is time to do so 288 //! 289 //! \param destination_rect The location and dimensions in the renderer on 290 //! which to draw the movie 291 void refresh(const SDL_Rect& destination_rect); 292 293 //! Deallocate the picture buffer and free any resources associated with it. 294 //! 295 //! \remark This destroys the textures and other resources that may lock 296 //! the renderer from being deleted. If the target changes you would call 297 //! this, then free and switch renderers in the outside program, then call 298 //! movie_player::set_renderer and finally 299 //! movie_player::allocate_picture_buffer. \remark Up to the size of the 300 //! picture buffer frames may be lost during this process. 301 void deallocate_picture_buffer(); 302 303 //! Allocate the picture buffer for the current renderer 304 void allocate_picture_buffer(); 305 306 //! Read packets from the movie and allocate them to the appropriate stream 307 //! packet queues. Signal if we have reached the end of the movie. 308 //! 309 //! \remark This should not be called externally. It is public as it is the 310 //! entry point of a thread. 311 void read_streams(); 312 313 //! Read video frames from the video packet queue and write them to the 314 //! picture queue. 315 //! 316 //! \remark This should not be called externally. It is public as it is the 317 //! entry point of a thread. 318 void run_video(); 319 320 //! Read audio from the audio packet queue, and copy it into the audio 321 //! buffer for playback 322 void copy_audio_to_stream(uint8_t* pbStream, int iStreamSize); 323 324 private: 325 #if (defined(CORSIX_TH_USE_FFMPEG) || defined(CORSIX_TH_USE_LIBAV)) && \ 326 defined(CORSIX_TH_USE_SDL_MIXER) 327 static constexpr size_t movie_error_buffer_capacity = 328 128; ///< Buffer to hold last error description 329 static constexpr size_t audio_chunk_buffer_capacity = 330 1024; ///< Buffer for audio playback 331 332 //! Get the AVCodecContext associated with a given stream 333 AVCodecContext* get_codec_context_for_stream(AVCodec* codec, 334 AVStream* stream) const; 335 336 //! Get the time the given frame should be played (from the start of the 337 //! stream) 338 //! 339 //! \param frame The video or audio frame 340 //! \param streamIndex The position of the stream in m_pFormatContexts 341 //! streams array 342 double get_presentation_time_for_frame(AVFrame* frame, int streamIndex) const; 343 344 //! Decode audio from the movie into a format suitable for playback 345 int decode_audio_frame(bool fFirst); 346 347 #ifdef CORSIX_TH_MOVIE_USE_SEND_PACKET_API 348 //! Convert packet data into frames 349 //! 350 //! \param stream The index of the stream to get the frame for 351 //! \param pFrame An empty frame which gets populated by the data in the 352 //! packet queue. 353 //! \returns FFMPEG result of avcodec_recieve_frame 354 int get_frame(int stream, AVFrame* pFrame); 355 #else 356 //! Convert video packet data into a frame. 357 //! 358 //! \param pFrame An empty frame which gets populated by the data in the 359 //! video packet queue. 360 //! \returns 1 if the frame was received, 0 if it was not, and < 0 on error 361 int get_video_frame(AVFrame* pFrame); 362 #endif 363 364 SDL_Renderer* renderer; ///< The renderer to draw to 365 366 //! A description of the last error 367 std::string last_error; 368 369 //! A buffer for passing to ffmpeg to get error details 370 char error_buffer[movie_error_buffer_capacity]; 371 372 // TODO: Should be atomic 373 bool aborting; ///< Indicate that we are in process of aborting playback 374 375 std::mutex decoding_audio_mutex; ///< Synchronize access to #m_pAudioBuffer 376 377 AVFormatContext* format_context; ///< Information related to the loaded 378 ///< movie and all of its streams 379 int video_stream_index; ///< The index of the video stream 380 int audio_stream_index; ///< The index of the audio stream 381 AVCodecContext* video_codec_context; ///< The video codec and information 382 ///< related to video 383 AVCodecContext* audio_codec_context; ///< The audio codec and information 384 ///< related to audio 385 386 // queues for transferring data between threads 387 av_packet_queue* video_queue; ///< Packets from the video stream 388 av_packet_queue* audio_queue; ///< Packets from the audio stream 389 ::movie_picture_buffer* movie_picture_buffer; ///< Buffer of processed video 390 391 // clock sync parameters 392 int current_sync_pts_system_time; ///< System time matching #m_iCurSyncPts 393 double current_sync_pts; ///< The current presentation time stamp (from the 394 ///< audio stream) 395 396 #ifdef CORSIX_TH_USE_FFMPEG 397 SwrContext* audio_resample_context; ///< Context for resampling audio for 398 ///< playback with ffmpeg 399 #elif defined(CORSIX_TH_USE_LIBAV) 400 AVAudioResampleContext* 401 audio_resample_context; ///< Context for resampling audio for 402 ///< playback with libav 403 #endif 404 405 int audio_buffer_size; ///< The current size of audio data in 406 ///< #m_pbAudioBuffer 407 int audio_buffer_index; ///< The current position for writing in 408 ///< #m_pbAudioBuffer 409 int audio_buffer_max_size; ///< The capacity of #m_pbAudioBuffer (allocated 410 ///< size) 411 uint8_t* audio_buffer; ///< An audio buffer for playback 412 413 AVPacket* audio_packet; ///< The current audio packet being decoded (audio 414 ///< frames don't necessarily line up with packets) 415 int audio_packet_size; ///< The size of #m_pbAudioPacketData 416 uint8_t* audio_packet_data; ///< Original data for #m_pAudioPacket, kept so 417 ///< that it can be freed after the packet is 418 ///< processed 419 AVFrame* audio_frame; ///< The frame we are decoding audio into 420 421 Mix_Chunk* empty_audio_chunk; ///< Empty chunk needed for SDL_mixer 422 uint8_t* audio_chunk_buffer; ///< 0'd out buffer for the SDL_Mixer chunk 423 424 int audio_channel; ///< The channel to play audio on, -1 for none 425 int mixer_channels; ///< How many channels to play on (1 - mono, 2 - 426 ///< stereo) 427 int mixer_frequency; ///< The frequency of audio expected by SDL_Mixer 428 429 #ifndef CORSIX_TH_MOVIE_USE_SEND_PACKET_API 430 AVPacket* flush_packet; ///< A representative packet indicating a flush is 431 ///< required. 432 #endif 433 434 std::thread stream_thread; ///< The thread responsible for reading the 435 ///< movie streams 436 std::thread video_thread; ///< The thread responsible for decoding the 437 ///< video stream 438 #endif // CORSIX_TH_USE_FFMPEG || CORSIX_TH_USE_LIBAV 439 }; 440 441 #endif // TH_VIDEO_H 442