1 /** 2 * @file 3 * @brief Header file for FFmpegReader class 4 * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard 5 * 6 * @ref License 7 */ 8 9 /* LICENSE 10 * 11 * Copyright (c) 2008-2019 OpenShot Studios, LLC, Fabrice Bellard 12 * (http://www.openshotstudios.com). This file is part of 13 * OpenShot Library (http://www.openshot.org), an open-source project 14 * dedicated to delivering high quality video editing and animation solutions 15 * to the world. 16 * 17 * This file is originally based on the Libavformat API example, and then modified 18 * by the libopenshot project. 19 * 20 * OpenShot Library (libopenshot) is free software: you can redistribute it 21 * and/or modify it under the terms of the GNU Lesser General Public License 22 * as published by the Free Software Foundation, either version 3 of the 23 * License, or (at your option) any later version. 24 * 25 * OpenShot Library (libopenshot) is distributed in the hope that it will be 26 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of 27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 28 * GNU Lesser General Public License for more details. 29 * 30 * You should have received a copy of the GNU Lesser General Public License 31 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>. 32 */ 33 34 #ifndef OPENSHOT_FFMPEG_READER_H 35 #define OPENSHOT_FFMPEG_READER_H 36 37 #include "ReaderBase.h" 38 39 // Include FFmpeg headers and macros 40 #include "FFmpegUtilities.h" 41 42 #include <cmath> 43 #include <ctime> 44 #include <iostream> 45 #include <stdio.h> 46 #include <memory> 47 #include "CacheMemory.h" 48 #include "Clip.h" 49 #include "OpenMPUtilities.h" 50 #include "Settings.h" 51 52 53 namespace openshot { 54 /** 55 * @brief This struct holds the associated video frame and starting sample # for an audio packet. 56 * 57 * Because audio packets do not match up with video frames, this helps determine exactly 58 * where the audio packet's samples belong. 59 */ 60 struct AudioLocation { 61 int64_t frame; 62 int sample_start; 63 64 bool is_near(AudioLocation location, int samples_per_frame, int64_t amount); 65 }; 66 67 /** 68 * @brief This class uses the FFmpeg libraries, to open video files and audio files, and return 69 * openshot::Frame objects for any frame in the file. 70 * 71 * All seeking and caching is handled internally, and the primary public interface is the GetFrame() 72 * method. To use this reader, simply create an instance of this class, and call the GetFrame method 73 * to start retrieving frames. Use the <b>info</b> struct to obtain information on the file, such as the length 74 * (# of frames), height, width, bit rate, frames per second (fps), etc... 75 * 76 * @code 77 * // Create a reader for a video 78 * openshot::FFmpegReader r("MyAwesomeVideo.webm"); 79 * r.Open(); // Open the reader 80 * 81 * // Get frame number 1 from the video 82 * std::shared_ptr<openshot::Frame> f = r.GetFrame(1); 83 * 84 * // Now that we have an openshot::Frame object, lets have some fun! 85 * f->Display(); // Display the frame on the screen 86 * f->DisplayWaveform(); // Display the audio waveform as an image 87 * f->Play(); // Play the audio through your speaker 88 * 89 * // Close the reader 90 * r.Close(); 91 * @endcode 92 */ 93 class FFmpegReader : public ReaderBase { 94 private: 95 std::string path; 96 97 AVFormatContext *pFormatCtx; 98 int i, videoStream, audioStream; 99 AVCodecContext *pCodecCtx, *aCodecCtx; 100 #if USE_HW_ACCEL 101 AVBufferRef *hw_device_ctx = NULL; //PM 102 #endif 103 AVStream *pStream, *aStream; 104 AVPacket *packet; 105 AVFrame *pFrame; 106 bool is_open; 107 bool is_duration_known; 108 bool check_interlace; 109 bool check_fps; 110 bool has_missing_frames; 111 int max_concurrent_frames; 112 113 CacheMemory working_cache; 114 CacheMemory missing_frames; 115 std::map<int64_t, int64_t> processing_video_frames; 116 std::multimap<int64_t, int64_t> processing_audio_frames; 117 std::map<int64_t, int64_t> processed_video_frames; 118 std::map<int64_t, int64_t> processed_audio_frames; 119 std::multimap<int64_t, int64_t> missing_video_frames; 120 std::multimap<int64_t, int64_t> missing_video_frames_source; 121 std::multimap<int64_t, int64_t> missing_audio_frames; 122 std::multimap<int64_t, int64_t> missing_audio_frames_source; 123 std::map<int64_t, int> checked_frames; 124 AudioLocation previous_packet_location; 125 126 // DEBUG VARIABLES (FOR AUDIO ISSUES) 127 int prev_samples; 128 int64_t prev_pts; 129 int64_t pts_total; 130 int64_t pts_counter; 131 int64_t num_packets_since_video_frame; 132 int64_t num_checks_since_final; 133 std::shared_ptr<openshot::Frame> last_video_frame; 134 135 bool is_seeking; 136 int64_t seeking_pts; 137 int64_t seeking_frame; 138 bool is_video_seek; 139 int seek_count; 140 int64_t seek_audio_frame_found; 141 int64_t seek_video_frame_found; 142 143 int64_t audio_pts_offset; 144 int64_t video_pts_offset; 145 int64_t last_frame; 146 int64_t largest_frame_processed; 147 int64_t current_video_frame; // can't reliably use PTS of video to determine this 148 149 int hw_de_supported = 0; // Is set by FFmpegReader 150 #if USE_HW_ACCEL 151 AVPixelFormat hw_de_av_pix_fmt = AV_PIX_FMT_NONE; 152 AVHWDeviceType hw_de_av_device_type = AV_HWDEVICE_TYPE_NONE; 153 int IsHardwareDecodeSupported(int codecid); 154 #endif 155 156 /// Check for the correct frames per second value by scanning the 1st few seconds of video packets. 157 void CheckFPS(); 158 159 /// Check the current seek position and determine if we need to seek again 160 bool CheckSeek(bool is_video); 161 162 /// Check if a frame is missing and attempt to replace its frame image (and 163 bool CheckMissingFrame(int64_t requested_frame); 164 165 /// Check the working queue, and move finished frames to the finished queue 166 void CheckWorkingFrames(bool end_of_stream, int64_t requested_frame); 167 168 /// Convert Frame Number into Audio PTS 169 int64_t ConvertFrameToAudioPTS(int64_t frame_number); 170 171 /// Convert Frame Number into Video PTS 172 int64_t ConvertFrameToVideoPTS(int64_t frame_number); 173 174 /// Convert Video PTS into Frame Number 175 int64_t ConvertVideoPTStoFrame(int64_t pts); 176 177 /// Create a new Frame (or return an existing one) and add it to the working queue. 178 std::shared_ptr<openshot::Frame> CreateFrame(int64_t requested_frame); 179 180 /// Calculate Starting video frame and sample # for an audio PTS 181 AudioLocation GetAudioPTSLocation(int64_t pts); 182 183 /// Get an AVFrame (if any) 184 bool GetAVFrame(); 185 186 /// Get the next packet (if any) 187 int GetNextPacket(); 188 189 /// Get the smallest video frame that is still being processed 190 int64_t GetSmallestVideoFrame(); 191 192 /// Get the smallest audio frame that is still being processed 193 int64_t GetSmallestAudioFrame(); 194 195 /// Get the PTS for the current video packet 196 int64_t GetVideoPTS(); 197 198 /// Check if there's an album art 199 bool HasAlbumArt(); 200 201 /// Remove partial frames due to seek 202 bool IsPartialFrame(int64_t requested_frame); 203 204 /// Process a video packet 205 void ProcessVideoPacket(int64_t requested_frame); 206 207 /// Process an audio packet 208 void ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample); 209 210 /// Read the stream until we find the requested Frame 211 std::shared_ptr<openshot::Frame> ReadStream(int64_t requested_frame); 212 213 /// Remove AVFrame from cache (and deallocate its memory) 214 void RemoveAVFrame(AVFrame *); 215 216 /// Remove AVPacket from cache (and deallocate its memory) 217 void RemoveAVPacket(AVPacket *); 218 219 /// Seek to a specific Frame. This is not always frame accurate, it's more of an estimation on many codecs. 220 void Seek(int64_t requested_frame); 221 222 /// Update PTS Offset (if any) 223 void UpdatePTSOffset(bool is_video); 224 225 /// Update File Info for audio streams 226 void UpdateAudioInfo(); 227 228 /// Update File Info for video streams 229 void UpdateVideoInfo(); 230 231 public: 232 /// Final cache object used to hold final frames 233 CacheMemory final_cache; 234 235 /// Enable or disable seeking. Seeking can more quickly locate the requested frame, but some 236 /// codecs have trouble seeking, and can introduce artifacts or blank images into the video. 237 bool enable_seek; 238 239 /// @brief Constructor for FFmpegReader. 240 /// 241 /// Sets (and possibly opens) the media file path, 242 /// or throws an exception. 243 /// @param path The filesystem location to load 244 /// @param inspect_reader if true (the default), automatically open the media file and loads frame 1. 245 FFmpegReader(const std::string& path, bool inspect_reader=true); 246 247 /// Destructor 248 virtual ~FFmpegReader(); 249 250 /// Close File 251 void Close() override; 252 253 /// Get the cache object used by this reader GetCache()254 CacheMemory *GetCache() override { return &final_cache; }; 255 256 /// Get a shared pointer to a openshot::Frame object for a specific frame number of this reader. 257 /// 258 /// @returns The requested frame of video 259 /// @param requested_frame The frame number that is requested. 260 std::shared_ptr<openshot::Frame> GetFrame(int64_t requested_frame) override; 261 262 /// Determine if reader is open or closed IsOpen()263 bool IsOpen() override { return is_open; }; 264 265 /// Return the type name of the class Name()266 std::string Name() override { return "FFmpegReader"; }; 267 268 // Get and Set JSON methods 269 std::string Json() const override; ///< Generate JSON string of this object 270 void SetJson(const std::string value) override; ///< Load JSON string into this object 271 Json::Value JsonValue() const override; ///< Generate Json::Value for this object 272 void SetJsonValue(const Json::Value root) override; ///< Load Json::Value into this object 273 274 /// Open File - which is called by the constructor automatically 275 void Open() override; 276 277 /// Return true if frame can be read with GetFrame() 278 bool GetIsDurationKnown(); 279 }; 280 281 } 282 283 #endif 284