1 /*
2 This file is part of Telegram Desktop,
3 the official desktop application for the Telegram messaging service.
4
5 For license and copyright information please follow this link:
6 https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
7 */
8 #include "ffmpeg/ffmpeg_utility.h"
9
10 #include "base/algorithm.h"
11 #include "logs.h"
12
13 #include <QImage>
14
15 #ifdef LIB_FFMPEG_USE_QT_PRIVATE_API
16 #include <private/qdrawhelper_p.h>
17 #endif // LIB_FFMPEG_USE_QT_PRIVATE_API
18
19 extern "C" {
20 #include <libavutil/opt.h>
21 } // extern "C"
22
23 namespace FFmpeg {
24 namespace {
25
26 // See https://github.com/telegramdesktop/tdesktop/issues/7225
27 constexpr auto kAlignImageBy = 64;
28 constexpr auto kImageFormat = QImage::Format_ARGB32_Premultiplied;
29 constexpr auto kMaxScaleByAspectRatio = 16;
30 constexpr auto kAvioBlockSize = 4096;
31 constexpr auto kTimeUnknown = std::numeric_limits<crl::time>::min();
32 constexpr auto kDurationMax = crl::time(std::numeric_limits<int>::max());
33
AlignedImageBufferCleanupHandler(void * data)34 void AlignedImageBufferCleanupHandler(void* data) {
35 const auto buffer = static_cast<uchar*>(data);
36 delete[] buffer;
37 }
38
IsValidAspectRatio(AVRational aspect)39 [[nodiscard]] bool IsValidAspectRatio(AVRational aspect) {
40 return (aspect.num > 0)
41 && (aspect.den > 0)
42 && (aspect.num <= aspect.den * kMaxScaleByAspectRatio)
43 && (aspect.den <= aspect.num * kMaxScaleByAspectRatio);
44 }
45
IsAlignedImage(const QImage & image)46 [[nodiscard]] bool IsAlignedImage(const QImage &image) {
47 return !(reinterpret_cast<uintptr_t>(image.bits()) % kAlignImageBy)
48 && !(image.bytesPerLine() % kAlignImageBy);
49 }
50
UnPremultiplyLine(uchar * dst,const uchar * src,int intsCount)51 void UnPremultiplyLine(uchar *dst, const uchar *src, int intsCount) {
52 [[maybe_unused]] const auto udst = reinterpret_cast<uint*>(dst);
53 const auto usrc = reinterpret_cast<const uint*>(src);
54
55 #ifndef LIB_FFMPEG_USE_QT_PRIVATE_API
56 for (auto i = 0; i != intsCount; ++i) {
57 udst[i] = qUnpremultiply(usrc[i]);
58 }
59 #else // !LIB_FFMPEG_USE_QT_PRIVATE_API
60 static const auto layout = &qPixelLayouts[QImage::Format_ARGB32];
61 layout->storeFromARGB32PM(dst, usrc, 0, intsCount, nullptr, nullptr);
62 #endif // LIB_FFMPEG_USE_QT_PRIVATE_API
63 }
64
PremultiplyLine(uchar * dst,const uchar * src,int intsCount)65 void PremultiplyLine(uchar *dst, const uchar *src, int intsCount) {
66 const auto udst = reinterpret_cast<uint*>(dst);
67 [[maybe_unused]] const auto usrc = reinterpret_cast<const uint*>(src);
68
69 #ifndef LIB_FFMPEG_USE_QT_PRIVATE_API
70 for (auto i = 0; i != intsCount; ++i) {
71 udst[i] = qPremultiply(usrc[i]);
72 }
73 #else // !LIB_FFMPEG_USE_QT_PRIVATE_API
74 static const auto layout = &qPixelLayouts[QImage::Format_ARGB32];
75 layout->fetchToARGB32PM(udst, src, 0, intsCount, nullptr, nullptr);
76 #endif // LIB_FFMPEG_USE_QT_PRIVATE_API
77 }
78
79 } // namespace
80
MakeIOPointer(void * opaque,int (* read)(void * opaque,uint8_t * buffer,int bufferSize),int (* write)(void * opaque,uint8_t * buffer,int bufferSize),int64_t (* seek)(void * opaque,int64_t offset,int whence))81 IOPointer MakeIOPointer(
82 void *opaque,
83 int(*read)(void *opaque, uint8_t *buffer, int bufferSize),
84 int(*write)(void *opaque, uint8_t *buffer, int bufferSize),
85 int64_t(*seek)(void *opaque, int64_t offset, int whence)) {
86 auto buffer = reinterpret_cast<uchar*>(av_malloc(kAvioBlockSize));
87 if (!buffer) {
88 LogError(qstr("av_malloc"));
89 return {};
90 }
91 auto result = IOPointer(avio_alloc_context(
92 buffer,
93 kAvioBlockSize,
94 write ? 1 : 0,
95 opaque,
96 read,
97 write,
98 seek));
99 if (!result) {
100 av_freep(&buffer);
101 LogError(qstr("avio_alloc_context"));
102 return {};
103 }
104 return result;
105 }
106
operator ()(AVIOContext * value)107 void IODeleter::operator()(AVIOContext *value) {
108 if (value) {
109 av_freep(&value->buffer);
110 avio_context_free(&value);
111 }
112 }
113
MakeFormatPointer(void * opaque,int (* read)(void * opaque,uint8_t * buffer,int bufferSize),int (* write)(void * opaque,uint8_t * buffer,int bufferSize),int64_t (* seek)(void * opaque,int64_t offset,int whence))114 FormatPointer MakeFormatPointer(
115 void *opaque,
116 int(*read)(void *opaque, uint8_t *buffer, int bufferSize),
117 int(*write)(void *opaque, uint8_t *buffer, int bufferSize),
118 int64_t(*seek)(void *opaque, int64_t offset, int whence)) {
119 auto io = MakeIOPointer(opaque, read, write, seek);
120 if (!io) {
121 return {};
122 }
123 auto result = avformat_alloc_context();
124 if (!result) {
125 LogError(qstr("avformat_alloc_context"));
126 return {};
127 }
128 result->pb = io.get();
129
130 auto options = (AVDictionary*)nullptr;
131 const auto guard = gsl::finally([&] { av_dict_free(&options); });
132 av_dict_set(&options, "usetoc", "1", 0);
133 const auto error = AvErrorWrap(avformat_open_input(
134 &result,
135 nullptr,
136 nullptr,
137 &options));
138 if (error) {
139 // avformat_open_input freed 'result' in case an error happened.
140 LogError(qstr("avformat_open_input"), error);
141 return {};
142 }
143 result->flags |= AVFMT_FLAG_FAST_SEEK;
144
145 // Now FormatPointer will own and free the IO context.
146 io.release();
147 return FormatPointer(result);
148 }
149
operator ()(AVFormatContext * value)150 void FormatDeleter::operator()(AVFormatContext *value) {
151 if (value) {
152 const auto deleter = IOPointer(value->pb);
153 avformat_close_input(&value);
154 }
155 }
156
MakeCodecPointer(not_null<AVStream * > stream)157 CodecPointer MakeCodecPointer(not_null<AVStream*> stream) {
158 auto error = AvErrorWrap();
159
160 auto result = CodecPointer(avcodec_alloc_context3(nullptr));
161 const auto context = result.get();
162 if (!context) {
163 LogError(qstr("avcodec_alloc_context3"));
164 return {};
165 }
166 error = avcodec_parameters_to_context(context, stream->codecpar);
167 if (error) {
168 LogError(qstr("avcodec_parameters_to_context"), error);
169 return {};
170 }
171 context->pkt_timebase = stream->time_base;
172 av_opt_set(context, "threads", "auto", 0);
173 av_opt_set_int(context, "refcounted_frames", 1, 0);
174
175 const auto codec = avcodec_find_decoder(context->codec_id);
176 if (!codec) {
177 LogError(qstr("avcodec_find_decoder"), context->codec_id);
178 return {};
179 } else if ((error = avcodec_open2(context, codec, nullptr))) {
180 LogError(qstr("avcodec_open2"), error);
181 return {};
182 }
183 return result;
184 }
185
operator ()(AVCodecContext * value)186 void CodecDeleter::operator()(AVCodecContext *value) {
187 if (value) {
188 avcodec_free_context(&value);
189 }
190 }
191
MakeFramePointer()192 FramePointer MakeFramePointer() {
193 return FramePointer(av_frame_alloc());
194 }
195
FrameHasData(AVFrame * frame)196 bool FrameHasData(AVFrame *frame) {
197 return (frame && frame->data[0] != nullptr);
198 }
199
ClearFrameMemory(AVFrame * frame)200 void ClearFrameMemory(AVFrame *frame) {
201 if (FrameHasData(frame)) {
202 av_frame_unref(frame);
203 }
204 }
205
operator ()(AVFrame * value)206 void FrameDeleter::operator()(AVFrame *value) {
207 av_frame_free(&value);
208 }
209
MakeSwscalePointer(QSize srcSize,int srcFormat,QSize dstSize,int dstFormat,SwscalePointer * existing)210 SwscalePointer MakeSwscalePointer(
211 QSize srcSize,
212 int srcFormat,
213 QSize dstSize,
214 int dstFormat,
215 SwscalePointer *existing) {
216 // We have to use custom caching for SwsContext, because
217 // sws_getCachedContext checks passed flags with existing context flags,
218 // and re-creates context if they're different, but in the process of
219 // context creation the passed flags are modified before being written
220 // to the resulting context, so the caching doesn't work.
221 if (existing && (*existing) != nullptr) {
222 const auto &deleter = existing->get_deleter();
223 if (deleter.srcSize == srcSize
224 && deleter.srcFormat == srcFormat
225 && deleter.dstSize == dstSize
226 && deleter.dstFormat == dstFormat) {
227 return std::move(*existing);
228 }
229 }
230 if (srcFormat <= AV_PIX_FMT_NONE || srcFormat >= AV_PIX_FMT_NB) {
231 LogError(qstr("frame->format"));
232 return SwscalePointer();
233 }
234
235 const auto result = sws_getCachedContext(
236 existing ? existing->release() : nullptr,
237 srcSize.width(),
238 srcSize.height(),
239 AVPixelFormat(srcFormat),
240 dstSize.width(),
241 dstSize.height(),
242 AVPixelFormat(dstFormat),
243 0,
244 nullptr,
245 nullptr,
246 nullptr);
247 if (!result) {
248 LogError(qstr("sws_getCachedContext"));
249 }
250 return SwscalePointer(
251 result,
252 { srcSize, srcFormat, dstSize, dstFormat });
253 }
254
MakeSwscalePointer(not_null<AVFrame * > frame,QSize resize,SwscalePointer * existing)255 SwscalePointer MakeSwscalePointer(
256 not_null<AVFrame*> frame,
257 QSize resize,
258 SwscalePointer *existing) {
259 return MakeSwscalePointer(
260 QSize(frame->width, frame->height),
261 frame->format,
262 resize,
263 AV_PIX_FMT_BGRA,
264 existing);
265 }
266
operator ()(SwsContext * value)267 void SwscaleDeleter::operator()(SwsContext *value) {
268 if (value) {
269 sws_freeContext(value);
270 }
271 }
272
LogError(QLatin1String method)273 void LogError(QLatin1String method) {
274 LOG(("Streaming Error: Error in %1.").arg(method));
275 }
276
LogError(QLatin1String method,AvErrorWrap error)277 void LogError(QLatin1String method, AvErrorWrap error) {
278 LOG(("Streaming Error: Error in %1 (code: %2, text: %3)."
279 ).arg(method
280 ).arg(error.code()
281 ).arg(error.text()));
282 }
283
PtsToTime(int64_t pts,AVRational timeBase)284 crl::time PtsToTime(int64_t pts, AVRational timeBase) {
285 return (pts == AV_NOPTS_VALUE || !timeBase.den)
286 ? kTimeUnknown
287 : ((pts * 1000LL * timeBase.num) / timeBase.den);
288 }
289
PtsToTimeCeil(int64_t pts,AVRational timeBase)290 crl::time PtsToTimeCeil(int64_t pts, AVRational timeBase) {
291 return (pts == AV_NOPTS_VALUE || !timeBase.den)
292 ? kTimeUnknown
293 : ((pts * 1000LL * timeBase.num + timeBase.den - 1) / timeBase.den);
294 }
295
TimeToPts(crl::time time,AVRational timeBase)296 int64_t TimeToPts(crl::time time, AVRational timeBase) {
297 return (time == kTimeUnknown || !timeBase.num)
298 ? AV_NOPTS_VALUE
299 : (time * timeBase.den) / (1000LL * timeBase.num);
300 }
301
PacketPosition(const Packet & packet,AVRational timeBase)302 crl::time PacketPosition(const Packet &packet, AVRational timeBase) {
303 const auto &native = packet.fields();
304 return PtsToTime(
305 (native.pts == AV_NOPTS_VALUE) ? native.dts : native.pts,
306 timeBase);
307 }
308
PacketDuration(const Packet & packet,AVRational timeBase)309 crl::time PacketDuration(const Packet &packet, AVRational timeBase) {
310 return PtsToTime(packet.fields().duration, timeBase);
311 }
312
DurationByPacket(const Packet & packet,AVRational timeBase)313 int DurationByPacket(const Packet &packet, AVRational timeBase) {
314 const auto position = PacketPosition(packet, timeBase);
315 const auto duration = std::max(
316 PacketDuration(packet, timeBase),
317 crl::time(1));
318 const auto bad = [](crl::time time) {
319 return (time < 0) || (time > kDurationMax);
320 };
321 if (bad(position) || bad(duration) || bad(position + duration + 1)) {
322 LOG(("Streaming Error: Wrong duration by packet: %1 + %2"
323 ).arg(position
324 ).arg(duration));
325 return -1;
326 }
327 return int(position + duration + 1);
328 }
329
ReadRotationFromMetadata(not_null<AVStream * > stream)330 int ReadRotationFromMetadata(not_null<AVStream*> stream) {
331 const auto tag = av_dict_get(stream->metadata, "rotate", nullptr, 0);
332 if (tag && *tag->value) {
333 const auto string = QString::fromUtf8(tag->value);
334 auto ok = false;
335 const auto degrees = string.toInt(&ok);
336 if (ok && (degrees == 90 || degrees == 180 || degrees == 270)) {
337 return degrees;
338 }
339 }
340 return 0;
341 }
342
ValidateAspectRatio(AVRational aspect)343 AVRational ValidateAspectRatio(AVRational aspect) {
344 return IsValidAspectRatio(aspect) ? aspect : kNormalAspect;
345 }
346
CorrectByAspect(QSize size,AVRational aspect)347 QSize CorrectByAspect(QSize size, AVRational aspect) {
348 Expects(IsValidAspectRatio(aspect));
349
350 return QSize(size.width() * aspect.num / aspect.den, size.height());
351 }
352
RotationSwapWidthHeight(int rotation)353 bool RotationSwapWidthHeight(int rotation) {
354 return (rotation == 90 || rotation == 270);
355 }
356
GoodStorageForFrame(const QImage & storage,QSize size)357 bool GoodStorageForFrame(const QImage &storage, QSize size) {
358 return !storage.isNull()
359 && (storage.format() == kImageFormat)
360 && (storage.size() == size)
361 && storage.isDetached()
362 && IsAlignedImage(storage);
363 }
364
365 // Create a QImage of desired size where all the data is properly aligned.
CreateFrameStorage(QSize size)366 QImage CreateFrameStorage(QSize size) {
367 const auto width = size.width();
368 const auto height = size.height();
369 const auto widthAlign = kAlignImageBy / kPixelBytesSize;
370 const auto neededWidth = width + ((width % widthAlign)
371 ? (widthAlign - (width % widthAlign))
372 : 0);
373 const auto perLine = neededWidth * kPixelBytesSize;
374 const auto buffer = new uchar[perLine * height + kAlignImageBy];
375 const auto cleanupData = static_cast<void *>(buffer);
376 const auto address = reinterpret_cast<uintptr_t>(buffer);
377 const auto alignedBuffer = buffer + ((address % kAlignImageBy)
378 ? (kAlignImageBy - (address % kAlignImageBy))
379 : 0);
380 return QImage(
381 alignedBuffer,
382 width,
383 height,
384 perLine,
385 kImageFormat,
386 AlignedImageBufferCleanupHandler,
387 cleanupData);
388 }
389
UnPremultiply(QImage & to,const QImage & from)390 void UnPremultiply(QImage &to, const QImage &from) {
391 // This creates QImage::Format_ARGB32_Premultiplied, but we use it
392 // as an image in QImage::Format_ARGB32 format.
393 if (!GoodStorageForFrame(to, from.size())) {
394 to = CreateFrameStorage(from.size());
395 }
396 const auto fromPerLine = from.bytesPerLine();
397 const auto toPerLine = to.bytesPerLine();
398 const auto width = from.width();
399 const auto height = from.height();
400 auto fromBytes = from.bits();
401 auto toBytes = to.bits();
402 if (fromPerLine != width * 4 || toPerLine != width * 4) {
403 for (auto i = 0; i != height; ++i) {
404 UnPremultiplyLine(toBytes, fromBytes, width);
405 fromBytes += fromPerLine;
406 toBytes += toPerLine;
407 }
408 } else {
409 UnPremultiplyLine(toBytes, fromBytes, width * height);
410 }
411 }
412
PremultiplyInplace(QImage & image)413 void PremultiplyInplace(QImage &image) {
414 const auto perLine = image.bytesPerLine();
415 const auto width = image.width();
416 const auto height = image.height();
417 auto bytes = image.bits();
418 if (perLine != width * 4) {
419 for (auto i = 0; i != height; ++i) {
420 PremultiplyLine(bytes, bytes, width);
421 bytes += perLine;
422 }
423 } else {
424 PremultiplyLine(bytes, bytes, width * height);
425 }
426 }
427
428 } // namespace FFmpeg
429