1 /*
2 * This file is part of libplacebo.
3 *
4 * libplacebo is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * libplacebo is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with libplacebo. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18 #ifndef LIBPLACEBO_LIBAV_H_
19 #error This header should be included as part of <libplacebo/utils/libav.h>
20 #else
21
22 #include <assert.h>
23
24 #include <libavutil/hwcontext.h>
25 #include <libavutil/imgutils.h>
26 #include <libavutil/mastering_display_metadata.h>
27 #include <libavutil/pixdesc.h>
28
29 #if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(56, 61, 100)
30 #define HAVE_LAV_FILM_GRAIN
31 #include <libavutil/film_grain_params.h>
32 #endif
33
pl_system_from_av(enum AVColorSpace spc)34 static inline enum pl_color_system pl_system_from_av(enum AVColorSpace spc)
35 {
36 switch (spc) {
37 case AVCOL_SPC_RGB: return PL_COLOR_SYSTEM_RGB;
38 case AVCOL_SPC_BT709: return PL_COLOR_SYSTEM_BT_709;
39 case AVCOL_SPC_UNSPECIFIED: return PL_COLOR_SYSTEM_UNKNOWN;
40 case AVCOL_SPC_RESERVED: return PL_COLOR_SYSTEM_UNKNOWN;
41 case AVCOL_SPC_FCC: return PL_COLOR_SYSTEM_UNKNOWN; // missing
42 case AVCOL_SPC_BT470BG: return PL_COLOR_SYSTEM_BT_601;
43 case AVCOL_SPC_SMPTE170M: return PL_COLOR_SYSTEM_BT_601;
44 case AVCOL_SPC_SMPTE240M: return PL_COLOR_SYSTEM_SMPTE_240M;
45 case AVCOL_SPC_YCGCO: return PL_COLOR_SYSTEM_YCGCO;
46 case AVCOL_SPC_BT2020_NCL: return PL_COLOR_SYSTEM_BT_2020_NC;
47 case AVCOL_SPC_BT2020_CL: return PL_COLOR_SYSTEM_BT_2020_C;
48 case AVCOL_SPC_SMPTE2085: return PL_COLOR_SYSTEM_UNKNOWN; // missing
49 case AVCOL_SPC_CHROMA_DERIVED_NCL: return PL_COLOR_SYSTEM_UNKNOWN; // missing
50 case AVCOL_SPC_CHROMA_DERIVED_CL: return PL_COLOR_SYSTEM_UNKNOWN; // missing
51 // Note: this colorspace is confused between PQ and HLG, which libav*
52 // requires inferring from other sources, but libplacebo makes explicit.
53 // Default to PQ as it's the more common scenario.
54 case AVCOL_SPC_ICTCP: return PL_COLOR_SYSTEM_BT_2100_PQ;
55 case AVCOL_SPC_NB: return PL_COLOR_SYSTEM_COUNT;
56 }
57
58 return PL_COLOR_SYSTEM_UNKNOWN;
59 }
60
pl_system_to_av(enum pl_color_system sys)61 static inline enum AVColorSpace pl_system_to_av(enum pl_color_system sys)
62 {
63 switch (sys) {
64 case PL_COLOR_SYSTEM_UNKNOWN: return AVCOL_SPC_UNSPECIFIED;
65 case PL_COLOR_SYSTEM_BT_601: return AVCOL_SPC_SMPTE170M;
66 case PL_COLOR_SYSTEM_BT_709: return AVCOL_SPC_BT709;
67 case PL_COLOR_SYSTEM_SMPTE_240M: return AVCOL_SPC_SMPTE240M;
68 case PL_COLOR_SYSTEM_BT_2020_NC: return AVCOL_SPC_BT2020_NCL;
69 case PL_COLOR_SYSTEM_BT_2020_C: return AVCOL_SPC_BT2020_CL;
70 case PL_COLOR_SYSTEM_BT_2100_PQ: return AVCOL_SPC_ICTCP;
71 case PL_COLOR_SYSTEM_BT_2100_HLG: return AVCOL_SPC_ICTCP;
72 case PL_COLOR_SYSTEM_YCGCO: return AVCOL_SPC_YCGCO;
73 case PL_COLOR_SYSTEM_RGB: return AVCOL_SPC_RGB;
74 case PL_COLOR_SYSTEM_XYZ: return AVCOL_SPC_UNSPECIFIED; // handled differently
75 case PL_COLOR_SYSTEM_COUNT: return AVCOL_SPC_NB;
76 }
77
78 return AVCOL_SPC_UNSPECIFIED;
79 }
80
pl_levels_from_av(enum AVColorRange range)81 static inline enum pl_color_levels pl_levels_from_av(enum AVColorRange range)
82 {
83 switch (range) {
84 case AVCOL_RANGE_UNSPECIFIED: return PL_COLOR_LEVELS_UNKNOWN;
85 case AVCOL_RANGE_MPEG: return PL_COLOR_LEVELS_LIMITED;
86 case AVCOL_RANGE_JPEG: return PL_COLOR_LEVELS_FULL;
87 case AVCOL_RANGE_NB: return PL_COLOR_LEVELS_COUNT;
88 }
89
90 return PL_COLOR_LEVELS_UNKNOWN;
91 }
92
pl_levels_to_av(enum pl_color_levels levels)93 static inline enum AVColorRange pl_levels_to_av(enum pl_color_levels levels)
94 {
95 switch (levels) {
96 case PL_COLOR_LEVELS_UNKNOWN: return AVCOL_RANGE_UNSPECIFIED;
97 case PL_COLOR_LEVELS_LIMITED: return AVCOL_RANGE_MPEG;
98 case PL_COLOR_LEVELS_FULL: return AVCOL_RANGE_JPEG;
99 case PL_COLOR_LEVELS_COUNT: return AVCOL_RANGE_NB;
100 }
101
102 return AVCOL_RANGE_UNSPECIFIED;
103 }
104
pl_primaries_from_av(enum AVColorPrimaries prim)105 static inline enum pl_color_primaries pl_primaries_from_av(enum AVColorPrimaries prim)
106 {
107 switch (prim) {
108 case AVCOL_PRI_RESERVED0: return PL_COLOR_PRIM_UNKNOWN;
109 case AVCOL_PRI_BT709: return PL_COLOR_PRIM_BT_709;
110 case AVCOL_PRI_UNSPECIFIED: return PL_COLOR_PRIM_UNKNOWN;
111 case AVCOL_PRI_RESERVED: return PL_COLOR_PRIM_UNKNOWN;
112 case AVCOL_PRI_BT470M: return PL_COLOR_PRIM_BT_470M;
113 case AVCOL_PRI_BT470BG: return PL_COLOR_PRIM_BT_601_625;
114 case AVCOL_PRI_SMPTE170M: return PL_COLOR_PRIM_BT_601_525;
115 case AVCOL_PRI_SMPTE240M: return PL_COLOR_PRIM_BT_601_525;
116 case AVCOL_PRI_FILM: return PL_COLOR_PRIM_FILM_C;
117 case AVCOL_PRI_BT2020: return PL_COLOR_PRIM_BT_2020;
118 case AVCOL_PRI_SMPTE428: return PL_COLOR_PRIM_CIE_1931;
119 case AVCOL_PRI_SMPTE431: return PL_COLOR_PRIM_DCI_P3;
120 case AVCOL_PRI_SMPTE432: return PL_COLOR_PRIM_DISPLAY_P3;
121 case AVCOL_PRI_JEDEC_P22: return PL_COLOR_PRIM_EBU_3213;
122 case AVCOL_PRI_NB: return PL_COLOR_PRIM_COUNT;
123 }
124
125 return PL_COLOR_PRIM_UNKNOWN;
126 }
127
pl_primaries_to_av(enum pl_color_primaries prim)128 static inline enum AVColorPrimaries pl_primaries_to_av(enum pl_color_primaries prim)
129 {
130 switch (prim) {
131 case PL_COLOR_PRIM_UNKNOWN: return AVCOL_PRI_UNSPECIFIED;
132 case PL_COLOR_PRIM_BT_601_525: return AVCOL_PRI_SMPTE170M;
133 case PL_COLOR_PRIM_BT_601_625: return AVCOL_PRI_BT470BG;
134 case PL_COLOR_PRIM_BT_709: return AVCOL_PRI_BT709;
135 case PL_COLOR_PRIM_BT_470M: return AVCOL_PRI_BT470M;
136 case PL_COLOR_PRIM_EBU_3213: return AVCOL_PRI_JEDEC_P22;
137 case PL_COLOR_PRIM_BT_2020: return AVCOL_PRI_BT2020;
138 case PL_COLOR_PRIM_APPLE: return AVCOL_PRI_UNSPECIFIED; // missing
139 case PL_COLOR_PRIM_ADOBE: return AVCOL_PRI_UNSPECIFIED; // missing
140 case PL_COLOR_PRIM_PRO_PHOTO: return AVCOL_PRI_UNSPECIFIED; // missing
141 case PL_COLOR_PRIM_CIE_1931: return AVCOL_PRI_SMPTE428;
142 case PL_COLOR_PRIM_DCI_P3: return AVCOL_PRI_SMPTE431;
143 case PL_COLOR_PRIM_DISPLAY_P3: return AVCOL_PRI_SMPTE432;
144 case PL_COLOR_PRIM_V_GAMUT: return AVCOL_PRI_UNSPECIFIED; // missing
145 case PL_COLOR_PRIM_S_GAMUT: return AVCOL_PRI_UNSPECIFIED; // missing
146 case PL_COLOR_PRIM_FILM_C: return AVCOL_PRI_FILM;
147 case PL_COLOR_PRIM_COUNT: return AVCOL_PRI_NB;
148 }
149
150 return AVCOL_PRI_UNSPECIFIED;
151 }
152
pl_transfer_from_av(enum AVColorTransferCharacteristic trc)153 static inline enum pl_color_transfer pl_transfer_from_av(enum AVColorTransferCharacteristic trc)
154 {
155 switch (trc) {
156 case AVCOL_TRC_RESERVED0: return PL_COLOR_TRC_UNKNOWN;
157 case AVCOL_TRC_BT709: return PL_COLOR_TRC_BT_1886; // EOTF != OETF
158 case AVCOL_TRC_UNSPECIFIED: return PL_COLOR_TRC_UNKNOWN;
159 case AVCOL_TRC_RESERVED: return PL_COLOR_TRC_UNKNOWN;
160 case AVCOL_TRC_GAMMA22: return PL_COLOR_TRC_GAMMA22;
161 case AVCOL_TRC_GAMMA28: return PL_COLOR_TRC_GAMMA28;
162 case AVCOL_TRC_SMPTE170M: return PL_COLOR_TRC_BT_1886; // EOTF != OETF
163 case AVCOL_TRC_SMPTE240M: return PL_COLOR_TRC_BT_1886; // EOTF != OETF
164 case AVCOL_TRC_LINEAR: return PL_COLOR_TRC_LINEAR;
165 case AVCOL_TRC_LOG: return PL_COLOR_TRC_UNKNOWN; // missing
166 case AVCOL_TRC_LOG_SQRT: return PL_COLOR_TRC_UNKNOWN; // missing
167 case AVCOL_TRC_IEC61966_2_4: return PL_COLOR_TRC_BT_1886; // EOTF != OETF
168 case AVCOL_TRC_BT1361_ECG: return PL_COLOR_TRC_BT_1886; // ETOF != OETF
169 case AVCOL_TRC_IEC61966_2_1: return PL_COLOR_TRC_SRGB;
170 case AVCOL_TRC_BT2020_10: return PL_COLOR_TRC_BT_1886; // EOTF != OETF
171 case AVCOL_TRC_BT2020_12: return PL_COLOR_TRC_BT_1886; // EOTF != OETF
172 case AVCOL_TRC_SMPTE2084: return PL_COLOR_TRC_PQ;
173 case AVCOL_TRC_SMPTE428: return PL_COLOR_TRC_UNKNOWN; // missing
174 case AVCOL_TRC_ARIB_STD_B67: return PL_COLOR_TRC_HLG;
175 case AVCOL_TRC_NB: return PL_COLOR_TRC_COUNT;
176 }
177
178 return PL_COLOR_TRC_UNKNOWN;
179 }
180
pl_transfer_to_av(enum pl_color_transfer trc)181 static inline enum AVColorTransferCharacteristic pl_transfer_to_av(enum pl_color_transfer trc)
182 {
183 switch (trc) {
184 case PL_COLOR_TRC_UNKNOWN: return AVCOL_TRC_UNSPECIFIED;
185 case PL_COLOR_TRC_BT_1886: return AVCOL_TRC_BT709; // EOTF != OETF
186 case PL_COLOR_TRC_SRGB: return AVCOL_TRC_IEC61966_2_1;
187 case PL_COLOR_TRC_LINEAR: return AVCOL_TRC_LINEAR;
188 case PL_COLOR_TRC_GAMMA18: return AVCOL_TRC_UNSPECIFIED; // missing
189 case PL_COLOR_TRC_GAMMA20: return AVCOL_TRC_UNSPECIFIED; // missing
190 case PL_COLOR_TRC_GAMMA22: return AVCOL_TRC_GAMMA22;
191 case PL_COLOR_TRC_GAMMA24: return AVCOL_TRC_UNSPECIFIED; // missing
192 case PL_COLOR_TRC_GAMMA26: return AVCOL_TRC_UNSPECIFIED; // missing
193 case PL_COLOR_TRC_GAMMA28: return AVCOL_TRC_GAMMA28;
194 case PL_COLOR_TRC_PRO_PHOTO: return AVCOL_TRC_UNSPECIFIED; // missing
195 case PL_COLOR_TRC_PQ: return AVCOL_TRC_SMPTE2084;
196 case PL_COLOR_TRC_HLG: return AVCOL_TRC_ARIB_STD_B67;
197 case PL_COLOR_TRC_V_LOG: return AVCOL_TRC_UNSPECIFIED; // missing
198 case PL_COLOR_TRC_S_LOG1: return AVCOL_TRC_UNSPECIFIED; // missing
199 case PL_COLOR_TRC_S_LOG2: return AVCOL_TRC_UNSPECIFIED; // missing
200 case PL_COLOR_TRC_COUNT: return AVCOL_TRC_NB;
201 }
202
203 return AVCOL_TRC_UNSPECIFIED;
204 }
205
pl_chroma_from_av(enum AVChromaLocation loc)206 static inline enum pl_chroma_location pl_chroma_from_av(enum AVChromaLocation loc)
207 {
208 switch (loc) {
209 case AVCHROMA_LOC_UNSPECIFIED: return PL_CHROMA_UNKNOWN;
210 case AVCHROMA_LOC_LEFT: return PL_CHROMA_LEFT;
211 case AVCHROMA_LOC_CENTER: return PL_CHROMA_CENTER;
212 case AVCHROMA_LOC_TOPLEFT: return PL_CHROMA_TOP_LEFT;
213 case AVCHROMA_LOC_TOP: return PL_CHROMA_TOP_CENTER;
214 case AVCHROMA_LOC_BOTTOMLEFT: return PL_CHROMA_BOTTOM_LEFT;
215 case AVCHROMA_LOC_BOTTOM: return PL_CHROMA_BOTTOM_CENTER;
216 case AVCHROMA_LOC_NB: return PL_CHROMA_COUNT;
217 }
218
219 return PL_CHROMA_UNKNOWN;
220 }
221
pl_chroma_to_av(enum pl_chroma_location loc)222 static inline enum AVChromaLocation pl_chroma_to_av(enum pl_chroma_location loc)
223 {
224 switch (loc) {
225 case PL_CHROMA_UNKNOWN: return AVCHROMA_LOC_UNSPECIFIED;
226 case PL_CHROMA_LEFT: return AVCHROMA_LOC_LEFT;
227 case PL_CHROMA_CENTER: return AVCHROMA_LOC_CENTER;
228 case PL_CHROMA_TOP_LEFT: return AVCHROMA_LOC_TOPLEFT;
229 case PL_CHROMA_TOP_CENTER: return AVCHROMA_LOC_TOP;
230 case PL_CHROMA_BOTTOM_LEFT: return AVCHROMA_LOC_BOTTOMLEFT;
231 case PL_CHROMA_BOTTOM_CENTER: return AVCHROMA_LOC_BOTTOM;
232 case PL_CHROMA_COUNT: return AVCHROMA_LOC_NB;
233 }
234
235 return AVCHROMA_LOC_UNSPECIFIED;
236 }
237
pl_plane_data_num_comps(const struct pl_plane_data * data)238 static inline int pl_plane_data_num_comps(const struct pl_plane_data *data)
239 {
240 for (int i = 0; i < 4; i++) {
241 if (data->component_size[i] == 0)
242 return i;
243 }
244
245 return 4;
246 }
247
pl_plane_data_from_pixfmt(struct pl_plane_data out_data[4],struct pl_bit_encoding * out_bits,enum AVPixelFormat pix_fmt)248 static inline int pl_plane_data_from_pixfmt(struct pl_plane_data out_data[4],
249 struct pl_bit_encoding *out_bits,
250 enum AVPixelFormat pix_fmt)
251 {
252 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
253 int planes = av_pix_fmt_count_planes(pix_fmt);
254 struct pl_plane_data aligned_data[4];
255 struct pl_bit_encoding bits;
256 bool first;
257 if (!desc || planes < 0) // e.g. AV_PIX_FMT_NONE
258 return 0;
259
260 if (desc->flags & AV_PIX_FMT_FLAG_BE) {
261 // Big endian formats are almost definitely not supported in any
262 // reasonable manner, erroring as a safety precation
263 return 0;
264 }
265
266 if (desc->flags & AV_PIX_FMT_FLAG_BITSTREAM) {
267 // Bitstream formats will most likely never be supported
268 return 0;
269 }
270
271 if (desc->flags & AV_PIX_FMT_FLAG_PAL) {
272 // Palette formats are (currently) not supported
273 return 0;
274 }
275
276 if (desc->flags & AV_PIX_FMT_FLAG_BAYER) {
277 // Bayer format don't have valid `desc->offset` values, so we can't
278 // use `pl_plane_data_from_mask` on them.
279 return 0;
280 }
281
282 if (desc->nb_components == 0 || desc->nb_components > 4) {
283 // Bogus components, possibly fake/virtual/hwaccel format?
284 return 0;
285 }
286
287 if (planes > 4)
288 return 0; // This shouldn't ever happen
289
290 // Fill in the details for each plane
291 for (int p = 0; p < planes; p++) {
292 struct pl_plane_data *data = &out_data[p];
293 uint64_t masks[4] = {0};
294 data->type = (desc->flags & AV_PIX_FMT_FLAG_FLOAT)
295 ? PL_FMT_FLOAT
296 : PL_FMT_UNORM;
297
298 data->pixel_stride = 0;
299
300 for (int c = 0; c < desc->nb_components; c++) {
301 const AVComponentDescriptor *comp = &desc->comp[c];
302 if (comp->plane != p)
303 continue;
304
305 masks[c] = (1LLU << comp->depth) - 1; // e.g. 0xFF for depth=8
306 masks[c] <<= comp->shift;
307 masks[c] <<= comp->offset * 8;
308
309 if (data->pixel_stride && (int) data->pixel_stride != comp->step) {
310 // Pixel format contains components with different pixel stride
311 // (e.g. packed YUYV), this is currently not supported
312 return 0;
313 }
314 data->pixel_stride = comp->step;
315 }
316
317 pl_plane_data_from_mask(data, masks);
318 }
319
320 if (!out_bits)
321 return planes;
322
323 // Attempt aligning all of the planes for optimum compatibility
324 first = true;
325 for (int p = 0; p < planes; p++) {
326 aligned_data[p] = out_data[p];
327
328 // Planes with only an alpha component should be ignored
329 if (pl_plane_data_num_comps(&aligned_data[p]) == 1 &&
330 aligned_data[p].component_map[0] == PL_CHANNEL_A)
331 {
332 continue;
333 }
334
335 if (!pl_plane_data_align(&aligned_data[p], &bits))
336 goto misaligned;
337
338 if (first) {
339 *out_bits = bits;
340 first = false;
341 } else {
342 if (!pl_bit_encoding_equal(&bits, out_bits))
343 goto misaligned;
344 }
345 }
346
347 // Overwrite the planes by their aligned versions
348 for (int p = 0; p < planes; p++)
349 out_data[p] = aligned_data[p];
350
351 return planes;
352
353 misaligned:
354 *out_bits = (struct pl_bit_encoding) {0};
355 return planes;
356 }
357
pl_test_pixfmt(pl_gpu gpu,enum AVPixelFormat pixfmt)358 static inline bool pl_test_pixfmt(pl_gpu gpu,
359 enum AVPixelFormat pixfmt)
360 {
361 struct pl_bit_encoding bits;
362 struct pl_plane_data data[4];
363 int planes = pl_plane_data_from_pixfmt(data, &bits, pixfmt);
364 if (!planes)
365 return false;
366
367 for (int i = 0; i < planes; i++) {
368 if (!pl_plane_find_fmt(gpu, NULL, &data[i]))
369 return false;
370 }
371
372 return true;
373 }
374
pl_avframe_set_color(AVFrame * frame,struct pl_color_space space)375 static inline void pl_avframe_set_color(AVFrame *frame, struct pl_color_space space)
376 {
377 const AVFrameSideData *sd;
378
379 frame->color_primaries = pl_primaries_to_av(space.primaries);
380 frame->color_trc = pl_transfer_to_av(space.transfer);
381 // No way to map space.light, so just ignore it
382
383 if (space.sig_peak > 1 && space.sig_peak < pl_color_transfer_nominal_peak(space.transfer)) {
384 sd = av_frame_get_side_data(frame, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL);
385 if (!sd) {
386 sd = av_frame_new_side_data(frame, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL,
387 sizeof(AVContentLightMetadata));
388 }
389
390 if (sd) {
391 AVContentLightMetadata *clm = (AVContentLightMetadata *) sd->data;
392 *clm = (AVContentLightMetadata) {
393 .MaxCLL = space.sig_peak * PL_COLOR_SDR_WHITE,
394 .MaxFALL = space.sig_avg * PL_COLOR_SDR_WHITE,
395 };
396
397 if (!clm->MaxFALL)
398 clm->MaxFALL = clm->MaxCLL;
399 }
400 }
401
402 if (space.sig_floor > 0.0) {
403 sd = av_frame_get_side_data(frame, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA);
404 if (!sd) {
405 sd = av_frame_new_side_data(frame, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA,
406 sizeof(AVMasteringDisplayMetadata));
407 }
408
409 if (sd) {
410 AVMasteringDisplayMetadata *mdm = (AVMasteringDisplayMetadata *) sd->data;
411 *mdm = (AVMasteringDisplayMetadata) {
412 .max_luminance = av_d2q(space.sig_peak * PL_COLOR_SDR_WHITE, 100000),
413 .min_luminance = av_d2q(space.sig_floor * PL_COLOR_SDR_WHITE, 100000),
414 .has_luminance = 1,
415 };
416 }
417 }
418
419 // No way to map space.sig_scale, so just ignore it
420 }
421
pl_avframe_set_repr(AVFrame * frame,struct pl_color_repr repr)422 static inline void pl_avframe_set_repr(AVFrame *frame, struct pl_color_repr repr)
423 {
424 frame->colorspace = pl_system_to_av(repr.sys);
425 frame->color_range = pl_levels_to_av(repr.levels);
426
427 // No real way to map repr.bits, the image format already has to match
428 }
429
pl_avframe_set_profile(AVFrame * frame,struct pl_icc_profile profile)430 static inline void pl_avframe_set_profile(AVFrame *frame, struct pl_icc_profile profile)
431 {
432 const AVFrameSideData *sd;
433 av_frame_remove_side_data(frame, AV_FRAME_DATA_ICC_PROFILE);
434
435 if (!profile.len)
436 return;
437
438 sd = av_frame_new_side_data(frame, AV_FRAME_DATA_ICC_PROFILE, profile.len);
439 memcpy(sd->data, profile.data, profile.len);
440 }
441
pl_frame_from_avframe(struct pl_frame * out,const AVFrame * frame)442 static inline void pl_frame_from_avframe(struct pl_frame *out,
443 const AVFrame *frame)
444 {
445 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
446 int planes = av_pix_fmt_count_planes(frame->format);
447 const AVFrameSideData *sd;
448 assert(desc);
449
450 if (desc->flags & AV_PIX_FMT_FLAG_HWACCEL) {
451 const AVHWFramesContext *hwfc = (AVHWFramesContext *) frame->hw_frames_ctx->data;
452 desc = av_pix_fmt_desc_get(hwfc->sw_format);
453 planes = av_pix_fmt_count_planes(hwfc->sw_format);
454 }
455
456 // This should never fail, and there's nothing really useful we can do in
457 // this failure case anyway, since this is a `void` function.
458 assert(planes <= 4);
459
460 *out = (struct pl_frame) {
461 .num_planes = planes,
462 .crop = {
463 .x0 = frame->crop_left,
464 .y0 = frame->crop_top,
465 .x1 = frame->width - frame->crop_right,
466 .y1 = frame->height - frame->crop_bottom,
467 },
468 .color = {
469 .primaries = pl_primaries_from_av(frame->color_primaries),
470 .transfer = pl_transfer_from_av(frame->color_trc),
471 .light = PL_COLOR_LIGHT_UNKNOWN,
472 },
473 .repr = {
474 .sys = pl_system_from_av(frame->colorspace),
475 .levels = pl_levels_from_av(frame->color_range),
476 .alpha = (desc->flags & AV_PIX_FMT_FLAG_ALPHA)
477 ? PL_ALPHA_INDEPENDENT
478 : PL_ALPHA_UNKNOWN,
479
480 // For sake of simplicity, just use the first component's depth as
481 // the authoritative color depth for the whole image. Usually, this
482 // will be overwritten by more specific information when using e.g.
483 // `pl_upload_avframe`, but for the sake of e.g. users only wishing
484 // to map hwaccel frames, this is a good default.
485 .bits.color_depth = desc->comp[0].depth,
486 },
487 };
488
489 if (frame->colorspace == AVCOL_SPC_ICTCP &&
490 frame->color_trc == AVCOL_TRC_ARIB_STD_B67)
491 {
492 // libav* makes no distinction between PQ and HLG ICtCp, so we need
493 // to manually fix it in the case that we have HLG ICtCp data.
494 out->repr.sys = PL_COLOR_SYSTEM_BT_2100_HLG;
495
496 } else if (strncmp(desc->name, "xyz", 3) == 0) {
497
498 // libav* handles this as a special case, but doesn't provide an
499 // explicit flag for it either, so we have to resort to this ugly
500 // hack...
501 out->repr.sys= PL_COLOR_SYSTEM_XYZ;
502
503 } else if (desc->flags & AV_PIX_FMT_FLAG_RGB) {
504
505 out->repr.sys = PL_COLOR_SYSTEM_RGB;
506 out->repr.levels = PL_COLOR_LEVELS_FULL; // libav* ignores levels for RGB
507
508 } else if (!out->repr.sys) {
509
510 // libav* likes leaving this as UNKNOWN for YCbCr frames, which
511 // confuses libplacebo since we infer UNKNOWN as RGB. To get around
512 // this, explicitly infer a suitable colorspace for non-RGB formats.
513 out->repr.sys = pl_color_system_guess_ycbcr(frame->width, frame->height);
514 }
515
516 if ((sd = av_frame_get_side_data(frame, AV_FRAME_DATA_ICC_PROFILE))) {
517 out->profile = (struct pl_icc_profile) {
518 .data = sd->data,
519 .len = sd->size,
520 };
521
522 // Needed to ensure profile uniqueness
523 pl_icc_profile_compute_signature(&out->profile);
524 }
525
526 if ((sd = av_frame_get_side_data(frame, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL))) {
527 const AVContentLightMetadata *clm = (AVContentLightMetadata *) sd->data;
528 out->color.sig_peak = clm->MaxCLL / PL_COLOR_SDR_WHITE;
529 out->color.sig_avg = clm->MaxFALL / PL_COLOR_SDR_WHITE;
530 }
531
532 // This overrides the CLL values above, if both are present
533 if ((sd = av_frame_get_side_data(frame, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA))) {
534 const AVMasteringDisplayMetadata *mdm = (AVMasteringDisplayMetadata *) sd->data;
535 if (mdm->has_luminance) {
536 out->color.sig_peak = av_q2d(mdm->max_luminance) / PL_COLOR_SDR_WHITE;
537 out->color.sig_floor = av_q2d(mdm->min_luminance) / PL_COLOR_SDR_WHITE;
538 }
539 }
540
541 // Make sure this value is more or less legal
542 if (out->color.sig_peak < 1.0 || out->color.sig_peak > 50.0)
543 out->color.sig_peak = 0.0;
544
545 #ifdef HAVE_LAV_FILM_GRAIN
546 if ((sd = av_frame_get_side_data(frame, AV_FRAME_DATA_FILM_GRAIN_PARAMS))) {
547 const AVFilmGrainParams *fgp = (AVFilmGrainParams *) sd->data;
548 switch (fgp->type) {
549 case AV_FILM_GRAIN_PARAMS_NONE:
550 break;
551 case AV_FILM_GRAIN_PARAMS_AV1: {
552 const AVFilmGrainAOMParams *aom = &fgp->codec.aom;
553 struct pl_av1_grain_data *av1 = &out->av1_grain;
554 *av1 = (struct pl_av1_grain_data) {
555 .grain_seed = fgp->seed,
556 .num_points_y = aom->num_y_points,
557 .chroma_scaling_from_luma = aom->chroma_scaling_from_luma,
558 .num_points_uv = { aom->num_uv_points[0], aom->num_uv_points[1] },
559 .scaling_shift = aom->scaling_shift,
560 .ar_coeff_lag = aom->ar_coeff_lag,
561 .ar_coeff_shift = aom->ar_coeff_shift,
562 .grain_scale_shift = aom->grain_scale_shift,
563 .uv_mult = { aom->uv_mult[0], aom->uv_mult[1] },
564 .uv_mult_luma = { aom->uv_mult_luma[0], aom->uv_mult_luma[1] },
565 .uv_offset = { aom->uv_offset[0], aom->uv_offset[1] },
566 .overlap = aom->overlap_flag,
567 };
568
569 assert(sizeof(av1->ar_coeffs_uv) == sizeof(aom->ar_coeffs_uv));
570 memcpy(av1->points_y, aom->y_points, sizeof(av1->points_y));
571 memcpy(av1->points_uv, aom->uv_points, sizeof(av1->points_uv));
572 memcpy(av1->ar_coeffs_y, aom->ar_coeffs_y, sizeof(av1->ar_coeffs_y));
573 memcpy(av1->ar_coeffs_uv, aom->ar_coeffs_uv, sizeof(av1->ar_coeffs_uv));
574 break;
575 }
576 }
577 }
578 #endif // HAVE_LAV_AV1_GRAIN
579
580 for (int p = 0; p < out->num_planes; p++) {
581 struct pl_plane *plane = &out->planes[p];
582
583 // Fill in the component mapping array
584 for (int c = 0; c < desc->nb_components; c++) {
585 if (desc->comp[c].plane == p)
586 plane->component_mapping[plane->components++] = c;
587 }
588
589 // Clear the superfluous components
590 for (int c = plane->components; c < 4; c++)
591 plane->component_mapping[c] = PL_CHANNEL_NONE;
592 }
593
594 // Only set the chroma location for definitely subsampled images, makes no
595 // sense otherwise
596 if (desc->log2_chroma_w || desc->log2_chroma_h) {
597 enum pl_chroma_location loc = pl_chroma_from_av(frame->chroma_location);
598 pl_frame_set_chroma_location(out, loc);
599 }
600 }
601
pl_swapchain_colors_from_avframe(struct pl_swapchain_colors * out_colors,const AVFrame * avframe)602 static inline void pl_swapchain_colors_from_avframe(struct pl_swapchain_colors *out_colors,
603 const AVFrame *avframe)
604 {
605 const AVFrameSideData *sd;
606 struct pl_frame frame;
607 pl_frame_from_avframe(&frame, avframe);
608
609 *out_colors = (struct pl_swapchain_colors) {
610 .primaries = frame.color.primaries,
611 .transfer = frame.color.transfer,
612 };
613
614 if ((sd = av_frame_get_side_data(avframe, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL))) {
615 const AVContentLightMetadata *clm = (AVContentLightMetadata *) sd->data;
616 out_colors->hdr.max_cll = clm->MaxCLL;
617 out_colors->hdr.max_fall = clm->MaxFALL;
618 }
619
620 if ((sd = av_frame_get_side_data(avframe, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA))) {
621 const AVMasteringDisplayMetadata *mdm = (AVMasteringDisplayMetadata *) sd->data;
622 if (mdm->has_luminance) {
623 out_colors->hdr.min_luma = av_q2d(mdm->min_luminance);
624 out_colors->hdr.max_luma = av_q2d(mdm->max_luminance);
625 }
626
627 if (mdm->has_primaries) {
628 out_colors->hdr.prim.red.x = av_q2d(mdm->display_primaries[0][0]);
629 out_colors->hdr.prim.red.y = av_q2d(mdm->display_primaries[0][1]);
630 out_colors->hdr.prim.green.x = av_q2d(mdm->display_primaries[1][0]);
631 out_colors->hdr.prim.green.y = av_q2d(mdm->display_primaries[1][1]);
632 out_colors->hdr.prim.blue.x = av_q2d(mdm->display_primaries[2][0]);
633 out_colors->hdr.prim.blue.y = av_q2d(mdm->display_primaries[2][1]);
634 out_colors->hdr.prim.white.x = av_q2d(mdm->white_point[0]);
635 out_colors->hdr.prim.white.y = av_q2d(mdm->white_point[1]);
636 }
637 }
638 }
639
pl_frame_recreate_from_avframe(pl_gpu gpu,struct pl_frame * out,pl_tex tex[4],const AVFrame * frame)640 static inline bool pl_frame_recreate_from_avframe(pl_gpu gpu,
641 struct pl_frame *out,
642 pl_tex tex[4],
643 const AVFrame *frame)
644 {
645 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
646 struct pl_plane_data data[4] = {0};
647 int planes;
648
649 pl_frame_from_avframe(out, frame);
650 planes = pl_plane_data_from_pixfmt(data, &out->repr.bits, frame->format);
651 if (!planes)
652 return false;
653
654 for (int p = 0; p < planes; p++) {
655 bool is_chroma = p == 1 || p == 2; // matches lavu logic
656 data[p].width = AV_CEIL_RSHIFT(frame->width, is_chroma ? desc->log2_chroma_w : 0);
657 data[p].height = AV_CEIL_RSHIFT(frame->height, is_chroma ? desc->log2_chroma_h : 0);
658
659 if (!pl_recreate_plane(gpu, &out->planes[p], &tex[p], &data[p]))
660 return false;
661 }
662
663 return true;
664 }
665
pl_avframe_free(void * priv)666 static void pl_avframe_free(void *priv)
667 {
668 AVFrame *frame = priv;
669 av_frame_free(&frame);
670 }
671
672 #define PL_MAGIC0 0xfb5b3b8b
673 #define PL_MAGIC1 0xee659f6d
674
675 struct pl_avalloc {
676 uint32_t magic[2];
677 pl_gpu gpu;
678 pl_buf buf;
679 };
680
pl_upload_avframe(pl_gpu gpu,struct pl_frame * out,pl_tex tex[4],const AVFrame * frame)681 static inline bool pl_upload_avframe(pl_gpu gpu,
682 struct pl_frame *out,
683 pl_tex tex[4],
684 const AVFrame *frame)
685 {
686 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
687 struct pl_plane_data data[4] = {0};
688 int planes;
689
690 pl_frame_from_avframe(out, frame);
691
692 // TODO: support HW-accelerated formats (e.g. wrapping vulkan frames,
693 // importing DRM buffers, and so on)
694
695 planes = pl_plane_data_from_pixfmt(data, &out->repr.bits, frame->format);
696 if (!planes)
697 return false;
698
699 // Probe for frames allocated by our get_buffer2
700 struct pl_avalloc *alloc = frame->buf[0] ? av_buffer_get_opaque(frame->buf[0]) : NULL;
701 pl_buf buf = NULL;
702 if (alloc && alloc->magic[0] == PL_MAGIC0 && alloc->magic[1] == PL_MAGIC1) {
703 assert(alloc->gpu == gpu);
704 buf = alloc->buf;
705 }
706
707 for (int p = 0; p < planes; p++) {
708 bool is_chroma = p == 1 || p == 2; // matches lavu logic
709 data[p].width = AV_CEIL_RSHIFT(frame->width, is_chroma ? desc->log2_chroma_w : 0);
710 data[p].height = AV_CEIL_RSHIFT(frame->height, is_chroma ? desc->log2_chroma_h : 0);
711 data[p].row_stride = frame->linesize[p];
712 data[p].pixels = frame->data[p];
713
714 if (buf) {
715 data[p].pixels = NULL;
716 data[p].buf = buf;
717 data[p].buf_offset = (uintptr_t) frame->data[p] - (uintptr_t) buf->data;
718 } else if (gpu->limits.callbacks) {
719 // Use asynchronous upload if possible
720 data[p].callback = pl_avframe_free;
721 data[p].priv = av_frame_clone(frame);
722 }
723
724 if (!pl_upload_plane(gpu, &out->planes[p], &tex[p], &data[p])) {
725 pl_avframe_free(data[p].priv);
726 return false;
727 }
728 }
729
730 return true;
731 }
732
pl_done_cb(void * priv)733 static void pl_done_cb(void *priv)
734 {
735 bool *status = priv;
736 *status = true;
737 }
738
pl_download_avframe(pl_gpu gpu,const struct pl_frame * frame,AVFrame * out_frame)739 static inline bool pl_download_avframe(pl_gpu gpu,
740 const struct pl_frame *frame,
741 AVFrame *out_frame)
742 {
743 bool done[4] = {0};
744 if (frame->num_planes != av_pix_fmt_count_planes(out_frame->format))
745 return false;
746
747 for (int p = 0; p < frame->num_planes; p++) {
748 size_t texel_size = frame->planes[p].texture->params.format->texel_size;
749 bool ok = pl_tex_download(gpu, &(struct pl_tex_transfer_params) {
750 .tex = frame->planes[p].texture,
751 .stride_w = out_frame->linesize[p] / texel_size,
752 .ptr = out_frame->data[p],
753 // Use synchronous transfer for the last plane
754 .callback = (p+1) < frame->num_planes ? pl_done_cb : NULL,
755 .priv = &done[p],
756 });
757
758 if (!ok)
759 return false;
760 }
761
762 for (int p = 0; p < frame->num_planes - 1; p++) {
763 while (!done[p])
764 pl_tex_poll(gpu, frame->planes[p].texture, UINT64_MAX);
765 }
766
767 return true;
768 }
769
770 #define PL_ALIGN(x, align) ((align) ? ((x) + (align) - 1) / (align) * (align) : (x))
771 #define PL_ALIGN2(x, align) (((x) + (align) - 1) & ~((align) - 1))
772 #define PL_MAX(x, y) ((x) > (y) ? (x) : (y))
773
pl_avalloc_free(void * opaque,uint8_t * data)774 static inline void pl_avalloc_free(void *opaque, uint8_t *data)
775 {
776 struct pl_avalloc *alloc = opaque;
777 assert(alloc->magic[0] == PL_MAGIC0);
778 assert(alloc->magic[1] == PL_MAGIC1);
779 assert(alloc->buf->data == data);
780 pl_buf_destroy(alloc->gpu, &alloc->buf);
781 free(alloc);
782 }
783
pl_get_buffer2(AVCodecContext * avctx,AVFrame * pic,int flags)784 static inline int pl_get_buffer2(AVCodecContext *avctx, AVFrame *pic, int flags)
785 {
786 int linesize_align[AV_NUM_DATA_POINTERS];
787 uint8_t *pointers[5];
788 size_t offsets[4];
789 int width = pic->width;
790 int height = pic->height;
791 size_t total_size = 0;
792 int ret;
793
794 pl_gpu *pgpu = avctx->opaque;
795 pl_gpu gpu = pgpu ? *pgpu : NULL;
796 pl_buf buf;
797 struct pl_plane_data data[4];
798 struct pl_avalloc *alloc = malloc(sizeof(struct pl_avalloc));
799 int planes = pl_plane_data_from_pixfmt(data, NULL, pic->format);
800 if (!(avctx->codec->capabilities & AV_CODEC_CAP_DR1) || !planes)
801 goto fallback;
802 if (!gpu || !gpu->limits.thread_safe || !gpu->limits.max_mapped_size)
803 goto fallback;
804
805 memset(pic->data, 0, sizeof(pic->data));
806 memset(pic->linesize, 0, sizeof(pic->linesize));
807 memset(pic->buf, 0, sizeof(pic->buf));
808 pic->extended_data = pic->data;
809 pic->extended_buf = NULL;
810
811 avcodec_align_dimensions2(avctx, &width, &height, linesize_align);
812 if ((ret = av_image_fill_linesizes(pic->linesize, pic->format, width)))
813 return ret;
814
815 for (int p = 0; p < planes; p++) {
816 int align = PL_MAX(linesize_align[p], gpu->limits.align_tex_xfer_stride);
817 pic->linesize[p] = PL_ALIGN2(pic->linesize[p], align);
818 pic->linesize[p] = PL_ALIGN(pic->linesize[p], data[p].pixel_stride);
819 }
820
821 // Properly align each plane offset, using the pointers as a proxy to
822 // figure out the size of each plane
823 ret = av_image_fill_pointers(pointers, pic->format, height, NULL, pic->linesize);
824 if (ret < 0) {
825 free(alloc);
826 return ret;
827 }
828 pointers[planes] = (uint8_t *) (uintptr_t) ret;
829
830 for (int p = 0; p < planes; p++) {
831 size_t plane_size = (uintptr_t) pointers[p+1] - (uintptr_t) pointers[p];
832 offsets[p] = PL_ALIGN2(total_size, gpu->limits.align_tex_xfer_offset);
833 offsets[p] = PL_ALIGN(offsets[p], data[p].pixel_stride);
834 total_size = offsets[p] + plane_size;
835 }
836
837 if (total_size > gpu->limits.max_mapped_size)
838 goto fallback;
839
840 // Create data buffer
841 buf = pl_buf_create(gpu, &(struct pl_buf_params) {
842 .size = total_size,
843 .memory_type = PL_BUF_MEM_HOST,
844 .host_mapped = true,
845 });
846 if (!buf)
847 goto fallback;
848
849 for (int p = 0; p < planes; p++)
850 pic->data[p] = buf->data + offsets[p];
851
852 // Create buffer ref
853 pic->buf[0] = av_buffer_create(buf->data, total_size, pl_avalloc_free, alloc, 0);
854 *alloc = (struct pl_avalloc) {
855 .magic = { PL_MAGIC0, PL_MAGIC1 },
856 .gpu = gpu,
857 .buf = buf,
858 };
859
860 if (!pic->buf[0]) {
861 pl_buf_destroy(gpu, &buf);
862 free(alloc);
863 return AVERROR(ENOMEM);
864 }
865
866 return 0;
867
868 fallback:
869 free(alloc);
870 return avcodec_default_get_buffer2(avctx, pic, flags);
871 }
872
873 #undef PL_MAGIC0
874 #undef PL_MAGIC1
875 #undef PL_ALIGN
876 #undef PL_ALIGN2
877 #undef PL_MAX
878
879 #endif // LIBPLACEBO_LIBAV_H_
880