1 /* 2 * Copyright (c) 2017, Alliance for Open Media. All rights reserved 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at https://www.aomedia.org/license/software-license. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at https://www.aomedia.org/license/patent-license. 10 */ 11 12 #ifndef AOM_AOM_DSP_NOISE_MODEL_H_ 13 #define AOM_AOM_DSP_NOISE_MODEL_H_ 14 15 #ifdef __cplusplus 16 extern "C" { 17 #endif // __cplusplus 18 19 #include <stdint.h> 20 #include "grainSynthesis.h" 21 #include "EbPictureBufferDesc.h" 22 #include "EbObject.h" 23 24 #define DENOISING_BlockSize 32 25 26 /*!\brief Wrapper of data required to represent linear system of eqns and soln. 27 */ 28 typedef struct { 29 double *A; 30 double *b; 31 double *x; 32 int32_t n; 33 } AomEquationSystem; 34 35 /*!\brief Representation of a piecewise linear curve 36 * 37 * Holds n points as (x, y) pairs, that store the curve. 38 */ 39 typedef struct { 40 double (*points)[2]; 41 int32_t num_points; 42 } AomNoiseStrengthLut; 43 44 /*!\brief Init the noise strength lut with the given number of points*/ 45 int32_t svt_aom_noise_strength_lut_init(AomNoiseStrengthLut *lut, int32_t num_points); 46 47 /*!\brief Frees the noise strength lut. */ 48 void svt_aom_noise_strength_lut_free(AomNoiseStrengthLut *lut); 49 50 /*!\brief Helper struct to model noise strength as a function of intensity. 51 * 52 * Internally, this structure holds a representation of a linear system 53 * of equations that models noise strength (standard deviation) as a 54 * function of intensity. The mapping is initially stored using a 55 * piecewise representation with evenly spaced bins that cover the entire 56 * domain from [min_intensity, max_intensity]. Each observation (x,y) gives a 57 * constraint of the form: 58 * y_{i} (1 - a) + y_{i+1} a = y 59 * where y_{i} is the value of bin i and x_{i} <= x <= x_{i+1} and 60 * a = x/(x_{i+1} - x{i}). The equation system holds the corresponding 61 * normal equations. 62 * 63 * As there may be missing data, the solution is regularized to get a 64 * complete set of values for the bins. A reduced representation after 65 * solving can be obtained by getting the corresponding noise_strength_lut_t. 66 */ 67 typedef struct { 68 AomEquationSystem eqns; 69 double min_intensity; 70 double max_intensity; 71 int32_t num_bins; 72 int32_t num_equations; 73 double total; 74 } AomNoiseStrengthSolver; 75 76 /*!\brief Initializes the noise solver with the given number of bins. 77 * 78 * Returns 0 if initialization fails. 79 * 80 * \param[in] solver The noise solver to be initialized. 81 * \param[in] num_bins Number of bins to use in the internal representation. 82 * \param[in] bit_depth The bit depth used to derive {min,max}_intensity. 83 */ 84 int32_t svt_aom_noise_strength_solver_init(AomNoiseStrengthSolver *solver, int32_t num_bins, 85 int32_t bit_depth); 86 /*!\brief Gets the x coordinate of bin i. 87 * 88 * \param[in] i The bin whose coordinate to query. 89 */ 90 double svt_aom_noise_strength_solver_get_center(const AomNoiseStrengthSolver *solver, int32_t i); 91 92 /*!\brief Add an observation of the block mean intensity to its noise strength. 93 * 94 * \param[in] block_mean The average block intensity, 95 * \param[in] noise_std The observed noise strength. 96 */ 97 void svt_aom_noise_strength_solver_add_measurement(AomNoiseStrengthSolver *solver, 98 double block_mean, double noise_std); 99 100 /*!\brief Solves the current set of equations for the noise strength. */ 101 int32_t svt_aom_noise_strength_solver_solve(AomNoiseStrengthSolver *solver); 102 103 /*!\brief Fits a reduced piecewise linear lut to the internal solution 104 * 105 * \param[in] max_num_points The maximum number of output points 106 * \param[out] lut The output piecewise linear lut. 107 */ 108 int32_t svt_aom_noise_strength_solver_fit_piecewise(const AomNoiseStrengthSolver *solver, 109 int32_t max_num_points, 110 AomNoiseStrengthLut * lut); 111 112 /*!\brief Helper for holding precomputed data for finding flat blocks. 113 * 114 * Internally a block is modeled with a low-order polynomial model. A 115 * planar model would be a bunch of equations like: 116 * <[y_i x_i 1], [a_1, a_2, a_3]> = b_i 117 * for each point in the block. The system matrix A with row i as [y_i x_i 1] 118 * is maintained as is the inverse, inv(A'*A), so that the plane parameters 119 * can be fit for each block. 120 */ 121 typedef struct { 122 double *at_a_inv; 123 double *A; 124 int32_t num_params; // The number of parameters used for internal low-order model 125 int32_t block_size; // The block size the finder was initialized with 126 double normalization; // Normalization factor (1 / (2^(bit_depth) - 1)) 127 int32_t use_highbd; // Whether input data should be interpreted as uint16 128 } AomFlatBlockFinder; 129 130 /*!\brief Init the block_finder with the given block size, bit_depth */ 131 int32_t svt_aom_flat_block_finder_init(AomFlatBlockFinder *block_finder, int32_t block_size, 132 int32_t bit_depth, int32_t use_highbd); 133 void svt_aom_flat_block_finder_free(AomFlatBlockFinder *block_finder); 134 135 /*!\brief Helper to extract a block and low order "planar" model. */ 136 void svt_aom_flat_block_finder_extract_block(const AomFlatBlockFinder *block_finder, 137 const uint8_t *const data, int32_t w, int32_t h, 138 int32_t stride, int32_t offsx, int32_t offsy, 139 double *plane, double *block); 140 141 /*!\brief Runs the flat block finder on the input data. 142 * 143 * Find flat blocks in the input image data. Returns a map of 144 * flat_blocks, where the value of flat_blocks map will be non-zero 145 * when a block is determined to be flat. A higher value indicates a bigger 146 * confidence in the decision. 147 */ 148 int32_t svt_aom_flat_block_finder_run(const AomFlatBlockFinder *block_finder, 149 const uint8_t *const data, int32_t w, int32_t h, 150 int32_t stride, uint8_t *flat_blocks); 151 152 // The noise shape indicates the allowed coefficients in the AR model. 153 typedef enum { AOM_NOISE_SHAPE_DIAMOND = 0, AOM_NOISE_SHAPE_SQUARE = 1 } AomNoiseShape; 154 155 // The parameters of the noise model include the shape type, lag, the 156 // bit depth of the input images provided, and whether the input images 157 // will be using uint16 (or uint8) representation. 158 typedef struct { 159 AomNoiseShape shape; 160 int32_t lag; 161 int32_t bit_depth; 162 int32_t use_highbd; 163 } AomNoiseModelParams; 164 165 /*!\brief State of a noise model estimate for a single channel. 166 * 167 * This contains a system of equations that can be used to solve 168 * for the auto-regressive coefficients as well as a noise strength 169 * solver that can be used to model noise strength as a function of 170 * intensity. 171 */ 172 typedef struct { 173 AomEquationSystem eqns; 174 AomNoiseStrengthSolver strength_solver; 175 int32_t num_observations; // The number of observations in the eqn system 176 double ar_gain; // The gain of the current AR filter 177 } AomNoiseState; 178 179 /*!\brief Complete model of noise for a planar video 180 * 181 * This includes a noise model for the latest frame and an aggregated 182 * estimate over all previous frames that had similar parameters. 183 */ 184 typedef struct { 185 AomNoiseModelParams params; 186 AomNoiseState combined_state[3]; // Combined state per channel 187 AomNoiseState latest_state[3]; // Latest state per channel 188 int32_t (*coords)[2]; // Offsets (x,y) of the coefficient samples 189 int32_t n; // Number of parameters (size of coords) 190 int32_t bit_depth; 191 } AomNoiseModel; 192 193 /*!\brief Result of a noise model update. */ 194 typedef enum { 195 AOM_NOISE_STATUS_OK = 0, 196 AOM_NOISE_STATUS_INVALID_ARGUMENT, 197 AOM_NOISE_STATUS_INSUFFICIENT_FLAT_BLOCKS, 198 AOM_NOISE_STATUS_DIFFERENT_NOISE_TYPE, 199 AOM_NOISE_STATUS_INTERNAL_ERROR, 200 } AomNoiseStatus; 201 202 /************************************ 203 * DenoiseAndModelInitData 204 ************************************/ 205 typedef struct DenoiseAndModelInitData { 206 uint16_t noise_level; 207 uint32_t encoder_bit_depth; 208 uint32_t encoder_color_format; 209 210 uint16_t width; 211 uint16_t height; 212 uint16_t stride_y; 213 uint16_t stride_cb; 214 uint16_t stride_cr; 215 } DenoiseAndModelInitData; 216 217 typedef struct AomDenoiseAndModel { 218 EbDctor dctor; 219 int32_t block_size; 220 int32_t bit_depth; 221 float noise_level; 222 223 // Size of current denoised buffer and flat_block buffer 224 int32_t width; 225 int32_t height; 226 int32_t y_stride; 227 int32_t uv_stride; 228 int32_t num_blocks_w; 229 int32_t num_blocks_h; 230 231 // Buffers for image and noise_psd allocated on the fly 232 float * noise_psd[3]; 233 uint8_t * denoised[3]; 234 uint8_t * flat_blocks; 235 uint16_t * packed[3]; 236 EbPictureBufferDesc *denoised_pic; 237 EbPictureBufferDesc *packed_pic; 238 239 AomFlatBlockFinder flat_block_finder; 240 AomNoiseModel noise_model; 241 } AomDenoiseAndModel; 242 243 /************************************ 244 * denoise and model constructor 245 ************************************/ 246 EbErrorType denoise_and_model_ctor(AomDenoiseAndModel *object_ptr, EbPtr object_init_data_ptr); 247 248 /*!\brief Initializes a noise model with the given parameters. 249 * 250 * Returns 0 on failure. 251 */ 252 int32_t svt_aom_noise_model_init(AomNoiseModel *model, const AomNoiseModelParams params); 253 void svt_aom_noise_model_free(AomNoiseModel *model); 254 255 /*!\brief Updates the noise model with a new frame observation. 256 * 257 * Updates the noise model with measurements from the given input frame and a 258 * denoised variant of it. Noise is sampled from flat blocks using the flat 259 * block map. 260 * 261 * Returns a noise_status indicating if the update was successful. If the 262 * Update was successful, the combined_state is updated with measurements from 263 * the provided frame. If status is OK or DIFFERENT_NOISE_TYPE, the latest noise 264 * state will be updated with measurements from the provided frame. 265 * 266 * \param[in,out] noise_model The noise model to be updated 267 * \param[in] data Raw frame data 268 * \param[in] denoised Denoised frame data. 269 * \param[in] w Frame width 270 * \param[in] h Frame height 271 * \param[in] strides Stride of the planes 272 * \param[in] chroma_sub_log2 Chroma subsampling for planes != 0. 273 * \param[in] flat_blocks A map to blocks that have been determined flat 274 * \param[in] block_size The size of blocks. 275 */ 276 AomNoiseStatus svt_aom_noise_model_update(AomNoiseModel *const noise_model, 277 const uint8_t *const data[3], 278 const uint8_t *const denoised[3], int32_t w, int32_t h, 279 int32_t strides[3], int32_t chroma_sub_log2[2], 280 const uint8_t *const flat_blocks, int32_t block_size); 281 282 /*\brief Save the "latest" estimate into the "combined" estimate. 283 * 284 * This is meant to be called when the noise modeling detected a change 285 * in parameters (or for example, if a user wanted to reset estimation at 286 * a shot boundary). 287 */ 288 void svt_aom_noise_model_save_latest(AomNoiseModel *noise_model); 289 290 /*!\brief Converts the noise_model parameters to the corresponding 291 * grain_parameters. 292 * 293 * The noise structs in this file are suitable for estimation (e.g., using 294 * floats), but the grain parameters in the Bitstream are quantized. This 295 * function does the conversion by selecting the correct quantization levels. 296 */ 297 int32_t svt_aom_noise_model_get_grain_parameters(AomNoiseModel *const noise_model, 298 AomFilmGrain * film_grain); 299 300 /*!\brief Perform a Wiener filter denoising in 2D using the provided noise psd. 301 * 302 * \param[in] data Raw frame data 303 * \param[out] denoised Denoised frame data 304 * \param[in] w Frame width 305 * \param[in] h Frame height 306 * \param[in] stride Stride of the planes 307 * \param[in] chroma_sub_log2 Chroma subsampling for planes != 0. 308 * \param[in] noise_psd The power spectral density of the noise 309 * \param[in] block_size The size of blocks 310 * \param[in] bit_depth Bit depth of the image 311 * \param[in] use_highbd If true, uint8 pointers are interpreted as 312 * uint16 and stride is measured in uint16. 313 * This must be true when bit_depth >= 10. 314 */ 315 int32_t svt_aom_wiener_denoise_2d(const uint8_t *const data[3], uint8_t *denoised[3], int32_t w, 316 int32_t h, int32_t stride[3], int32_t chroma_sub_log2[2], 317 float *noise_psd[3], int32_t block_size, int32_t bit_depth, 318 int32_t use_highbd); 319 320 struct AomDenoiseAndModel; 321 322 /*!\brief Denoise the buffer and model the residual noise. 323 * 324 * This is meant to be called sequentially on input frames. The input buffer 325 * is denoised and the residual noise is modelled. The current noise estimate 326 * is populated in film_grain. Returns true on success. The grain.apply_grain 327 * parameter will be true when the input buffer was successfully denoised and 328 * grain was modelled. Returns false on error. 329 * 330 * \param[in] ctx Struct allocated with svt_aom_denoise_and_model_alloc 331 * that holds some buffers for denoising and the current 332 * noise estimate. 333 * \param[in/out] buf The raw input buffer to be denoised. 334 * \param[out] grain Output film grain parameters 335 */ 336 int32_t svt_aom_denoise_and_model_run(struct AomDenoiseAndModel *ctx, EbPictureBufferDesc *sd, 337 AomFilmGrain *film_grain, int32_t use_highbd); 338 339 /*!\brief Allocates a context that can be used for denoising and noise modeling. 340 * 341 * \param[in] bit_depth Bit depth of buffers this will be run on. 342 * \param[in] block_size Block size for noise modeling and flat block 343 * estimation 344 * \param[in] noise_level The noise_level (2.5 for moderate noise, and 5 for 345 * higher levels of noise) 346 */ 347 348 /*!\brief Frees the denoise context allocated with svt_aom_denoise_and_model_alloc 349 */ 350 void aom_denoise_and_model_free(struct AomDenoiseAndModel *denoise_model, int32_t use_highbd); 351 352 int32_t is_ref_noise_model_different(AomNoiseModel *const noise_model, 353 AomNoiseModel *const ref_noise_model); 354 355 #ifdef __cplusplus 356 } // extern "C" 357 #endif // __cplusplus 358 #endif // AOM_AOM_DSP_NOISE_MODEL_H_ 359