1 /*
2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <memory.h>
15 #include <math.h>
16 #include <assert.h>
17
18 #include "av1/encoder/global_motion.h"
19
20 #include "av1/common/warped_motion.h"
21
22 #include "av1/encoder/segmentation.h"
23 #include "av1/encoder/corner_detect.h"
24 #include "av1/encoder/corner_match.h"
25 #include "av1/encoder/ransac.h"
26
27 #define MAX_CORNERS 4096
28 #define MIN_INLIER_PROB 0.1
29
30 #define MIN_TRANS_THRESH (1 * GM_TRANS_DECODE_FACTOR)
31
32 // Border over which to compute the global motion
33 #define ERRORADV_BORDER 0
34
35 static const double erroradv_tr[] = { 0.65, 0.60, 0.55 };
36 static const double erroradv_prod_tr[] = { 20000, 18000, 16000 };
37
is_enough_erroradvantage(double best_erroradvantage,int params_cost,int erroradv_type)38 int is_enough_erroradvantage(double best_erroradvantage, int params_cost,
39 int erroradv_type) {
40 assert(erroradv_type < GM_ERRORADV_TR_TYPES);
41 return best_erroradvantage < erroradv_tr[erroradv_type] &&
42 best_erroradvantage * params_cost < erroradv_prod_tr[erroradv_type];
43 }
44
convert_to_params(const double * params,int32_t * model)45 static void convert_to_params(const double *params, int32_t *model) {
46 int i;
47 int alpha_present = 0;
48 model[0] = (int32_t)floor(params[0] * (1 << GM_TRANS_PREC_BITS) + 0.5);
49 model[1] = (int32_t)floor(params[1] * (1 << GM_TRANS_PREC_BITS) + 0.5);
50 model[0] = (int32_t)clamp(model[0], GM_TRANS_MIN, GM_TRANS_MAX) *
51 GM_TRANS_DECODE_FACTOR;
52 model[1] = (int32_t)clamp(model[1], GM_TRANS_MIN, GM_TRANS_MAX) *
53 GM_TRANS_DECODE_FACTOR;
54
55 for (i = 2; i < 6; ++i) {
56 const int diag_value = ((i == 2 || i == 5) ? (1 << GM_ALPHA_PREC_BITS) : 0);
57 model[i] = (int32_t)floor(params[i] * (1 << GM_ALPHA_PREC_BITS) + 0.5);
58 model[i] =
59 (int32_t)clamp(model[i] - diag_value, GM_ALPHA_MIN, GM_ALPHA_MAX);
60 alpha_present |= (model[i] != 0);
61 model[i] = (model[i] + diag_value) * GM_ALPHA_DECODE_FACTOR;
62 }
63 for (; i < 8; ++i) {
64 model[i] = (int32_t)floor(params[i] * (1 << GM_ROW3HOMO_PREC_BITS) + 0.5);
65 model[i] = (int32_t)clamp(model[i], GM_ROW3HOMO_MIN, GM_ROW3HOMO_MAX) *
66 GM_ROW3HOMO_DECODE_FACTOR;
67 alpha_present |= (model[i] != 0);
68 }
69
70 if (!alpha_present) {
71 if (abs(model[0]) < MIN_TRANS_THRESH && abs(model[1]) < MIN_TRANS_THRESH) {
72 model[0] = 0;
73 model[1] = 0;
74 }
75 }
76 }
77
convert_model_to_params(const double * params,WarpedMotionParams * model)78 void convert_model_to_params(const double *params, WarpedMotionParams *model) {
79 convert_to_params(params, model->wmmat);
80 model->wmtype = get_gmtype(model);
81 model->invalid = 0;
82 }
83
84 // Adds some offset to a global motion parameter and handles
85 // all of the necessary precision shifts, clamping, and
86 // zero-centering.
add_param_offset(int param_index,int32_t param_value,int32_t offset)87 static int32_t add_param_offset(int param_index, int32_t param_value,
88 int32_t offset) {
89 const int scale_vals[3] = { GM_TRANS_PREC_DIFF, GM_ALPHA_PREC_DIFF,
90 GM_ROW3HOMO_PREC_DIFF };
91 const int clamp_vals[3] = { GM_TRANS_MAX, GM_ALPHA_MAX, GM_ROW3HOMO_MAX };
92 // type of param: 0 - translation, 1 - affine, 2 - homography
93 const int param_type = (param_index < 2 ? 0 : (param_index < 6 ? 1 : 2));
94 const int is_one_centered = (param_index == 2 || param_index == 5);
95
96 // Make parameter zero-centered and offset the shift that was done to make
97 // it compatible with the warped model
98 param_value = (param_value - (is_one_centered << WARPEDMODEL_PREC_BITS)) >>
99 scale_vals[param_type];
100 // Add desired offset to the rescaled/zero-centered parameter
101 param_value += offset;
102 // Clamp the parameter so it does not overflow the number of bits allotted
103 // to it in the bitstream
104 param_value = (int32_t)clamp(param_value, -clamp_vals[param_type],
105 clamp_vals[param_type]);
106 // Rescale the parameter to WARPEDMODEL_PRECISION_BITS so it is compatible
107 // with the warped motion library
108 param_value *= (1 << scale_vals[param_type]);
109
110 // Undo the zero-centering step if necessary
111 return param_value + (is_one_centered << WARPEDMODEL_PREC_BITS);
112 }
113
force_wmtype(WarpedMotionParams * wm,TransformationType wmtype)114 static void force_wmtype(WarpedMotionParams *wm, TransformationType wmtype) {
115 switch (wmtype) {
116 case IDENTITY:
117 wm->wmmat[0] = 0;
118 wm->wmmat[1] = 0;
119 AOM_FALLTHROUGH_INTENDED;
120 case TRANSLATION:
121 wm->wmmat[2] = 1 << WARPEDMODEL_PREC_BITS;
122 wm->wmmat[3] = 0;
123 AOM_FALLTHROUGH_INTENDED;
124 case ROTZOOM:
125 wm->wmmat[4] = -wm->wmmat[3];
126 wm->wmmat[5] = wm->wmmat[2];
127 AOM_FALLTHROUGH_INTENDED;
128 case AFFINE: wm->wmmat[6] = wm->wmmat[7] = 0; break;
129 default: assert(0);
130 }
131 wm->wmtype = wmtype;
132 }
133
refine_integerized_param(WarpedMotionParams * wm,TransformationType wmtype,int use_hbd,int bd,uint8_t * ref,int r_width,int r_height,int r_stride,uint8_t * dst,int d_width,int d_height,int d_stride,int n_refinements,int64_t best_frame_error)134 int64_t refine_integerized_param(WarpedMotionParams *wm,
135 TransformationType wmtype, int use_hbd, int bd,
136 uint8_t *ref, int r_width, int r_height,
137 int r_stride, uint8_t *dst, int d_width,
138 int d_height, int d_stride, int n_refinements,
139 int64_t best_frame_error) {
140 static const int max_trans_model_params[TRANS_TYPES] = { 0, 2, 4, 6 };
141 const int border = ERRORADV_BORDER;
142 int i = 0, p;
143 int n_params = max_trans_model_params[wmtype];
144 int32_t *param_mat = wm->wmmat;
145 int64_t step_error, best_error;
146 int32_t step;
147 int32_t *param;
148 int32_t curr_param;
149 int32_t best_param;
150
151 force_wmtype(wm, wmtype);
152 best_error = av1_warp_error(wm, use_hbd, bd, ref, r_width, r_height, r_stride,
153 dst + border * d_stride + border, border, border,
154 d_width - 2 * border, d_height - 2 * border,
155 d_stride, 0, 0, best_frame_error);
156 best_error = AOMMIN(best_error, best_frame_error);
157 step = 1 << (n_refinements - 1);
158 for (i = 0; i < n_refinements; i++, step >>= 1) {
159 for (p = 0; p < n_params; ++p) {
160 int step_dir = 0;
161 // Skip searches for parameters that are forced to be 0
162 param = param_mat + p;
163 curr_param = *param;
164 best_param = curr_param;
165 // look to the left
166 *param = add_param_offset(p, curr_param, -step);
167 step_error =
168 av1_warp_error(wm, use_hbd, bd, ref, r_width, r_height, r_stride,
169 dst + border * d_stride + border, border, border,
170 d_width - 2 * border, d_height - 2 * border, d_stride,
171 0, 0, best_error);
172 if (step_error < best_error) {
173 best_error = step_error;
174 best_param = *param;
175 step_dir = -1;
176 }
177
178 // look to the right
179 *param = add_param_offset(p, curr_param, step);
180 step_error =
181 av1_warp_error(wm, use_hbd, bd, ref, r_width, r_height, r_stride,
182 dst + border * d_stride + border, border, border,
183 d_width - 2 * border, d_height - 2 * border, d_stride,
184 0, 0, best_error);
185 if (step_error < best_error) {
186 best_error = step_error;
187 best_param = *param;
188 step_dir = 1;
189 }
190 *param = best_param;
191
192 // look to the direction chosen above repeatedly until error increases
193 // for the biggest step size
194 while (step_dir) {
195 *param = add_param_offset(p, best_param, step * step_dir);
196 step_error =
197 av1_warp_error(wm, use_hbd, bd, ref, r_width, r_height, r_stride,
198 dst + border * d_stride + border, border, border,
199 d_width - 2 * border, d_height - 2 * border,
200 d_stride, 0, 0, best_error);
201 if (step_error < best_error) {
202 best_error = step_error;
203 best_param = *param;
204 } else {
205 *param = best_param;
206 step_dir = 0;
207 }
208 }
209 }
210 }
211 force_wmtype(wm, wmtype);
212 wm->wmtype = get_gmtype(wm);
213 return best_error;
214 }
215
get_ransac_type(TransformationType type)216 static INLINE RansacFunc get_ransac_type(TransformationType type) {
217 switch (type) {
218 case AFFINE: return ransac_affine;
219 case ROTZOOM: return ransac_rotzoom;
220 case TRANSLATION: return ransac_translation;
221 default: assert(0); return NULL;
222 }
223 }
224
downconvert_frame(YV12_BUFFER_CONFIG * frm,int bit_depth)225 static unsigned char *downconvert_frame(YV12_BUFFER_CONFIG *frm,
226 int bit_depth) {
227 int i, j;
228 uint16_t *orig_buf = CONVERT_TO_SHORTPTR(frm->y_buffer);
229 uint8_t *buf_8bit = frm->y_buffer_8bit;
230 assert(buf_8bit);
231 if (!frm->buf_8bit_valid) {
232 for (i = 0; i < frm->y_height; ++i) {
233 for (j = 0; j < frm->y_width; ++j) {
234 buf_8bit[i * frm->y_stride + j] =
235 orig_buf[i * frm->y_stride + j] >> (bit_depth - 8);
236 }
237 }
238 frm->buf_8bit_valid = 1;
239 }
240 return buf_8bit;
241 }
242
compute_global_motion_feature_based(TransformationType type,YV12_BUFFER_CONFIG * frm,YV12_BUFFER_CONFIG * ref,int bit_depth,int * num_inliers_by_motion,double * params_by_motion,int num_motions)243 int compute_global_motion_feature_based(TransformationType type,
244 YV12_BUFFER_CONFIG *frm,
245 YV12_BUFFER_CONFIG *ref, int bit_depth,
246 int *num_inliers_by_motion,
247 double *params_by_motion,
248 int num_motions) {
249 int i;
250 int num_frm_corners, num_ref_corners;
251 int num_correspondences;
252 int *correspondences;
253 int frm_corners[2 * MAX_CORNERS], ref_corners[2 * MAX_CORNERS];
254 unsigned char *frm_buffer = frm->y_buffer;
255 unsigned char *ref_buffer = ref->y_buffer;
256 RansacFunc ransac = get_ransac_type(type);
257
258 if (frm->flags & YV12_FLAG_HIGHBITDEPTH) {
259 // The frame buffer is 16-bit, so we need to convert to 8 bits for the
260 // following code. We cache the result until the frame is released.
261 frm_buffer = downconvert_frame(frm, bit_depth);
262 }
263 if (ref->flags & YV12_FLAG_HIGHBITDEPTH) {
264 ref_buffer = downconvert_frame(ref, bit_depth);
265 }
266
267 // compute interest points in images using FAST features
268 num_frm_corners = fast_corner_detect(frm_buffer, frm->y_width, frm->y_height,
269 frm->y_stride, frm_corners, MAX_CORNERS);
270 num_ref_corners = fast_corner_detect(ref_buffer, ref->y_width, ref->y_height,
271 ref->y_stride, ref_corners, MAX_CORNERS);
272
273 // find correspondences between the two images
274 correspondences =
275 (int *)malloc(num_frm_corners * 4 * sizeof(*correspondences));
276 num_correspondences = determine_correspondence(
277 frm_buffer, (int *)frm_corners, num_frm_corners, ref_buffer,
278 (int *)ref_corners, num_ref_corners, frm->y_width, frm->y_height,
279 frm->y_stride, ref->y_stride, correspondences);
280
281 ransac(correspondences, num_correspondences, num_inliers_by_motion,
282 params_by_motion, num_motions);
283
284 free(correspondences);
285
286 // Set num_inliers = 0 for motions with too few inliers so they are ignored.
287 for (i = 0; i < num_motions; ++i) {
288 if (num_inliers_by_motion[i] < MIN_INLIER_PROB * num_correspondences) {
289 num_inliers_by_motion[i] = 0;
290 }
291 }
292
293 // Return true if any one of the motions has inliers.
294 for (i = 0; i < num_motions; ++i) {
295 if (num_inliers_by_motion[i] > 0) return 1;
296 }
297 return 0;
298 }
299