Line data Source code
1 : /*
2 : * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3 : *
4 : * This source code is subject to the terms of the BSD 2 Clause License and
5 : * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 : * was not distributed with this source code in the LICENSE file, you can
7 : * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 : * Media Patent License 1.0 was not distributed with this source code in the
9 : * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 : */
11 :
12 : #include <stdio.h>
13 : #include <stdlib.h>
14 : #include <memory.h>
15 : #include <math.h>
16 : #include <assert.h>
17 :
18 : //#include "config/aom_dsp_rtcd.h"
19 :
20 : #include "global_motion.h"
21 :
22 : //#include "segmentation.h"
23 : #include "corner_detect.h"
24 : #include "corner_match.h"
25 : #include "ransac.h"
26 :
27 : #include "EbWarpedMotion.h"
28 :
29 : #define MIN_INLIER_PROB 0.1
30 :
31 : #define MIN_TRANS_THRESH (1 * GM_TRANS_DECODE_FACTOR)
32 :
33 : // Border over which to compute the global motion
34 : #define ERRORADV_BORDER 0
35 :
36 : // TODO(sarahparker) These need to be retuned for speed 0 and 1 to
37 : // maximize gains from segmented error metric
38 : static const double erroradv_tr[] = { 0.65, 0.60, 0.65 };
39 : static const double erroradv_prod_tr[] = { 20000, 18000, 16000 };
40 :
41 113 : int av1_is_enough_erroradvantage(double best_erroradvantage, int params_cost,
42 : int erroradv_type) {
43 113 : assert(erroradv_type < GM_ERRORADV_TR_TYPES);
44 226 : return best_erroradvantage < erroradv_tr[erroradv_type] &&
45 113 : best_erroradvantage * params_cost < erroradv_prod_tr[erroradv_type];
46 : }
47 :
48 2173 : static void convert_to_params(const double *params, int32_t *model) {
49 : int i;
50 2173 : int alpha_present = 0;
51 2173 : model[0] = (int32_t)floor(params[0] * (1 << GM_TRANS_PREC_BITS) + 0.5);
52 2173 : model[1] = (int32_t)floor(params[1] * (1 << GM_TRANS_PREC_BITS) + 0.5);
53 2173 : model[0] = (int32_t)clamp(model[0], GM_TRANS_MIN, GM_TRANS_MAX) *
54 : GM_TRANS_DECODE_FACTOR;
55 2173 : model[1] = (int32_t)clamp(model[1], GM_TRANS_MIN, GM_TRANS_MAX) *
56 : GM_TRANS_DECODE_FACTOR;
57 :
58 10865 : for (i = 2; i < 6; ++i) {
59 8692 : const int diag_value = ((i == 2 || i == 5) ? (1 << GM_ALPHA_PREC_BITS) : 0);
60 8692 : model[i] = (int32_t)floor(params[i] * (1 << GM_ALPHA_PREC_BITS) + 0.5);
61 17384 : model[i] =
62 8692 : (int32_t)clamp(model[i] - diag_value, GM_ALPHA_MIN, GM_ALPHA_MAX);
63 8692 : alpha_present |= (model[i] != 0);
64 8692 : model[i] = (model[i] + diag_value) * GM_ALPHA_DECODE_FACTOR;
65 : }
66 6519 : for (; i < 8; ++i) {
67 4346 : model[i] = (int32_t)floor(params[i] * (1 << GM_ROW3HOMO_PREC_BITS) + 0.5);
68 4346 : model[i] = (int32_t)clamp(model[i], GM_ROW3HOMO_MIN, GM_ROW3HOMO_MAX) *
69 : GM_ROW3HOMO_DECODE_FACTOR;
70 4346 : alpha_present |= (model[i] != 0);
71 : }
72 :
73 2173 : if (!alpha_present) {
74 0 : if (abs(model[0]) < MIN_TRANS_THRESH && abs(model[1]) < MIN_TRANS_THRESH) {
75 0 : model[0] = 0;
76 0 : model[1] = 0;
77 : }
78 : }
79 2173 : }
80 :
81 :
82 4346 : static INLINE TransformationType get_wmtype(const EbWarpedMotionParams *gm) {
83 4346 : if (gm->wmmat[5] == (1 << WARPEDMODEL_PREC_BITS) && !gm->wmmat[4] &&
84 0 : gm->wmmat[2] == (1 << WARPEDMODEL_PREC_BITS) && !gm->wmmat[3]) {
85 0 : return ((!gm->wmmat[1] && !gm->wmmat[0]) ? IDENTITY : TRANSLATION);
86 : }
87 4346 : if (gm->wmmat[2] == gm->wmmat[5] && gm->wmmat[3] == -gm->wmmat[4])
88 4346 : return ROTZOOM;
89 : else
90 0 : return AFFINE;
91 : }
92 :
93 2173 : void av1_convert_model_to_params(const double *params,
94 : EbWarpedMotionParams *model) {
95 2173 : convert_to_params(params, model->wmmat);
96 2173 : model->wmtype = get_wmtype(model);
97 2173 : model->invalid = 0;
98 2173 : }
99 :
100 : // Adds some offset to a global motion parameter and handles
101 : // all of the necessary precision shifts, clamping, and
102 : // zero-centering.
103 89060 : static int32_t add_param_offset(int param_index, int32_t param_value,
104 : int32_t offset) {
105 89060 : const int scale_vals[3] = { GM_TRANS_PREC_DIFF, GM_ALPHA_PREC_DIFF,
106 : GM_ROW3HOMO_PREC_DIFF };
107 89060 : const int clamp_vals[3] = { GM_TRANS_MAX, GM_ALPHA_MAX, GM_ROW3HOMO_MAX };
108 : // type of param: 0 - translation, 1 - affine, 2 - homography
109 89060 : const int param_type = (param_index < 2 ? 0 : (param_index < 6 ? 1 : 2));
110 89060 : const int is_one_centered = (param_index == 2 || param_index == 5);
111 :
112 : // Make parameter zero-centered and offset the shift that was done to make
113 : // it compatible with the warped model
114 89060 : param_value = (param_value - (is_one_centered << WARPEDMODEL_PREC_BITS)) >>
115 89060 : scale_vals[param_type];
116 : // Add desired offset to the rescaled/zero-centered parameter
117 89060 : param_value += offset;
118 : // Clamp the parameter so it does not overflow the number of bits allotted
119 : // to it in the bitstream
120 89060 : param_value = (int32_t)clamp(param_value, -clamp_vals[param_type],
121 : clamp_vals[param_type]);
122 : // Rescale the parameter to WARPEDMODEL_PRECISION_BITS so it is compatible
123 : // with the warped motion library
124 89060 : param_value *= (1 << scale_vals[param_type]);
125 :
126 : // Undo the zero-centering step if necessary
127 89060 : return param_value + (is_one_centered << WARPEDMODEL_PREC_BITS);
128 : }
129 :
130 4346 : static void force_wmtype(EbWarpedMotionParams *wm, TransformationType wmtype) {
131 4346 : switch (wmtype) {
132 0 : case IDENTITY:
133 0 : wm->wmmat[0] = 0;
134 0 : wm->wmmat[1] = 0;
135 : AOM_FALLTHROUGH_INTENDED;
136 0 : case TRANSLATION:
137 0 : wm->wmmat[2] = 1 << WARPEDMODEL_PREC_BITS;
138 0 : wm->wmmat[3] = 0;
139 : AOM_FALLTHROUGH_INTENDED;
140 4346 : case ROTZOOM:
141 4346 : wm->wmmat[4] = -wm->wmmat[3];
142 4346 : wm->wmmat[5] = wm->wmmat[2];
143 : AOM_FALLTHROUGH_INTENDED;
144 4346 : case AFFINE: wm->wmmat[6] = wm->wmmat[7] = 0; break;
145 0 : default: assert(0);
146 : }
147 4346 : wm->wmtype = wmtype;
148 4346 : }
149 :
150 2173 : int64_t av1_refine_integerized_param(
151 : EbWarpedMotionParams *wm, TransformationType wmtype, int use_hbd, int bd,
152 : uint8_t *ref, int r_width, int r_height, int r_stride, uint8_t *dst,
153 : int d_width, int d_height, int d_stride, int n_refinements,
154 : int64_t best_frame_error) {
155 : static const int max_trans_model_params[TRANS_TYPES] = { 0, 2, 4, 6 };
156 2173 : const int border = ERRORADV_BORDER;
157 2173 : int i = 0, p;
158 2173 : int n_params = max_trans_model_params[wmtype];
159 2173 : int32_t *param_mat = wm->wmmat;
160 : int64_t step_error, best_error;
161 : int32_t step;
162 : int32_t *param;
163 : int32_t curr_param;
164 : int32_t best_param;
165 :
166 2173 : force_wmtype(wm, wmtype);
167 : best_error =
168 2173 : eb_av1_warp_error(wm, use_hbd, bd, ref, r_width, r_height, r_stride,
169 2173 : dst + border * d_stride + border, border, border,
170 2173 : d_width - 2 * border, d_height - 2 * border, d_stride, 0,
171 : 0, best_frame_error);
172 2173 : best_error = AOMMIN(best_error, best_frame_error);
173 2173 : step = 1 << (n_refinements - 1);
174 13038 : for (i = 0; i < n_refinements; i++, step >>= 1) {
175 54325 : for (p = 0; p < n_params; ++p) {
176 43460 : int step_dir = 0;
177 : // Skip searches for parameters that are forced to be 0
178 43460 : param = param_mat + p;
179 43460 : curr_param = *param;
180 43460 : best_param = curr_param;
181 : // look to the left
182 43460 : *param = add_param_offset(p, curr_param, -step);
183 : step_error =
184 43460 : eb_av1_warp_error(wm, use_hbd, bd, ref, r_width, r_height, r_stride,
185 43460 : dst + border * d_stride + border, border, border,
186 43460 : d_width - 2 * border, d_height - 2 * border, d_stride,
187 : 0, 0, best_error);
188 43460 : if (step_error < best_error) {
189 753 : best_error = step_error;
190 753 : best_param = *param;
191 753 : step_dir = -1;
192 : }
193 :
194 : // look to the right
195 43460 : *param = add_param_offset(p, curr_param, step);
196 : step_error =
197 43460 : eb_av1_warp_error(wm, use_hbd, bd, ref, r_width, r_height, r_stride,
198 43460 : dst + border * d_stride + border, border, border,
199 43460 : d_width - 2 * border, d_height - 2 * border, d_stride,
200 : 0, 0, best_error);
201 43460 : if (step_error < best_error) {
202 835 : best_error = step_error;
203 835 : best_param = *param;
204 835 : step_dir = 1;
205 : }
206 43460 : *param = best_param;
207 :
208 : // look to the direction chosen above repeatedly until error increases
209 : // for the biggest step size
210 45600 : while (step_dir) {
211 2140 : *param = add_param_offset(p, best_param, step * step_dir);
212 2140 : step_error = eb_av1_warp_error(
213 : wm, use_hbd, bd, ref, r_width, r_height, r_stride,
214 2140 : dst + border * d_stride + border, border, border,
215 2140 : d_width - 2 * border, d_height - 2 * border, d_stride, 0, 0,
216 : best_error);
217 2140 : if (step_error < best_error) {
218 552 : best_error = step_error;
219 552 : best_param = *param;
220 : } else {
221 1588 : *param = best_param;
222 1588 : step_dir = 0;
223 : }
224 : }
225 : }
226 : }
227 2173 : force_wmtype(wm, wmtype);
228 2173 : wm->wmtype = get_wmtype(wm);
229 2173 : return best_error;
230 : }
231 :
232 :
233 2173 : static void get_inliers_from_indices(MotionModel *params,
234 : int *correspondences) {
235 2173 : int *inliers_tmp = (int *)eb_aom_malloc(2 * MAX_CORNERS * sizeof(*inliers_tmp));
236 2173 : memset(inliers_tmp, 0, 2 * MAX_CORNERS * sizeof(*inliers_tmp));
237 :
238 4715300 : for (int i = 0; i < params->num_inliers; i++) {
239 4713120 : int index = params->inliers[i];
240 4713120 : inliers_tmp[2 * i] = correspondences[4 * index];
241 4713120 : inliers_tmp[2 * i + 1] = correspondences[4 * index + 1];
242 : }
243 2173 : memcpy(params->inliers, inliers_tmp, sizeof(*inliers_tmp) * 2 * MAX_CORNERS);
244 2173 : eb_aom_free(inliers_tmp);
245 2173 : }
246 :
247 :
248 113 : static int compute_global_motion_feature_based(
249 : TransformationType type, unsigned char *frm_buffer, int frm_width,
250 : int frm_height, int frm_stride, int *frm_corners, int num_frm_corners,
251 : uint8_t *ref, int ref_stride, int bit_depth, int *num_inliers_by_motion,
252 : MotionModel *params_by_motion, int num_motions) {
253 : (void)bit_depth;
254 113 : assert(bit_depth == EB_8BIT);
255 : int i;
256 : int num_ref_corners;
257 : int num_correspondences;
258 : int *correspondences;
259 : int ref_corners[2 * MAX_CORNERS];
260 113 : unsigned char *ref_buffer = ref;
261 113 : RansacFunc ransac = av1_get_ransac_type(type);
262 :
263 : num_ref_corners =
264 113 : av1_fast_corner_detect(ref_buffer, frm_width, frm_height,
265 : ref_stride, ref_corners, MAX_CORNERS);
266 :
267 : // find correspondences between the two images
268 : correspondences =
269 113 : (int *)malloc(num_frm_corners * 4 * sizeof(*correspondences));
270 113 : num_correspondences = av1_determine_correspondence(
271 : frm_buffer, (int *)frm_corners, num_frm_corners, ref_buffer,
272 : (int *)ref_corners, num_ref_corners, frm_width, frm_height, frm_stride,
273 : ref_stride, correspondences);
274 :
275 113 : ransac(correspondences, num_correspondences, num_inliers_by_motion,
276 : params_by_motion, num_motions);
277 :
278 : // Set num_inliers = 0 for motions with too few inliers so they are ignored.
279 11413 : for (i = 0; i < num_motions; ++i) {
280 11300 : if (num_inliers_by_motion[i] < MIN_INLIER_PROB * num_correspondences ||
281 : num_correspondences == 0) {
282 9127 : num_inliers_by_motion[i] = 0;
283 : } else {
284 2173 : get_inliers_from_indices(¶ms_by_motion[i], correspondences);
285 : }
286 : }
287 :
288 113 : free(correspondences);
289 :
290 : // Return true if any one of the motions has inliers.
291 113 : for (i = 0; i < num_motions; ++i) {
292 113 : if (num_inliers_by_motion[i] > 0) return 1;
293 : }
294 0 : return 0;
295 : }
296 :
297 :
298 113 : int av1_compute_global_motion(TransformationType type,
299 : unsigned char *frm_buffer, int frm_width,
300 : int frm_height, int frm_stride, int *frm_corners,
301 : int num_frm_corners, uint8_t *ref, int ref_stride,
302 : int bit_depth,
303 : GlobalMotionEstimationType gm_estimation_type,
304 : int *num_inliers_by_motion,
305 : MotionModel *params_by_motion, int num_motions) {
306 113 : switch (gm_estimation_type) {
307 113 : case GLOBAL_MOTION_FEATURE_BASED:
308 113 : return compute_global_motion_feature_based(
309 : type, frm_buffer, frm_width, frm_height, frm_stride, frm_corners,
310 : num_frm_corners, ref, ref_stride, bit_depth, num_inliers_by_motion,
311 : params_by_motion, num_motions);
312 0 : default: assert(0 && "Unknown global motion estimation type");
313 : }
314 : return 0;
315 : }
|
