1 // Copyright (c) 2019-2021, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::context::MAX_TX_SIZE;
11 use crate::cpu_features::CpuFeatureLevel;
12 use crate::predict::{
13 rust, IntraEdgeFilterParameters, PredictionMode, PredictionVariant,
14 };
15 use crate::tiling::PlaneRegionMut;
16 use crate::transform::TxSize;
17 use crate::util::Aligned;
18 use crate::{Pixel, PixelType};
19 use libc;
20
21 macro_rules! decl_angular_ipred_fn {
22 ($($f:ident),+) => {
23 extern {
24 $(
25 fn $f(
26 dst: *mut u8, stride: libc::ptrdiff_t, topleft: *const u8,
27 width: libc::c_int, height: libc::c_int, angle: libc::c_int,
28 );
29 )*
30 }
31 };
32 }
33
34 decl_angular_ipred_fn! {
35 rav1e_ipred_dc_8bpc_neon,
36 rav1e_ipred_dc_128_8bpc_neon,
37 rav1e_ipred_dc_left_8bpc_neon,
38 rav1e_ipred_dc_top_8bpc_neon,
39 rav1e_ipred_v_8bpc_neon,
40 rav1e_ipred_h_8bpc_neon,
41 rav1e_ipred_smooth_8bpc_neon,
42 rav1e_ipred_smooth_v_8bpc_neon,
43 rav1e_ipred_smooth_h_8bpc_neon,
44 rav1e_ipred_paeth_8bpc_neon
45 }
46
47 macro_rules! decl_angular_ipred_hbd_fn {
48 ($($f:ident),+) => {
49 extern {
50 $(
51 fn $f(
52 dst: *mut u16, stride: libc::ptrdiff_t, topleft: *const u16,
53 width: libc::c_int, height: libc::c_int, angle: libc::c_int,
54 max_width: libc::c_int, max_height: libc::c_int,
55 bit_depth_max: libc::c_int,
56 );
57 )*
58 }
59 };
60 }
61
62 decl_angular_ipred_hbd_fn! {
63 rav1e_ipred_dc_16bpc_neon,
64 rav1e_ipred_dc_128_16bpc_neon,
65 rav1e_ipred_dc_left_16bpc_neon,
66 rav1e_ipred_dc_top_16bpc_neon,
67 rav1e_ipred_v_16bpc_neon,
68 rav1e_ipred_h_16bpc_neon,
69 rav1e_ipred_smooth_16bpc_neon,
70 rav1e_ipred_smooth_v_16bpc_neon,
71 rav1e_ipred_smooth_h_16bpc_neon,
72 rav1e_ipred_paeth_16bpc_neon
73 }
74
75 macro_rules! decl_cfl_pred_fn {
76 ($($f:ident),+) => {
77 extern {
78 $(
79 fn $f(
80 dst: *mut u8, stride: libc::ptrdiff_t, topleft: *const u8,
81 width: libc::c_int, height: libc::c_int, ac: *const i16,
82 alpha: libc::c_int,
83 );
84 )*
85 }
86 };
87 }
88
89 decl_cfl_pred_fn! {
90 rav1e_ipred_cfl_8bpc_neon,
91 rav1e_ipred_cfl_128_8bpc_neon,
92 rav1e_ipred_cfl_left_8bpc_neon,
93 rav1e_ipred_cfl_top_8bpc_neon
94 }
95
96 macro_rules! decl_cfl_pred_hbd_fn {
97 ($($f:ident),+) => {
98 extern {
99 $(
100 fn $f(
101 dst: *mut u16, stride: libc::ptrdiff_t, topleft: *const u16,
102 width: libc::c_int, height: libc::c_int, ac: *const i16,
103 alpha: libc::c_int, bit_depth_max: libc::c_int,
104 );
105 )*
106 }
107 };
108 }
109
110 decl_cfl_pred_hbd_fn! {
111 rav1e_ipred_cfl_16bpc_neon,
112 rav1e_ipred_cfl_128_16bpc_neon,
113 rav1e_ipred_cfl_left_16bpc_neon,
114 rav1e_ipred_cfl_top_16bpc_neon
115 }
116
117 #[inline(always)]
dispatch_predict_intra<T: Pixel>( mode: PredictionMode, variant: PredictionVariant, dst: &mut PlaneRegionMut<'_, T>, tx_size: TxSize, bit_depth: usize, ac: &[i16], angle: isize, ief_params: Option<IntraEdgeFilterParameters>, edge_buf: &Aligned<[T; 4 * MAX_TX_SIZE + 1]>, cpu: CpuFeatureLevel, )118 pub fn dispatch_predict_intra<T: Pixel>(
119 mode: PredictionMode, variant: PredictionVariant,
120 dst: &mut PlaneRegionMut<'_, T>, tx_size: TxSize, bit_depth: usize,
121 ac: &[i16], angle: isize, ief_params: Option<IntraEdgeFilterParameters>,
122 edge_buf: &Aligned<[T; 4 * MAX_TX_SIZE + 1]>, cpu: CpuFeatureLevel,
123 ) {
124 let call_rust = |dst: &mut PlaneRegionMut<'_, T>| {
125 rust::dispatch_predict_intra(
126 mode, variant, dst, tx_size, bit_depth, ac, angle, ief_params, edge_buf,
127 cpu,
128 );
129 };
130
131 if cpu < CpuFeatureLevel::NEON {
132 return call_rust(dst);
133 }
134
135 unsafe {
136 let dst_ptr = dst.data_ptr_mut() as *mut _;
137 let dst_u16 = dst.data_ptr_mut() as *mut u16;
138 let stride = T::to_asm_stride(dst.plane_cfg.stride) as libc::ptrdiff_t;
139 let edge_ptr =
140 edge_buf.data.as_ptr().offset(2 * MAX_TX_SIZE as isize) as *const _;
141 let edge_u16 =
142 edge_buf.data.as_ptr().offset(2 * MAX_TX_SIZE as isize) as *const u16;
143 let w = tx_size.width() as libc::c_int;
144 let h = tx_size.height() as libc::c_int;
145 let angle = angle as libc::c_int;
146 let bd_max = (1 << bit_depth) - 1;
147 match T::type_enum() {
148 PixelType::U8 => match mode {
149 PredictionMode::DC_PRED => {
150 (match variant {
151 PredictionVariant::NONE => rav1e_ipred_dc_128_8bpc_neon,
152 PredictionVariant::LEFT => rav1e_ipred_dc_left_8bpc_neon,
153 PredictionVariant::TOP => rav1e_ipred_dc_top_8bpc_neon,
154 PredictionVariant::BOTH => rav1e_ipred_dc_8bpc_neon,
155 })(dst_ptr, stride, edge_ptr, w, h, angle);
156 }
157 PredictionMode::V_PRED if angle == 90 => {
158 rav1e_ipred_v_8bpc_neon(dst_ptr, stride, edge_ptr, w, h, angle);
159 }
160 PredictionMode::H_PRED if angle == 180 => {
161 rav1e_ipred_h_8bpc_neon(dst_ptr, stride, edge_ptr, w, h, angle);
162 }
163 PredictionMode::SMOOTH_PRED => {
164 rav1e_ipred_smooth_8bpc_neon(dst_ptr, stride, edge_ptr, w, h, angle);
165 }
166 PredictionMode::SMOOTH_V_PRED => {
167 rav1e_ipred_smooth_v_8bpc_neon(
168 dst_ptr, stride, edge_ptr, w, h, angle,
169 );
170 }
171 PredictionMode::SMOOTH_H_PRED => {
172 rav1e_ipred_smooth_h_8bpc_neon(
173 dst_ptr, stride, edge_ptr, w, h, angle,
174 );
175 }
176 PredictionMode::PAETH_PRED => {
177 rav1e_ipred_paeth_8bpc_neon(dst_ptr, stride, edge_ptr, w, h, angle);
178 }
179 PredictionMode::UV_CFL_PRED => {
180 let ac_ptr = ac.as_ptr() as *const _;
181 (match variant {
182 PredictionVariant::NONE => rav1e_ipred_cfl_128_8bpc_neon,
183 PredictionVariant::LEFT => rav1e_ipred_cfl_left_8bpc_neon,
184 PredictionVariant::TOP => rav1e_ipred_cfl_top_8bpc_neon,
185 PredictionVariant::BOTH => rav1e_ipred_cfl_8bpc_neon,
186 })(dst_ptr, stride, edge_ptr, w, h, ac_ptr, angle);
187 }
188 _ => call_rust(dst),
189 },
190 PixelType::U16 if bit_depth > 8 => match mode {
191 PredictionMode::DC_PRED => {
192 (match variant {
193 PredictionVariant::NONE => rav1e_ipred_dc_128_16bpc_neon,
194 PredictionVariant::LEFT => rav1e_ipred_dc_left_16bpc_neon,
195 PredictionVariant::TOP => rav1e_ipred_dc_top_16bpc_neon,
196 PredictionVariant::BOTH => rav1e_ipred_dc_16bpc_neon,
197 })(dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max);
198 }
199 PredictionMode::V_PRED if angle == 90 => {
200 rav1e_ipred_v_16bpc_neon(
201 dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max,
202 );
203 }
204 PredictionMode::H_PRED if angle == 180 => {
205 rav1e_ipred_h_16bpc_neon(
206 dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max,
207 );
208 }
209 PredictionMode::SMOOTH_PRED => {
210 rav1e_ipred_smooth_16bpc_neon(
211 dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max,
212 );
213 }
214 PredictionMode::SMOOTH_V_PRED => {
215 rav1e_ipred_smooth_v_16bpc_neon(
216 dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max,
217 );
218 }
219 PredictionMode::SMOOTH_H_PRED => {
220 rav1e_ipred_smooth_h_16bpc_neon(
221 dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max,
222 );
223 }
224 PredictionMode::PAETH_PRED => {
225 rav1e_ipred_paeth_16bpc_neon(
226 dst_u16, stride, edge_u16, w, h, angle, 0, 0, bd_max,
227 );
228 }
229 PredictionMode::UV_CFL_PRED => {
230 let ac_ptr = ac.as_ptr() as *const _;
231 (match variant {
232 PredictionVariant::NONE => rav1e_ipred_cfl_128_16bpc_neon,
233 PredictionVariant::LEFT => rav1e_ipred_cfl_left_16bpc_neon,
234 PredictionVariant::TOP => rav1e_ipred_cfl_top_16bpc_neon,
235 PredictionVariant::BOTH => rav1e_ipred_cfl_16bpc_neon,
236 })(dst_u16, stride, edge_u16, w, h, ac_ptr, angle, bd_max);
237 }
238 _ => call_rust(dst),
239 },
240 _ => call_rust(dst),
241 }
242 }
243 }
244