1 /*
2 * Copyright (c) 2021-2022 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "src/core/helpers/ScaleHelpers.h"
25 #include "src/cpu/kernels/scale/neon/list.h"
26
27 namespace arm_compute
28 {
29 namespace
30 {
qasymm8_signed_neon_scale_bilinear(const ITensor * src,ITensor * dst,const ITensor * offsets,const ITensor * dx,const ITensor * dy,BorderMode border_mode,PixelValue constant_border_value,float sampling_offset,bool align_corners,const Window & window)31 void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
32 BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
33 bool align_corners, const Window &window)
34 {
35 // Data layout is NHWC
36 const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform();
37 const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform();
38
39 const int32_t input_width = src->info()->dimension(1);
40 const int32_t input_height = src->info()->dimension(2);
41
42 // Compute the ratio between source and destination dimensions
43 const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners);
44 const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
45
46 if(border_mode == BorderMode::CONSTANT)
47 {
48 const int32_t in_stride_y = src->info()->strides_in_bytes()[1];
49 const int32_t in_stride_z = src->info()->strides_in_bytes()[2];
50
51 Window win_off;
52 win_off.set(Window::DimX, Window::Dimension(0, 0, 0));
53 win_off.set(Window::DimY, Window::Dimension(0, 0, 0));
54
55 // Don't increment in X and Y direction for the input tensor
56 // A pointer to the start of this plane is needed as base for the precomputed offsets
57 Window win_in(window);
58 win_in.set(1, Window::Dimension(0, 0, 0));
59 win_in.set(2, Window::Dimension(0, 0, 0));
60
61 for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d)
62 {
63 win_off.set(d, Window::Dimension(0, 0, 0));
64 }
65
66 Iterator in(src, win_in);
67 Iterator out(dst, window);
68
69 const int8_t const_border_value = static_cast<int8_t>(constant_border_value.get<int8_t>());
70 execute_window_loop(window, [&](const Coordinates & id)
71 {
72 const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset);
73 const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2]))));
74 const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2]))));
75 const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2]))));
76 const auto pixel_row_ptr = reinterpret_cast<const int8_t *>(in.ptr());
77
78 const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) ?
79 (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) :
80 const_border_value;
81 const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) ?
82 (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) :
83 const_border_value;
84 const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) ?
85 (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) :
86 const_border_value;
87 const auto a11 = (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) ?
88 (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) :
89 const_border_value;
90
91 const float inp00 = Qasymm8QuantizationHelper<int8_t>::dequantize(a00, iq_info);
92 const float inp01 = Qasymm8QuantizationHelper<int8_t>::dequantize(a01, iq_info);
93 const float inp10 = Qasymm8QuantizationHelper<int8_t>::dequantize(a10, iq_info);
94 const float inp11 = Qasymm8QuantizationHelper<int8_t>::dequantize(a11, iq_info);
95 *reinterpret_cast<int8_t *>(out.ptr()) = Qasymm8QuantizationHelper<int8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info);
96 },
97 in, out);
98 }
99 else if(border_mode == BorderMode::REPLICATE)
100 {
101 using FloatTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>;
102 using Int32TagType = typename wrapper::traits::neon_bitvector_tag_t<int32_t, wrapper::traits::BitWidth::W128>;
103
104 const int in_stride_x = src->info()->strides_in_bytes()[1];
105 const int in_stride_y = src->info()->strides_in_bytes()[2];
106 const int in_stride_b = src->info()->strides_in_bytes()[3];
107 const int out_stride_x = dst->info()->strides_in_bytes()[1];
108 const int out_stride_y = dst->info()->strides_in_bytes()[2];
109 const int out_stride_b = dst->info()->strides_in_bytes()[3];
110 const int out_dim_ch = dst->info()->dimension(0);
111 constexpr int step_cout = 16;
112
113 Window window_execution = window;
114 window_execution.set(Window::DimX, Window::Dimension(0, 1, 1));
115 Window win_in_out(window);
116 win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0));
117 win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0));
118 Iterator in(src, win_in_out);
119 Iterator out(dst, win_in_out);
120
121 const int xo_start = window_execution[1].start();
122 const int xo_end = window_execution[1].end();
123 const int xo_step = window_execution[1].step();
124 const int yo_start = window_execution[2].start();
125 const int yo_end = window_execution[2].end();
126 const int yo_step = window_execution[2].step();
127 const int bo_start = window_execution[3].start();
128 const int bo_end = window_execution[3].end();
129 const int bo_step = window_execution[3].step();
130
131 const float fp_coord_offset_y = sampling_offset * (scale_y - 1);
132 const float fp_coord_offset_x = sampling_offset * (scale_x - 1);
133
134 const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform();
135 const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform();
136
137 const float32x4_t vscale_in = wrapper::vdup_n(iq_info.scale, FloatTagType{});
138 const int32x4_t voffset_in = wrapper::vdup_n(iq_info.offset, Int32TagType{}); // Offsets will be Int32
139
140 const float32x4_t invvscale_o = wrapper::vdup_n(1.f / oq_info.scale, FloatTagType{});
141 const float32x4_t voffset_o = vdupq_n_f32(oq_info.offset);
142
143 for(int bo = bo_start; bo < bo_end; bo += bo_step)
144 {
145 const int8_t *in_ptr = reinterpret_cast<int8_t *>(in.ptr() + bo * in_stride_b);
146 int8_t *out_ptr = reinterpret_cast<int8_t *>(out.ptr() + bo * out_stride_b);
147
148 for(int yo = yo_start; yo < yo_end; yo += yo_step)
149 {
150 // Floating-point coordinate
151 const float yi_f = yo * scale_y + fp_coord_offset_y;
152 // Integer coordinate
153 const int yi = static_cast<int>(std::floor(yi_f));
154 // Weight for the y coordinate
155 const float a1 = (yi_f - static_cast<float>(yi));
156 const float b1 = (1.f - a1);
157
158 const int yi0 = utility::clamp<int>(yi, 0, input_height - 1);
159 const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1);
160
161 const int8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y;
162 const int8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y;
163
164 int8_t *out_ptr_yo = out_ptr + yo * out_stride_y;
165 for(int xo = xo_start; xo < xo_end; xo += xo_step)
166 {
167 // Floating-point coordinate
168 const float xi_f = xo * scale_x + fp_coord_offset_x;
169 // Integer coordinate
170 const int xi = static_cast<int>(std::floor(xi_f));
171 // Weight for the x coordinate
172 const float a = (xi_f - static_cast<float>(xi));
173 const float b = (1.f - a);
174
175 const float s00_s = b * b1;
176 const float s01_s = a * b1;
177 const float s10_s = b * a1;
178 const float s11_s = a * a1;
179
180 const auto s00 = wrapper::vdup_n(s00_s, FloatTagType{});
181 const auto s01 = wrapper::vdup_n(s01_s, FloatTagType{});
182 const auto s10 = wrapper::vdup_n(s10_s, FloatTagType{});
183 const auto s11 = wrapper::vdup_n(s11_s, FloatTagType{});
184
185 const int xi0 = utility::clamp<int>(xi, 0, input_width - 1);
186 const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1);
187
188 const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x;
189 const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x;
190 const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x;
191 const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x;
192
193 int8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x;
194
195 int cout = 0;
196 for(; cout <= (out_dim_ch - step_cout); cout += step_cout)
197 {
198 const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(int8_t));
199 const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(int8_t));
200 const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(int8_t));
201 const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(int8_t));
202
203 const int16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00));
204 const int16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00));
205
206 const auto in00_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in00_low)), voffset_in)), vscale_in);
207 const auto in00_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in00_low)), voffset_in)), vscale_in);
208 const auto in00_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in00_high)), voffset_in)), vscale_in);
209 const auto in00_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in00_high)), voffset_in)), vscale_in);
210
211 const int16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01));
212 const int16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01));
213
214 const auto in01_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in01_low)), voffset_in)), vscale_in);
215 const auto in01_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in01_low)), voffset_in)), vscale_in);
216 const auto in01_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in01_high)), voffset_in)), vscale_in);
217 const auto in01_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in01_high)), voffset_in)), vscale_in);
218
219 const int16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10));
220 const int16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10));
221
222 const auto in10_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in10_low)), voffset_in)), vscale_in);
223 const auto in10_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in10_low)), voffset_in)), vscale_in);
224 const auto in10_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in10_high)), voffset_in)), vscale_in);
225 const auto in10_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in10_high)), voffset_in)), vscale_in);
226
227 const int16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11));
228 const int16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11));
229
230 const auto in11_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in11_low)), voffset_in)), vscale_in);
231 const auto in11_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in11_low)), voffset_in)), vscale_in);
232 const auto in11_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in11_high)), voffset_in)), vscale_in);
233 const auto in11_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in11_high)), voffset_in)), vscale_in);
234
235 auto out_0 = wrapper::vmul(in00_0, s00);
236 out_0 = wrapper::vmla(out_0, in01_0, s01);
237 out_0 = wrapper::vmla(out_0, in10_0, s10);
238 out_0 = wrapper::vmla(out_0, in11_0, s11);
239
240 auto out_1 = wrapper::vmul(in00_1, s00);
241 out_1 = wrapper::vmla(out_1, in01_1, s01);
242 out_1 = wrapper::vmla(out_1, in10_1, s10);
243 out_1 = wrapper::vmla(out_1, in11_1, s11);
244
245 auto out_2 = wrapper::vmul(in00_2, s00);
246 out_2 = wrapper::vmla(out_2, in01_2, s01);
247 out_2 = wrapper::vmla(out_2, in10_2, s10);
248 out_2 = wrapper::vmla(out_2, in11_2, s11);
249
250 auto out_3 = wrapper::vmul(in00_3, s00);
251 out_3 = wrapper::vmla(out_3, in01_3, s01);
252 out_3 = wrapper::vmla(out_3, in10_3, s10);
253 out_3 = wrapper::vmla(out_3, in11_3, s11);
254
255 #if defined(__aarch64__) && !defined(BARE_METAL)
256 const auto out_0_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o));
257 const auto out_1_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o));
258 const auto out_2_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o));
259 const auto out_3_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o));
260 #else // defined(__aarch64__) && !defined(BARE_METAL)
261 const auto out_0_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o));
262 const auto out_1_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o));
263 const auto out_2_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o));
264 const auto out_3_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o));
265 #endif // defined(__aarch64__) && !defined(BARE_METAL)
266 const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int)));
267 const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int)));
268 const auto out = wrapper::vcombine(low_part, high_part);
269
270 wrapper::vstore(out_ptr_xo_yo + cout * sizeof(int8_t), out);
271 }
272
273 for(; cout < out_dim_ch; ++cout)
274 {
275 const int8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(int8_t));
276 const int8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(int8_t));
277 const int8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(int8_t));
278 const int8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(int8_t));
279
280 const float in00_f = (static_cast<int32_t>(in00) - iq_info.offset) * iq_info.scale;
281 const float in01_f = (static_cast<int32_t>(in01) - iq_info.offset) * iq_info.scale;
282 const float in10_f = (static_cast<int32_t>(in10) - iq_info.offset) * iq_info.scale;
283 const float in11_f = (static_cast<int32_t>(in11) - iq_info.offset) * iq_info.scale;
284
285 float out = in00_f * s00_s;
286 out += in01_f * s01_s;
287 out += in10_f * s10_s;
288 out += in11_f * s11_s;
289
290 // Rounding modes of vector and scalar loops should match
291 #if defined(__aarch64__) && !defined(BARE_METAL)
292 *(out_ptr_xo_yo + cout * sizeof(int8_t)) = quantize_qasymm8_signed(out, oq_info);
293 #else // defined(__aarch64__) && !defined(BARE_METAL)
294 *(out_ptr_xo_yo + cout * sizeof(int8_t)) = quantize_qasymm8_signed(out, oq_info, RoundingPolicy::TO_ZERO);
295 #endif // defined(__aarch64__) && !defined(BARE_METAL)
296 }
297 }
298 }
299 }
300 }
301 else
302 {
303 ARM_COMPUTE_ERROR("Not implemented");
304 }
305 }
306 }
307 namespace cpu
308 {
qasymm8_signed_neon_scale(const ITensor * src,ITensor * dst,const ITensor * offsets,const ITensor * dx,const ITensor * dy,InterpolationPolicy policy,BorderMode border_mode,PixelValue constant_border_value,float sampling_offset,bool align_corners,const Window & window)309 void qasymm8_signed_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
310 InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
311 bool align_corners, const Window &window)
312 {
313 if(policy == InterpolationPolicy::BILINEAR)
314 {
315 if(src->info()->quantization_info() == dst->info()->quantization_info() && border_mode == BorderMode::REPLICATE)
316 {
317 s8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, align_corners, window);
318 }
319 else
320 {
321 qasymm8_signed_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
322 }
323 }
324 else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
325 {
326 nearest_neon_scale<int8_t>(src, dst, offsets, sampling_offset, align_corners, window);
327 }
328 }
329 } // namespace cpu
330 } // namespace arm_compute