1 /*
2 * Copyright (c) 2021-2022 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "arm_compute/core/Helpers.h"
25 #include "arm_compute/core/ITensor.h"
26 #include "arm_compute/core/Types.h"
27 #include "arm_compute/core/utils/misc/Traits.h"
28 #include "src/core/NEON/wrapper/intrinsics/intrinsics.h"
29 #include "src/core/helpers/WindowHelpers.h"
30 #include "src/cpu/kernels/pool2d/neon/list.h"
31
32 #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS)
33
34 namespace arm_compute
35 {
36 namespace cpu
37 {
38 namespace
39 {
pooling2_f16_maxpool_indices(const ITensor * src,ITensor * dst0,ITensor * dst1,PoolingLayerInfo & pool_info,const Window & window_src,const Window & window)40 void pooling2_f16_maxpool_indices(const ITensor *src, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &pool_info, const Window &window_src, const Window &window)
41 {
42 const int window_start_x = window.x().start();
43 const int window_end_x = window.x().end();
44 const int window_step_x = 8;
45
46 Window window_out = window;
47 window_out.set(Window::DimX, Window::Dimension(0, 1, 1));
48
49 Iterator in(src, window_src);
50 Iterator out(dst0, window_out);
51 Iterator indices(dst1, window_out);
52
53 const int pool_pad_top = pool_info.pad_stride_info.pad_top();
54 const int pool_pad_left = pool_info.pad_stride_info.pad_left();
55
56 int pool_stride_x = 0;
57 int pool_stride_y = 0;
58 std::tie(pool_stride_x, pool_stride_y) = pool_info.pad_stride_info.stride();
59
60 const int pad_right = src->info()->padding().right;
61 const int pad_left = src->info()->padding().left;
62 const int pad_horizontal = pad_right + pad_left;
63 const int in_stride_y = static_cast<int>(src->info()->strides_in_bytes().y());
64 const int in_stride_z = static_cast<int>(src->info()->strides_in_bytes().z());
65
66 execute_window_loop(window_out, [&](const Coordinates & id)
67 {
68 const int idx_width = id.y() * pool_stride_x;
69 const int idx_height = id.z() * pool_stride_y;
70 const int pool_limit_y = pool_pad_top - idx_height;
71 const int pool_limit_x = pool_pad_left - idx_width;
72
73 const int pool_start_y = std::max(0, window_src.z().start() + pool_limit_y);
74 const int pool_start_x = std::max(0, window_src.y().start() + pool_limit_x);
75 const int in_x0_offset = (pool_start_x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y - pool_pad_top) * static_cast<int>(src->info()->strides_in_bytes().z());
76 const int in_x1_offset = (pool_start_x + 1 - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y - pool_pad_top) * static_cast<int>
77 (src->info()->strides_in_bytes().z());
78 const int in_x2_offset = (pool_start_x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y + 1 - pool_pad_top) * static_cast<int>
79 (src->info()->strides_in_bytes().z());
80 const int in_x3_offset = (pool_start_x + 1 - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y + 1 - pool_pad_top) * static_cast<int>
81 (src->info()->strides_in_bytes().z());
82
83 int x_off = window_start_x;
84 for(; x_off <= (window_end_x - window_step_x); x_off += window_step_x)
85 {
86 const auto in_x0_ptr = reinterpret_cast<const float16_t *>(in.ptr() + in_x0_offset) + x_off;
87 const auto in_x1_ptr = reinterpret_cast<const float16_t *>(in.ptr() + in_x1_offset) + x_off;
88 const auto in_x2_ptr = reinterpret_cast<const float16_t *>(in.ptr() + in_x2_offset) + x_off;
89 const auto in_x3_ptr = reinterpret_cast<const float16_t *>(in.ptr() + in_x3_offset) + x_off;
90 const auto v_x0 = vld1q_f16(in_x0_ptr);
91 const auto v_x1 = vld1q_f16(in_x1_ptr);
92 const auto v_x2 = vld1q_f16(in_x2_ptr);
93 const auto v_x3 = vld1q_f16(in_x3_ptr);
94 float16x8_t vres = vmaxq_f16(vmaxq_f16(v_x2, v_x3), vmaxq_f16(v_x0, v_x1));
95 // Store result
96 vst1q_f16(reinterpret_cast<float16_t *>(out.ptr()) + x_off, vres);
97
98 const uint32_t offset_base = offset_no_padding<float16_t>(in.offset(), id, *src->info(), pool_stride_x, pool_stride_y, DataLayout::NHWC);
99 const uint32_t offset_x0 = (uint32_t)offset_base / sizeof(float16_t) + x_off;
100 const uint32_t offset_x1 = (uint32_t)offset_x0 + in_stride_y / sizeof(float16_t) - pad_horizontal;
101 const uint32_t offset_x2 = (uint32_t)offset_x0 + in_stride_z / sizeof(float16_t) - pad_horizontal * src->info()->tensor_shape()[1];
102 const uint32_t offset_x3 = (uint32_t)offset_x2 + in_stride_y / sizeof(float16_t) - pad_horizontal;
103 const uint32x4_t voffset_x0_0 = { offset_x0, offset_x0 + 1, offset_x0 + 2, offset_x0 + 3 };
104 const uint32x4_t voffset_x0_1 = { offset_x0 + 4, offset_x0 + 5, offset_x0 + 6, offset_x0 + 7 };
105 const uint16x8_t voffset_x0 = vcombine_u16(vmovn_u32(voffset_x0_0), vmovn_u32(voffset_x0_1));
106 const uint32x4_t voffset_x1_0 = { offset_x1, offset_x1 + 1, offset_x1 + 2, offset_x1 + 3 };
107 const uint32x4_t voffset_x1_1 = { offset_x1 + 4, offset_x1 + 5, offset_x1 + 6, offset_x1 + 7 };
108 const uint16x8_t voffset_x1 = vcombine_u16(vmovn_u32(voffset_x1_0), vmovn_u32(voffset_x1_1));
109 const uint32x4_t voffset_x2_0 = { offset_x2, offset_x2 + 1, offset_x2 + 2, offset_x2 + 3 };
110 const uint32x4_t voffset_x2_1 = { offset_x2 + 4, offset_x2 + 5, offset_x2 + 6, offset_x2 + 7 };
111 const uint16x8_t voffset_x2 = vcombine_u16(vmovn_u32(voffset_x2_0), vmovn_u32(voffset_x2_1));
112 const uint32x4_t voffset_x3_0 = { offset_x3, offset_x3 + 1, offset_x3 + 2, offset_x3 + 3 };
113 const uint32x4_t voffset_x3_1 = { offset_x3 + 4, offset_x3 + 5, offset_x3 + 6, offset_x3 + 7 };
114 const uint16x8_t voffset_x3 = vcombine_u16(vmovn_u32(voffset_x3_0), vmovn_u32(voffset_x3_1));
115 const uint16x8_t tmp_indices0 = vbslq_u16(vcgeq_f16(v_x0, v_x1), voffset_x0, voffset_x1);
116 const uint16x8_t tmp_indices1 = vbslq_u16(vcgeq_f16(v_x2, v_x3), voffset_x2, voffset_x3);
117 const uint16x8_t tmp_indices2 = vbslq_u16(vcgeq_f16(vmaxq_f16(v_x0, v_x1), vmaxq_f16(v_x2, v_x3)), tmp_indices0, tmp_indices1);
118 const uint32x4_t tmp_indeces3_0 = vmovl_u16(vget_low_u16(tmp_indices2));
119 const uint32x4_t tmp_indeces3_1 = vmovl_u16(vget_high_u16(tmp_indices2));
120 // Store indicies
121 vst1q_u32(reinterpret_cast<uint32_t *>(indices.ptr()) + x_off, tmp_indeces3_0);
122 vst1q_u32(reinterpret_cast<uint32_t *>(indices.ptr() + 16) + x_off, tmp_indeces3_1);
123 }
124
125 // Left-overs loop
126 for(; x_off < window_end_x; ++x_off)
127 {
128 const auto x0 = *(reinterpret_cast<const float16_t *>(in.ptr() + in_x0_offset) + x_off);
129 const auto x1 = *(reinterpret_cast<const float16_t *>(in.ptr() + in_x1_offset) + x_off);
130 const auto x2 = *(reinterpret_cast<const float16_t *>(in.ptr() + in_x2_offset) + x_off);
131 const auto x3 = *(reinterpret_cast<const float16_t *>(in.ptr() + in_x3_offset) + x_off);
132 float16_t res = std::max(std::max(x2, x3), std::max(x0, x1));
133
134 // Store result
135 *(reinterpret_cast<float16_t *>(out.ptr()) + x_off) = res;
136
137 const uint32_t offset_base = offset_no_padding<float16_t>(in.offset(), id, *src->info(), pool_stride_x, pool_stride_y, DataLayout::NHWC);
138 const uint32_t offset_x0 = (uint32_t)offset_base / sizeof(float16_t) + x_off;
139 const uint32_t offset_x1 = (uint32_t)offset_x0 + in_stride_y / sizeof(float16_t) - pad_horizontal;
140 const uint32_t offset_x2 = (uint32_t)offset_x0 + in_stride_z / sizeof(float16_t) - pad_horizontal * src->info()->tensor_shape()[1];
141 const uint32_t offset_x3 = (uint32_t)offset_x2 + in_stride_y / sizeof(float16_t) - pad_horizontal;
142 const uint32_t tmp_idx0 = (x0 >= x1) ? offset_x0 : offset_x1;
143 const uint32_t tmp_idx1 = (x2 >= x3) ? offset_x2 : offset_x3;
144 const uint32_t tmp_idx2 = (std::max(x0, x1) >= std::max(x2, x3)) ? tmp_idx0 : tmp_idx1;
145
146 // Store indices
147 *(reinterpret_cast<uint32_t *>(indices.ptr()) + x_off) = tmp_idx2;
148 }
149 },
150 in, out, indices);
151 }
152 }
153
poolingMxN_fp16_neon_nhwc(const ITensor * src,ITensor * dst0,ITensor * dst1,PoolingLayerInfo & pool_info,const Window & window_src,const Window & window)154 void poolingMxN_fp16_neon_nhwc(const ITensor *src, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &pool_info, const Window &window_src, const Window &window)
155 {
156 if(pool_info.pool_size == Size2D(2, 2) && pool_info.pool_type == PoolingType::MAX && dst1)
157 {
158 pooling2_f16_maxpool_indices(src, dst0, dst1, pool_info, window_src, window);
159 }
160 const int window_start_x = window.x().start();
161 const int window_end_x = window.x().end();
162 const int window_step_x = 8;
163
164 Window window_out = window;
165 window_out.set(Window::DimX, Window::Dimension(0, 1, 1));
166
167 Iterator in(src, window_src);
168 Iterator out(dst0, window_out);
169
170 const int pool_size_x = pool_info.is_global_pooling ? src->info()->tensor_shape().y() : pool_info.pool_size.width;
171 const int pool_size_y = pool_info.is_global_pooling ? src->info()->tensor_shape().z() : pool_info.pool_size.height;
172 const int pool_pad_right = pool_info.pad_stride_info.pad_right();
173 const int pool_pad_top = pool_info.pad_stride_info.pad_top();
174 const int pool_pad_left = pool_info.pad_stride_info.pad_left();
175 const int pool_pad_bottom = pool_info.pad_stride_info.pad_bottom();
176 int pool_stride_x = 0;
177 int pool_stride_y = 0;
178 std::tie(pool_stride_x, pool_stride_y) = pool_info.pad_stride_info.stride();
179 const int upper_bound_w = src->info()->dimension(1) + (pool_info.exclude_padding ? 0 : pool_pad_right);
180 const int upper_bound_h = src->info()->dimension(2) + (pool_info.exclude_padding ? 0 : pool_pad_bottom);
181
182 float16x8_t vres;
183
184 execute_window_loop(window_out, [&](const Coordinates & id)
185 {
186 const int idx_width = id.y() * pool_stride_x;
187 const int idx_height = id.z() * pool_stride_y;
188 const int pool_limit_y = pool_pad_top - idx_height;
189 const int pool_limit_x = pool_pad_left - idx_width;
190
191 const int pool_start_y = std::max(0, window_src.z().start() + pool_limit_y);
192 const int pool_end_y = std::min(pool_size_y, window_src.z().end() + pool_limit_y);
193 const int pool_start_x = std::max(0, window_src.y().start() + pool_limit_x);
194 const int pool_end_x = std::min(pool_size_x, window_src.y().end() + pool_limit_x);
195
196 int x_off = window_start_x;
197 for(; x_off <= (window_end_x - window_step_x); x_off += window_step_x)
198 {
199 if(pool_info.pool_type != PoolingType::MAX)
200 {
201 // Calculate scale
202 const float scale = calculate_avg_scale_pool2d(pool_info.exclude_padding, DataLayout::NHWC, id, pool_size_x, pool_size_y, upper_bound_w, upper_bound_h, pool_pad_left, pool_pad_top, pool_stride_x,
203 pool_stride_y);
204 const float16x8_t scale_v = vdupq_n_f16(scale);
205
206 // Perform pooling
207 vres = vdupq_n_f16(0.0f);
208 for(int y = pool_start_y; y < pool_end_y; ++y)
209 {
210 for(int x = pool_start_x; x < pool_end_x; ++x)
211 {
212 const float16x8_t data = vld1q_f16(reinterpret_cast<const float16_t *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>
213 (src->info()->strides_in_bytes().z())) + x_off);
214
215 // Get power of 2 in case of l2 pooling and accumulate
216 if(pool_info.pool_type == PoolingType::L2)
217 {
218 vres = vaddq_f16(vres, vmulq_f16(data, data));
219 }
220 else
221 {
222 vres = vaddq_f16(vres, data);
223 }
224 }
225 }
226 // Divide by scale
227 vres = vmulq_f16(vres, scale_v);
228 }
229 else
230 {
231 vres = vdupq_n_f16(-std::numeric_limits<float>::infinity());
232
233 for(int y = pool_start_y; y < pool_end_y; ++y)
234 {
235 for(int x = pool_start_x; x < pool_end_x; ++x)
236 {
237 const float16x8_t data = vld1q_f16(reinterpret_cast<const float16_t *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>
238 (src->info()->strides_in_bytes().z())) + x_off);
239 vres = vmaxq_f16(vres, data);
240 }
241 }
242 }
243
244 // Calculate square-root in case of l2 pooling
245 if(pool_info.pool_type == PoolingType::L2)
246 {
247 float16x8_t sqrt_reciprocal = vrsqrteq_f16(vres);
248 vres = vmulq_f16(vres, vmulq_f16(vrsqrtsq_f16(vmulq_f16(vres, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal));
249 }
250
251 // Store result
252 vst1q_f16(reinterpret_cast<float16_t *>(out.ptr()) + x_off, vres);
253 }
254
255 // Left-overs loop
256 for(; x_off < window_end_x; ++x_off)
257 {
258 float16_t res = 0.0f;
259
260 if(pool_info.pool_type != PoolingType::MAX)
261 {
262 // Calculate scale
263 const float16_t scale = calculate_avg_scale_pool2d(pool_info.exclude_padding, DataLayout::NHWC, id, pool_size_x, pool_size_y, upper_bound_w, upper_bound_h, pool_pad_left, pool_pad_top, pool_stride_x,
264 pool_stride_y);
265
266 for(int y = pool_start_y; y < pool_end_y; ++y)
267 {
268 for(int x = pool_start_x; x < pool_end_x; ++x)
269 {
270 const float data = *(reinterpret_cast<const float16_t *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>
271 (src->info()->strides_in_bytes().z())) + x_off);
272
273 // Get power of 2 in case of l2 pooling and accumulate
274 if(pool_info.pool_type == PoolingType::L2)
275 {
276 res += data * data;
277 }
278 else
279 {
280 res += data;
281 }
282 }
283 }
284
285 // Divide by scale
286 res *= scale;
287 }
288 else
289 {
290 res = -std::numeric_limits<float>::infinity();
291 for(int y = pool_start_y; y < pool_end_y; ++y)
292 {
293 for(int x = pool_start_x; x < pool_end_x; ++x)
294 {
295 const float16_t data = *(reinterpret_cast<const float16_t *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>
296 (src->info()->strides_in_bytes().z())) + x_off);
297 res = std::max(res, data);
298 }
299 }
300 }
301
302 // Calculate square-root in case of l2 pooling
303 if(pool_info.pool_type == PoolingType::L2)
304 {
305 res = std::sqrt(res);
306 }
307
308 // Store result
309 *(reinterpret_cast<float16_t *>(out.ptr()) + x_off) = res;
310 }
311 },
312 in, out);
313 }
314 } // namespace cpu
315 } // namespace arm_compute
316
317 #endif /* defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) */