xref: /aosp_15_r20/external/ComputeLibrary/src/cpu/kernels/CpuAddKernel.cpp (revision c217d954acce2dbc11938adb493fc0abd69584f3) 
1 /*
2  * Copyright (c) 2021-2022 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
24 #include "src/cpu/kernels/CpuAddKernel.h"
25 
26 #include "arm_compute/core/ITensor.h"
27 #include "arm_compute/core/TensorInfo.h"
28 #include "arm_compute/core/Validate.h"
29 #include "src/core/CPP/Validate.h"
30 #include "src/core/common/Registrars.h"
31 #include "src/core/helpers/AutoConfiguration.h"
32 #include "src/core/helpers/WindowHelpers.h"
33 #include "src/cpu/kernels/add/list.h"
34 #include <array>
35 
36 #if defined(ENABLE_FP32_KERNELS)
37 namespace
38 {
39     static constexpr size_t default_mws_N1_fp32_neon = 24536;
40     static constexpr size_t default_mws_V1_fp32_neon = 40510;
41 }
42 #endif /* ENABLE_FP32_KERNELS */
43 
44 namespace arm_compute
45 {
46 namespace cpu
47 {
48 namespace kernels
49 {
50 namespace
51 {
52 static const std::vector<CpuAddKernel::AddKernel> available_kernels =
53 {
54     {
55         "neon_qu8_add_fixedpoint",
56         [](const CpuAddKernelDataTypeISASelectorData & data)
__anon86e7f2dc0302() 57         {
58             return (data.dt == DataType::QASYMM8) && data.can_use_fixedpoint;
59         },
60         REGISTER_FP32_NEON(arm_compute::cpu::add_q8_neon_fixedpoint<uint8_t>)
61     },
62     {
63         "neon_qs8_add_fixedpoint",
64         [](const CpuAddKernelDataTypeISASelectorData & data)
__anon86e7f2dc0402() 65         {
66             return (data.dt == DataType::QASYMM8_SIGNED) && data.can_use_fixedpoint;
67         },
68         REGISTER_FP32_NEON(arm_compute::cpu::add_q8_neon_fixedpoint<int8_t>)
69     },
70     {
71         "sve2_qu8_add",
72         [](const CpuAddKernelDataTypeISASelectorData & data)
__anon86e7f2dc0502() 73         {
74             return (data.dt == DataType::QASYMM8) && data.isa.sve2;
75         },
76         REGISTER_QASYMM8_SVE2(arm_compute::cpu::add_qasymm8_sve2)
77     },
78     {
79         "sve2_qs8_add",
80         [](const CpuAddKernelDataTypeISASelectorData & data)
__anon86e7f2dc0602() 81         {
82             return (data.dt == DataType::QASYMM8_SIGNED) && data.isa.sve2;
83         },
84         REGISTER_QASYMM8_SIGNED_SVE2(arm_compute::cpu::add_qasymm8_signed_sve2)
85     },
86     {
87         "sve2_qs16_add",
88         [](const CpuAddKernelDataTypeISASelectorData & data)
__anon86e7f2dc0702() 89         {
90             return (data.dt == DataType::QSYMM16) && data.isa.sve2;
91         },
92         REGISTER_QSYMM16_SVE2(arm_compute::cpu::add_qsymm16_sve2)
93     },
94     {
95         "sve_fp32_add",
96         [](const CpuAddKernelDataTypeISASelectorData & data)
__anon86e7f2dc0802() 97         {
98             return (data.dt == DataType::F32) && data.isa.sve;
99         },
100         REGISTER_FP32_SVE(arm_compute::cpu::add_fp32_sve)
101     },
102     {
103         "sve_fp16_add",
104         [](const CpuAddKernelDataTypeISASelectorData & data)
__anon86e7f2dc0902() 105         {
106             return (data.dt == DataType::F16) && data.isa.sve && data.isa.fp16;
107         },
108         REGISTER_FP16_SVE(arm_compute::cpu::add_fp16_sve)
109     },
110     {
111         "sve_u8_add",
112         [](const CpuAddKernelDataTypeISASelectorData & data)
__anon86e7f2dc0a02() 113         {
114             return (data.dt == DataType::U8) && data.isa.sve;
115         },
116         REGISTER_INTEGER_SVE(arm_compute::cpu::add_u8_sve)
117     },
118     {
119         "sve_s16_add",
120         [](const CpuAddKernelDataTypeISASelectorData & data)
__anon86e7f2dc0b02() 121         {
122             return (data.dt == DataType::S16) && data.isa.sve;
123         },
124         REGISTER_INTEGER_SVE(arm_compute::cpu::add_s16_sve)
125     },
126     {
127         "sve_s32_add",
128         [](const CpuAddKernelDataTypeISASelectorData & data)
__anon86e7f2dc0c02() 129         {
130             return (data.dt == DataType::S32) && data.isa.sve;
131         },
132         REGISTER_INTEGER_SVE(arm_compute::cpu::add_s32_sve)
133     },
134     {
135         "neon_fp32_add",
__anon86e7f2dc0d02() 136         [](const CpuAddKernelDataTypeISASelectorData & data) { return (data.dt == DataType::F32); },
137         REGISTER_FP32_NEON(arm_compute::cpu::add_fp32_neon)
138     },
139     {
140         "neon_fp16_add",
141         [](const CpuAddKernelDataTypeISASelectorData & data)
__anon86e7f2dc0e02() 142         {
143             return (data.dt == DataType::F16) && data.isa.fp16;
144         },
145         REGISTER_FP16_NEON(arm_compute::cpu::add_fp16_neon)
146     },
147     {
148         "neon_u8_add",
__anon86e7f2dc0f02() 149         [](const CpuAddKernelDataTypeISASelectorData & data) { return (data.dt == DataType::U8); },
150         REGISTER_INTEGER_NEON(arm_compute::cpu::add_u8_neon)
151     },
152     {
153         "neon_s16_add",
__anon86e7f2dc1002() 154         [](const CpuAddKernelDataTypeISASelectorData & data) { return (data.dt == DataType::S16); },
155         REGISTER_INTEGER_NEON(arm_compute::cpu::add_s16_neon)
156     },
157     {
158         "neon_s32_add",
__anon86e7f2dc1102() 159         [](const CpuAddKernelDataTypeISASelectorData & data) { return (data.dt == DataType::S32); },
160         REGISTER_INTEGER_NEON(arm_compute::cpu::add_s32_neon)
161     },
162     {
163         "neon_qu8_add",
__anon86e7f2dc1202() 164         [](const CpuAddKernelDataTypeISASelectorData & data) { return (data.dt == DataType::QASYMM8); },
165         REGISTER_QASYMM8_NEON(arm_compute::cpu::add_qasymm8_neon)
166     },
167     {
168         "neon_qs8_add",
__anon86e7f2dc1302() 169         [](const CpuAddKernelDataTypeISASelectorData & data) { return (data.dt == DataType::QASYMM8_SIGNED); },
170         REGISTER_QASYMM8_SIGNED_NEON(arm_compute::cpu::add_qasymm8_signed_neon)
171     },
172     {
173         "neon_qs16_add",
__anon86e7f2dc1402() 174         [](const CpuAddKernelDataTypeISASelectorData & data) { return (data.dt == DataType::QSYMM16); },
175         REGISTER_QSYMM16_NEON(arm_compute::cpu::add_qsymm16_neon)
176     }
177 };
178 
validate_arguments(const ITensorInfo & src0,const ITensorInfo & src1,const ITensorInfo & dst,ConvertPolicy policy)179 Status validate_arguments(const ITensorInfo &src0, const ITensorInfo &src1, const ITensorInfo &dst, ConvertPolicy policy)
180 {
181     ARM_COMPUTE_UNUSED(policy);
182 
183     ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(&src0);
184     ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(&src0, 1, DataType::U8, DataType::QASYMM8, DataType::QASYMM8_SIGNED,
185                                                          DataType::S16, DataType::QSYMM16, DataType::F16,
186                                                          DataType::S32, DataType::F32);
187     ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(&src0, &src1);
188 
189     const TensorShape out_shape = TensorShape::broadcast_shape(src0.tensor_shape(), src1.tensor_shape());
190 
191     ARM_COMPUTE_RETURN_ERROR_ON_MSG(out_shape.total_size() == 0, "Inputs are not broadcast compatible");
192     ARM_COMPUTE_RETURN_ERROR_ON_MSG((src0.tensor_shape().x() != src1.tensor_shape().x()) && ((src0.data_type() != src1.data_type()) || (src0.data_type() != dst.data_type())
193                                                                                              || (src1.data_type() != dst.data_type())),
194                                     "Broadcasting across width is supported on configurations where all tensors have the same data type");
195 
196     // Validate in case of configured dst
197     if(dst.total_size() > 0)
198     {
199         ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(&src0, &dst);
200         ARM_COMPUTE_RETURN_ERROR_ON_MSG(detail::have_different_dimensions(out_shape, dst.tensor_shape(), 0),
201                                         "Wrong shape for dst");
202     }
203 
204     const auto can_use_fixedpoint = add_q8_neon_fixedpoint_possible(&src0, &src1, &dst);
205     const auto uk = CpuAddKernel::get_implementation<CpuAddKernelDataTypeISASelectorData>(CpuAddKernelDataTypeISASelectorData{ src0.data_type(),
206                                                                                           CPUInfo::get().get_isa(), can_use_fixedpoint });
207     ARM_COMPUTE_RETURN_ERROR_ON(uk == nullptr || uk->ukernel == nullptr);
208 
209     return Status{};
210 }
211 } // namespace
212 
configure(const ITensorInfo * src0,const ITensorInfo * src1,ITensorInfo * dst,ConvertPolicy policy)213 void CpuAddKernel::configure(const ITensorInfo *src0, const ITensorInfo *src1, ITensorInfo *dst, ConvertPolicy policy)
214 {
215     ARM_COMPUTE_ERROR_ON_NULLPTR(src0, src1, dst);
216     ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(*src0, *src1, *dst, policy));
217 
218     const auto can_use_fixedpoint     = add_q8_neon_fixedpoint_possible(src0, src1, dst);
219     const auto uk                     = CpuAddKernel::get_implementation<CpuAddKernelDataTypeISASelectorData>(CpuAddKernelDataTypeISASelectorData{ src0->data_type(),
220                                                                                                               CPUInfo::get().get_isa(), can_use_fixedpoint });
221 
222     ARM_COMPUTE_ERROR_ON_NULLPTR(uk);
223 
224     _policy     = policy;
225     _run_method = uk->ukernel;
226     _name       = std::string("CpuAddKernel").append("/").append(uk->name);
227 
228     // Auto initialize dst if not initialized
229     const TensorShape &out_shape = TensorShape::broadcast_shape(src0->tensor_shape(), src1->tensor_shape());
230     set_shape_if_empty(*dst, out_shape);
231     set_data_type_if_unknown(*dst, src0->data_type());
232 
233     // Configure kernel window
234     Window win;
235     std::tie(win, _split_dimension) = calculate_squashed_or_max_window(*src0, *src1);
236 
237     ICpuKernel::configure(win);
238 }
239 
validate(const ITensorInfo * src0,const ITensorInfo * src1,const ITensorInfo * dst,ConvertPolicy policy)240 Status CpuAddKernel::validate(const ITensorInfo *src0, const ITensorInfo *src1, const ITensorInfo *dst, ConvertPolicy policy)
241 {
242     ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(src0, src1, dst);
243 
244     ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(*src0, *src1, *dst, policy));
245 
246     return Status{};
247 }
248 
run_op(ITensorPack & tensors,const Window & window,const ThreadInfo & info)249 void CpuAddKernel::run_op(ITensorPack &tensors, const Window &window, const ThreadInfo &info)
250 {
251     ARM_COMPUTE_UNUSED(info);
252     ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
253     ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICpuKernel::window(), window);
254 
255     ARM_COMPUTE_ERROR_ON(tensors.empty());
256     ARM_COMPUTE_ERROR_ON(_run_method == nullptr);
257 
258     const ITensor *src0 = tensors.get_const_tensor(TensorType::ACL_SRC_0);
259     const ITensor *src1 = tensors.get_const_tensor(TensorType::ACL_SRC_1);
260     ITensor       *dst  = tensors.get_tensor(TensorType::ACL_DST);
261 
262     _run_method(src0, src1, dst, _policy, window);
263 }
264 
name() const265 const char *CpuAddKernel::name() const
266 {
267     return _name.c_str();
268 }
269 
get_available_kernels()270 const std::vector<CpuAddKernel::AddKernel> &CpuAddKernel::get_available_kernels()
271 {
272     return available_kernels;
273 }
274 
get_mws(const CPUInfo & platform,size_t thread_count) const275 size_t CpuAddKernel::get_mws(const CPUInfo &platform, size_t thread_count) const
276 {
277     ARM_COMPUTE_UNUSED(thread_count);
278 
279 #if defined(ENABLE_FP32_KERNELS)
280     if(this->_run_method == &add_fp32_neon)
281     {
282         size_t mws = ICPPKernel::default_mws;
283         if(platform.get_cpu_model() == CPUModel::N1)
284         {
285             mws = default_mws_N1_fp32_neon;
286         }
287         else if(platform.get_cpu_model() == CPUModel::V1)
288         {
289             mws = default_mws_V1_fp32_neon;
290         }
291         else
292         {
293             return ICPPKernel::default_mws;
294         }
295 
296         // tensor is 1D or was re-interpreted as 1D
297         if(this->window().shape().num_dimensions() == 1)
298         {
299             return mws;
300         }
301         else
302         {
303             // scale mws down by the number of elements along all the dimensions (x, z, w, etc) except the one
304             // that we parallelize along (the y dimension). This allows for parallelization when the Y_SIZE is small
305             // but the other sizes are large, which boosts performance.
306             mws = static_cast<size_t>(mws / (this->window().num_iterations_total() / this->window().num_iterations(1)));
307             return std::max(static_cast<size_t>(1), mws);
308         }
309     }
310 #else /* ENABLE_FP32_KERNELS */
311     ARM_COMPUTE_UNUSED(platform);
312 #endif /* ENABLE_FP32_KERNELS */
313     return ICPPKernel::default_mws;
314 }
315 
316 } // namespace kernels
317 } // namespace cpu
318 } // namespace arm_compute
319