1 /*
2 * Copyright (c) 2016-2023 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #if defined(ARM_COMPUTE_ENABLE_BF16)
25
26 #include "arm_compute/core/TensorInfo.h"
27 #include "src/core/NEON/wrapper/wrapper.h"
28 #include "src/cpu/kernels/CpuCastKernel.h"
29 #include "src/cpu/kernels/cast/list.h"
30 #include "support/SaturateCast.h"
31
32 namespace arm_compute
33 {
34 namespace cpu
35 {
neon_fp32_to_bfloat16_cast(const ITensor * _src,ITensor * _dst,const ThreadInfo & info,ConvertPolicy _policy,const Window & window)36 void neon_fp32_to_bfloat16_cast(const ITensor *_src, ITensor *_dst, const ThreadInfo &info, ConvertPolicy _policy, const Window &window)
37 {
38 ARM_COMPUTE_UNUSED(info);
39 ARM_COMPUTE_UNUSED(_policy);
40
41 const auto window_start_x = static_cast<int>(window.x().start());
42 const auto window_end_x = static_cast<int>(window.x().end());
43 const int window_step_x = 16;
44
45 ARM_COMPUTE_ERROR_ON_NULLPTR(_src, _dst);
46 ARM_COMPUTE_ERROR_ON(_src == _dst);
47
48 ARM_COMPUTE_ERROR_ON_NULLPTR(_src, _dst);
49
50 Window win{ window };
51 win.set(Window::DimX, Window::Dimension(0, 1, 1));
52
53 Iterator src(_src, win);
54 Iterator dst(_dst, win);
55
56 /* Down-conversion F32 -> BFLOAT16 */
57 execute_window_loop(win, [&](const Coordinates &)
58 {
59 const auto src_ptr = reinterpret_cast<const float *>(src.ptr());
60 const auto dst_ptr = reinterpret_cast<bfloat16 *>(dst.ptr());
61
62 int x = window_start_x;
63 for(; x <= (window_end_x - window_step_x); x += window_step_x)
64 {
65 wrapper::vcvt_bf16_f32(reinterpret_cast<float *>(src.ptr()),
66 reinterpret_cast<uint16_t *>(dst.ptr()));
67 wrapper::vcvt_bf16_f32(reinterpret_cast<float *>(src.ptr()) + 8,
68 reinterpret_cast<uint16_t *>(dst.ptr()) + 8);
69 }
70
71 for(; x < window_end_x; ++x)
72 {
73 *(dst_ptr + x) = *(src_ptr + x);
74 }
75 },
76 src, dst);
77 }
78
neon_bfloat16_to_fp32_cast(const ITensor * _src,ITensor * _dst,const ThreadInfo & info,ConvertPolicy _policy,const Window & window)79 void neon_bfloat16_to_fp32_cast(const ITensor *_src, ITensor *_dst, const ThreadInfo &info, ConvertPolicy _policy, const Window &window)
80 {
81 ARM_COMPUTE_UNUSED(info);
82 ARM_COMPUTE_UNUSED(_policy);
83
84 const auto window_start_x = static_cast<int>(window.x().start());
85 const auto window_end_x = static_cast<int>(window.x().end());
86 const int window_step_x = 16;
87
88 ARM_COMPUTE_ERROR_ON_NULLPTR(_src, _dst);
89 ARM_COMPUTE_ERROR_ON(_src == _dst);
90
91 ARM_COMPUTE_ERROR_ON_NULLPTR(_src, _dst);
92
93 Window win{ window };
94 win.set(Window::DimX, Window::Dimension(0, 1, 1));
95
96 Iterator src(_src, win);
97 Iterator dst(_dst, win);
98 switch(_dst->info()->data_type())
99 {
100 case DataType::F32:
101 {
102 /* Up-conversion BFLOAT16 -> F32 */
103 execute_window_loop(win, [&](const Coordinates &)
104 {
105 const auto src_ptr = reinterpret_cast<const bfloat16 *>(src.ptr());
106 const auto dst_ptr = reinterpret_cast<float *>(dst.ptr());
107
108 int x = window_start_x;
109 for(; x <= (window_end_x - window_step_x); x += window_step_x)
110 {
111 const uint16x8x2_t texels =
112 {
113 {
114 vld1q_u16(reinterpret_cast<uint16_t *>(src.ptr())),
115 vld1q_u16(reinterpret_cast<uint16_t *>(src.ptr()) + 8)
116 }
117 };
118
119 vst1q_f32(reinterpret_cast<float *>(dst.ptr()),
120 vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(vget_low_u16(texels.val[0])), 16)));
121 vst1q_f32(reinterpret_cast<float *>(dst.ptr()) + 4,
122 vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(vget_high_u16(texels.val[0])), 16)));
123 vst1q_f32(reinterpret_cast<float *>(dst.ptr()) + 8,
124 vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(vget_low_u16(texels.val[1])), 16)));
125 vst1q_f32(reinterpret_cast<float *>(dst.ptr()) + 12,
126 vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(vget_high_u16(texels.val[1])), 16)));
127 }
128
129 for(; x < window_end_x; ++x)
130 {
131 *(dst_ptr + x) = float(*(src_ptr + x));
132 }
133 },
134 src, dst);
135 break;
136 }
137 default:
138 ARM_COMPUTE_ERROR("dst data type unsupported");
139 }
140 }
141
142 } // namespace cpu
143 } // namespace arm_compute
144
145 #endif /* defined(ARM_COMPUTE_ENABLE_BF16) */
146