1 // Copyright 2021 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7 #include <stddef.h>
8 #include <stdint.h>
9
10 #include <arm_neon.h>
11
12 #include <xnnpack/math-stubs.h>
13
14
xnn_math_f32_f16_cvt__neon(size_t n,const float * input,void * output)15 void xnn_math_f32_f16_cvt__neon(
16 size_t n,
17 const float* input,
18 void* output)
19 {
20 assert(n % (8 * sizeof(uint16_t)) == 0);
21
22 const uint32x4_t vexp_bias = vdupq_n_u32(UINT32_C(0x07800000));
23 const float32x4_t vscale_to_inf = vdupq_n_f32(0x1.0p+112f);
24 const uint32x4_t vexpw_max = vdupq_n_u32(UINT32_C(0x7F800000));
25 const float32x4_t vscale_to_zero = vdupq_n_f32(0x1.0p-110f);
26 const uint32x4_t vbias_min = vdupq_n_u32(UINT32_C(0x40000000));
27 const uint16x8_t vexph_mask = vdupq_n_u16(UINT16_C(0x7C00));
28 const uint16x8_t vmanth_mask = vdupq_n_u16(UINT16_C(0x0FFF));
29 const uint16x8_t vsignh_mask = vdupq_n_u16(UINT16_C(0x8000));
30 const uint16x8_t vnanh = vdupq_n_u16(UINT16_C(0x7E00));
31
32 uint16_t* o = (uint16_t*) output;
33 for (; n != 0; n -= 8 * sizeof(uint16_t)) {
34 const float32x4_t vx_lo = vld1q_f32(input); input += 4;
35 const float32x4_t vx_hi = vld1q_f32(input); input += 4;
36
37 const float32x4_t vabsx_lo = vabsq_f32(vx_lo);
38 const float32x4_t vabsx_hi = vabsq_f32(vx_hi);
39
40 uint32x4_t vbias_lo = vaddq_u32(vreinterpretq_u32_f32(vabsx_lo), vexp_bias);
41 uint32x4_t vbias_hi = vaddq_u32(vreinterpretq_u32_f32(vabsx_hi), vexp_bias);
42
43 float32x4_t vf_lo = vmulq_f32(vabsx_lo, vscale_to_inf);
44 float32x4_t vf_hi = vmulq_f32(vabsx_hi, vscale_to_inf);
45 const uint32x4_t vnanmaskw_lo = vcgtq_u32(vreinterpretq_u32_f32(vabsx_lo), vexpw_max);
46 const uint32x4_t vnanmaskw_hi = vcgtq_u32(vreinterpretq_u32_f32(vabsx_hi), vexpw_max);
47
48 vbias_lo = vandq_u32(vbias_lo, vexpw_max);
49 vbias_hi = vandq_u32(vbias_hi, vexpw_max);
50 vf_lo = vmulq_f32(vf_lo, vscale_to_zero);
51 vf_hi = vmulq_f32(vf_hi, vscale_to_zero);
52
53 const uint16x8_t vnanmaskh = vcombine_u16(vmovn_u32(vnanmaskw_lo), vmovn_u32(vnanmaskw_hi));
54 vbias_lo = vmaxq_u32(vbias_lo, vbias_min);
55 vbias_hi = vmaxq_u32(vbias_hi, vbias_min);
56
57 vf_lo = vaddq_f32(vf_lo, vreinterpretq_f32_u32(vbias_lo));
58 vf_hi = vaddq_f32(vf_hi, vreinterpretq_f32_u32(vbias_hi));
59
60 uint16x8_t vexph = vcombine_u16(vshrn_n_u32(vreinterpretq_u32_f32(vf_lo), 13), vshrn_n_u32(vreinterpretq_u32_f32(vf_hi), 13));
61 uint16x8_t vmanth = vcombine_u16(vmovn_u32(vreinterpretq_u32_f32(vf_lo)), vmovn_u32(vreinterpretq_u32_f32(vf_hi)));
62 uint16x8_t vsignh = vcombine_u16(vshrn_n_u32(vreinterpretq_u32_f32(vx_lo), 16), vshrn_n_u32(vreinterpretq_u32_f32(vx_hi), 16));
63
64 vexph = vandq_u16(vexph, vexph_mask);
65 vmanth = vandq_u16(vmanth, vmanth_mask);
66 vsignh = vandq_u16(vsignh, vsignh_mask);
67
68 uint16x8_t vh = vaddq_u16(vmanth, vexph);
69 vh = vbslq_u16(vnanmaskh, vnanh, vh);
70 vh = vorrq_u16(vh, vsignh);
71
72 vst1q_u16(o, vh); o += 8;
73 }
74 }
75