1 // Auto-generated file. Do not edit!
2 // Template: src/f16-gavgpool/unipass-f16c.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2022 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include <xnnpack/gavgpool.h>
15 #include <xnnpack/intrinsics-polyfill.h>
16
17
xnn_f16_gavgpool_minmax_ukernel_7x__f16c_c16(size_t rows,size_t channels,const void * input,size_t input_stride,const void * zero,void * output,const union xnn_f16_scaleminmax_params params[restrict XNN_MIN_ELEMENTS (1)])18 void xnn_f16_gavgpool_minmax_ukernel_7x__f16c_c16(
19 size_t rows,
20 size_t channels,
21 const void* input,
22 size_t input_stride,
23 const void* zero,
24 void* output,
25 const union xnn_f16_scaleminmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
26 {
27 assert(rows != 0);
28 assert(rows <= 7);
29 assert(channels != 0);
30
31 const uint16_t* i0 = input;
32 const uint16_t* i1 = (const uint16_t*) ((uintptr_t) i0 + input_stride);
33 if XNN_UNPREDICTABLE(rows < 2) {
34 i1 = (const uint16_t*) zero;
35 }
36 const uint16_t* i2 = (const uint16_t*) ((uintptr_t) i1 + input_stride);
37 if XNN_UNPREDICTABLE(rows <= 2) {
38 i2 = (const uint16_t*) zero;
39 }
40 const uint16_t* i3 = (const uint16_t*) ((uintptr_t) i2 + input_stride);
41 if XNN_UNPREDICTABLE(rows < 4) {
42 i3 = (const uint16_t*) zero;
43 }
44 const uint16_t* i4 = (const uint16_t*) ((uintptr_t) i3 + input_stride);
45 if XNN_UNPREDICTABLE(rows <= 4) {
46 i4 = (const uint16_t*) zero;
47 }
48 const uint16_t* i5 = (const uint16_t*) ((uintptr_t) i4 + input_stride);
49 if XNN_UNPREDICTABLE(rows < 6) {
50 i5 = (const uint16_t*) zero;
51 }
52 const uint16_t* i6 = (const uint16_t*) ((uintptr_t) i5 + input_stride);
53 if XNN_UNPREDICTABLE(rows <= 6) {
54 i6 = (const uint16_t*) zero;
55 }
56 uint16_t* o = (uint16_t*) output;
57
58 const __m256 vscale = _mm256_load_ps(params->avx.scale);
59 const __m256 vmin = _mm256_load_ps(params->avx.min);
60 const __m256 vmax = _mm256_load_ps(params->avx.max);
61 for (; channels >= 16; channels -= 16) {
62 const __m256 vi0x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i0));
63 const __m256 vi0x89ABCDEF = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) (i0 + 8)));
64 i0 += 16;
65 const __m256 vi1x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i1));
66 const __m256 vi1x89ABCDEF = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) (i1 + 8)));
67 i1 += 16;
68
69 const __m256 vi2x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i2));
70 __m128i vacc01234567 = _mm256_cvtps_ph(_mm256_add_ps(vi0x01234567, vi1x01234567), _MM_FROUND_NO_EXC);
71 const __m256 vi2x89ABCDEF = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) (i2 + 8)));
72 __m128i vacc89ABCDEF = _mm256_cvtps_ph(_mm256_add_ps(vi0x89ABCDEF, vi1x89ABCDEF), _MM_FROUND_NO_EXC);
73 i2 += 16;
74
75 const __m256 vi3x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i3));
76 vacc01234567 = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc01234567), vi2x01234567), _MM_FROUND_NO_EXC);
77 const __m256 vi3x89ABCDEF = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) (i3 + 8)));
78 i3 += 16;
79 vacc89ABCDEF = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc89ABCDEF), vi2x89ABCDEF), _MM_FROUND_NO_EXC);
80 const __m256 vi4x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i4));
81 vacc01234567 = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc01234567), vi3x01234567), _MM_FROUND_NO_EXC);
82 const __m256 vi4x89ABCDEF = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) (i4 + 8)));
83 i4 += 16;
84 vacc89ABCDEF = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc89ABCDEF), vi3x89ABCDEF), _MM_FROUND_NO_EXC);
85 const __m256 vi5x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i5));
86 vacc01234567 = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc01234567), vi4x01234567), _MM_FROUND_NO_EXC);
87 const __m256 vi5x89ABCDEF = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) (i5 + 8)));
88 i5 += 16;
89 vacc89ABCDEF = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc89ABCDEF), vi4x89ABCDEF), _MM_FROUND_NO_EXC);
90 const __m256 vi6x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i6));
91 vacc01234567 = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc01234567), vi5x01234567), _MM_FROUND_NO_EXC);
92 const __m256 vi6x89ABCDEF = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) (i6 + 8)));
93 i6 += 16;
94 vacc89ABCDEF = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc89ABCDEF), vi5x89ABCDEF), _MM_FROUND_NO_EXC);
95 vacc01234567 = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc01234567), vi6x01234567), _MM_FROUND_NO_EXC);
96 vacc89ABCDEF = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc89ABCDEF), vi6x89ABCDEF), _MM_FROUND_NO_EXC);
97
98 vacc01234567 = _mm256_cvtps_ph(_mm256_mul_ps(_mm256_cvtph_ps(vacc01234567), vscale), _MM_FROUND_NO_EXC);
99 vacc89ABCDEF = _mm256_cvtps_ph(_mm256_mul_ps(_mm256_cvtph_ps(vacc89ABCDEF), vscale), _MM_FROUND_NO_EXC);
100
101 __m256 vout01234567 = _mm256_max_ps(_mm256_cvtph_ps(vacc01234567), vmin);
102 __m256 vout89ABCDEF = _mm256_max_ps(_mm256_cvtph_ps(vacc89ABCDEF), vmin);
103
104 vout01234567 = _mm256_min_ps(vout01234567, vmax);
105 vout89ABCDEF = _mm256_min_ps(vout89ABCDEF, vmax);
106
107 _mm_storeu_si128((__m128i*) o, _mm256_cvtps_ph(vout01234567, _MM_FROUND_NO_EXC));
108 _mm_storeu_si128((__m128i*) (o + 8), _mm256_cvtps_ph(vout89ABCDEF, _MM_FROUND_NO_EXC));
109 o += 16;
110 }
111 if XNN_UNLIKELY(channels != 0) {
112 do {
113 const __m256 vi0x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i0));
114 i0 += 8;
115 const __m256 vi1x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i1));
116 i1 += 8;
117
118 const __m256 vi2x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i2));
119 __m128i vacc01234567 = _mm256_cvtps_ph(_mm256_add_ps(vi0x01234567, vi1x01234567), _MM_FROUND_NO_EXC);
120 i2 += 8;
121
122 const __m256 vi3x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i3));
123 i3 += 8;
124 vacc01234567 = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc01234567), vi2x01234567), _MM_FROUND_NO_EXC);
125 const __m256 vi4x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i4));
126 i4 += 8;
127 vacc01234567 = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc01234567), vi3x01234567), _MM_FROUND_NO_EXC);
128 const __m256 vi5x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i5));
129 i5 += 8;
130 vacc01234567 = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc01234567), vi4x01234567), _MM_FROUND_NO_EXC);
131 const __m256 vi6x01234567 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) i6));
132 i6 += 8;
133 vacc01234567 = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc01234567), vi5x01234567), _MM_FROUND_NO_EXC);
134 vacc01234567 = _mm256_cvtps_ph(_mm256_add_ps(_mm256_cvtph_ps(vacc01234567), vi6x01234567), _MM_FROUND_NO_EXC);
135
136 vacc01234567 = _mm256_cvtps_ph(_mm256_mul_ps(_mm256_cvtph_ps(vacc01234567), vscale), _MM_FROUND_NO_EXC);
137 __m256 vout01234567 = _mm256_max_ps(_mm256_cvtph_ps(vacc01234567), vmin);
138 vout01234567 = _mm256_min_ps(vout01234567, vmax);
139
140 if XNN_LIKELY(channels >= 8) {
141 _mm_storeu_si128((__m128i*) o, _mm256_cvtps_ph(vout01234567, _MM_FROUND_NO_EXC));
142 o += 8;
143 channels -= 8;
144 } else {
145 __m128i vh01234567 = _mm256_cvtps_ph(vout01234567, _MM_FROUND_NO_EXC);
146 if (channels & 4) {
147 _mm_storel_epi64((__m128i*) o, vh01234567);
148 o += 4;
149 vh01234567 = _mm_unpackhi_epi64(vh01234567, vh01234567);
150 }
151 if (channels & 2) {
152 _mm_storeu_si32(o, vh01234567);
153 o += 2;
154 vh01234567 = _mm_srli_epi64(vh01234567, 32);
155 }
156 if (channels & 1) {
157 *o = (uint16_t) _mm_extract_epi16(vh01234567, 0);
158 }
159 channels = 0;
160 }
161 } while (channels != 0);
162 }
163 }
164