1 // Copyright 2019 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <math.h>
7 #include <stddef.h>
8 #include <stdint.h>
9 #include <stdlib.h>
10 #include <string.h>
11
12 #include <xnnpack.h>
13 #include <xnnpack/allocator.h>
14 #include <xnnpack/cache.h>
15 #include <xnnpack/log.h>
16 #include <xnnpack/operator.h>
17 #include <xnnpack/pack.h>
18 #include <xnnpack/microparams-init.h>
19 #include <xnnpack/params.h>
20
21
create_prelu_nc(size_t channels,size_t input_stride,size_t output_stride,const void * negative_slope,uint32_t flags,uint32_t log2_weights_element_size,xnn_pack_prelu_w_function pack_prelu_w,uint32_t datatype_init_flags,enum xnn_operator_type operator_type,xnn_caches_t caches,xnn_operator_t * prelu_op_out)22 static enum xnn_status create_prelu_nc(
23 size_t channels,
24 size_t input_stride,
25 size_t output_stride,
26 const void* negative_slope,
27 uint32_t flags,
28 uint32_t log2_weights_element_size,
29 xnn_pack_prelu_w_function pack_prelu_w,
30 uint32_t datatype_init_flags,
31 enum xnn_operator_type operator_type,
32 xnn_caches_t caches,
33 xnn_operator_t* prelu_op_out)
34 {
35 xnn_operator_t prelu_op = NULL;
36 enum xnn_status status = xnn_status_uninitialized;
37
38 if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
39 xnn_log_error("failed to setup %s operator: XNNPACK is not initialized",
40 xnn_operator_type_to_string(operator_type));
41 return xnn_status_uninitialized;
42 }
43
44 status = xnn_status_unsupported_hardware;
45
46 if ((xnn_params.init_flags & datatype_init_flags) != datatype_init_flags) {
47 xnn_log_error(
48 "failed to create %s operator: operations on data type are not supported",
49 xnn_operator_type_to_string(operator_type));
50 goto error;
51 }
52
53 status = xnn_status_invalid_parameter;
54
55 if (channels == 0) {
56 xnn_log_error(
57 "failed to create %s operator with %zu channels: number of channels must be non-zero",
58 xnn_operator_type_to_string(operator_type), channels);
59 goto error;
60 }
61
62 if (input_stride < channels) {
63 xnn_log_error(
64 "failed to create %s operator with input element stride of %zu: "
65 "stride must be at least as large as the number of channels (%zu)",
66 xnn_operator_type_to_string(operator_type), input_stride, channels);
67 goto error;
68 }
69
70 if (output_stride < channels) {
71 xnn_log_error(
72 "failed to create %s operator with output element stride of %zu: "
73 "stride must be at least as large as the number of channels (%zu)",
74 xnn_operator_type_to_string(operator_type), output_stride, channels);
75 goto error;
76 }
77
78 status = xnn_status_out_of_memory;
79
80 prelu_op = xnn_allocate_zero_simd_memory(sizeof(struct xnn_operator));
81 if (prelu_op == NULL) {
82 xnn_log_error(
83 "failed to allocate %zu bytes for %s operator descriptor",
84 sizeof(struct xnn_operator), xnn_operator_type_to_string(operator_type));
85 goto error;
86 }
87
88 if (caches != NULL) {
89 prelu_op->weights_cache = caches->weights_cache;
90 }
91
92 const size_t packed_weights_size = (channels << log2_weights_element_size) + XNN_EXTRA_BYTES;
93 const size_t aligned_total_weights_size = round_up_po2(packed_weights_size, XNN_ALLOCATION_ALIGNMENT);
94 void* weights_ptr = xnn_get_pointer_to_write_weights(prelu_op, aligned_total_weights_size, 0);
95 pack_prelu_w(channels, negative_slope, weights_ptr);
96
97 if (use_weights_cache(prelu_op)) {
98 prelu_op->packed_weights.offset = xnn_get_or_insert_weights_cache(
99 prelu_op->weights_cache, weights_ptr, aligned_total_weights_size);
100 }
101
102 prelu_op->channels = channels;
103 prelu_op->input_pixel_stride = input_stride;
104 prelu_op->output_pixel_stride = output_stride;
105
106 prelu_op->type = operator_type;
107 prelu_op->flags = flags;
108
109 prelu_op->state = xnn_run_state_invalid;
110
111 *prelu_op_out = prelu_op;
112 return xnn_status_success;
113
114 error:
115 xnn_delete_operator(prelu_op);
116 return status;
117 }
118
119
xnn_create_prelu_nc_f16(size_t channels,size_t input_stride,size_t output_stride,const void * negative_slope,uint32_t flags,xnn_caches_t caches,xnn_operator_t * prelu_op_out)120 enum xnn_status xnn_create_prelu_nc_f16(
121 size_t channels,
122 size_t input_stride,
123 size_t output_stride,
124 const void* negative_slope,
125 uint32_t flags,
126 xnn_caches_t caches,
127 xnn_operator_t* prelu_op_out)
128 {
129 xnn_pack_prelu_w_function pack_prelu_w = (xnn_pack_prelu_w_function) xnn_pack_f16_prelu_w;
130 if (flags & XNN_FLAG_FP32_STATIC_WEIGHTS) {
131 pack_prelu_w = (xnn_pack_prelu_w_function) xnn_pack_f32_to_f16_prelu_w;
132 }
133
134 return create_prelu_nc(
135 channels, input_stride, output_stride,
136 negative_slope, flags,
137 1 /* log2(sizeof(uint16_t)) */,
138 pack_prelu_w,
139 XNN_INIT_FLAG_F16, xnn_operator_type_prelu_nc_f16,
140 caches,
141 prelu_op_out);
142 }
143
xnn_create_prelu_nc_f32(size_t channels,size_t input_stride,size_t output_stride,const float * negative_slope,uint32_t flags,xnn_caches_t caches,xnn_operator_t * prelu_op_out)144 enum xnn_status xnn_create_prelu_nc_f32(
145 size_t channels,
146 size_t input_stride,
147 size_t output_stride,
148 const float* negative_slope,
149 uint32_t flags,
150 xnn_caches_t caches,
151 xnn_operator_t* prelu_op_out)
152 {
153 return create_prelu_nc(
154 channels, input_stride, output_stride,
155 negative_slope, flags,
156 2 /* log2(sizeof(float)) */,
157 (xnn_pack_prelu_w_function) xnn_pack_f32_prelu_w,
158 XNN_INIT_FLAG_F32, xnn_operator_type_prelu_nc_f32,
159 caches,
160 prelu_op_out);
161 }
162
setup_prelu_nc(xnn_operator_t prelu_op,enum xnn_operator_type expected_operator_type,size_t batch_size,const float * input,float * output,uint32_t datatype_init_flags,uint32_t log2_element_size,const struct prelu_parameters prelu[restrict XNN_MIN_ELEMENTS (1)],size_t num_threads)163 static enum xnn_status setup_prelu_nc(
164 xnn_operator_t prelu_op,
165 enum xnn_operator_type expected_operator_type,
166 size_t batch_size,
167 const float* input,
168 float* output,
169 uint32_t datatype_init_flags,
170 uint32_t log2_element_size,
171 const struct prelu_parameters prelu[restrict XNN_MIN_ELEMENTS(1)],
172 size_t num_threads)
173 {
174 if (prelu_op->type != expected_operator_type) {
175 xnn_log_error("failed to setup operator: operator type mismatch (expected %s, got %s)",
176 xnn_operator_type_to_string(expected_operator_type),
177 xnn_operator_type_to_string(prelu_op->type));
178 return xnn_status_invalid_parameter;
179 }
180 prelu_op->state = xnn_run_state_invalid;
181
182 if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
183 xnn_log_error("failed to setup %s operator: XNNPACK is not initialized",
184 xnn_operator_type_to_string(expected_operator_type));
185 return xnn_status_uninitialized;
186 }
187
188 if ((xnn_params.init_flags & datatype_init_flags) != datatype_init_flags) {
189 xnn_log_error("failed to setup %s operator: operations on data type are not supported",
190 xnn_operator_type_to_string(expected_operator_type));
191 return xnn_status_unsupported_hardware;
192 }
193
194 if (batch_size == 0) {
195 prelu_op->state = xnn_run_state_skip;
196 return xnn_status_success;
197 }
198
199 if (prelu_op->weights_cache != NULL && !xnn_weights_cache_is_finalized(prelu_op->weights_cache)) {
200 xnn_log_error("failed to setup %s operator: weights cache is not finalized",
201 xnn_operator_type_to_string(expected_operator_type));
202 return xnn_status_invalid_state;
203 }
204
205 const size_t channels = prelu_op->channels;
206 prelu_op->context.prelu = (struct prelu_context) {
207 .n = channels << log2_element_size,
208 .x = input,
209 .x_stride = prelu_op->input_pixel_stride << log2_element_size,
210 .w = packed_weights(prelu_op),
211 .y = output,
212 .y_stride = prelu_op->output_pixel_stride << log2_element_size,
213 .ukernel = prelu->ukernel,
214 };
215
216 #if XNN_TEST_MODE
217 const size_t batch_tile = prelu->row_tile;
218 #else
219 size_t batch_tile = batch_size;
220 if (num_threads > 1) {
221 const size_t target_tiles_per_thread = 5;
222 const size_t max_batch_tile = divide_round_up(batch_size, num_threads * target_tiles_per_thread);
223 if (max_batch_tile < batch_tile) {
224 const uint32_t row_tile = prelu->row_tile;
225 batch_tile = min(batch_tile, divide_round_up(batch_tile, max_batch_tile * row_tile) * row_tile);
226 }
227 }
228 #endif
229 prelu_op->compute.type = xnn_parallelization_type_1d_tile_1d;
230 prelu_op->compute.task_1d_tile_1d = (pthreadpool_task_1d_tile_1d_t) xnn_compute_prelu;
231 prelu_op->compute.range[0] = batch_size;
232 prelu_op->compute.tile[0] = batch_tile;
233 prelu_op->state = xnn_run_state_ready;
234
235 return xnn_status_success;
236 }
237
xnn_setup_prelu_nc_f16(xnn_operator_t prelu_op,size_t batch_size,const void * input,void * output,pthreadpool_t threadpool)238 enum xnn_status xnn_setup_prelu_nc_f16(
239 xnn_operator_t prelu_op,
240 size_t batch_size,
241 const void* input,
242 void* output,
243 pthreadpool_t threadpool)
244 {
245 return setup_prelu_nc(
246 prelu_op, xnn_operator_type_prelu_nc_f16,
247 batch_size, input, output,
248 XNN_INIT_FLAG_F16,
249 1 /* log2(sizeof(uint16_t)) */,
250 &xnn_params.f16.prelu,
251 pthreadpool_get_threads_count(threadpool));
252 }
253
xnn_setup_prelu_nc_f32(xnn_operator_t prelu_op,size_t batch_size,const float * input,float * output,pthreadpool_t threadpool)254 enum xnn_status xnn_setup_prelu_nc_f32(
255 xnn_operator_t prelu_op,
256 size_t batch_size,
257 const float* input,
258 float* output,
259 pthreadpool_t threadpool)
260 {
261 return setup_prelu_nc(
262 prelu_op, xnn_operator_type_prelu_nc_f32,
263 batch_size, input, output,
264 XNN_INIT_FLAG_F32,
265 2 /* log2(sizeof(float)) */,
266 &xnn_params.f32.prelu,
267 pthreadpool_get_threads_count(threadpool));
268 }
269