1 // Copyright 2019 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7
8 #include <xnnpack/pad.h>
9 #include <xnnpack/unaligned.h>
10
11
xnn_xx_pad_ukernel__scalar(size_t rows,size_t channels,size_t pre_padding,size_t post_padding,const void * input,size_t input_stride,void * output,size_t output_stride,const uint32_t fill_pattern)12 void xnn_xx_pad_ukernel__scalar(
13 size_t rows,
14 size_t channels,
15 size_t pre_padding,
16 size_t post_padding,
17 const void* input,
18 size_t input_stride,
19 void* output,
20 size_t output_stride,
21 const uint32_t fill_pattern) XNN_OOB_READS
22 {
23 const size_t input_increment = input_stride - channels;
24 const size_t output_increment = output_stride - (pre_padding + channels + post_padding);
25
26 do {
27 // Pre-pad input channels.
28 size_t l = pre_padding;
29 if XNN_LIKELY(l != 0) {
30 uint32_t vfill_pattern = fill_pattern;
31 for (; l >= 4 * sizeof(uint8_t); l -= 4 * sizeof(uint8_t)) {
32 unaligned_store_u32(output, vfill_pattern);
33 output = (uint8_t*) output + 4;
34 }
35 if XNN_LIKELY(l & (2 * sizeof(uint8_t))) {
36 unaligned_store_u16(output, (uint16_t) vfill_pattern);
37 vfill_pattern >>= 16;
38 output = (uint8_t*) output + 2;
39 }
40 if XNN_LIKELY(l & (1 * sizeof(uint8_t))) {
41 *((uint8_t*) output) = (uint8_t) vfill_pattern;
42 output = (uint8_t*) output + 1;
43 }
44 }
45
46 // Copy input channels.
47 size_t c = channels;
48 for (; c >= 16 * sizeof(uint8_t); c -= 16 * sizeof(uint8_t)) {
49 const uint32_t vdata0 = unaligned_indexed_load_u32(input, 0);
50 const uint32_t vdata1 = unaligned_indexed_load_u32(input, 1);
51 const uint32_t vdata2 = unaligned_indexed_load_u32(input, 2);
52 const uint32_t vdata3 = unaligned_indexed_load_u32(input, 3);
53 input = (const uint8_t*) input + 16;
54
55 unaligned_indexed_store_u32(output, 0, vdata0);
56 unaligned_indexed_store_u32(output, 1, vdata1);
57 unaligned_indexed_store_u32(output, 2, vdata2);
58 unaligned_indexed_store_u32(output, 3, vdata3);
59 output = (uint8_t*) output + 16;
60 }
61 if XNN_UNLIKELY(c != 0) {
62 for (; c >= 4 * sizeof(uint8_t); c -= 4 * sizeof(uint8_t)) {
63 unaligned_store_u32(output, unaligned_load_u32(input));
64 input = (const uint8_t*) input + 4;
65 output = (uint8_t*) output + 4;
66 }
67 if XNN_UNLIKELY(c != 0) {
68 uint32_t vdata = unaligned_load_u32(input);
69 input = (const void*) ((uintptr_t) input + c);
70
71 if XNN_LIKELY(c & (2 * sizeof(uint8_t))) {
72 unaligned_store_u16(output, vdata);
73 vdata >>= 16;
74 output = (uint8_t*) output + 2;
75 }
76 if XNN_LIKELY(c & (1 * sizeof(uint8_t))) {
77 *((uint8_t*) output) = (uint8_t) vdata;
78 output = (uint8_t*) output + 1;
79 }
80 }
81 }
82
83 // Post-pad input channels.
84 size_t r = post_padding;
85 if XNN_LIKELY(r != 0) {
86 uint32_t vfill_pattern = fill_pattern;
87 for (; r >= 4 * sizeof(uint8_t); r -= 4 * sizeof(uint8_t)) {
88 unaligned_store_u32(output, vfill_pattern);
89 output = (uint8_t*) output + 4;
90 }
91 if XNN_LIKELY(r & (2 * sizeof(uint8_t))) {
92 unaligned_store_u16(output, vfill_pattern);
93 vfill_pattern >>= 16;
94 output = (uint8_t*) output + 2;
95 }
96 if XNN_LIKELY(r & (1 * sizeof(uint8_t))) {
97 *((uint8_t*) output) = (uint8_t) vfill_pattern;
98 output = (uint8_t*) output + 1;
99 }
100 }
101
102 input = (const uint32_t*) ((uintptr_t) input + input_increment);
103 output = (uint32_t*) ((uintptr_t) output + output_increment);
104 } while (--rows != 0);
105 }
106