1// Copyright 2019 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert NR % 8 == 0 7$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 8 9#include <assert.h> 10 11#include <arm_neon.h> 12 13#include <xnnpack/igemm.h> 14 15 16void xnn_f16_igemm_minmax_ukernel_${MR}x${NR}__neonfp16arith_ld64( 17 size_t mr, 18 size_t nc, 19 size_t kc, 20 size_t ks, 21 const void** restrict a, 22 const void* restrict w, 23 void* restrict c, 24 size_t cm_stride, 25 size_t cn_stride, 26 size_t a_offset, 27 const void* zero, 28 const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) 29{ 30 assert(mr != 0); 31 assert(mr <= ${MR}); 32 assert(nc != 0); 33 assert(kc != 0); 34 assert(kc % sizeof(__fp16) == 0); 35 assert(ks != 0); 36 assert(ks % (${MR} * sizeof(void*)) == 0); 37 assert(a_offset % sizeof(__fp16) == 0); 38 assert(a != NULL); 39 assert(w != NULL); 40 assert(c != NULL); 41 42 __fp16* c0 = (__fp16*) c; 43 $for M in range(1, MR): 44 __fp16* c${M} = (__fp16*) ((uintptr_t) c${M-1} + cm_stride); 45 $if M % 2 == 0: 46 if XNN_UNPREDICTABLE(mr <= ${M}) { 47 c${M} = c${M-1}; 48 } 49 $elif M + 1 == MR: 50 if XNN_UNPREDICTABLE(mr != ${M+1}) { 51 c${M} = c${M-1}; 52 } 53 $else: 54 if XNN_UNPREDICTABLE(mr < ${M+1}) { 55 c${M} = c${M-1}; 56 } 57 58 do { 59 $for N in range(0, NR, 8): 60 float16x8_t vacc0x${ABC[N:N+8]} = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t)); 61 $for M in range(1, MR): 62 $for N in range(0, NR, 8): 63 float16x8_t vacc${M}x${ABC[N:N+8]} = vacc0x${ABC[N:N+8]}; 64 65 size_t p = ks; 66 do { 67 $for M in range(MR): 68 const __fp16* restrict a${M} = (const __fp16*) a[${M}]; 69 assert(a${M} != NULL); 70 if XNN_UNPREDICTABLE(a${M} != zero) { 71 a${M} = (const __fp16*) ((uintptr_t) a${M} + a_offset); 72 } 73 a += ${MR}; 74 75 size_t k = kc; 76 for (; k >= 4 * sizeof(__fp16); k -= 4 * sizeof(__fp16)) { 77 $for M in range(MR): 78 const float16x4_t va${M} = vld1_f16(a${M}); a${M} += 4; 79 80 $for L in range(4): 81 $for N in range(0, NR, 8): 82 const float16x8_t vb${ABC[N:N+8]}c${L} = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t)); 83 84 #if XNN_ARCH_ARM64 85 $for N in range(0, NR, 8): 86 $for M in range(MR): 87 vacc${M}x${ABC[N:N+8]} = vfmaq_lane_f16(vacc${M}x${ABC[N:N+8]}, vb${ABC[N:N+8]}c${L}, va${M}, ${L}); 88 #else 89 $for M in range(MR): 90 const float16x8_t va${M}c${L} = vdupq_lane_f16(va${M}, ${L}); 91 92 $for N in range(0, NR, 8): 93 $for M in range(MR): 94 vacc${M}x${ABC[N:N+8]} = vfmaq_f16(vacc${M}x${ABC[N:N+8]}, va${M}c${L}, vb${ABC[N:N+8]}c${L}); 95 #endif 96 } 97 if XNN_UNLIKELY(k != 0) { 98 do { 99 $for M in range(MR): 100 const float16x8_t va${M} = vld1q_dup_f16(a${M}); a${M} += 1; 101 102 $for N in range(0, NR, 8): 103 const float16x8_t vb${ABC[N:N+8]} = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t)); 104 105 $for N in range(0, NR, 8): 106 $for M in range(MR): 107 vacc${M}x${ABC[N:N+8]} = vfmaq_f16(vacc${M}x${ABC[N:N+8]}, va${M}, vb${ABC[N:N+8]}); 108 109 k -= sizeof(__fp16); 110 } while (k != 0); 111 } 112 p -= ${MR} * sizeof(void*); 113 } while (p != 0); 114 115 116 const float16x8_t vmax = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->neon.max)); 117 $for N in range(0, NR, 8): 118 $for M in range(MR): 119 vacc${M}x${ABC[N:N+8]} = vminq_f16(vacc${M}x${ABC[N:N+8]}, vmax); 120 121 const float16x8_t vmin = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->neon.min)); 122 $for N in range(0, NR, 8): 123 $for M in range(MR): 124 vacc${M}x${ABC[N:N+8]} = vmaxq_f16(vacc${M}x${ABC[N:N+8]}, vmin); 125 126 if XNN_LIKELY(nc >= ${NR}) { 127 $for M in reversed(range(MR)): 128 vst1q_f16(c${M}, vacc${M}x${ABC[0:8]}); 129 $for N in range(8, NR, 8): 130 vst1q_f16(c${M} + ${N}, vacc${M}x${ABC[N:N+8]}); 131 c${M} = (__fp16*) ((uintptr_t) c${M} + cn_stride); 132 133 a = (const void**restrict) ((uintptr_t) a - ks); 134 nc -= ${NR}; 135 } else { 136 $for LOG2N in reversed(range(NR.bit_length())): 137 $if NR != 1 << LOG2N: 138 if (nc & ${1 << LOG2N}) { 139 $if LOG2N >= 3: 140 $for N in range(0, 1 << LOG2N, 8): 141 $for M in reversed(range(MR)): 142 vst1q_f16(c${M}, vacc${M}x${ABC[N:N+8]}); c${M} += 8; 143 144 $for M in reversed(range(MR)): 145 $for N in range(0, 1 << (LOG2N - 1), 8): 146 vacc${M}x${ABC[N:N+8]} = vacc${M}x${ABC[N + (1 << LOG2N):N + (1 << LOG2N)+8]}; 147 $elif LOG2N == 2: 148 $for M in reversed(range(MR)): 149 vst1_f16(c${M}, vacc${M}x${ABC[0:4]}); c${M} += 4; 150 151 $for M in reversed(range(MR)): 152 vacc${M}x${ABC[0:4]} = vget_high_f16(vacc${M}x${ABC[0:8]}); 153 $elif LOG2N == 1: 154 $for M in reversed(range(MR)): 155 vst1_lane_u32((void*) c${M}, vreinterpret_u32_f16(vacc${M}x${ABC[0:4]}), 0); c${M} += 2; 156 157 $for M in reversed(range(MR)): 158 vacc${M}x${ABC[0:4]} = vext_f16(vacc${M}x${ABC[0:4]}, vacc${M}x${ABC[0:4]}, 2); 159 $elif LOG2N == 0: 160 $for M in reversed(range(MR)): 161 vst1_lane_f16(c${M}, vacc${M}x${ABC[0:4]}, 0); 162 } 163 $if LOG2N == 3: 164 $for M in reversed(range(MR)): 165 float16x4_t vacc${M}x${ABC[0:4]} = vget_low_f16(vacc${M}x${ABC[0:8]}); 166 167 nc = 0; 168 } 169 } while (nc != 0); 170} 171