1 // Copyright (c) Facebook, Inc. and its affiliates.
2 // All rights reserved.
3 //
4 // Copyright 2019 Google LLC
5 //
6 // This source code is licensed under the BSD-style license found in the
7 // LICENSE file in the root directory of this source tree.
8
9 #include <assert.h>
10 #include <stdint.h>
11 #include <stddef.h>
12
13 #include <xnnpack/math.h>
14 #include <xnnpack/requantization-stubs.h>
15
16
xnn_qu8_requantize_rndna__scalar_signed64(size_t n,const int32_t * input,float scale,uint8_t zero_point,uint8_t qmin,uint8_t qmax,uint8_t * output)17 void xnn_qu8_requantize_rndna__scalar_signed64(
18 size_t n,
19 const int32_t* input,
20 float scale,
21 uint8_t zero_point,
22 uint8_t qmin,
23 uint8_t qmax,
24 uint8_t* output)
25 {
26 assert(n % 4 == 0);
27 assert(scale < 1.0f);
28 assert(scale >= 0x1.0p-32f);
29
30 const uint32_t scale_bits = float_as_uint32(scale);
31 const int32_t multiplier = ((int32_t) scale_bits & INT32_C(0x007FFFFF)) | INT32_C(0x00800000);
32 const uint32_t shift = 127 + 23 - (scale_bits >> 23);
33 assert(shift >= 24);
34 assert(shift < 56);
35
36 const int64_t rounding = INT64_C(1) << (shift - 1);
37 const int32_t smin = (int32_t) (uint32_t) qmin - (int32_t) (uint32_t) zero_point;
38 const int32_t smax = (int32_t) (uint32_t) qmax - (int32_t) (uint32_t) zero_point;
39 for (; n != 0; n -= 4) {
40 const int32_t x = input[0];
41 const int32_t y = input[1];
42 const int32_t z = input[2];
43 const int32_t w = input[3];
44 input += 4;
45
46 // Compute full 64-bit product of signed 32-bit factors.
47 //
48 // Note: multiplier can be treated as either signed or unsigned.
49 const int64_t x_product = (int64_t) x * (int64_t) multiplier;
50 const int64_t y_product = (int64_t) y * (int64_t) multiplier;
51 const int64_t z_product = (int64_t) z * (int64_t) multiplier;
52 const int64_t w_product = (int64_t) w * (int64_t) multiplier;
53
54 // Adjust product before subsequent shift with rounding up to simulate shift with rounding away from zero.
55 const int64_t x_adjusted_product = x_product - (int64_t) (x < 0);
56 const int64_t y_adjusted_product = y_product - (int64_t) (y < 0);
57 const int64_t z_adjusted_product = z_product - (int64_t) (z < 0);
58 const int64_t w_adjusted_product = w_product - (int64_t) (w < 0);
59
60 // Arithmetically shift the full 64-bit product right with rounding.
61 // Rounding is performed towards closest integer, with midpoints rounded up.
62 //
63 // Note that although rounding is precomputed, it is dependent on shift value, and on processors with 64-bit
64 // "right shift with rounding" instruction each line below can be represented by just one such instruction
65 // (e.g. VRSHL.S64 on ARM NEON, SRSHL in ARM64 Advanced SIMD).
66 const int32_t x_scaled = (int32_t) math_asr_s64(x_adjusted_product + rounding, shift);
67 const int32_t y_scaled = (int32_t) math_asr_s64(y_adjusted_product + rounding, shift);
68 const int32_t z_scaled = (int32_t) math_asr_s64(z_adjusted_product + rounding, shift);
69 const int32_t w_scaled = (int32_t) math_asr_s64(w_adjusted_product + rounding, shift);
70
71 // Clamp scaled value with zero point between (qmin - zero point) and (qmax - zero point).
72 const int32_t x_clamped = math_min_s32(math_max_s32(x_scaled, smin), smax);
73 const int32_t y_clamped = math_min_s32(math_max_s32(y_scaled, smin), smax);
74 const int32_t z_clamped = math_min_s32(math_max_s32(z_scaled, smin), smax);
75 const int32_t w_clamped = math_min_s32(math_max_s32(w_scaled, smin), smax);
76
77 // Add zero point to clamped value.
78 // The result is guaranteed to be in [qmin, qmax] range.
79 //
80 // This addition can not be safely done before clamping, because scaled values are in [-2147483520, 2147483519]
81 // range, so addition of zero point (which can be up to 255) can overflow signed 32-bit integer.
82 const int32_t x_biased = x_clamped + zero_point;
83 const int32_t y_biased = y_clamped + zero_point;
84 const int32_t z_biased = z_clamped + zero_point;
85 const int32_t w_biased = w_clamped + zero_point;
86
87 output[0] = (uint8_t) x_biased;
88 output[1] = (uint8_t) y_biased;
89 output[2] = (uint8_t) z_biased;
90 output[3] = (uint8_t) w_biased;
91 output += 4;
92 }
93 }
94