1 /*
2 * Copyright (c) Facebook, Inc. and its affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 #pragma once
10
11 #include <assert.h>
12 #include <limits.h>
13 #include <stdbool.h>
14 #include <stdint.h>
15
16 #include <fp16/bitcasts.h>
17
18 #if defined(__clang__)
19 #if __clang_major__ == 3 && __clang_minor__ >= 7 || __clang_major__ > 3
20 #define PYTORCH_QNNP_IGNORE_SHIFT_BASE_UB \
21 __attribute__((__no_sanitize__("shift-base")))
22 #else
23 #define PYTORCH_QNNP_IGNORE_SHIFT_BASE_UB
24 #endif
25 #elif defined(__GNUC__)
26 #if __GNUC__ >= 8
27 #define PYTORCH_QNNP_IGNORE_SHIFT_BASE_UB \
28 __attribute__((__no_sanitize__("shift-base")))
29 #elif __GNUC__ == 4 && __GNUC_MINOR__ >= 9 || __GNUC__ > 4
30 /* 4.9 <= gcc < 8 support ubsan, but doesn't support no_sanitize attribute */
31 #define PYTORCH_QNNP_IGNORE_SHIFT_BASE_UB
32 #ifndef PYTORCH_QNNP_USE_SHIFT_BASE_UB_WORKAROUND
33 #define PYTORCH_QNNP_USE_SHIFT_BASE_UB_WORKAROUND 1
34 #endif
35 #else
36 #define PYTORCH_QNNP_IGNORE_SHIFT_BASE_UB
37 #endif
38 #else
39 #define PYTORCH_QNNP_IGNORE_SHIFT_BASE_UB
40 #endif
41
42 PYTORCH_QNNP_IGNORE_SHIFT_BASE_UB
asr_s32(int32_t x,uint32_t n)43 inline static int32_t asr_s32(int32_t x, uint32_t n) {
44 #ifdef PYTORCH_QNNP_USE_SHIFT_BASE_UB_WORKAROUND
45 #if defined(__x86_64__) || defined(__aarch64__)
46 return (int32_t)((uint64_t)(int64_t)x >> n);
47 #else
48 return x >= 0 ? x >> n : ~(~x >> n);
49 #endif
50 #else
51 return x >> n;
52 #endif
53 }
54
55 PYTORCH_QNNP_IGNORE_SHIFT_BASE_UB
asr_s64(int64_t x,uint32_t n)56 inline static int64_t asr_s64(int64_t x, uint32_t n) {
57 #ifdef PYTORCH_QNNP_USE_SHIFT_BASE_UB_WORKAROUND
58 return x >= 0 ? x >> n : ~(~x >> n);
59 #else
60 return x >> n;
61 #endif
62 }
63
pytorch_scalar_requantize_precise(int32_t value,float scale,uint8_t zero_point,uint8_t qmin,uint8_t qmax)64 inline static uint8_t pytorch_scalar_requantize_precise(
65 int32_t value,
66 float scale,
67 uint8_t zero_point,
68 uint8_t qmin,
69 uint8_t qmax) {
70 assert(scale < 1.0f);
71 assert(scale >= 0x1.0p-32f);
72
73 const uint32_t scale_bits = fp32_to_bits(scale);
74 const uint32_t multiplier =
75 (scale_bits & UINT32_C(0x007FFFFF)) | UINT32_C(0x00800000);
76 const uint32_t shift = 127 + 23 - (scale_bits >> 23);
77 assert(shift >= 24);
78 assert(shift < 56);
79
80 /*
81 * Compute absolute value of input as unsigned 32-bit int.
82 * All further computations will work with unsigned values to avoid undefined
83 * behaviour on signed operations.
84 */
85 const uint32_t abs_value = (value >= 0) ? (uint32_t)value : -(uint32_t)value;
86
87 /* Compute full 64-bit product of 32-bit factors */
88 const uint64_t product = (uint64_t)abs_value * (uint64_t)multiplier;
89
90 /*
91 * Shift the full 64-bit product right with rounding.
92 * Rounding is performed towards closest integer, with midpoints rounded up
93 * (same as away from zero).
94 */
95 const uint64_t rounding = UINT64_C(1) << (shift - 1);
96 const uint32_t abs_scaled_value = (uint32_t)((product + rounding) >> shift);
97
98 /*
99 * Copy the sign of input to scaled absolute input value.
100 */
101 const int32_t scaled_value =
102 (int32_t)(value >= 0 ? abs_scaled_value : -abs_scaled_value);
103
104 /* Clamp scaled value with zero point between smin and smax */
105 int32_t clamped_value = scaled_value;
106 const int32_t smin = (int32_t)(uint32_t)qmin - (int32_t)(uint32_t)zero_point;
107 if (clamped_value < smin) {
108 clamped_value = smin;
109 }
110 const int32_t smax = (int32_t)(uint32_t)qmax - (int32_t)(uint32_t)zero_point;
111 if (clamped_value > smax) {
112 clamped_value = smax;
113 }
114
115 /* Add zero point to clamped value */
116 const int32_t biased_value = clamped_value + (int32_t)(uint32_t)zero_point;
117
118 return biased_value;
119 }
120