1 /*
2  * Copyright (C) 2023 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <inttypes.h>
18 #include <sys/mman.h>
19 
20 #include "berberis/base/bit_util.h"
21 #include "berberis/base/mmap.h"
22 #include "berberis/base/struct_check.h"
23 #include "berberis/intrinsics/simd_register.h"
24 
25 #include "berberis/intrinsics/macro_assembler.h"
26 
27 namespace berberis::constants_pool {
28 
29 // All constants we refer in macroinstructions are collected in MacroAssemblerConstants.
30 // This allows us:
31 //   1. To save memory (they are not duplicated when reused).
32 //   2. Make it possible to access in text assembler without complex dance with hash-tables.
33 //   3. Allocate below-2GB-copy in x86_64 mode easily.
34 struct MacroAssemblerConstants {
35   alignas(16) const uint64_t kNanBoxFloat32[2] = {0xffff'ffff'0000'0000, 0xffff'ffff'0000'0000};
36   alignas(16) const uint64_t kNanBoxedNansFloat32[2] = {0xffff'ffff'7fc0'0000,
37                                                         0xffff'ffff'7fc0'0000};
38   alignas(16) const uint32_t kCanonicalNansFloat32[4] = {0x7fc'00000,
39                                                          0x7fc'00000,
40                                                          0x7fc'00000,
41                                                          0x7fc'00000};
42   alignas(16) const uint64_t kCanonicalNansFloat64[2] = {0x7ff8'0000'0000'0000,
43                                                          0x7ff8'0000'0000'0000};
44   alignas(16) const uint32_t kFloat32One[4] = {0x3f80'0000, 0x3f80'0000, 0x3f80'0000, 0x3f80'0000};
45   alignas(16) const uint64_t kFloat64One[2] = {0x3ff0'0000'0000'0000, 0x3ff0'0000'0000'0000};
46   alignas(16) const uint32_t kFloat32PInf[4] = {0x7f80'0000, 0x7f80'0000, 0x7f80'0000, 0x7f80'0000};
47   alignas(16) const uint32_t kFloat32NInf[4] = {0xff80'0000, 0xff80'0000, 0xff80'0000, 0xff80'0000};
48   alignas(16) const uint64_t kFloat64PInf[2] = {0x7ff0'0000'0000'0000, 0x7ff0'0000'0000'0000};
49   alignas(16) const uint64_t kFloat64NInf[2] = {0xfff0'0000'0000'0000, 0xfff0'0000'0000'0000};
50   alignas(16) const int8_t kMinInt8[16] = {
51       -128,
52       -128,
53       -128,
54       -128,
55       -128,
56       -128,
57       -128,
58       -128,
59       -128,
60       -128,
61       -128,
62       -128,
63       -128,
64       -128,
65       -128,
66       -128,
67   };
68   alignas(16) const int8_t kMaxInt8[16] =
69       {127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127};
70   alignas(16) const int16_t kMinInt16[8] =
71       {-0x8000, -0x8000, -0x8000, -0x8000, -0x8000, -0x8000, -0x8000, -0x8000};
72   alignas(16)
73       const int16_t kMaxInt16[8] = {0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff};
74   alignas(16) const int32_t kMinInt32[4] = {
75       static_cast<int32_t>(-0x8000'0000),
76       static_cast<int32_t>(-0x8000'0000),
77       static_cast<int32_t>(-0x8000'0000),
78       static_cast<int32_t>(-0x8000'0000),
79   };
80   alignas(16) const int32_t kMaxInt32[4] = {0x7fff'ffff, 0x7fff'ffff, 0x7fff'ffff, 0x7fff'ffff};
81   alignas(16) const int64_t kMinInt64[2] = {
82       static_cast<int64_t>(-0x8000'0000'0000'0000),
83       static_cast<int64_t>(-0x8000'0000'0000'0000),
84   };
85   alignas(16) const int64_t kMaxInt64[2] = {0x7fff'ffff'ffff'ffff, 0x7fff'ffff'ffff'ffff};
86   int64_t kBsrToClzInt64 = 127;
87   int64_t kWidthInBits64 = 64;
88   int32_t kBsrToClzInt32 = 63;
89   int32_t kWidthInBits32 = 32;
90   // 64 bit constants for use with arithmetic operations.
91   // Used because only 32 bit immediates are supported on x86-64.
92   int64_t k0x8000_0000_0000_00ff = 0x8000'0000'0000'00ff;
93   alignas(16) const int8_t kPMovmskwToPMovmskb[16] =
94       {0, 2, 4, 6, 8, 10, 12, 14, -63, -24, -19, -27, -28, -128, -128, -128};
95   alignas(16) const int8_t kPMovmskdToPMovmskb[16] =
96       {0, 4, 8, 12, -128, -128, -128, -128, -51, -17, -24, -31, -19, -27, -28, -128};
97   alignas(16) const int8_t kPMovmskqToPMovmskb[16] =
98       {0, 8, -128, -128, -128, -128, -128, -128, -57, -24, -31, -6, -7, -128, -128, -128};
99   alignas(16) const uint8_t kRiscVToX87Exceptions[32] = {
100       0x00, 0x20, 0x10, 0x30, 0x08, 0x28, 0x18, 0x38,
101       0x04, 0x24, 0x14, 0x34, 0x0c, 0x2c, 0x1c, 0x3c,
102       0x01, 0x21, 0x11, 0x31, 0x09, 0x29, 0x19, 0x39,
103       0x05, 0x25, 0x15, 0x35, 0x0d, 0x2d, 0x1d, 0x3d};
104   alignas(16) const uint8_t kX87ToRiscVExceptions[64] = {
105       0x00, 0x10, 0x00, 0x10, 0x08, 0x18, 0x08, 0x18,
106       0x04, 0x14, 0x04, 0x14, 0x0c, 0x1c, 0x0c, 0x1c,
107       0x02, 0x12, 0x02, 0x12, 0x0a, 0x1a, 0x0a, 0x1a,
108       0x06, 0x16, 0x06, 0x16, 0x0e, 0x1e, 0x0e, 0x1e,
109       0x01, 0x11, 0x01, 0x11, 0x09, 0x19, 0x09, 0x19,
110       0x05, 0x15, 0x05, 0x15, 0x0d, 0x1d, 0x0d, 0x1d,
111       0x03, 0x13, 0x03, 0x13, 0x0b, 0x1b, 0x0b, 0x1b,
112       0x07, 0x17, 0x07, 0x17, 0x0f, 0x1f, 0x0f, 0x1f};
113   // This table represents exactly what you see: ������ + �� unset bits and then ������ - �� set bits for ��
114   // in range from �� to ��.  The last ������ bits from line �� then it's mask for ���� equal to �� and if
115   // you shift start address down by ��  bytes then you get mask for �� * �� + �� bits.
116   // Using this approach we may load appropriate bitmask from memory with one load instruction and
117   // the table itself takes ������ bytes.
118   // Since valid values for ���� are from �� to ���� �� ������ (including ������), �� may be from �� to ����,
119   // that's why we need 16 zero bytes in that table.
120   // On AMD CPUs with misalignsse feature we may access data from that table without using movups,
121   // and on all CPUs we may use 2KiB table instead.
122   // But for now we are using movups and small table as probably-adequate compromise.
123   alignas(16) const uint64_t kBitMaskTable[8][4] = {
124       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'ffff, 0xffff'ffff'ffff'ffff},
125       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'fffe, 0xffff'ffff'ffff'ffff},
126       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'fffc, 0xffff'ffff'ffff'ffff},
127       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'fff8, 0xffff'ffff'ffff'ffff},
128       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'fff0, 0xffff'ffff'ffff'ffff},
129       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'ffe0, 0xffff'ffff'ffff'ffff},
130       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'ffc0, 0xffff'ffff'ffff'ffff},
131       {0x0000'0000'0000'0000, 0x0000'0000'0000'0000, 0xffff'ffff'ffff'ff80, 0xffff'ffff'ffff'ffff},
132   };
133   // RISC-V manual strongly implies that vid.v may be implemented similarly to viota.m
134   // This may be true for hardware implementation, but in software vid.v may be implemented with a
135   // simple precomputed table which implementation of viota.m is much more tricky and slow.
136   // Here are precomputed values for Vid.v
137   alignas(16) Int64x2 kVid64Bit[8] = {
138       {0, 1},
139       {2, 3},
140       {4, 5},
141       {6, 7},
142       {8, 9},
143       {10, 11},
144       {12, 13},
145       {14, 15},
146   };
147   alignas(16) Int32x4 kVid32Bit[8] = {
148       {0, 1, 2, 3},
149       {4, 5, 6, 7},
150       {8, 9, 10, 11},
151       {12, 13, 14, 15},
152       {16, 17, 18, 19},
153       {20, 21, 22, 23},
154       {24, 25, 26, 27},
155       {28, 29, 30, 31},
156   };
157   alignas(16) Int16x8 kVid16Bit[8] = {
158       {0, 1, 2, 3, 4, 5, 6, 7},
159       {8, 9, 10, 11, 12, 13, 14, 15},
160       {16, 17, 18, 19, 20, 21, 22, 23},
161       {24, 25, 26, 27, 28, 29, 30, 31},
162       {32, 33, 34, 35, 36, 37, 38, 39},
163       {40, 41, 42, 43, 44, 45, 46, 47},
164       {48, 49, 50, 51, 52, 53, 54, 55},
165       {56, 57, 58, 59, 60, 61, 62, 63},
166   };
167   alignas(16) Int8x16 kVid8Bit[8] = {
168       {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
169       {16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31},
170       {32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47},
171       {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63},
172       {64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79},
173       {80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95},
174       {96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111},
175       {112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127},
176   };
177   alignas(16) const uint64_t kBitMaskTo32bitMask[4] = {
178       0x0000'0000'0000'0000,
179       0x0000'0000'ffff'ffff,
180       0xffff'ffff'0000'0000,
181       0xffff'ffff'ffff'ffff,
182   };
183   alignas(16) const uint64_t kBitMaskTo16bitMask[16] = {
184       0x0000'0000'0000'0000,
185       0x0000'0000'0000'ffff,
186       0x0000'0000'ffff'0000,
187       0x0000'0000'ffff'ffff,
188       0x0000'ffff'0000'0000,
189       0x0000'ffff'0000'ffff,
190       0x0000'ffff'ffff'0000,
191       0x0000'ffff'ffff'ffff,
192       0xffff'0000'0000'0000,
193       0xffff'0000'0000'ffff,
194       0xffff'0000'ffff'0000,
195       0xffff'0000'ffff'ffff,
196       0xffff'ffff'0000'0000,
197       0xffff'ffff'0000'ffff,
198       0xffff'ffff'ffff'0000,
199       0xffff'ffff'ffff'ffff,
200   };
201   alignas(16) const uint64_t kBitMaskTo8bitMask[256] = {
202       0x0000'0000'0000'0000, 0x0000'0000'0000'00ff, 0x0000'0000'0000'ff00, 0x0000'0000'0000'ffff,
203       0x0000'0000'00ff'0000, 0x0000'0000'00ff'00ff, 0x0000'0000'00ff'ff00, 0x0000'0000'00ff'ffff,
204       0x0000'0000'ff00'0000, 0x0000'0000'ff00'00ff, 0x0000'0000'ff00'ff00, 0x0000'0000'ff00'ffff,
205       0x0000'0000'ffff'0000, 0x0000'0000'ffff'00ff, 0x0000'0000'ffff'ff00, 0x0000'0000'ffff'ffff,
206       0x0000'00ff'0000'0000, 0x0000'00ff'0000'00ff, 0x0000'00ff'0000'ff00, 0x0000'00ff'0000'ffff,
207       0x0000'00ff'00ff'0000, 0x0000'00ff'00ff'00ff, 0x0000'00ff'00ff'ff00, 0x0000'00ff'00ff'ffff,
208       0x0000'00ff'ff00'0000, 0x0000'00ff'ff00'00ff, 0x0000'00ff'ff00'ff00, 0x0000'00ff'ff00'ffff,
209       0x0000'00ff'ffff'0000, 0x0000'00ff'ffff'00ff, 0x0000'00ff'ffff'ff00, 0x0000'00ff'ffff'ffff,
210       0x0000'ff00'0000'0000, 0x0000'ff00'0000'00ff, 0x0000'ff00'0000'ff00, 0x0000'ff00'0000'ffff,
211       0x0000'ff00'00ff'0000, 0x0000'ff00'00ff'00ff, 0x0000'ff00'00ff'ff00, 0x0000'ff00'00ff'ffff,
212       0x0000'ff00'ff00'0000, 0x0000'ff00'ff00'00ff, 0x0000'ff00'ff00'ff00, 0x0000'ff00'ff00'ffff,
213       0x0000'ff00'ffff'0000, 0x0000'ff00'ffff'00ff, 0x0000'ff00'ffff'ff00, 0x0000'ff00'ffff'ffff,
214       0x0000'ffff'0000'0000, 0x0000'ffff'0000'00ff, 0x0000'ffff'0000'ff00, 0x0000'ffff'0000'ffff,
215       0x0000'ffff'00ff'0000, 0x0000'ffff'00ff'00ff, 0x0000'ffff'00ff'ff00, 0x0000'ffff'00ff'ffff,
216       0x0000'ffff'ff00'0000, 0x0000'ffff'ff00'00ff, 0x0000'ffff'ff00'ff00, 0x0000'ffff'ff00'ffff,
217       0x0000'ffff'ffff'0000, 0x0000'ffff'ffff'00ff, 0x0000'ffff'ffff'ff00, 0x0000'ffff'ffff'ffff,
218       0x00ff'0000'0000'0000, 0x00ff'0000'0000'00ff, 0x00ff'0000'0000'ff00, 0x00ff'0000'0000'ffff,
219       0x00ff'0000'00ff'0000, 0x00ff'0000'00ff'00ff, 0x00ff'0000'00ff'ff00, 0x00ff'0000'00ff'ffff,
220       0x00ff'0000'ff00'0000, 0x00ff'0000'ff00'00ff, 0x00ff'0000'ff00'ff00, 0x00ff'0000'ff00'ffff,
221       0x00ff'0000'ffff'0000, 0x00ff'0000'ffff'00ff, 0x00ff'0000'ffff'ff00, 0x00ff'0000'ffff'ffff,
222       0x00ff'00ff'0000'0000, 0x00ff'00ff'0000'00ff, 0x00ff'00ff'0000'ff00, 0x00ff'00ff'0000'ffff,
223       0x00ff'00ff'00ff'0000, 0x00ff'00ff'00ff'00ff, 0x00ff'00ff'00ff'ff00, 0x00ff'00ff'00ff'ffff,
224       0x00ff'00ff'ff00'0000, 0x00ff'00ff'ff00'00ff, 0x00ff'00ff'ff00'ff00, 0x00ff'00ff'ff00'ffff,
225       0x00ff'00ff'ffff'0000, 0x00ff'00ff'ffff'00ff, 0x00ff'00ff'ffff'ff00, 0x00ff'00ff'ffff'ffff,
226       0x00ff'ff00'0000'0000, 0x00ff'ff00'0000'00ff, 0x00ff'ff00'0000'ff00, 0x00ff'ff00'0000'ffff,
227       0x00ff'ff00'00ff'0000, 0x00ff'ff00'00ff'00ff, 0x00ff'ff00'00ff'ff00, 0x00ff'ff00'00ff'ffff,
228       0x00ff'ff00'ff00'0000, 0x00ff'ff00'ff00'00ff, 0x00ff'ff00'ff00'ff00, 0x00ff'ff00'ff00'ffff,
229       0x00ff'ff00'ffff'0000, 0x00ff'ff00'ffff'00ff, 0x00ff'ff00'ffff'ff00, 0x00ff'ff00'ffff'ffff,
230       0x00ff'ffff'0000'0000, 0x00ff'ffff'0000'00ff, 0x00ff'ffff'0000'ff00, 0x00ff'ffff'0000'ffff,
231       0x00ff'ffff'00ff'0000, 0x00ff'ffff'00ff'00ff, 0x00ff'ffff'00ff'ff00, 0x00ff'ffff'00ff'ffff,
232       0x00ff'ffff'ff00'0000, 0x00ff'ffff'ff00'00ff, 0x00ff'ffff'ff00'ff00, 0x00ff'ffff'ff00'ffff,
233       0x00ff'ffff'ffff'0000, 0x00ff'ffff'ffff'00ff, 0x00ff'ffff'ffff'ff00, 0x00ff'ffff'ffff'ffff,
234       0xff00'0000'0000'0000, 0xff00'0000'0000'00ff, 0xff00'0000'0000'ff00, 0xff00'0000'0000'ffff,
235       0xff00'0000'00ff'0000, 0xff00'0000'00ff'00ff, 0xff00'0000'00ff'ff00, 0xff00'0000'00ff'ffff,
236       0xff00'0000'ff00'0000, 0xff00'0000'ff00'00ff, 0xff00'0000'ff00'ff00, 0xff00'0000'ff00'ffff,
237       0xff00'0000'ffff'0000, 0xff00'0000'ffff'00ff, 0xff00'0000'ffff'ff00, 0xff00'0000'ffff'ffff,
238       0xff00'00ff'0000'0000, 0xff00'00ff'0000'00ff, 0xff00'00ff'0000'ff00, 0xff00'00ff'0000'ffff,
239       0xff00'00ff'00ff'0000, 0xff00'00ff'00ff'00ff, 0xff00'00ff'00ff'ff00, 0xff00'00ff'00ff'ffff,
240       0xff00'00ff'ff00'0000, 0xff00'00ff'ff00'00ff, 0xff00'00ff'ff00'ff00, 0xff00'00ff'ff00'ffff,
241       0xff00'00ff'ffff'0000, 0xff00'00ff'ffff'00ff, 0xff00'00ff'ffff'ff00, 0xff00'00ff'ffff'ffff,
242       0xff00'ff00'0000'0000, 0xff00'ff00'0000'00ff, 0xff00'ff00'0000'ff00, 0xff00'ff00'0000'ffff,
243       0xff00'ff00'00ff'0000, 0xff00'ff00'00ff'00ff, 0xff00'ff00'00ff'ff00, 0xff00'ff00'00ff'ffff,
244       0xff00'ff00'ff00'0000, 0xff00'ff00'ff00'00ff, 0xff00'ff00'ff00'ff00, 0xff00'ff00'ff00'ffff,
245       0xff00'ff00'ffff'0000, 0xff00'ff00'ffff'00ff, 0xff00'ff00'ffff'ff00, 0xff00'ff00'ffff'ffff,
246       0xff00'ffff'0000'0000, 0xff00'ffff'0000'00ff, 0xff00'ffff'0000'ff00, 0xff00'ffff'0000'ffff,
247       0xff00'ffff'00ff'0000, 0xff00'ffff'00ff'00ff, 0xff00'ffff'00ff'ff00, 0xff00'ffff'00ff'ffff,
248       0xff00'ffff'ff00'0000, 0xff00'ffff'ff00'00ff, 0xff00'ffff'ff00'ff00, 0xff00'ffff'ff00'ffff,
249       0xff00'ffff'ffff'0000, 0xff00'ffff'ffff'00ff, 0xff00'ffff'ffff'ff00, 0xff00'ffff'ffff'ffff,
250       0xffff'0000'0000'0000, 0xffff'0000'0000'00ff, 0xffff'0000'0000'ff00, 0xffff'0000'0000'ffff,
251       0xffff'0000'00ff'0000, 0xffff'0000'00ff'00ff, 0xffff'0000'00ff'ff00, 0xffff'0000'00ff'ffff,
252       0xffff'0000'ff00'0000, 0xffff'0000'ff00'00ff, 0xffff'0000'ff00'ff00, 0xffff'0000'ff00'ffff,
253       0xffff'0000'ffff'0000, 0xffff'0000'ffff'00ff, 0xffff'0000'ffff'ff00, 0xffff'0000'ffff'ffff,
254       0xffff'00ff'0000'0000, 0xffff'00ff'0000'00ff, 0xffff'00ff'0000'ff00, 0xffff'00ff'0000'ffff,
255       0xffff'00ff'00ff'0000, 0xffff'00ff'00ff'00ff, 0xffff'00ff'00ff'ff00, 0xffff'00ff'00ff'ffff,
256       0xffff'00ff'ff00'0000, 0xffff'00ff'ff00'00ff, 0xffff'00ff'ff00'ff00, 0xffff'00ff'ff00'ffff,
257       0xffff'00ff'ffff'0000, 0xffff'00ff'ffff'00ff, 0xffff'00ff'ffff'ff00, 0xffff'00ff'ffff'ffff,
258       0xffff'ff00'0000'0000, 0xffff'ff00'0000'00ff, 0xffff'ff00'0000'ff00, 0xffff'ff00'0000'ffff,
259       0xffff'ff00'00ff'0000, 0xffff'ff00'00ff'00ff, 0xffff'ff00'00ff'ff00, 0xffff'ff00'00ff'ffff,
260       0xffff'ff00'ff00'0000, 0xffff'ff00'ff00'00ff, 0xffff'ff00'ff00'ff00, 0xffff'ff00'ff00'ffff,
261       0xffff'ff00'ffff'0000, 0xffff'ff00'ffff'00ff, 0xffff'ff00'ffff'ff00, 0xffff'ff00'ffff'ffff,
262       0xffff'ffff'0000'0000, 0xffff'ffff'0000'00ff, 0xffff'ffff'0000'ff00, 0xffff'ffff'0000'ffff,
263       0xffff'ffff'00ff'0000, 0xffff'ffff'00ff'00ff, 0xffff'ffff'00ff'ff00, 0xffff'ffff'00ff'ffff,
264       0xffff'ffff'ff00'0000, 0xffff'ffff'ff00'00ff, 0xffff'ffff'ff00'ff00, 0xffff'ffff'ff00'ffff,
265       0xffff'ffff'ffff'0000, 0xffff'ffff'ffff'00ff, 0xffff'ffff'ffff'ff00, 0xffff'ffff'ffff'ffff,
266   };
267 };
268 
269 // Make sure Layout is the same in 32-bit mode and 64-bit mode.
270 CHECK_STRUCT_LAYOUT(MacroAssemblerConstants, 27520, 128);
271 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kNanBoxFloat32, 0, 128);
272 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kNanBoxedNansFloat32, 128, 128);
273 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kCanonicalNansFloat32, 256, 128);
274 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kCanonicalNansFloat64, 384, 128);
275 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kFloat32One, 512, 128);
276 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kFloat64One, 640, 128);
277 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kFloat32PInf, 768, 128);
278 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kFloat32NInf, 896, 128);
279 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kFloat64PInf, 1024, 128);
280 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kFloat64NInf, 1152, 128);
281 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMinInt8, 1280, 128);
282 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMaxInt8, 1408, 128);
283 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMinInt16, 1536, 128);
284 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMaxInt16, 1664, 128);
285 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMinInt32, 1792, 128);
286 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMaxInt32, 1920, 128);
287 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMinInt64, 2048, 128);
288 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kMaxInt64, 2176, 128);
289 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kBsrToClzInt64, 2304, 64);
290 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kWidthInBits64, 2368, 64);
291 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kBsrToClzInt32, 2432, 32);
292 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kWidthInBits32, 2464, 32);
293 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, k0x8000_0000_0000_00ff, 2496, 64);
294 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kRiscVToX87Exceptions, 2944, 256);
295 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kX87ToRiscVExceptions, 3200, 512);
296 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kBitMaskTable, 3712, 2048);
297 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kVid64Bit, 5760, 1024);
298 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kVid32Bit, 6784, 1024);
299 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kVid16Bit, 7808, 1024);
300 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kVid8Bit, 8832, 1024);
301 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kBitMaskTo32bitMask, 9856, 256);
302 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kBitMaskTo16bitMask, 10112, 1024);
303 CHECK_FIELD_LAYOUT(MacroAssemblerConstants, kBitMaskTo8bitMask, 11136, 16384);
304 
305 // Note: because we have aligned fields and thus padding in that data structure
306 // value-initialization is both slower and larger than copy-initialization for
307 // that structure.
308 //
309 // Also assembler intrinsics for interpreter use kMacroAssemblerConstants
310 // directly (they couldn't use consts below because these addresses are only
311 // known during runtime)
312 extern const MacroAssemblerConstants kBerberisMacroAssemblerConstants
313     __attribute__((visibility("hidden")));
314 const MacroAssemblerConstants kBerberisMacroAssemblerConstants;
315 
316 namespace {
317 
GetConstants()318 int32_t GetConstants() {
319   static const MacroAssemblerConstants* Constants =
320       new (mmap(nullptr,
321                 AlignUpPageSize(sizeof(MacroAssemblerConstants)),
322                 PROT_READ | PROT_WRITE,
323                 MAP_PRIVATE | MAP_ANONYMOUS | MAP_32BIT,
324                 -1,
325                 0)) MacroAssemblerConstants(kBerberisMacroAssemblerConstants);
326   // Note that we are returning only 32-bit address here, but it's guaranteed to
327   // be enough since struct is in low memory because of MAP_32BIT flag.
328   return bit_cast<intptr_t>(Constants);
329 }
330 
331 }  // namespace
332 
333 extern const int32_t kBerberisMacroAssemblerConstantsRelocated;
334 const int32_t kBerberisMacroAssemblerConstantsRelocated = GetConstants();
335 template <>
336 extern const int32_t kVectorConst<int8_t{-128}> =
337     GetConstants() + offsetof(MacroAssemblerConstants, kMinInt8);
338 template <>
339 extern const int32_t kVectorConst<int8_t{127}> =
340     GetConstants() + offsetof(MacroAssemblerConstants, kMaxInt8);
341 template <>
342 extern const int32_t kVectorConst<int16_t{-0x8000}> =
343     GetConstants() + offsetof(MacroAssemblerConstants, kMinInt16);
344 template <>
345 extern const int32_t kVectorConst<int16_t{0x7fff}> =
346     GetConstants() + offsetof(MacroAssemblerConstants, kMaxInt16);
347 template <>
348 extern const int32_t kVectorConst<int32_t{static_cast<int32_t>(-0x8000'0000)}> =
349     GetConstants() + offsetof(MacroAssemblerConstants, kMinInt32);
350 template <>
351 extern const int32_t kVectorConst<int32_t{0x3f80'0000}> =
352     GetConstants() + offsetof(MacroAssemblerConstants, kFloat32One);
353 template <>
354 extern const int32_t kVectorConst<int32_t{0x7f80'0000}> =
355     GetConstants() + offsetof(MacroAssemblerConstants, kFloat32PInf);
356 template <>
357 extern const int32_t kVectorConst<int32_t{0x7fff'ffff}> =
358     GetConstants() + offsetof(MacroAssemblerConstants, kMaxInt32);
359 template <>
360 extern const int32_t kVectorConst<int32_t{-0x0080'0000}> =
361     GetConstants() + offsetof(MacroAssemblerConstants, kFloat32NInf);
362 template <>
363 extern const int32_t kVectorConst<int64_t{static_cast<int64_t>(-0x8000'0000'0000'0000)}> =
364     GetConstants() + offsetof(MacroAssemblerConstants, kMinInt64);
365 template <>
366 extern const int32_t kVectorConst<int64_t{0x3ff0'0000'0000'0000}> =
367     GetConstants() + offsetof(MacroAssemblerConstants, kFloat64One);
368 template <>
369 extern const int32_t kVectorConst<int64_t{0x7ff0'0000'0000'0000}> =
370     GetConstants() + offsetof(MacroAssemblerConstants, kFloat64PInf);
371 template <>
372 extern const int32_t kVectorConst<int64_t{0x7fff'ffff'ffff'ffff}> =
373     GetConstants() + offsetof(MacroAssemblerConstants, kMaxInt64);
374 template <>
375 extern const int32_t kVectorConst<int64_t{-0x0010'0000'0000'0000}> =
376     GetConstants() + offsetof(MacroAssemblerConstants, kFloat64NInf);
377 template <>
378 const int32_t kVectorConst<uint64_t{0x0000'0000'0000'0000}> =
379     GetConstants() + offsetof(MacroAssemblerConstants, kBitMaskTable);
380 template <>
381 const int32_t kVectorConst<uint64_t{0xffff'ffff'ffff'ffff}> =
382     GetConstants() + offsetof(MacroAssemblerConstants, kBitMaskTable) + 16;
383 template <>
384 const int32_t kVectorConst<uint64_t{0xffff'ffff'0000'0000}> =
385     GetConstants() + offsetof(MacroAssemblerConstants, kNanBoxFloat32);
386 template <>
387 const int32_t kVectorConst<uint64_t{0xffff'ffff'7fc0'0000}> =
388     GetConstants() + offsetof(MacroAssemblerConstants, kNanBoxedNansFloat32);
389 template <>
390 const int32_t kVectorConst<uint64_t{0x7fc0'0000'7fc0'0000}> =
391     GetConstants() + offsetof(MacroAssemblerConstants, kCanonicalNansFloat32);
392 template <>
393 const int32_t kVectorConst<uint64_t{0x7ff8'0000'0000'0000}> =
394     GetConstants() + offsetof(MacroAssemblerConstants, kCanonicalNansFloat64);
395 template <>
396 const int32_t kConst<uint64_t{127}> =
397     GetConstants() + offsetof(MacroAssemblerConstants, kBsrToClzInt64);
398 template <>
399 const int32_t kConst<uint64_t{64}> =
400     GetConstants() + offsetof(MacroAssemblerConstants, kWidthInBits64);
401 template <>
402 const int32_t kConst<uint32_t{63}> =
403     GetConstants() + offsetof(MacroAssemblerConstants, kBsrToClzInt32);
404 template <>
405 const int32_t kConst<uint32_t{32}> =
406     GetConstants() + offsetof(MacroAssemblerConstants, kWidthInBits32);
407 template <>
408 const int32_t kConst<uint64_t{0x8000'0000'0000'00ff}> =
409     GetConstants() + offsetof(MacroAssemblerConstants, k0x8000_0000_0000_00ff);
410 const int32_t kRiscVToX87Exceptions =
411     GetConstants() + offsetof(MacroAssemblerConstants, kRiscVToX87Exceptions);
412 const int32_t kX87ToRiscVExceptions =
413     GetConstants() + offsetof(MacroAssemblerConstants, kX87ToRiscVExceptions);
414 const int32_t kBitMaskTable = GetConstants() + offsetof(MacroAssemblerConstants, kBitMaskTable);
415 const int32_t kVid64Bit = GetConstants() + offsetof(MacroAssemblerConstants, kVid64Bit);
416 const int32_t kVid32Bit = GetConstants() + offsetof(MacroAssemblerConstants, kVid32Bit);
417 const int32_t kVid16Bit = GetConstants() + offsetof(MacroAssemblerConstants, kVid16Bit);
418 const int32_t kVid8Bit = GetConstants() + offsetof(MacroAssemblerConstants, kVid8Bit);
419 const int32_t kBitMaskTo32bitMask =
420     GetConstants() + offsetof(MacroAssemblerConstants, kBitMaskTo32bitMask);
421 const int32_t kBitMaskTo16bitMask =
422     GetConstants() + offsetof(MacroAssemblerConstants, kBitMaskTo16bitMask);
423 const int32_t kBitMaskTo8bitMask =
424     GetConstants() + offsetof(MacroAssemblerConstants, kBitMaskTo8bitMask);
425 const int32_t kPMovmskwToPMovmskb =
426     GetConstants() + offsetof(MacroAssemblerConstants, kPMovmskwToPMovmskb);
427 const int32_t kPMovmskdToPMovmskb =
428     GetConstants() + offsetof(MacroAssemblerConstants, kPMovmskdToPMovmskb);
429 const int32_t kPMovmskqToPMovmskb =
430     GetConstants() + offsetof(MacroAssemblerConstants, kPMovmskqToPMovmskb);
431 
432 }  // namespace berberis::constants_pool
433