1 // Copyright 2022 The Abseil Authors.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // https://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #ifndef ABSL_BASE_INTERNAL_PREFETCH_H_
16 #define ABSL_BASE_INTERNAL_PREFETCH_H_
17
18 #include "absl/base/config.h"
19
20 #ifdef __SSE__
21 #include <xmmintrin.h>
22 #endif
23
24 #if defined(_MSC_VER) && defined(ABSL_INTERNAL_HAVE_SSE)
25 #include <intrin.h>
26 #pragma intrinsic(_mm_prefetch)
27 #endif
28
29 // Compatibility wrappers around __builtin_prefetch, to prefetch data
30 // for read if supported by the toolchain.
31
32 // Move data into the cache before it is read, or "prefetch" it.
33 //
34 // The value of `addr` is the address of the memory to prefetch. If
35 // the target and compiler support it, data prefetch instructions are
36 // generated. If the prefetch is done some time before the memory is
37 // read, it may be in the cache by the time the read occurs.
38 //
39 // The function names specify the temporal locality heuristic applied,
40 // using the names of Intel prefetch instructions:
41 //
42 // T0 - high degree of temporal locality; data should be left in as
43 // many levels of the cache possible
44 // T1 - moderate degree of temporal locality
45 // T2 - low degree of temporal locality
46 // Nta - no temporal locality, data need not be left in the cache
47 // after the read
48 //
49 // Incorrect or gratuitous use of these functions can degrade
50 // performance, so use them only when representative benchmarks show
51 // an improvement.
52 //
53 // Example usage:
54 //
55 // absl::base_internal::PrefetchT0(addr);
56 //
57 // Currently, the different prefetch calls behave on some Intel
58 // architectures as follows:
59 //
60 // SNB..SKL SKX
61 // PrefetchT0() L1/L2/L3 L1/L2
62 // PrefetchT1() L2/L3 L2
63 // PrefetchT2() L2/L3 L2
64 // PrefetchNta() L1/--/L3 L1*
65 //
66 // * On SKX PrefetchNta() will bring the line into L1 but will evict
67 // from L3 cache. This might result in surprising behavior.
68 //
69 // SNB = Sandy Bridge, SKL = Skylake, SKX = Skylake Xeon.
70 //
71 namespace absl {
72 ABSL_NAMESPACE_BEGIN
73 namespace base_internal {
74
75 void PrefetchT0(const void* addr);
76 void PrefetchT1(const void* addr);
77 void PrefetchT2(const void* addr);
78 void PrefetchNta(const void* addr);
79
80 // Implementation details follow.
81
82 #if ABSL_HAVE_BUILTIN(__builtin_prefetch) || defined(__GNUC__)
83
84 #define ABSL_INTERNAL_HAVE_PREFETCH 1
85
86 // See __builtin_prefetch:
87 // https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html.
88 //
89 // These functions speculatively load for read only. This is
90 // safe for all currently supported platforms. However, prefetch for
91 // store may have problems depending on the target platform.
92 //
PrefetchT0(const void * addr)93 inline void PrefetchT0(const void* addr) {
94 // Note: this uses prefetcht0 on Intel.
95 __builtin_prefetch(addr, 0, 3);
96 }
PrefetchT1(const void * addr)97 inline void PrefetchT1(const void* addr) {
98 // Note: this uses prefetcht1 on Intel.
99 __builtin_prefetch(addr, 0, 2);
100 }
PrefetchT2(const void * addr)101 inline void PrefetchT2(const void* addr) {
102 // Note: this uses prefetcht2 on Intel.
103 __builtin_prefetch(addr, 0, 1);
104 }
PrefetchNta(const void * addr)105 inline void PrefetchNta(const void* addr) {
106 // Note: this uses prefetchtnta on Intel.
107 __builtin_prefetch(addr, 0, 0);
108 }
109
110 #elif defined(ABSL_INTERNAL_HAVE_SSE)
111
112 #define ABSL_INTERNAL_HAVE_PREFETCH 1
113
PrefetchT0(const void * addr)114 inline void PrefetchT0(const void* addr) {
115 _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T0);
116 }
PrefetchT1(const void * addr)117 inline void PrefetchT1(const void* addr) {
118 _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T1);
119 }
PrefetchT2(const void * addr)120 inline void PrefetchT2(const void* addr) {
121 _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T2);
122 }
PrefetchNta(const void * addr)123 inline void PrefetchNta(const void* addr) {
124 _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_NTA);
125 }
126
127 #else
PrefetchT0(const void *)128 inline void PrefetchT0(const void*) {}
PrefetchT1(const void *)129 inline void PrefetchT1(const void*) {}
PrefetchT2(const void *)130 inline void PrefetchT2(const void*) {}
PrefetchNta(const void *)131 inline void PrefetchNta(const void*) {}
132 #endif
133
134 } // namespace base_internal
135 ABSL_NAMESPACE_END
136 } // namespace absl
137
138 #endif // ABSL_BASE_INTERNAL_PREFETCH_H_
139