xref: /aosp_15_r20/art/libartbase/base/mem_map.cc (revision 795d594fd825385562da6b089ea9b2033f3abf5a)
1*795d594fSAndroid Build Coastguard Worker /*
2*795d594fSAndroid Build Coastguard Worker  * Copyright (C) 2008 The Android Open Source Project
3*795d594fSAndroid Build Coastguard Worker  *
4*795d594fSAndroid Build Coastguard Worker  * Licensed under the Apache License, Version 2.0 (the "License");
5*795d594fSAndroid Build Coastguard Worker  * you may not use this file except in compliance with the License.
6*795d594fSAndroid Build Coastguard Worker  * You may obtain a copy of the License at
7*795d594fSAndroid Build Coastguard Worker  *
8*795d594fSAndroid Build Coastguard Worker  *      http://www.apache.org/licenses/LICENSE-2.0
9*795d594fSAndroid Build Coastguard Worker  *
10*795d594fSAndroid Build Coastguard Worker  * Unless required by applicable law or agreed to in writing, software
11*795d594fSAndroid Build Coastguard Worker  * distributed under the License is distributed on an "AS IS" BASIS,
12*795d594fSAndroid Build Coastguard Worker  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13*795d594fSAndroid Build Coastguard Worker  * See the License for the specific language governing permissions and
14*795d594fSAndroid Build Coastguard Worker  * limitations under the License.
15*795d594fSAndroid Build Coastguard Worker  */
16*795d594fSAndroid Build Coastguard Worker 
17*795d594fSAndroid Build Coastguard Worker #include "mem_map.h"
18*795d594fSAndroid Build Coastguard Worker 
19*795d594fSAndroid Build Coastguard Worker #include <inttypes.h>
20*795d594fSAndroid Build Coastguard Worker #include <stdlib.h>
21*795d594fSAndroid Build Coastguard Worker #if !defined(ANDROID_OS) && !defined(__Fuchsia__) && !defined(_WIN32)
22*795d594fSAndroid Build Coastguard Worker #include <sys/resource.h>
23*795d594fSAndroid Build Coastguard Worker #endif
24*795d594fSAndroid Build Coastguard Worker 
25*795d594fSAndroid Build Coastguard Worker #if defined(__linux__)
26*795d594fSAndroid Build Coastguard Worker #include <sys/prctl.h>
27*795d594fSAndroid Build Coastguard Worker #endif
28*795d594fSAndroid Build Coastguard Worker 
29*795d594fSAndroid Build Coastguard Worker #include <map>
30*795d594fSAndroid Build Coastguard Worker #include <memory>
31*795d594fSAndroid Build Coastguard Worker #include <sstream>
32*795d594fSAndroid Build Coastguard Worker 
33*795d594fSAndroid Build Coastguard Worker #include "android-base/stringprintf.h"
34*795d594fSAndroid Build Coastguard Worker #include "android-base/unique_fd.h"
35*795d594fSAndroid Build Coastguard Worker 
36*795d594fSAndroid Build Coastguard Worker #include "allocator.h"
37*795d594fSAndroid Build Coastguard Worker #include "bit_utils.h"
38*795d594fSAndroid Build Coastguard Worker #include "globals.h"
39*795d594fSAndroid Build Coastguard Worker #include "logging.h"  // For VLOG_IS_ON.
40*795d594fSAndroid Build Coastguard Worker #include "memory_tool.h"
41*795d594fSAndroid Build Coastguard Worker #include "mman.h"  // For the PROT_* and MAP_* constants.
42*795d594fSAndroid Build Coastguard Worker #include "utils.h"
43*795d594fSAndroid Build Coastguard Worker 
44*795d594fSAndroid Build Coastguard Worker #ifndef MAP_ANONYMOUS
45*795d594fSAndroid Build Coastguard Worker #define MAP_ANONYMOUS MAP_ANON
46*795d594fSAndroid Build Coastguard Worker #endif
47*795d594fSAndroid Build Coastguard Worker 
48*795d594fSAndroid Build Coastguard Worker namespace art {
49*795d594fSAndroid Build Coastguard Worker 
50*795d594fSAndroid Build Coastguard Worker using android::base::StringPrintf;
51*795d594fSAndroid Build Coastguard Worker using android::base::unique_fd;
52*795d594fSAndroid Build Coastguard Worker 
53*795d594fSAndroid Build Coastguard Worker template<class Key, class T, AllocatorTag kTag, class Compare = std::less<Key>>
54*795d594fSAndroid Build Coastguard Worker using AllocationTrackingMultiMap =
55*795d594fSAndroid Build Coastguard Worker     std::multimap<Key, T, Compare, TrackingAllocator<std::pair<const Key, T>, kTag>>;
56*795d594fSAndroid Build Coastguard Worker 
57*795d594fSAndroid Build Coastguard Worker using Maps = AllocationTrackingMultiMap<void*, MemMap*, kAllocatorTagMaps>;
58*795d594fSAndroid Build Coastguard Worker 
59*795d594fSAndroid Build Coastguard Worker // All the non-empty MemMaps. Use a multimap as we do a reserve-and-divide (eg ElfMap::Load()).
60*795d594fSAndroid Build Coastguard Worker static Maps* gMaps GUARDED_BY(MemMap::GetMemMapsLock()) = nullptr;
61*795d594fSAndroid Build Coastguard Worker 
62*795d594fSAndroid Build Coastguard Worker // A map containing unique strings used for indentifying anonymous mappings
63*795d594fSAndroid Build Coastguard Worker static std::map<std::string, int> debugStrMap GUARDED_BY(MemMap::GetMemMapsLock());
64*795d594fSAndroid Build Coastguard Worker 
65*795d594fSAndroid Build Coastguard Worker // Retrieve iterator to a `gMaps` entry that is known to exist.
GetGMapsEntry(const MemMap & map)66*795d594fSAndroid Build Coastguard Worker Maps::iterator GetGMapsEntry(const MemMap& map) REQUIRES(MemMap::GetMemMapsLock()) {
67*795d594fSAndroid Build Coastguard Worker   DCHECK(map.IsValid());
68*795d594fSAndroid Build Coastguard Worker   DCHECK(gMaps != nullptr);
69*795d594fSAndroid Build Coastguard Worker   for (auto it = gMaps->lower_bound(map.BaseBegin()), end = gMaps->end();
70*795d594fSAndroid Build Coastguard Worker        it != end && it->first == map.BaseBegin();
71*795d594fSAndroid Build Coastguard Worker        ++it) {
72*795d594fSAndroid Build Coastguard Worker     if (it->second == &map) {
73*795d594fSAndroid Build Coastguard Worker       return it;
74*795d594fSAndroid Build Coastguard Worker     }
75*795d594fSAndroid Build Coastguard Worker   }
76*795d594fSAndroid Build Coastguard Worker   LOG(FATAL) << "MemMap not found";
77*795d594fSAndroid Build Coastguard Worker   UNREACHABLE();
78*795d594fSAndroid Build Coastguard Worker }
79*795d594fSAndroid Build Coastguard Worker 
operator <<(std::ostream & os,const Maps & mem_maps)80*795d594fSAndroid Build Coastguard Worker std::ostream& operator<<(std::ostream& os, const Maps& mem_maps) {
81*795d594fSAndroid Build Coastguard Worker   os << "MemMap:" << std::endl;
82*795d594fSAndroid Build Coastguard Worker   for (auto it = mem_maps.begin(); it != mem_maps.end(); ++it) {
83*795d594fSAndroid Build Coastguard Worker     void* base = it->first;
84*795d594fSAndroid Build Coastguard Worker     MemMap* map = it->second;
85*795d594fSAndroid Build Coastguard Worker     CHECK_EQ(base, map->BaseBegin());
86*795d594fSAndroid Build Coastguard Worker     os << *map << std::endl;
87*795d594fSAndroid Build Coastguard Worker   }
88*795d594fSAndroid Build Coastguard Worker   return os;
89*795d594fSAndroid Build Coastguard Worker }
90*795d594fSAndroid Build Coastguard Worker 
91*795d594fSAndroid Build Coastguard Worker std::mutex* MemMap::mem_maps_lock_ = nullptr;
92*795d594fSAndroid Build Coastguard Worker #ifdef ART_PAGE_SIZE_AGNOSTIC
93*795d594fSAndroid Build Coastguard Worker size_t MemMap::page_size_ = 0;
94*795d594fSAndroid Build Coastguard Worker #endif
95*795d594fSAndroid Build Coastguard Worker 
96*795d594fSAndroid Build Coastguard Worker #if USE_ART_LOW_4G_ALLOCATOR
97*795d594fSAndroid Build Coastguard Worker // Handling mem_map in 32b address range for 64b architectures that do not support MAP_32BIT.
98*795d594fSAndroid Build Coastguard Worker 
99*795d594fSAndroid Build Coastguard Worker // The regular start of memory allocations. The first 64KB is protected by SELinux.
100*795d594fSAndroid Build Coastguard Worker static constexpr uintptr_t LOW_MEM_START = 64 * KB;
101*795d594fSAndroid Build Coastguard Worker 
102*795d594fSAndroid Build Coastguard Worker // Generate random starting position.
103*795d594fSAndroid Build Coastguard Worker // To not interfere with image position, take the image's address and only place it below. Current
104*795d594fSAndroid Build Coastguard Worker // formula (sketch):
105*795d594fSAndroid Build Coastguard Worker //
106*795d594fSAndroid Build Coastguard Worker // ART_BASE_ADDR      = 0001XXXXXXXXXXXXXXX
107*795d594fSAndroid Build Coastguard Worker // ----------------------------------------
108*795d594fSAndroid Build Coastguard Worker //                    = 0000111111111111111
109*795d594fSAndroid Build Coastguard Worker // & ~(page_size - 1) =~0000000000000001111
110*795d594fSAndroid Build Coastguard Worker // ----------------------------------------
111*795d594fSAndroid Build Coastguard Worker // mask               = 0000111111111110000
112*795d594fSAndroid Build Coastguard Worker // & random data      = YYYYYYYYYYYYYYYYYYY
113*795d594fSAndroid Build Coastguard Worker // -----------------------------------
114*795d594fSAndroid Build Coastguard Worker // tmp                = 0000YYYYYYYYYYY0000
115*795d594fSAndroid Build Coastguard Worker // + LOW_MEM_START    = 0000000000001000000
116*795d594fSAndroid Build Coastguard Worker // --------------------------------------
117*795d594fSAndroid Build Coastguard Worker // start
118*795d594fSAndroid Build Coastguard Worker //
119*795d594fSAndroid Build Coastguard Worker // arc4random as an entropy source is exposed in Bionic, but not in glibc. When we
120*795d594fSAndroid Build Coastguard Worker // do not have Bionic, simply start with LOW_MEM_START.
121*795d594fSAndroid Build Coastguard Worker 
122*795d594fSAndroid Build Coastguard Worker // Function is standalone so it can be tested somewhat in mem_map_test.cc.
123*795d594fSAndroid Build Coastguard Worker #ifdef __BIONIC__
CreateStartPos(uint64_t input,size_t page_size)124*795d594fSAndroid Build Coastguard Worker uintptr_t CreateStartPos(uint64_t input, size_t page_size) {
125*795d594fSAndroid Build Coastguard Worker   CHECK_NE(0, ART_BASE_ADDRESS);
126*795d594fSAndroid Build Coastguard Worker 
127*795d594fSAndroid Build Coastguard Worker   // Start with all bits below highest bit in ART_BASE_ADDRESS.
128*795d594fSAndroid Build Coastguard Worker   constexpr size_t leading_zeros = CLZ(static_cast<uint32_t>(ART_BASE_ADDRESS));
129*795d594fSAndroid Build Coastguard Worker   constexpr uintptr_t mask_ones = (1 << (31 - leading_zeros)) - 1;
130*795d594fSAndroid Build Coastguard Worker 
131*795d594fSAndroid Build Coastguard Worker   // Lowest (usually 12) bits are not used, as aligned by page size.
132*795d594fSAndroid Build Coastguard Worker   const uintptr_t mask = mask_ones & ~(page_size - 1);
133*795d594fSAndroid Build Coastguard Worker 
134*795d594fSAndroid Build Coastguard Worker   // Mask input data.
135*795d594fSAndroid Build Coastguard Worker   return (input & mask) + LOW_MEM_START;
136*795d594fSAndroid Build Coastguard Worker }
137*795d594fSAndroid Build Coastguard Worker #endif
138*795d594fSAndroid Build Coastguard Worker 
GenerateNextMemPos(size_t page_size)139*795d594fSAndroid Build Coastguard Worker static uintptr_t GenerateNextMemPos(size_t page_size) {
140*795d594fSAndroid Build Coastguard Worker #ifdef __BIONIC__
141*795d594fSAndroid Build Coastguard Worker   uint64_t random_data;
142*795d594fSAndroid Build Coastguard Worker   arc4random_buf(&random_data, sizeof(random_data));
143*795d594fSAndroid Build Coastguard Worker   return CreateStartPos(random_data, page_size);
144*795d594fSAndroid Build Coastguard Worker #else
145*795d594fSAndroid Build Coastguard Worker   UNUSED(page_size);
146*795d594fSAndroid Build Coastguard Worker   // No arc4random on host, see above.
147*795d594fSAndroid Build Coastguard Worker   return LOW_MEM_START;
148*795d594fSAndroid Build Coastguard Worker #endif
149*795d594fSAndroid Build Coastguard Worker }
150*795d594fSAndroid Build Coastguard Worker 
151*795d594fSAndroid Build Coastguard Worker uintptr_t MemMap::next_mem_pos_;
152*795d594fSAndroid Build Coastguard Worker #endif
153*795d594fSAndroid Build Coastguard Worker 
154*795d594fSAndroid Build Coastguard Worker // Return true if the address range is contained in a single memory map by either reading
155*795d594fSAndroid Build Coastguard Worker // the gMaps variable or the /proc/self/map entry.
ContainedWithinExistingMap(uint8_t * ptr,size_t size,std::string * error_msg)156*795d594fSAndroid Build Coastguard Worker bool MemMap::ContainedWithinExistingMap(uint8_t* ptr, size_t size, std::string* error_msg) {
157*795d594fSAndroid Build Coastguard Worker   uintptr_t begin = reinterpret_cast<uintptr_t>(ptr);
158*795d594fSAndroid Build Coastguard Worker   uintptr_t end = begin + size;
159*795d594fSAndroid Build Coastguard Worker 
160*795d594fSAndroid Build Coastguard Worker   {
161*795d594fSAndroid Build Coastguard Worker     std::lock_guard<std::mutex> mu(*mem_maps_lock_);
162*795d594fSAndroid Build Coastguard Worker     for (auto& pair : *gMaps) {
163*795d594fSAndroid Build Coastguard Worker       MemMap* const map = pair.second;
164*795d594fSAndroid Build Coastguard Worker       if (begin >= reinterpret_cast<uintptr_t>(map->Begin()) &&
165*795d594fSAndroid Build Coastguard Worker           end <= reinterpret_cast<uintptr_t>(map->End())) {
166*795d594fSAndroid Build Coastguard Worker         return true;
167*795d594fSAndroid Build Coastguard Worker       }
168*795d594fSAndroid Build Coastguard Worker     }
169*795d594fSAndroid Build Coastguard Worker   }
170*795d594fSAndroid Build Coastguard Worker 
171*795d594fSAndroid Build Coastguard Worker   if (error_msg != nullptr) {
172*795d594fSAndroid Build Coastguard Worker     PrintFileToLog("/proc/self/maps", LogSeverity::ERROR);
173*795d594fSAndroid Build Coastguard Worker     *error_msg = StringPrintf("Requested region 0x%08" PRIxPTR "-0x%08" PRIxPTR " does not overlap "
174*795d594fSAndroid Build Coastguard Worker                               "any existing map. See process maps in the log.", begin, end);
175*795d594fSAndroid Build Coastguard Worker   }
176*795d594fSAndroid Build Coastguard Worker   return false;
177*795d594fSAndroid Build Coastguard Worker }
178*795d594fSAndroid Build Coastguard Worker 
179*795d594fSAndroid Build Coastguard Worker // CheckMapRequest to validate a non-MAP_FAILED mmap result based on
180*795d594fSAndroid Build Coastguard Worker // the expected value, calling munmap if validation fails, giving the
181*795d594fSAndroid Build Coastguard Worker // reason in error_msg.
182*795d594fSAndroid Build Coastguard Worker //
183*795d594fSAndroid Build Coastguard Worker // If the expected_ptr is null, nothing is checked beyond the fact
184*795d594fSAndroid Build Coastguard Worker // that the actual_ptr is not MAP_FAILED. However, if expected_ptr is
185*795d594fSAndroid Build Coastguard Worker // non-null, we check that pointer is the actual_ptr == expected_ptr,
186*795d594fSAndroid Build Coastguard Worker // and if not, report in error_msg what the conflict mapping was if
187*795d594fSAndroid Build Coastguard Worker // found, or a generic error in other cases.
CheckMapRequest(uint8_t * expected_ptr,void * actual_ptr,size_t byte_count,std::string * error_msg)188*795d594fSAndroid Build Coastguard Worker bool MemMap::CheckMapRequest(uint8_t* expected_ptr, void* actual_ptr, size_t byte_count,
189*795d594fSAndroid Build Coastguard Worker                             std::string* error_msg) {
190*795d594fSAndroid Build Coastguard Worker   // Handled first by caller for more specific error messages.
191*795d594fSAndroid Build Coastguard Worker   CHECK(actual_ptr != MAP_FAILED);
192*795d594fSAndroid Build Coastguard Worker 
193*795d594fSAndroid Build Coastguard Worker   if (expected_ptr == nullptr) {
194*795d594fSAndroid Build Coastguard Worker     return true;
195*795d594fSAndroid Build Coastguard Worker   }
196*795d594fSAndroid Build Coastguard Worker 
197*795d594fSAndroid Build Coastguard Worker   uintptr_t actual = reinterpret_cast<uintptr_t>(actual_ptr);
198*795d594fSAndroid Build Coastguard Worker   uintptr_t expected = reinterpret_cast<uintptr_t>(expected_ptr);
199*795d594fSAndroid Build Coastguard Worker 
200*795d594fSAndroid Build Coastguard Worker   if (expected_ptr == actual_ptr) {
201*795d594fSAndroid Build Coastguard Worker     return true;
202*795d594fSAndroid Build Coastguard Worker   }
203*795d594fSAndroid Build Coastguard Worker 
204*795d594fSAndroid Build Coastguard Worker   // We asked for an address but didn't get what we wanted, all paths below here should fail.
205*795d594fSAndroid Build Coastguard Worker   int result = TargetMUnmap(actual_ptr, byte_count);
206*795d594fSAndroid Build Coastguard Worker   if (result == -1) {
207*795d594fSAndroid Build Coastguard Worker     PLOG(WARNING) << StringPrintf("munmap(%p, %zd) failed", actual_ptr, byte_count);
208*795d594fSAndroid Build Coastguard Worker   }
209*795d594fSAndroid Build Coastguard Worker 
210*795d594fSAndroid Build Coastguard Worker   if (error_msg != nullptr) {
211*795d594fSAndroid Build Coastguard Worker     // We call this here so that we can try and generate a full error
212*795d594fSAndroid Build Coastguard Worker     // message with the overlapping mapping. There's no guarantee that
213*795d594fSAndroid Build Coastguard Worker     // that there will be an overlap though, since
214*795d594fSAndroid Build Coastguard Worker     // - The kernel is not *required* to honor expected_ptr unless MAP_FIXED is
215*795d594fSAndroid Build Coastguard Worker     //   true, even if there is no overlap
216*795d594fSAndroid Build Coastguard Worker     // - There might have been an overlap at the point of mmap, but the
217*795d594fSAndroid Build Coastguard Worker     //   overlapping region has since been unmapped.
218*795d594fSAndroid Build Coastguard Worker 
219*795d594fSAndroid Build Coastguard Worker     // Tell the client the mappings that were in place at the time.
220*795d594fSAndroid Build Coastguard Worker     if (kIsDebugBuild) {
221*795d594fSAndroid Build Coastguard Worker       PrintFileToLog("/proc/self/maps", LogSeverity::WARNING);
222*795d594fSAndroid Build Coastguard Worker     }
223*795d594fSAndroid Build Coastguard Worker 
224*795d594fSAndroid Build Coastguard Worker     std::ostringstream os;
225*795d594fSAndroid Build Coastguard Worker     os <<  StringPrintf("Failed to mmap at expected address, mapped at "
226*795d594fSAndroid Build Coastguard Worker                         "0x%08" PRIxPTR " instead of 0x%08" PRIxPTR,
227*795d594fSAndroid Build Coastguard Worker                         actual, expected);
228*795d594fSAndroid Build Coastguard Worker     *error_msg = os.str();
229*795d594fSAndroid Build Coastguard Worker   }
230*795d594fSAndroid Build Coastguard Worker   return false;
231*795d594fSAndroid Build Coastguard Worker }
232*795d594fSAndroid Build Coastguard Worker 
CheckReservation(uint8_t * expected_ptr,size_t byte_count,const char * name,const MemMap & reservation,std::string * error_msg)233*795d594fSAndroid Build Coastguard Worker bool MemMap::CheckReservation(uint8_t* expected_ptr,
234*795d594fSAndroid Build Coastguard Worker                               size_t byte_count,
235*795d594fSAndroid Build Coastguard Worker                               const char* name,
236*795d594fSAndroid Build Coastguard Worker                               const MemMap& reservation,
237*795d594fSAndroid Build Coastguard Worker                               /*out*/std::string* error_msg) {
238*795d594fSAndroid Build Coastguard Worker   if (!reservation.IsValid()) {
239*795d594fSAndroid Build Coastguard Worker     *error_msg = StringPrintf("Invalid reservation for %s", name);
240*795d594fSAndroid Build Coastguard Worker     return false;
241*795d594fSAndroid Build Coastguard Worker   }
242*795d594fSAndroid Build Coastguard Worker   DCHECK_ALIGNED_PARAM(reservation.Begin(), GetPageSize());
243*795d594fSAndroid Build Coastguard Worker   if (reservation.Begin() != expected_ptr) {
244*795d594fSAndroid Build Coastguard Worker     *error_msg = StringPrintf("Bad image reservation start for %s: %p instead of %p",
245*795d594fSAndroid Build Coastguard Worker                               name,
246*795d594fSAndroid Build Coastguard Worker                               reservation.Begin(),
247*795d594fSAndroid Build Coastguard Worker                               expected_ptr);
248*795d594fSAndroid Build Coastguard Worker     return false;
249*795d594fSAndroid Build Coastguard Worker   }
250*795d594fSAndroid Build Coastguard Worker   if (byte_count > reservation.Size()) {
251*795d594fSAndroid Build Coastguard Worker     *error_msg = StringPrintf("Insufficient reservation, required %zu, available %zu",
252*795d594fSAndroid Build Coastguard Worker                               byte_count,
253*795d594fSAndroid Build Coastguard Worker                               reservation.Size());
254*795d594fSAndroid Build Coastguard Worker     return false;
255*795d594fSAndroid Build Coastguard Worker   }
256*795d594fSAndroid Build Coastguard Worker   return true;
257*795d594fSAndroid Build Coastguard Worker }
258*795d594fSAndroid Build Coastguard Worker 
259*795d594fSAndroid Build Coastguard Worker 
260*795d594fSAndroid Build Coastguard Worker #if USE_ART_LOW_4G_ALLOCATOR
TryMemMapLow4GB(void * ptr,size_t page_aligned_byte_count,int prot,int flags,int fd,off_t offset)261*795d594fSAndroid Build Coastguard Worker void* MemMap::TryMemMapLow4GB(void* ptr,
262*795d594fSAndroid Build Coastguard Worker                                     size_t page_aligned_byte_count,
263*795d594fSAndroid Build Coastguard Worker                                     int prot,
264*795d594fSAndroid Build Coastguard Worker                                     int flags,
265*795d594fSAndroid Build Coastguard Worker                                     int fd,
266*795d594fSAndroid Build Coastguard Worker                                     off_t offset) {
267*795d594fSAndroid Build Coastguard Worker   void* actual = TargetMMap(ptr, page_aligned_byte_count, prot, flags, fd, offset);
268*795d594fSAndroid Build Coastguard Worker   if (actual != MAP_FAILED) {
269*795d594fSAndroid Build Coastguard Worker     // Since we didn't use MAP_FIXED the kernel may have mapped it somewhere not in the low
270*795d594fSAndroid Build Coastguard Worker     // 4GB. If this is the case, unmap and retry.
271*795d594fSAndroid Build Coastguard Worker     if (reinterpret_cast<uintptr_t>(actual) + page_aligned_byte_count >= 4 * GB) {
272*795d594fSAndroid Build Coastguard Worker       TargetMUnmap(actual, page_aligned_byte_count);
273*795d594fSAndroid Build Coastguard Worker       actual = MAP_FAILED;
274*795d594fSAndroid Build Coastguard Worker     }
275*795d594fSAndroid Build Coastguard Worker   }
276*795d594fSAndroid Build Coastguard Worker   return actual;
277*795d594fSAndroid Build Coastguard Worker }
278*795d594fSAndroid Build Coastguard Worker #endif
279*795d594fSAndroid Build Coastguard Worker 
SetDebugName(void * map_ptr,const char * name,size_t size)280*795d594fSAndroid Build Coastguard Worker void MemMap::SetDebugName(void* map_ptr, const char* name, size_t size) {
281*795d594fSAndroid Build Coastguard Worker   // Debug naming is only used for Android target builds. For Linux targets,
282*795d594fSAndroid Build Coastguard Worker   // we'll still call prctl but it wont do anything till we upstream the prctl.
283*795d594fSAndroid Build Coastguard Worker   if (kIsTargetFuchsia || !kIsTargetBuild) {
284*795d594fSAndroid Build Coastguard Worker     return;
285*795d594fSAndroid Build Coastguard Worker   }
286*795d594fSAndroid Build Coastguard Worker 
287*795d594fSAndroid Build Coastguard Worker   // lock as std::map is not thread-safe
288*795d594fSAndroid Build Coastguard Worker   std::lock_guard<std::mutex> mu(*mem_maps_lock_);
289*795d594fSAndroid Build Coastguard Worker 
290*795d594fSAndroid Build Coastguard Worker   std::string debug_friendly_name("dalvik-");
291*795d594fSAndroid Build Coastguard Worker   debug_friendly_name += name;
292*795d594fSAndroid Build Coastguard Worker   auto it = debugStrMap.find(debug_friendly_name);
293*795d594fSAndroid Build Coastguard Worker 
294*795d594fSAndroid Build Coastguard Worker   if (it == debugStrMap.end()) {
295*795d594fSAndroid Build Coastguard Worker     it = debugStrMap.insert(std::make_pair(std::move(debug_friendly_name), 1)).first;
296*795d594fSAndroid Build Coastguard Worker   }
297*795d594fSAndroid Build Coastguard Worker 
298*795d594fSAndroid Build Coastguard Worker   DCHECK(it != debugStrMap.end());
299*795d594fSAndroid Build Coastguard Worker #if defined(PR_SET_VMA) && defined(__linux__)
300*795d594fSAndroid Build Coastguard Worker   prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, map_ptr, size, it->first.c_str());
301*795d594fSAndroid Build Coastguard Worker #else
302*795d594fSAndroid Build Coastguard Worker   // Prevent variable unused compiler errors.
303*795d594fSAndroid Build Coastguard Worker   UNUSED(map_ptr, size);
304*795d594fSAndroid Build Coastguard Worker #endif
305*795d594fSAndroid Build Coastguard Worker }
306*795d594fSAndroid Build Coastguard Worker 
MapAnonymous(const char * name,uint8_t * addr,size_t byte_count,int prot,bool low_4gb,bool reuse,MemMap * reservation,std::string * error_msg,bool use_debug_name)307*795d594fSAndroid Build Coastguard Worker MemMap MemMap::MapAnonymous(const char* name,
308*795d594fSAndroid Build Coastguard Worker                             uint8_t* addr,
309*795d594fSAndroid Build Coastguard Worker                             size_t byte_count,
310*795d594fSAndroid Build Coastguard Worker                             int prot,
311*795d594fSAndroid Build Coastguard Worker                             bool low_4gb,
312*795d594fSAndroid Build Coastguard Worker                             bool reuse,
313*795d594fSAndroid Build Coastguard Worker                             /*inout*/MemMap* reservation,
314*795d594fSAndroid Build Coastguard Worker                             /*out*/std::string* error_msg,
315*795d594fSAndroid Build Coastguard Worker                             bool use_debug_name) {
316*795d594fSAndroid Build Coastguard Worker #ifndef __LP64__
317*795d594fSAndroid Build Coastguard Worker   UNUSED(low_4gb);
318*795d594fSAndroid Build Coastguard Worker #endif
319*795d594fSAndroid Build Coastguard Worker   if (byte_count == 0) {
320*795d594fSAndroid Build Coastguard Worker     *error_msg = "Empty MemMap requested.";
321*795d594fSAndroid Build Coastguard Worker     return Invalid();
322*795d594fSAndroid Build Coastguard Worker   }
323*795d594fSAndroid Build Coastguard Worker   size_t page_aligned_byte_count = RoundUp(byte_count, GetPageSize());
324*795d594fSAndroid Build Coastguard Worker 
325*795d594fSAndroid Build Coastguard Worker   int flags = MAP_PRIVATE | MAP_ANONYMOUS;
326*795d594fSAndroid Build Coastguard Worker   if (reuse) {
327*795d594fSAndroid Build Coastguard Worker     // reuse means it is okay that it overlaps an existing page mapping.
328*795d594fSAndroid Build Coastguard Worker     // Only use this if you actually made the page reservation yourself.
329*795d594fSAndroid Build Coastguard Worker     CHECK(addr != nullptr);
330*795d594fSAndroid Build Coastguard Worker     DCHECK(reservation == nullptr);
331*795d594fSAndroid Build Coastguard Worker 
332*795d594fSAndroid Build Coastguard Worker     DCHECK(ContainedWithinExistingMap(addr, byte_count, error_msg)) << *error_msg;
333*795d594fSAndroid Build Coastguard Worker     flags |= MAP_FIXED;
334*795d594fSAndroid Build Coastguard Worker   } else if (reservation != nullptr) {
335*795d594fSAndroid Build Coastguard Worker     CHECK(addr != nullptr);
336*795d594fSAndroid Build Coastguard Worker     if (!CheckReservation(addr, byte_count, name, *reservation, error_msg)) {
337*795d594fSAndroid Build Coastguard Worker       return MemMap::Invalid();
338*795d594fSAndroid Build Coastguard Worker     }
339*795d594fSAndroid Build Coastguard Worker     flags |= MAP_FIXED;
340*795d594fSAndroid Build Coastguard Worker   }
341*795d594fSAndroid Build Coastguard Worker 
342*795d594fSAndroid Build Coastguard Worker   unique_fd fd;
343*795d594fSAndroid Build Coastguard Worker 
344*795d594fSAndroid Build Coastguard Worker   // We need to store and potentially set an error number for pretty printing of errors
345*795d594fSAndroid Build Coastguard Worker   int saved_errno = 0;
346*795d594fSAndroid Build Coastguard Worker 
347*795d594fSAndroid Build Coastguard Worker   void* actual = nullptr;
348*795d594fSAndroid Build Coastguard Worker 
349*795d594fSAndroid Build Coastguard Worker   // New Ubuntu linux kerners seem to ignore the address hint, so make it a firm request.
350*795d594fSAndroid Build Coastguard Worker   // Whereas old kernels allocated at 'addr' if provided, newer kernels seem to ignore it.
351*795d594fSAndroid Build Coastguard Worker   // However, MAP_FIXED_NOREPLACE tells the kernel it must allocate at the address or fail.
352*795d594fSAndroid Build Coastguard Worker   // Do this only on host since android kernels still obey the hint without flag (for now).
353*795d594fSAndroid Build Coastguard Worker   if (!kIsTargetBuild && (flags & MAP_FIXED) == 0 && addr != nullptr) {
354*795d594fSAndroid Build Coastguard Worker     actual = MapInternal(
355*795d594fSAndroid Build Coastguard Worker         addr, page_aligned_byte_count, prot, flags | MAP_FIXED_NOREPLACE, fd.get(), 0, low_4gb);
356*795d594fSAndroid Build Coastguard Worker     // If the fixed-address allocation failed, fallback to the default path (random address).
357*795d594fSAndroid Build Coastguard Worker   }
358*795d594fSAndroid Build Coastguard Worker   if (actual == nullptr || actual == MAP_FAILED) {
359*795d594fSAndroid Build Coastguard Worker     actual = MapInternal(addr, page_aligned_byte_count, prot, flags, fd.get(), 0, low_4gb);
360*795d594fSAndroid Build Coastguard Worker   }
361*795d594fSAndroid Build Coastguard Worker   saved_errno = errno;
362*795d594fSAndroid Build Coastguard Worker 
363*795d594fSAndroid Build Coastguard Worker   if (actual == MAP_FAILED) {
364*795d594fSAndroid Build Coastguard Worker     if (error_msg != nullptr) {
365*795d594fSAndroid Build Coastguard Worker       PrintFileToLog("/proc/self/maps", LogSeverity::WARNING);
366*795d594fSAndroid Build Coastguard Worker       *error_msg = StringPrintf("Failed anonymous mmap(%p, %zd, 0x%x, 0x%x, %d, 0): %s. "
367*795d594fSAndroid Build Coastguard Worker                                     "See process maps in the log.",
368*795d594fSAndroid Build Coastguard Worker                                 addr,
369*795d594fSAndroid Build Coastguard Worker                                 page_aligned_byte_count,
370*795d594fSAndroid Build Coastguard Worker                                 prot,
371*795d594fSAndroid Build Coastguard Worker                                 flags,
372*795d594fSAndroid Build Coastguard Worker                                 fd.get(),
373*795d594fSAndroid Build Coastguard Worker                                 strerror(saved_errno));
374*795d594fSAndroid Build Coastguard Worker     }
375*795d594fSAndroid Build Coastguard Worker     return Invalid();
376*795d594fSAndroid Build Coastguard Worker   }
377*795d594fSAndroid Build Coastguard Worker   if (!CheckMapRequest(addr, actual, page_aligned_byte_count, error_msg)) {
378*795d594fSAndroid Build Coastguard Worker     return Invalid();
379*795d594fSAndroid Build Coastguard Worker   }
380*795d594fSAndroid Build Coastguard Worker 
381*795d594fSAndroid Build Coastguard Worker   if (use_debug_name) {
382*795d594fSAndroid Build Coastguard Worker     SetDebugName(actual, name, page_aligned_byte_count);
383*795d594fSAndroid Build Coastguard Worker   }
384*795d594fSAndroid Build Coastguard Worker 
385*795d594fSAndroid Build Coastguard Worker   if (reservation != nullptr) {
386*795d594fSAndroid Build Coastguard Worker     // Re-mapping was successful, transfer the ownership of the memory to the new MemMap.
387*795d594fSAndroid Build Coastguard Worker     DCHECK_EQ(actual, reservation->Begin());
388*795d594fSAndroid Build Coastguard Worker     reservation->ReleaseReservedMemory(byte_count);
389*795d594fSAndroid Build Coastguard Worker   }
390*795d594fSAndroid Build Coastguard Worker   return MemMap(name,
391*795d594fSAndroid Build Coastguard Worker                 reinterpret_cast<uint8_t*>(actual),
392*795d594fSAndroid Build Coastguard Worker                 byte_count,
393*795d594fSAndroid Build Coastguard Worker                 actual,
394*795d594fSAndroid Build Coastguard Worker                 page_aligned_byte_count,
395*795d594fSAndroid Build Coastguard Worker                 prot,
396*795d594fSAndroid Build Coastguard Worker                 reuse);
397*795d594fSAndroid Build Coastguard Worker }
398*795d594fSAndroid Build Coastguard Worker 
MapAnonymousAligned(const char * name,size_t byte_count,int prot,bool low_4gb,size_t alignment,std::string * error_msg)399*795d594fSAndroid Build Coastguard Worker MemMap MemMap::MapAnonymousAligned(const char* name,
400*795d594fSAndroid Build Coastguard Worker                                    size_t byte_count,
401*795d594fSAndroid Build Coastguard Worker                                    int prot,
402*795d594fSAndroid Build Coastguard Worker                                    bool low_4gb,
403*795d594fSAndroid Build Coastguard Worker                                    size_t alignment,
404*795d594fSAndroid Build Coastguard Worker                                    /*out=*/std::string* error_msg) {
405*795d594fSAndroid Build Coastguard Worker   DCHECK(IsPowerOfTwo(alignment));
406*795d594fSAndroid Build Coastguard Worker   DCHECK_GT(alignment, GetPageSize());
407*795d594fSAndroid Build Coastguard Worker 
408*795d594fSAndroid Build Coastguard Worker   // Allocate extra 'alignment - GetPageSize()' bytes so that the mapping can be aligned.
409*795d594fSAndroid Build Coastguard Worker   MemMap ret = MapAnonymous(name,
410*795d594fSAndroid Build Coastguard Worker                             /*addr=*/nullptr,
411*795d594fSAndroid Build Coastguard Worker                             // AlignBy requires the size to be page-aligned, so
412*795d594fSAndroid Build Coastguard Worker                             // rounding it here. It is corrected afterwards with
413*795d594fSAndroid Build Coastguard Worker                             // SetSize after AlignBy.
414*795d594fSAndroid Build Coastguard Worker                             RoundUp(byte_count, GetPageSize()) + alignment - GetPageSize(),
415*795d594fSAndroid Build Coastguard Worker                             prot,
416*795d594fSAndroid Build Coastguard Worker                             low_4gb,
417*795d594fSAndroid Build Coastguard Worker                             /*reuse=*/false,
418*795d594fSAndroid Build Coastguard Worker                             /*reservation=*/nullptr,
419*795d594fSAndroid Build Coastguard Worker                             error_msg);
420*795d594fSAndroid Build Coastguard Worker   if (LIKELY(ret.IsValid())) {
421*795d594fSAndroid Build Coastguard Worker     ret.AlignBy(alignment, /*align_both_ends=*/false);
422*795d594fSAndroid Build Coastguard Worker     ret.SetSize(byte_count);
423*795d594fSAndroid Build Coastguard Worker     DCHECK_EQ(ret.Size(), byte_count);
424*795d594fSAndroid Build Coastguard Worker     DCHECK_ALIGNED_PARAM(ret.Begin(), alignment);
425*795d594fSAndroid Build Coastguard Worker   }
426*795d594fSAndroid Build Coastguard Worker   return ret;
427*795d594fSAndroid Build Coastguard Worker }
428*795d594fSAndroid Build Coastguard Worker 
MapPlaceholder(const char * name,uint8_t * addr,size_t byte_count)429*795d594fSAndroid Build Coastguard Worker MemMap MemMap::MapPlaceholder(const char* name, uint8_t* addr, size_t byte_count) {
430*795d594fSAndroid Build Coastguard Worker   if (byte_count == 0) {
431*795d594fSAndroid Build Coastguard Worker     return Invalid();
432*795d594fSAndroid Build Coastguard Worker   }
433*795d594fSAndroid Build Coastguard Worker   const size_t page_aligned_byte_count = RoundUp(byte_count, GetPageSize());
434*795d594fSAndroid Build Coastguard Worker   return MemMap(name, addr, byte_count, addr, page_aligned_byte_count, 0, /* reuse= */ true);
435*795d594fSAndroid Build Coastguard Worker }
436*795d594fSAndroid Build Coastguard Worker 
437*795d594fSAndroid Build Coastguard Worker template<typename A, typename B>
PointerDiff(A * a,B * b)438*795d594fSAndroid Build Coastguard Worker static ptrdiff_t PointerDiff(A* a, B* b) {
439*795d594fSAndroid Build Coastguard Worker   return static_cast<ptrdiff_t>(reinterpret_cast<intptr_t>(a) - reinterpret_cast<intptr_t>(b));
440*795d594fSAndroid Build Coastguard Worker }
441*795d594fSAndroid Build Coastguard Worker 
ReplaceWith(MemMap * source,std::string * error)442*795d594fSAndroid Build Coastguard Worker bool MemMap::ReplaceWith(MemMap* source, /*out*/std::string* error) {
443*795d594fSAndroid Build Coastguard Worker #if !HAVE_MREMAP_SYSCALL
444*795d594fSAndroid Build Coastguard Worker   UNUSED(source);
445*795d594fSAndroid Build Coastguard Worker   *error = "Cannot perform atomic replace because we are missing the required mremap syscall";
446*795d594fSAndroid Build Coastguard Worker   return false;
447*795d594fSAndroid Build Coastguard Worker #else  // !HAVE_MREMAP_SYSCALL
448*795d594fSAndroid Build Coastguard Worker   CHECK(source != nullptr);
449*795d594fSAndroid Build Coastguard Worker   CHECK(source->IsValid());
450*795d594fSAndroid Build Coastguard Worker   if (!MemMap::kCanReplaceMapping) {
451*795d594fSAndroid Build Coastguard Worker     *error = "Unable to perform atomic replace due to runtime environment!";
452*795d594fSAndroid Build Coastguard Worker     return false;
453*795d594fSAndroid Build Coastguard Worker   }
454*795d594fSAndroid Build Coastguard Worker   // neither can be reuse.
455*795d594fSAndroid Build Coastguard Worker   if (source->reuse_ || reuse_) {
456*795d594fSAndroid Build Coastguard Worker     *error = "One or both mappings is not a real mmap!";
457*795d594fSAndroid Build Coastguard Worker     return false;
458*795d594fSAndroid Build Coastguard Worker   }
459*795d594fSAndroid Build Coastguard Worker   // TODO Support redzones.
460*795d594fSAndroid Build Coastguard Worker   if (source->redzone_size_ != 0 || redzone_size_ != 0) {
461*795d594fSAndroid Build Coastguard Worker     *error = "source and dest have different redzone sizes";
462*795d594fSAndroid Build Coastguard Worker     return false;
463*795d594fSAndroid Build Coastguard Worker   }
464*795d594fSAndroid Build Coastguard Worker   // Make sure they have the same offset from the actual mmap'd address
465*795d594fSAndroid Build Coastguard Worker   if (PointerDiff(BaseBegin(), Begin()) != PointerDiff(source->BaseBegin(), source->Begin())) {
466*795d594fSAndroid Build Coastguard Worker     *error =
467*795d594fSAndroid Build Coastguard Worker         "source starts at a different offset from the mmap. Cannot atomically replace mappings";
468*795d594fSAndroid Build Coastguard Worker     return false;
469*795d594fSAndroid Build Coastguard Worker   }
470*795d594fSAndroid Build Coastguard Worker   // mremap doesn't allow the final [start, end] to overlap with the initial [start, end] (it's like
471*795d594fSAndroid Build Coastguard Worker   // memcpy but the check is explicit and actually done).
472*795d594fSAndroid Build Coastguard Worker   if (source->BaseBegin() > BaseBegin() &&
473*795d594fSAndroid Build Coastguard Worker       reinterpret_cast<uint8_t*>(BaseBegin()) + source->BaseSize() >
474*795d594fSAndroid Build Coastguard Worker       reinterpret_cast<uint8_t*>(source->BaseBegin())) {
475*795d594fSAndroid Build Coastguard Worker     *error = "destination memory pages overlap with source memory pages";
476*795d594fSAndroid Build Coastguard Worker     return false;
477*795d594fSAndroid Build Coastguard Worker   }
478*795d594fSAndroid Build Coastguard Worker   // Change the protection to match the new location.
479*795d594fSAndroid Build Coastguard Worker   int old_prot = source->GetProtect();
480*795d594fSAndroid Build Coastguard Worker   if (!source->Protect(GetProtect())) {
481*795d594fSAndroid Build Coastguard Worker     *error = "Could not change protections for source to those required for dest.";
482*795d594fSAndroid Build Coastguard Worker     return false;
483*795d594fSAndroid Build Coastguard Worker   }
484*795d594fSAndroid Build Coastguard Worker 
485*795d594fSAndroid Build Coastguard Worker   // Do the mremap.
486*795d594fSAndroid Build Coastguard Worker   void* res = mremap(/*old_address*/source->BaseBegin(),
487*795d594fSAndroid Build Coastguard Worker                      /*old_size*/source->BaseSize(),
488*795d594fSAndroid Build Coastguard Worker                      /*new_size*/source->BaseSize(),
489*795d594fSAndroid Build Coastguard Worker                      /*flags*/MREMAP_MAYMOVE | MREMAP_FIXED,
490*795d594fSAndroid Build Coastguard Worker                      /*new_address*/BaseBegin());
491*795d594fSAndroid Build Coastguard Worker   if (res == MAP_FAILED) {
492*795d594fSAndroid Build Coastguard Worker     int saved_errno = errno;
493*795d594fSAndroid Build Coastguard Worker     // Wasn't able to move mapping. Change the protection of source back to the original one and
494*795d594fSAndroid Build Coastguard Worker     // return.
495*795d594fSAndroid Build Coastguard Worker     source->Protect(old_prot);
496*795d594fSAndroid Build Coastguard Worker     *error = std::string("Failed to mremap source to dest. Error was ") + strerror(saved_errno);
497*795d594fSAndroid Build Coastguard Worker     return false;
498*795d594fSAndroid Build Coastguard Worker   }
499*795d594fSAndroid Build Coastguard Worker   CHECK(res == BaseBegin());
500*795d594fSAndroid Build Coastguard Worker 
501*795d594fSAndroid Build Coastguard Worker   // The new base_size is all the pages of the 'source' plus any remaining dest pages. We will unmap
502*795d594fSAndroid Build Coastguard Worker   // them later.
503*795d594fSAndroid Build Coastguard Worker   size_t new_base_size = std::max(source->base_size_, base_size_);
504*795d594fSAndroid Build Coastguard Worker 
505*795d594fSAndroid Build Coastguard Worker   // Invalidate *source, don't unmap it though since it is already gone.
506*795d594fSAndroid Build Coastguard Worker   size_t source_size = source->size_;
507*795d594fSAndroid Build Coastguard Worker   source->Invalidate();
508*795d594fSAndroid Build Coastguard Worker 
509*795d594fSAndroid Build Coastguard Worker   size_ = source_size;
510*795d594fSAndroid Build Coastguard Worker   base_size_ = new_base_size;
511*795d594fSAndroid Build Coastguard Worker   // Reduce base_size if needed (this will unmap the extra pages).
512*795d594fSAndroid Build Coastguard Worker   SetSize(source_size);
513*795d594fSAndroid Build Coastguard Worker 
514*795d594fSAndroid Build Coastguard Worker   return true;
515*795d594fSAndroid Build Coastguard Worker #endif  // !HAVE_MREMAP_SYSCALL
516*795d594fSAndroid Build Coastguard Worker }
517*795d594fSAndroid Build Coastguard Worker 
MapFileAtAddress(uint8_t * expected_ptr,size_t byte_count,int prot,int flags,int fd,off_t start,bool low_4gb,const char * filename,bool reuse,MemMap * reservation,std::string * error_msg)518*795d594fSAndroid Build Coastguard Worker MemMap MemMap::MapFileAtAddress(uint8_t* expected_ptr,
519*795d594fSAndroid Build Coastguard Worker                                 size_t byte_count,
520*795d594fSAndroid Build Coastguard Worker                                 int prot,
521*795d594fSAndroid Build Coastguard Worker                                 int flags,
522*795d594fSAndroid Build Coastguard Worker                                 int fd,
523*795d594fSAndroid Build Coastguard Worker                                 off_t start,
524*795d594fSAndroid Build Coastguard Worker                                 bool low_4gb,
525*795d594fSAndroid Build Coastguard Worker                                 const char* filename,
526*795d594fSAndroid Build Coastguard Worker                                 bool reuse,
527*795d594fSAndroid Build Coastguard Worker                                 /*inout*/MemMap* reservation,
528*795d594fSAndroid Build Coastguard Worker                                 /*out*/std::string* error_msg) {
529*795d594fSAndroid Build Coastguard Worker   CHECK_NE(0, prot);
530*795d594fSAndroid Build Coastguard Worker   CHECK_NE(0, flags & (MAP_SHARED | MAP_PRIVATE));
531*795d594fSAndroid Build Coastguard Worker 
532*795d594fSAndroid Build Coastguard Worker   // Note that we do not allow MAP_FIXED unless reuse == true or we have an existing
533*795d594fSAndroid Build Coastguard Worker   // reservation, i.e we expect this mapping to be contained within an existing map.
534*795d594fSAndroid Build Coastguard Worker   if (reuse && expected_ptr != nullptr) {
535*795d594fSAndroid Build Coastguard Worker     // reuse means it is okay that it overlaps an existing page mapping.
536*795d594fSAndroid Build Coastguard Worker     // Only use this if you actually made the page reservation yourself.
537*795d594fSAndroid Build Coastguard Worker     DCHECK(reservation == nullptr);
538*795d594fSAndroid Build Coastguard Worker     DCHECK(error_msg != nullptr);
539*795d594fSAndroid Build Coastguard Worker     DCHECK(ContainedWithinExistingMap(expected_ptr, byte_count, error_msg))
540*795d594fSAndroid Build Coastguard Worker         << ((error_msg != nullptr) ? *error_msg : std::string());
541*795d594fSAndroid Build Coastguard Worker     flags |= MAP_FIXED;
542*795d594fSAndroid Build Coastguard Worker   } else if (reservation != nullptr) {
543*795d594fSAndroid Build Coastguard Worker     DCHECK(error_msg != nullptr);
544*795d594fSAndroid Build Coastguard Worker     if (!CheckReservation(expected_ptr, byte_count, filename, *reservation, error_msg)) {
545*795d594fSAndroid Build Coastguard Worker       return Invalid();
546*795d594fSAndroid Build Coastguard Worker     }
547*795d594fSAndroid Build Coastguard Worker     flags |= MAP_FIXED;
548*795d594fSAndroid Build Coastguard Worker   } else {
549*795d594fSAndroid Build Coastguard Worker     CHECK_EQ(0, flags & MAP_FIXED);
550*795d594fSAndroid Build Coastguard Worker     // Don't bother checking for an overlapping region here. We'll
551*795d594fSAndroid Build Coastguard Worker     // check this if required after the fact inside CheckMapRequest.
552*795d594fSAndroid Build Coastguard Worker   }
553*795d594fSAndroid Build Coastguard Worker 
554*795d594fSAndroid Build Coastguard Worker   if (byte_count == 0) {
555*795d594fSAndroid Build Coastguard Worker     *error_msg = "Empty MemMap requested";
556*795d594fSAndroid Build Coastguard Worker     return Invalid();
557*795d594fSAndroid Build Coastguard Worker   }
558*795d594fSAndroid Build Coastguard Worker   // Adjust 'offset' to be page-aligned as required by mmap.
559*795d594fSAndroid Build Coastguard Worker   int page_offset = start % GetPageSize();
560*795d594fSAndroid Build Coastguard Worker   off_t page_aligned_offset = start - page_offset;
561*795d594fSAndroid Build Coastguard Worker   // Adjust 'byte_count' to be page-aligned as we will map this anyway.
562*795d594fSAndroid Build Coastguard Worker   size_t page_aligned_byte_count = RoundUp(byte_count + page_offset, GetPageSize());
563*795d594fSAndroid Build Coastguard Worker   // The 'expected_ptr' is modified (if specified, ie non-null) to be page aligned to the file but
564*795d594fSAndroid Build Coastguard Worker   // not necessarily to virtual memory. mmap will page align 'expected' for us.
565*795d594fSAndroid Build Coastguard Worker   uint8_t* page_aligned_expected =
566*795d594fSAndroid Build Coastguard Worker       (expected_ptr == nullptr) ? nullptr : (expected_ptr - page_offset);
567*795d594fSAndroid Build Coastguard Worker 
568*795d594fSAndroid Build Coastguard Worker   size_t redzone_size = 0;
569*795d594fSAndroid Build Coastguard Worker   if (kRunningOnMemoryTool && kMemoryToolAddsRedzones && expected_ptr == nullptr) {
570*795d594fSAndroid Build Coastguard Worker     redzone_size = GetPageSize();
571*795d594fSAndroid Build Coastguard Worker     page_aligned_byte_count += redzone_size;
572*795d594fSAndroid Build Coastguard Worker   }
573*795d594fSAndroid Build Coastguard Worker 
574*795d594fSAndroid Build Coastguard Worker   uint8_t* actual = reinterpret_cast<uint8_t*>(MapInternal(page_aligned_expected,
575*795d594fSAndroid Build Coastguard Worker                                                            page_aligned_byte_count,
576*795d594fSAndroid Build Coastguard Worker                                                            prot,
577*795d594fSAndroid Build Coastguard Worker                                                            flags,
578*795d594fSAndroid Build Coastguard Worker                                                            fd,
579*795d594fSAndroid Build Coastguard Worker                                                            page_aligned_offset,
580*795d594fSAndroid Build Coastguard Worker                                                            low_4gb));
581*795d594fSAndroid Build Coastguard Worker   if (actual == MAP_FAILED) {
582*795d594fSAndroid Build Coastguard Worker     if (error_msg != nullptr) {
583*795d594fSAndroid Build Coastguard Worker       auto saved_errno = errno;
584*795d594fSAndroid Build Coastguard Worker 
585*795d594fSAndroid Build Coastguard Worker       if (kIsDebugBuild || VLOG_IS_ON(oat)) {
586*795d594fSAndroid Build Coastguard Worker         PrintFileToLog("/proc/self/maps", LogSeverity::WARNING);
587*795d594fSAndroid Build Coastguard Worker       }
588*795d594fSAndroid Build Coastguard Worker 
589*795d594fSAndroid Build Coastguard Worker       *error_msg = StringPrintf("mmap(%p, %zd, 0x%x, 0x%x, %d, %" PRId64
590*795d594fSAndroid Build Coastguard Worker                                 ") of file '%s' failed: %s. See process maps in the log.",
591*795d594fSAndroid Build Coastguard Worker                                 page_aligned_expected, page_aligned_byte_count, prot, flags, fd,
592*795d594fSAndroid Build Coastguard Worker                                 static_cast<int64_t>(page_aligned_offset), filename,
593*795d594fSAndroid Build Coastguard Worker                                 strerror(saved_errno));
594*795d594fSAndroid Build Coastguard Worker     }
595*795d594fSAndroid Build Coastguard Worker     return Invalid();
596*795d594fSAndroid Build Coastguard Worker   }
597*795d594fSAndroid Build Coastguard Worker   if (!CheckMapRequest(expected_ptr, actual, page_aligned_byte_count, error_msg)) {
598*795d594fSAndroid Build Coastguard Worker     return Invalid();
599*795d594fSAndroid Build Coastguard Worker   }
600*795d594fSAndroid Build Coastguard Worker   if (redzone_size != 0) {
601*795d594fSAndroid Build Coastguard Worker     const uint8_t *real_start = actual + page_offset;
602*795d594fSAndroid Build Coastguard Worker     const uint8_t *real_end = actual + page_offset + byte_count;
603*795d594fSAndroid Build Coastguard Worker     const uint8_t *mapping_end = actual + page_aligned_byte_count;
604*795d594fSAndroid Build Coastguard Worker 
605*795d594fSAndroid Build Coastguard Worker     MEMORY_TOOL_MAKE_NOACCESS(actual, real_start - actual);
606*795d594fSAndroid Build Coastguard Worker     MEMORY_TOOL_MAKE_NOACCESS(real_end, mapping_end - real_end);
607*795d594fSAndroid Build Coastguard Worker     page_aligned_byte_count -= redzone_size;
608*795d594fSAndroid Build Coastguard Worker   }
609*795d594fSAndroid Build Coastguard Worker 
610*795d594fSAndroid Build Coastguard Worker   if (reservation != nullptr) {
611*795d594fSAndroid Build Coastguard Worker     // Re-mapping was successful, transfer the ownership of the memory to the new MemMap.
612*795d594fSAndroid Build Coastguard Worker     DCHECK_EQ(actual, reservation->Begin());
613*795d594fSAndroid Build Coastguard Worker     reservation->ReleaseReservedMemory(byte_count);
614*795d594fSAndroid Build Coastguard Worker   }
615*795d594fSAndroid Build Coastguard Worker   return MemMap(filename,
616*795d594fSAndroid Build Coastguard Worker                 actual + page_offset,
617*795d594fSAndroid Build Coastguard Worker                 byte_count,
618*795d594fSAndroid Build Coastguard Worker                 actual,
619*795d594fSAndroid Build Coastguard Worker                 page_aligned_byte_count,
620*795d594fSAndroid Build Coastguard Worker                 prot,
621*795d594fSAndroid Build Coastguard Worker                 reuse,
622*795d594fSAndroid Build Coastguard Worker                 redzone_size);
623*795d594fSAndroid Build Coastguard Worker }
624*795d594fSAndroid Build Coastguard Worker 
MemMap(MemMap && other)625*795d594fSAndroid Build Coastguard Worker MemMap::MemMap(MemMap&& other) noexcept
626*795d594fSAndroid Build Coastguard Worker     : MemMap() {
627*795d594fSAndroid Build Coastguard Worker   swap(other);
628*795d594fSAndroid Build Coastguard Worker }
629*795d594fSAndroid Build Coastguard Worker 
~MemMap()630*795d594fSAndroid Build Coastguard Worker MemMap::~MemMap() {
631*795d594fSAndroid Build Coastguard Worker   Reset();
632*795d594fSAndroid Build Coastguard Worker }
633*795d594fSAndroid Build Coastguard Worker 
DoReset()634*795d594fSAndroid Build Coastguard Worker void MemMap::DoReset() {
635*795d594fSAndroid Build Coastguard Worker   DCHECK(IsValid());
636*795d594fSAndroid Build Coastguard Worker   size_t real_base_size = base_size_;
637*795d594fSAndroid Build Coastguard Worker   // Unlike Valgrind, AddressSanitizer requires that all manually poisoned memory is unpoisoned
638*795d594fSAndroid Build Coastguard Worker   // before it is returned to the system.
639*795d594fSAndroid Build Coastguard Worker   if (redzone_size_ != 0) {
640*795d594fSAndroid Build Coastguard Worker     // Add redzone_size_ back to base_size or it will cause a mmap leakage.
641*795d594fSAndroid Build Coastguard Worker     real_base_size += redzone_size_;
642*795d594fSAndroid Build Coastguard Worker     MEMORY_TOOL_MAKE_UNDEFINED(
643*795d594fSAndroid Build Coastguard Worker         reinterpret_cast<char*>(base_begin_) + real_base_size - redzone_size_,
644*795d594fSAndroid Build Coastguard Worker         redzone_size_);
645*795d594fSAndroid Build Coastguard Worker   }
646*795d594fSAndroid Build Coastguard Worker 
647*795d594fSAndroid Build Coastguard Worker   if (!reuse_) {
648*795d594fSAndroid Build Coastguard Worker     MEMORY_TOOL_MAKE_UNDEFINED(base_begin_, base_size_);
649*795d594fSAndroid Build Coastguard Worker     if (!already_unmapped_) {
650*795d594fSAndroid Build Coastguard Worker       int result = TargetMUnmap(base_begin_, real_base_size);
651*795d594fSAndroid Build Coastguard Worker       if (result == -1) {
652*795d594fSAndroid Build Coastguard Worker         PLOG(FATAL) << "munmap failed";
653*795d594fSAndroid Build Coastguard Worker       }
654*795d594fSAndroid Build Coastguard Worker     }
655*795d594fSAndroid Build Coastguard Worker   }
656*795d594fSAndroid Build Coastguard Worker 
657*795d594fSAndroid Build Coastguard Worker   Invalidate();
658*795d594fSAndroid Build Coastguard Worker }
659*795d594fSAndroid Build Coastguard Worker 
ResetInForkedProcess()660*795d594fSAndroid Build Coastguard Worker void MemMap::ResetInForkedProcess() {
661*795d594fSAndroid Build Coastguard Worker   // This should be called on a map that has MADV_DONTFORK.
662*795d594fSAndroid Build Coastguard Worker   // The kernel has already unmapped this.
663*795d594fSAndroid Build Coastguard Worker   already_unmapped_ = true;
664*795d594fSAndroid Build Coastguard Worker   Reset();
665*795d594fSAndroid Build Coastguard Worker }
666*795d594fSAndroid Build Coastguard Worker 
Invalidate()667*795d594fSAndroid Build Coastguard Worker void MemMap::Invalidate() {
668*795d594fSAndroid Build Coastguard Worker   DCHECK(IsValid());
669*795d594fSAndroid Build Coastguard Worker 
670*795d594fSAndroid Build Coastguard Worker   // Remove it from gMaps.
671*795d594fSAndroid Build Coastguard Worker   // TODO(b/307704260) Move MemMap::Init MemMap::Shutdown out of Runtime init/shutdown.
672*795d594fSAndroid Build Coastguard Worker   if (mem_maps_lock_ != nullptr) {  // Runtime was shutdown.
673*795d594fSAndroid Build Coastguard Worker     std::lock_guard<std::mutex> mu(*mem_maps_lock_);
674*795d594fSAndroid Build Coastguard Worker     auto it = GetGMapsEntry(*this);
675*795d594fSAndroid Build Coastguard Worker     gMaps->erase(it);
676*795d594fSAndroid Build Coastguard Worker   }
677*795d594fSAndroid Build Coastguard Worker 
678*795d594fSAndroid Build Coastguard Worker   // Mark it as invalid.
679*795d594fSAndroid Build Coastguard Worker   base_size_ = 0u;
680*795d594fSAndroid Build Coastguard Worker   DCHECK(!IsValid());
681*795d594fSAndroid Build Coastguard Worker }
682*795d594fSAndroid Build Coastguard Worker 
swap(MemMap & other)683*795d594fSAndroid Build Coastguard Worker void MemMap::swap(MemMap& other) {
684*795d594fSAndroid Build Coastguard Worker   if (IsValid() || other.IsValid()) {
685*795d594fSAndroid Build Coastguard Worker     std::lock_guard<std::mutex> mu(*mem_maps_lock_);
686*795d594fSAndroid Build Coastguard Worker     DCHECK(gMaps != nullptr);
687*795d594fSAndroid Build Coastguard Worker     auto this_it = IsValid() ? GetGMapsEntry(*this) : gMaps->end();
688*795d594fSAndroid Build Coastguard Worker     auto other_it = other.IsValid() ? GetGMapsEntry(other) : gMaps->end();
689*795d594fSAndroid Build Coastguard Worker     if (IsValid()) {
690*795d594fSAndroid Build Coastguard Worker       DCHECK(this_it != gMaps->end());
691*795d594fSAndroid Build Coastguard Worker       DCHECK_EQ(this_it->second, this);
692*795d594fSAndroid Build Coastguard Worker       this_it->second = &other;
693*795d594fSAndroid Build Coastguard Worker     }
694*795d594fSAndroid Build Coastguard Worker     if (other.IsValid()) {
695*795d594fSAndroid Build Coastguard Worker       DCHECK(other_it != gMaps->end());
696*795d594fSAndroid Build Coastguard Worker       DCHECK_EQ(other_it->second, &other);
697*795d594fSAndroid Build Coastguard Worker       other_it->second = this;
698*795d594fSAndroid Build Coastguard Worker     }
699*795d594fSAndroid Build Coastguard Worker     // Swap members with the `mem_maps_lock_` held so that `base_begin_` matches
700*795d594fSAndroid Build Coastguard Worker     // with the `gMaps` key when other threads try to use `gMaps`.
701*795d594fSAndroid Build Coastguard Worker     SwapMembers(other);
702*795d594fSAndroid Build Coastguard Worker   } else {
703*795d594fSAndroid Build Coastguard Worker     SwapMembers(other);
704*795d594fSAndroid Build Coastguard Worker   }
705*795d594fSAndroid Build Coastguard Worker }
706*795d594fSAndroid Build Coastguard Worker 
SwapMembers(MemMap & other)707*795d594fSAndroid Build Coastguard Worker void MemMap::SwapMembers(MemMap& other) {
708*795d594fSAndroid Build Coastguard Worker   name_.swap(other.name_);
709*795d594fSAndroid Build Coastguard Worker   std::swap(begin_, other.begin_);
710*795d594fSAndroid Build Coastguard Worker   std::swap(size_, other.size_);
711*795d594fSAndroid Build Coastguard Worker   std::swap(base_begin_, other.base_begin_);
712*795d594fSAndroid Build Coastguard Worker   std::swap(base_size_, other.base_size_);
713*795d594fSAndroid Build Coastguard Worker   std::swap(prot_, other.prot_);
714*795d594fSAndroid Build Coastguard Worker   std::swap(reuse_, other.reuse_);
715*795d594fSAndroid Build Coastguard Worker   std::swap(already_unmapped_, other.already_unmapped_);
716*795d594fSAndroid Build Coastguard Worker   std::swap(redzone_size_, other.redzone_size_);
717*795d594fSAndroid Build Coastguard Worker }
718*795d594fSAndroid Build Coastguard Worker 
MemMap(const std::string & name,uint8_t * begin,size_t size,void * base_begin,size_t base_size,int prot,bool reuse,size_t redzone_size)719*795d594fSAndroid Build Coastguard Worker MemMap::MemMap(const std::string& name, uint8_t* begin, size_t size, void* base_begin,
720*795d594fSAndroid Build Coastguard Worker                size_t base_size, int prot, bool reuse, size_t redzone_size)
721*795d594fSAndroid Build Coastguard Worker     : name_(name), begin_(begin), size_(size), base_begin_(base_begin), base_size_(base_size),
722*795d594fSAndroid Build Coastguard Worker       prot_(prot), reuse_(reuse), already_unmapped_(false), redzone_size_(redzone_size) {
723*795d594fSAndroid Build Coastguard Worker   if (size_ == 0) {
724*795d594fSAndroid Build Coastguard Worker     CHECK(begin_ == nullptr);
725*795d594fSAndroid Build Coastguard Worker     CHECK(base_begin_ == nullptr);
726*795d594fSAndroid Build Coastguard Worker     CHECK_EQ(base_size_, 0U);
727*795d594fSAndroid Build Coastguard Worker   } else {
728*795d594fSAndroid Build Coastguard Worker     CHECK(begin_ != nullptr);
729*795d594fSAndroid Build Coastguard Worker     CHECK(base_begin_ != nullptr);
730*795d594fSAndroid Build Coastguard Worker     CHECK_NE(base_size_, 0U);
731*795d594fSAndroid Build Coastguard Worker 
732*795d594fSAndroid Build Coastguard Worker     // Add it to gMaps.
733*795d594fSAndroid Build Coastguard Worker     std::lock_guard<std::mutex> mu(*mem_maps_lock_);
734*795d594fSAndroid Build Coastguard Worker     DCHECK(gMaps != nullptr);
735*795d594fSAndroid Build Coastguard Worker     gMaps->insert(std::make_pair(base_begin_, this));
736*795d594fSAndroid Build Coastguard Worker   }
737*795d594fSAndroid Build Coastguard Worker }
738*795d594fSAndroid Build Coastguard Worker 
RemapAtEnd(uint8_t * new_end,const char * tail_name,int tail_prot,std::string * error_msg,bool use_debug_name)739*795d594fSAndroid Build Coastguard Worker MemMap MemMap::RemapAtEnd(uint8_t* new_end,
740*795d594fSAndroid Build Coastguard Worker                           const char* tail_name,
741*795d594fSAndroid Build Coastguard Worker                           int tail_prot,
742*795d594fSAndroid Build Coastguard Worker                           std::string* error_msg,
743*795d594fSAndroid Build Coastguard Worker                           bool use_debug_name) {
744*795d594fSAndroid Build Coastguard Worker   return RemapAtEnd(new_end,
745*795d594fSAndroid Build Coastguard Worker                     tail_name,
746*795d594fSAndroid Build Coastguard Worker                     tail_prot,
747*795d594fSAndroid Build Coastguard Worker                     MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS,
748*795d594fSAndroid Build Coastguard Worker                     /* fd= */ -1,
749*795d594fSAndroid Build Coastguard Worker                     /* offset= */ 0,
750*795d594fSAndroid Build Coastguard Worker                     error_msg,
751*795d594fSAndroid Build Coastguard Worker                     use_debug_name);
752*795d594fSAndroid Build Coastguard Worker }
753*795d594fSAndroid Build Coastguard Worker 
RemapAtEnd(uint8_t * new_end,const char * tail_name,int tail_prot,int flags,int fd,off_t offset,std::string * error_msg,bool use_debug_name)754*795d594fSAndroid Build Coastguard Worker MemMap MemMap::RemapAtEnd(uint8_t* new_end,
755*795d594fSAndroid Build Coastguard Worker                           const char* tail_name,
756*795d594fSAndroid Build Coastguard Worker                           int tail_prot,
757*795d594fSAndroid Build Coastguard Worker                           int flags,
758*795d594fSAndroid Build Coastguard Worker                           int fd,
759*795d594fSAndroid Build Coastguard Worker                           off_t offset,
760*795d594fSAndroid Build Coastguard Worker                           std::string* error_msg,
761*795d594fSAndroid Build Coastguard Worker                           bool use_debug_name) {
762*795d594fSAndroid Build Coastguard Worker   DCHECK_GE(new_end, Begin());
763*795d594fSAndroid Build Coastguard Worker   DCHECK_LE(new_end, End());
764*795d594fSAndroid Build Coastguard Worker   DCHECK_LE(begin_ + size_, reinterpret_cast<uint8_t*>(base_begin_) + base_size_);
765*795d594fSAndroid Build Coastguard Worker   DCHECK_ALIGNED_PARAM(begin_, GetPageSize());
766*795d594fSAndroid Build Coastguard Worker   DCHECK_ALIGNED_PARAM(base_begin_, GetPageSize());
767*795d594fSAndroid Build Coastguard Worker   DCHECK_ALIGNED_PARAM(reinterpret_cast<uint8_t*>(base_begin_) + base_size_, GetPageSize());
768*795d594fSAndroid Build Coastguard Worker   DCHECK_ALIGNED_PARAM(new_end, GetPageSize());
769*795d594fSAndroid Build Coastguard Worker   uint8_t* old_end = begin_ + size_;
770*795d594fSAndroid Build Coastguard Worker   uint8_t* old_base_end = reinterpret_cast<uint8_t*>(base_begin_) + base_size_;
771*795d594fSAndroid Build Coastguard Worker   uint8_t* new_base_end = new_end;
772*795d594fSAndroid Build Coastguard Worker   DCHECK_LE(new_base_end, old_base_end);
773*795d594fSAndroid Build Coastguard Worker   if (new_base_end == old_base_end) {
774*795d594fSAndroid Build Coastguard Worker     return Invalid();
775*795d594fSAndroid Build Coastguard Worker   }
776*795d594fSAndroid Build Coastguard Worker   size_t new_size = new_end - reinterpret_cast<uint8_t*>(begin_);
777*795d594fSAndroid Build Coastguard Worker   size_t new_base_size = new_base_end - reinterpret_cast<uint8_t*>(base_begin_);
778*795d594fSAndroid Build Coastguard Worker   DCHECK_LE(begin_ + new_size, reinterpret_cast<uint8_t*>(base_begin_) + new_base_size);
779*795d594fSAndroid Build Coastguard Worker   size_t tail_size = old_end - new_end;
780*795d594fSAndroid Build Coastguard Worker   uint8_t* tail_base_begin = new_base_end;
781*795d594fSAndroid Build Coastguard Worker   size_t tail_base_size = old_base_end - new_base_end;
782*795d594fSAndroid Build Coastguard Worker   DCHECK_EQ(tail_base_begin + tail_base_size, old_base_end);
783*795d594fSAndroid Build Coastguard Worker   DCHECK_ALIGNED_PARAM(tail_base_size, GetPageSize());
784*795d594fSAndroid Build Coastguard Worker 
785*795d594fSAndroid Build Coastguard Worker   MEMORY_TOOL_MAKE_UNDEFINED(tail_base_begin, tail_base_size);
786*795d594fSAndroid Build Coastguard Worker   // Note: Do not explicitly unmap the tail region, mmap() with MAP_FIXED automatically
787*795d594fSAndroid Build Coastguard Worker   // removes old mappings for the overlapping region. This makes the operation atomic
788*795d594fSAndroid Build Coastguard Worker   // and prevents other threads from racing to allocate memory in the requested region.
789*795d594fSAndroid Build Coastguard Worker   uint8_t* actual = reinterpret_cast<uint8_t*>(TargetMMap(tail_base_begin,
790*795d594fSAndroid Build Coastguard Worker                                                           tail_base_size,
791*795d594fSAndroid Build Coastguard Worker                                                           tail_prot,
792*795d594fSAndroid Build Coastguard Worker                                                           flags,
793*795d594fSAndroid Build Coastguard Worker                                                           fd,
794*795d594fSAndroid Build Coastguard Worker                                                           offset));
795*795d594fSAndroid Build Coastguard Worker   if (actual == MAP_FAILED) {
796*795d594fSAndroid Build Coastguard Worker     *error_msg = StringPrintf("map(%p, %zd, 0x%x, 0x%x, %d, 0) failed: %s. See process "
797*795d594fSAndroid Build Coastguard Worker                               "maps in the log.", tail_base_begin, tail_base_size, tail_prot, flags,
798*795d594fSAndroid Build Coastguard Worker                               fd, strerror(errno));
799*795d594fSAndroid Build Coastguard Worker     PrintFileToLog("/proc/self/maps", LogSeverity::WARNING);
800*795d594fSAndroid Build Coastguard Worker     return Invalid();
801*795d594fSAndroid Build Coastguard Worker   }
802*795d594fSAndroid Build Coastguard Worker   // Update *this.
803*795d594fSAndroid Build Coastguard Worker   if (new_base_size == 0u) {
804*795d594fSAndroid Build Coastguard Worker     std::lock_guard<std::mutex> mu(*mem_maps_lock_);
805*795d594fSAndroid Build Coastguard Worker     auto it = GetGMapsEntry(*this);
806*795d594fSAndroid Build Coastguard Worker     gMaps->erase(it);
807*795d594fSAndroid Build Coastguard Worker   }
808*795d594fSAndroid Build Coastguard Worker 
809*795d594fSAndroid Build Coastguard Worker   if (use_debug_name) {
810*795d594fSAndroid Build Coastguard Worker     SetDebugName(actual, tail_name, tail_base_size);
811*795d594fSAndroid Build Coastguard Worker   }
812*795d594fSAndroid Build Coastguard Worker 
813*795d594fSAndroid Build Coastguard Worker   size_ = new_size;
814*795d594fSAndroid Build Coastguard Worker   base_size_ = new_base_size;
815*795d594fSAndroid Build Coastguard Worker   // Return the new mapping.
816*795d594fSAndroid Build Coastguard Worker   return MemMap(tail_name, actual, tail_size, actual, tail_base_size, tail_prot, false);
817*795d594fSAndroid Build Coastguard Worker }
818*795d594fSAndroid Build Coastguard Worker 
TakeReservedMemory(size_t byte_count,bool reuse)819*795d594fSAndroid Build Coastguard Worker MemMap MemMap::TakeReservedMemory(size_t byte_count, bool reuse) {
820*795d594fSAndroid Build Coastguard Worker   uint8_t* begin = Begin();
821*795d594fSAndroid Build Coastguard Worker   ReleaseReservedMemory(byte_count);  // Performs necessary DCHECK()s on this reservation.
822*795d594fSAndroid Build Coastguard Worker   size_t base_size = RoundUp(byte_count, GetPageSize());
823*795d594fSAndroid Build Coastguard Worker   return MemMap(name_, begin, byte_count, begin, base_size, prot_, reuse);
824*795d594fSAndroid Build Coastguard Worker }
825*795d594fSAndroid Build Coastguard Worker 
ReleaseReservedMemory(size_t byte_count)826*795d594fSAndroid Build Coastguard Worker void MemMap::ReleaseReservedMemory(size_t byte_count) {
827*795d594fSAndroid Build Coastguard Worker   // Check the reservation mapping.
828*795d594fSAndroid Build Coastguard Worker   DCHECK(IsValid());
829*795d594fSAndroid Build Coastguard Worker   DCHECK(!reuse_);
830*795d594fSAndroid Build Coastguard Worker   DCHECK(!already_unmapped_);
831*795d594fSAndroid Build Coastguard Worker   DCHECK_EQ(redzone_size_, 0u);
832*795d594fSAndroid Build Coastguard Worker   DCHECK_EQ(begin_, base_begin_);
833*795d594fSAndroid Build Coastguard Worker   DCHECK_EQ(size_, base_size_);
834*795d594fSAndroid Build Coastguard Worker   DCHECK_ALIGNED_PARAM(begin_, GetPageSize());
835*795d594fSAndroid Build Coastguard Worker   DCHECK_ALIGNED_PARAM(size_, GetPageSize());
836*795d594fSAndroid Build Coastguard Worker 
837*795d594fSAndroid Build Coastguard Worker   // Check and round up the `byte_count`.
838*795d594fSAndroid Build Coastguard Worker   DCHECK_NE(byte_count, 0u);
839*795d594fSAndroid Build Coastguard Worker   DCHECK_LE(byte_count, size_);
840*795d594fSAndroid Build Coastguard Worker   byte_count = RoundUp(byte_count, GetPageSize());
841*795d594fSAndroid Build Coastguard Worker 
842*795d594fSAndroid Build Coastguard Worker   if (byte_count == size_) {
843*795d594fSAndroid Build Coastguard Worker     Invalidate();
844*795d594fSAndroid Build Coastguard Worker   } else {
845*795d594fSAndroid Build Coastguard Worker     // Shrink the reservation MemMap and update its `gMaps` entry.
846*795d594fSAndroid Build Coastguard Worker     std::lock_guard<std::mutex> mu(*mem_maps_lock_);
847*795d594fSAndroid Build Coastguard Worker     auto it = GetGMapsEntry(*this);
848*795d594fSAndroid Build Coastguard Worker     auto node = gMaps->extract(it);
849*795d594fSAndroid Build Coastguard Worker     begin_ += byte_count;
850*795d594fSAndroid Build Coastguard Worker     size_ -= byte_count;
851*795d594fSAndroid Build Coastguard Worker     base_begin_ = begin_;
852*795d594fSAndroid Build Coastguard Worker     base_size_ = size_;
853*795d594fSAndroid Build Coastguard Worker     node.key() = base_begin_;
854*795d594fSAndroid Build Coastguard Worker     gMaps->insert(std::move(node));
855*795d594fSAndroid Build Coastguard Worker   }
856*795d594fSAndroid Build Coastguard Worker }
857*795d594fSAndroid Build Coastguard Worker 
FillWithZero(bool release_eagerly)858*795d594fSAndroid Build Coastguard Worker void MemMap::FillWithZero(bool release_eagerly) {
859*795d594fSAndroid Build Coastguard Worker   if (base_begin_ != nullptr && base_size_ != 0) {
860*795d594fSAndroid Build Coastguard Worker     ZeroMemory(base_begin_, base_size_, release_eagerly);
861*795d594fSAndroid Build Coastguard Worker   }
862*795d594fSAndroid Build Coastguard Worker }
863*795d594fSAndroid Build Coastguard Worker 
MadviseDontFork()864*795d594fSAndroid Build Coastguard Worker int MemMap::MadviseDontFork() {
865*795d594fSAndroid Build Coastguard Worker #if defined(__linux__)
866*795d594fSAndroid Build Coastguard Worker   if (base_begin_ != nullptr || base_size_ != 0) {
867*795d594fSAndroid Build Coastguard Worker     return madvise(base_begin_, base_size_, MADV_DONTFORK);
868*795d594fSAndroid Build Coastguard Worker   }
869*795d594fSAndroid Build Coastguard Worker #endif
870*795d594fSAndroid Build Coastguard Worker   return -1;
871*795d594fSAndroid Build Coastguard Worker }
872*795d594fSAndroid Build Coastguard Worker 
Sync()873*795d594fSAndroid Build Coastguard Worker bool MemMap::Sync() {
874*795d594fSAndroid Build Coastguard Worker #ifdef _WIN32
875*795d594fSAndroid Build Coastguard Worker   // TODO: add FlushViewOfFile support.
876*795d594fSAndroid Build Coastguard Worker   PLOG(ERROR) << "MemMap::Sync unsupported on Windows.";
877*795d594fSAndroid Build Coastguard Worker   return false;
878*795d594fSAndroid Build Coastguard Worker #else
879*795d594fSAndroid Build Coastguard Worker   // Historical note: To avoid Valgrind errors, we temporarily lifted the lower-end noaccess
880*795d594fSAndroid Build Coastguard Worker   // protection before passing it to msync() when `redzone_size_` was non-null, as Valgrind
881*795d594fSAndroid Build Coastguard Worker   // only accepts page-aligned base address, and excludes the higher-end noaccess protection
882*795d594fSAndroid Build Coastguard Worker   // from the msync range. b/27552451.
883*795d594fSAndroid Build Coastguard Worker   return msync(BaseBegin(), BaseSize(), MS_SYNC) == 0;
884*795d594fSAndroid Build Coastguard Worker #endif
885*795d594fSAndroid Build Coastguard Worker }
886*795d594fSAndroid Build Coastguard Worker 
Protect(int prot)887*795d594fSAndroid Build Coastguard Worker bool MemMap::Protect(int prot) {
888*795d594fSAndroid Build Coastguard Worker   if (base_begin_ == nullptr && base_size_ == 0) {
889*795d594fSAndroid Build Coastguard Worker     prot_ = prot;
890*795d594fSAndroid Build Coastguard Worker     return true;
891*795d594fSAndroid Build Coastguard Worker   }
892*795d594fSAndroid Build Coastguard Worker 
893*795d594fSAndroid Build Coastguard Worker #ifndef _WIN32
894*795d594fSAndroid Build Coastguard Worker   if (mprotect(base_begin_, base_size_, prot) == 0) {
895*795d594fSAndroid Build Coastguard Worker     prot_ = prot;
896*795d594fSAndroid Build Coastguard Worker     return true;
897*795d594fSAndroid Build Coastguard Worker   }
898*795d594fSAndroid Build Coastguard Worker #endif
899*795d594fSAndroid Build Coastguard Worker 
900*795d594fSAndroid Build Coastguard Worker   PLOG(ERROR) << "mprotect(" << reinterpret_cast<void*>(base_begin_) << ", " << base_size_ << ", "
901*795d594fSAndroid Build Coastguard Worker               << prot << ") failed";
902*795d594fSAndroid Build Coastguard Worker   return false;
903*795d594fSAndroid Build Coastguard Worker }
904*795d594fSAndroid Build Coastguard Worker 
CheckNoGaps(MemMap & begin_map,MemMap & end_map)905*795d594fSAndroid Build Coastguard Worker bool MemMap::CheckNoGaps(MemMap& begin_map, MemMap& end_map) {
906*795d594fSAndroid Build Coastguard Worker   std::lock_guard<std::mutex> mu(*mem_maps_lock_);
907*795d594fSAndroid Build Coastguard Worker   CHECK(begin_map.IsValid());
908*795d594fSAndroid Build Coastguard Worker   CHECK(end_map.IsValid());
909*795d594fSAndroid Build Coastguard Worker   CHECK(HasMemMap(begin_map));
910*795d594fSAndroid Build Coastguard Worker   CHECK(HasMemMap(end_map));
911*795d594fSAndroid Build Coastguard Worker   CHECK_LE(begin_map.BaseBegin(), end_map.BaseBegin());
912*795d594fSAndroid Build Coastguard Worker   MemMap* map = &begin_map;
913*795d594fSAndroid Build Coastguard Worker   while (map->BaseBegin() != end_map.BaseBegin()) {
914*795d594fSAndroid Build Coastguard Worker     MemMap* next_map = GetLargestMemMapAt(map->BaseEnd());
915*795d594fSAndroid Build Coastguard Worker     if (next_map == nullptr) {
916*795d594fSAndroid Build Coastguard Worker       // Found a gap.
917*795d594fSAndroid Build Coastguard Worker       return false;
918*795d594fSAndroid Build Coastguard Worker     }
919*795d594fSAndroid Build Coastguard Worker     map = next_map;
920*795d594fSAndroid Build Coastguard Worker   }
921*795d594fSAndroid Build Coastguard Worker   return true;
922*795d594fSAndroid Build Coastguard Worker }
923*795d594fSAndroid Build Coastguard Worker 
DumpMaps(std::ostream & os,bool terse)924*795d594fSAndroid Build Coastguard Worker void MemMap::DumpMaps(std::ostream& os, bool terse) {
925*795d594fSAndroid Build Coastguard Worker   std::lock_guard<std::mutex> mu(*mem_maps_lock_);
926*795d594fSAndroid Build Coastguard Worker   DumpMapsLocked(os, terse);
927*795d594fSAndroid Build Coastguard Worker }
928*795d594fSAndroid Build Coastguard Worker 
DumpMapsLocked(std::ostream & os,bool terse)929*795d594fSAndroid Build Coastguard Worker void MemMap::DumpMapsLocked(std::ostream& os, bool terse) {
930*795d594fSAndroid Build Coastguard Worker   const auto& mem_maps = *gMaps;
931*795d594fSAndroid Build Coastguard Worker   if (!terse) {
932*795d594fSAndroid Build Coastguard Worker     os << mem_maps;
933*795d594fSAndroid Build Coastguard Worker     return;
934*795d594fSAndroid Build Coastguard Worker   }
935*795d594fSAndroid Build Coastguard Worker 
936*795d594fSAndroid Build Coastguard Worker   // Terse output example:
937*795d594fSAndroid Build Coastguard Worker   //   [MemMap: 0x409be000+0x20P~0x11dP+0x20P~0x61cP+0x20P prot=0x3 LinearAlloc]
938*795d594fSAndroid Build Coastguard Worker   //   [MemMap: 0x451d6000+0x6bP(3) prot=0x3 large object space allocation]
939*795d594fSAndroid Build Coastguard Worker   // The details:
940*795d594fSAndroid Build Coastguard Worker   //   "+0x20P" means 0x20 pages taken by a single mapping,
941*795d594fSAndroid Build Coastguard Worker   //   "~0x11dP" means a gap of 0x11d pages,
942*795d594fSAndroid Build Coastguard Worker   //   "+0x6bP(3)" means 3 mappings one after another, together taking 0x6b pages.
943*795d594fSAndroid Build Coastguard Worker   os << "MemMap:" << std::endl;
944*795d594fSAndroid Build Coastguard Worker   for (auto it = mem_maps.begin(), maps_end = mem_maps.end(); it != maps_end;) {
945*795d594fSAndroid Build Coastguard Worker     MemMap* map = it->second;
946*795d594fSAndroid Build Coastguard Worker     void* base = it->first;
947*795d594fSAndroid Build Coastguard Worker     CHECK_EQ(base, map->BaseBegin());
948*795d594fSAndroid Build Coastguard Worker     os << "[MemMap: " << base;
949*795d594fSAndroid Build Coastguard Worker     ++it;
950*795d594fSAndroid Build Coastguard Worker     // Merge consecutive maps with the same protect flags and name.
951*795d594fSAndroid Build Coastguard Worker     constexpr size_t kMaxGaps = 9;
952*795d594fSAndroid Build Coastguard Worker     size_t num_gaps = 0;
953*795d594fSAndroid Build Coastguard Worker     size_t num = 1u;
954*795d594fSAndroid Build Coastguard Worker     size_t size = map->BaseSize();
955*795d594fSAndroid Build Coastguard Worker     CHECK_ALIGNED_PARAM(size, GetPageSize());
956*795d594fSAndroid Build Coastguard Worker     void* end = map->BaseEnd();
957*795d594fSAndroid Build Coastguard Worker     while (it != maps_end &&
958*795d594fSAndroid Build Coastguard Worker         it->second->GetProtect() == map->GetProtect() &&
959*795d594fSAndroid Build Coastguard Worker         it->second->GetName() == map->GetName() &&
960*795d594fSAndroid Build Coastguard Worker         (it->second->BaseBegin() == end || num_gaps < kMaxGaps)) {
961*795d594fSAndroid Build Coastguard Worker       if (it->second->BaseBegin() != end) {
962*795d594fSAndroid Build Coastguard Worker         ++num_gaps;
963*795d594fSAndroid Build Coastguard Worker         os << "+0x" << std::hex << (size / GetPageSize()) << "P";
964*795d594fSAndroid Build Coastguard Worker         if (num != 1u) {
965*795d594fSAndroid Build Coastguard Worker           os << "(" << std::dec << num << ")";
966*795d594fSAndroid Build Coastguard Worker         }
967*795d594fSAndroid Build Coastguard Worker         size_t gap =
968*795d594fSAndroid Build Coastguard Worker             reinterpret_cast<uintptr_t>(it->second->BaseBegin()) - reinterpret_cast<uintptr_t>(end);
969*795d594fSAndroid Build Coastguard Worker         CHECK_ALIGNED_PARAM(gap, GetPageSize());
970*795d594fSAndroid Build Coastguard Worker         os << "~0x" << std::hex << (gap / GetPageSize()) << "P";
971*795d594fSAndroid Build Coastguard Worker         num = 0u;
972*795d594fSAndroid Build Coastguard Worker         size = 0u;
973*795d594fSAndroid Build Coastguard Worker       }
974*795d594fSAndroid Build Coastguard Worker       CHECK_ALIGNED_PARAM(it->second->BaseSize(), GetPageSize());
975*795d594fSAndroid Build Coastguard Worker       ++num;
976*795d594fSAndroid Build Coastguard Worker       size += it->second->BaseSize();
977*795d594fSAndroid Build Coastguard Worker       end = it->second->BaseEnd();
978*795d594fSAndroid Build Coastguard Worker       ++it;
979*795d594fSAndroid Build Coastguard Worker     }
980*795d594fSAndroid Build Coastguard Worker     os << "+0x" << std::hex << (size / GetPageSize()) << "P";
981*795d594fSAndroid Build Coastguard Worker     if (num != 1u) {
982*795d594fSAndroid Build Coastguard Worker       os << "(" << std::dec << num << ")";
983*795d594fSAndroid Build Coastguard Worker     }
984*795d594fSAndroid Build Coastguard Worker     os << " prot=0x" << std::hex << map->GetProtect() << " " << map->GetName() << "]" << std::endl;
985*795d594fSAndroid Build Coastguard Worker   }
986*795d594fSAndroid Build Coastguard Worker }
987*795d594fSAndroid Build Coastguard Worker 
HasMemMap(MemMap & map)988*795d594fSAndroid Build Coastguard Worker bool MemMap::HasMemMap(MemMap& map) {
989*795d594fSAndroid Build Coastguard Worker   void* base_begin = map.BaseBegin();
990*795d594fSAndroid Build Coastguard Worker   for (auto it = gMaps->lower_bound(base_begin), end = gMaps->end();
991*795d594fSAndroid Build Coastguard Worker        it != end && it->first == base_begin; ++it) {
992*795d594fSAndroid Build Coastguard Worker     if (it->second == &map) {
993*795d594fSAndroid Build Coastguard Worker       return true;
994*795d594fSAndroid Build Coastguard Worker     }
995*795d594fSAndroid Build Coastguard Worker   }
996*795d594fSAndroid Build Coastguard Worker   return false;
997*795d594fSAndroid Build Coastguard Worker }
998*795d594fSAndroid Build Coastguard Worker 
GetLargestMemMapAt(void * address)999*795d594fSAndroid Build Coastguard Worker MemMap* MemMap::GetLargestMemMapAt(void* address) {
1000*795d594fSAndroid Build Coastguard Worker   size_t largest_size = 0;
1001*795d594fSAndroid Build Coastguard Worker   MemMap* largest_map = nullptr;
1002*795d594fSAndroid Build Coastguard Worker   DCHECK(gMaps != nullptr);
1003*795d594fSAndroid Build Coastguard Worker   for (auto it = gMaps->lower_bound(address), end = gMaps->end();
1004*795d594fSAndroid Build Coastguard Worker        it != end && it->first == address; ++it) {
1005*795d594fSAndroid Build Coastguard Worker     MemMap* map = it->second;
1006*795d594fSAndroid Build Coastguard Worker     CHECK(map != nullptr);
1007*795d594fSAndroid Build Coastguard Worker     if (largest_size < map->BaseSize()) {
1008*795d594fSAndroid Build Coastguard Worker       largest_size = map->BaseSize();
1009*795d594fSAndroid Build Coastguard Worker       largest_map = map;
1010*795d594fSAndroid Build Coastguard Worker     }
1011*795d594fSAndroid Build Coastguard Worker   }
1012*795d594fSAndroid Build Coastguard Worker   return largest_map;
1013*795d594fSAndroid Build Coastguard Worker }
1014*795d594fSAndroid Build Coastguard Worker 
Init()1015*795d594fSAndroid Build Coastguard Worker void MemMap::Init() {
1016*795d594fSAndroid Build Coastguard Worker   if (mem_maps_lock_ != nullptr) {
1017*795d594fSAndroid Build Coastguard Worker     // dex2oat calls MemMap::Init twice since its needed before the runtime is created.
1018*795d594fSAndroid Build Coastguard Worker     return;
1019*795d594fSAndroid Build Coastguard Worker   }
1020*795d594fSAndroid Build Coastguard Worker 
1021*795d594fSAndroid Build Coastguard Worker   mem_maps_lock_ = new std::mutex();
1022*795d594fSAndroid Build Coastguard Worker   // Not for thread safety, but for the annotation that gMaps is GUARDED_BY(mem_maps_lock_).
1023*795d594fSAndroid Build Coastguard Worker   std::lock_guard<std::mutex> mu(*mem_maps_lock_);
1024*795d594fSAndroid Build Coastguard Worker #ifdef ART_PAGE_SIZE_AGNOSTIC
1025*795d594fSAndroid Build Coastguard Worker   page_size_ = GetPageSizeSlow();
1026*795d594fSAndroid Build Coastguard Worker #endif
1027*795d594fSAndroid Build Coastguard Worker   CHECK_GE(GetPageSize(), kMinPageSize);
1028*795d594fSAndroid Build Coastguard Worker   CHECK_LE(GetPageSize(), kMaxPageSize);
1029*795d594fSAndroid Build Coastguard Worker #if USE_ART_LOW_4G_ALLOCATOR
1030*795d594fSAndroid Build Coastguard Worker   // Initialize linear scan to random position.
1031*795d594fSAndroid Build Coastguard Worker   CHECK_EQ(next_mem_pos_, 0u);
1032*795d594fSAndroid Build Coastguard Worker   next_mem_pos_ = GenerateNextMemPos(GetPageSize());
1033*795d594fSAndroid Build Coastguard Worker #endif
1034*795d594fSAndroid Build Coastguard Worker   DCHECK(gMaps == nullptr);
1035*795d594fSAndroid Build Coastguard Worker   gMaps = new Maps;
1036*795d594fSAndroid Build Coastguard Worker 
1037*795d594fSAndroid Build Coastguard Worker   TargetMMapInit();
1038*795d594fSAndroid Build Coastguard Worker }
1039*795d594fSAndroid Build Coastguard Worker 
IsInitialized()1040*795d594fSAndroid Build Coastguard Worker bool MemMap::IsInitialized() { return mem_maps_lock_ != nullptr; }
1041*795d594fSAndroid Build Coastguard Worker 
Shutdown()1042*795d594fSAndroid Build Coastguard Worker void MemMap::Shutdown() {
1043*795d594fSAndroid Build Coastguard Worker   if (mem_maps_lock_ == nullptr) {
1044*795d594fSAndroid Build Coastguard Worker     // If MemMap::Shutdown is called more than once, there is no effect.
1045*795d594fSAndroid Build Coastguard Worker     return;
1046*795d594fSAndroid Build Coastguard Worker   }
1047*795d594fSAndroid Build Coastguard Worker   {
1048*795d594fSAndroid Build Coastguard Worker     // Not for thread safety, but for the annotation that gMaps is GUARDED_BY(mem_maps_lock_).
1049*795d594fSAndroid Build Coastguard Worker     std::lock_guard<std::mutex> mu(*mem_maps_lock_);
1050*795d594fSAndroid Build Coastguard Worker     DCHECK(gMaps != nullptr);
1051*795d594fSAndroid Build Coastguard Worker     delete gMaps;
1052*795d594fSAndroid Build Coastguard Worker     gMaps = nullptr;
1053*795d594fSAndroid Build Coastguard Worker   }
1054*795d594fSAndroid Build Coastguard Worker #if USE_ART_LOW_4G_ALLOCATOR
1055*795d594fSAndroid Build Coastguard Worker   next_mem_pos_ = 0u;
1056*795d594fSAndroid Build Coastguard Worker #endif
1057*795d594fSAndroid Build Coastguard Worker   delete mem_maps_lock_;
1058*795d594fSAndroid Build Coastguard Worker   mem_maps_lock_ = nullptr;
1059*795d594fSAndroid Build Coastguard Worker }
1060*795d594fSAndroid Build Coastguard Worker 
SetSize(size_t new_size)1061*795d594fSAndroid Build Coastguard Worker void MemMap::SetSize(size_t new_size) {
1062*795d594fSAndroid Build Coastguard Worker   CHECK_LE(new_size, size_);
1063*795d594fSAndroid Build Coastguard Worker   size_t new_base_size = RoundUp(new_size + static_cast<size_t>(PointerDiff(Begin(), BaseBegin())),
1064*795d594fSAndroid Build Coastguard Worker                                  GetPageSize());
1065*795d594fSAndroid Build Coastguard Worker   if (new_base_size == base_size_) {
1066*795d594fSAndroid Build Coastguard Worker     size_ = new_size;
1067*795d594fSAndroid Build Coastguard Worker     return;
1068*795d594fSAndroid Build Coastguard Worker   }
1069*795d594fSAndroid Build Coastguard Worker   CHECK_LT(new_base_size, base_size_);
1070*795d594fSAndroid Build Coastguard Worker   MEMORY_TOOL_MAKE_UNDEFINED(
1071*795d594fSAndroid Build Coastguard Worker       reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(BaseBegin()) +
1072*795d594fSAndroid Build Coastguard Worker                               new_base_size),
1073*795d594fSAndroid Build Coastguard Worker       base_size_ - new_base_size);
1074*795d594fSAndroid Build Coastguard Worker   CHECK_EQ(TargetMUnmap(reinterpret_cast<void*>(
1075*795d594fSAndroid Build Coastguard Worker                         reinterpret_cast<uintptr_t>(BaseBegin()) + new_base_size),
1076*795d594fSAndroid Build Coastguard Worker                         base_size_ - new_base_size), 0)
1077*795d594fSAndroid Build Coastguard Worker                         << new_base_size << " " << base_size_;
1078*795d594fSAndroid Build Coastguard Worker   base_size_ = new_base_size;
1079*795d594fSAndroid Build Coastguard Worker   size_ = new_size;
1080*795d594fSAndroid Build Coastguard Worker }
1081*795d594fSAndroid Build Coastguard Worker 
MapInternalArtLow4GBAllocator(size_t length,int prot,int flags,int fd,off_t offset)1082*795d594fSAndroid Build Coastguard Worker void* MemMap::MapInternalArtLow4GBAllocator(size_t length,
1083*795d594fSAndroid Build Coastguard Worker                                             int prot,
1084*795d594fSAndroid Build Coastguard Worker                                             int flags,
1085*795d594fSAndroid Build Coastguard Worker                                             int fd,
1086*795d594fSAndroid Build Coastguard Worker                                             off_t offset) {
1087*795d594fSAndroid Build Coastguard Worker #if USE_ART_LOW_4G_ALLOCATOR
1088*795d594fSAndroid Build Coastguard Worker   void* actual = MAP_FAILED;
1089*795d594fSAndroid Build Coastguard Worker 
1090*795d594fSAndroid Build Coastguard Worker   bool first_run = true;
1091*795d594fSAndroid Build Coastguard Worker 
1092*795d594fSAndroid Build Coastguard Worker   std::lock_guard<std::mutex> mu(*mem_maps_lock_);
1093*795d594fSAndroid Build Coastguard Worker   for (uintptr_t ptr = next_mem_pos_; ptr < 4 * GB; ptr += GetPageSize()) {
1094*795d594fSAndroid Build Coastguard Worker     // Use gMaps as an optimization to skip over large maps.
1095*795d594fSAndroid Build Coastguard Worker     // Find the first map which is address > ptr.
1096*795d594fSAndroid Build Coastguard Worker     auto it = gMaps->upper_bound(reinterpret_cast<void*>(ptr));
1097*795d594fSAndroid Build Coastguard Worker     if (it != gMaps->begin()) {
1098*795d594fSAndroid Build Coastguard Worker       auto before_it = it;
1099*795d594fSAndroid Build Coastguard Worker       --before_it;
1100*795d594fSAndroid Build Coastguard Worker       // Start at the end of the map before the upper bound.
1101*795d594fSAndroid Build Coastguard Worker       ptr = std::max(ptr, reinterpret_cast<uintptr_t>(before_it->second->BaseEnd()));
1102*795d594fSAndroid Build Coastguard Worker       CHECK_ALIGNED_PARAM(ptr, GetPageSize());
1103*795d594fSAndroid Build Coastguard Worker     }
1104*795d594fSAndroid Build Coastguard Worker     while (it != gMaps->end()) {
1105*795d594fSAndroid Build Coastguard Worker       // How much space do we have until the next map?
1106*795d594fSAndroid Build Coastguard Worker       size_t delta = reinterpret_cast<uintptr_t>(it->first) - ptr;
1107*795d594fSAndroid Build Coastguard Worker       // If the space may be sufficient, break out of the loop.
1108*795d594fSAndroid Build Coastguard Worker       if (delta >= length) {
1109*795d594fSAndroid Build Coastguard Worker         break;
1110*795d594fSAndroid Build Coastguard Worker       }
1111*795d594fSAndroid Build Coastguard Worker       // Otherwise, skip to the end of the map.
1112*795d594fSAndroid Build Coastguard Worker       ptr = reinterpret_cast<uintptr_t>(it->second->BaseEnd());
1113*795d594fSAndroid Build Coastguard Worker       CHECK_ALIGNED_PARAM(ptr, GetPageSize());
1114*795d594fSAndroid Build Coastguard Worker       ++it;
1115*795d594fSAndroid Build Coastguard Worker     }
1116*795d594fSAndroid Build Coastguard Worker 
1117*795d594fSAndroid Build Coastguard Worker     // Try to see if we get lucky with this address since none of the ART maps overlap.
1118*795d594fSAndroid Build Coastguard Worker     actual = TryMemMapLow4GB(reinterpret_cast<void*>(ptr), length, prot, flags, fd, offset);
1119*795d594fSAndroid Build Coastguard Worker     if (actual != MAP_FAILED) {
1120*795d594fSAndroid Build Coastguard Worker       next_mem_pos_ = reinterpret_cast<uintptr_t>(actual) + length;
1121*795d594fSAndroid Build Coastguard Worker       return actual;
1122*795d594fSAndroid Build Coastguard Worker     }
1123*795d594fSAndroid Build Coastguard Worker 
1124*795d594fSAndroid Build Coastguard Worker     if (4U * GB - ptr < length) {
1125*795d594fSAndroid Build Coastguard Worker       // Not enough memory until 4GB.
1126*795d594fSAndroid Build Coastguard Worker       if (first_run) {
1127*795d594fSAndroid Build Coastguard Worker         // Try another time from the bottom;
1128*795d594fSAndroid Build Coastguard Worker         ptr = LOW_MEM_START - GetPageSize();
1129*795d594fSAndroid Build Coastguard Worker         first_run = false;
1130*795d594fSAndroid Build Coastguard Worker         continue;
1131*795d594fSAndroid Build Coastguard Worker       } else {
1132*795d594fSAndroid Build Coastguard Worker         // Second try failed.
1133*795d594fSAndroid Build Coastguard Worker         break;
1134*795d594fSAndroid Build Coastguard Worker       }
1135*795d594fSAndroid Build Coastguard Worker     }
1136*795d594fSAndroid Build Coastguard Worker 
1137*795d594fSAndroid Build Coastguard Worker     uintptr_t tail_ptr;
1138*795d594fSAndroid Build Coastguard Worker 
1139*795d594fSAndroid Build Coastguard Worker     // Check pages are free.
1140*795d594fSAndroid Build Coastguard Worker     bool safe = true;
1141*795d594fSAndroid Build Coastguard Worker     for (tail_ptr = ptr; tail_ptr < ptr + length; tail_ptr += GetPageSize()) {
1142*795d594fSAndroid Build Coastguard Worker       if (msync(reinterpret_cast<void*>(tail_ptr), GetPageSize(), 0) == 0) {
1143*795d594fSAndroid Build Coastguard Worker         safe = false;
1144*795d594fSAndroid Build Coastguard Worker         break;
1145*795d594fSAndroid Build Coastguard Worker       } else {
1146*795d594fSAndroid Build Coastguard Worker         DCHECK_EQ(errno, ENOMEM);
1147*795d594fSAndroid Build Coastguard Worker       }
1148*795d594fSAndroid Build Coastguard Worker     }
1149*795d594fSAndroid Build Coastguard Worker 
1150*795d594fSAndroid Build Coastguard Worker     next_mem_pos_ = tail_ptr;  // update early, as we break out when we found and mapped a region
1151*795d594fSAndroid Build Coastguard Worker 
1152*795d594fSAndroid Build Coastguard Worker     if (safe == true) {
1153*795d594fSAndroid Build Coastguard Worker       actual = TryMemMapLow4GB(reinterpret_cast<void*>(ptr), length, prot, flags, fd, offset);
1154*795d594fSAndroid Build Coastguard Worker       if (actual != MAP_FAILED) {
1155*795d594fSAndroid Build Coastguard Worker         return actual;
1156*795d594fSAndroid Build Coastguard Worker       }
1157*795d594fSAndroid Build Coastguard Worker     } else {
1158*795d594fSAndroid Build Coastguard Worker       // Skip over last page.
1159*795d594fSAndroid Build Coastguard Worker       ptr = tail_ptr;
1160*795d594fSAndroid Build Coastguard Worker     }
1161*795d594fSAndroid Build Coastguard Worker   }
1162*795d594fSAndroid Build Coastguard Worker 
1163*795d594fSAndroid Build Coastguard Worker   if (actual == MAP_FAILED) {
1164*795d594fSAndroid Build Coastguard Worker     LOG(ERROR) << "Could not find contiguous low-memory space.";
1165*795d594fSAndroid Build Coastguard Worker     errno = ENOMEM;
1166*795d594fSAndroid Build Coastguard Worker   }
1167*795d594fSAndroid Build Coastguard Worker   return actual;
1168*795d594fSAndroid Build Coastguard Worker #else
1169*795d594fSAndroid Build Coastguard Worker   UNUSED(length, prot, flags, fd, offset);
1170*795d594fSAndroid Build Coastguard Worker   LOG(FATAL) << "Unreachable";
1171*795d594fSAndroid Build Coastguard Worker   UNREACHABLE();
1172*795d594fSAndroid Build Coastguard Worker #endif
1173*795d594fSAndroid Build Coastguard Worker }
1174*795d594fSAndroid Build Coastguard Worker 
MapInternal(void * addr,size_t length,int prot,int flags,int fd,off_t offset,bool low_4gb)1175*795d594fSAndroid Build Coastguard Worker void* MemMap::MapInternal(void* addr,
1176*795d594fSAndroid Build Coastguard Worker                           size_t length,
1177*795d594fSAndroid Build Coastguard Worker                           int prot,
1178*795d594fSAndroid Build Coastguard Worker                           int flags,
1179*795d594fSAndroid Build Coastguard Worker                           int fd,
1180*795d594fSAndroid Build Coastguard Worker                           off_t offset,
1181*795d594fSAndroid Build Coastguard Worker                           bool low_4gb) {
1182*795d594fSAndroid Build Coastguard Worker #ifdef __LP64__
1183*795d594fSAndroid Build Coastguard Worker   // When requesting low_4g memory and having an expectation, the requested range should fit into
1184*795d594fSAndroid Build Coastguard Worker   // 4GB.
1185*795d594fSAndroid Build Coastguard Worker   if (low_4gb && (
1186*795d594fSAndroid Build Coastguard Worker       // Start out of bounds.
1187*795d594fSAndroid Build Coastguard Worker       (reinterpret_cast<uintptr_t>(addr) >> 32) != 0 ||
1188*795d594fSAndroid Build Coastguard Worker       // End out of bounds. For simplicity, this will fail for the last page of memory.
1189*795d594fSAndroid Build Coastguard Worker       ((reinterpret_cast<uintptr_t>(addr) + length) >> 32) != 0)) {
1190*795d594fSAndroid Build Coastguard Worker     LOG(ERROR) << "The requested address space (" << addr << ", "
1191*795d594fSAndroid Build Coastguard Worker                << reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(addr) + length)
1192*795d594fSAndroid Build Coastguard Worker                << ") cannot fit in low_4gb";
1193*795d594fSAndroid Build Coastguard Worker     return MAP_FAILED;
1194*795d594fSAndroid Build Coastguard Worker   }
1195*795d594fSAndroid Build Coastguard Worker #else
1196*795d594fSAndroid Build Coastguard Worker   UNUSED(low_4gb);
1197*795d594fSAndroid Build Coastguard Worker #endif
1198*795d594fSAndroid Build Coastguard Worker   DCHECK_ALIGNED_PARAM(length, GetPageSize());
1199*795d594fSAndroid Build Coastguard Worker   // TODO:
1200*795d594fSAndroid Build Coastguard Worker   // A page allocator would be a useful abstraction here, as
1201*795d594fSAndroid Build Coastguard Worker   // 1) It is doubtful that MAP_32BIT on x86_64 is doing the right job for us
1202*795d594fSAndroid Build Coastguard Worker   void* actual = MAP_FAILED;
1203*795d594fSAndroid Build Coastguard Worker #if USE_ART_LOW_4G_ALLOCATOR
1204*795d594fSAndroid Build Coastguard Worker   // MAP_32BIT only available on x86_64.
1205*795d594fSAndroid Build Coastguard Worker   if (low_4gb && addr == nullptr) {
1206*795d594fSAndroid Build Coastguard Worker     // The linear-scan allocator has an issue when executable pages are denied (e.g., by selinux
1207*795d594fSAndroid Build Coastguard Worker     // policies in sensitive processes). In that case, the error code will still be ENOMEM. So
1208*795d594fSAndroid Build Coastguard Worker     // the allocator will scan all low 4GB twice, and still fail. This is *very* slow.
1209*795d594fSAndroid Build Coastguard Worker     //
1210*795d594fSAndroid Build Coastguard Worker     // To avoid the issue, always map non-executable first, and mprotect if necessary.
1211*795d594fSAndroid Build Coastguard Worker     const int orig_prot = prot;
1212*795d594fSAndroid Build Coastguard Worker     const int prot_non_exec = prot & ~PROT_EXEC;
1213*795d594fSAndroid Build Coastguard Worker     actual = MapInternalArtLow4GBAllocator(length, prot_non_exec, flags, fd, offset);
1214*795d594fSAndroid Build Coastguard Worker 
1215*795d594fSAndroid Build Coastguard Worker     if (actual == MAP_FAILED) {
1216*795d594fSAndroid Build Coastguard Worker       return MAP_FAILED;
1217*795d594fSAndroid Build Coastguard Worker     }
1218*795d594fSAndroid Build Coastguard Worker 
1219*795d594fSAndroid Build Coastguard Worker     // See if we need to remap with the executable bit now.
1220*795d594fSAndroid Build Coastguard Worker     if (orig_prot != prot_non_exec) {
1221*795d594fSAndroid Build Coastguard Worker       if (mprotect(actual, length, orig_prot) != 0) {
1222*795d594fSAndroid Build Coastguard Worker         PLOG(ERROR) << "Could not protect to requested prot: " << orig_prot;
1223*795d594fSAndroid Build Coastguard Worker         TargetMUnmap(actual, length);
1224*795d594fSAndroid Build Coastguard Worker         errno = ENOMEM;
1225*795d594fSAndroid Build Coastguard Worker         return MAP_FAILED;
1226*795d594fSAndroid Build Coastguard Worker       }
1227*795d594fSAndroid Build Coastguard Worker     }
1228*795d594fSAndroid Build Coastguard Worker     return actual;
1229*795d594fSAndroid Build Coastguard Worker   }
1230*795d594fSAndroid Build Coastguard Worker 
1231*795d594fSAndroid Build Coastguard Worker   actual = TargetMMap(addr, length, prot, flags, fd, offset);
1232*795d594fSAndroid Build Coastguard Worker #else
1233*795d594fSAndroid Build Coastguard Worker #if defined(__LP64__)
1234*795d594fSAndroid Build Coastguard Worker   if (low_4gb && addr == nullptr) {
1235*795d594fSAndroid Build Coastguard Worker     flags |= MAP_32BIT;
1236*795d594fSAndroid Build Coastguard Worker   }
1237*795d594fSAndroid Build Coastguard Worker #endif
1238*795d594fSAndroid Build Coastguard Worker   actual = TargetMMap(addr, length, prot, flags, fd, offset);
1239*795d594fSAndroid Build Coastguard Worker #endif
1240*795d594fSAndroid Build Coastguard Worker   return actual;
1241*795d594fSAndroid Build Coastguard Worker }
1242*795d594fSAndroid Build Coastguard Worker 
operator <<(std::ostream & os,const MemMap & mem_map)1243*795d594fSAndroid Build Coastguard Worker std::ostream& operator<<(std::ostream& os, const MemMap& mem_map) {
1244*795d594fSAndroid Build Coastguard Worker   os << StringPrintf("[MemMap: %p-%p prot=0x%x %s]",
1245*795d594fSAndroid Build Coastguard Worker                      mem_map.BaseBegin(), mem_map.BaseEnd(), mem_map.GetProtect(),
1246*795d594fSAndroid Build Coastguard Worker                      mem_map.GetName().c_str());
1247*795d594fSAndroid Build Coastguard Worker   return os;
1248*795d594fSAndroid Build Coastguard Worker }
1249*795d594fSAndroid Build Coastguard Worker 
TryReadable()1250*795d594fSAndroid Build Coastguard Worker void MemMap::TryReadable() {
1251*795d594fSAndroid Build Coastguard Worker   if (base_begin_ == nullptr && base_size_ == 0) {
1252*795d594fSAndroid Build Coastguard Worker     return;
1253*795d594fSAndroid Build Coastguard Worker   }
1254*795d594fSAndroid Build Coastguard Worker   CHECK_NE(prot_ & PROT_READ, 0);
1255*795d594fSAndroid Build Coastguard Worker   volatile uint8_t* begin = reinterpret_cast<volatile uint8_t*>(base_begin_);
1256*795d594fSAndroid Build Coastguard Worker   volatile uint8_t* end = begin + base_size_;
1257*795d594fSAndroid Build Coastguard Worker   DCHECK(IsAlignedParam(begin, GetPageSize()));
1258*795d594fSAndroid Build Coastguard Worker   DCHECK(IsAlignedParam(end, GetPageSize()));
1259*795d594fSAndroid Build Coastguard Worker   // Read the first byte of each page. Use volatile to prevent the compiler from optimizing away the
1260*795d594fSAndroid Build Coastguard Worker   // reads.
1261*795d594fSAndroid Build Coastguard Worker   for (volatile uint8_t* ptr = begin; ptr < end; ptr += GetPageSize()) {
1262*795d594fSAndroid Build Coastguard Worker     // This read could fault if protection wasn't set correctly.
1263*795d594fSAndroid Build Coastguard Worker     uint8_t value = *ptr;
1264*795d594fSAndroid Build Coastguard Worker     UNUSED(value);
1265*795d594fSAndroid Build Coastguard Worker   }
1266*795d594fSAndroid Build Coastguard Worker }
1267*795d594fSAndroid Build Coastguard Worker 
RawClearMemory(uint8_t * begin,uint8_t * end)1268*795d594fSAndroid Build Coastguard Worker static void inline RawClearMemory(uint8_t* begin, uint8_t* end) {
1269*795d594fSAndroid Build Coastguard Worker   std::fill(begin, end, 0);
1270*795d594fSAndroid Build Coastguard Worker }
1271*795d594fSAndroid Build Coastguard Worker 
1272*795d594fSAndroid Build Coastguard Worker #if defined(__linux__)
ClearMemory(uint8_t * page_begin,size_t size,bool resident,size_t page_size)1273*795d594fSAndroid Build Coastguard Worker static inline void ClearMemory(uint8_t* page_begin, size_t size, bool resident, size_t page_size) {
1274*795d594fSAndroid Build Coastguard Worker   DCHECK(IsAlignedParam(page_begin, page_size));
1275*795d594fSAndroid Build Coastguard Worker   DCHECK(IsAlignedParam(page_begin + size, page_size));
1276*795d594fSAndroid Build Coastguard Worker   if (resident) {
1277*795d594fSAndroid Build Coastguard Worker     RawClearMemory(page_begin, page_begin + size);
1278*795d594fSAndroid Build Coastguard Worker     // Note we check madvise return value against -1, as it seems old kernels
1279*795d594fSAndroid Build Coastguard Worker     // can return 1.
1280*795d594fSAndroid Build Coastguard Worker #ifdef MADV_FREE
1281*795d594fSAndroid Build Coastguard Worker     bool res = madvise(page_begin, size, MADV_FREE);
1282*795d594fSAndroid Build Coastguard Worker     CHECK_NE(res, -1) << "madvise failed";
1283*795d594fSAndroid Build Coastguard Worker #endif  // MADV_FREE
1284*795d594fSAndroid Build Coastguard Worker   } else {
1285*795d594fSAndroid Build Coastguard Worker     bool res = madvise(page_begin, size, MADV_DONTNEED);
1286*795d594fSAndroid Build Coastguard Worker     CHECK_NE(res, -1) << "madvise failed";
1287*795d594fSAndroid Build Coastguard Worker   }
1288*795d594fSAndroid Build Coastguard Worker }
1289*795d594fSAndroid Build Coastguard Worker #endif  // __linux__
1290*795d594fSAndroid Build Coastguard Worker 
ZeroMemory(void * address,size_t length,bool release_eagerly)1291*795d594fSAndroid Build Coastguard Worker void ZeroMemory(void* address, size_t length, bool release_eagerly) {
1292*795d594fSAndroid Build Coastguard Worker   if (length == 0) {
1293*795d594fSAndroid Build Coastguard Worker     return;
1294*795d594fSAndroid Build Coastguard Worker   }
1295*795d594fSAndroid Build Coastguard Worker   uint8_t* const mem_begin = reinterpret_cast<uint8_t*>(address);
1296*795d594fSAndroid Build Coastguard Worker   uint8_t* const mem_end = mem_begin + length;
1297*795d594fSAndroid Build Coastguard Worker   uint8_t* const page_begin = AlignUp(mem_begin, MemMap::GetPageSize());
1298*795d594fSAndroid Build Coastguard Worker   uint8_t* const page_end = AlignDown(mem_end, MemMap::GetPageSize());
1299*795d594fSAndroid Build Coastguard Worker   if (!kMadviseZeroes || page_begin >= page_end) {
1300*795d594fSAndroid Build Coastguard Worker     // No possible area to madvise.
1301*795d594fSAndroid Build Coastguard Worker     RawClearMemory(mem_begin, mem_end);
1302*795d594fSAndroid Build Coastguard Worker     return;
1303*795d594fSAndroid Build Coastguard Worker   }
1304*795d594fSAndroid Build Coastguard Worker   // Spans one or more pages.
1305*795d594fSAndroid Build Coastguard Worker   DCHECK_LE(mem_begin, page_begin);
1306*795d594fSAndroid Build Coastguard Worker   DCHECK_LE(page_begin, page_end);
1307*795d594fSAndroid Build Coastguard Worker   DCHECK_LE(page_end, mem_end);
1308*795d594fSAndroid Build Coastguard Worker #ifdef _WIN32
1309*795d594fSAndroid Build Coastguard Worker   UNUSED(release_eagerly);
1310*795d594fSAndroid Build Coastguard Worker   LOG(WARNING) << "ZeroMemory does not madvise on Windows.";
1311*795d594fSAndroid Build Coastguard Worker   RawClearMemory(mem_begin, mem_end);
1312*795d594fSAndroid Build Coastguard Worker #else
1313*795d594fSAndroid Build Coastguard Worker   RawClearMemory(mem_begin, page_begin);
1314*795d594fSAndroid Build Coastguard Worker   RawClearMemory(page_end, mem_end);
1315*795d594fSAndroid Build Coastguard Worker // mincore() is linux-specific syscall.
1316*795d594fSAndroid Build Coastguard Worker #if defined(__linux__)
1317*795d594fSAndroid Build Coastguard Worker   if (!release_eagerly) {
1318*795d594fSAndroid Build Coastguard Worker     size_t vec_len = (page_end - page_begin) / MemMap::GetPageSize();
1319*795d594fSAndroid Build Coastguard Worker     std::unique_ptr<unsigned char[]> vec(new unsigned char[vec_len]);
1320*795d594fSAndroid Build Coastguard Worker     if (mincore(page_begin, page_end - page_begin, vec.get()) == 0) {
1321*795d594fSAndroid Build Coastguard Worker       uint8_t* current_page = page_begin;
1322*795d594fSAndroid Build Coastguard Worker       size_t current_size = MemMap::GetPageSize();
1323*795d594fSAndroid Build Coastguard Worker       uint32_t old_state = vec[0] & 0x1;
1324*795d594fSAndroid Build Coastguard Worker       for (size_t i = 1; i < vec_len; ++i) {
1325*795d594fSAndroid Build Coastguard Worker         uint32_t new_state = vec[i] & 0x1;
1326*795d594fSAndroid Build Coastguard Worker         if (old_state == new_state) {
1327*795d594fSAndroid Build Coastguard Worker           current_size += MemMap::GetPageSize();
1328*795d594fSAndroid Build Coastguard Worker         } else {
1329*795d594fSAndroid Build Coastguard Worker           ClearMemory(current_page, current_size, old_state, MemMap::GetPageSize());
1330*795d594fSAndroid Build Coastguard Worker           current_page = current_page + current_size;
1331*795d594fSAndroid Build Coastguard Worker           current_size = MemMap::GetPageSize();
1332*795d594fSAndroid Build Coastguard Worker           old_state = new_state;
1333*795d594fSAndroid Build Coastguard Worker         }
1334*795d594fSAndroid Build Coastguard Worker       }
1335*795d594fSAndroid Build Coastguard Worker       ClearMemory(current_page, current_size, old_state, MemMap::GetPageSize());
1336*795d594fSAndroid Build Coastguard Worker       return;
1337*795d594fSAndroid Build Coastguard Worker     }
1338*795d594fSAndroid Build Coastguard Worker     static bool logged_about_mincore = false;
1339*795d594fSAndroid Build Coastguard Worker     if (!logged_about_mincore) {
1340*795d594fSAndroid Build Coastguard Worker       PLOG(WARNING) << "mincore failed, falling back to madvise MADV_DONTNEED";
1341*795d594fSAndroid Build Coastguard Worker       logged_about_mincore = true;
1342*795d594fSAndroid Build Coastguard Worker     }
1343*795d594fSAndroid Build Coastguard Worker     // mincore failed, fall through to MADV_DONTNEED.
1344*795d594fSAndroid Build Coastguard Worker   }
1345*795d594fSAndroid Build Coastguard Worker #else
1346*795d594fSAndroid Build Coastguard Worker   UNUSED(release_eagerly);
1347*795d594fSAndroid Build Coastguard Worker #endif  // __linux__
1348*795d594fSAndroid Build Coastguard Worker   bool res = madvise(page_begin, page_end - page_begin, MADV_DONTNEED);
1349*795d594fSAndroid Build Coastguard Worker   CHECK_NE(res, -1) << "madvise failed";
1350*795d594fSAndroid Build Coastguard Worker #endif  // _WIN32
1351*795d594fSAndroid Build Coastguard Worker }
1352*795d594fSAndroid Build Coastguard Worker 
AlignBy(size_t alignment,bool align_both_ends)1353*795d594fSAndroid Build Coastguard Worker void MemMap::AlignBy(size_t alignment, bool align_both_ends) {
1354*795d594fSAndroid Build Coastguard Worker   CHECK_EQ(begin_, base_begin_) << "Unsupported";
1355*795d594fSAndroid Build Coastguard Worker   CHECK_EQ(size_, base_size_) << "Unsupported";
1356*795d594fSAndroid Build Coastguard Worker   CHECK_GT(alignment, static_cast<size_t>(GetPageSize()));
1357*795d594fSAndroid Build Coastguard Worker   CHECK_ALIGNED_PARAM(alignment, GetPageSize());
1358*795d594fSAndroid Build Coastguard Worker   CHECK(!reuse_);
1359*795d594fSAndroid Build Coastguard Worker   if (IsAlignedParam(reinterpret_cast<uintptr_t>(base_begin_), alignment) &&
1360*795d594fSAndroid Build Coastguard Worker       (!align_both_ends || IsAlignedParam(base_size_, alignment))) {
1361*795d594fSAndroid Build Coastguard Worker     // Already aligned.
1362*795d594fSAndroid Build Coastguard Worker     return;
1363*795d594fSAndroid Build Coastguard Worker   }
1364*795d594fSAndroid Build Coastguard Worker   uint8_t* base_begin = reinterpret_cast<uint8_t*>(base_begin_);
1365*795d594fSAndroid Build Coastguard Worker   uint8_t* aligned_base_begin = AlignUp(base_begin, alignment);
1366*795d594fSAndroid Build Coastguard Worker   CHECK_LE(base_begin, aligned_base_begin);
1367*795d594fSAndroid Build Coastguard Worker   if (base_begin < aligned_base_begin) {
1368*795d594fSAndroid Build Coastguard Worker     MEMORY_TOOL_MAKE_UNDEFINED(base_begin, aligned_base_begin - base_begin);
1369*795d594fSAndroid Build Coastguard Worker     CHECK_EQ(TargetMUnmap(base_begin, aligned_base_begin - base_begin), 0)
1370*795d594fSAndroid Build Coastguard Worker         << "base_begin=" << reinterpret_cast<void*>(base_begin)
1371*795d594fSAndroid Build Coastguard Worker         << " aligned_base_begin=" << reinterpret_cast<void*>(aligned_base_begin);
1372*795d594fSAndroid Build Coastguard Worker   }
1373*795d594fSAndroid Build Coastguard Worker   uint8_t* base_end = base_begin + base_size_;
1374*795d594fSAndroid Build Coastguard Worker   size_t aligned_base_size;
1375*795d594fSAndroid Build Coastguard Worker   if (align_both_ends) {
1376*795d594fSAndroid Build Coastguard Worker     uint8_t* aligned_base_end = AlignDown(base_end, alignment);
1377*795d594fSAndroid Build Coastguard Worker     CHECK_LE(aligned_base_end, base_end);
1378*795d594fSAndroid Build Coastguard Worker     CHECK_LT(aligned_base_begin, aligned_base_end)
1379*795d594fSAndroid Build Coastguard Worker         << "base_begin = " << reinterpret_cast<void*>(base_begin)
1380*795d594fSAndroid Build Coastguard Worker         << " base_end = " << reinterpret_cast<void*>(base_end);
1381*795d594fSAndroid Build Coastguard Worker     aligned_base_size = aligned_base_end - aligned_base_begin;
1382*795d594fSAndroid Build Coastguard Worker     CHECK_GE(aligned_base_size, alignment);
1383*795d594fSAndroid Build Coastguard Worker     if (aligned_base_end < base_end) {
1384*795d594fSAndroid Build Coastguard Worker       MEMORY_TOOL_MAKE_UNDEFINED(aligned_base_end, base_end - aligned_base_end);
1385*795d594fSAndroid Build Coastguard Worker       CHECK_EQ(TargetMUnmap(aligned_base_end, base_end - aligned_base_end), 0)
1386*795d594fSAndroid Build Coastguard Worker           << "base_end=" << reinterpret_cast<void*>(base_end)
1387*795d594fSAndroid Build Coastguard Worker           << " aligned_base_end=" << reinterpret_cast<void*>(aligned_base_end);
1388*795d594fSAndroid Build Coastguard Worker     }
1389*795d594fSAndroid Build Coastguard Worker   } else {
1390*795d594fSAndroid Build Coastguard Worker     CHECK_LT(aligned_base_begin, base_end)
1391*795d594fSAndroid Build Coastguard Worker         << "base_begin = " << reinterpret_cast<void*>(base_begin);
1392*795d594fSAndroid Build Coastguard Worker     aligned_base_size = base_end - aligned_base_begin;
1393*795d594fSAndroid Build Coastguard Worker   }
1394*795d594fSAndroid Build Coastguard Worker   std::lock_guard<std::mutex> mu(*mem_maps_lock_);
1395*795d594fSAndroid Build Coastguard Worker   if (base_begin < aligned_base_begin) {
1396*795d594fSAndroid Build Coastguard Worker     auto it = GetGMapsEntry(*this);
1397*795d594fSAndroid Build Coastguard Worker     auto node = gMaps->extract(it);
1398*795d594fSAndroid Build Coastguard Worker     node.key() = aligned_base_begin;
1399*795d594fSAndroid Build Coastguard Worker     gMaps->insert(std::move(node));
1400*795d594fSAndroid Build Coastguard Worker   }
1401*795d594fSAndroid Build Coastguard Worker   base_begin_ = aligned_base_begin;
1402*795d594fSAndroid Build Coastguard Worker   base_size_ = aligned_base_size;
1403*795d594fSAndroid Build Coastguard Worker   begin_ = aligned_base_begin;
1404*795d594fSAndroid Build Coastguard Worker   size_ = aligned_base_size;
1405*795d594fSAndroid Build Coastguard Worker   DCHECK(gMaps != nullptr);
1406*795d594fSAndroid Build Coastguard Worker }
1407*795d594fSAndroid Build Coastguard Worker 
1408*795d594fSAndroid Build Coastguard Worker }  // namespace art
1409