1 #include "pin_utils.h"
2 #include <android-base/parseint.h>
3 #include <algorithm>
4 #include <fstream>
5 #include <map>
6 #include <string_view>
7 #include <utility>
8 #include <vector>
9
10 using namespace std;
11 using namespace android::base;
12
write_pinlist_file(const std::string & output_file,const std::vector<ZipEntryCoverage> & files_to_write,int64_t write_quota)13 int write_pinlist_file(const std::string& output_file,
14 const std::vector<ZipEntryCoverage>& files_to_write, int64_t write_quota) {
15 std::vector<VmaRange> ranges;
16 for (auto&& file : files_to_write) {
17 ranges.insert(ranges.end(), file.coverage.ranges.begin(), file.coverage.ranges.end());
18 }
19 return write_pinlist_file(output_file, ranges, write_quota);
20 }
21
write_pinlist_file(const std::string & output_file,const std::vector<VmaRange> & vmas_to_write,int64_t write_quota)22 int write_pinlist_file(const std::string& output_file, const std::vector<VmaRange>& vmas_to_write,
23 int64_t write_quota) {
24 ofstream pinlist_file(output_file);
25 if (pinlist_file.fail()) {
26 return 1;
27 }
28 int64_t total_written = 0;
29 unsigned int page_size = sysconf(_SC_PAGESIZE);
30 const bool has_quota = write_quota > 0;
31 bool reached_quota = false;
32
33 // The PinnerService does not require aligned offsets, however, aligning
34 // allows our summary results to be accurate and avoids over-accounting
35 // of pinning in PinnerService.
36 std::vector<VmaRange> processed_vmas_to_write = vmas_to_write;
37 align_ranges(processed_vmas_to_write, page_size);
38
39 // When we page-align the ranges, we may cause overlaps between ranges
40 // as we elongate the begin offset to match the page the previous
41 // range may end up overlapping the current one.
42 processed_vmas_to_write = merge_ranges(processed_vmas_to_write);
43
44 for (auto&& processed_vma_to_write : processed_vmas_to_write) {
45 uint32_t vma_start_offset = processed_vma_to_write.offset;
46 uint32_t vma_length = processed_vma_to_write.length;
47 if (has_quota && (total_written + vma_length > write_quota)) {
48 // We would go beyond quota, set the maximum allowed write and exit.
49 vma_length = write_quota - total_written;
50 reached_quota = true;
51 }
52 // Transform to BigEndian as PinnerService requires that endianness for reading.
53 uint32_t vma_start_offset_be = htobe32(vma_start_offset);
54 uint32_t vma_length_be = htobe32(vma_length);
55 cout << "Pinlist Writing start=" << vma_start_offset << " bytes=" << vma_length << endl;
56 pinlist_file.write(reinterpret_cast<char*>(&vma_start_offset_be),
57 sizeof(vma_start_offset_be));
58 if (pinlist_file.fail()) {
59 return 1;
60 }
61 pinlist_file.write(reinterpret_cast<char*>(&vma_length_be), sizeof(vma_length_be));
62 total_written += vma_length;
63 if (pinlist_file.fail()) {
64 return 1;
65 }
66
67 if (reached_quota) {
68 break;
69 }
70 }
71 return 0;
72 }
73
read_pinlist_file(const std::string & pinner_file,std::vector<VmaRange> & pinranges)74 int read_pinlist_file(const std::string& pinner_file, /*out*/ std::vector<VmaRange>& pinranges) {
75 ifstream pinlist_file(pinner_file);
76 if (pinlist_file.fail()) {
77 return 1;
78 }
79
80 uint32_t vma_start;
81 uint32_t vma_length;
82 while (!pinlist_file.eof()) {
83 pinlist_file.read(reinterpret_cast<char*>(&vma_start), sizeof(vma_start));
84 pinlist_file.read(reinterpret_cast<char*>(&vma_length), sizeof(vma_length));
85 if (pinlist_file.fail()) {
86 return 1;
87 }
88 vma_start = betoh32(vma_start);
89 vma_length = betoh32(vma_length);
90 pinranges.push_back(VmaRange(vma_start, vma_length));
91 }
92
93 return 0;
94 }
95
to_zipfilemem(const ZipEntryInfo & info)96 ZipEntryCoverage PinConfigFile::to_zipfilemem(const ZipEntryInfo& info) {
97 ZipEntryCoverage file;
98 file.info = info;
99
100 if (ranges.empty()) {
101 cout << "No ranges found for file " << info.name << " creating entire file range" << endl;
102 // Any file coming from pinconfig without explicit
103 // ranges will be assumed to be wanted in its entirety
104 ranges.push_back(VmaRange(0, info.file_size_bytes));
105 }
106
107 file.coverage.ranges = ranges;
108
109 // Offsets specified in pinconfig file are relative to the file
110 // so transform to zip global offsets which are used for coverage
111 // computations.
112 file.coverage.apply_offset(info.offset_in_zip);
113
114 file.coverage.compute_total_size();
115 return file;
116 }
117
parse(std::string config_file,bool verbose)118 int PinConfig::parse(std::string config_file, bool verbose) {
119 ifstream file(config_file);
120 string file_in_zip;
121 if (verbose) {
122 cout << "Parsing file: " << config_file << endl;
123 }
124 string token;
125 file >> token;
126 while (!file.eof()) {
127 if (token == "file") {
128 file >> file_in_zip;
129 PinConfigFile pin_config_file;
130 pin_config_file.filename = file_in_zip;
131 file >> token;
132 while (token != "file" && !file.eof()) {
133 VmaRange range;
134 // Inner parsing loop for per file config.
135 if (token == "offset") {
136 file >> token;
137 android::base::ParseUint(token, &range.offset);
138 file >> token;
139 if (token != "len") {
140 cerr << "Malformed file, expected 'len' after offset" << endl;
141 return 1;
142 }
143 file >> token;
144 android::base::ParseUint(token, &range.length);
145 pin_config_file.ranges.push_back(range);
146 }
147 file >> token;
148 }
149 files_.push_back(pin_config_file);
150 } else {
151 cerr << "Unexpected token: " << token << ". Exit read" << endl;
152 return 1;
153 }
154 }
155
156 if (files_.empty()) {
157 cerr << "Failed parsing pinconfig file, no entries found." << endl;
158 return 1;
159 }
160
161 if (verbose) {
162 cout << "Finished parsing Pinconfig file" << endl;
163 for (auto&& pin_file : files_) {
164 cout << "file=" << pin_file.filename << endl;
165 for (auto&& range : pin_file.ranges) {
166 cout << "offset=" << range.offset << " bytes=" << range.length << endl;
167 }
168 }
169 }
170
171 return 0;
172 }
173
set_custom_zip_inspector(ZipMemInspector * inspector)174 void PinTool::set_custom_zip_inspector(ZipMemInspector* inspector) {
175 delete zip_inspector_;
176 zip_inspector_ = inspector;
177 }
178
set_verbose_output(bool verbose)179 void PinTool::set_verbose_output(bool verbose) {
180 verbose_ = verbose;
181 }
182
read_probe_from_pinlist(std::string custom_probe_file)183 void PinTool::read_probe_from_pinlist(std::string custom_probe_file) {
184 custom_probe_file_ = custom_probe_file;
185 VmaRangeGroup* custom_probe = new VmaRangeGroup();
186 read_pinlist_file(custom_probe_file_, custom_probe->ranges);
187 custom_probe->compute_total_size();
188 if (custom_probe->ranges.empty()) {
189 cerr << "Did not find any memory range in " << custom_probe_file_ << endl;
190 delete custom_probe;
191 return;
192 }
193 zip_inspector_->set_existing_probe(custom_probe);
194 }
195
probe_resident()196 int PinTool::probe_resident() {
197 return zip_inspector_->probe_resident();
198 }
199
compute_zip_entry_coverages()200 void PinTool::compute_zip_entry_coverages() {
201 zip_inspector_->compute_per_file_coverage();
202 if (verbose_) {
203 std::vector<ZipEntryInfo> files = zip_inspector_->get_file_infos();
204 for (auto&& file : files) {
205 cout << "file found. name=" << file.name << " offset=" << file.offset_in_zip
206 << " uncompressed=" << file.uncompressed_size
207 << " compressed=" << file.file_size_bytes << endl
208 << endl;
209 }
210 }
211 }
212
dump_coverages(PinTool::DumpType dump_type)213 void PinTool::dump_coverages(PinTool::DumpType dump_type) {
214 std::vector<ZipEntryCoverage>* file_coverages;
215 if (dump_type == PinTool::DumpType::FILTERED) {
216 file_coverages = &filtered_files_;
217 } else if (dump_type == PinTool::DumpType::FILE_COVERAGE) {
218 file_coverages = &(zip_inspector_->get_file_coverages());
219 } else { // PinTool::DumpType::PROBE
220 VmaRangeGroup* probe = zip_inspector_->get_probe();
221 file_coverages = new vector<ZipEntryCoverage>();
222 ZipEntryCoverage file;
223 file.coverage = *probe;
224 file.info.name = input_file_;
225 file.info.offset_in_zip = 0;
226 uint64_t file_size_bytes = get_file_size(input_file_);
227 if (file_size_bytes == -1) {
228 cerr << "Failed to dump, cannot fstat file: " << input_file_ << endl;
229 delete file_coverages;
230 return;
231 }
232 file.info.file_size_bytes = file_size_bytes;
233 file_coverages->push_back(file);
234 }
235
236 for (auto&& file : *file_coverages) {
237 uint64_t total_size = file.coverage.compute_total_size();
238 cout << file.info.name << " size(B)=" << file.info.file_size_bytes
239 << " resident(B)=" << total_size
240 << " resident(%)=" << (double)(total_size) / file.info.file_size_bytes * 100.0 << endl;
241 if (verbose_) {
242 cout << "file_base_zip_offset=" << file.info.offset_in_zip << endl;
243 }
244 cout << "file resident ranges" << endl;
245 if (dump_type != DumpType::PROBE) {
246 for (auto&& range : file.coverage.ranges) {
247 // The offset in the range represents the absolute absolute offset relative to the
248 // zip so substract the file base offset to get the relative offset within the file
249 // which may be what is worth for a user to specify in pinconfig.txt files.
250 uint64_t offset_in_file = range.offset - file.info.offset_in_zip;
251
252 cout << "zip_offset=" << range.offset << " file_offset=" << offset_in_file
253 << " total_bytes=" << range.length << endl;
254 }
255 } else {
256 for (auto&& range : file.coverage.ranges) {
257 cout << "file_offset=" << range.offset << " total_bytes=" << range.length << endl;
258 }
259 }
260 cout << endl;
261 }
262 cout << endl;
263 if (dump_type == DumpType::PROBE) {
264 // For other dump types we do not create memory, we reuse from class.
265 delete file_coverages;
266 }
267 }
268
filter_zip_entry_coverages(const std::string & pinconfig_filename)269 void PinTool::filter_zip_entry_coverages(const std::string& pinconfig_filename) {
270 if (pinconfig_filename.length() == 0) {
271 // Nothing to do.
272 return;
273 }
274
275 PinConfig* pinconfig = new PinConfig();
276 if (pinconfig->parse(pinconfig_filename, verbose_) > 0) {
277 cerr << "Failed parsing pinconfig file " << pinconfig_filename << ". Skip filtering";
278 delete pinconfig;
279 return;
280 }
281
282 filter_zip_entry_coverages(pinconfig);
283 }
284
filter_zip_entry_coverages(PinConfig * pinconfig)285 void PinTool::filter_zip_entry_coverages(PinConfig* pinconfig) {
286 pinconfig_ = pinconfig;
287
288 // Filter based on the per file configuration.
289 vector<ZipEntryCoverage> file_coverages = zip_inspector_->get_file_coverages();
290 vector<ZipEntryCoverage>& filtered_files = filtered_files_;
291
292 for (auto&& file_coverage : file_coverages) {
293 for (auto&& pinconfig_file : pinconfig_->files_) {
294 // Match each zip entry against every pattern in filter file.
295 std::string_view file_coverage_view(file_coverage.info.name.c_str());
296 std::string_view pinconfig_view(pinconfig_file.filename.c_str());
297 if (file_coverage_view.find(pinconfig_view) != std::string_view::npos) {
298 // Now that we found a match, create a file with offsets that are global to zip file
299 ZipEntryCoverage file_in_config = pinconfig_file.to_zipfilemem(file_coverage.info);
300 if (verbose_) {
301 cout << "Found a match: file=" << file_coverage.info.name
302 << " matching filter=" << pinconfig_file.filename << endl;
303 for (auto&& range : file_in_config.coverage.ranges) {
304 cout << "zip_offset=" << range.offset << " bytes=" << range.length << endl;
305 }
306 }
307 ZipEntryCoverage filtered_file =
308 file_coverage.compute_coverage(file_in_config.coverage);
309 filtered_files.push_back(filtered_file);
310 break;
311 }
312 }
313 }
314 }
315
get_filtered_zip_entries()316 std::vector<ZipEntryCoverage> PinTool::get_filtered_zip_entries() {
317 return filtered_files_;
318 }
319
write_coverages_as_pinlist(std::string output_pinlist,int64_t write_quota)320 void PinTool::write_coverages_as_pinlist(std::string output_pinlist, int64_t write_quota) {
321 std::vector<ZipEntryCoverage>* pinlist_coverages = nullptr;
322 if (!filtered_files_.empty()) {
323 // Highest preference is writing filtered files if they exist
324 if (verbose_) {
325 cout << "Writing pinconfig filtered file coverages" << endl;
326 }
327 pinlist_coverages = &filtered_files_;
328 } else if (!zip_inspector_->get_file_coverages().empty()) {
329 // Fallback to looking for file coverage computation
330 pinlist_coverages = &zip_inspector_->get_file_coverages();
331 if (verbose_) {
332 cout << "Writing regular file coverages." << endl;
333 }
334 }
335 if (pinlist_coverages == nullptr) {
336 cerr << "Failed to find coverage to write to: " << output_pinlist << endl;
337 return;
338 }
339 int res = write_pinlist_file(output_pinlist, *pinlist_coverages, write_quota);
340 if (res > 0) {
341 cerr << "Failed to write pin file at: " << output_pinlist << endl;
342 } else {
343 if (verbose_) {
344 cout << "Finished writing pin file at: " << output_pinlist << endl;
345 }
346 }
347 }