xref: /aosp_15_r20/external/perfetto/src/trace_processor/importers/proto/system_probes_parser.cc (revision 6dbdd20afdafa5e3ca9b8809fa73465d530080dc) 
1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "src/trace_processor/importers/proto/system_probes_parser.h"
18 
19 #include <algorithm>
20 #include <cstddef>
21 #include <cstdint>
22 #include <optional>
23 #include <string>
24 #include <utility>
25 #include <variant>
26 #include <vector>
27 
28 #include "perfetto/base/logging.h"
29 #include "perfetto/ext/base/status_or.h"
30 #include "perfetto/ext/base/string_utils.h"
31 #include "perfetto/ext/base/string_view.h"
32 #include "perfetto/ext/traced/sys_stats_counters.h"
33 #include "perfetto/protozero/field.h"
34 #include "perfetto/protozero/proto_decoder.h"
35 #include "perfetto/public/compiler.h"
36 #include "src/kernel_utils/syscall_table.h"
37 #include "src/trace_processor/containers/string_pool.h"
38 #include "src/trace_processor/importers/common/args_tracker.h"
39 #include "src/trace_processor/importers/common/clock_tracker.h"
40 #include "src/trace_processor/importers/common/cpu_tracker.h"
41 #include "src/trace_processor/importers/common/event_tracker.h"
42 #include "src/trace_processor/importers/common/metadata_tracker.h"
43 #include "src/trace_processor/importers/common/process_tracker.h"
44 #include "src/trace_processor/importers/common/system_info_tracker.h"
45 #include "src/trace_processor/importers/common/track_tracker.h"
46 #include "src/trace_processor/importers/common/tracks.h"
47 #include "src/trace_processor/importers/common/tracks_common.h"
48 #include "src/trace_processor/importers/common/tracks_internal.h"
49 #include "src/trace_processor/importers/syscalls/syscall_tracker.h"
50 #include "src/trace_processor/storage/metadata.h"
51 #include "src/trace_processor/storage/stats.h"
52 #include "src/trace_processor/storage/trace_storage.h"
53 #include "src/trace_processor/tables/metadata_tables_py.h"
54 #include "src/trace_processor/types/trace_processor_context.h"
55 #include "src/trace_processor/types/variadic.h"
56 
57 #include "protos/perfetto/common/builtin_clock.pbzero.h"
58 #include "protos/perfetto/trace/ps/process_stats.pbzero.h"
59 #include "protos/perfetto/trace/ps/process_tree.pbzero.h"
60 #include "protos/perfetto/trace/sys_stats/sys_stats.pbzero.h"
61 #include "protos/perfetto/trace/system_info.pbzero.h"
62 #include "protos/perfetto/trace/system_info/cpu_info.pbzero.h"
63 
64 namespace perfetto::trace_processor {
65 
66 namespace {
67 
VersionStringToSdkVersion(const std::string & version)68 std::optional<int> VersionStringToSdkVersion(const std::string& version) {
69   // TODO(lalitm): remove this when the SDK version polling saturates
70   // S/T traces in practice.
71   if (base::StartsWith(version, "T") || base::StartsWith(version, "S")) {
72     return 31;
73   }
74 
75   // Documentation for this mapping can be found at
76   // https://source.android.com/compatibility/cdd.
77   if (version == "12") {
78     return 31;
79   }
80   if (version == "11") {
81     return 30;
82   }
83   if (version == "10") {
84     return 29;
85   }
86   if (version == "9") {
87     return 28;
88   }
89   if (version == "8.1") {
90     return 27;
91   }
92   if (version == "8.0") {
93     return 26;
94   }
95   if (version == "7.1") {
96     return 25;
97   }
98   if (version == "7.0") {
99     return 24;
100   }
101   if (version == "6.0") {
102     return 23;
103   }
104   if (version == "5.1" || version == "5.1.1") {
105     return 22;
106   }
107   if (version == "5.0" || version == "5.0.1" || version == "5.0.2") {
108     return 21;
109   }
110   // If we reached this point, we don't know how to parse this version
111   // so just return null.
112   return std::nullopt;
113 }
114 
FingerprintToSdkVersion(const std::string & fingerprint)115 std::optional<int> FingerprintToSdkVersion(const std::string& fingerprint) {
116   // Try to parse the SDK version from the fingerprint.
117   // Examples of fingerprints:
118   // google/shamu/shamu:7.0/NBD92F/3753956:userdebug/dev-keys
119   // google/coral/coral:12/SP1A.210812.015/7679548:userdebug/dev-keys
120   size_t colon = fingerprint.find(':');
121   if (colon == std::string::npos)
122     return std::nullopt;
123 
124   size_t slash = fingerprint.find('/', colon);
125   if (slash == std::string::npos)
126     return std::nullopt;
127 
128   std::string version = fingerprint.substr(colon + 1, slash - (colon + 1));
129   return VersionStringToSdkVersion(version);
130 }
131 
IsSupportedDiskStatDevice(const std::string & device_name)132 bool IsSupportedDiskStatDevice(const std::string& device_name) {
133   return device_name == "sda";  // Primary SCSI disk device name
134 }
135 
136 struct ArmCpuIdentifier {
137   uint32_t implementer;
138   uint32_t architecture;
139   uint32_t variant;
140   uint32_t part;
141   uint32_t revision;
142 };
143 
144 struct CpuInfo {
145   uint32_t cpu = 0;
146   std::optional<uint32_t> capacity;
147   std::vector<uint32_t> frequencies;
148   protozero::ConstChars processor;
149   // Extend the variant to support additional identifiers
150   std::variant<std::nullopt_t, ArmCpuIdentifier> identifier = std::nullopt;
151 };
152 
153 struct CpuMaxFrequency {
154   uint32_t cpu = 0;
155   uint32_t max_frequency = 0;
156 };
157 
GetPsiResourceKey(size_t resource)158 const char* GetPsiResourceKey(size_t resource) {
159   using PsiResource = protos::pbzero::SysStats::PsiSample::PsiResource;
160   switch (resource) {
161     case PsiResource::PSI_RESOURCE_UNSPECIFIED:
162       return "resource.unspecified";
163     case PsiResource::PSI_RESOURCE_CPU_SOME:
164       return "cpu.some";
165     case PsiResource::PSI_RESOURCE_CPU_FULL:
166       return "cpu.full";
167     case PsiResource::PSI_RESOURCE_IO_SOME:
168       return "io.some";
169     case PsiResource::PSI_RESOURCE_IO_FULL:
170       return "io.full";
171     case PsiResource::PSI_RESOURCE_MEMORY_SOME:
172       return "mem.some";
173     case PsiResource::PSI_RESOURCE_MEMORY_FULL:
174       return "mem.full";
175     default:
176       return nullptr;
177   }
178 }
179 
GetProcessMemoryKey(uint32_t field_id)180 const char* GetProcessMemoryKey(uint32_t field_id) {
181   using ProcessStats = protos::pbzero::ProcessStats;
182   switch (field_id) {
183     case ProcessStats::Process::kVmSizeKbFieldNumber:
184       return "virt";
185     case ProcessStats::Process::kVmRssKbFieldNumber:
186       return "rss";
187     case ProcessStats::Process::kRssAnonKbFieldNumber:
188       return "rss.anon";
189     case ProcessStats::Process::kRssFileKbFieldNumber:
190       return "rss.file";
191     case ProcessStats::Process::kRssShmemKbFieldNumber:
192       return "rss.shmem";
193     case ProcessStats::Process::kVmSwapKbFieldNumber:
194       return "swap";
195     case ProcessStats::Process::kVmLockedKbFieldNumber:
196       return "locked";
197     case ProcessStats::Process::kVmHwmKbFieldNumber:
198       return "rss.watermark";
199     default:
200       return nullptr;
201   }
202 }
203 
GetSmapsKey(uint32_t field_id)204 const char* GetSmapsKey(uint32_t field_id) {
205   using ProcessStats = protos::pbzero::ProcessStats;
206   switch (field_id) {
207     case ProcessStats::Process::kSmrRssKbFieldNumber:
208       return "rss";
209     case ProcessStats::Process::kSmrPssKbFieldNumber:
210       return "pss";
211     case ProcessStats::Process::kSmrPssAnonKbFieldNumber:
212       return "pss.anon";
213     case ProcessStats::Process::kSmrPssFileKbFieldNumber:
214       return "pss.file";
215     case ProcessStats::Process::kSmrPssShmemKbFieldNumber:
216       return "pss.smem";
217     case ProcessStats::Process::kSmrSwapPssKbFieldNumber:
218       return "swap.pss";
219     default:
220       return nullptr;
221   }
222 }
223 
224 }  // namespace
225 
SystemProbesParser(TraceProcessorContext * context)226 SystemProbesParser::SystemProbesParser(TraceProcessorContext* context)
227     : context_(context),
228       utid_name_id_(context->storage->InternString("utid")),
229       arm_cpu_implementer(
230           context->storage->InternString("arm_cpu_implementer")),
231       arm_cpu_architecture(
232           context->storage->InternString("arm_cpu_architecture")),
233       arm_cpu_variant(context->storage->InternString("arm_cpu_variant")),
234       arm_cpu_part(context->storage->InternString("arm_cpu_part")),
235       arm_cpu_revision(context->storage->InternString("arm_cpu_revision")),
236       meminfo_strs_(BuildMeminfoCounterNames()),
237       vmstat_strs_(BuildVmstatCounterNames()) {}
238 
ParseDiskStats(int64_t ts,ConstBytes blob)239 void SystemProbesParser::ParseDiskStats(int64_t ts, ConstBytes blob) {
240   protos::pbzero::SysStats::DiskStat::Decoder ds(blob);
241   static constexpr double SECTORS_PER_MB = 2048.0;
242   static constexpr double MS_PER_SEC = 1000.0;
243   std::string device_name = ds.device_name().ToStdString();
244   if (!IsSupportedDiskStatDevice(device_name)) {
245     return;
246   }
247 
248   static constexpr auto kBlueprint = tracks::CounterBlueprint(
249       "diskstat", tracks::UnknownUnitBlueprint(),
250       tracks::DimensionBlueprints(
251           tracks::StringDimensionBlueprint("device_name")),
252       tracks::DynamicNameBlueprint());
253 
254   base::StackString<512> tag_prefix("diskstat.[%s]", device_name.c_str());
255   auto push_counter = [&, this](const char* counter_name, double value) {
256     base::StackString<512> track_name("%s.%s", tag_prefix.c_str(),
257                                       counter_name);
258     StringId string_id = context_->storage->InternString(track_name.c_str());
259     TrackId track = context_->track_tracker->InternTrack(
260         kBlueprint, tracks::Dimensions(track_name.string_view()),
261         tracks::DynamicName(string_id));
262     context_->event_tracker->PushCounter(ts, value, track);
263   };
264 
265   // TODO(rsavitski): with the UI now supporting rate mode for counter tracks,
266   // this is likely redundant.
267   auto calculate_throughput = [](double amount, int64_t diff) {
268     return diff == 0 ? 0 : amount * MS_PER_SEC / static_cast<double>(diff);
269   };
270 
271   auto cur_read_amount = static_cast<int64_t>(ds.read_sectors());
272   auto cur_write_amount = static_cast<int64_t>(ds.write_sectors());
273   auto cur_discard_amount = static_cast<int64_t>(ds.discard_sectors());
274   auto cur_flush_count = static_cast<int64_t>(ds.flush_count());
275   auto cur_read_time = static_cast<int64_t>(ds.read_time_ms());
276   auto cur_write_time = static_cast<int64_t>(ds.write_time_ms());
277   auto cur_discard_time = static_cast<int64_t>(ds.discard_time_ms());
278   auto cur_flush_time = static_cast<int64_t>(ds.flush_time_ms());
279 
280   if (prev_read_amount != -1) {
281     double read_amount =
282         static_cast<double>(cur_read_amount - prev_read_amount) /
283         SECTORS_PER_MB;
284     double write_amount =
285         static_cast<double>(cur_write_amount - prev_write_amount) /
286         SECTORS_PER_MB;
287     double discard_amount =
288         static_cast<double>(cur_discard_amount - prev_discard_amount) /
289         SECTORS_PER_MB;
290     auto flush_count = static_cast<double>(cur_flush_count - prev_flush_count);
291     int64_t read_time_diff = cur_read_time - prev_read_time;
292     int64_t write_time_diff = cur_write_time - prev_write_time;
293     int64_t discard_time_diff = cur_discard_time - prev_discard_time;
294     auto flush_time_diff =
295         static_cast<double>(cur_flush_time - prev_flush_time);
296 
297     double read_thpt = calculate_throughput(read_amount, read_time_diff);
298     double write_thpt = calculate_throughput(write_amount, write_time_diff);
299     double discard_thpt =
300         calculate_throughput(discard_amount, discard_time_diff);
301 
302     push_counter("read_amount(mg)", read_amount);
303     push_counter("read_throughput(mg/s)", read_thpt);
304     push_counter("write_amount(mg)", write_amount);
305     push_counter("write_throughput(mg/s)", write_thpt);
306     push_counter("discard_amount(mg)", discard_amount);
307     push_counter("discard_throughput(mg/s)", discard_thpt);
308     push_counter("flush_amount(count)", flush_count);
309     push_counter("flush_time(ms)", flush_time_diff);
310   }
311 
312   prev_read_amount = cur_read_amount;
313   prev_write_amount = cur_write_amount;
314   prev_discard_amount = cur_discard_amount;
315   prev_flush_count = cur_flush_count;
316   prev_read_time = cur_read_time;
317   prev_write_time = cur_write_time;
318   prev_discard_time = cur_discard_time;
319   prev_flush_time = cur_flush_time;
320 }
321 
ParseSysStats(int64_t ts,ConstBytes blob)322 void SystemProbesParser::ParseSysStats(int64_t ts, ConstBytes blob) {
323   protos::pbzero::SysStats::Decoder sys_stats(blob);
324 
325   static constexpr auto kMeminfoBlueprint = tracks::CounterBlueprint(
326       "meminfo", tracks::kBytesUnitBlueprint,
327       tracks::DimensionBlueprints(
328           tracks::StringDimensionBlueprint("meminfo_key")),
329       tracks::FnNameBlueprint([](base::StringView name) {
330         return base::StackString<1024>("%.*s", int(name.size()), name.data());
331       }));
332   for (auto it = sys_stats.meminfo(); it; ++it) {
333     protos::pbzero::SysStats::MeminfoValue::Decoder mi(*it);
334     auto key = static_cast<size_t>(mi.key());
335     if (PERFETTO_UNLIKELY(key >= meminfo_strs_.size())) {
336       PERFETTO_ELOG("MemInfo key %zu is not recognized.", key);
337       context_->storage->IncrementStats(stats::meminfo_unknown_keys);
338       continue;
339     }
340     // /proc/meminfo counters are in kB, convert to bytes
341     TrackId track = context_->track_tracker->InternTrack(
342         kMeminfoBlueprint, tracks::Dimensions(meminfo_strs_[key]),
343         tracks::BlueprintName());
344     context_->event_tracker->PushCounter(
345         ts, static_cast<double>(mi.value()) * 1024, track);
346   }
347 
348   for (auto it = sys_stats.devfreq(); it; ++it) {
349     protos::pbzero::SysStats::DevfreqValue::Decoder vm(*it);
350     TrackId track = context_->track_tracker->InternTrack(
351         tracks::kClockFrequencyBlueprint, tracks::Dimensions(vm.key()));
352     context_->event_tracker->PushCounter(ts, static_cast<double>(vm.value()),
353                                          track);
354   }
355 
356   uint32_t c = 0;
357   for (auto it = sys_stats.cpufreq_khz(); it; ++it, ++c) {
358     TrackId track = context_->track_tracker->InternTrack(
359         tracks::kCpuFrequencyBlueprint, tracks::Dimensions(c));
360     context_->event_tracker->PushCounter(ts, static_cast<double>(*it), track);
361   }
362 
363   static constexpr auto kVmStatBlueprint = tracks::CounterBlueprint(
364       "vmstat", tracks::UnknownUnitBlueprint(),
365       tracks::DimensionBlueprints(
366           tracks::StringDimensionBlueprint("vmstat_key")),
367       tracks::FnNameBlueprint([](base::StringView name) {
368         return base::StackString<1024>("%.*s", int(name.size()), name.data());
369       }));
370   for (auto it = sys_stats.vmstat(); it; ++it) {
371     protos::pbzero::SysStats::VmstatValue::Decoder vm(*it);
372     auto key = static_cast<size_t>(vm.key());
373     if (PERFETTO_UNLIKELY(key >= vmstat_strs_.size())) {
374       PERFETTO_ELOG("VmStat key %zu is not recognized.", key);
375       context_->storage->IncrementStats(stats::vmstat_unknown_keys);
376       continue;
377     }
378     TrackId track = context_->track_tracker->InternTrack(
379         kVmStatBlueprint, tracks::Dimensions(vmstat_strs_[key]));
380     context_->event_tracker->PushCounter(ts, static_cast<double>(vm.value()),
381                                          track);
382   }
383 
384   for (auto it = sys_stats.cpu_stat(); it; ++it) {
385     protos::pbzero::SysStats::CpuTimes::Decoder ct(*it);
386     if (PERFETTO_UNLIKELY(!ct.has_cpu_id())) {
387       PERFETTO_ELOG("CPU field not found in CpuTimes");
388       context_->storage->IncrementStats(stats::invalid_cpu_times);
389       continue;
390     }
391 
392     static constexpr auto kCpuStatBlueprint = tracks::CounterBlueprint(
393         "cpustat", tracks::UnknownUnitBlueprint(),
394         tracks::DimensionBlueprints(
395             tracks::kCpuDimensionBlueprint,
396             tracks::StringDimensionBlueprint("cpustat_key")),
397         tracks::FnNameBlueprint([](uint32_t, base::StringView key) {
398           return base::StackString<1024>("cpu.times.%.*s", int(key.size()),
399                                          key.data());
400         }));
401     auto intern_track = [&](const char* name) {
402       return context_->track_tracker->InternTrack(
403           kCpuStatBlueprint, tracks::Dimensions(ct.cpu_id(), name));
404     };
405     context_->event_tracker->PushCounter(ts, static_cast<double>(ct.user_ns()),
406                                          intern_track("user_ns"));
407     context_->event_tracker->PushCounter(ts,
408                                          static_cast<double>(ct.user_nice_ns()),
409                                          intern_track("user_nice_ns"));
410     context_->event_tracker->PushCounter(
411         ts, static_cast<double>(ct.system_mode_ns()),
412         intern_track("system_mode_ns"));
413     context_->event_tracker->PushCounter(ts, static_cast<double>(ct.idle_ns()),
414                                          intern_track("idle_ns"));
415     context_->event_tracker->PushCounter(
416         ts, static_cast<double>(ct.io_wait_ns()), intern_track("io_wait_ns"));
417     context_->event_tracker->PushCounter(ts, static_cast<double>(ct.irq_ns()),
418                                          intern_track("irq_ns"));
419     context_->event_tracker->PushCounter(
420         ts, static_cast<double>(ct.softirq_ns()), intern_track("softirq_ns"));
421   }
422 
423   for (auto it = sys_stats.num_irq(); it; ++it) {
424     static constexpr auto kTrackBlueprint = tracks::CounterBlueprint(
425         "num_irq", tracks::UnknownUnitBlueprint(),
426         tracks::DimensionBlueprints(tracks::kIrqDimensionBlueprint),
427         tracks::StaticNameBlueprint("num_irq"));
428     protos::pbzero::SysStats::InterruptCount::Decoder ic(*it);
429     TrackId track = context_->track_tracker->InternTrack(
430         kTrackBlueprint, tracks::Dimensions(ic.irq()));
431     context_->event_tracker->PushCounter(ts, static_cast<double>(ic.count()),
432                                          track);
433   }
434 
435   for (auto it = sys_stats.num_softirq(); it; ++it) {
436     static constexpr auto kTrackBlueprint = tracks::CounterBlueprint(
437         "num_softirq", tracks::UnknownUnitBlueprint(),
438         tracks::DimensionBlueprints(tracks::kIrqDimensionBlueprint),
439         tracks::StaticNameBlueprint("num_softirq"));
440     protos::pbzero::SysStats::InterruptCount::Decoder ic(*it);
441     TrackId track = context_->track_tracker->InternTrack(
442         kTrackBlueprint, tracks::Dimensions(ic.irq()));
443     context_->event_tracker->PushCounter(ts, static_cast<double>(ic.count()),
444                                          track);
445   }
446 
447   if (sys_stats.has_num_forks()) {
448     static constexpr auto kBlueprint =
449         tracks::CounterBlueprint("num_forks", tracks::UnknownUnitBlueprint(),
450                                  tracks::DimensionBlueprints(),
451                                  tracks::StaticNameBlueprint("num_forks"));
452     TrackId track = context_->track_tracker->InternTrack(kBlueprint);
453     context_->event_tracker->PushCounter(
454         ts, static_cast<double>(sys_stats.num_forks()), track);
455   }
456 
457   if (sys_stats.has_num_irq_total()) {
458     static constexpr auto kBlueprint = tracks::CounterBlueprint(
459         "num_irq_total", tracks::UnknownUnitBlueprint(),
460         tracks::DimensionBlueprints(),
461         tracks::StaticNameBlueprint("num_irq_total"));
462     TrackId track = context_->track_tracker->InternTrack(kBlueprint);
463     context_->event_tracker->PushCounter(
464         ts, static_cast<double>(sys_stats.num_irq_total()), track);
465   }
466 
467   if (sys_stats.has_num_softirq_total()) {
468     static constexpr auto kBlueprint = tracks::CounterBlueprint(
469         "num_softirq_total", tracks::UnknownUnitBlueprint(),
470         tracks::DimensionBlueprints(),
471         tracks::StaticNameBlueprint("num_softirq_total"));
472     TrackId track = context_->track_tracker->InternTrack(kBlueprint);
473     context_->event_tracker->PushCounter(
474         ts, static_cast<double>(sys_stats.num_softirq_total()), track);
475   }
476 
477   // Fragmentation of the kernel binary buddy memory allocator.
478   // See /proc/buddyinfo in `man 5 proc`.
479   for (auto it = sys_stats.buddy_info(); it; ++it) {
480     static constexpr auto kBlueprint = tracks::CounterBlueprint(
481         "buddyinfo", tracks::UnknownUnitBlueprint(),
482         tracks::DimensionBlueprints(
483             tracks::StringDimensionBlueprint("buddyinfo_node"),
484             tracks::StringDimensionBlueprint("buddyinfo_zone"),
485             tracks::UintDimensionBlueprint("buddyinfo_chunk_size_kb")),
486         tracks::FnNameBlueprint([](base::StringView node, base::StringView zone,
487                                    uint32_t chunk_size_kb) {
488           return base::StackString<1024>(
489               "mem.buddyinfo[%.*s][%.*s][%u kB]", int(node.size()), node.data(),
490               int(zone.size()), zone.data(), chunk_size_kb);
491         }));
492     protos::pbzero::SysStats::BuddyInfo::Decoder bi(*it);
493     int order = 0;
494     for (auto order_it = bi.order_pages(); order_it; ++order_it) {
495       auto chunk_size_kb =
496           static_cast<uint32_t>(((1 << order) * page_size_) / 1024);
497       TrackId track = context_->track_tracker->InternTrack(
498           kBlueprint, tracks::Dimensions(bi.node(), bi.zone(), chunk_size_kb));
499       context_->event_tracker->PushCounter(ts, static_cast<double>(*order_it),
500                                            track);
501       order++;
502     }
503   }
504 
505   for (auto it = sys_stats.disk_stat(); it; ++it) {
506     ParseDiskStats(ts, *it);
507   }
508 
509   // Pressure Stall Information. See
510   // https://docs.kernel.org/accounting/psi.html.
511   for (auto it = sys_stats.psi(); it; ++it) {
512     protos::pbzero::SysStats::PsiSample::Decoder psi(*it);
513 
514     auto resource = static_cast<size_t>(psi.resource());
515     const char* resource_key = GetPsiResourceKey(resource);
516     if (!resource_key) {
517       context_->storage->IncrementStats(stats::psi_unknown_resource);
518       return;
519     }
520     static constexpr auto kBlueprint = tracks::CounterBlueprint(
521         "psi", tracks::UnknownUnitBlueprint(),
522         tracks::DimensionBlueprints(
523             tracks::StringDimensionBlueprint("psi_resource")),
524         tracks::FnNameBlueprint([](base::StringView resource) {
525           return base::StackString<1024>("psi.%.*s", int(resource.size()),
526                                          resource.data());
527         }));
528     // Unit = total blocked time on this resource in nanoseconds.
529     TrackId track = context_->track_tracker->InternTrack(
530         kBlueprint, tracks::Dimensions(resource_key));
531     context_->event_tracker->PushCounter(
532         ts, static_cast<double>(psi.total_ns()), track);
533   }
534 
535   for (auto it = sys_stats.thermal_zone(); it; ++it) {
536     static constexpr auto kBlueprint = tracks::CounterBlueprint(
537         "thermal_temperature_sys", tracks::StaticUnitBlueprint("C"),
538         tracks::DimensionBlueprints(tracks::kThermalZoneDimensionBlueprint),
539         tracks::FnNameBlueprint([](base::StringView thermal_zone) {
540           return base::StackString<1024>("%.*s", int(thermal_zone.size()),
541                                          thermal_zone.data());
542         }));
543     protos::pbzero::SysStats::ThermalZone::Decoder thermal(*it);
544     TrackId track = context_->track_tracker->InternTrack(
545         kBlueprint, tracks::Dimensions(thermal.type()));
546     context_->event_tracker->PushCounter(
547         ts, static_cast<double>(thermal.temp()), track);
548   }
549 
550   for (auto it = sys_stats.cpuidle_state(); it; ++it) {
551     ParseCpuIdleStats(ts, *it);
552   }
553 
554   for (auto it = sys_stats.gpufreq_mhz(); it; ++it, ++c) {
555     TrackId track = context_->track_tracker->InternTrack(
556         tracks::kGpuFrequencyBlueprint, tracks::Dimensions(0));
557     context_->event_tracker->PushCounter(ts, static_cast<double>(*it), track);
558   }
559 }
560 
ParseCpuIdleStats(int64_t ts,ConstBytes blob)561 void SystemProbesParser::ParseCpuIdleStats(int64_t ts, ConstBytes blob) {
562   static constexpr auto kBlueprint = tracks::CounterBlueprint(
563       "cpu_idle_state", tracks::UnknownUnitBlueprint(),
564       tracks::DimensionBlueprints(
565           tracks::kCpuDimensionBlueprint,
566           tracks::StringDimensionBlueprint("cpu_idle_state")));
567 
568   protos::pbzero::SysStats::CpuIdleState::Decoder cpuidle_state(blob);
569   uint32_t cpu = cpuidle_state.cpu_id();
570   for (auto f = cpuidle_state.cpuidle_state_entry(); f; ++f) {
571     protos::pbzero::SysStats::CpuIdleStateEntry::Decoder idle(*f);
572     TrackId track =
573         context_->track_tracker->InternTrack(kBlueprint, {cpu, idle.state()});
574     context_->event_tracker->PushCounter(
575         ts, static_cast<double>(idle.duration_us()), track);
576   }
577 }
578 
ParseProcessTree(ConstBytes blob)579 void SystemProbesParser::ParseProcessTree(ConstBytes blob) {
580   protos::pbzero::ProcessTree::Decoder ps(blob.data, blob.size);
581 
582   for (auto it = ps.processes(); it; ++it) {
583     protos::pbzero::ProcessTree::Process::Decoder proc(*it);
584     if (!proc.has_cmdline())
585       continue;
586     auto pid = static_cast<uint32_t>(proc.pid());
587     auto ppid = static_cast<uint32_t>(proc.ppid());
588 
589     if (proc.has_nspid()) {
590       std::vector<uint32_t> nspid;
591       for (auto nspid_it = proc.nspid(); nspid_it; nspid_it++) {
592         nspid.emplace_back(static_cast<uint32_t>(*nspid_it));
593       }
594       context_->process_tracker->UpdateNamespacedProcess(pid, std::move(nspid));
595     }
596 
597     protozero::RepeatedFieldIterator<protozero::ConstChars> raw_cmdline =
598         proc.cmdline();
599     base::StringView argv0 = raw_cmdline ? *raw_cmdline : base::StringView();
600     base::StringView joined_cmdline{};
601 
602     // Special case: workqueue kernel threads (kworker). Worker threads are
603     // organised in pools, which can process work from different workqueues.
604     // When we read their thread name via procfs, the kernel takes a dedicated
605     // codepath that appends the name of the current/last workqueue that the
606     // worker processed. This is highly transient and therefore misleading to
607     // users if we keep using this name for the kernel thread.
608     // Example:
609     //   kworker/45:2-mm_percpu_wq
610     //   ^           ^
611     //   [worker id ][last queue ]
612     //
613     // Instead, use a truncated version of the process name that identifies just
614     // the worker itself. For the above example, this would be "kworker/45:2".
615     //
616     // https://github.com/torvalds/linux/blob/6d280f4d760e3bcb4a8df302afebf085b65ec982/kernel/workqueue.c#L5336
617     uint32_t kThreaddPid = 2;
618     if (ppid == kThreaddPid && argv0.StartsWith("kworker/")) {
619       size_t delim_loc = std::min(argv0.find('+', 8), argv0.find('-', 8));
620       if (delim_loc != base::StringView::npos) {
621         argv0 = argv0.substr(0, delim_loc);
622         joined_cmdline = argv0;
623       }
624     }
625 
626     // Special case: some processes rewrite their cmdline with spaces as a
627     // separator instead of a NUL byte. Assume that's the case if there's only a
628     // single cmdline element. This will be wrong for binaries that have spaces
629     // in their path and are invoked without additional arguments, but those are
630     // very rare. The full cmdline will still be correct either way.
631     if (!static_cast<bool>(++proc.cmdline())) {
632       size_t delim_pos = argv0.find(' ');
633       if (delim_pos != base::StringView::npos) {
634         argv0 = argv0.substr(0, delim_pos);
635       }
636     }
637 
638     std::string cmdline_str;
639     if (joined_cmdline.empty()) {
640       for (auto cmdline_it = raw_cmdline; cmdline_it;) {
641         auto cmdline_part = *cmdline_it;
642         cmdline_str.append(cmdline_part.data, cmdline_part.size);
643 
644         if (++cmdline_it)
645           cmdline_str.append(" ");
646       }
647       joined_cmdline = base::StringView(cmdline_str);
648     }
649     UniquePid upid = context_->process_tracker->SetProcessMetadata(
650         pid, ppid, argv0, joined_cmdline);
651 
652     if (proc.has_uid()) {
653       context_->process_tracker->SetProcessUid(
654           upid, static_cast<uint32_t>(proc.uid()));
655     }
656 
657     // note: early kernel threads can have an age of zero (at tick resolution)
658     if (proc.has_process_start_from_boot()) {
659       base::StatusOr<int64_t> start_ts = context_->clock_tracker->ToTraceTime(
660           protos::pbzero::BUILTIN_CLOCK_BOOTTIME,
661           static_cast<int64_t>(proc.process_start_from_boot()));
662       if (start_ts.ok()) {
663         context_->process_tracker->SetStartTsIfUnset(upid, *start_ts);
664       }
665     }
666   }
667 
668   for (auto it = ps.threads(); it; ++it) {
669     protos::pbzero::ProcessTree::Thread::Decoder thd(*it);
670     auto tid = static_cast<uint32_t>(thd.tid());
671     auto tgid = static_cast<uint32_t>(thd.tgid());
672     context_->process_tracker->UpdateThread(tid, tgid);
673 
674     if (thd.has_name()) {
675       StringId thread_name_id = context_->storage->InternString(thd.name());
676       context_->process_tracker->UpdateThreadName(
677           tid, thread_name_id, ThreadNamePriority::kProcessTree);
678     }
679 
680     if (thd.has_nstid()) {
681       std::vector<uint32_t> nstid;
682       for (auto nstid_it = thd.nstid(); nstid_it; nstid_it++) {
683         nstid.emplace_back(static_cast<uint32_t>(*nstid_it));
684       }
685       context_->process_tracker->UpdateNamespacedThread(tgid, tid,
686                                                         std::move(nstid));
687     }
688   }
689 }
690 
ParseProcessStats(int64_t ts,ConstBytes blob)691 void SystemProbesParser::ParseProcessStats(int64_t ts, ConstBytes blob) {
692   // Maps a process counter field it to its value.
693   // E.g., 4 := 1024 -> "mem.rss.anon" := 1024.
694   // std::array<int64_t, kProcStatsProcessSize> counter_values{};
695   // std::array<bool, kProcStatsProcessSize> has_counter{};
696 
697   using Process = protos::pbzero::ProcessStats::Process;
698   protos::pbzero::ProcessStats::Decoder stats(blob);
699   for (auto it = stats.processes(); it; ++it) {
700     protozero::ProtoDecoder proc(*it);
701     uint32_t pid = proc.FindField(Process::kPidFieldNumber).as_uint32();
702     for (auto fld = proc.ReadField(); fld.valid(); fld = proc.ReadField()) {
703       if (fld.id() == Process::kPidFieldNumber) {
704         continue;
705       }
706       if (fld.id() == Process::kThreadsFieldNumber) {
707         ParseThreadStats(ts, pid, fld.as_bytes());
708         continue;
709       }
710       if (fld.id() == Process::kFdsFieldNumber) {
711         ParseProcessFds(ts, pid, fld.as_bytes());
712         continue;
713       }
714       // Chrome fields are processed by ChromeSystemProbesParser.
715       if (fld.id() == Process::kIsPeakRssResettableFieldNumber ||
716           fld.id() == Process::kChromePrivateFootprintKbFieldNumber ||
717           fld.id() == Process::kChromePrivateFootprintKbFieldNumber) {
718         continue;
719       }
720 
721       UniquePid upid = context_->process_tracker->GetOrCreateProcess(pid);
722       if (fld.id() == Process::kOomScoreAdjFieldNumber) {
723         TrackId track = context_->track_tracker->InternTrack(
724             tracks::kOomScoreAdjBlueprint, tracks::DimensionBlueprints(upid));
725         context_->event_tracker->PushCounter(
726             ts, static_cast<double>(fld.as_int64()), track);
727         continue;
728       }
729       {
730         const char* process_memory_key = GetProcessMemoryKey(fld.id());
731         if (process_memory_key) {
732           // Memory counters are in KB, keep values in bytes in the trace
733           // processor.
734           int64_t value = fld.as_int64() * 1024;
735           TrackId track = context_->track_tracker->InternTrack(
736               tracks::kProcessMemoryBlueprint,
737               tracks::DimensionBlueprints(upid, process_memory_key));
738           context_->event_tracker->PushCounter(ts, static_cast<double>(value),
739                                                track);
740           continue;
741         }
742       }
743       {
744         const char* smaps = GetSmapsKey(fld.id());
745         if (smaps) {
746           static constexpr auto kBlueprint = tracks::CounterBlueprint(
747               "smaps", tracks::UnknownUnitBlueprint(),
748               tracks::DimensionBlueprints(
749                   tracks::kProcessDimensionBlueprint,
750                   tracks::StringDimensionBlueprint("smaps_key")),
751               tracks::FnNameBlueprint([](UniquePid, base::StringView key) {
752                 return base::StackString<1024>("mem.smaps.%.*s",
753                                                int(key.size()), key.data());
754               }));
755 
756           // Memory counters are in KB, keep values in bytes in the trace
757           // processor.
758           int64_t value = fld.as_int64() * 1024;
759           TrackId track = context_->track_tracker->InternTrack(
760               kBlueprint, tracks::DimensionBlueprints(upid, smaps));
761           context_->event_tracker->PushCounter(ts, static_cast<double>(value),
762                                                track);
763           continue;
764         }
765       }
766       if (fld.id() == Process::kRuntimeUserModeFieldNumber ||
767           fld.id() == Process::kRuntimeKernelModeFieldNumber) {
768         static constexpr auto kBlueprint = tracks::CounterBlueprint(
769             "proc_stat_runtime", tracks::UnknownUnitBlueprint(),
770             tracks::DimensionBlueprints(
771                 tracks::kProcessDimensionBlueprint,
772                 tracks::StringDimensionBlueprint("proc_stat_runtime_key")),
773             tracks::FnNameBlueprint([](UniquePid, base::StringView key) {
774               return base::StackString<1024>("runtime.%.*s", int(key.size()),
775                                              key.data());
776             }));
777         const char* key = fld.id() == Process::kRuntimeUserModeFieldNumber
778                               ? "user_ns"
779                               : "kernel_ns";
780         TrackId track = context_->track_tracker->InternTrack(
781             kBlueprint, tracks::DimensionBlueprints(upid, key));
782         context_->event_tracker->PushCounter(
783             ts, static_cast<double>(fld.as_int64()), track);
784         continue;
785       }
786 
787       // No handling for this field, so increment the error counter.
788       context_->storage->IncrementStats(stats::proc_stat_unknown_counters);
789     }
790   }
791 }
792 
ParseThreadStats(int64_t,uint32_t pid,ConstBytes blob)793 void SystemProbesParser::ParseThreadStats(int64_t,
794                                           uint32_t pid,
795                                           ConstBytes blob) {
796   protos::pbzero::ProcessStats::Thread::Decoder stats(blob.data, blob.size);
797   context_->process_tracker->UpdateThread(static_cast<uint32_t>(stats.tid()),
798                                           pid);
799 }
800 
ParseProcessFds(int64_t ts,uint32_t pid,ConstBytes blob)801 void SystemProbesParser::ParseProcessFds(int64_t ts,
802                                          uint32_t pid,
803                                          ConstBytes blob) {
804   protos::pbzero::ProcessStats::FDInfo::Decoder fd_info(blob.data, blob.size);
805 
806   tables::FiledescriptorTable::Row row;
807   row.fd = static_cast<int64_t>(fd_info.fd());
808   row.ts = ts;
809   row.path = context_->storage->InternString(fd_info.path());
810   row.upid = context_->process_tracker->GetOrCreateProcess(pid);
811 
812   auto* fd_table = context_->storage->mutable_filedescriptor_table();
813   fd_table->Insert(row);
814 }
815 
ParseSystemInfo(ConstBytes blob)816 void SystemProbesParser::ParseSystemInfo(ConstBytes blob) {
817   protos::pbzero::SystemInfo::Decoder packet(blob.data, blob.size);
818   SystemInfoTracker* system_info_tracker =
819       SystemInfoTracker::GetOrCreate(context_);
820   if (packet.has_utsname()) {
821     ConstBytes utsname_blob = packet.utsname();
822     protos::pbzero::Utsname::Decoder utsname(utsname_blob.data,
823                                              utsname_blob.size);
824     base::StringView machine = utsname.machine();
825     SyscallTracker* syscall_tracker = SyscallTracker::GetOrCreate(context_);
826     Architecture arch = SyscallTable::ArchFromString(machine);
827     if (arch != Architecture::kUnknown) {
828       syscall_tracker->SetArchitecture(arch);
829     } else {
830       PERFETTO_ELOG("Unknown architecture %s. Syscall traces will not work.",
831                     machine.ToStdString().c_str());
832     }
833 
834     system_info_tracker->SetKernelVersion(utsname.sysname(), utsname.release());
835 
836     StringPool::Id sysname_id =
837         context_->storage->InternString(utsname.sysname());
838     StringPool::Id version_id =
839         context_->storage->InternString(utsname.version());
840     StringPool::Id release_id =
841         context_->storage->InternString(utsname.release());
842     StringPool::Id machine_id =
843         context_->storage->InternString(utsname.machine());
844 
845     MetadataTracker* metadata = context_->metadata_tracker.get();
846     metadata->SetMetadata(metadata::system_name, Variadic::String(sysname_id));
847     metadata->SetMetadata(metadata::system_version,
848                           Variadic::String(version_id));
849     metadata->SetMetadata(metadata::system_release,
850                           Variadic::String(release_id));
851     metadata->SetMetadata(metadata::system_machine,
852                           Variadic::String(machine_id));
853   }
854 
855   if (packet.has_timezone_off_mins()) {
856     static constexpr int64_t kNanosInMinute =
857         60ull * 1000ull * 1000ull * 1000ull;
858     context_->metadata_tracker->SetMetadata(
859         metadata::timezone_off_mins,
860         Variadic::Integer(packet.timezone_off_mins()));
861     context_->clock_tracker->set_timezone_offset(packet.timezone_off_mins() *
862                                                  kNanosInMinute);
863   }
864 
865   if (packet.has_android_build_fingerprint()) {
866     context_->metadata_tracker->SetMetadata(
867         metadata::android_build_fingerprint,
868         Variadic::String(context_->storage->InternString(
869             packet.android_build_fingerprint())));
870   }
871 
872   if (packet.has_android_device_manufacturer()) {
873     context_->metadata_tracker->SetMetadata(
874         metadata::android_device_manufacturer,
875         Variadic::String(context_->storage->InternString(
876             packet.android_device_manufacturer())));
877   }
878 
879   // If we have the SDK version in the trace directly just use that.
880   // Otherwise, try and parse it from the fingerprint.
881   std::optional<int64_t> opt_sdk_version;
882   if (packet.has_android_sdk_version()) {
883     opt_sdk_version = static_cast<int64_t>(packet.android_sdk_version());
884   } else if (packet.has_android_build_fingerprint()) {
885     opt_sdk_version = FingerprintToSdkVersion(
886         packet.android_build_fingerprint().ToStdString());
887   }
888 
889   if (opt_sdk_version) {
890     context_->metadata_tracker->SetMetadata(
891         metadata::android_sdk_version, Variadic::Integer(*opt_sdk_version));
892   }
893 
894   if (packet.has_android_soc_model()) {
895     context_->metadata_tracker->SetMetadata(
896         metadata::android_soc_model,
897         Variadic::String(
898             context_->storage->InternString(packet.android_soc_model())));
899   }
900 
901   if (packet.has_android_guest_soc_model()) {
902     context_->metadata_tracker->SetMetadata(
903         metadata::android_guest_soc_model,
904         Variadic::String(
905             context_->storage->InternString(packet.android_guest_soc_model())));
906   }
907 
908   if (packet.has_android_hardware_revision()) {
909     context_->metadata_tracker->SetMetadata(
910         metadata::android_hardware_revision,
911         Variadic::String(context_->storage->InternString(
912             packet.android_hardware_revision())));
913   }
914 
915   if (packet.has_android_storage_model()) {
916     context_->metadata_tracker->SetMetadata(
917         metadata::android_storage_model,
918         Variadic::String(
919             context_->storage->InternString(packet.android_storage_model())));
920   }
921 
922   if (packet.has_android_ram_model()) {
923     context_->metadata_tracker->SetMetadata(
924         metadata::android_ram_model,
925         Variadic::String(
926             context_->storage->InternString(packet.android_ram_model())));
927   }
928 
929   page_size_ = packet.page_size();
930   if (!page_size_) {
931     page_size_ = 4096;
932   }
933 
934   if (packet.has_num_cpus()) {
935     system_info_tracker->SetNumCpus(packet.num_cpus());
936   }
937 }
938 
ParseCpuInfo(ConstBytes blob)939 void SystemProbesParser::ParseCpuInfo(ConstBytes blob) {
940   protos::pbzero::CpuInfo::Decoder packet(blob.data, blob.size);
941   std::vector<CpuInfo> cpu_infos;
942 
943   // Decode CpuInfo packet
944   uint32_t cpu_id = 0;
945   for (auto it = packet.cpus(); it; it++, cpu_id++) {
946     protos::pbzero::CpuInfo::Cpu::Decoder cpu(*it);
947 
948     CpuInfo current_cpu_info;
949     current_cpu_info.cpu = cpu_id;
950     current_cpu_info.processor = cpu.processor();
951 
952     for (auto freq_it = cpu.frequencies(); freq_it; freq_it++) {
953       uint32_t current_cpu_frequency = *freq_it;
954       current_cpu_info.frequencies.push_back(current_cpu_frequency);
955     }
956     if (cpu.has_capacity()) {
957       current_cpu_info.capacity = cpu.capacity();
958     }
959 
960     if (cpu.has_arm_identifier()) {
961       protos::pbzero::CpuInfo::ArmCpuIdentifier::Decoder identifier(
962           cpu.arm_identifier());
963 
964       current_cpu_info.identifier = ArmCpuIdentifier{
965           identifier.implementer(), identifier.architecture(),
966           identifier.variant(),     identifier.part(),
967           identifier.revision(),
968       };
969     }
970 
971     cpu_infos.push_back(current_cpu_info);
972   }
973 
974   // Calculate cluster ids
975   // We look to use capacities as it is an ARM provided metric which is designed
976   // to measure the heterogeneity of CPU clusters however we fallback on the
977   // maximum frequency as an estimate
978 
979   // Capacities are defined as existing on all CPUs if present and so we set
980   // them as invalid if any is missing
981   bool valid_capacities = std::all_of(
982       cpu_infos.begin(), cpu_infos.end(),
983       [](const CpuInfo& info) { return info.capacity.has_value(); });
984 
985   bool valid_frequencies = std::all_of(
986       cpu_infos.begin(), cpu_infos.end(),
987       [](const CpuInfo& info) { return !info.frequencies.empty(); });
988 
989   std::vector<uint32_t> cluster_ids(cpu_infos.size());
990   uint32_t cluster_id = 0;
991 
992   if (valid_capacities) {
993     std::sort(cpu_infos.begin(), cpu_infos.end(),
994               [](auto a, auto b) { return a.capacity < b.capacity; });
995     uint32_t previous_capacity = *cpu_infos[0].capacity;
996     for (CpuInfo& cpu_info : cpu_infos) {
997       uint32_t capacity = *cpu_info.capacity;
998       // If cpus have the same capacity, they should have the same cluster id
999       if (previous_capacity < capacity) {
1000         previous_capacity = capacity;
1001         cluster_id++;
1002       }
1003       cluster_ids[cpu_info.cpu] = cluster_id;
1004     }
1005   } else if (valid_frequencies) {
1006     // Use max frequency if capacities are invalid
1007     std::vector<CpuMaxFrequency> cpu_max_freqs;
1008     cpu_max_freqs.reserve(cpu_infos.size());
1009     for (CpuInfo& info : cpu_infos) {
1010       cpu_max_freqs.push_back(
1011           {info.cpu, *std::max_element(info.frequencies.begin(),
1012                                        info.frequencies.end())});
1013     }
1014     std::sort(cpu_max_freqs.begin(), cpu_max_freqs.end(),
1015               [](auto a, auto b) { return a.max_frequency < b.max_frequency; });
1016 
1017     uint32_t previous_max_freq = cpu_max_freqs[0].max_frequency;
1018     for (CpuMaxFrequency& cpu_max_freq : cpu_max_freqs) {
1019       uint32_t max_freq = cpu_max_freq.max_frequency;
1020       // If cpus have the same max frequency, they should have the same
1021       // cluster_id
1022       if (previous_max_freq < max_freq) {
1023         previous_max_freq = max_freq;
1024         cluster_id++;
1025       }
1026       cluster_ids[cpu_max_freq.cpu] = cluster_id;
1027     }
1028   }
1029 
1030   // Add values to tables
1031   for (CpuInfo& cpu_info : cpu_infos) {
1032     tables::CpuTable::Id ucpu = context_->cpu_tracker->SetCpuInfo(
1033         cpu_info.cpu, cpu_info.processor, cluster_ids[cpu_info.cpu],
1034         cpu_info.capacity);
1035     for (uint32_t frequency : cpu_info.frequencies) {
1036       tables::CpuFreqTable::Row cpu_freq_row;
1037       cpu_freq_row.ucpu = ucpu;
1038       cpu_freq_row.freq = frequency;
1039       context_->storage->mutable_cpu_freq_table()->Insert(cpu_freq_row);
1040     }
1041 
1042     if (auto* id = std::get_if<ArmCpuIdentifier>(&cpu_info.identifier)) {
1043       context_->args_tracker->AddArgsTo(ucpu)
1044           .AddArg(arm_cpu_implementer,
1045                   Variadic::UnsignedInteger(id->implementer))
1046           .AddArg(arm_cpu_architecture,
1047                   Variadic::UnsignedInteger(id->architecture))
1048           .AddArg(arm_cpu_variant, Variadic::UnsignedInteger(id->variant))
1049           .AddArg(arm_cpu_part, Variadic::UnsignedInteger(id->part))
1050           .AddArg(arm_cpu_revision, Variadic::UnsignedInteger(id->revision));
1051     }
1052   }
1053 }
1054 
1055 }  // namespace perfetto::trace_processor
1056