1 /*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/importers/proto/memory_tracker_snapshot_parser.h"
18
19 #include <cstddef>
20 #include <cstdint>
21 #include <memory>
22 #include <optional>
23 #include <string>
24 #include <utility>
25 #include <vector>
26
27 #include "perfetto/base/logging.h"
28 #include "perfetto/base/proc_utils.h"
29 #include "perfetto/ext/base/string_view.h"
30 #include "perfetto/ext/trace_processor/importers/memory_tracker/graph.h"
31 #include "perfetto/ext/trace_processor/importers/memory_tracker/graph_processor.h"
32 #include "perfetto/ext/trace_processor/importers/memory_tracker/memory_allocator_node_id.h"
33 #include "perfetto/ext/trace_processor/importers/memory_tracker/memory_graph_edge.h"
34 #include "perfetto/ext/trace_processor/importers/memory_tracker/raw_memory_graph_node.h"
35 #include "perfetto/ext/trace_processor/importers/memory_tracker/raw_process_memory_node.h"
36 #include "protos/perfetto/trace/memory_graph.pbzero.h"
37 #include "src/trace_processor/containers/string_pool.h"
38 #include "src/trace_processor/importers/common/args_tracker.h"
39 #include "src/trace_processor/importers/common/track_tracker.h"
40 #include "src/trace_processor/importers/common/tracks.h"
41 #include "src/trace_processor/importers/common/tracks_common.h"
42 #include "src/trace_processor/storage/stats.h"
43 #include "src/trace_processor/storage/trace_storage.h"
44 #include "src/trace_processor/tables/memory_tables_py.h"
45 #include "src/trace_processor/types/variadic.h"
46
47 namespace perfetto::trace_processor {
48
MemoryTrackerSnapshotParser(TraceProcessorContext * context)49 MemoryTrackerSnapshotParser::MemoryTrackerSnapshotParser(
50 TraceProcessorContext* context)
51 : context_(context),
52 level_of_detail_ids_{{context_->storage->InternString("background"),
53 context_->storage->InternString("light"),
54 context_->storage->InternString("detailed")}},
55 unit_ids_{{context_->storage->InternString("objects"),
56 context_->storage->InternString("bytes")}},
57 aggregate_raw_nodes_(),
58 last_snapshot_timestamp_(-1),
59 last_snapshot_level_of_detail_(LevelOfDetail::kFirst) {}
60
ParseMemoryTrackerSnapshot(int64_t ts,ConstBytes blob)61 void MemoryTrackerSnapshotParser::ParseMemoryTrackerSnapshot(int64_t ts,
62 ConstBytes blob) {
63 PERFETTO_DCHECK(last_snapshot_timestamp_ <= ts);
64 if (!aggregate_raw_nodes_.empty() && ts != last_snapshot_timestamp_) {
65 GenerateGraphFromRawNodesAndEmitRows();
66 }
67 ReadProtoSnapshot(blob, aggregate_raw_nodes_, last_snapshot_level_of_detail_);
68 last_snapshot_timestamp_ = ts;
69 }
70
NotifyEndOfFile()71 void MemoryTrackerSnapshotParser::NotifyEndOfFile() {
72 if (!aggregate_raw_nodes_.empty()) {
73 GenerateGraphFromRawNodesAndEmitRows();
74 }
75 }
76
ReadProtoSnapshot(ConstBytes blob,RawMemoryNodeMap & raw_nodes,LevelOfDetail & level_of_detail)77 void MemoryTrackerSnapshotParser::ReadProtoSnapshot(
78 ConstBytes blob,
79 RawMemoryNodeMap& raw_nodes,
80 LevelOfDetail& level_of_detail) {
81 protos::pbzero::MemoryTrackerSnapshot::Decoder snapshot(blob.data, blob.size);
82 level_of_detail = LevelOfDetail::kDetailed;
83
84 switch (snapshot.level_of_detail()) {
85 case 0: // FULL
86 level_of_detail = LevelOfDetail::kDetailed;
87 break;
88 case 1: // LIGHT
89 level_of_detail = LevelOfDetail::kLight;
90 break;
91 case 2: // BACKGROUND
92 level_of_detail = LevelOfDetail::kBackground;
93 break;
94 }
95
96 for (auto process_it = snapshot.process_memory_dumps(); process_it;
97 ++process_it) {
98 protos::pbzero::MemoryTrackerSnapshot::ProcessSnapshot::Decoder
99 process_memory_dump(*process_it);
100
101 auto pid = static_cast<base::PlatformProcessId>(process_memory_dump.pid());
102
103 RawProcessMemoryNode::MemoryNodesMap nodes_map;
104 RawProcessMemoryNode::AllocatorNodeEdgesMap edges_map;
105
106 for (auto node_it = process_memory_dump.allocator_dumps(); node_it;
107 ++node_it) {
108 protos::pbzero::MemoryTrackerSnapshot::ProcessSnapshot::MemoryNode::
109 Decoder node(*node_it);
110
111 MemoryAllocatorNodeId node_id(node.id());
112 const std::string absolute_name = node.absolute_name().ToStdString();
113 int flags;
114 if (node.weak()) {
115 flags = RawMemoryGraphNode::kWeak;
116 } else {
117 flags = RawMemoryGraphNode::kDefault;
118 }
119
120 std::vector<RawMemoryGraphNode::MemoryNodeEntry> entries;
121
122 if (node.has_size_bytes()) {
123 entries.emplace_back("size", RawMemoryGraphNode::kUnitsBytes,
124 node.size_bytes());
125 }
126
127 for (auto entry_it = node.entries(); entry_it; ++entry_it) {
128 protos::pbzero::MemoryTrackerSnapshot::ProcessSnapshot::MemoryNode::
129 MemoryNodeEntry::Decoder entry(*entry_it);
130
131 std::string unit;
132
133 switch (entry.units()) {
134 case 1: // BYTES
135 unit = RawMemoryGraphNode::kUnitsBytes;
136 break;
137 case 2: // COUNT
138 unit = RawMemoryGraphNode::kUnitsObjects;
139 break;
140 }
141 if (entry.has_value_uint64()) {
142 entries.emplace_back(entry.name().ToStdString(), unit,
143 entry.value_uint64());
144 } else if (entry.has_value_string()) {
145 entries.emplace_back(entry.name().ToStdString(), unit,
146 entry.value_string().ToStdString());
147 } else {
148 context_->storage->IncrementStats(
149 stats::memory_snapshot_parser_failure);
150 }
151 }
152 std::unique_ptr<RawMemoryGraphNode> raw_graph_node(new RawMemoryGraphNode(
153 absolute_name, level_of_detail, node_id, std::move(entries)));
154 raw_graph_node->set_flags(flags);
155 nodes_map.insert(
156 std::make_pair(absolute_name, std::move(raw_graph_node)));
157 }
158
159 for (auto edge_it = process_memory_dump.memory_edges(); edge_it;
160 ++edge_it) {
161 protos::pbzero::MemoryTrackerSnapshot::ProcessSnapshot::MemoryEdge::
162 Decoder edge(*edge_it);
163
164 std::unique_ptr<MemoryGraphEdge> graph_edge(new MemoryGraphEdge(
165 MemoryAllocatorNodeId(edge.source_id()),
166 MemoryAllocatorNodeId(edge.target_id()),
167 static_cast<int>(edge.importance()), edge.overridable()));
168
169 edges_map.insert(std::make_pair(MemoryAllocatorNodeId(edge.source_id()),
170 std::move(graph_edge)));
171 }
172 std::unique_ptr<RawProcessMemoryNode> raw_node(new RawProcessMemoryNode(
173 level_of_detail, std::move(edges_map), std::move(nodes_map)));
174 raw_nodes.insert(std::make_pair(pid, std::move(raw_node)));
175 }
176 }
177
GenerateGraph(RawMemoryNodeMap & raw_nodes)178 std::unique_ptr<GlobalNodeGraph> MemoryTrackerSnapshotParser::GenerateGraph(
179 RawMemoryNodeMap& raw_nodes) {
180 auto graph = GraphProcessor::CreateMemoryGraph(raw_nodes);
181 GraphProcessor::CalculateSizesForGraph(graph.get());
182 return graph;
183 }
184
EmitRows(int64_t ts,GlobalNodeGraph & graph,LevelOfDetail level_of_detail)185 void MemoryTrackerSnapshotParser::EmitRows(int64_t ts,
186 GlobalNodeGraph& graph,
187 LevelOfDetail level_of_detail) {
188 IdNodeMap id_node_map;
189
190 // For now, we use the existing global instant event track for chrome events,
191 // since memory dumps are global.
192 TrackId track_id = context_->track_tracker->InternTrack(
193 tracks::kLegacyGlobalInstantsBlueprint, tracks::Dimensions(),
194 tracks::BlueprintName(), [this](ArgsTracker::BoundInserter& inserter) {
195 inserter.AddArg(
196 context_->storage->InternString("source"),
197 Variadic::String(context_->storage->InternString("chrome")));
198 });
199
200 tables::MemorySnapshotTable::Row snapshot_row(
201 ts, track_id, level_of_detail_ids_[static_cast<size_t>(level_of_detail)]);
202 tables::MemorySnapshotTable::Id snapshot_row_id =
203 context_->storage->mutable_memory_snapshot_table()
204 ->Insert(snapshot_row)
205 .id;
206
207 for (auto const& it_process : graph.process_node_graphs()) {
208 tables::ProcessMemorySnapshotTable::Row process_row;
209 process_row.upid = context_->process_tracker->GetOrCreateProcess(
210 static_cast<uint32_t>(it_process.first));
211 process_row.snapshot_id = snapshot_row_id;
212 tables::ProcessMemorySnapshotTable::Id proc_snapshot_row_id =
213 context_->storage->mutable_process_memory_snapshot_table()
214 ->Insert(process_row)
215 .id;
216 EmitMemorySnapshotNodeRows(*(it_process.second->root()),
217 proc_snapshot_row_id, id_node_map);
218 }
219
220 // For each snapshot nodes from shared_memory_graph will be associated
221 // with a fabricated process_memory_snapshot entry whose pid == 0.
222 // TODO([email protected]): Track the shared memory graph
223 // in a separate table.
224 tables::ProcessMemorySnapshotTable::Row fake_process_row;
225 fake_process_row.upid = context_->process_tracker->GetOrCreateProcess(0u);
226 fake_process_row.snapshot_id = snapshot_row_id;
227 tables::ProcessMemorySnapshotTable::Id fake_proc_snapshot_row_id =
228 context_->storage->mutable_process_memory_snapshot_table()
229 ->Insert(fake_process_row)
230 .id;
231 EmitMemorySnapshotNodeRows(*(graph.shared_memory_graph()->root()),
232 fake_proc_snapshot_row_id, id_node_map);
233
234 for (const auto& edge : graph.edges()) {
235 tables::MemorySnapshotEdgeTable::Row edge_row;
236 auto source_it = id_node_map.find(edge.source()->id());
237 if (source_it == id_node_map.end())
238 continue;
239 edge_row.source_node_id =
240 static_cast<tables::MemorySnapshotNodeTable::Id>(source_it->second);
241 auto target_it = id_node_map.find(edge.target()->id());
242 if (target_it == id_node_map.end())
243 continue;
244 edge_row.target_node_id =
245 static_cast<tables::MemorySnapshotNodeTable::Id>(target_it->second);
246 edge_row.importance = static_cast<uint32_t>(edge.priority());
247 context_->storage->mutable_memory_snapshot_edge_table()->Insert(edge_row);
248 }
249 }
250
EmitMemorySnapshotNodeRows(GlobalNodeGraph::Node & root_node_graph,ProcessMemorySnapshotId & proc_snapshot_row_id,IdNodeMap & id_node_map)251 void MemoryTrackerSnapshotParser::EmitMemorySnapshotNodeRows(
252 GlobalNodeGraph::Node& root_node_graph,
253 ProcessMemorySnapshotId& proc_snapshot_row_id,
254 IdNodeMap& id_node_map) {
255 EmitMemorySnapshotNodeRowsRecursively(root_node_graph, std::string(),
256 std::nullopt, proc_snapshot_row_id,
257 id_node_map);
258 }
259
EmitMemorySnapshotNodeRowsRecursively(GlobalNodeGraph::Node & node,const std::string & path,std::optional<tables::MemorySnapshotNodeTable::Id> parent_node_row_id,ProcessMemorySnapshotId & proc_snapshot_row_id,IdNodeMap & id_node_map)260 void MemoryTrackerSnapshotParser::EmitMemorySnapshotNodeRowsRecursively(
261 GlobalNodeGraph::Node& node,
262 const std::string& path,
263 std::optional<tables::MemorySnapshotNodeTable::Id> parent_node_row_id,
264 ProcessMemorySnapshotId& proc_snapshot_row_id,
265 IdNodeMap& id_node_map) {
266 std::optional<tables::MemorySnapshotNodeTable::Id> node_id;
267 // Skip emitting the root node into the tables - it is not a real node.
268 if (!path.empty()) {
269 node_id = EmitNode(node, path, parent_node_row_id, proc_snapshot_row_id,
270 id_node_map);
271 }
272
273 for (const auto& name_and_child : *node.children()) {
274 std::string child_path = path;
275 if (!child_path.empty())
276 child_path += "/";
277 child_path += name_and_child.first;
278
279 EmitMemorySnapshotNodeRowsRecursively(*(name_and_child.second), child_path,
280 /*parent_node_row_id=*/node_id,
281 proc_snapshot_row_id, id_node_map);
282 }
283 }
284
285 std::optional<tables::MemorySnapshotNodeTable::Id>
EmitNode(const GlobalNodeGraph::Node & node,const std::string & path,std::optional<tables::MemorySnapshotNodeTable::Id> parent_node_row_id,ProcessMemorySnapshotId & proc_snapshot_row_id,IdNodeMap & id_node_map)286 MemoryTrackerSnapshotParser::EmitNode(
287 const GlobalNodeGraph::Node& node,
288 const std::string& path,
289 std::optional<tables::MemorySnapshotNodeTable::Id> parent_node_row_id,
290 ProcessMemorySnapshotId& proc_snapshot_row_id,
291 IdNodeMap& id_node_map) {
292 tables::MemorySnapshotNodeTable::Row node_row;
293 node_row.process_snapshot_id = proc_snapshot_row_id;
294 node_row.parent_node_id = parent_node_row_id;
295 node_row.path = context_->storage->InternString(base::StringView(path));
296
297 tables::MemorySnapshotNodeTable::Id node_row_id =
298 context_->storage->mutable_memory_snapshot_node_table()
299 ->Insert(node_row)
300 .id;
301
302 auto* node_table = context_->storage->mutable_memory_snapshot_node_table();
303 auto rr = *node_table->FindById(node_row_id);
304 ArgsTracker::BoundInserter args =
305 context_->args_tracker->AddArgsTo(node_row_id);
306
307 for (const auto& entry : node.const_entries()) {
308 switch (entry.second.type) {
309 case GlobalNodeGraph::Node::Entry::Type::kUInt64: {
310 int64_t value_int = static_cast<int64_t>(entry.second.value_uint64);
311
312 if (entry.first == "size") {
313 rr.set_size(value_int);
314 } else if (entry.first == "effective_size") {
315 rr.set_effective_size(value_int);
316 } else {
317 args.AddArg(context_->storage->InternString(
318 base::StringView(entry.first + ".value")),
319 Variadic::Integer(value_int));
320 if (entry.second.units < unit_ids_.size()) {
321 args.AddArg(context_->storage->InternString(
322 base::StringView(entry.first + ".unit")),
323 Variadic::String(unit_ids_[entry.second.units]));
324 }
325 }
326 break;
327 }
328 case GlobalNodeGraph::Node::Entry::Type::kString: {
329 args.AddArg(context_->storage->InternString(
330 base::StringView(entry.first + ".value")),
331 Variadic::String(context_->storage->InternString(
332 base::StringView(entry.second.value_string))));
333 break;
334 }
335 }
336 }
337 id_node_map.emplace(std::make_pair(node.id(), node_row_id));
338 return node_row_id;
339 }
340
GenerateGraphFromRawNodesAndEmitRows()341 void MemoryTrackerSnapshotParser::GenerateGraphFromRawNodesAndEmitRows() {
342 std::unique_ptr<GlobalNodeGraph> graph = GenerateGraph(aggregate_raw_nodes_);
343 EmitRows(last_snapshot_timestamp_, *graph, last_snapshot_level_of_detail_);
344 aggregate_raw_nodes_.clear();
345 }
346
347 } // namespace perfetto::trace_processor
348