1 /*
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "src/trace_processor/importers/proto/statsd_module.h"
17
18 #include "perfetto/ext/base/string_utils.h"
19 #include "perfetto/protozero/scattered_heap_buffer.h"
20 #include "protos/perfetto/trace/statsd/statsd_atom.pbzero.h"
21 #include "protos/perfetto/trace/trace_packet.pbzero.h"
22 #include "src/trace_processor/importers/common/async_track_set_tracker.h"
23 #include "src/trace_processor/importers/common/machine_tracker.h"
24 #include "src/trace_processor/importers/common/slice_tracker.h"
25 #include "src/trace_processor/importers/common/track_tracker.h"
26 #include "src/trace_processor/importers/proto/args_parser.h"
27 #include "src/trace_processor/importers/proto/packet_sequence_state_generation.h"
28 #include "src/trace_processor/sorter/trace_sorter.h"
29 #include "src/trace_processor/storage/stats.h"
30 #include "src/trace_processor/storage/trace_storage.h"
31 #include "src/trace_processor/util/descriptors.h"
32
33 #include "src/trace_processor/importers/proto/atoms.descriptor.h"
34
35 namespace perfetto {
36 namespace trace_processor {
37 namespace {
38
39 constexpr const char* kAtomProtoName = ".android.os.statsd.Atom";
40
41 // If we don't know about the atom format put whatever details we
42 // can. This has the following restrictions:
43 // - We can't tell the difference between double, fixed64, sfixed64
44 // so those all show up as double
45 // - We can't tell the difference between float, fixed32, sfixed32
46 // so those all show up as float
47 // - We can't tell the difference between int32, int64 and sint32
48 // and sint64. We assume int32/int64.
49 // - We only show the length of strings, nested messages, packed ints
50 // and any other length delimited fields.
ParseGenericEvent(const protozero::ConstBytes & cb,util::ProtoToArgsParser::Delegate & delegate)51 base::Status ParseGenericEvent(const protozero::ConstBytes& cb,
52 util::ProtoToArgsParser::Delegate& delegate) {
53 protozero::ProtoDecoder decoder(cb);
54 for (auto f = decoder.ReadField(); f.valid(); f = decoder.ReadField()) {
55 switch (f.type()) {
56 case protozero::proto_utils::ProtoWireType::kLengthDelimited: {
57 base::StackString<64> name("field_%u", f.id());
58 std::string name_str = name.ToStdString();
59 util::ProtoToArgsParser::Key key{name_str, name_str};
60 delegate.AddBytes(key, f.as_bytes());
61 break;
62 }
63 case protozero::proto_utils::ProtoWireType::kVarInt: {
64 base::StackString<64> name("field_%u", f.id());
65 std::string name_str = name.ToStdString();
66 util::ProtoToArgsParser::Key key{name_str, name_str};
67 delegate.AddInteger(key, f.as_int64());
68 break;
69 }
70 case protozero::proto_utils::ProtoWireType::kFixed32: {
71 base::StackString<64> name("field_%u_assuming_float", f.id());
72 std::string name_str = name.ToStdString();
73 util::ProtoToArgsParser::Key key{name_str, name_str};
74 delegate.AddDouble(key, static_cast<double>(f.as_float()));
75 break;
76 }
77 case protozero::proto_utils::ProtoWireType::kFixed64: {
78 base::StackString<64> name("field_%u_assuming_double", f.id());
79 std::string name_str = name.ToStdString();
80 util::ProtoToArgsParser::Key key{name_str, name_str};
81 delegate.AddDouble(key, f.as_double());
82 break;
83 }
84 }
85 }
86 return base::OkStatus();
87 }
88
89 } // namespace
90
91 using perfetto::protos::pbzero::StatsdAtom;
92 using perfetto::protos::pbzero::TracePacket;
93
PoolAndDescriptor(const uint8_t * data,size_t size,const char * name)94 PoolAndDescriptor::PoolAndDescriptor(const uint8_t* data,
95 size_t size,
96 const char* name) {
97 pool_.AddFromFileDescriptorSet(data, size);
98 std::optional<uint32_t> opt_idx = pool_.FindDescriptorIdx(name);
99 if (opt_idx.has_value()) {
100 descriptor_ = &pool_.descriptors()[opt_idx.value()];
101 }
102 }
103
104 PoolAndDescriptor::~PoolAndDescriptor() = default;
105
StatsdModule(TraceProcessorContext * context)106 StatsdModule::StatsdModule(TraceProcessorContext* context)
107 : context_(context),
108 pool_(kAtomsDescriptor.data(), kAtomsDescriptor.size(), kAtomProtoName),
109 args_parser_(*(pool_.pool())) {
110 RegisterForField(TracePacket::kStatsdAtomFieldNumber, context);
111 }
112
113 StatsdModule::~StatsdModule() = default;
114
TokenizePacket(const TracePacket::Decoder & decoder,TraceBlobView *,int64_t packet_timestamp,RefPtr<PacketSequenceStateGeneration> state,uint32_t field_id)115 ModuleResult StatsdModule::TokenizePacket(
116 const TracePacket::Decoder& decoder,
117 TraceBlobView* /*packet*/,
118 int64_t packet_timestamp,
119 RefPtr<PacketSequenceStateGeneration> state,
120 uint32_t field_id) {
121 if (field_id != TracePacket::kStatsdAtomFieldNumber) {
122 return ModuleResult::Ignored();
123 }
124 const auto& atoms_wrapper = StatsdAtom::Decoder(decoder.statsd_atom());
125 auto it_timestamps = atoms_wrapper.timestamp_nanos();
126 for (auto it = atoms_wrapper.atom(); it; ++it) {
127 int64_t atom_timestamp;
128
129 if (it_timestamps) {
130 atom_timestamp = *it_timestamps++;
131 } else {
132 context_->storage->IncrementStats(stats::atom_timestamp_missing);
133 atom_timestamp = packet_timestamp;
134 }
135
136 protozero::HeapBuffered<TracePacket> forged;
137
138 forged->set_timestamp(static_cast<uint64_t>(atom_timestamp));
139
140 auto* statsd = forged->set_statsd_atom();
141 statsd->AppendBytes(StatsdAtom::kAtomFieldNumber, (*it).data, (*it).size);
142
143 std::vector<uint8_t> vec = forged.SerializeAsArray();
144 TraceBlob blob = TraceBlob::CopyFrom(vec.data(), vec.size());
145
146 context_->sorter->PushTracePacket(atom_timestamp, state,
147 TraceBlobView(std::move(blob)),
148 context_->machine_id());
149 }
150
151 return ModuleResult::Handled();
152 }
153
ParseTracePacketData(const TracePacket::Decoder & decoder,int64_t ts,const TracePacketData &,uint32_t field_id)154 void StatsdModule::ParseTracePacketData(const TracePacket::Decoder& decoder,
155 int64_t ts,
156 const TracePacketData&,
157 uint32_t field_id) {
158 if (field_id != TracePacket::kStatsdAtomFieldNumber) {
159 return;
160 }
161 const auto& atoms_wrapper = StatsdAtom::Decoder(decoder.statsd_atom());
162 auto it = atoms_wrapper.atom();
163 // There should be exactly one atom per trace packet at this point.
164 // If not something has gone wrong in tokenization above.
165 PERFETTO_CHECK(it);
166 ParseAtom(ts, *it++);
167 PERFETTO_CHECK(!it);
168 }
169
ParseAtom(int64_t ts,protozero::ConstBytes nested_bytes)170 void StatsdModule::ParseAtom(int64_t ts, protozero::ConstBytes nested_bytes) {
171 // nested_bytes is an Atom proto. We (deliberately) don't generate
172 // decoding code for every kind of atom (or the parent Atom proto)
173 // and instead use the descriptor to parse the args/name.
174
175 // Atom is a giant oneof of all the possible 'kinds' of atom so here
176 // we use the protozero decoder implementation to grab the first
177 // field id which we we use to look up the field name:
178 protozero::ProtoDecoder nested_decoder(nested_bytes);
179 protozero::Field field = nested_decoder.ReadField();
180 uint32_t nested_field_id = 0;
181 if (field.valid()) {
182 nested_field_id = field.id();
183 }
184 StringId atom_name = GetAtomName(nested_field_id);
185
186 AsyncTrackSetTracker::TrackSetId track_set = InternAsyncTrackSetId();
187 TrackId track = context_->async_track_set_tracker->Scoped(track_set, ts, 0);
188 std::optional<SliceId> opt_slice =
189 context_->slice_tracker->Scoped(ts, track, kNullStringId, atom_name, 0);
190 if (!opt_slice) {
191 return;
192 }
193 SliceId slice = opt_slice.value();
194 auto inserter = context_->args_tracker->AddArgsTo(slice);
195 ArgsParser delegate(ts, inserter, *context_->storage.get());
196
197 const auto& fields = pool_.descriptor()->fields();
198 const auto& field_it = fields.find(nested_field_id);
199 base::Status status;
200
201 if (field_it == fields.end()) {
202 /// Field ids 100000 and over are OEM atoms - we can't have the
203 // descriptor for them so don't report errors. See:
204 // https://cs.android.com/android/platform/superproject/main/+/main:frameworks/proto_logging/stats/atoms.proto;l=1290;drc=a34b11bfebe897259a0340a59f1793ae2dffd762
205 if (nested_field_id < 100000) {
206 context_->storage->IncrementStats(stats::atom_unknown);
207 }
208
209 status = ParseGenericEvent(field.as_bytes(), delegate);
210 } else {
211 status = args_parser_.ParseMessage(
212 nested_bytes, kAtomProtoName, nullptr /* parse all fields */, delegate);
213 }
214
215 if (!status.ok()) {
216 context_->storage->IncrementStats(stats::atom_unknown);
217 }
218 }
219
GetAtomName(uint32_t atom_field_id)220 StringId StatsdModule::GetAtomName(uint32_t atom_field_id) {
221 StringId* cached_name = atom_names_.Find(atom_field_id);
222 if (cached_name == nullptr) {
223 if (pool_.descriptor() == nullptr) {
224 context_->storage->IncrementStats(stats::atom_unknown);
225 return context_->storage->InternString("Could not load atom descriptor");
226 }
227
228 StringId name_id;
229 const auto& fields = pool_.descriptor()->fields();
230 const auto& field_it = fields.find(atom_field_id);
231 if (field_it == fields.end()) {
232 base::StackString<255> name("atom_%u", atom_field_id);
233 name_id = context_->storage->InternString(name.string_view());
234 } else {
235 const FieldDescriptor& field = field_it->second;
236 name_id = context_->storage->InternString(base::StringView(field.name()));
237 }
238 atom_names_[atom_field_id] = name_id;
239 return name_id;
240 }
241 return *cached_name;
242 }
243
InternAsyncTrackSetId()244 AsyncTrackSetTracker::TrackSetId StatsdModule::InternAsyncTrackSetId() {
245 if (!track_set_id_) {
246 StringId name = context_->storage->InternString("Statsd Atoms");
247 track_set_id_ =
248 context_->async_track_set_tracker->InternGlobalTrackSet(name);
249 }
250 return track_set_id_.value();
251 }
252
253 } // namespace trace_processor
254 } // namespace perfetto
255