1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/util/proto_to_args_parser.h"
18
19 #include <unordered_set>
20
21 #include "perfetto/base/status.h"
22 #include "perfetto/ext/base/string_utils.h"
23 #include "perfetto/protozero/field.h"
24 #include "perfetto/protozero/proto_decoder.h"
25 #include "perfetto/protozero/proto_utils.h"
26 #include "protos/perfetto/common/descriptor.pbzero.h"
27 #include "src/trace_processor/util/descriptors.h"
28 #include "src/trace_processor/util/status_macros.h"
29
30 namespace perfetto {
31 namespace trace_processor {
32 namespace util {
33
34 namespace {
35
36 template <protozero::proto_utils::ProtoWireType wire_type, typename cpp_type>
37 using PRFI = protozero::PackedRepeatedFieldIterator<wire_type, cpp_type>;
38
AppendProtoType(std::string & target,const std::string & value)39 void AppendProtoType(std::string& target, const std::string& value) {
40 if (!target.empty())
41 target += '.';
42 target += value;
43 }
44
IsFieldAllowed(const FieldDescriptor & field,const std::vector<uint32_t> * allowed_fields)45 bool IsFieldAllowed(const FieldDescriptor& field,
46 const std::vector<uint32_t>* allowed_fields) {
47 // If allowlist is not provided, reflect all fields. Otherwise, check if the
48 // current field either an extension or is in allowlist.
49 return field.is_extension() || !allowed_fields ||
50 std::find(allowed_fields->begin(), allowed_fields->end(),
51 field.number()) != allowed_fields->end();
52 }
53
54 } // namespace
55
56 ProtoToArgsParser::Key::Key() = default;
Key(const std::string & k)57 ProtoToArgsParser::Key::Key(const std::string& k) : flat_key(k), key(k) {}
Key(const std::string & fk,const std::string & k)58 ProtoToArgsParser::Key::Key(const std::string& fk, const std::string& k)
59 : flat_key(fk), key(k) {}
60 ProtoToArgsParser::Key::~Key() = default;
61
ScopedNestedKeyContext(Key & key)62 ProtoToArgsParser::ScopedNestedKeyContext::ScopedNestedKeyContext(Key& key)
63 : key_(key),
64 old_flat_key_length_(key.flat_key.length()),
65 old_key_length_(key.key.length()) {}
66
ScopedNestedKeyContext(ProtoToArgsParser::ScopedNestedKeyContext && other)67 ProtoToArgsParser::ScopedNestedKeyContext::ScopedNestedKeyContext(
68 ProtoToArgsParser::ScopedNestedKeyContext&& other) noexcept
69 : key_(other.key_),
70 old_flat_key_length_(other.old_flat_key_length_),
71 old_key_length_(other.old_key_length_) {
72 other.old_flat_key_length_ = std::nullopt;
73 other.old_key_length_ = std::nullopt;
74 }
75
~ScopedNestedKeyContext()76 ProtoToArgsParser::ScopedNestedKeyContext::~ScopedNestedKeyContext() {
77 RemoveFieldSuffix();
78 }
79
RemoveFieldSuffix()80 void ProtoToArgsParser::ScopedNestedKeyContext::RemoveFieldSuffix() {
81 if (old_flat_key_length_)
82 key_.flat_key.resize(old_flat_key_length_.value());
83 if (old_key_length_)
84 key_.key.resize(old_key_length_.value());
85 old_flat_key_length_ = std::nullopt;
86 old_key_length_ = std::nullopt;
87 }
88
89 ProtoToArgsParser::Delegate::~Delegate() = default;
90
ProtoToArgsParser(const DescriptorPool & pool)91 ProtoToArgsParser::ProtoToArgsParser(const DescriptorPool& pool) : pool_(pool) {
92 constexpr int kDefaultSize = 64;
93 key_prefix_.key.reserve(kDefaultSize);
94 key_prefix_.flat_key.reserve(kDefaultSize);
95 }
96
ParseMessage(const protozero::ConstBytes & cb,const std::string & type,const std::vector<uint32_t> * allowed_fields,Delegate & delegate,int * unknown_extensions,bool add_defaults)97 base::Status ProtoToArgsParser::ParseMessage(
98 const protozero::ConstBytes& cb,
99 const std::string& type,
100 const std::vector<uint32_t>* allowed_fields,
101 Delegate& delegate,
102 int* unknown_extensions,
103 bool add_defaults) {
104 ScopedNestedKeyContext key_context(key_prefix_);
105 return ParseMessageInternal(key_context, cb, type, allowed_fields, delegate,
106 unknown_extensions, add_defaults);
107 }
108
ParseMessageInternal(ScopedNestedKeyContext & key_context,const protozero::ConstBytes & cb,const std::string & type,const std::vector<uint32_t> * allowed_fields,Delegate & delegate,int * unknown_extensions,bool add_defaults)109 base::Status ProtoToArgsParser::ParseMessageInternal(
110 ScopedNestedKeyContext& key_context,
111 const protozero::ConstBytes& cb,
112 const std::string& type,
113 const std::vector<uint32_t>* allowed_fields,
114 Delegate& delegate,
115 int* unknown_extensions,
116 bool add_defaults) {
117 if (auto override_result =
118 MaybeApplyOverrideForType(type, key_context, cb, delegate)) {
119 return override_result.value();
120 }
121
122 auto idx = pool_.FindDescriptorIdx(type);
123 if (!idx) {
124 return base::Status("Failed to find proto descriptor");
125 }
126
127 auto& descriptor = pool_.descriptors()[*idx];
128
129 std::unordered_map<size_t, int> repeated_field_index;
130 bool empty_message = true;
131 protozero::ProtoDecoder decoder(cb);
132 std::unordered_set<std::string_view> existing_fields;
133 for (protozero::Field f = decoder.ReadField(); f.valid();
134 f = decoder.ReadField()) {
135 empty_message = false;
136 auto field = descriptor.FindFieldByTag(f.id());
137 if (!field) {
138 if (unknown_extensions != nullptr) {
139 (*unknown_extensions)++;
140 }
141 // Unknown field, possibly an unknown extension.
142 continue;
143 }
144
145 if (add_defaults) {
146 existing_fields.insert(field->name());
147 }
148
149 if (!IsFieldAllowed(*field, allowed_fields)) {
150 // Field is neither an extension, nor is allowed to be
151 // reflected.
152 continue;
153 }
154
155 // Packed fields need to be handled specially because
156 if (field->is_packed()) {
157 RETURN_IF_ERROR(ParsePackedField(*field, repeated_field_index, f,
158 delegate, unknown_extensions,
159 add_defaults));
160 continue;
161 }
162
163 RETURN_IF_ERROR(ParseField(*field, repeated_field_index[f.id()], f,
164 delegate, unknown_extensions, add_defaults));
165 if (field->is_repeated()) {
166 repeated_field_index[f.id()]++;
167 }
168 }
169
170 if (empty_message) {
171 delegate.AddNull(key_prefix_);
172 } else if (add_defaults) {
173 for (const auto& [id, field] : descriptor.fields()) {
174 if (!IsFieldAllowed(field, allowed_fields)) {
175 continue;
176 }
177 const std::string& field_name = field.name();
178 bool field_exists =
179 existing_fields.find(field_name) != existing_fields.cend();
180 if (field_exists) {
181 continue;
182 }
183 ScopedNestedKeyContext key_context_default(key_prefix_);
184 AppendProtoType(key_prefix_.flat_key, field_name);
185 AppendProtoType(key_prefix_.key, field_name);
186 RETURN_IF_ERROR(AddDefault(field, delegate));
187 }
188 }
189
190 return base::OkStatus();
191 }
192
ParseField(const FieldDescriptor & field_descriptor,int repeated_field_number,protozero::Field field,Delegate & delegate,int * unknown_extensions,bool add_defaults)193 base::Status ProtoToArgsParser::ParseField(
194 const FieldDescriptor& field_descriptor,
195 int repeated_field_number,
196 protozero::Field field,
197 Delegate& delegate,
198 int* unknown_extensions,
199 bool add_defaults) {
200 std::string prefix_part = field_descriptor.name();
201 if (field_descriptor.is_repeated()) {
202 std::string number = std::to_string(repeated_field_number);
203 prefix_part.reserve(prefix_part.length() + number.length() + 2);
204 prefix_part.append("[");
205 prefix_part.append(number);
206 prefix_part.append("]");
207 }
208
209 // In the args table we build up message1.message2.field1 as the column
210 // name. This will append the ".field1" suffix to |key_prefix| and then
211 // remove it when it goes out of scope.
212 ScopedNestedKeyContext key_context(key_prefix_);
213 AppendProtoType(key_prefix_.flat_key, field_descriptor.name());
214 AppendProtoType(key_prefix_.key, prefix_part);
215
216 // If we have an override parser then use that instead and move onto the
217 // next loop.
218 if (std::optional<base::Status> status =
219 MaybeApplyOverrideForField(field, delegate)) {
220 return *status;
221 }
222
223 // If this is not a message we can just immediately add the column name and
224 // get the value out of |field|. However if it is a message we need to
225 // recurse into it.
226 if (field_descriptor.type() ==
227 protos::pbzero::FieldDescriptorProto::TYPE_MESSAGE) {
228 return ParseMessageInternal(key_context, field.as_bytes(),
229 field_descriptor.resolved_type_name(), nullptr,
230 delegate, unknown_extensions, add_defaults);
231 }
232 return ParseSimpleField(field_descriptor, field, delegate);
233 }
234
ParsePackedField(const FieldDescriptor & field_descriptor,std::unordered_map<size_t,int> & repeated_field_index,protozero::Field field,Delegate & delegate,int * unknown_extensions,bool add_defaults)235 base::Status ProtoToArgsParser::ParsePackedField(
236 const FieldDescriptor& field_descriptor,
237 std::unordered_map<size_t, int>& repeated_field_index,
238 protozero::Field field,
239 Delegate& delegate,
240 int* unknown_extensions,
241 bool add_defaults) {
242 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
243 using PWT = protozero::proto_utils::ProtoWireType;
244
245 if (!field_descriptor.is_repeated()) {
246 return base::ErrStatus("Packed field %s must be repeated",
247 field_descriptor.name().c_str());
248 }
249 if (field.type() != PWT::kLengthDelimited) {
250 return base::ErrStatus(
251 "Packed field %s must have a length delimited wire type",
252 field_descriptor.name().c_str());
253 }
254
255 auto parse = [&](uint64_t new_value, PWT wire_type) {
256 protozero::Field f;
257 f.initialize(field.id(), static_cast<uint8_t>(wire_type), new_value, 0);
258 return ParseField(field_descriptor, repeated_field_index[field.id()]++, f,
259 delegate, unknown_extensions, add_defaults);
260 };
261
262 const uint8_t* data = field.as_bytes().data;
263 size_t size = field.as_bytes().size;
264 bool perr = false;
265 switch (field_descriptor.type()) {
266 case FieldDescriptorProto::TYPE_INT32:
267 case FieldDescriptorProto::TYPE_INT64:
268 case FieldDescriptorProto::TYPE_UINT32:
269 case FieldDescriptorProto::TYPE_UINT64:
270 case FieldDescriptorProto::TYPE_ENUM:
271 for (PRFI<PWT::kVarInt, uint64_t> it(data, size, &perr); it; ++it) {
272 parse(*it, PWT::kVarInt);
273 }
274 break;
275 case FieldDescriptorProto::TYPE_FIXED32:
276 case FieldDescriptorProto::TYPE_SFIXED32:
277 case FieldDescriptorProto::TYPE_FLOAT:
278 for (PRFI<PWT::kFixed32, uint32_t> it(data, size, &perr); it; ++it) {
279 parse(*it, PWT::kFixed32);
280 }
281 break;
282 case FieldDescriptorProto::TYPE_FIXED64:
283 case FieldDescriptorProto::TYPE_SFIXED64:
284 case FieldDescriptorProto::TYPE_DOUBLE:
285 for (PRFI<PWT::kFixed64, uint64_t> it(data, size, &perr); it; ++it) {
286 parse(*it, PWT::kFixed64);
287 }
288 break;
289 default:
290 return base::ErrStatus("Unsupported packed repeated field");
291 }
292 return base::OkStatus();
293 }
294
AddParsingOverrideForField(const std::string & field,ParsingOverrideForField func)295 void ProtoToArgsParser::AddParsingOverrideForField(
296 const std::string& field,
297 ParsingOverrideForField func) {
298 field_overrides_[field] = std::move(func);
299 }
300
AddParsingOverrideForType(const std::string & type,ParsingOverrideForType func)301 void ProtoToArgsParser::AddParsingOverrideForType(const std::string& type,
302 ParsingOverrideForType func) {
303 type_overrides_[type] = std::move(func);
304 }
305
MaybeApplyOverrideForField(const protozero::Field & field,Delegate & delegate)306 std::optional<base::Status> ProtoToArgsParser::MaybeApplyOverrideForField(
307 const protozero::Field& field,
308 Delegate& delegate) {
309 auto it = field_overrides_.find(key_prefix_.flat_key);
310 if (it == field_overrides_.end())
311 return std::nullopt;
312 return it->second(field, delegate);
313 }
314
MaybeApplyOverrideForType(const std::string & message_type,ScopedNestedKeyContext & key,const protozero::ConstBytes & data,Delegate & delegate)315 std::optional<base::Status> ProtoToArgsParser::MaybeApplyOverrideForType(
316 const std::string& message_type,
317 ScopedNestedKeyContext& key,
318 const protozero::ConstBytes& data,
319 Delegate& delegate) {
320 auto it = type_overrides_.find(message_type);
321 if (it == type_overrides_.end())
322 return std::nullopt;
323 return it->second(key, data, delegate);
324 }
325
ParseSimpleField(const FieldDescriptor & descriptor,const protozero::Field & field,Delegate & delegate)326 base::Status ProtoToArgsParser::ParseSimpleField(
327 const FieldDescriptor& descriptor,
328 const protozero::Field& field,
329 Delegate& delegate) {
330 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
331 switch (descriptor.type()) {
332 case FieldDescriptorProto::TYPE_INT32:
333 case FieldDescriptorProto::TYPE_SFIXED32:
334 delegate.AddInteger(key_prefix_, field.as_int32());
335 return base::OkStatus();
336 case FieldDescriptorProto::TYPE_SINT32:
337 delegate.AddInteger(key_prefix_, field.as_sint32());
338 return base::OkStatus();
339 case FieldDescriptorProto::TYPE_INT64:
340 case FieldDescriptorProto::TYPE_SFIXED64:
341 delegate.AddInteger(key_prefix_, field.as_int64());
342 return base::OkStatus();
343 case FieldDescriptorProto::TYPE_SINT64:
344 delegate.AddInteger(key_prefix_, field.as_sint64());
345 return base::OkStatus();
346 case FieldDescriptorProto::TYPE_UINT32:
347 case FieldDescriptorProto::TYPE_FIXED32:
348 delegate.AddUnsignedInteger(key_prefix_, field.as_uint32());
349 return base::OkStatus();
350 case FieldDescriptorProto::TYPE_UINT64:
351 case FieldDescriptorProto::TYPE_FIXED64:
352 delegate.AddUnsignedInteger(key_prefix_, field.as_uint64());
353 return base::OkStatus();
354 case FieldDescriptorProto::TYPE_BOOL:
355 delegate.AddBoolean(key_prefix_, field.as_bool());
356 return base::OkStatus();
357 case FieldDescriptorProto::TYPE_DOUBLE:
358 delegate.AddDouble(key_prefix_, field.as_double());
359 return base::OkStatus();
360 case FieldDescriptorProto::TYPE_FLOAT:
361 delegate.AddDouble(key_prefix_, static_cast<double>(field.as_float()));
362 return base::OkStatus();
363 case FieldDescriptorProto::TYPE_BYTES:
364 delegate.AddBytes(key_prefix_, field.as_bytes());
365 return base::OkStatus();
366 case FieldDescriptorProto::TYPE_STRING:
367 delegate.AddString(key_prefix_, field.as_string());
368 return base::OkStatus();
369 case FieldDescriptorProto::TYPE_ENUM:
370 return AddEnum(descriptor, field.as_int32(), delegate);
371 default:
372 return base::ErrStatus(
373 "Tried to write value of type field %s (in proto type "
374 "%s) which has type enum %u",
375 descriptor.name().c_str(), descriptor.resolved_type_name().c_str(),
376 descriptor.type());
377 }
378 }
379
EnterArray(size_t index)380 ProtoToArgsParser::ScopedNestedKeyContext ProtoToArgsParser::EnterArray(
381 size_t index) {
382 ScopedNestedKeyContext context(key_prefix_);
383 key_prefix_.key += "[" + std::to_string(index) + "]";
384 return context;
385 }
386
EnterDictionary(const std::string & name)387 ProtoToArgsParser::ScopedNestedKeyContext ProtoToArgsParser::EnterDictionary(
388 const std::string& name) {
389 ScopedNestedKeyContext context(key_prefix_);
390 AppendProtoType(key_prefix_.key, name);
391 AppendProtoType(key_prefix_.flat_key, name);
392 return context;
393 }
394
AddDefault(const FieldDescriptor & descriptor,Delegate & delegate)395 base::Status ProtoToArgsParser::AddDefault(const FieldDescriptor& descriptor,
396 Delegate& delegate) {
397 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
398 if (descriptor.is_repeated()) {
399 delegate.AddNull(key_prefix_);
400 return base::OkStatus();
401 }
402 const auto& default_value = descriptor.default_value();
403 const auto& default_value_if_number =
404 default_value ? default_value.value() : "0";
405 switch (descriptor.type()) {
406 case FieldDescriptorProto::TYPE_INT32:
407 case FieldDescriptorProto::TYPE_SFIXED32:
408 delegate.AddInteger(key_prefix_,
409 base::StringToInt32(default_value_if_number).value());
410 return base::OkStatus();
411 case FieldDescriptorProto::TYPE_SINT32:
412 delegate.AddInteger(
413 key_prefix_,
414 protozero::proto_utils::ZigZagDecode(
415 base::StringToInt64(default_value_if_number).value()));
416 return base::OkStatus();
417 case FieldDescriptorProto::TYPE_INT64:
418 case FieldDescriptorProto::TYPE_SFIXED64:
419 delegate.AddInteger(key_prefix_,
420 base::StringToInt64(default_value_if_number).value());
421 return base::OkStatus();
422 case FieldDescriptorProto::TYPE_SINT64:
423 delegate.AddInteger(
424 key_prefix_,
425 protozero::proto_utils::ZigZagDecode(
426 base::StringToInt64(default_value_if_number).value()));
427 return base::OkStatus();
428 case FieldDescriptorProto::TYPE_UINT32:
429 case FieldDescriptorProto::TYPE_FIXED32:
430 delegate.AddUnsignedInteger(
431 key_prefix_, base::StringToUInt32(default_value_if_number).value());
432 return base::OkStatus();
433 case FieldDescriptorProto::TYPE_UINT64:
434 case FieldDescriptorProto::TYPE_FIXED64:
435 delegate.AddUnsignedInteger(
436 key_prefix_, base::StringToUInt64(default_value_if_number).value());
437 return base::OkStatus();
438 case FieldDescriptorProto::TYPE_BOOL:
439 delegate.AddBoolean(key_prefix_, default_value == "true");
440 return base::OkStatus();
441 case FieldDescriptorProto::TYPE_DOUBLE:
442 case FieldDescriptorProto::TYPE_FLOAT:
443 delegate.AddDouble(key_prefix_,
444 base::StringToDouble(default_value_if_number).value());
445 return base::OkStatus();
446 case FieldDescriptorProto::TYPE_BYTES:
447 delegate.AddBytes(key_prefix_, protozero::ConstBytes{});
448 return base::OkStatus();
449 case FieldDescriptorProto::TYPE_STRING:
450 if (default_value) {
451 delegate.AddString(key_prefix_, default_value.value());
452 } else {
453 delegate.AddNull(key_prefix_);
454 }
455 return base::OkStatus();
456 case FieldDescriptorProto::TYPE_MESSAGE:
457 delegate.AddNull(key_prefix_);
458 return base::OkStatus();
459 case FieldDescriptorProto::TYPE_ENUM:
460 return AddEnum(descriptor,
461 base::StringToInt32(default_value_if_number).value(),
462 delegate);
463 default:
464 return base::ErrStatus(
465 "Tried to write default value of type field %s (in proto type "
466 "%s) which has type enum %u",
467 descriptor.name().c_str(), descriptor.resolved_type_name().c_str(),
468 descriptor.type());
469 }
470 }
471
AddEnum(const FieldDescriptor & descriptor,int32_t value,Delegate & delegate)472 base::Status ProtoToArgsParser::AddEnum(const FieldDescriptor& descriptor,
473 int32_t value,
474 Delegate& delegate) {
475 auto opt_enum_descriptor_idx =
476 pool_.FindDescriptorIdx(descriptor.resolved_type_name());
477 if (!opt_enum_descriptor_idx) {
478 delegate.AddInteger(key_prefix_, value);
479 return base::OkStatus();
480 }
481 auto opt_enum_string =
482 pool_.descriptors()[*opt_enum_descriptor_idx].FindEnumString(value);
483 if (!opt_enum_string) {
484 // Fall back to the integer representation of the field.
485 // We add the string representation of the int value here in order that
486 // EXTRACT_ARG() should return consistent types under error conditions and
487 // that CREATE PERFETTO TABLE AS EXTRACT_ARG(...) should be generally safe
488 // to use.
489 delegate.AddString(key_prefix_, std::to_string(value));
490 return base::OkStatus();
491 }
492 delegate.AddString(
493 key_prefix_,
494 protozero::ConstChars{opt_enum_string->data(), opt_enum_string->size()});
495 return base::OkStatus();
496 }
497 } // namespace util
498 } // namespace trace_processor
499 } // namespace perfetto
500