1 // Copyright 2023 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "components/metrics/structured/structured_metrics_recorder.h"
6
7 #include <sstream>
8 #include <utility>
9
10 #include "base/feature_list.h"
11 #include "base/logging.h"
12 #include "base/metrics/histogram_macros.h"
13 #include "base/metrics/metrics_hashes.h"
14 #include "base/strings/string_number_conversions.h"
15 #include "base/strings/string_split.h"
16 #include "base/task/current_thread.h"
17 #include "components/metrics/metrics_features.h"
18 #include "components/metrics/structured/enums.h"
19 #include "components/metrics/structured/histogram_util.h"
20 #include "components/metrics/structured/project_validator.h"
21 #include "components/metrics/structured/proto/event_storage.pb.h"
22 #include "components/metrics/structured/structured_metrics_features.h"
23 #include "components/metrics/structured/structured_metrics_validator.h"
24 #include "third_party/metrics_proto/chrome_user_metrics_extension.pb.h"
25 #include "third_party/metrics_proto/structured_data.pb.h"
26
27 namespace metrics::structured {
28
StructuredMetricsRecorder(std::unique_ptr<KeyDataProvider> key_data_provider,std::unique_ptr<EventStorage<StructuredEventProto>> event_storage)29 StructuredMetricsRecorder::StructuredMetricsRecorder(
30 std::unique_ptr<KeyDataProvider> key_data_provider,
31 std::unique_ptr<EventStorage<StructuredEventProto>> event_storage)
32 : key_data_provider_(std::move(key_data_provider)),
33 event_storage_(std::move(event_storage)) {
34 CHECK(key_data_provider_);
35 CHECK(event_storage_);
36 Recorder::GetInstance()->AddObserver(this);
37 key_data_provider_->AddObserver(this);
38 }
39
~StructuredMetricsRecorder()40 StructuredMetricsRecorder::~StructuredMetricsRecorder() {
41 Recorder::GetInstance()->RemoveObserver(this);
42 key_data_provider_->RemoveObserver(this);
43 }
44
EnableRecording()45 void StructuredMetricsRecorder::EnableRecording() {
46 DCHECK(base::CurrentUIThread::IsSet());
47 // Enable recording only if structured metrics' feature flag is enabled.
48 recording_enabled_ =
49 base::FeatureList::IsEnabled(features::kStructuredMetrics);
50 if (recording_enabled_) {
51 CacheDisallowedProjectsSet();
52 }
53 }
54
DisableRecording()55 void StructuredMetricsRecorder::DisableRecording() {
56 DCHECK(base::CurrentUIThread::IsSet());
57 recording_enabled_ = false;
58 disallowed_projects_.clear();
59 }
ProvideUmaEventMetrics(ChromeUserMetricsExtension & uma_proto)60 void StructuredMetricsRecorder::ProvideUmaEventMetrics(
61 ChromeUserMetricsExtension& uma_proto) {
62 // no-op
63 }
64
ProvideEventMetrics(ChromeUserMetricsExtension & uma_proto)65 void StructuredMetricsRecorder::ProvideEventMetrics(
66 ChromeUserMetricsExtension& uma_proto) {
67 if (!CanProvideMetrics()) {
68 return;
69 }
70
71 // Get the events from event storage.
72 auto events = event_storage_->TakeEvents();
73
74 if (events.size() == 0) {
75 return;
76 }
77
78 StructuredDataProto& structured_data = *uma_proto.mutable_structured_data();
79 *structured_data.mutable_events() = std::move(events);
80
81 LogUploadSizeBytes(structured_data.ByteSizeLong());
82 LogNumEventsInUpload(structured_data.events_size());
83
84 // Applies custom metadata providers.
85 Recorder::GetInstance()->OnProvideIndependentMetrics(&uma_proto);
86 }
87
CanProvideMetrics()88 bool StructuredMetricsRecorder::CanProvideMetrics() {
89 // We can provide metrics once device or profile keys have been loaded.
90 return recording_enabled() && (IsInitialized() || IsProfileInitialized());
91 }
92
HasMetricsToProvide()93 bool StructuredMetricsRecorder::HasMetricsToProvide() {
94 return event_storage()->HasEvents();
95 }
96
OnKeyReady()97 void StructuredMetricsRecorder::OnKeyReady() {
98 DCHECK(base::CurrentUIThread::IsSet());
99
100 // If key data has not been initialized, it is highly likely that the key data
101 // is initialized.
102 if (!init_state_.Has(State::kKeyDataInitialized)) {
103 init_state_.Put(State::kKeyDataInitialized);
104 } else {
105 // If kKeyDataInitialized, then this is the second time this callback is
106 // being called, which must be the profile keys.
107 init_state_.Put(State::kProfileKeyDataInitialized);
108 }
109
110 // If recorder is now ready then hash events in-memory and store them in
111 // persistent storage.
112 if (CanProvideMetrics()) {
113 HashUnhashedEventsAndPersist();
114 if (!on_ready_callback_.is_null()) {
115 std::move(on_ready_callback_).Run();
116 }
117 }
118 }
119
AddEventsObserver(Observer * watcher)120 void StructuredMetricsRecorder::AddEventsObserver(Observer* watcher) {
121 watchers_.AddObserver(watcher);
122 }
123
RemoveEventsObserver(Observer * watcher)124 void StructuredMetricsRecorder::RemoveEventsObserver(Observer* watcher) {
125 watchers_.RemoveObserver(watcher);
126 }
127
OnEventRecord(const Event & event)128 void StructuredMetricsRecorder::OnEventRecord(const Event& event) {
129 DCHECK(base::CurrentUIThread::IsSet());
130
131 // One more state for the EventRecordingState exists: kMetricsProviderMissing.
132 // This is recorded in Recorder::Record.
133 if (!recording_enabled_ && !CanForceRecord(event)) {
134 // Events should be ignored if recording is disabled.
135 LogEventRecordingState(EventRecordingState::kRecordingDisabled);
136 return;
137 }
138
139 if (IsDeviceEvent(event) && !IsInitialized()) {
140 RecordEventBeforeInitialization(event);
141 return;
142 }
143
144 if (IsProfileEvent(event) && !IsProfileInitialized()) {
145 RecordProfileEventBeforeInitialization(event);
146 return;
147 }
148
149 RecordEvent(event);
150 test_callback_on_record_.Run();
151 }
152
HasState(State state) const153 bool StructuredMetricsRecorder::HasState(State state) const {
154 return init_state_.Has(state);
155 }
156
Purge()157 void StructuredMetricsRecorder::Purge() {
158 CHECK(event_storage_);
159 event_storage_->Purge();
160 key_data_provider_->Purge();
161
162 unhashed_events_.clear();
163 unhashed_profile_events_.clear();
164 }
165
RecordEventBeforeInitialization(const Event & event)166 void StructuredMetricsRecorder::RecordEventBeforeInitialization(
167 const Event& event) {
168 DCHECK(!IsInitialized());
169 unhashed_events_.emplace_back(event.Clone());
170 }
171
RecordProfileEventBeforeInitialization(const Event & event)172 void StructuredMetricsRecorder::RecordProfileEventBeforeInitialization(
173 const Event& event) {
174 DCHECK(!IsProfileInitialized());
175 unhashed_profile_events_.emplace_back(event.Clone());
176 }
177
RecordEvent(const Event & event)178 void StructuredMetricsRecorder::RecordEvent(const Event& event) {
179 DCHECK(IsKeyDataInitialized());
180
181 // Retrieve key for the project.
182 KeyData* key_data = key_data_provider_->GetKeyData(event.project_name());
183 if (!key_data) {
184 return;
185 }
186
187 // Validates the event. If valid, retrieve the metadata associated
188 // with the event.
189 const auto validators = GetEventValidators(event);
190 if (!validators) {
191 return;
192 }
193
194 const auto* project_validator = validators->first;
195 const auto* event_validator = validators->second;
196
197 if (!CanUploadProject(project_validator->project_hash())) {
198 LogEventRecordingState(EventRecordingState::kProjectDisallowed);
199 return;
200 }
201
202 LogEventRecordingState(EventRecordingState::kRecorded);
203
204 // Events associated with UMA are deprecated.
205 if (!IsIndependentMetricsUploadEnabled() ||
206 project_validator->id_type() == IdType::kUmaId) {
207 return;
208 }
209
210 StructuredEventProto event_proto;
211
212 // Initialize event proto from validator metadata.
213 InitializeEventProto(&event_proto, event, *project_validator,
214 *event_validator);
215
216 // Sequence-related metadata.
217 if (project_validator->event_type() == StructuredEventProto::SEQUENCE &&
218 base::FeatureList::IsEnabled(kEventSequenceLogging)) {
219 AddSequenceMetadata(&event_proto, event, *project_validator, *key_data);
220 }
221
222 // Populate the metrics and add to proto.
223 AddMetricsToProto(&event_proto, event, *project_validator, *event_validator);
224
225 // Log size information about the event.
226 LogEventSerializedSizeBytes(event_proto.ByteSizeLong());
227
228 Recorder::GetInstance()->OnEventRecorded(&event_proto);
229 NotifyEventRecorded(event_proto);
230
231 // Add new event to storage.
232 event_storage_->AddEvent(event_proto);
233
234 test_callback_on_record_.Run();
235 }
236
InitializeEventProto(StructuredEventProto * proto,const Event & event,const ProjectValidator & project_validator,const EventValidator & event_validator)237 void StructuredMetricsRecorder::InitializeEventProto(
238 StructuredEventProto* proto,
239 const Event& event,
240 const ProjectValidator& project_validator,
241 const EventValidator& event_validator) {
242 proto->set_project_name_hash(project_validator.project_hash());
243 proto->set_event_name_hash(event_validator.event_hash());
244
245 // Set the event type. Do this with a switch statement to catch when the
246 // event type is UNKNOWN or uninitialized.
247 CHECK_NE(project_validator.event_type(), StructuredEventProto::UNKNOWN);
248
249 proto->set_event_type(project_validator.event_type());
250
251 // Set the ID for this event, if any.
252 switch (project_validator.id_type()) {
253 case IdType::kProjectId: {
254 std::optional<uint64_t> primary_id =
255 key_data_provider_->GetId(event.project_name());
256 if (primary_id.has_value()) {
257 proto->set_profile_event_id(primary_id.value());
258 }
259 } break;
260 case IdType::kUmaId:
261 // TODO(crbug.com/1148168): Unimplemented.
262 break;
263 case IdType::kUnidentified:
264 // Do nothing.
265 break;
266 }
267 }
268
AddMetricsToProto(StructuredEventProto * proto,const Event & event,const ProjectValidator & project_validator,const EventValidator & event_validator)269 void StructuredMetricsRecorder::AddMetricsToProto(
270 StructuredEventProto* proto,
271 const Event& event,
272 const ProjectValidator& project_validator,
273 const EventValidator& event_validator) {
274 KeyData* key = key_data_provider_->GetKeyData(event.project_name());
275
276 // Key is checked by the calling function.
277 CHECK(key);
278
279 // Set each metric's name hash and value.
280 for (const auto& metric : event.metric_values()) {
281 const std::string& metric_name = metric.first;
282 const Event::MetricValue& metric_value = metric.second;
283
284 // Validate that both name and metric type are valid structured metrics.
285 // If a metric is invalid, then ignore the metric so that other valid
286 // metrics are added to the proto.
287 std::optional<EventValidator::MetricMetadata> metadata =
288 event_validator.GetMetricMetadata(metric_name);
289
290 // Checks that the metrics defined are valid. If not valid, then the
291 // metric will be ignored.
292 bool is_valid =
293 metadata.has_value() && metadata->metric_type == metric_value.type;
294 DCHECK(is_valid);
295 if (!is_valid) {
296 continue;
297 }
298
299 StructuredEventProto::Metric* metric_proto = proto->add_metrics();
300 int64_t metric_name_hash = metadata->metric_name_hash;
301 metric_proto->set_name_hash(metric_name_hash);
302
303 const auto& value = metric_value.value;
304 switch (metadata->metric_type) {
305 case Event::MetricType::kHmac:
306 metric_proto->set_value_hmac(key->HmacMetric(
307 project_validator.project_hash(), metric_name_hash,
308 value.GetString(), project_validator.key_rotation_period()));
309 break;
310 case Event::MetricType::kLong:
311 int64_t long_value;
312 base::StringToInt64(value.GetString(), &long_value);
313 metric_proto->set_value_int64(long_value);
314 break;
315 case Event::MetricType::kRawString:
316 metric_proto->set_value_string(value.GetString());
317 break;
318 case Event::MetricType::kDouble:
319 metric_proto->set_value_double(value.GetDouble());
320 break;
321 // Represents an enum.
322 case Event::MetricType::kInt:
323 metric_proto->set_value_int64(value.GetInt());
324 break;
325 // Not supported yet.
326 case Event::MetricType::kBoolean:
327 break;
328 }
329 }
330 }
331
HashUnhashedEventsAndPersist()332 void StructuredMetricsRecorder::HashUnhashedEventsAndPersist() {
333 if (IsInitialized()) {
334 LogNumEventsRecordedBeforeInit(unhashed_events_.size());
335 while (!unhashed_events_.empty()) {
336 OnEventRecord(unhashed_events_.front());
337 unhashed_events_.pop_front();
338 }
339 }
340 if (IsProfileInitialized()) {
341 LogNumEventsRecordedBeforeInit(unhashed_profile_events_.size());
342 while (!unhashed_profile_events_.empty()) {
343 OnEventRecord(unhashed_profile_events_.front());
344 unhashed_profile_events_.pop_front();
345 }
346 }
347 }
348
CanUploadProject(uint64_t project_name_hash) const349 bool StructuredMetricsRecorder::CanUploadProject(
350 uint64_t project_name_hash) const {
351 return !disallowed_projects_.contains(project_name_hash);
352 }
353
CacheDisallowedProjectsSet()354 void StructuredMetricsRecorder::CacheDisallowedProjectsSet() {
355 const std::string& disallowed_list = GetDisabledProjects();
356 if (disallowed_list.empty()) {
357 return;
358 }
359
360 for (const auto& value :
361 base::SplitString(disallowed_list, ",", base::TRIM_WHITESPACE,
362 base::SPLIT_WANT_NONEMPTY)) {
363 uint64_t project_name_hash;
364 // Parse the string and keep only perfect conversions.
365 if (base::StringToUint64(value, &project_name_hash)) {
366 disallowed_projects_.insert(project_name_hash);
367 }
368 }
369 }
370
IsKeyDataInitialized()371 bool StructuredMetricsRecorder::IsKeyDataInitialized() {
372 return key_data_provider_->IsReady();
373 }
374
IsInitialized()375 bool StructuredMetricsRecorder::IsInitialized() {
376 return init_state_.Has(State::kKeyDataInitialized);
377 }
378
IsProfileInitialized()379 bool StructuredMetricsRecorder::IsProfileInitialized() {
380 return init_state_.Has(State::kProfileKeyDataInitialized);
381 }
382
CanForceRecord(const Event & event) const383 bool StructuredMetricsRecorder::CanForceRecord(const Event& event) const {
384 const auto validators = GetEventValidators(event);
385 if (!validators) {
386 return false;
387 }
388 return validators->second->can_force_record();
389 }
390
IsDeviceEvent(const Event & event) const391 bool StructuredMetricsRecorder::IsDeviceEvent(const Event& event) const {
392 // Validates the event. If valid, retrieve the metadata associated
393 // with the event.
394 const auto validators = GetEventValidators(event);
395 if (!validators) {
396 return false;
397 }
398 const auto* project_validator = validators->first;
399
400 // Sequence events are marked as per-device but use the profile keys.
401 return !event.IsEventSequenceType() &&
402 project_validator->id_scope() == IdScope::kPerDevice;
403 }
404
IsProfileEvent(const Event & event) const405 bool StructuredMetricsRecorder::IsProfileEvent(const Event& event) const {
406 // Validates the event. If valid, retrieve the metadata associated
407 // with the event.
408 const auto validators = GetEventValidators(event);
409 if (!validators) {
410 return false;
411 }
412 const auto* project_validator = validators->first;
413
414 // Sequence events are marked as per-device but use the profile keys.
415 return event.IsEventSequenceType() ||
416 project_validator->id_scope() == IdScope::kPerProfile;
417 }
418
419 std::optional<std::pair<const ProjectValidator*, const EventValidator*>>
GetEventValidators(const Event & event) const420 StructuredMetricsRecorder::GetEventValidators(const Event& event) const {
421 const auto* project_validator =
422 validator::Validators::Get()->GetProjectValidator(event.project_name());
423
424 if (!project_validator) {
425 return std::nullopt;
426 }
427
428 const auto* event_validator =
429 project_validator->GetEventValidator(event.event_name());
430
431 if (!event_validator) {
432 return std::nullopt;
433 }
434
435 return std::make_pair(project_validator, event_validator);
436 }
437
SetOnReadyToRecord(base::OnceClosure callback)438 void StructuredMetricsRecorder::SetOnReadyToRecord(base::OnceClosure callback) {
439 on_ready_callback_ = std::move(callback);
440
441 if (IsInitialized()) {
442 std::move(on_ready_callback_).Run();
443 }
444 }
445
SetEventRecordCallbackForTest(base::RepeatingClosure callback)446 void StructuredMetricsRecorder::SetEventRecordCallbackForTest(
447 base::RepeatingClosure callback) {
448 test_callback_on_record_ = std::move(callback);
449 }
450
AddDisallowedProjectForTest(uint64_t project_name_hash)451 void StructuredMetricsRecorder::AddDisallowedProjectForTest(
452 uint64_t project_name_hash) {
453 disallowed_projects_.insert(project_name_hash);
454 }
455
NotifyEventRecorded(const StructuredEventProto & event)456 void StructuredMetricsRecorder::NotifyEventRecorded(
457 const StructuredEventProto& event) {
458 for (Observer& watcher : watchers_) {
459 watcher.OnEventRecorded(event);
460 }
461 }
462
463 } // namespace metrics::structured
464