1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #define STATSD_DEBUG false // STOPSHIP if true
17 #if !defined(NDEBUG) && !defined(DEBUG)
18 #define NDEBUG // comment to enable assert
19 #endif /* !defined(NDEBUG) && !defined(DEBUG) */
20 #include "Log.h"
21
22 #include "MetricsManager.h"
23
24 #include <assert.h>
25 #include <private/android_filesystem_config.h>
26
27 #include "CountMetricProducer.h"
28 #include "condition/CombinationConditionTracker.h"
29 #include "condition/SimpleConditionTracker.h"
30 #include "flags/FlagProvider.h"
31 #include "guardrail/StatsdStats.h"
32 #include "matchers/CombinationAtomMatchingTracker.h"
33 #include "matchers/SimpleAtomMatchingTracker.h"
34 #include "parsing_utils/config_update_utils.h"
35 #include "parsing_utils/metrics_manager_util.h"
36 #include "state/StateManager.h"
37 #include "stats_log_util.h"
38 #include "stats_util.h"
39 #include "statslog_statsd.h"
40 #include "utils/DbUtils.h"
41 #include "utils/api_tracing.h"
42
43 using android::util::FIELD_COUNT_REPEATED;
44 using android::util::FIELD_TYPE_INT32;
45 using android::util::FIELD_TYPE_INT64;
46 using android::util::FIELD_TYPE_MESSAGE;
47 using android::util::FIELD_TYPE_STRING;
48 using android::util::ProtoOutputStream;
49
50 using std::set;
51 using std::string;
52 using std::unique_ptr;
53 using std::vector;
54
55 namespace android {
56 namespace os {
57 namespace statsd {
58
59 const int FIELD_ID_METRICS = 1;
60 const int FIELD_ID_ANNOTATIONS = 7;
61 const int FIELD_ID_ANNOTATIONS_INT64 = 1;
62 const int FIELD_ID_ANNOTATIONS_INT32 = 2;
63
64 // for ActiveConfig
65 const int FIELD_ID_ACTIVE_CONFIG_ID = 1;
66 const int FIELD_ID_ACTIVE_CONFIG_UID = 2;
67 const int FIELD_ID_ACTIVE_CONFIG_METRIC = 3;
68
MetricsManager(const ConfigKey & key,const StatsdConfig & config,const int64_t timeBaseNs,const int64_t currentTimeNs,const sp<UidMap> & uidMap,const sp<StatsPullerManager> & pullerManager,const sp<AlarmMonitor> & anomalyAlarmMonitor,const sp<AlarmMonitor> & periodicAlarmMonitor)69 MetricsManager::MetricsManager(const ConfigKey& key, const StatsdConfig& config,
70 const int64_t timeBaseNs, const int64_t currentTimeNs,
71 const sp<UidMap>& uidMap,
72 const sp<StatsPullerManager>& pullerManager,
73 const sp<AlarmMonitor>& anomalyAlarmMonitor,
74 const sp<AlarmMonitor>& periodicAlarmMonitor)
75 : mConfigKey(key),
76 mUidMap(uidMap),
77 mPackageCertificateHashSizeBytes(
78 static_cast<uint8_t>(config.package_certificate_hash_size_bytes())),
79 mTtlNs(config.has_ttl_in_seconds() ? config.ttl_in_seconds() * NS_PER_SEC : -1),
80 mTtlEndNs(-1),
81 mLastReportTimeNs(currentTimeNs),
82 mLastReportWallClockNs(getWallClockNs()),
83 mPullerManager(pullerManager),
84 mWhitelistedAtomIds(config.whitelisted_atom_ids().begin(),
85 config.whitelisted_atom_ids().end()),
86 mShouldPersistHistory(config.persist_locally()),
87 mUseV2SoftMemoryCalculation(config.statsd_config_options().use_v2_soft_memory_limit()),
88 mOmitSystemUidsInUidMap(config.statsd_config_options().omit_system_uids_in_uidmap()),
89 mOmitUnusedUidsInUidMap(config.statsd_config_options().omit_unused_uids_in_uidmap()),
90 mAllowlistedUidMapPackages(
91 set<string>(config.statsd_config_options().uidmap_package_allowlist().begin(),
92 config.statsd_config_options().uidmap_package_allowlist().end())) {
93 if (!isAtLeastU() && config.has_restricted_metrics_delegate_package_name()) {
94 mInvalidConfigReason =
95 InvalidConfigReason(INVALID_CONFIG_REASON_RESTRICTED_METRIC_NOT_ENABLED);
96 return;
97 }
98 if (config.has_restricted_metrics_delegate_package_name()) {
99 mRestrictedMetricsDelegatePackageName = config.restricted_metrics_delegate_package_name();
100 }
101 // Init the ttl end timestamp.
102 refreshTtl(timeBaseNs);
103 mInvalidConfigReason = initStatsdConfig(
104 key, config, uidMap, pullerManager, anomalyAlarmMonitor, periodicAlarmMonitor,
105 timeBaseNs, currentTimeNs, this, mTagIdsToMatchersMap, mAllAtomMatchingTrackers,
106 mAtomMatchingTrackerMap, mAllConditionTrackers, mConditionTrackerMap,
107 mAllMetricProducers, mMetricProducerMap, mAllAnomalyTrackers, mAllPeriodicAlarmTrackers,
108 mConditionToMetricMap, mTrackerToMetricMap, mTrackerToConditionMap,
109 mActivationAtomTrackerToMetricMap, mDeactivationAtomTrackerToMetricMap,
110 mAlertTrackerMap, mMetricIndexesWithActivation, mStateProtoHashes, mNoReportMetricIds);
111
112 mHashStringsInReport = config.hash_strings_in_metric_report();
113 mVersionStringsInReport = config.version_strings_in_metric_report();
114 mInstallerInReport = config.installer_in_metric_report();
115
116 createAllLogSourcesFromConfig(config);
117 setMaxMetricsBytesFromConfig(config);
118 setTriggerGetDataBytesFromConfig(config);
119 mPullerManager->RegisterPullUidProvider(mConfigKey, this);
120
121 // Store the sub-configs used.
122 for (const auto& annotation : config.annotation()) {
123 mAnnotations.emplace_back(annotation.field_int64(), annotation.field_int32());
124 }
125 verifyGuardrailsAndUpdateStatsdStats();
126 initializeConfigActiveStatus();
127 }
128
~MetricsManager()129 MetricsManager::~MetricsManager() {
130 for (auto it : mAllMetricProducers) {
131 for (int atomId : it->getSlicedStateAtoms()) {
132 StateManager::getInstance().unregisterListener(atomId, it);
133 }
134 }
135 mPullerManager->UnregisterPullUidProvider(mConfigKey, this);
136
137 VLOG("~MetricsManager()");
138 }
139
updateConfig(const StatsdConfig & config,const int64_t timeBaseNs,const int64_t currentTimeNs,const sp<AlarmMonitor> & anomalyAlarmMonitor,const sp<AlarmMonitor> & periodicAlarmMonitor)140 bool MetricsManager::updateConfig(const StatsdConfig& config, const int64_t timeBaseNs,
141 const int64_t currentTimeNs,
142 const sp<AlarmMonitor>& anomalyAlarmMonitor,
143 const sp<AlarmMonitor>& periodicAlarmMonitor) {
144 if (!isAtLeastU() && config.has_restricted_metrics_delegate_package_name()) {
145 mInvalidConfigReason =
146 InvalidConfigReason(INVALID_CONFIG_REASON_RESTRICTED_METRIC_NOT_ENABLED);
147 return false;
148 }
149 if (config.has_restricted_metrics_delegate_package_name()) {
150 mRestrictedMetricsDelegatePackageName = config.restricted_metrics_delegate_package_name();
151 } else {
152 mRestrictedMetricsDelegatePackageName = nullopt;
153 }
154 vector<sp<AtomMatchingTracker>> newAtomMatchingTrackers;
155 unordered_map<int64_t, int> newAtomMatchingTrackerMap;
156 vector<sp<ConditionTracker>> newConditionTrackers;
157 unordered_map<int64_t, int> newConditionTrackerMap;
158 map<int64_t, uint64_t> newStateProtoHashes;
159 vector<sp<MetricProducer>> newMetricProducers;
160 unordered_map<int64_t, int> newMetricProducerMap;
161 vector<sp<AnomalyTracker>> newAnomalyTrackers;
162 unordered_map<int64_t, int> newAlertTrackerMap;
163 vector<sp<AlarmTracker>> newPeriodicAlarmTrackers;
164 mTagIdsToMatchersMap.clear();
165 mConditionToMetricMap.clear();
166 mTrackerToMetricMap.clear();
167 mTrackerToConditionMap.clear();
168 mActivationAtomTrackerToMetricMap.clear();
169 mDeactivationAtomTrackerToMetricMap.clear();
170 mMetricIndexesWithActivation.clear();
171 mNoReportMetricIds.clear();
172 mInvalidConfigReason = updateStatsdConfig(
173 mConfigKey, config, mUidMap, mPullerManager, anomalyAlarmMonitor, periodicAlarmMonitor,
174 timeBaseNs, currentTimeNs, mAllAtomMatchingTrackers, mAtomMatchingTrackerMap,
175 mAllConditionTrackers, mConditionTrackerMap, mAllMetricProducers, mMetricProducerMap,
176 mAllAnomalyTrackers, mAlertTrackerMap, mStateProtoHashes, this, mTagIdsToMatchersMap,
177 newAtomMatchingTrackers, newAtomMatchingTrackerMap, newConditionTrackers,
178 newConditionTrackerMap, newMetricProducers, newMetricProducerMap, newAnomalyTrackers,
179 newAlertTrackerMap, newPeriodicAlarmTrackers, mConditionToMetricMap,
180 mTrackerToMetricMap, mTrackerToConditionMap, mActivationAtomTrackerToMetricMap,
181 mDeactivationAtomTrackerToMetricMap, mMetricIndexesWithActivation, newStateProtoHashes,
182 mNoReportMetricIds);
183 mAllAtomMatchingTrackers = newAtomMatchingTrackers;
184 mAtomMatchingTrackerMap = newAtomMatchingTrackerMap;
185 mAllConditionTrackers = newConditionTrackers;
186 mConditionTrackerMap = newConditionTrackerMap;
187 mAllMetricProducers = newMetricProducers;
188 mMetricProducerMap = newMetricProducerMap;
189 mStateProtoHashes = newStateProtoHashes;
190 mAllAnomalyTrackers = newAnomalyTrackers;
191 mAlertTrackerMap = newAlertTrackerMap;
192 mAllPeriodicAlarmTrackers = newPeriodicAlarmTrackers;
193
194 mTtlNs = config.has_ttl_in_seconds() ? config.ttl_in_seconds() * NS_PER_SEC : -1;
195 refreshTtl(currentTimeNs);
196
197 mHashStringsInReport = config.hash_strings_in_metric_report();
198 mVersionStringsInReport = config.version_strings_in_metric_report();
199 mInstallerInReport = config.installer_in_metric_report();
200 mWhitelistedAtomIds.clear();
201 mWhitelistedAtomIds.insert(config.whitelisted_atom_ids().begin(),
202 config.whitelisted_atom_ids().end());
203 mShouldPersistHistory = config.persist_locally();
204 mPackageCertificateHashSizeBytes = config.package_certificate_hash_size_bytes();
205 mUseV2SoftMemoryCalculation = config.statsd_config_options().use_v2_soft_memory_limit();
206 mOmitSystemUidsInUidMap = config.statsd_config_options().omit_system_uids_in_uidmap();
207 mOmitUnusedUidsInUidMap = config.statsd_config_options().omit_unused_uids_in_uidmap();
208 mAllowlistedUidMapPackages =
209 set<string>(config.statsd_config_options().uidmap_package_allowlist().begin(),
210 config.statsd_config_options().uidmap_package_allowlist().end());
211
212 // Store the sub-configs used.
213 mAnnotations.clear();
214 for (const auto& annotation : config.annotation()) {
215 mAnnotations.emplace_back(annotation.field_int64(), annotation.field_int32());
216 }
217
218 mAllowedUid.clear();
219 mAllowedPkg.clear();
220 mDefaultPullUids.clear();
221 mPullAtomUids.clear();
222 mPullAtomPackages.clear();
223 createAllLogSourcesFromConfig(config);
224 setMaxMetricsBytesFromConfig(config);
225 setTriggerGetDataBytesFromConfig(config);
226
227 verifyGuardrailsAndUpdateStatsdStats();
228 initializeConfigActiveStatus();
229 return !mInvalidConfigReason.has_value();
230 }
231
createAllLogSourcesFromConfig(const StatsdConfig & config)232 void MetricsManager::createAllLogSourcesFromConfig(const StatsdConfig& config) {
233 // Init allowed pushed atom uids.
234 for (const auto& source : config.allowed_log_source()) {
235 auto it = UidMap::sAidToUidMapping.find(source);
236 if (it != UidMap::sAidToUidMapping.end()) {
237 mAllowedUid.push_back(it->second);
238 } else {
239 mAllowedPkg.push_back(source);
240 }
241 }
242
243 if (mAllowedUid.size() + mAllowedPkg.size() > StatsdStats::kMaxLogSourceCount) {
244 ALOGE("Too many log sources. This is likely to be an error in the config.");
245 mInvalidConfigReason = InvalidConfigReason(INVALID_CONFIG_REASON_TOO_MANY_LOG_SOURCES);
246 } else {
247 initAllowedLogSources();
248 }
249
250 // Init default allowed pull atom uids.
251 int numPullPackages = 0;
252 for (const string& pullSource : config.default_pull_packages()) {
253 auto it = UidMap::sAidToUidMapping.find(pullSource);
254 if (it != UidMap::sAidToUidMapping.end()) {
255 numPullPackages++;
256 mDefaultPullUids.insert(it->second);
257 } else {
258 ALOGE("Default pull atom packages must be in sAidToUidMapping");
259 mInvalidConfigReason =
260 InvalidConfigReason(INVALID_CONFIG_REASON_DEFAULT_PULL_PACKAGES_NOT_IN_MAP);
261 }
262 }
263 // Init per-atom pull atom packages.
264 for (const PullAtomPackages& pullAtomPackages : config.pull_atom_packages()) {
265 int32_t atomId = pullAtomPackages.atom_id();
266 for (const string& pullPackage : pullAtomPackages.packages()) {
267 numPullPackages++;
268 auto it = UidMap::sAidToUidMapping.find(pullPackage);
269 if (it != UidMap::sAidToUidMapping.end()) {
270 mPullAtomUids[atomId].insert(it->second);
271 } else {
272 mPullAtomPackages[atomId].insert(pullPackage);
273 }
274 }
275 }
276 if (numPullPackages > StatsdStats::kMaxPullAtomPackages) {
277 ALOGE("Too many sources in default_pull_packages and pull_atom_packages. This is likely to "
278 "be an error in the config");
279 mInvalidConfigReason =
280 InvalidConfigReason(INVALID_CONFIG_REASON_TOO_MANY_SOURCES_IN_PULL_PACKAGES);
281 } else {
282 initPullAtomSources();
283 }
284 }
285
setMaxMetricsBytesFromConfig(const StatsdConfig & config)286 void MetricsManager::setMaxMetricsBytesFromConfig(const StatsdConfig& config) {
287 if (!config.has_max_metrics_memory_kb()) {
288 mMaxMetricsBytes = StatsdStats::kDefaultMaxMetricsBytesPerConfig;
289 return;
290 }
291 if (config.max_metrics_memory_kb() <= 0 ||
292 static_cast<size_t>(config.max_metrics_memory_kb() * 1024) >
293 StatsdStats::kHardMaxMetricsBytesPerConfig) {
294 ALOGW("Memory limit must be between 0KB and 20MB. Setting to default value (2MB).");
295 mMaxMetricsBytes = StatsdStats::kDefaultMaxMetricsBytesPerConfig;
296 } else {
297 mMaxMetricsBytes = config.max_metrics_memory_kb() * 1024;
298 }
299 }
300
setTriggerGetDataBytesFromConfig(const StatsdConfig & config)301 void MetricsManager::setTriggerGetDataBytesFromConfig(const StatsdConfig& config) {
302 if (!config.has_soft_metrics_memory_kb()) {
303 mTriggerGetDataBytes = StatsdStats::kDefaultBytesPerConfigTriggerGetData;
304 return;
305 }
306 if (config.soft_metrics_memory_kb() <= 0 ||
307 static_cast<size_t>(config.soft_metrics_memory_kb() * 1024) >
308 StatsdStats::kHardMaxTriggerGetDataBytes) {
309 ALOGW("Memory limit ust be between 0KB and 10MB. Setting to default value (192KB).");
310 mTriggerGetDataBytes = StatsdStats::kDefaultBytesPerConfigTriggerGetData;
311 } else {
312 mTriggerGetDataBytes = config.soft_metrics_memory_kb() * 1024;
313 }
314 }
315
verifyGuardrailsAndUpdateStatsdStats()316 void MetricsManager::verifyGuardrailsAndUpdateStatsdStats() {
317 // Guardrail. Reject the config if it's too big.
318 if (mAllMetricProducers.size() > StatsdStats::kMaxMetricCountPerConfig) {
319 ALOGE("This config has too many metrics! Reject!");
320 mInvalidConfigReason = InvalidConfigReason(INVALID_CONFIG_REASON_TOO_MANY_METRICS);
321 }
322 if (mAllConditionTrackers.size() > StatsdStats::kMaxConditionCountPerConfig) {
323 ALOGE("This config has too many predicates! Reject!");
324 mInvalidConfigReason = InvalidConfigReason(INVALID_CONFIG_REASON_TOO_MANY_CONDITIONS);
325 }
326 if (mAllAtomMatchingTrackers.size() > StatsdStats::kMaxMatcherCountPerConfig) {
327 ALOGE("This config has too many matchers! Reject!");
328 mInvalidConfigReason = InvalidConfigReason(INVALID_CONFIG_REASON_TOO_MANY_MATCHERS);
329 }
330 if (mAllAnomalyTrackers.size() > StatsdStats::kMaxAlertCountPerConfig) {
331 ALOGE("This config has too many alerts! Reject!");
332 mInvalidConfigReason = InvalidConfigReason(INVALID_CONFIG_REASON_TOO_MANY_ALERTS);
333 }
334 // no matter whether this config is valid, log it in the stats.
335 StatsdStats::getInstance().noteConfigReceived(
336 mConfigKey, mAllMetricProducers.size(), mAllConditionTrackers.size(),
337 mAllAtomMatchingTrackers.size(), mAllAnomalyTrackers.size(), mAnnotations,
338 mInvalidConfigReason);
339 }
340
initializeConfigActiveStatus()341 void MetricsManager::initializeConfigActiveStatus() {
342 mIsAlwaysActive = (mMetricIndexesWithActivation.size() != mAllMetricProducers.size()) ||
343 (mAllMetricProducers.size() == 0);
344 mIsActive = mIsAlwaysActive;
345 for (int metric : mMetricIndexesWithActivation) {
346 mIsActive |= mAllMetricProducers[metric]->isActive();
347 }
348 VLOG("mIsActive is initialized to %d", mIsActive);
349 }
350
initAllowedLogSources()351 void MetricsManager::initAllowedLogSources() {
352 std::lock_guard<std::mutex> lock(mAllowedLogSourcesMutex);
353 mAllowedLogSources.clear();
354 mAllowedLogSources.insert(mAllowedUid.begin(), mAllowedUid.end());
355
356 for (const auto& pkg : mAllowedPkg) {
357 auto uids = mUidMap->getAppUid(pkg);
358 mAllowedLogSources.insert(uids.begin(), uids.end());
359 }
360 if (STATSD_DEBUG) {
361 for (const auto& uid : mAllowedLogSources) {
362 VLOG("Allowed uid %d", uid);
363 }
364 }
365 }
366
initPullAtomSources()367 void MetricsManager::initPullAtomSources() {
368 std::lock_guard<std::mutex> lock(mAllowedLogSourcesMutex);
369 mCombinedPullAtomUids.clear();
370 for (const auto& [atomId, uids] : mPullAtomUids) {
371 mCombinedPullAtomUids[atomId].insert(uids.begin(), uids.end());
372 }
373 for (const auto& [atomId, packages] : mPullAtomPackages) {
374 for (const string& pkg : packages) {
375 set<int32_t> uids = mUidMap->getAppUid(pkg);
376 mCombinedPullAtomUids[atomId].insert(uids.begin(), uids.end());
377 }
378 }
379 }
380
isConfigValid() const381 bool MetricsManager::isConfigValid() const {
382 return !mInvalidConfigReason.has_value();
383 }
384
notifyAppUpgrade(const int64_t eventTimeNs,const string & apk,const int uid,const int64_t version)385 void MetricsManager::notifyAppUpgrade(const int64_t eventTimeNs, const string& apk, const int uid,
386 const int64_t version) {
387 // Inform all metric producers.
388 for (const auto& it : mAllMetricProducers) {
389 it->notifyAppUpgrade(eventTimeNs);
390 }
391 // check if we care this package
392 if (std::find(mAllowedPkg.begin(), mAllowedPkg.end(), apk) != mAllowedPkg.end()) {
393 // We will re-initialize the whole list because we don't want to keep the multi mapping of
394 // UID<->pkg inside MetricsManager to reduce the memory usage.
395 initAllowedLogSources();
396 }
397
398 for (const auto& it : mPullAtomPackages) {
399 if (it.second.find(apk) != it.second.end()) {
400 initPullAtomSources();
401 return;
402 }
403 }
404 }
405
notifyAppRemoved(const int64_t eventTimeNs,const string & apk,const int uid)406 void MetricsManager::notifyAppRemoved(const int64_t eventTimeNs, const string& apk, const int uid) {
407 // Inform all metric producers.
408 for (const auto& it : mAllMetricProducers) {
409 it->notifyAppRemoved(eventTimeNs);
410 }
411 // check if we care this package
412 if (std::find(mAllowedPkg.begin(), mAllowedPkg.end(), apk) != mAllowedPkg.end()) {
413 // We will re-initialize the whole list because we don't want to keep the multi mapping of
414 // UID<->pkg inside MetricsManager to reduce the memory usage.
415 initAllowedLogSources();
416 }
417
418 for (const auto& it : mPullAtomPackages) {
419 if (it.second.find(apk) != it.second.end()) {
420 initPullAtomSources();
421 return;
422 }
423 }
424 }
425
onUidMapReceived(const int64_t eventTimeNs)426 void MetricsManager::onUidMapReceived(const int64_t eventTimeNs) {
427 // Purposefully don't inform metric producers on a new snapshot
428 // because we don't need to flush partial buckets.
429 // This occurs if a new user is added/removed or statsd crashes.
430 initPullAtomSources();
431
432 if (mAllowedPkg.size() == 0) {
433 return;
434 }
435 initAllowedLogSources();
436 }
437
onStatsdInitCompleted(const int64_t eventTimeNs)438 void MetricsManager::onStatsdInitCompleted(const int64_t eventTimeNs) {
439 ATRACE_CALL();
440 // Inform all metric producers.
441 for (const auto& it : mAllMetricProducers) {
442 it->onStatsdInitCompleted(eventTimeNs);
443 }
444 }
445
init()446 void MetricsManager::init() {
447 for (const auto& producer : mAllMetricProducers) {
448 producer->prepareFirstBucket();
449 }
450 }
451
getPullAtomUids(int32_t atomId)452 vector<int32_t> MetricsManager::getPullAtomUids(int32_t atomId) {
453 std::lock_guard<std::mutex> lock(mAllowedLogSourcesMutex);
454 vector<int32_t> uids;
455 const auto& it = mCombinedPullAtomUids.find(atomId);
456 if (it != mCombinedPullAtomUids.end()) {
457 uids.insert(uids.end(), it->second.begin(), it->second.end());
458 }
459 uids.insert(uids.end(), mDefaultPullUids.begin(), mDefaultPullUids.end());
460 return uids;
461 }
462
useV2SoftMemoryCalculation()463 bool MetricsManager::useV2SoftMemoryCalculation() {
464 return mUseV2SoftMemoryCalculation;
465 }
466
dumpStates(int out,bool verbose)467 void MetricsManager::dumpStates(int out, bool verbose) {
468 dprintf(out, "ConfigKey %s, allowed source:", mConfigKey.ToString().c_str());
469 {
470 std::lock_guard<std::mutex> lock(mAllowedLogSourcesMutex);
471 for (const auto& source : mAllowedLogSources) {
472 dprintf(out, "%d ", source);
473 }
474 }
475 dprintf(out, "\n");
476 for (const auto& producer : mAllMetricProducers) {
477 producer->dumpStates(out, verbose);
478 }
479 }
480
dropData(const int64_t dropTimeNs)481 void MetricsManager::dropData(const int64_t dropTimeNs) {
482 for (const auto& producer : mAllMetricProducers) {
483 producer->dropData(dropTimeNs);
484 }
485 }
486
onDumpReport(const int64_t dumpTimeStampNs,const int64_t wallClockNs,const bool include_current_partial_bucket,const bool erase_data,const DumpLatency dumpLatency,std::set<string> * str_set,std::set<int32_t> & usedUids,ProtoOutputStream * protoOutput)487 void MetricsManager::onDumpReport(const int64_t dumpTimeStampNs, const int64_t wallClockNs,
488 const bool include_current_partial_bucket, const bool erase_data,
489 const DumpLatency dumpLatency, std::set<string>* str_set,
490 std::set<int32_t>& usedUids, ProtoOutputStream* protoOutput) {
491 if (hasRestrictedMetricsDelegate()) {
492 // TODO(b/268150038): report error to statsdstats
493 VLOG("Unexpected call to onDumpReport in restricted metricsmanager.");
494 return;
495 }
496
497 processQueueOverflowStats();
498
499 VLOG("=========================Metric Reports Start==========================");
500 // one StatsLogReport per MetricProduer
501 for (const auto& producer : mAllMetricProducers) {
502 if (mNoReportMetricIds.find(producer->getMetricId()) == mNoReportMetricIds.end()) {
503 uint64_t token = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
504 FIELD_ID_METRICS);
505 if (mHashStringsInReport) {
506 producer->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, erase_data,
507 dumpLatency, str_set, usedUids, protoOutput);
508 } else {
509 producer->onDumpReport(dumpTimeStampNs, include_current_partial_bucket, erase_data,
510 dumpLatency, nullptr, usedUids, protoOutput);
511 }
512 protoOutput->end(token);
513 } else {
514 producer->clearPastBuckets(dumpTimeStampNs);
515 }
516 }
517 for (const auto& annotation : mAnnotations) {
518 uint64_t token = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
519 FIELD_ID_ANNOTATIONS);
520 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ANNOTATIONS_INT64,
521 (long long)annotation.first);
522 protoOutput->write(FIELD_TYPE_INT32 | FIELD_ID_ANNOTATIONS_INT32, annotation.second);
523 protoOutput->end(token);
524 }
525
526 // Do not update the timestamps when data is not cleared to avoid timestamps from being
527 // misaligned.
528 if (erase_data) {
529 mLastReportTimeNs = dumpTimeStampNs;
530 mLastReportWallClockNs = wallClockNs;
531 }
532 VLOG("=========================Metric Reports End==========================");
533 }
534
checkLogCredentials(const int32_t uid,const int32_t atomId) const535 bool MetricsManager::checkLogCredentials(const int32_t uid, const int32_t atomId) const {
536 if (mWhitelistedAtomIds.find(atomId) != mWhitelistedAtomIds.end()) {
537 return true;
538 }
539
540 if (uid == AID_ROOT || (uid >= AID_SYSTEM && uid < AID_SHELL)) {
541 // enable atoms logged from pre-installed Android system services
542 return true;
543 }
544
545 std::lock_guard<std::mutex> lock(mAllowedLogSourcesMutex);
546 if (mAllowedLogSources.find(uid) == mAllowedLogSources.end()) {
547 VLOG("log source %d not on the whitelist", uid);
548 return false;
549 }
550 return true;
551 }
552
553 // Consume the stats log if it's interesting to this metric.
onLogEvent(const LogEvent & event)554 void MetricsManager::onLogEvent(const LogEvent& event) {
555 if (!isConfigValid()) {
556 return;
557 }
558
559 const int tagId = event.GetTagId();
560
561 if (tagId == util::STATS_SOCKET_LOSS_REPORTED) {
562 // Hard coded logic to handle socket loss info to highlight metric corruption reason
563 // STATS_SOCKET_LOSS_REPORTED might not be part of atoms allow list - but some of lost
564 // atoms can be always allowed - that is the reason to evaluate SocketLossInfo content prior
565 // the checkLogCredentials below
566 const std::optional<SocketLossInfo>& lossInfo = toSocketLossInfo(event);
567 if (lossInfo) {
568 onLogEventLost(*lossInfo);
569 }
570 // next, atom is going to be propagated to be consumed by metrics if any
571 }
572
573 if (!checkLogCredentials(event)) {
574 return;
575 }
576
577 const int64_t eventTimeNs = event.GetElapsedTimestampNs();
578
579 bool isActive = mIsAlwaysActive;
580
581 // Set of metrics that are still active after flushing.
582 unordered_set<int> activeMetricsIndices;
583
584 // Update state of all metrics w/ activation conditions as of eventTimeNs.
585 for (int metricIndex : mMetricIndexesWithActivation) {
586 const sp<MetricProducer>& metric = mAllMetricProducers[metricIndex];
587 metric->flushIfExpire(eventTimeNs);
588 if (metric->isActive()) {
589 // If this metric w/ activation condition is still active after
590 // flushing, remember it.
591 activeMetricsIndices.insert(metricIndex);
592 }
593 }
594
595 mIsActive = isActive || !activeMetricsIndices.empty();
596
597 const auto matchersIt = mTagIdsToMatchersMap.find(tagId);
598
599 if (matchersIt == mTagIdsToMatchersMap.end()) {
600 // Not interesting...
601 return;
602 }
603
604 if (event.isParsedHeaderOnly()) {
605 // This should not happen if metric config is defined for certain atom id
606 const int64_t firstMatcherId =
607 mAllAtomMatchingTrackers[*matchersIt->second.begin()]->getId();
608 ALOGW("Atom %d is mistakenly skipped - there is a matcher %lld for it", tagId,
609 (long long)firstMatcherId);
610 return;
611 }
612
613 vector<MatchingState> matcherCache(mAllAtomMatchingTrackers.size(),
614 MatchingState::kNotComputed);
615 vector<shared_ptr<LogEvent>> matcherTransformations(matcherCache.size(), nullptr);
616
617 for (const auto& matcherIndex : matchersIt->second) {
618 mAllAtomMatchingTrackers[matcherIndex]->onLogEvent(event, matcherIndex,
619 mAllAtomMatchingTrackers, matcherCache,
620 matcherTransformations);
621 }
622
623 // Set of metrics that received an activation cancellation.
624 unordered_set<int> metricIndicesWithCanceledActivations;
625
626 // Determine which metric activations received a cancellation and cancel them.
627 for (const auto& it : mDeactivationAtomTrackerToMetricMap) {
628 if (matcherCache[it.first] == MatchingState::kMatched) {
629 for (int metricIndex : it.second) {
630 mAllMetricProducers[metricIndex]->cancelEventActivation(it.first);
631 metricIndicesWithCanceledActivations.insert(metricIndex);
632 }
633 }
634 }
635
636 // Determine whether any metrics are no longer active after cancelling metric activations.
637 for (const int metricIndex : metricIndicesWithCanceledActivations) {
638 const sp<MetricProducer>& metric = mAllMetricProducers[metricIndex];
639 metric->flushIfExpire(eventTimeNs);
640 if (!metric->isActive()) {
641 activeMetricsIndices.erase(metricIndex);
642 }
643 }
644
645 isActive |= !activeMetricsIndices.empty();
646
647 // Determine which metric activations should be turned on and turn them on
648 for (const auto& it : mActivationAtomTrackerToMetricMap) {
649 if (matcherCache[it.first] == MatchingState::kMatched) {
650 for (int metricIndex : it.second) {
651 mAllMetricProducers[metricIndex]->activate(it.first, eventTimeNs);
652 isActive |= mAllMetricProducers[metricIndex]->isActive();
653 }
654 }
655 }
656
657 mIsActive = isActive;
658
659 // A bitmap to see which ConditionTracker needs to be re-evaluated.
660 vector<uint8_t> conditionToBeEvaluated(mAllConditionTrackers.size(), false);
661 vector<shared_ptr<LogEvent>> conditionToTransformedLogEvents(mAllConditionTrackers.size(),
662 nullptr);
663
664 for (const auto& [matcherIndex, conditionList] : mTrackerToConditionMap) {
665 if (matcherCache[matcherIndex] == MatchingState::kMatched) {
666 for (const int conditionIndex : conditionList) {
667 conditionToBeEvaluated[conditionIndex] = true;
668 conditionToTransformedLogEvents[conditionIndex] =
669 matcherTransformations[matcherIndex];
670 }
671 }
672 }
673
674 vector<ConditionState> conditionCache(mAllConditionTrackers.size(),
675 ConditionState::kNotEvaluated);
676 // A bitmap to track if a condition has changed value.
677 vector<uint8_t> changedCache(mAllConditionTrackers.size(), false);
678 for (size_t i = 0; i < mAllConditionTrackers.size(); i++) {
679 if (!conditionToBeEvaluated[i]) {
680 continue;
681 }
682 sp<ConditionTracker>& condition = mAllConditionTrackers[i];
683 const LogEvent& conditionEvent = conditionToTransformedLogEvents[i] == nullptr
684 ? event
685 : *conditionToTransformedLogEvents[i];
686 condition->evaluateCondition(conditionEvent, matcherCache, mAllConditionTrackers,
687 conditionCache, changedCache);
688 }
689
690 for (size_t i = 0; i < mAllConditionTrackers.size(); i++) {
691 if (!changedCache[i]) {
692 continue;
693 }
694 auto it = mConditionToMetricMap.find(i);
695 if (it == mConditionToMetricMap.end()) {
696 continue;
697 }
698 auto& metricList = it->second;
699 for (auto metricIndex : metricList) {
700 // Metric cares about non sliced condition, and it's changed.
701 // Push the new condition to it directly.
702 if (!mAllMetricProducers[metricIndex]->isConditionSliced()) {
703 mAllMetricProducers[metricIndex]->onConditionChanged(conditionCache[i],
704 eventTimeNs);
705 // Metric cares about sliced conditions, and it may have changed. Send
706 // notification, and the metric can query the sliced conditions that are
707 // interesting to it.
708 } else {
709 mAllMetricProducers[metricIndex]->onSlicedConditionMayChange(conditionCache[i],
710 eventTimeNs);
711 }
712 }
713 }
714 // For matched AtomMatchers, tell relevant metrics that a matched event has come.
715 for (size_t i = 0; i < mAllAtomMatchingTrackers.size(); i++) {
716 if (matcherCache[i] == MatchingState::kMatched) {
717 StatsdStats::getInstance().noteMatcherMatched(mConfigKey,
718 mAllAtomMatchingTrackers[i]->getId());
719 auto it = mTrackerToMetricMap.find(i);
720 if (it == mTrackerToMetricMap.end()) {
721 continue;
722 }
723 auto& metricList = it->second;
724 const LogEvent& metricEvent =
725 matcherTransformations[i] == nullptr ? event : *matcherTransformations[i];
726 for (const int metricIndex : metricList) {
727 // pushed metrics are never scheduled pulls
728 mAllMetricProducers[metricIndex]->onMatchedLogEvent(i, metricEvent);
729 }
730 }
731 }
732 }
733
onLogEventLost(const SocketLossInfo & socketLossInfo)734 void MetricsManager::onLogEventLost(const SocketLossInfo& socketLossInfo) {
735 // socketLossInfo stores atomId per UID - to eliminate duplicates using set
736 const unordered_set<int> uniqueLostAtomIds(socketLossInfo.atomIds.begin(),
737 socketLossInfo.atomIds.end());
738
739 // pass lost atom id to all relevant metrics
740 for (const auto lostAtomId : uniqueLostAtomIds) {
741 /**
742 * Socket loss atom:
743 * - comes from a specific uid (originUid)
744 * - specifies the uid in the atom payload (socketLossInfo.uid)
745 * - provides a list of atom ids that are lost
746 *
747 * For atom id that is lost (lostAtomId below):
748 * - if that atom id is allowed from any uid, then always count this atom as lost
749 * - else, if the originUid (from ucred) (socketLossInfo.uid below and is the same for all
750 * uniqueLostAtomIds) is in the allowed log sources - count this atom as lost
751 */
752
753 if (!checkLogCredentials(socketLossInfo.uid, lostAtomId)) {
754 continue;
755 }
756
757 notifyMetricsAboutLostAtom(lostAtomId, DATA_CORRUPTED_SOCKET_LOSS);
758 }
759 }
760
notifyMetricsAboutLostAtom(int32_t lostAtomId,DataCorruptedReason reason)761 int MetricsManager::notifyMetricsAboutLostAtom(int32_t lostAtomId, DataCorruptedReason reason) {
762 const auto matchersIt = mTagIdsToMatchersMap.find(lostAtomId);
763 if (matchersIt == mTagIdsToMatchersMap.end()) {
764 // atom is lost - but no metrics in config reference it
765 return 0;
766 }
767 int numberOfNotifiedMetrics = 0;
768
769 const auto& matchersIndexesListForLostAtom = matchersIt->second;
770 for (const auto matcherIndex : matchersIndexesListForLostAtom) {
771 // look through any metric which depends on matcher
772 auto metricMapIt = mTrackerToMetricMap.find(matcherIndex);
773 if (metricMapIt != mTrackerToMetricMap.end()) {
774 const auto& metricsList = metricMapIt->second;
775 for (const int metricIndex : metricsList) {
776 mAllMetricProducers[metricIndex]->onMatchedLogEventLost(
777 lostAtomId, reason, MetricProducer::LostAtomType::kWhat);
778 numberOfNotifiedMetrics++;
779 }
780 }
781
782 // look through any condition tracker which depends on matcher
783 const auto conditionMapIt = mTrackerToConditionMap.find(matcherIndex);
784 if (conditionMapIt != mTrackerToConditionMap.end()) {
785 const auto& conditionTrackersList = conditionMapIt->second;
786 for (const int conditionTrackerIndex : conditionTrackersList) {
787 metricMapIt = mConditionToMetricMap.find(conditionTrackerIndex);
788 if (metricMapIt != mConditionToMetricMap.end()) {
789 const auto& metricsList = metricMapIt->second;
790 for (const int metricIndex : metricsList) {
791 mAllMetricProducers[metricIndex]->onMatchedLogEventLost(
792 lostAtomId, reason, MetricProducer::LostAtomType::kCondition);
793 numberOfNotifiedMetrics++;
794 }
795 }
796 }
797 }
798 }
799 return numberOfNotifiedMetrics;
800 }
801
onAnomalyAlarmFired(const int64_t timestampNs,unordered_set<sp<const InternalAlarm>,SpHash<InternalAlarm>> & alarmSet)802 void MetricsManager::onAnomalyAlarmFired(
803 const int64_t timestampNs,
804 unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>>& alarmSet) {
805 for (const auto& itr : mAllAnomalyTrackers) {
806 itr->informAlarmsFired(timestampNs, alarmSet);
807 }
808 }
809
onPeriodicAlarmFired(const int64_t timestampNs,unordered_set<sp<const InternalAlarm>,SpHash<InternalAlarm>> & alarmSet)810 void MetricsManager::onPeriodicAlarmFired(
811 const int64_t timestampNs,
812 unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>>& alarmSet) {
813 for (const auto& itr : mAllPeriodicAlarmTrackers) {
814 itr->informAlarmsFired(timestampNs, alarmSet);
815 }
816 }
817
818 // Returns the total byte size of all metrics managed by a single config source.
byteSize()819 size_t MetricsManager::byteSize() {
820 size_t totalSize = 0;
821 for (const auto& metricProducer : mAllMetricProducers) {
822 totalSize += metricProducer->byteSize();
823 }
824 return totalSize;
825 }
826
loadActiveConfig(const ActiveConfig & config,int64_t currentTimeNs)827 void MetricsManager::loadActiveConfig(const ActiveConfig& config, int64_t currentTimeNs) {
828 if (config.metric_size() == 0) {
829 ALOGW("No active metric for config %s", mConfigKey.ToString().c_str());
830 return;
831 }
832
833 for (int i = 0; i < config.metric_size(); i++) {
834 const auto& activeMetric = config.metric(i);
835 for (int metricIndex : mMetricIndexesWithActivation) {
836 const auto& metric = mAllMetricProducers[metricIndex];
837 if (metric->getMetricId() == activeMetric.id()) {
838 VLOG("Setting active metric: %lld", (long long)metric->getMetricId());
839 metric->loadActiveMetric(activeMetric, currentTimeNs);
840 if (!mIsActive && metric->isActive()) {
841 StatsdStats::getInstance().noteActiveStatusChanged(mConfigKey,
842 /*activate=*/true);
843 }
844 mIsActive |= metric->isActive();
845 }
846 }
847 }
848 }
849
writeActiveConfigToProtoOutputStream(int64_t currentTimeNs,const DumpReportReason reason,ProtoOutputStream * proto)850 void MetricsManager::writeActiveConfigToProtoOutputStream(int64_t currentTimeNs,
851 const DumpReportReason reason,
852 ProtoOutputStream* proto) {
853 proto->write(FIELD_TYPE_INT64 | FIELD_ID_ACTIVE_CONFIG_ID, (long long)mConfigKey.GetId());
854 proto->write(FIELD_TYPE_INT32 | FIELD_ID_ACTIVE_CONFIG_UID, mConfigKey.GetUid());
855 for (int metricIndex : mMetricIndexesWithActivation) {
856 const auto& metric = mAllMetricProducers[metricIndex];
857 const uint64_t metricToken = proto->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
858 FIELD_ID_ACTIVE_CONFIG_METRIC);
859 metric->writeActiveMetricToProtoOutputStream(currentTimeNs, reason, proto);
860 proto->end(metricToken);
861 }
862 }
863
writeMetadataToProto(int64_t currentWallClockTimeNs,int64_t systemElapsedTimeNs,metadata::StatsMetadata * statsMetadata)864 bool MetricsManager::writeMetadataToProto(int64_t currentWallClockTimeNs,
865 int64_t systemElapsedTimeNs,
866 metadata::StatsMetadata* statsMetadata) {
867 bool metadataWritten = false;
868 metadata::ConfigKey* configKey = statsMetadata->mutable_config_key();
869 configKey->set_config_id(mConfigKey.GetId());
870 configKey->set_uid(mConfigKey.GetUid());
871 for (const auto& anomalyTracker : mAllAnomalyTrackers) {
872 metadata::AlertMetadata* alertMetadata = statsMetadata->add_alert_metadata();
873 bool alertWritten = anomalyTracker->writeAlertMetadataToProto(
874 currentWallClockTimeNs, systemElapsedTimeNs, alertMetadata);
875 if (!alertWritten) {
876 statsMetadata->mutable_alert_metadata()->RemoveLast();
877 }
878 metadataWritten |= alertWritten;
879 }
880
881 for (const auto& metricProducer : mAllMetricProducers) {
882 metadata::MetricMetadata* metricMetadata = statsMetadata->add_metric_metadata();
883 bool metricWritten = metricProducer->writeMetricMetadataToProto(metricMetadata);
884 if (!metricWritten) {
885 statsMetadata->mutable_metric_metadata()->RemoveLast();
886 }
887 metadataWritten |= metricWritten;
888 }
889 return metadataWritten;
890 }
891
loadMetadata(const metadata::StatsMetadata & metadata,int64_t currentWallClockTimeNs,int64_t systemElapsedTimeNs)892 void MetricsManager::loadMetadata(const metadata::StatsMetadata& metadata,
893 int64_t currentWallClockTimeNs, int64_t systemElapsedTimeNs) {
894 for (const metadata::AlertMetadata& alertMetadata : metadata.alert_metadata()) {
895 int64_t alertId = alertMetadata.alert_id();
896 const auto& it = mAlertTrackerMap.find(alertId);
897 if (it == mAlertTrackerMap.end()) {
898 ALOGE("No anomalyTracker found for alertId %lld", (long long)alertId);
899 continue;
900 }
901 mAllAnomalyTrackers[it->second]->loadAlertMetadata(alertMetadata, currentWallClockTimeNs,
902 systemElapsedTimeNs);
903 }
904 for (const metadata::MetricMetadata& metricMetadata : metadata.metric_metadata()) {
905 int64_t metricId = metricMetadata.metric_id();
906 const auto& it = mMetricProducerMap.find(metricId);
907 if (it == mMetricProducerMap.end()) {
908 ALOGE("No metricProducer found for metricId %lld", (long long)metricId);
909 }
910 mAllMetricProducers[it->second]->loadMetricMetadataFromProto(metricMetadata);
911 }
912 }
913
enforceRestrictedDataTtls(const int64_t wallClockNs)914 void MetricsManager::enforceRestrictedDataTtls(const int64_t wallClockNs) {
915 if (!hasRestrictedMetricsDelegate()) {
916 return;
917 }
918 sqlite3* db = dbutils::getDb(mConfigKey);
919 if (db == nullptr) {
920 ALOGE("Failed to open sqlite db");
921 dbutils::closeDb(db);
922 return;
923 }
924 for (const auto& producer : mAllMetricProducers) {
925 producer->enforceRestrictedDataTtl(db, wallClockNs);
926 }
927 dbutils::closeDb(db);
928 }
929
validateRestrictedMetricsDelegate(const int32_t callingUid)930 bool MetricsManager::validateRestrictedMetricsDelegate(const int32_t callingUid) {
931 if (!hasRestrictedMetricsDelegate()) {
932 return false;
933 }
934
935 set<int32_t> possibleUids = mUidMap->getAppUid(mRestrictedMetricsDelegatePackageName.value());
936
937 return possibleUids.find(callingUid) != possibleUids.end();
938 }
939
flushRestrictedData()940 void MetricsManager::flushRestrictedData() {
941 if (!hasRestrictedMetricsDelegate()) {
942 return;
943 }
944 int64_t flushStartNs = getElapsedRealtimeNs();
945 for (const auto& producer : mAllMetricProducers) {
946 producer->flushRestrictedData();
947 }
948 StatsdStats::getInstance().noteRestrictedConfigFlushLatency(
949 mConfigKey, getElapsedRealtimeNs() - flushStartNs);
950 }
951
getAllMetricIds() const952 vector<int64_t> MetricsManager::getAllMetricIds() const {
953 vector<int64_t> metricIds;
954 metricIds.reserve(mMetricProducerMap.size());
955 for (const auto& [metricId, _] : mMetricProducerMap) {
956 metricIds.push_back(metricId);
957 }
958 return metricIds;
959 }
960
addAllAtomIds(LogEventFilter::AtomIdSet & allIds) const961 void MetricsManager::addAllAtomIds(LogEventFilter::AtomIdSet& allIds) const {
962 for (const auto& [atomId, _] : mTagIdsToMatchersMap) {
963 allIds.insert(atomId);
964 }
965 }
966
processQueueOverflowStats()967 void MetricsManager::processQueueOverflowStats() {
968 auto queueOverflowStats = StatsdStats::getInstance().getQueueOverflowAtomsStats();
969 assert((queueOverflowStats.size() < mQueueOverflowAtomsStats.size()) &&
970 "StatsdStats reset unexpected");
971
972 for (const auto [atomId, count] : queueOverflowStats) {
973 // are there new atoms dropped due to queue overflow since previous request
974 auto droppedAtomStatsIt = mQueueOverflowAtomsStats.find(atomId);
975 if (droppedAtomStatsIt != mQueueOverflowAtomsStats.end() &&
976 droppedAtomStatsIt->second == count) {
977 // no new dropped atoms detected for the atomId
978 continue;
979 }
980
981 notifyMetricsAboutLostAtom(atomId, DATA_CORRUPTED_EVENT_QUEUE_OVERFLOW);
982 }
983 mQueueOverflowAtomsStats = std::move(queueOverflowStats);
984 }
985
986 } // namespace statsd
987 } // namespace os
988 } // namespace android
989