1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Classes representing individual metrics that can be sent.""" 6 7import re 8 9import six 10 11from infra_libs.ts_mon.protos import metrics_pb2 12 13from infra_libs.ts_mon.common import distribution 14from infra_libs.ts_mon.common import errors 15from infra_libs.ts_mon.common import interface 16 17 18MICROSECONDS_PER_SECOND = 1000000 19 20 21class Field(object): 22 FIELD_NAME_PATTERN = re.compile(r'[A-Za-z_][A-Za-z0-9_]*') 23 24 allowed_python_types = None 25 type_enum = None 26 field_name = None 27 28 def __init__(self, name): 29 if not self.FIELD_NAME_PATTERN.match(name): 30 raise errors.MetricDefinitionError( 31 'Invalid metric field name "%s" - must match the regex "%s"' % ( 32 name, self.FIELD_NAME_PATTERN.pattern)) 33 34 self.name = name 35 36 def __eq__(self, other): 37 return (type(self) == type(other) and 38 self.__dict__ == other.__dict__) 39 40 def validate_value(self, metric_name, value): 41 if not isinstance(value, self.allowed_python_types): 42 raise errors.MonitoringInvalidFieldTypeError( 43 metric_name, self.name, value) 44 45 def populate_proto(self, proto, value): 46 setattr(proto, self.field_name, value) 47 48 49class StringField(Field): 50 allowed_python_types = six.string_types 51 type_enum = metrics_pb2.MetricsDataSet.MetricFieldDescriptor.STRING 52 field_name = 'string_value' 53 54 55class IntegerField(Field): 56 allowed_python_types = six.integer_types 57 type_enum = metrics_pb2.MetricsDataSet.MetricFieldDescriptor.INT64 58 field_name = 'int64_value' 59 60 61class BooleanField(Field): 62 allowed_python_types = bool 63 type_enum = metrics_pb2.MetricsDataSet.MetricFieldDescriptor.BOOL 64 field_name = 'bool_value' 65 66 67class Metric(object): 68 """Abstract base class for a metric. 69 70 A Metric is an attribute that may be monitored across many targets. Examples 71 include disk usage or the number of requests a server has received. A single 72 process may keep track of many metrics. 73 74 Note that Metric objects may be initialized at any time (for example, at the 75 top of a library), but cannot be sent until the underlying Monitor object 76 has been set up (usually by the top-level process parsing the command line). 77 78 A Metric can actually store multiple values that are identified by a set of 79 fields (which are themselves key-value pairs). Fields can be passed to the 80 set() or increment() methods to modify a particular value, or passed to the 81 constructor in which case they will be used as the defaults for this Metric. 82 83 The unit of measurement for Metric data should be specified with 84 MetricsDataUnits when a Metric object is created: 85 e.g., MetricsDataUnits.SECONDS, MetricsDataUnits.BYTES, and etc.., 86 See `MetricsDataUnits` class for a full list of units. 87 88 Do not directly instantiate an object of this class. 89 Use the concrete child classes instead: 90 * StringMetric for metrics with string value 91 * BooleanMetric for metrics with boolean values 92 * CounterMetric for metrics with monotonically increasing integer values 93 * GaugeMetric for metrics with arbitrarily varying integer values 94 * CumulativeMetric for metrics with monotonically increasing float values 95 * FloatMetric for metrics with arbitrarily varying float values 96 97 See http://go/inframon-doc for help designing and using your metrics. 98 """ 99 100 def __init__(self, name, description, field_spec, units=None): 101 """Create an instance of a Metric. 102 103 Args: 104 name (str): the file-like name of this metric 105 description (string): help string for the metric. Should be enough to 106 know what the metric is about. 107 field_spec (list): a list of Field subclasses to define the fields that 108 are allowed on this metric. Pass a list of either 109 StringField, IntegerField or BooleanField here. 110 units (string): the unit used to measure data for given metric. Some 111 common units are pre-defined in the MetricsDataUnits 112 class. 113 """ 114 field_spec = field_spec or [] 115 116 self._name = name.lstrip('/') 117 118 if not isinstance(description, six.string_types): 119 raise errors.MetricDefinitionError('Metric description must be a string') 120 if not description: 121 raise errors.MetricDefinitionError('Metric must have a description') 122 if (not isinstance(field_spec, (list, tuple)) or 123 any(not isinstance(x, Field) for x in field_spec)): 124 raise errors.MetricDefinitionError( 125 'Metric constructor takes a list of Fields, or None') 126 if len(field_spec) > 7: 127 raise errors.MonitoringTooManyFieldsError(self._name, field_spec) 128 129 self._start_time = None 130 self._field_spec = field_spec 131 self._sorted_field_names = sorted(x.name for x in field_spec) 132 self._description = description 133 self._units = units 134 135 interface.register(self) 136 137 def __eq__(self, other): 138 return (type(self) == type(other) 139 and self.__dict__ == other.__dict__) 140 141 @property 142 def field_spec(self): 143 return list(self._field_spec) 144 145 @property 146 def name(self): 147 return self._name 148 149 @property 150 def start_time(self): 151 return self._start_time 152 153 @property 154 def units(self): 155 return self._units 156 157 def is_cumulative(self): 158 raise NotImplementedError() 159 160 def unregister(self): 161 interface.unregister(self) 162 163 def populate_data_set(self, data_set): 164 """Populate MetricsDataSet.""" 165 data_set.metric_name = '%s%s' % (interface.state.metric_name_prefix, 166 self._name) 167 data_set.description = self._description or '' 168 if self._units is not None: 169 data_set.annotations.unit = self._units 170 171 if self.is_cumulative(): 172 data_set.stream_kind = metrics_pb2.CUMULATIVE 173 else: 174 data_set.stream_kind = metrics_pb2.GAUGE 175 176 self._populate_value_type(data_set) 177 self._populate_field_descriptors(data_set) 178 179 def populate_data(self, data, start_time, end_time, fields, value): 180 """Populate a new metrics_pb2.MetricsData. 181 182 Args: 183 data (metrics_pb2.MetricsData): protocol buffer into 184 which to populate the current metric values. 185 start_time (int): timestamp in microseconds since UNIX epoch. 186 """ 187 data.start_timestamp.seconds = int(start_time) 188 data.end_timestamp.seconds = int(end_time) 189 190 self._populate_fields(data, fields) 191 self._populate_value(data, value) 192 193 def _populate_field_descriptors(self, data_set): 194 """Populate `field_descriptor` in MetricsDataSet. 195 196 Args: 197 data_set (metrics_pb2.MetricsDataSet): a data set protobuf to populate 198 """ 199 for spec in self._field_spec: 200 descriptor = data_set.field_descriptor.add() 201 descriptor.name = spec.name 202 descriptor.field_type = spec.type_enum 203 204 def _populate_fields(self, data, field_values): 205 """Fill in the fields attribute of a metric protocol buffer. 206 207 Args: 208 metric (metrics_pb2.MetricsData): a metrics protobuf to populate 209 field_values (tuple): field values 210 """ 211 for spec, value in zip(self._field_spec, field_values): 212 field = data.field.add() 213 field.name = spec.name 214 spec.populate_proto(field, value) 215 216 def _validate_fields(self, fields): 217 """Checks the correct number and types of field values were provided. 218 219 Args: 220 fields (dict): A dict of field values given by the user, or None. 221 222 Returns: 223 fields' values as a tuple, in the same order as the field_spec. 224 225 Raises: 226 WrongFieldsError: if you provide a different number of fields to those 227 the metric was defined with. 228 MonitoringInvalidFieldTypeError: if the field value was the wrong type for 229 the field spec. 230 """ 231 fields = fields or {} 232 233 if not isinstance(fields, dict): 234 raise ValueError('fields should be a dict, got %r (%s)' % ( 235 fields, type(fields))) 236 237 if sorted(fields) != self._sorted_field_names: 238 raise errors.WrongFieldsError( 239 self.name, fields.keys(), self._sorted_field_names) 240 241 for spec in self._field_spec: 242 spec.validate_value(self.name, fields[spec.name]) 243 244 return tuple(fields[spec.name] for spec in self._field_spec) 245 246 def _populate_value(self, data, value): 247 """Fill in the the data values of a metric protocol buffer. 248 249 Args: 250 data (metrics_pb2.MetricsData): a metrics protobuf to populate 251 value (see concrete class): the value of the metric to be set 252 """ 253 raise NotImplementedError() 254 255 def _populate_value_type(self, data_set): 256 """Fill in the the data values of a metric protocol buffer. 257 258 Args: 259 data_set (metrics_pb2.MetricsDataSet): a MetricsDataSet protobuf to 260 populate 261 """ 262 raise NotImplementedError() 263 264 def set(self, value, fields=None, target_fields=None): 265 """Set a new value for this metric. Results in sending a new value. 266 267 The subclass should do appropriate type checking on value and then call 268 self._set_and_send_value. 269 270 Args: 271 value (see concrete class): the value of the metric to be set 272 fields (dict): metric field values 273 target_fields (dict): overwrite some of the default target fields 274 """ 275 raise NotImplementedError() 276 277 def get(self, fields=None, target_fields=None): 278 """Returns the current value for this metric. 279 280 Subclasses should never use this to get a value, modify it and set it again. 281 Instead use _incr with a modify_fn. 282 """ 283 return interface.state.store.get( 284 self.name, self._validate_fields(fields), target_fields) 285 286 def get_all(self): 287 return interface.state.store.iter_field_values(self.name) 288 289 def reset(self): 290 """Clears the values of this metric. Useful in unit tests. 291 292 It might be easier to call ts_mon.reset_for_unittest() in your setUp() 293 method instead of resetting every individual metric. 294 """ 295 296 interface.state.store.reset_for_unittest(self.name) 297 298 def _set(self, fields, target_fields, value, enforce_ge=False): 299 interface.state.store.set( 300 self.name, self._validate_fields(fields), target_fields, 301 value, enforce_ge=enforce_ge) 302 303 def _incr(self, fields, target_fields, delta, modify_fn=None): 304 interface.state.store.incr( 305 self.name, self._validate_fields(fields), target_fields, 306 delta, modify_fn=modify_fn) 307 308 309class StringMetric(Metric): 310 """A metric whose value type is a string.""" 311 312 def _populate_value(self, data, value): 313 data.string_value = value 314 315 def _populate_value_type(self, data_set): 316 data_set.value_type = metrics_pb2.STRING 317 318 def set(self, value, fields=None, target_fields=None): 319 if not isinstance(value, six.string_types): 320 raise errors.MonitoringInvalidValueTypeError(self._name, value) 321 self._set(fields, target_fields, value) 322 323 def is_cumulative(self): 324 return False 325 326 327class BooleanMetric(Metric): 328 """A metric whose value type is a boolean.""" 329 330 def _populate_value(self, data, value): 331 data.bool_value = value 332 333 def _populate_value_type(self, data_set): 334 data_set.value_type = metrics_pb2.BOOL 335 336 def set(self, value, fields=None, target_fields=None): 337 if not isinstance(value, bool): 338 raise errors.MonitoringInvalidValueTypeError(self._name, value) 339 self._set(fields, target_fields, value) 340 341 def is_cumulative(self): 342 return False 343 344 345class NumericMetric(Metric): # pylint: disable=abstract-method 346 """Abstract base class for numeric (int or float) metrics.""" 347 348 def increment(self, fields=None, target_fields=None): 349 self._incr(fields, target_fields, 1) 350 351 def increment_by(self, step, fields=None, target_fields=None): 352 self._incr(fields, target_fields, step) 353 354 355class CounterMetric(NumericMetric): 356 """A metric whose value type is a monotonically increasing integer.""" 357 358 def __init__(self, name, description, field_spec, start_time=None, 359 units=None): 360 self._start_time = start_time 361 super(CounterMetric, self).__init__( 362 name, description, field_spec, units=units) 363 364 def _populate_value(self, data, value): 365 data.int64_value = value 366 367 def _populate_value_type(self, data_set): 368 data_set.value_type = metrics_pb2.INT64 369 370 def set(self, value, fields=None, target_fields=None): 371 if not isinstance(value, six.integer_types): 372 raise errors.MonitoringInvalidValueTypeError(self._name, value) 373 self._set(fields, target_fields, value, enforce_ge=True) 374 375 def increment_by(self, step, fields=None, target_fields=None): 376 if not isinstance(step, six.integer_types): 377 raise errors.MonitoringInvalidValueTypeError(self._name, step) 378 self._incr(fields, target_fields, step) 379 380 def is_cumulative(self): 381 return True 382 383 384class GaugeMetric(NumericMetric): 385 """A metric whose value type is an integer.""" 386 387 def _populate_value(self, data, value): 388 data.int64_value = value 389 390 def _populate_value_type(self, data_set): 391 data_set.value_type = metrics_pb2.INT64 392 393 def set(self, value, fields=None, target_fields=None): 394 if not isinstance(value, six.integer_types): 395 raise errors.MonitoringInvalidValueTypeError(self._name, value) 396 self._set(fields, target_fields, value) 397 398 def is_cumulative(self): 399 return False 400 401 402class CumulativeMetric(NumericMetric): 403 """A metric whose value type is a monotonically increasing float.""" 404 405 def __init__(self, name, description, field_spec, start_time=None, 406 units=None): 407 self._start_time = start_time 408 super(CumulativeMetric, self).__init__( 409 name, description, field_spec, units=units) 410 411 def _populate_value(self, data, value): 412 data.double_value = value 413 414 def _populate_value_type(self, data_set): 415 data_set.value_type = metrics_pb2.DOUBLE 416 417 def set(self, value, fields=None, target_fields=None): 418 if not isinstance(value, (float, int)): 419 raise errors.MonitoringInvalidValueTypeError(self._name, value) 420 self._set(fields, target_fields, float(value), enforce_ge=True) 421 422 def is_cumulative(self): 423 return True 424 425 426class FloatMetric(NumericMetric): 427 """A metric whose value type is a float.""" 428 429 def _populate_value(self, metric, value): 430 metric.double_value = value 431 432 def _populate_value_type(self, data_set_pb): 433 data_set_pb.value_type = metrics_pb2.DOUBLE 434 435 def set(self, value, fields=None, target_fields=None): 436 if not isinstance(value, (float, int)): 437 raise errors.MonitoringInvalidValueTypeError(self._name, value) 438 self._set(fields, target_fields, float(value)) 439 440 def is_cumulative(self): 441 return False 442 443 444class _DistributionMetricBase(Metric): 445 """A metric that holds a distribution of values. 446 447 By default buckets are chosen from a geometric progression, each bucket being 448 approximately 1.59 times bigger than the last. In practice this is suitable 449 for many kinds of data, but you may want to provide a FixedWidthBucketer or 450 GeometricBucketer with different parameters.""" 451 452 def __init__(self, name, description, field_spec, is_cumulative=True, 453 bucketer=None, start_time=None, units=None): 454 self._start_time = start_time 455 456 if bucketer is None: 457 bucketer = distribution.GeometricBucketer() 458 459 self._is_cumulative = is_cumulative 460 self.bucketer = bucketer 461 super(_DistributionMetricBase, self).__init__( 462 name, description, field_spec, units=units) 463 464 def _populate_value(self, metric, value): 465 pb = metric.distribution_value 466 467 # Copy the bucketer params. 468 if value.bucketer.width == 0: 469 pb.exponential_buckets.growth_factor = value.bucketer.growth_factor 470 pb.exponential_buckets.scale = value.bucketer.scale 471 pb.exponential_buckets.num_finite_buckets = ( 472 value.bucketer.num_finite_buckets) 473 else: 474 pb.linear_buckets.width = value.bucketer.width 475 pb.linear_buckets.offset = 0.0 476 pb.linear_buckets.num_finite_buckets = value.bucketer.num_finite_buckets 477 478 # Copy the distribution bucket values. Include the overflow buckets on 479 # either end. 480 pb.bucket_count.extend( 481 value.buckets.get(i, 0) for i in 482 range(0, value.bucketer.total_buckets)) 483 484 pb.count = value.count 485 pb.mean = float(value.sum) / max(value.count, 1) 486 487 def _populate_value_type(self, data_set_pb): 488 data_set_pb.value_type = metrics_pb2.DISTRIBUTION 489 490 def add(self, value, fields=None, target_fields=None): 491 def modify_fn(dist, value): 492 if dist == 0: 493 dist = distribution.Distribution(self.bucketer) 494 dist.add(value) 495 return dist 496 497 self._incr(fields, target_fields, value, modify_fn=modify_fn) 498 499 def set(self, value, fields=None, target_fields=None): 500 """Replaces the distribution with the given fields with another one. 501 502 This only makes sense on non-cumulative DistributionMetrics. 503 504 Args: 505 value: A infra_libs.ts_mon.Distribution. 506 """ 507 508 if self._is_cumulative: 509 raise TypeError( 510 'Cannot set() a cumulative DistributionMetric (use add() instead)') 511 512 if not isinstance(value, distribution.Distribution): 513 raise errors.MonitoringInvalidValueTypeError(self._name, value) 514 515 self._set(fields, target_fields, value) 516 517 def is_cumulative(self): 518 return self._is_cumulative 519 520 521class CumulativeDistributionMetric(_DistributionMetricBase): 522 """A DistributionMetric with is_cumulative set to True.""" 523 524 def __init__(self, name, description, field_spec, bucketer=None, units=None): 525 super(CumulativeDistributionMetric, self).__init__( 526 name, description, field_spec, 527 is_cumulative=True, 528 bucketer=bucketer, 529 units=units) 530 531 532class NonCumulativeDistributionMetric(_DistributionMetricBase): 533 """A DistributionMetric with is_cumulative set to False.""" 534 535 def __init__(self, name, description, field_spec, bucketer=None, units=None): 536 super(NonCumulativeDistributionMetric, self).__init__( 537 name, description, field_spec, 538 is_cumulative=False, 539 bucketer=bucketer, 540 units=units) 541 542 543class MetricsDataUnits(object): 544 """An container for units of measurement for Metrics data.""" 545 546 UNKNOWN_UNITS = '{unknown}' 547 SECONDS = 's' 548 MILLISECONDS = 'ms' 549 MICROSECONDS = 'us' 550 NANOSECONDS = 'ns' 551 BITS = 'B' 552 BYTES = 'By' 553 KILOBYTES = 'kBy' 554 MEGABYTES = 'MBy' 555 GIGABYTES = 'GBy' 556 KIBIBYTES = 'kiBy' 557 MEBIBYTES = 'MiBy' 558 GIBIBYTES = 'GiBy' 559 AMPS = 'A' 560 MILLIAMPS = 'mA' 561 DEGREES_CELSIUS = 'Cel' 562