1# -*- coding: utf-8 -*- 2# Copyright 2013 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Table generating, analyzing and printing functions. 7 8This defines several classes that are used to generate, analyze and print 9tables. 10 11Example usage: 12 13 from cros_utils import tabulator 14 15 data = [["benchmark1", "33", "44"],["benchmark2", "44", "33"]] 16 tabulator.GetSimpleTable(data) 17 18You could also use it to generate more complex tables with analysis such as 19p-values, custom colors, etc. Tables are generated by TableGenerator and 20analyzed/formatted by TableFormatter. TableFormatter can take in a list of 21columns with custom result computation and coloring, and will compare values in 22each row according to taht scheme. Here is a complex example on printing a 23table: 24 25 from cros_utils import tabulator 26 27 runs = [[{"k1": "10", "k2": "12", "k5": "40", "k6": "40", 28 "ms_1": "20", "k7": "FAIL", "k8": "PASS", "k9": "PASS", 29 "k10": "0"}, 30 {"k1": "13", "k2": "14", "k3": "15", "ms_1": "10", "k8": "PASS", 31 "k9": "FAIL", "k10": "0"}], 32 [{"k1": "50", "k2": "51", "k3": "52", "k4": "53", "k5": "35", "k6": 33 "45", "ms_1": "200", "ms_2": "20", "k7": "FAIL", "k8": "PASS", "k9": 34 "PASS"}]] 35 labels = ["vanilla", "modified"] 36 tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC) 37 table = tg.GetTable() 38 columns = [Column(LiteralResult(), 39 Format(), 40 "Literal"), 41 Column(AmeanResult(), 42 Format()), 43 Column(StdResult(), 44 Format()), 45 Column(CoeffVarResult(), 46 CoeffVarFormat()), 47 Column(NonEmptyCountResult(), 48 Format()), 49 Column(AmeanRatioResult(), 50 PercentFormat()), 51 Column(AmeanRatioResult(), 52 RatioFormat()), 53 Column(GmeanRatioResult(), 54 RatioFormat()), 55 Column(PValueResult(), 56 PValueFormat()), 57 ] 58 tf = TableFormatter(table, columns) 59 cell_table = tf.GetCellTable() 60 tp = TablePrinter(cell_table, out_to) 61 print tp.Print() 62""" 63 64 65import collections 66import getpass 67import math 68import statistics 69import sys 70from typing import Tuple, Union 71 72from cros_utils import misc 73from cros_utils.email_sender import EmailSender 74import numpy as np 75 76 77def _ttest_ind( 78 sample: Union[np.ndarray, list], baseline: Union[np.ndarray, list] 79) -> Tuple[float, float]: 80 """Independent, two-sided student's T test. 81 82 Reimplementation of scipy.stats.ttest_ind. 83 """ 84 if isinstance(sample, list): 85 sample = np.asarray(sample) 86 if isinstance(baseline, list): 87 baseline = np.asarray(baseline) 88 diff = np.mean(sample) - np.mean(baseline) 89 diff_stderr = np.sqrt(sample.var(ddof=1) + baseline.var(ddof=1)) 90 t_value = np.mean(diff) / (diff_stderr / np.sqrt(len(sample))) 91 samples = _sample_student_t(len(sample), 1000) 92 # Assuming two-sided student's t 93 if t_value < 0: 94 # Lower tail 95 return t_value, 2 * np.sum(samples < t_value) / len(samples) 96 # Upper tail 97 return t_value, 2 * np.sum(samples > t_value) / len(samples) 98 99 100def _sample_student_t( 101 dof: float, num_samples: int 102) -> np.ndarray: 103 # In theory this probably should be memoized. However, 104 # that's a lot of data points to store in memory for 105 # the lifetime of the program? 106 sample_generator = np.random.default_rng() 107 return sample_generator.standard_t(dof, num_samples) 108 109 110def _AllFloat(values): 111 return all([misc.IsFloat(v) for v in values]) 112 113 114def _GetFloats(values): 115 return [float(v) for v in values] 116 117 118def _StripNone(results): 119 res = [] 120 for result in results: 121 if result is not None: 122 res.append(result) 123 return res 124 125 126def _RemoveMinMax(cell, values): 127 if len(values) < 3: 128 print( 129 "WARNING: Values count is less than 3, not ignoring min/max values" 130 ) 131 print("WARNING: Cell name:", cell.name, "Values:", values) 132 return values 133 134 values.remove(min(values)) 135 values.remove(max(values)) 136 return values 137 138 139class TableGenerator(object): 140 """Creates a table from a list of list of dicts. 141 142 The main public function is called GetTable(). 143 """ 144 145 SORT_BY_KEYS = 0 146 SORT_BY_KEYS_DESC = 1 147 SORT_BY_VALUES = 2 148 SORT_BY_VALUES_DESC = 3 149 NO_SORT = 4 150 151 MISSING_VALUE = "x" 152 153 def __init__(self, d, l, sort=NO_SORT, key_name="keys"): 154 self._runs = d 155 self._labels = l 156 self._sort = sort 157 self._key_name = key_name 158 159 def _AggregateKeys(self): 160 keys = collections.OrderedDict() 161 for run_list in self._runs: 162 for run in run_list: 163 keys.update(dict.fromkeys(run.keys())) 164 return list(keys.keys()) 165 166 def _GetHighestValue(self, key): 167 values = [] 168 for run_list in self._runs: 169 for run in run_list: 170 if key in run: 171 values.append(run[key]) 172 values = _StripNone(values) 173 if _AllFloat(values): 174 values = _GetFloats(values) 175 values = [ 176 float(v) 177 for v in values 178 if isinstance(v, float) 179 or isinstance(v, int) 180 or v.lower() in ("nan", "inf") 181 ] 182 if not values: 183 return float("nan") 184 return max(values) 185 186 def _GetLowestValue(self, key): 187 values = [] 188 for run_list in self._runs: 189 for run in run_list: 190 if key in run: 191 values.append(run[key]) 192 values = _StripNone(values) 193 if _AllFloat(values): 194 values = _GetFloats(values) 195 values = [ 196 float(v) 197 for v in values 198 if isinstance(v, float) 199 or isinstance(v, int) 200 or v.lower() in ("nan", "inf") 201 ] 202 if not values: 203 return float("nan") 204 return min(values) 205 206 def _SortKeys(self, keys): 207 if self._sort == self.SORT_BY_KEYS: 208 return sorted(keys) 209 elif self._sort == self.SORT_BY_VALUES: 210 # pylint: disable=unnecessary-lambda 211 return sorted(keys, key=lambda x: self._GetLowestValue(x)) 212 elif self._sort == self.SORT_BY_VALUES_DESC: 213 # pylint: disable=unnecessary-lambda 214 return sorted( 215 keys, key=lambda x: self._GetHighestValue(x), reverse=True 216 ) 217 elif self._sort == self.NO_SORT: 218 return keys 219 else: 220 assert 0, "Unimplemented sort %s" % self._sort 221 222 def _GetKeys(self): 223 keys = self._AggregateKeys() 224 return self._SortKeys(keys) 225 226 def GetTable(self, number_of_rows=sys.maxsize): 227 """Returns a table from a list of list of dicts. 228 229 Examples: 230 We have the following runs: 231 [[{"k1": "v1", "k2": "v2"}, {"k1": "v3"}], 232 [{"k1": "v4", "k4": "v5"}]] 233 and the following labels: 234 ["vanilla", "modified"] 235 it will return: 236 [["Key", "vanilla", "modified"] 237 ["k1", ["v1", "v3"], ["v4"]] 238 ["k2", ["v2"], []] 239 ["k4", [], ["v5"]]] 240 The returned table can then be processed further by other classes in this 241 module. 242 243 The list of list of dicts is passed into the constructor of TableGenerator. 244 This method converts that into a canonical list of lists which represents a 245 table of values. 246 247 Args: 248 number_of_rows: Maximum number of rows to return from the table. 249 250 Returns: 251 A list of lists which is the table. 252 """ 253 keys = self._GetKeys() 254 header = [self._key_name] + self._labels 255 table = [header] 256 rows = 0 257 for k in keys: 258 row = [k] 259 unit = None 260 for run_list in self._runs: 261 v = [] 262 for run in run_list: 263 if k in run: 264 if isinstance(run[k], list): 265 val = run[k][0] 266 unit = run[k][1] 267 else: 268 val = run[k] 269 v.append(val) 270 else: 271 v.append(None) 272 row.append(v) 273 # If we got a 'unit' value, append the units name to the key name. 274 if unit: 275 keyname = row[0] + " (%s) " % unit 276 row[0] = keyname 277 table.append(row) 278 rows += 1 279 if rows == number_of_rows: 280 break 281 return table 282 283 284class SamplesTableGenerator(TableGenerator): 285 """Creates a table with only samples from the results 286 287 The main public function is called GetTable(). 288 289 Different than TableGenerator, self._runs is now a dict of {benchmark: runs} 290 We are expecting there is 'samples' in `runs`. 291 """ 292 293 def __init__(self, run_keyvals, label_list, iter_counts, weights): 294 TableGenerator.__init__( 295 self, run_keyvals, label_list, key_name="Benchmarks" 296 ) 297 self._iter_counts = iter_counts 298 self._weights = weights 299 300 def _GetKeys(self): 301 keys = self._runs.keys() 302 return self._SortKeys(keys) 303 304 def GetTable(self, number_of_rows=sys.maxsize): 305 """Returns a tuple, which contains three args: 306 307 1) a table from a list of list of dicts. 308 2) updated benchmark_results run_keyvals with composite benchmark 309 3) updated benchmark_results iter_count with composite benchmark 310 311 The dict of list of list of dicts is passed into the constructor of 312 SamplesTableGenerator. 313 This method converts that into a canonical list of lists which 314 represents a table of values. 315 316 Examples: 317 We have the following runs: 318 {bench1: [[{"samples": "v1"}, {"samples": "v2"}], 319 [{"samples": "v3"}, {"samples": "v4"}]] 320 bench2: [[{"samples": "v21"}, None], 321 [{"samples": "v22"}, {"samples": "v23"}]]} 322 and weights of benchmarks: 323 {bench1: w1, bench2: w2} 324 and the following labels: 325 ["vanilla", "modified"] 326 it will return: 327 [["Benchmark", "Weights", "vanilla", "modified"] 328 ["bench1", w1, 329 ((2, 0), ["v1*w1", "v2*w1"]), ((2, 0), ["v3*w1", "v4*w1"])] 330 ["bench2", w2, 331 ((1, 1), ["v21*w2", None]), ((2, 0), ["v22*w2", "v23*w2"])] 332 ["Composite Benchmark", N/A, 333 ((1, 1), ["v1*w1+v21*w2", None]), 334 ((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]] 335 The returned table can then be processed further by other classes in this 336 module. 337 338 Args: 339 number_of_rows: Maximum number of rows to return from the table. 340 341 Returns: 342 A list of lists which is the table. 343 """ 344 keys = self._GetKeys() 345 header = [self._key_name, "Weights"] + self._labels 346 table = [header] 347 rows = 0 348 iterations = 0 349 350 for k in keys: 351 bench_runs = self._runs[k] 352 unit = None 353 all_runs_empty = all( 354 not dict for label in bench_runs for dict in label 355 ) 356 if all_runs_empty: 357 cell = Cell() 358 cell.string_value = ( 359 "Benchmark %s contains no result." 360 " Is the benchmark name valid?" % k 361 ) 362 table.append([cell]) 363 else: 364 row = [k] 365 row.append(self._weights[k]) 366 for run_list in bench_runs: 367 run_pass = 0 368 run_fail = 0 369 v = [] 370 for run in run_list: 371 if "samples" in run: 372 if isinstance(run["samples"], list): 373 val = run["samples"][0] * self._weights[k] 374 unit = run["samples"][1] 375 else: 376 val = run["samples"] * self._weights[k] 377 v.append(val) 378 run_pass += 1 379 else: 380 v.append(None) 381 run_fail += 1 382 one_tuple = ((run_pass, run_fail), v) 383 if iterations not in (0, run_pass + run_fail): 384 raise ValueError( 385 "Iterations of each benchmark run " 386 "are not the same" 387 ) 388 iterations = run_pass + run_fail 389 row.append(one_tuple) 390 if unit: 391 keyname = row[0] + " (%s) " % unit 392 row[0] = keyname 393 table.append(row) 394 rows += 1 395 if rows == number_of_rows: 396 break 397 398 k = "Composite Benchmark" 399 if k in keys: 400 raise RuntimeError("Composite benchmark already exists in results") 401 402 # Create a new composite benchmark row at the bottom of the summary table 403 # The new row will be like the format in example: 404 # ["Composite Benchmark", N/A, 405 # ((1, 1), ["v1*w1+v21*w2", None]), 406 # ((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]] 407 # First we will create a row of [key, weight, [[0] * iterations] * labels] 408 row = [None] * len(header) 409 row[0] = "%s (samples)" % k 410 row[1] = "N/A" 411 for label_index in range(2, len(row)): 412 row[label_index] = [0] * iterations 413 414 for cur_row in table[1:]: 415 # Iterate through each benchmark 416 if len(cur_row) > 1: 417 for label_index in range(2, len(cur_row)): 418 # Iterate through each run in a single benchmark 419 # each result should look like ((pass, fail), [values_list]) 420 bench_runs = cur_row[label_index][1] 421 for index in range(iterations): 422 # Accumulate each run result to composite benchmark run 423 # If any run fails, then we set this run for composite benchmark 424 # to None so that we know it fails. 425 if ( 426 bench_runs[index] 427 and row[label_index][index] is not None 428 ): 429 row[label_index][index] += bench_runs[index] 430 else: 431 row[label_index][index] = None 432 else: 433 # One benchmark totally fails, no valid data will be in final result 434 for label_index in range(2, len(row)): 435 row[label_index] = [None] * iterations 436 break 437 # Calculate pass and fail count for composite benchmark 438 for label_index in range(2, len(row)): 439 run_pass = 0 440 run_fail = 0 441 for run in row[label_index]: 442 if run: 443 run_pass += 1 444 else: 445 run_fail += 1 446 row[label_index] = ((run_pass, run_fail), row[label_index]) 447 table.append(row) 448 449 # Now that we have the table genearted, we want to store this new composite 450 # benchmark into the benchmark_result in ResultReport object. 451 # This will be used to generate a full table which contains our composite 452 # benchmark. 453 # We need to create composite benchmark result and add it to keyvals in 454 # benchmark_results. 455 v = [] 456 for label in row[2:]: 457 # each label's result looks like ((pass, fail), [values]) 458 benchmark_runs = label[1] 459 # List of values of each label 460 single_run_list = [] 461 for run in benchmark_runs: 462 # Result of each run under the same label is a dict of keys. 463 # Here the only key we will add for composite benchmark is the 464 # weighted_samples we added up. 465 one_dict = {} 466 if run: 467 one_dict["weighted_samples"] = [run, "samples"] 468 one_dict["retval"] = 0 469 else: 470 one_dict["retval"] = 1 471 single_run_list.append(one_dict) 472 v.append(single_run_list) 473 474 self._runs[k] = v 475 self._iter_counts[k] = iterations 476 477 return (table, self._runs, self._iter_counts) 478 479 480class Result(object): 481 """A class that respresents a single result. 482 483 This single result is obtained by condensing the information from a list of 484 runs and a list of baseline runs. 485 """ 486 487 def __init__(self): 488 pass 489 490 def _AllStringsSame(self, values): 491 values_set = set(values) 492 return len(values_set) == 1 493 494 def NeedsBaseline(self): 495 return False 496 497 # pylint: disable=unused-argument 498 def _Literal(self, cell, values, baseline_values): 499 cell.value = " ".join([str(v) for v in values]) 500 501 def _ComputeFloat(self, cell, values, baseline_values): 502 self._Literal(cell, values, baseline_values) 503 504 def _ComputeString(self, cell, values, baseline_values): 505 self._Literal(cell, values, baseline_values) 506 507 def _InvertIfLowerIsBetter(self, cell): 508 pass 509 510 def _GetGmean(self, values): 511 if not values: 512 return float("nan") 513 if any([v < 0 for v in values]): 514 return float("nan") 515 if any([v == 0 for v in values]): 516 return 0.0 517 log_list = [math.log(v) for v in values] 518 gmean_log = sum(log_list) / len(log_list) 519 return math.exp(gmean_log) 520 521 def Compute(self, cell, values, baseline_values): 522 """Compute the result given a list of values and baseline values. 523 524 Args: 525 cell: A cell data structure to populate. 526 values: List of values. 527 baseline_values: List of baseline values. Can be none if this is the 528 baseline itself. 529 """ 530 all_floats = True 531 values = _StripNone(values) 532 if not values: 533 cell.value = "" 534 return 535 if _AllFloat(values): 536 float_values = _GetFloats(values) 537 else: 538 all_floats = False 539 if baseline_values: 540 baseline_values = _StripNone(baseline_values) 541 if baseline_values: 542 if _AllFloat(baseline_values): 543 float_baseline_values = _GetFloats(baseline_values) 544 else: 545 all_floats = False 546 else: 547 if self.NeedsBaseline(): 548 cell.value = "" 549 return 550 float_baseline_values = None 551 if all_floats: 552 self._ComputeFloat(cell, float_values, float_baseline_values) 553 self._InvertIfLowerIsBetter(cell) 554 else: 555 self._ComputeString(cell, values, baseline_values) 556 557 558class LiteralResult(Result): 559 """A literal result.""" 560 561 def __init__(self, iteration=0): 562 super(LiteralResult, self).__init__() 563 self.iteration = iteration 564 565 def Compute(self, cell, values, baseline_values): 566 try: 567 cell.value = values[self.iteration] 568 except IndexError: 569 cell.value = "-" 570 571 572class NonEmptyCountResult(Result): 573 """A class that counts the number of non-empty results. 574 575 The number of non-empty values will be stored in the cell. 576 """ 577 578 def Compute(self, cell, values, baseline_values): 579 """Put the number of non-empty values in the cell result. 580 581 Args: 582 cell: Put the result in cell.value. 583 values: A list of values for the row. 584 baseline_values: A list of baseline values for the row. 585 """ 586 cell.value = len(_StripNone(values)) 587 if not baseline_values: 588 return 589 base_value = len(_StripNone(baseline_values)) 590 if cell.value == base_value: 591 return 592 f = ColorBoxFormat() 593 len_values = len(values) 594 len_baseline_values = len(baseline_values) 595 tmp_cell = Cell() 596 tmp_cell.value = 1.0 + ( 597 float(cell.value - base_value) 598 / (max(len_values, len_baseline_values)) 599 ) 600 f.Compute(tmp_cell) 601 cell.bgcolor = tmp_cell.bgcolor 602 603 604class StringMeanResult(Result): 605 """Mean of string values.""" 606 607 def _ComputeString(self, cell, values, baseline_values): 608 if self._AllStringsSame(values): 609 cell.value = str(values[0]) 610 else: 611 cell.value = "?" 612 613 614class AmeanResult(StringMeanResult): 615 """Arithmetic mean.""" 616 617 def __init__(self, ignore_min_max=False): 618 super(AmeanResult, self).__init__() 619 self.ignore_min_max = ignore_min_max 620 621 def _ComputeFloat(self, cell, values, baseline_values): 622 if self.ignore_min_max: 623 values = _RemoveMinMax(cell, values) 624 cell.value = statistics.mean(values) 625 626 627class RawResult(Result): 628 """Raw result.""" 629 630 631class IterationResult(Result): 632 """Iteration result.""" 633 634 635class MinResult(Result): 636 """Minimum.""" 637 638 def _ComputeFloat(self, cell, values, baseline_values): 639 cell.value = min(values) 640 641 def _ComputeString(self, cell, values, baseline_values): 642 if values: 643 cell.value = min(values) 644 else: 645 cell.value = "" 646 647 648class MaxResult(Result): 649 """Maximum.""" 650 651 def _ComputeFloat(self, cell, values, baseline_values): 652 cell.value = max(values) 653 654 def _ComputeString(self, cell, values, baseline_values): 655 if values: 656 cell.value = max(values) 657 else: 658 cell.value = "" 659 660 661class NumericalResult(Result): 662 """Numerical result.""" 663 664 def _ComputeString(self, cell, values, baseline_values): 665 cell.value = "?" 666 667 668class StdResult(NumericalResult): 669 """Standard deviation.""" 670 671 def __init__(self, ignore_min_max=False): 672 super(StdResult, self).__init__() 673 self.ignore_min_max = ignore_min_max 674 675 def _ComputeFloat(self, cell, values, baseline_values): 676 if self.ignore_min_max: 677 values = _RemoveMinMax(cell, values) 678 cell.value = statistics.pstdev(values) 679 680 681class CoeffVarResult(NumericalResult): 682 """Standard deviation / Mean""" 683 684 def __init__(self, ignore_min_max=False): 685 super(CoeffVarResult, self).__init__() 686 self.ignore_min_max = ignore_min_max 687 688 def _ComputeFloat(self, cell, values, baseline_values): 689 if self.ignore_min_max: 690 values = _RemoveMinMax(cell, values) 691 if statistics.mean(values) != 0.0: 692 noise = abs(statistics.pstdev(values) / statistics.mean(values)) 693 else: 694 noise = 0.0 695 cell.value = noise 696 697 698class ComparisonResult(Result): 699 """Same or Different.""" 700 701 def NeedsBaseline(self): 702 return True 703 704 def _ComputeString(self, cell, values, baseline_values): 705 value = None 706 baseline_value = None 707 if self._AllStringsSame(values): 708 value = values[0] 709 if self._AllStringsSame(baseline_values): 710 baseline_value = baseline_values[0] 711 if value is not None and baseline_value is not None: 712 if value == baseline_value: 713 cell.value = "SAME" 714 else: 715 cell.value = "DIFFERENT" 716 else: 717 cell.value = "?" 718 719 720class PValueResult(ComparisonResult): 721 """P-value.""" 722 723 def __init__(self, ignore_min_max=False): 724 super(PValueResult, self).__init__() 725 self.ignore_min_max = ignore_min_max 726 727 def _ComputeFloat(self, cell, values, baseline_values): 728 if self.ignore_min_max: 729 values = _RemoveMinMax(cell, values) 730 baseline_values = _RemoveMinMax(cell, baseline_values) 731 if len(values) < 2 or len(baseline_values) < 2: 732 cell.value = float("nan") 733 return 734 _, cell.value = _ttest_ind(values, baseline_values) 735 736 def _ComputeString(self, cell, values, baseline_values): 737 return float("nan") 738 739 740class KeyAwareComparisonResult(ComparisonResult): 741 """Automatic key aware comparison.""" 742 743 def _IsLowerBetter(self, key): 744 # Units in histograms should include directions 745 if "smallerIsBetter" in key: 746 return True 747 if "biggerIsBetter" in key: 748 return False 749 750 # For units in chartjson: 751 # TODO(llozano): Trying to guess direction by looking at the name of the 752 # test does not seem like a good idea. Test frameworks should provide this 753 # info explicitly. I believe Telemetry has this info. Need to find it out. 754 # 755 # Below are some test names for which we are not sure what the 756 # direction is. 757 # 758 # For these we dont know what the direction is. But, since we dont 759 # specify anything, crosperf will assume higher is better: 760 # --percent_impl_scrolled--percent_impl_scrolled--percent 761 # --solid_color_tiles_analyzed--solid_color_tiles_analyzed--count 762 # --total_image_cache_hit_count--total_image_cache_hit_count--count 763 # --total_texture_upload_time_by_url 764 # 765 # About these we are doubtful but we made a guess: 766 # --average_num_missing_tiles_by_url--*--units (low is good) 767 # --experimental_mean_frame_time_by_url--*--units (low is good) 768 # --experimental_median_frame_time_by_url--*--units (low is good) 769 # --texture_upload_count--texture_upload_count--count (high is good) 770 # --total_deferred_image_decode_count--count (low is good) 771 # --total_tiles_analyzed--total_tiles_analyzed--count (high is good) 772 lower_is_better_keys = [ 773 "milliseconds", 774 "ms_", 775 "seconds_", 776 "KB", 777 "rdbytes", 778 "wrbytes", 779 "dropped_percent", 780 "(ms)", 781 "(seconds)", 782 "--ms", 783 "--average_num_missing_tiles", 784 "--experimental_jank", 785 "--experimental_mean_frame", 786 "--experimental_median_frame_time", 787 "--total_deferred_image_decode_count", 788 "--seconds", 789 "samples", 790 "bytes", 791 ] 792 793 return any([l in key for l in lower_is_better_keys]) 794 795 def _InvertIfLowerIsBetter(self, cell): 796 if self._IsLowerBetter(cell.name): 797 if cell.value: 798 cell.value = 1.0 / cell.value 799 800 801class AmeanRatioResult(KeyAwareComparisonResult): 802 """Ratio of arithmetic means of values vs. baseline values.""" 803 804 def __init__(self, ignore_min_max=False): 805 super(AmeanRatioResult, self).__init__() 806 self.ignore_min_max = ignore_min_max 807 808 def _ComputeFloat(self, cell, values, baseline_values): 809 if self.ignore_min_max: 810 values = _RemoveMinMax(cell, values) 811 baseline_values = _RemoveMinMax(cell, baseline_values) 812 813 baseline_mean = statistics.mean(baseline_values) 814 values_mean = statistics.mean(values) 815 if baseline_mean != 0: 816 cell.value = values_mean / baseline_mean 817 elif values_mean != 0: 818 cell.value = 0.00 819 # cell.value = 0 means the values and baseline_values have big difference 820 else: 821 cell.value = 1.00 822 # no difference if both values and baseline_values are 0 823 824 825class GmeanRatioResult(KeyAwareComparisonResult): 826 """Ratio of geometric means of values vs. baseline values.""" 827 828 def __init__(self, ignore_min_max=False): 829 super(GmeanRatioResult, self).__init__() 830 self.ignore_min_max = ignore_min_max 831 832 def _ComputeFloat(self, cell, values, baseline_values): 833 if self.ignore_min_max: 834 values = _RemoveMinMax(cell, values) 835 baseline_values = _RemoveMinMax(cell, baseline_values) 836 if self._GetGmean(baseline_values) != 0: 837 cell.value = self._GetGmean(values) / self._GetGmean( 838 baseline_values 839 ) 840 elif self._GetGmean(values) != 0: 841 cell.value = 0.00 842 else: 843 cell.value = 1.00 844 845 846class Color(object): 847 """Class that represents color in RGBA format.""" 848 849 def __init__(self, r=0, g=0, b=0, a=0): 850 self.r = r 851 self.g = g 852 self.b = b 853 self.a = a 854 855 def __str__(self): 856 return "r: %s g: %s: b: %s: a: %s" % (self.r, self.g, self.b, self.a) 857 858 def Round(self): 859 """Round RGBA values to the nearest integer.""" 860 self.r = int(self.r) 861 self.g = int(self.g) 862 self.b = int(self.b) 863 self.a = int(self.a) 864 865 def GetRGB(self): 866 """Get a hex representation of the color.""" 867 return "%02x%02x%02x" % (self.r, self.g, self.b) 868 869 @classmethod 870 def Lerp(cls, ratio, a, b): 871 """Perform linear interpolation between two colors. 872 873 Args: 874 ratio: The ratio to use for linear polation. 875 a: The first color object (used when ratio is 0). 876 b: The second color object (used when ratio is 1). 877 878 Returns: 879 Linearly interpolated color. 880 """ 881 ret = cls() 882 ret.r = (b.r - a.r) * ratio + a.r 883 ret.g = (b.g - a.g) * ratio + a.g 884 ret.b = (b.b - a.b) * ratio + a.b 885 ret.a = (b.a - a.a) * ratio + a.a 886 return ret 887 888 889class Format(object): 890 """A class that represents the format of a column.""" 891 892 def __init__(self): 893 pass 894 895 def Compute(self, cell): 896 """Computes the attributes of a cell based on its value. 897 898 Attributes typically are color, width, etc. 899 900 Args: 901 cell: The cell whose attributes are to be populated. 902 """ 903 if cell.value is None: 904 cell.string_value = "" 905 if isinstance(cell.value, float): 906 self._ComputeFloat(cell) 907 else: 908 self._ComputeString(cell) 909 910 def _ComputeFloat(self, cell): 911 cell.string_value = "{0:.2f}".format(cell.value) 912 913 def _ComputeString(self, cell): 914 cell.string_value = str(cell.value) 915 916 def _GetColor(self, value, low, mid, high, power=6, mid_value=1.0): 917 min_value = 0.0 918 max_value = 2.0 919 if math.isnan(value): 920 return mid 921 if value > mid_value: 922 value = max_value - mid_value / value 923 924 return self._GetColorBetweenRange( 925 value, min_value, mid_value, max_value, low, mid, high, power 926 ) 927 928 def _GetColorBetweenRange( 929 self, 930 value, 931 min_value, 932 mid_value, 933 max_value, 934 low_color, 935 mid_color, 936 high_color, 937 power, 938 ): 939 assert value <= max_value 940 assert value >= min_value 941 if value > mid_value: 942 value = (max_value - value) / (max_value - mid_value) 943 value **= power 944 ret = Color.Lerp(value, high_color, mid_color) 945 else: 946 value = (value - min_value) / (mid_value - min_value) 947 value **= power 948 ret = Color.Lerp(value, low_color, mid_color) 949 ret.Round() 950 return ret 951 952 953class PValueFormat(Format): 954 """Formatting for p-value.""" 955 956 def _ComputeFloat(self, cell): 957 cell.string_value = "%0.2f" % float(cell.value) 958 if float(cell.value) < 0.05: 959 cell.bgcolor = self._GetColor( 960 cell.value, 961 Color(255, 255, 0, 0), 962 Color(255, 255, 255, 0), 963 Color(255, 255, 255, 0), 964 mid_value=0.05, 965 power=1, 966 ) 967 968 969class WeightFormat(Format): 970 """Formatting for weight in cwp mode.""" 971 972 def _ComputeFloat(self, cell): 973 cell.string_value = "%0.4f" % float(cell.value) 974 975 976class StorageFormat(Format): 977 """Format the cell as a storage number. 978 979 Examples: 980 If the cell contains a value of 1024, the string_value will be 1.0K. 981 """ 982 983 def _ComputeFloat(self, cell): 984 base = 1024 985 suffices = ["K", "M", "G"] 986 v = float(cell.value) 987 current = 0 988 while v >= base ** (current + 1) and current < len(suffices): 989 current += 1 990 991 if current: 992 divisor = base**current 993 cell.string_value = "%1.1f%s" % ( 994 (v / divisor), 995 suffices[current - 1], 996 ) 997 else: 998 cell.string_value = str(cell.value) 999 1000 1001class CoeffVarFormat(Format): 1002 """Format the cell as a percent. 1003 1004 Examples: 1005 If the cell contains a value of 1.5, the string_value will be +150%. 1006 """ 1007 1008 def _ComputeFloat(self, cell): 1009 cell.string_value = "%1.1f%%" % (float(cell.value) * 100) 1010 cell.color = self._GetColor( 1011 cell.value, 1012 Color(0, 255, 0, 0), 1013 Color(0, 0, 0, 0), 1014 Color(255, 0, 0, 0), 1015 mid_value=0.02, 1016 power=1, 1017 ) 1018 1019 1020class PercentFormat(Format): 1021 """Format the cell as a percent. 1022 1023 Examples: 1024 If the cell contains a value of 1.5, the string_value will be +50%. 1025 """ 1026 1027 def _ComputeFloat(self, cell): 1028 cell.string_value = "%+1.1f%%" % ((float(cell.value) - 1) * 100) 1029 cell.color = self._GetColor( 1030 cell.value, 1031 Color(255, 0, 0, 0), 1032 Color(0, 0, 0, 0), 1033 Color(0, 255, 0, 0), 1034 ) 1035 1036 1037class RatioFormat(Format): 1038 """Format the cell as a ratio. 1039 1040 Examples: 1041 If the cell contains a value of 1.5642, the string_value will be 1.56. 1042 """ 1043 1044 def _ComputeFloat(self, cell): 1045 cell.string_value = "%+1.1f%%" % ((cell.value - 1) * 100) 1046 cell.color = self._GetColor( 1047 cell.value, 1048 Color(255, 0, 0, 0), 1049 Color(0, 0, 0, 0), 1050 Color(0, 255, 0, 0), 1051 ) 1052 1053 1054class ColorBoxFormat(Format): 1055 """Format the cell as a color box. 1056 1057 Examples: 1058 If the cell contains a value of 1.5, it will get a green color. 1059 If the cell contains a value of 0.5, it will get a red color. 1060 The intensity of the green/red will be determined by how much above or below 1061 1.0 the value is. 1062 """ 1063 1064 def _ComputeFloat(self, cell): 1065 cell.string_value = "--" 1066 bgcolor = self._GetColor( 1067 cell.value, 1068 Color(255, 0, 0, 0), 1069 Color(255, 255, 255, 0), 1070 Color(0, 255, 0, 0), 1071 ) 1072 cell.bgcolor = bgcolor 1073 cell.color = bgcolor 1074 1075 1076class Cell(object): 1077 """A class to represent a cell in a table. 1078 1079 Attributes: 1080 value: The raw value of the cell. 1081 color: The color of the cell. 1082 bgcolor: The background color of the cell. 1083 string_value: The string value of the cell. 1084 suffix: A string suffix to be attached to the value when displaying. 1085 prefix: A string prefix to be attached to the value when displaying. 1086 color_row: Indicates whether the whole row is to inherit this cell's color. 1087 bgcolor_row: Indicates whether the whole row is to inherit this cell's 1088 bgcolor. 1089 width: Optional specifier to make a column narrower than the usual width. 1090 The usual width of a column is the max of all its cells widths. 1091 colspan: Set the colspan of the cell in the HTML table, this is used for 1092 table headers. Default value is 1. 1093 name: the test name of the cell. 1094 header: Whether this is a header in html. 1095 """ 1096 1097 def __init__(self): 1098 self.value = None 1099 self.color = None 1100 self.bgcolor = None 1101 self.string_value = None 1102 self.suffix = None 1103 self.prefix = None 1104 # Entire row inherits this color. 1105 self.color_row = False 1106 self.bgcolor_row = False 1107 self.width = 0 1108 self.colspan = 1 1109 self.name = None 1110 self.header = False 1111 1112 def __str__(self): 1113 l = [] 1114 l.append("value: %s" % self.value) 1115 l.append("string_value: %s" % self.string_value) 1116 return " ".join(l) 1117 1118 1119class Column(object): 1120 """Class representing a column in a table. 1121 1122 Attributes: 1123 result: an object of the Result class. 1124 fmt: an object of the Format class. 1125 """ 1126 1127 def __init__(self, result, fmt, name=""): 1128 self.result = result 1129 self.fmt = fmt 1130 self.name = name 1131 1132 1133# Takes in: 1134# ["Key", "Label1", "Label2"] 1135# ["k", ["v", "v2"], [v3]] 1136# etc. 1137# Also takes in a format string. 1138# Returns a table like: 1139# ["Key", "Label1", "Label2"] 1140# ["k", avg("v", "v2"), stddev("v", "v2"), etc.]] 1141# according to format string 1142class TableFormatter(object): 1143 """Class to convert a plain table into a cell-table. 1144 1145 This class takes in a table generated by TableGenerator and a list of column 1146 formats to apply to the table and returns a table of cells. 1147 """ 1148 1149 def __init__(self, table, columns, samples_table=False): 1150 """The constructor takes in a table and a list of columns. 1151 1152 Args: 1153 table: A list of lists of values. 1154 columns: A list of column containing what to produce and how to format 1155 it. 1156 samples_table: A flag to check whether we are generating a table of 1157 samples in CWP apporximation mode. 1158 """ 1159 self._table = table 1160 self._columns = columns 1161 self._samples_table = samples_table 1162 self._table_columns = [] 1163 self._out_table = [] 1164 1165 def GenerateCellTable(self, table_type): 1166 row_index = 0 1167 all_failed = False 1168 1169 for row in self._table[1:]: 1170 # If we are generating samples_table, the second value will be weight 1171 # rather than values. 1172 start_col = 2 if self._samples_table else 1 1173 # It does not make sense to put retval in the summary table. 1174 if str(row[0]) == "retval" and table_type == "summary": 1175 # Check to see if any runs passed, and update all_failed. 1176 all_failed = True 1177 for values in row[start_col:]: 1178 if 0 in values: 1179 all_failed = False 1180 continue 1181 key = Cell() 1182 key.string_value = str(row[0]) 1183 out_row = [key] 1184 if self._samples_table: 1185 # Add one column for weight if in samples_table mode 1186 weight = Cell() 1187 weight.value = row[1] 1188 f = WeightFormat() 1189 f.Compute(weight) 1190 out_row.append(weight) 1191 baseline = None 1192 for results in row[start_col:]: 1193 column_start = 0 1194 values = None 1195 # If generating sample table, we will split a tuple of iterations info 1196 # from the results 1197 if isinstance(results, tuple): 1198 it, values = results 1199 column_start = 1 1200 cell = Cell() 1201 cell.string_value = "[%d: %d]" % (it[0], it[1]) 1202 out_row.append(cell) 1203 if not row_index: 1204 self._table_columns.append(self._columns[0]) 1205 else: 1206 values = results 1207 # Parse each column 1208 for column in self._columns[column_start:]: 1209 cell = Cell() 1210 cell.name = key.string_value 1211 if ( 1212 not column.result.NeedsBaseline() 1213 or baseline is not None 1214 ): 1215 column.result.Compute(cell, values, baseline) 1216 column.fmt.Compute(cell) 1217 out_row.append(cell) 1218 if not row_index: 1219 self._table_columns.append(column) 1220 1221 if baseline is None: 1222 baseline = values 1223 self._out_table.append(out_row) 1224 row_index += 1 1225 1226 # If this is a summary table, and the only row in it is 'retval', and 1227 # all the test runs failed, we need to a 'Results' row to the output 1228 # table. 1229 if table_type == "summary" and all_failed and len(self._table) == 2: 1230 labels_row = self._table[0] 1231 key = Cell() 1232 key.string_value = "Results" 1233 out_row = [key] 1234 baseline = None 1235 for _ in labels_row[1:]: 1236 for column in self._columns: 1237 cell = Cell() 1238 cell.name = key.string_value 1239 column.result.Compute(cell, ["Fail"], baseline) 1240 column.fmt.Compute(cell) 1241 out_row.append(cell) 1242 if not row_index: 1243 self._table_columns.append(column) 1244 self._out_table.append(out_row) 1245 1246 def AddColumnName(self): 1247 """Generate Column name at the top of table.""" 1248 key = Cell() 1249 key.header = True 1250 key.string_value = "Keys" if not self._samples_table else "Benchmarks" 1251 header = [key] 1252 if self._samples_table: 1253 weight = Cell() 1254 weight.header = True 1255 weight.string_value = "Weights" 1256 header.append(weight) 1257 for column in self._table_columns: 1258 cell = Cell() 1259 cell.header = True 1260 if column.name: 1261 cell.string_value = column.name 1262 else: 1263 result_name = column.result.__class__.__name__ 1264 format_name = column.fmt.__class__.__name__ 1265 1266 cell.string_value = "%s %s" % ( 1267 result_name.replace("Result", ""), 1268 format_name.replace("Format", ""), 1269 ) 1270 1271 header.append(cell) 1272 1273 self._out_table = [header] + self._out_table 1274 1275 def AddHeader(self, s): 1276 """Put additional string on the top of the table.""" 1277 cell = Cell() 1278 cell.header = True 1279 cell.string_value = str(s) 1280 header = [cell] 1281 colspan = max(1, max(len(row) for row in self._table)) 1282 cell.colspan = colspan 1283 self._out_table = [header] + self._out_table 1284 1285 def GetPassesAndFails(self, values): 1286 passes = 0 1287 fails = 0 1288 for val in values: 1289 if val == 0: 1290 passes = passes + 1 1291 else: 1292 fails = fails + 1 1293 return passes, fails 1294 1295 def AddLabelName(self): 1296 """Put label on the top of the table.""" 1297 top_header = [] 1298 base_colspan = len( 1299 [c for c in self._columns if not c.result.NeedsBaseline()] 1300 ) 1301 compare_colspan = len(self._columns) 1302 # Find the row with the key 'retval', if it exists. This 1303 # will be used to calculate the number of iterations that passed and 1304 # failed for each image label. 1305 retval_row = None 1306 for row in self._table: 1307 if row[0] == "retval": 1308 retval_row = row 1309 # The label is organized as follows 1310 # "keys" label_base, label_comparison1, label_comparison2 1311 # The first cell has colspan 1, the second is base_colspan 1312 # The others are compare_colspan 1313 column_position = 0 1314 for label in self._table[0]: 1315 cell = Cell() 1316 cell.header = True 1317 # Put the number of pass/fail iterations in the image label header. 1318 if column_position > 0 and retval_row: 1319 retval_values = retval_row[column_position] 1320 if isinstance(retval_values, list): 1321 passes, fails = self.GetPassesAndFails(retval_values) 1322 cell.string_value = str(label) + " (pass:%d fail:%d)" % ( 1323 passes, 1324 fails, 1325 ) 1326 else: 1327 cell.string_value = str(label) 1328 else: 1329 cell.string_value = str(label) 1330 if top_header: 1331 if not self._samples_table or ( 1332 self._samples_table and len(top_header) == 2 1333 ): 1334 cell.colspan = base_colspan 1335 if len(top_header) > 1: 1336 if not self._samples_table or ( 1337 self._samples_table and len(top_header) > 2 1338 ): 1339 cell.colspan = compare_colspan 1340 top_header.append(cell) 1341 column_position = column_position + 1 1342 self._out_table = [top_header] + self._out_table 1343 1344 def _PrintOutTable(self): 1345 o = "" 1346 for row in self._out_table: 1347 for cell in row: 1348 o += str(cell) + " " 1349 o += "\n" 1350 print(o) 1351 1352 def GetCellTable(self, table_type="full", headers=True): 1353 """Function to return a table of cells. 1354 1355 The table (list of lists) is converted into a table of cells by this 1356 function. 1357 1358 Args: 1359 table_type: Can be 'full' or 'summary' 1360 headers: A boolean saying whether we want default headers 1361 1362 Returns: 1363 A table of cells with each cell having the properties and string values as 1364 requiested by the columns passed in the constructor. 1365 """ 1366 # Generate the cell table, creating a list of dynamic columns on the fly. 1367 if not self._out_table: 1368 self.GenerateCellTable(table_type) 1369 if headers: 1370 self.AddColumnName() 1371 self.AddLabelName() 1372 return self._out_table 1373 1374 1375class TablePrinter(object): 1376 """Class to print a cell table to the console, file or html.""" 1377 1378 PLAIN = 0 1379 CONSOLE = 1 1380 HTML = 2 1381 TSV = 3 1382 EMAIL = 4 1383 1384 def __init__(self, table, output_type): 1385 """Constructor that stores the cell table and output type.""" 1386 self._table = table 1387 self._output_type = output_type 1388 self._row_styles = [] 1389 self._column_styles = [] 1390 1391 # Compute whole-table properties like max-size, etc. 1392 def _ComputeStyle(self): 1393 self._row_styles = [] 1394 for row in self._table: 1395 row_style = Cell() 1396 for cell in row: 1397 if cell.color_row: 1398 assert cell.color, "Cell color not set but color_row set!" 1399 assert ( 1400 not row_style.color 1401 ), "Multiple row_style.colors found!" 1402 row_style.color = cell.color 1403 if cell.bgcolor_row: 1404 assert ( 1405 cell.bgcolor 1406 ), "Cell bgcolor not set but bgcolor_row set!" 1407 assert ( 1408 not row_style.bgcolor 1409 ), "Multiple row_style.bgcolors found!" 1410 row_style.bgcolor = cell.bgcolor 1411 self._row_styles.append(row_style) 1412 1413 self._column_styles = [] 1414 if len(self._table) < 2: 1415 return 1416 1417 for i in range(max(len(row) for row in self._table)): 1418 column_style = Cell() 1419 for row in self._table: 1420 if not any([cell.colspan != 1 for cell in row]): 1421 column_style.width = max( 1422 column_style.width, len(row[i].string_value) 1423 ) 1424 self._column_styles.append(column_style) 1425 1426 def _GetBGColorFix(self, color): 1427 if self._output_type == self.CONSOLE: 1428 prefix = misc.rgb2short(color.r, color.g, color.b) 1429 # pylint: disable=anomalous-backslash-in-string 1430 prefix = "\033[48;5;%sm" % prefix 1431 suffix = "\033[0m" 1432 elif self._output_type in [self.EMAIL, self.HTML]: 1433 rgb = color.GetRGB() 1434 prefix = '<FONT style="BACKGROUND-COLOR:#{0}">'.format(rgb) 1435 suffix = "</FONT>" 1436 elif self._output_type in [self.PLAIN, self.TSV]: 1437 prefix = "" 1438 suffix = "" 1439 return prefix, suffix 1440 1441 def _GetColorFix(self, color): 1442 if self._output_type == self.CONSOLE: 1443 prefix = misc.rgb2short(color.r, color.g, color.b) 1444 # pylint: disable=anomalous-backslash-in-string 1445 prefix = "\033[38;5;%sm" % prefix 1446 suffix = "\033[0m" 1447 elif self._output_type in [self.EMAIL, self.HTML]: 1448 rgb = color.GetRGB() 1449 prefix = "<FONT COLOR=#{0}>".format(rgb) 1450 suffix = "</FONT>" 1451 elif self._output_type in [self.PLAIN, self.TSV]: 1452 prefix = "" 1453 suffix = "" 1454 return prefix, suffix 1455 1456 def Print(self): 1457 """Print the table to a console, html, etc. 1458 1459 Returns: 1460 A string that contains the desired representation of the table. 1461 """ 1462 self._ComputeStyle() 1463 return self._GetStringValue() 1464 1465 def _GetCellValue(self, i, j): 1466 cell = self._table[i][j] 1467 out = cell.string_value 1468 raw_width = len(out) 1469 1470 if cell.color: 1471 p, s = self._GetColorFix(cell.color) 1472 out = "%s%s%s" % (p, out, s) 1473 1474 if cell.bgcolor: 1475 p, s = self._GetBGColorFix(cell.bgcolor) 1476 out = "%s%s%s" % (p, out, s) 1477 1478 if self._output_type in [self.PLAIN, self.CONSOLE, self.EMAIL]: 1479 if cell.width: 1480 width = cell.width 1481 else: 1482 if self._column_styles: 1483 width = self._column_styles[j].width 1484 else: 1485 width = len(cell.string_value) 1486 if cell.colspan > 1: 1487 width = 0 1488 start = 0 1489 for k in range(j): 1490 start += self._table[i][k].colspan 1491 for k in range(cell.colspan): 1492 width += self._column_styles[start + k].width 1493 if width > raw_width: 1494 padding = ("%" + str(width - raw_width) + "s") % "" 1495 out = padding + out 1496 1497 if self._output_type == self.HTML: 1498 if cell.header: 1499 tag = "th" 1500 else: 1501 tag = "td" 1502 out = '<{0} colspan = "{2}"> {1} </{0}>'.format( 1503 tag, out, cell.colspan 1504 ) 1505 1506 return out 1507 1508 def _GetHorizontalSeparator(self): 1509 if self._output_type in [self.CONSOLE, self.PLAIN, self.EMAIL]: 1510 return " " 1511 if self._output_type == self.HTML: 1512 return "" 1513 if self._output_type == self.TSV: 1514 return "\t" 1515 1516 def _GetVerticalSeparator(self): 1517 if self._output_type in [ 1518 self.PLAIN, 1519 self.CONSOLE, 1520 self.TSV, 1521 self.EMAIL, 1522 ]: 1523 return "\n" 1524 if self._output_type == self.HTML: 1525 return "</tr>\n<tr>" 1526 1527 def _GetPrefix(self): 1528 if self._output_type in [ 1529 self.PLAIN, 1530 self.CONSOLE, 1531 self.TSV, 1532 self.EMAIL, 1533 ]: 1534 return "" 1535 if self._output_type == self.HTML: 1536 return '<p></p><table id="box-table-a">\n<tr>' 1537 1538 def _GetSuffix(self): 1539 if self._output_type in [ 1540 self.PLAIN, 1541 self.CONSOLE, 1542 self.TSV, 1543 self.EMAIL, 1544 ]: 1545 return "" 1546 if self._output_type == self.HTML: 1547 return "</tr>\n</table>" 1548 1549 def _GetStringValue(self): 1550 o = "" 1551 o += self._GetPrefix() 1552 for i in range(len(self._table)): 1553 row = self._table[i] 1554 # Apply row color and bgcolor. 1555 p = s = bgp = bgs = "" 1556 if self._row_styles[i].bgcolor: 1557 bgp, bgs = self._GetBGColorFix(self._row_styles[i].bgcolor) 1558 if self._row_styles[i].color: 1559 p, s = self._GetColorFix(self._row_styles[i].color) 1560 o += p + bgp 1561 for j in range(len(row)): 1562 out = self._GetCellValue(i, j) 1563 o += out + self._GetHorizontalSeparator() 1564 o += s + bgs 1565 o += self._GetVerticalSeparator() 1566 o += self._GetSuffix() 1567 return o 1568 1569 1570# Some common drivers 1571def GetSimpleTable(table, out_to=TablePrinter.CONSOLE): 1572 """Prints a simple table. 1573 1574 This is used by code that has a very simple list-of-lists and wants to 1575 produce a table with ameans, a percentage ratio of ameans and a colorbox. 1576 1577 Examples: 1578 GetSimpleConsoleTable([["binary", "b1", "b2"],["size", "300", "400"]]) 1579 will produce a colored table that can be printed to the console. 1580 1581 Args: 1582 table: a list of lists. 1583 out_to: specify the fomat of output. Currently it supports HTML and CONSOLE. 1584 1585 Returns: 1586 A string version of the table that can be printed to the console. 1587 """ 1588 columns = [ 1589 Column(AmeanResult(), Format()), 1590 Column(AmeanRatioResult(), PercentFormat()), 1591 Column(AmeanRatioResult(), ColorBoxFormat()), 1592 ] 1593 our_table = [table[0]] 1594 for row in table[1:]: 1595 our_row = [row[0]] 1596 for v in row[1:]: 1597 our_row.append([v]) 1598 our_table.append(our_row) 1599 1600 tf = TableFormatter(our_table, columns) 1601 cell_table = tf.GetCellTable() 1602 tp = TablePrinter(cell_table, out_to) 1603 return tp.Print() 1604 1605 1606# pylint: disable=redefined-outer-name 1607def GetComplexTable(runs, labels, out_to=TablePrinter.CONSOLE): 1608 """Prints a complex table. 1609 1610 This can be used to generate a table with arithmetic mean, standard deviation, 1611 coefficient of variation, p-values, etc. 1612 1613 Args: 1614 runs: A list of lists with data to tabulate. 1615 labels: A list of labels that correspond to the runs. 1616 out_to: specifies the format of the table (example CONSOLE or HTML). 1617 1618 Returns: 1619 A string table that can be printed to the console or put in an HTML file. 1620 """ 1621 tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC) 1622 table = tg.GetTable() 1623 columns = [ 1624 Column(LiteralResult(), Format(), "Literal"), 1625 Column(AmeanResult(), Format()), 1626 Column(StdResult(), Format()), 1627 Column(CoeffVarResult(), CoeffVarFormat()), 1628 Column(NonEmptyCountResult(), Format()), 1629 Column(AmeanRatioResult(), PercentFormat()), 1630 Column(AmeanRatioResult(), RatioFormat()), 1631 Column(GmeanRatioResult(), RatioFormat()), 1632 Column(PValueResult(), PValueFormat()), 1633 ] 1634 tf = TableFormatter(table, columns) 1635 cell_table = tf.GetCellTable() 1636 tp = TablePrinter(cell_table, out_to) 1637 return tp.Print() 1638 1639 1640if __name__ == "__main__": 1641 # Run a few small tests here. 1642 run1 = { 1643 "k1": "10", 1644 "k2": "12", 1645 "k5": "40", 1646 "k6": "40", 1647 "ms_1": "20", 1648 "k7": "FAIL", 1649 "k8": "PASS", 1650 "k9": "PASS", 1651 "k10": "0", 1652 } 1653 run2 = { 1654 "k1": "13", 1655 "k2": "14", 1656 "k3": "15", 1657 "ms_1": "10", 1658 "k8": "PASS", 1659 "k9": "FAIL", 1660 "k10": "0", 1661 } 1662 run3 = { 1663 "k1": "50", 1664 "k2": "51", 1665 "k3": "52", 1666 "k4": "53", 1667 "k5": "35", 1668 "k6": "45", 1669 "ms_1": "200", 1670 "ms_2": "20", 1671 "k7": "FAIL", 1672 "k8": "PASS", 1673 "k9": "PASS", 1674 } 1675 runs = [[run1, run2], [run3]] 1676 labels = ["vanilla", "modified"] 1677 t = GetComplexTable(runs, labels, TablePrinter.CONSOLE) 1678 print(t) 1679 email = GetComplexTable(runs, labels, TablePrinter.EMAIL) 1680 1681 runs = [ 1682 [{"k1": "1"}, {"k1": "1.1"}, {"k1": "1.2"}], 1683 [{"k1": "5"}, {"k1": "5.1"}, {"k1": "5.2"}], 1684 ] 1685 t = GetComplexTable(runs, labels, TablePrinter.CONSOLE) 1686 print(t) 1687 1688 simple_table = [ 1689 ["binary", "b1", "b2", "b3"], 1690 ["size", 100, 105, 108], 1691 ["rodata", 100, 80, 70], 1692 ["data", 100, 100, 100], 1693 ["debug", 100, 140, 60], 1694 ] 1695 t = GetSimpleTable(simple_table) 1696 print(t) 1697 email += GetSimpleTable(simple_table, TablePrinter.HTML) 1698 email_to = [getpass.getuser()] 1699 email = "<pre style='font-size: 13px'>%s</pre>" % email 1700 EmailSender().SendEmail(email_to, "SimpleTableTest", email, msg_type="html") 1701