1# -*- coding: utf-8 -*- 2# Copyright 2013 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""A module to generate experiments.""" 7 8 9import os 10import re 11import socket 12import sys 13 14from benchmark import Benchmark 15from cros_utils import command_executer 16from cros_utils import logger 17from experiment import Experiment 18import file_lock_machine 19from label import Label 20from label import MockLabel 21from results_cache import CacheConditions 22import test_flag 23 24import config 25 26 27# Users may want to run Telemetry tests either individually, or in 28# specified sets. Here we define sets of tests that users may want 29# to run together. 30 31telemetry_toolchain_perf_tests = [ 32 "octane", 33 "speedometer", 34 "speedometer2", 35 "jetstream2", 36] 37graphics_perf_tests = [ 38 "graphics_GLBench", 39 "graphics_GLMark2", 40 "graphics_SanAngeles", 41 "graphics_WebGLAquarium", 42 "graphics_WebGLPerformance", 43] 44# TODO: disable rendering.desktop by default as the benchmark is 45# currently in a bad state 46telemetry_crosbolt_perf_tests = [ 47 "octane", 48 "speedometer2", 49 "jetstream2", 50 "loading.desktop", 51 # 'rendering.desktop', 52] 53 54crosbolt_perf_tests = [ 55 "graphics_WebGLAquarium", 56 "tast.video.PlaybackPerfVP91080P30FPS", 57] 58 59dso_list = [ 60 "all", 61 "chrome", 62 "kallsyms", 63] 64 65 66class ExperimentFactory(object): 67 """Factory class for building an Experiment, given an ExperimentFile as input. 68 69 This factory is currently hardcoded to produce an experiment for running 70 ChromeOS benchmarks, but the idea is that in the future, other types 71 of experiments could be produced. 72 """ 73 74 def AppendBenchmarkSet( 75 self, 76 benchmarks, 77 benchmark_list, 78 test_args, 79 iterations, 80 rm_chroot_tmp, 81 perf_args, 82 suite, 83 show_all_results, 84 retries, 85 run_local, 86 cwp_dso, 87 weight, 88 ): 89 """Add all the tests in a set to the benchmarks list.""" 90 for test_name in benchmark_list: 91 telemetry_benchmark = Benchmark( 92 test_name, 93 test_name, 94 test_args, 95 iterations, 96 rm_chroot_tmp, 97 perf_args, 98 suite, 99 show_all_results, 100 retries, 101 run_local, 102 cwp_dso, 103 weight, 104 ) 105 benchmarks.append(telemetry_benchmark) 106 107 def GetExperiment(self, experiment_file, working_directory, log_dir): 108 """Construct an experiment from an experiment file.""" 109 global_settings = experiment_file.GetGlobalSettings() 110 experiment_name = global_settings.GetField("name") 111 board = global_settings.GetField("board") 112 chromeos_root = global_settings.GetField("chromeos_root") 113 log_level = global_settings.GetField("logging_level") 114 if log_level not in ("quiet", "average", "verbose"): 115 log_level = "verbose" 116 117 crosfleet = global_settings.GetField("crosfleet") 118 no_lock = bool(global_settings.GetField("no_lock")) 119 # Check whether crosfleet tool is installed correctly for crosfleet mode. 120 if crosfleet and not self.CheckCrosfleetTool(chromeos_root, log_level): 121 sys.exit(0) 122 123 remote = global_settings.GetField("remote") 124 # This is used to remove the ",' from the remote if user 125 # add them to the remote string. 126 new_remote = [] 127 if remote: 128 for i in remote: 129 c = re.sub("[\"']", "", i) 130 new_remote.append(c) 131 remote = new_remote 132 rm_chroot_tmp = global_settings.GetField("rm_chroot_tmp") 133 perf_args = global_settings.GetField("perf_args") 134 download_debug = global_settings.GetField("download_debug") 135 # Do not download debug symbols when perf_args is not specified. 136 if not perf_args and download_debug: 137 download_debug = False 138 acquire_timeout = global_settings.GetField("acquire_timeout") 139 cache_dir = global_settings.GetField("cache_dir") 140 cache_only = global_settings.GetField("cache_only") 141 config.AddConfig("no_email", global_settings.GetField("no_email")) 142 share_cache = global_settings.GetField("share_cache") 143 results_dir = global_settings.GetField("results_dir") 144 compress_results = global_settings.GetField("compress_results") 145 # Warn user that option use_file_locks is deprecated. 146 use_file_locks = global_settings.GetField("use_file_locks") 147 if use_file_locks: 148 l = logger.GetLogger() 149 l.LogWarning( 150 "Option use_file_locks is deprecated, please remove it " 151 "from your experiment settings." 152 ) 153 locks_dir = global_settings.GetField("locks_dir") 154 # If not specified, set the locks dir to the default locks dir in 155 # file_lock_machine. 156 if not locks_dir: 157 locks_dir = file_lock_machine.Machine.LOCKS_DIR 158 if not os.path.exists(locks_dir): 159 raise RuntimeError( 160 "Cannot access default lock directory. " 161 "Please run prodaccess or specify a local directory" 162 ) 163 chrome_src = global_settings.GetField("chrome_src") 164 show_all_results = global_settings.GetField("show_all_results") 165 cwp_dso = global_settings.GetField("cwp_dso") 166 if cwp_dso and not cwp_dso in dso_list: 167 raise RuntimeError("The DSO specified is not supported") 168 ignore_min_max = global_settings.GetField("ignore_min_max") 169 dut_config = { 170 "enable_aslr": global_settings.GetField("enable_aslr"), 171 "intel_pstate": global_settings.GetField("intel_pstate"), 172 "cooldown_time": global_settings.GetField("cooldown_time"), 173 "cooldown_temp": global_settings.GetField("cooldown_temp"), 174 "governor": global_settings.GetField("governor"), 175 "cpu_usage": global_settings.GetField("cpu_usage"), 176 "cpu_freq_pct": global_settings.GetField("cpu_freq_pct"), 177 "turbostat": global_settings.GetField("turbostat"), 178 "top_interval": global_settings.GetField("top_interval"), 179 } 180 keep_stateful = global_settings.GetField("keep_stateful") 181 182 # Default cache hit conditions. The image checksum in the cache and the 183 # computed checksum of the image must match. Also a cache file must exist. 184 cache_conditions = [ 185 CacheConditions.CACHE_FILE_EXISTS, 186 CacheConditions.CHECKSUMS_MATCH, 187 ] 188 if global_settings.GetField("rerun_if_failed"): 189 cache_conditions.append(CacheConditions.RUN_SUCCEEDED) 190 if global_settings.GetField("rerun") or global_settings.GetField( 191 "ignore_cache" 192 ): 193 cache_conditions.append(CacheConditions.FALSE) 194 if global_settings.GetField("same_machine"): 195 cache_conditions.append(CacheConditions.SAME_MACHINE_MATCH) 196 if global_settings.GetField("same_specs"): 197 cache_conditions.append(CacheConditions.MACHINES_MATCH) 198 199 # Construct benchmarks. 200 # Some fields are common with global settings. The values are 201 # inherited and/or merged with the global settings values. 202 benchmarks = [] 203 all_benchmark_settings = experiment_file.GetSettings("benchmark") 204 205 # Check if there is duplicated benchmark name 206 benchmark_names = {} 207 # Check if in cwp_dso mode, all benchmarks should have same iterations 208 cwp_dso_iterations = 0 209 210 for benchmark_settings in all_benchmark_settings: 211 benchmark_name = benchmark_settings.name 212 test_name = benchmark_settings.GetField("test_name") 213 if not test_name: 214 test_name = benchmark_name 215 test_args = benchmark_settings.GetField("test_args") 216 217 # Rename benchmark name if 'story-filter' or 'story-tag-filter' specified 218 # in test_args. Make sure these two tags only appear once. 219 story_count = 0 220 for arg in test_args.split(): 221 if "--story-filter=" in arg or "--story-tag-filter=" in arg: 222 story_count += 1 223 if story_count > 1: 224 raise RuntimeError( 225 "Only one story or story-tag filter allowed in " 226 "a single benchmark run" 227 ) 228 # Rename benchmark name with an extension of 'story'-option 229 benchmark_name = "%s@@%s" % ( 230 benchmark_name, 231 arg.split("=")[-1], 232 ) 233 234 # Check for duplicated benchmark name after renaming 235 if not benchmark_name in benchmark_names: 236 benchmark_names[benchmark_name] = True 237 else: 238 raise SyntaxError( 239 "Duplicate benchmark name: '%s'." % benchmark_name 240 ) 241 242 iterations = benchmark_settings.GetField("iterations") 243 if cwp_dso: 244 if cwp_dso_iterations not in (0, iterations): 245 raise RuntimeError( 246 "Iterations of each benchmark run are not the " "same" 247 ) 248 cwp_dso_iterations = iterations 249 250 suite = benchmark_settings.GetField("suite") 251 retries = benchmark_settings.GetField("retries") 252 run_local = benchmark_settings.GetField("run_local") 253 weight = benchmark_settings.GetField("weight") 254 if weight: 255 if not cwp_dso: 256 raise RuntimeError( 257 "Weight can only be set when DSO specified" 258 ) 259 if suite != "telemetry_Crosperf": 260 raise RuntimeError( 261 "CWP approximation weight only works with " 262 "telemetry_Crosperf suite" 263 ) 264 if run_local: 265 raise RuntimeError( 266 "run_local must be set to False to use CWP " 267 "approximation" 268 ) 269 if weight < 0: 270 raise RuntimeError("Weight should be a float >=0") 271 elif cwp_dso: 272 raise RuntimeError( 273 "With DSO specified, each benchmark should have a " "weight" 274 ) 275 276 if suite == "telemetry_Crosperf": 277 if test_name == "all_crosbolt_perf": 278 self.AppendBenchmarkSet( 279 benchmarks, 280 telemetry_crosbolt_perf_tests, 281 test_args, 282 iterations, 283 rm_chroot_tmp, 284 perf_args, 285 "telemetry_Crosperf", 286 show_all_results, 287 retries, 288 run_local, 289 cwp_dso, 290 weight, 291 ) 292 self.AppendBenchmarkSet( 293 benchmarks, 294 crosbolt_perf_tests, 295 "", 296 iterations, 297 rm_chroot_tmp, 298 perf_args, 299 "", 300 show_all_results, 301 retries, 302 run_local=False, 303 cwp_dso=cwp_dso, 304 weight=weight, 305 ) 306 elif test_name == "all_toolchain_perf": 307 self.AppendBenchmarkSet( 308 benchmarks, 309 telemetry_toolchain_perf_tests, 310 test_args, 311 iterations, 312 rm_chroot_tmp, 313 perf_args, 314 suite, 315 show_all_results, 316 retries, 317 run_local, 318 cwp_dso, 319 weight, 320 ) 321 # Add non-telemetry toolchain-perf benchmarks: 322 323 # TODO: crbug.com/1057755 Do not enable graphics_WebGLAquarium until 324 # it gets fixed. 325 # 326 # benchmarks.append( 327 # Benchmark( 328 # 'graphics_WebGLAquarium', 329 # 'graphics_WebGLAquarium', 330 # '', 331 # iterations, 332 # rm_chroot_tmp, 333 # perf_args, 334 # 'crosperf_Wrapper', # Use client wrapper in Autotest 335 # show_all_results, 336 # retries, 337 # run_local=False, 338 # cwp_dso=cwp_dso, 339 # weight=weight)) 340 else: 341 benchmark = Benchmark( 342 benchmark_name, 343 test_name, 344 test_args, 345 iterations, 346 rm_chroot_tmp, 347 perf_args, 348 suite, 349 show_all_results, 350 retries, 351 run_local, 352 cwp_dso, 353 weight, 354 ) 355 benchmarks.append(benchmark) 356 else: 357 if test_name == "all_graphics_perf": 358 self.AppendBenchmarkSet( 359 benchmarks, 360 graphics_perf_tests, 361 "", 362 iterations, 363 rm_chroot_tmp, 364 perf_args, 365 "", 366 show_all_results, 367 retries, 368 run_local=False, 369 cwp_dso=cwp_dso, 370 weight=weight, 371 ) 372 else: 373 # Add the single benchmark. 374 benchmark = Benchmark( 375 benchmark_name, 376 test_name, 377 test_args, 378 iterations, 379 rm_chroot_tmp, 380 perf_args, 381 suite, 382 show_all_results, 383 retries, 384 run_local=False, 385 cwp_dso=cwp_dso, 386 weight=weight, 387 ) 388 benchmarks.append(benchmark) 389 390 if not benchmarks: 391 raise RuntimeError("No benchmarks specified") 392 393 # Construct labels. 394 # Some fields are common with global settings. The values are 395 # inherited and/or merged with the global settings values. 396 labels = [] 397 all_label_settings = experiment_file.GetSettings("label") 398 all_remote = list(remote) 399 for label_settings in all_label_settings: 400 label_name = label_settings.name 401 image = label_settings.GetField("chromeos_image") 402 build = label_settings.GetField("build") 403 autotest_path = label_settings.GetField("autotest_path") 404 debug_path = label_settings.GetField("debug_path") 405 chromeos_root = label_settings.GetField("chromeos_root") 406 my_remote = label_settings.GetField("remote") 407 compiler = label_settings.GetField("compiler") 408 new_remote = [] 409 if my_remote: 410 for i in my_remote: 411 c = re.sub("[\"']", "", i) 412 new_remote.append(c) 413 my_remote = new_remote 414 415 if image: 416 if crosfleet: 417 raise RuntimeError( 418 "In crosfleet mode, local image should not be used." 419 ) 420 if build: 421 raise RuntimeError( 422 "Image path and build are provided at the same " 423 "time, please use only one of them." 424 ) 425 else: 426 if not build: 427 raise RuntimeError("Can not have empty 'build' field!") 428 image, autotest_path, debug_path = label_settings.GetXbuddyPath( 429 build, 430 autotest_path, 431 debug_path, 432 board, 433 chromeos_root, 434 log_level, 435 download_debug, 436 ) 437 438 cache_dir = label_settings.GetField("cache_dir") 439 chrome_src = label_settings.GetField("chrome_src") 440 441 # TODO(yunlian): We should consolidate code in machine_manager.py 442 # to derermine whether we are running from within google or not 443 if ( 444 "corp.google.com" in socket.gethostname() 445 and not my_remote 446 and not crosfleet 447 ): 448 my_remote = self.GetDefaultRemotes(board) 449 if global_settings.GetField("same_machine") and len(my_remote) > 1: 450 raise RuntimeError( 451 "Only one remote is allowed when same_machine " 452 "is turned on" 453 ) 454 all_remote += my_remote 455 image_args = label_settings.GetField("image_args") 456 if test_flag.GetTestMode(): 457 # pylint: disable=too-many-function-args 458 label = MockLabel( 459 label_name, 460 build, 461 image, 462 autotest_path, 463 debug_path, 464 chromeos_root, 465 board, 466 my_remote, 467 image_args, 468 cache_dir, 469 cache_only, 470 log_level, 471 compiler, 472 crosfleet, 473 chrome_src, 474 ) 475 else: 476 label = Label( 477 label_name, 478 build, 479 image, 480 autotest_path, 481 debug_path, 482 chromeos_root, 483 board, 484 my_remote, 485 image_args, 486 cache_dir, 487 cache_only, 488 log_level, 489 compiler, 490 crosfleet, 491 chrome_src, 492 ) 493 labels.append(label) 494 495 if not labels: 496 raise RuntimeError("No labels specified") 497 498 email = global_settings.GetField("email") 499 all_remote += list(set(my_remote)) 500 all_remote = list(set(all_remote)) 501 if crosfleet: 502 for remote in all_remote: 503 self.CheckRemotesInCrosfleet(remote) 504 experiment = Experiment( 505 experiment_name, 506 all_remote, 507 working_directory, 508 chromeos_root, 509 cache_conditions, 510 labels, 511 benchmarks, 512 experiment_file.Canonicalize(), 513 email, 514 acquire_timeout, 515 log_dir, 516 log_level, 517 share_cache, 518 results_dir, 519 compress_results, 520 locks_dir, 521 cwp_dso, 522 ignore_min_max, 523 crosfleet, 524 dut_config, 525 keep_stateful, 526 no_lock=no_lock, 527 ) 528 529 return experiment 530 531 def GetDefaultRemotes(self, board): 532 default_remotes_file = os.path.join( 533 os.path.dirname(__file__), "default_remotes" 534 ) 535 try: 536 with open(default_remotes_file, encoding="utf-8") as f: 537 for line in f: 538 key, v = line.split(":") 539 if key.strip() == board: 540 remotes = v.strip().split() 541 if remotes: 542 return remotes 543 else: 544 raise RuntimeError( 545 f"There is no remote for {board}" 546 ) 547 except IOError: 548 # TODO: rethrow instead of throwing different exception. 549 raise RuntimeError( 550 f"IOError while reading file {default_remotes_file}" 551 ) 552 else: 553 raise RuntimeError(f"There is no remote for {board}") 554 555 def CheckRemotesInCrosfleet(self, remote): 556 # TODO: (AI:zhizhouy) need to check whether a remote is a local or lab 557 # machine. If not lab machine, raise an error. 558 pass 559 560 def CheckCrosfleetTool(self, chromeos_root, log_level): 561 CROSFLEET_PATH = "crosfleet" 562 if os.path.exists(CROSFLEET_PATH): 563 return True 564 l = logger.GetLogger() 565 l.LogOutput("Crosfleet tool not installed, trying to install it.") 566 ce = command_executer.GetCommandExecuter(l, log_level=log_level) 567 setup_lab_tools = os.path.join( 568 chromeos_root, "chromeos-admin", "lab-tools", "setup_lab_tools" 569 ) 570 cmd = "%s" % setup_lab_tools 571 status = ce.RunCommand(cmd) 572 if status != 0: 573 raise RuntimeError( 574 "Crosfleet tool not installed correctly, please try to " 575 "manually install it from %s" % setup_lab_tools 576 ) 577 l.LogOutput( 578 "Crosfleet is installed at %s, please login before first use. " 579 'Login by running "crosfleet login" and follow instructions.' 580 % CROSFLEET_PATH 581 ) 582 return False 583