1# -*- coding: utf-8 -*- 2# Copyright 2011 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""The experiment file module. It manages the input file of crosperf.""" 7 8 9import os.path 10import re 11 12from settings_factory import SettingsFactory 13 14 15class ExperimentFile(object): 16 """Class for parsing the experiment file format. 17 18 The grammar for this format is: 19 20 experiment = { _FIELD_VALUE_RE | settings } 21 settings = _OPEN_SETTINGS_RE 22 { _FIELD_VALUE_RE } 23 _CLOSE_SETTINGS_RE 24 25 Where the regexes are terminals defined below. This results in an format 26 which looks something like: 27 28 field_name: value 29 settings_type: settings_name { 30 field_name: value 31 field_name: value 32 } 33 """ 34 35 # Field regex, e.g. "iterations: 3" 36 _FIELD_VALUE_RE = re.compile(r"(\+)?\s*(\w+?)(?:\.(\S+))?\s*:\s*(.*)") 37 # Open settings regex, e.g. "label {" 38 _OPEN_SETTINGS_RE = re.compile(r"(?:([\w.-]+):)?\s*([\w.-]+)\s*{") 39 # Close settings regex. 40 _CLOSE_SETTINGS_RE = re.compile(r"}") 41 42 def __init__(self, experiment_file, overrides=None): 43 """Construct object from file-like experiment_file. 44 45 Args: 46 experiment_file: file-like object with text description of experiment. 47 overrides: A settings object that will override fields in other settings. 48 49 Raises: 50 Exception: if invalid build type or description is invalid. 51 """ 52 self.all_settings = [] 53 self.global_settings = SettingsFactory().GetSettings("global", "global") 54 self.all_settings.append(self.global_settings) 55 56 self._Parse(experiment_file) 57 58 for settings in self.all_settings: 59 settings.Inherit() 60 settings.Validate() 61 if overrides: 62 settings.Override(overrides) 63 64 def GetSettings(self, settings_type): 65 """Return nested fields from the experiment file.""" 66 res = [] 67 for settings in self.all_settings: 68 if settings.settings_type == settings_type: 69 res.append(settings) 70 return res 71 72 def GetGlobalSettings(self): 73 """Return the global fields from the experiment file.""" 74 return self.global_settings 75 76 def _ParseField(self, reader): 77 """Parse a key/value field.""" 78 line = reader.CurrentLine().strip() 79 match = ExperimentFile._FIELD_VALUE_RE.match(line) 80 append, name, _, text_value = match.groups() 81 return (name, text_value, append) 82 83 def _ParseSettings(self, reader): 84 """Parse a settings block.""" 85 line = reader.CurrentLine().strip() 86 match = ExperimentFile._OPEN_SETTINGS_RE.match(line) 87 settings_type = match.group(1) 88 if settings_type is None: 89 settings_type = "" 90 settings_name = match.group(2) 91 settings = SettingsFactory().GetSettings(settings_name, settings_type) 92 settings.SetParentSettings(self.global_settings) 93 94 while reader.NextLine(): 95 line = reader.CurrentLine().strip() 96 97 if not line: 98 continue 99 100 if ExperimentFile._FIELD_VALUE_RE.match(line): 101 field = self._ParseField(reader) 102 settings.SetField(field[0], field[1], field[2]) 103 elif ExperimentFile._CLOSE_SETTINGS_RE.match(line): 104 return settings, settings_type 105 106 raise EOFError("Unexpected EOF while parsing settings block.") 107 108 def _Parse(self, experiment_file): 109 """Parse experiment file and create settings.""" 110 reader = ExperimentFileReader(experiment_file) 111 settings_names = {} 112 try: 113 while reader.NextLine(): 114 line = reader.CurrentLine().strip() 115 116 if not line: 117 continue 118 119 if ExperimentFile._OPEN_SETTINGS_RE.match(line): 120 new_settings, settings_type = self._ParseSettings(reader) 121 # We will allow benchmarks with duplicated settings name for now. 122 # Further decision will be made when parsing benchmark details in 123 # ExperimentFactory.GetExperiment(). 124 if settings_type != "benchmark": 125 if new_settings.name in settings_names: 126 raise SyntaxError( 127 "Duplicate settings name: '%s'." 128 % new_settings.name 129 ) 130 settings_names[new_settings.name] = True 131 self.all_settings.append(new_settings) 132 elif ExperimentFile._FIELD_VALUE_RE.match(line): 133 field = self._ParseField(reader) 134 self.global_settings.SetField(field[0], field[1], field[2]) 135 else: 136 raise IOError("Unexpected line.") 137 except Exception as err: 138 raise RuntimeError( 139 "Line %d: %s\n==> %s" 140 % (reader.LineNo(), str(err), reader.CurrentLine(False)) 141 ) 142 143 def Canonicalize(self): 144 """Convert parsed experiment file back into an experiment file.""" 145 res = "" 146 board = "" 147 for field_name in self.global_settings.fields: 148 field = self.global_settings.fields[field_name] 149 if field.assigned: 150 res += "%s: %s\n" % (field.name, field.GetString()) 151 if field.name == "board": 152 board = field.GetString() 153 res += "\n" 154 155 for settings in self.all_settings: 156 if settings.settings_type != "global": 157 res += "%s: %s {\n" % (settings.settings_type, settings.name) 158 for field_name in settings.fields: 159 field = settings.fields[field_name] 160 if field.assigned: 161 res += "\t%s: %s\n" % (field.name, field.GetString()) 162 if field.name == "chromeos_image": 163 real_file = os.path.realpath( 164 os.path.expanduser(field.GetString()) 165 ) 166 if real_file != field.GetString(): 167 res += "\t#actual_image: %s\n" % real_file 168 if field.name == "build": 169 chromeos_root_field = settings.fields[ 170 "chromeos_root" 171 ] 172 if chromeos_root_field: 173 chromeos_root = chromeos_root_field.GetString() 174 value = field.GetString() 175 autotest_field = settings.fields["autotest_path"] 176 autotest_path = "" 177 if autotest_field.assigned: 178 autotest_path = autotest_field.GetString() 179 debug_field = settings.fields["debug_path"] 180 debug_path = "" 181 if debug_field.assigned: 182 debug_path = autotest_field.GetString() 183 # Do not download the debug symbols since this function is for 184 # canonicalizing experiment file. 185 downlad_debug = False 186 ( 187 image_path, 188 autotest_path, 189 debug_path, 190 ) = settings.GetXbuddyPath( 191 value, 192 autotest_path, 193 debug_path, 194 board, 195 chromeos_root, 196 "quiet", 197 downlad_debug, 198 ) 199 res += "\t#actual_image: %s\n" % image_path 200 if not autotest_field.assigned: 201 res += ( 202 "\t#actual_autotest_path: %s\n" 203 % autotest_path 204 ) 205 if not debug_field.assigned: 206 res += "\t#actual_debug_path: %s\n" % debug_path 207 208 res += "}\n\n" 209 210 return res 211 212 213class ExperimentFileReader(object): 214 """Handle reading lines from an experiment file.""" 215 216 def __init__(self, file_object): 217 self.file_object = file_object 218 self.current_line = None 219 self.current_line_no = 0 220 221 def CurrentLine(self, strip_comment=True): 222 """Return the next line from the file, without advancing the iterator.""" 223 if strip_comment: 224 return self._StripComment(self.current_line) 225 return self.current_line 226 227 def NextLine(self, strip_comment=True): 228 """Advance the iterator and return the next line of the file.""" 229 self.current_line_no += 1 230 self.current_line = self.file_object.readline() 231 return self.CurrentLine(strip_comment) 232 233 def _StripComment(self, line): 234 """Strip comments starting with # from a line.""" 235 if "#" in line: 236 line = line[: line.find("#")] + line[-1] 237 return line 238 239 def LineNo(self): 240 """Return the current line number.""" 241 return self.current_line_no 242