xref: /aosp_15_r20/external/toolchain-utils/crosperf/experiment_file_unittest.py (revision 760c253c1ed00ce9abd48f8546f08516e57485fe) 
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3
4# Copyright 2011 The ChromiumOS Authors
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8"""The unittest of experiment_file."""
9
10import io
11import unittest
12
13from experiment_file import ExperimentFile
14
15
16EXPERIMENT_FILE_1 = """
17  board: x86-alex
18  remote: chromeos-alex3
19  perf_args: record -a -e cycles
20  benchmark: PageCycler {
21    iterations: 3
22  }
23
24  image1 {
25    chromeos_image: /usr/local/google/cros_image1.bin
26  }
27
28  image2 {
29    remote: chromeos-lumpy1
30    chromeos_image: /usr/local/google/cros_image2.bin
31  }
32  """
33
34EXPERIMENT_FILE_2 = """
35  board: x86-alex
36  remote: chromeos-alex3
37  iterations: 3
38
39  benchmark: PageCycler {
40  }
41
42  benchmark: AndroidBench {
43    iterations: 2
44  }
45
46  image1 {
47    chromeos_image:/usr/local/google/cros_image1.bin
48  }
49
50  image2 {
51    chromeos_image: /usr/local/google/cros_image2.bin
52  }
53  """
54
55EXPERIMENT_FILE_3 = """
56  board: x86-alex
57  remote: chromeos-alex3
58  iterations: 3
59
60  benchmark: PageCycler {
61  }
62
63  image1 {
64    chromeos_image:/usr/local/google/cros_image1.bin
65  }
66
67  image1 {
68    chromeos_image: /usr/local/google/cros_image2.bin
69  }
70  """
71
72EXPERIMENT_FILE_4 = """
73  board: x86-alex
74  remote: chromeos-alex3
75  iterations: 3
76
77  benchmark: webrtc {
78    test_args: --story-filter=datachannel
79  }
80
81  benchmark: webrtc {
82    test_args: --story-tag-filter=smoothness
83  }
84
85  image1 {
86    chromeos_image:/usr/local/google/cros_image1.bin
87  }
88  """
89
90DUT_CONFIG_EXPERIMENT_FILE_GOOD = """
91  board: kevin64
92  remote: chromeos-kevin.cros
93  turbostat: False
94  intel_pstate: no_hwp
95  cooldown_temp: 38
96  cooldown_time: 5
97  governor: powersave
98  cpu_usage: exclusive_cores
99  cpu_freq_pct: 50
100  top_interval: 5
101
102  benchmark: speedometer {
103    iterations: 3
104    suite: telemetry_Crosperf
105  }
106
107  image1 {
108    chromeos_image:/usr/local/google/cros_image1.bin
109  }
110  """
111
112DUT_CONFIG_EXPERIMENT_FILE_BAD_GOV = """
113  board: kevin64
114  remote: chromeos-kevin.cros
115  intel_pstate: active
116  governor: misspelled_governor
117
118  benchmark: speedometer2 {
119    iterations: 3
120    suite: telemetry_Crosperf
121  }
122  """
123
124DUT_CONFIG_EXPERIMENT_FILE_BAD_CPUUSE = """
125  board: kevin64
126  remote: chromeos-kevin.cros
127  turbostat: False
128  governor: ondemand
129  cpu_usage: unknown
130
131  benchmark: speedometer2 {
132    iterations: 3
133    suite: telemetry_Crosperf
134  }
135
136  image1 {
137    chromeos_image:/usr/local/google/cros_image1.bin
138  }
139  """
140
141OUTPUT_FILE = """board: x86-alex
142remote: chromeos-alex3
143perf_args: record -a -e cycles
144
145benchmark: PageCycler {
146\titerations: 3
147}
148
149label: image1 {
150\tchromeos_image: /usr/local/google/cros_image1.bin
151\tremote: chromeos-alex3
152}
153
154label: image2 {
155\tchromeos_image: /usr/local/google/cros_image2.bin
156\tremote: chromeos-lumpy1
157}\n\n"""
158
159
160class ExperimentFileTest(unittest.TestCase):
161    """The main class for Experiment File test."""
162
163    def testLoadExperimentFile1(self):
164        input_file = io.StringIO(EXPERIMENT_FILE_1)
165        experiment_file = ExperimentFile(input_file)
166        global_settings = experiment_file.GetGlobalSettings()
167        self.assertEqual(global_settings.GetField("remote"), ["chromeos-alex3"])
168        self.assertEqual(
169            global_settings.GetField("perf_args"), "record -a -e cycles"
170        )
171        benchmark_settings = experiment_file.GetSettings("benchmark")
172        self.assertEqual(len(benchmark_settings), 1)
173        self.assertEqual(benchmark_settings[0].name, "PageCycler")
174        self.assertEqual(benchmark_settings[0].GetField("iterations"), 3)
175
176        label_settings = experiment_file.GetSettings("label")
177        self.assertEqual(len(label_settings), 2)
178        self.assertEqual(label_settings[0].name, "image1")
179        self.assertEqual(
180            label_settings[0].GetField("chromeos_image"),
181            "/usr/local/google/cros_image1.bin",
182        )
183        self.assertEqual(
184            label_settings[1].GetField("remote"), ["chromeos-lumpy1"]
185        )
186        self.assertEqual(
187            label_settings[0].GetField("remote"), ["chromeos-alex3"]
188        )
189
190    def testOverrideSetting(self):
191        input_file = io.StringIO(EXPERIMENT_FILE_2)
192        experiment_file = ExperimentFile(input_file)
193        global_settings = experiment_file.GetGlobalSettings()
194        self.assertEqual(global_settings.GetField("remote"), ["chromeos-alex3"])
195
196        benchmark_settings = experiment_file.GetSettings("benchmark")
197        self.assertEqual(len(benchmark_settings), 2)
198        self.assertEqual(benchmark_settings[0].name, "PageCycler")
199        self.assertEqual(benchmark_settings[0].GetField("iterations"), 3)
200        self.assertEqual(benchmark_settings[1].name, "AndroidBench")
201        self.assertEqual(benchmark_settings[1].GetField("iterations"), 2)
202
203    def testDuplicateLabel(self):
204        input_file = io.StringIO(EXPERIMENT_FILE_3)
205        self.assertRaises(Exception, ExperimentFile, input_file)
206
207    def testDuplicateBenchmark(self):
208        input_file = io.StringIO(EXPERIMENT_FILE_4)
209        experiment_file = ExperimentFile(input_file)
210        benchmark_settings = experiment_file.GetSettings("benchmark")
211        self.assertEqual(benchmark_settings[0].name, "webrtc")
212        self.assertEqual(
213            benchmark_settings[0].GetField("test_args"),
214            "--story-filter=datachannel",
215        )
216        self.assertEqual(benchmark_settings[1].name, "webrtc")
217        self.assertEqual(
218            benchmark_settings[1].GetField("test_args"),
219            "--story-tag-filter=smoothness",
220        )
221
222    def testCanonicalize(self):
223        input_file = io.StringIO(EXPERIMENT_FILE_1)
224        experiment_file = ExperimentFile(input_file)
225        res = experiment_file.Canonicalize()
226        self.assertEqual(res, OUTPUT_FILE)
227
228    def testLoadDutConfigExperimentFile_Good(self):
229        input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_GOOD)
230        experiment_file = ExperimentFile(input_file)
231        global_settings = experiment_file.GetGlobalSettings()
232        self.assertEqual(global_settings.GetField("turbostat"), False)
233        self.assertEqual(global_settings.GetField("intel_pstate"), "no_hwp")
234        self.assertEqual(global_settings.GetField("governor"), "powersave")
235        self.assertEqual(
236            global_settings.GetField("cpu_usage"), "exclusive_cores"
237        )
238        self.assertEqual(global_settings.GetField("cpu_freq_pct"), 50)
239        self.assertEqual(global_settings.GetField("cooldown_time"), 5)
240        self.assertEqual(global_settings.GetField("cooldown_temp"), 38)
241        self.assertEqual(global_settings.GetField("top_interval"), 5)
242
243    def testLoadDutConfigExperimentFile_WrongGovernor(self):
244        input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_GOV)
245        with self.assertRaises(RuntimeError) as msg:
246            ExperimentFile(input_file)
247        self.assertRegex(str(msg.exception), "governor: misspelled_governor")
248        self.assertRegex(
249            str(msg.exception),
250            "Invalid enum value for field 'governor'."
251            r" Must be one of \(performance, powersave, userspace, ondemand,"
252            r" conservative, schedutils, sched, interactive\)",
253        )
254
255    def testLoadDutConfigExperimentFile_WrongCpuUsage(self):
256        input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_CPUUSE)
257        with self.assertRaises(RuntimeError) as msg:
258            ExperimentFile(input_file)
259        self.assertRegex(str(msg.exception), "cpu_usage: unknown")
260        self.assertRegex(
261            str(msg.exception),
262            "Invalid enum value for field 'cpu_usage'."
263            r" Must be one of \(all, big_only, little_only, exclusive_cores\)",
264        )
265
266
267if __name__ == "__main__":
268    unittest.main()
269