xref: /aosp_15_r20/external/mesa3d/src/gallium/drivers/radeonsi/ci/radeonsi-run-tests.py (revision 6104692788411f58d303aa86923a9ff6ecaded22) 
1#!/usr/bin/python3
2#
3# Copyright 2021 Advanced Micro Devices, Inc.
4#
5# SPDX-License-Identifier: MIT
6#
7
8import os
9import sys
10import argparse
11import subprocess
12import shutil
13from datetime import datetime
14import tempfile
15import itertools
16import filecmp
17import multiprocessing
18import csv
19
20
21def print_red(txt, end_line=True, prefix=None):
22    if prefix:
23        print(prefix, end="")
24    print("\033[0;31m{}\033[0m".format(txt), end="\n" if end_line else " ")
25
26
27def print_yellow(txt, end_line=True, prefix=None):
28    if prefix:
29        print(prefix, end="")
30    print("\033[1;33m{}\033[0m".format(txt), end="\n" if end_line else " ")
31
32
33def print_green(txt, end_line=True, prefix=None):
34    if prefix:
35        print(prefix, end="")
36    print("\033[1;32m{}\033[0m".format(txt), end="\n" if end_line else " ")
37
38
39parser = argparse.ArgumentParser(
40    description="radeonsi tester",
41    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
42)
43parser.add_argument(
44    "--jobs",
45    "-j",
46    type=int,
47    help="Number of processes/threads to use.",
48    default=multiprocessing.cpu_count(),
49)
50
51# The path to above the mesa directory, i.e. ../../../../../..
52path_above_mesa = os.path.realpath(os.path.join(os.path.dirname(__file__), *['..'] * 6))
53
54parser.add_argument("--piglit-path", type=str, help="Path to piglit source folder.")
55parser.add_argument("--glcts-path", type=str, help="Path to GLCTS source folder.")
56parser.add_argument(
57    "--parent-path",
58    type=str,
59    help="Path to folder containing piglit/GLCTS and dEQP source folders.",
60    default=os.getenv('MAREKO_BUILD_PATH', path_above_mesa),
61)
62parser.add_argument("--verbose", "-v", action="count", default=0)
63parser.add_argument(
64    "--include-tests",
65    "-t",
66    action="append",
67    dest="include_tests",
68    default=[],
69    help="Only run the test matching this expression. This can only be a filename containing a list of failing tests to re-run.",
70)
71parser.add_argument(
72    "--baseline",
73    dest="baseline",
74    help="Folder containing expected results files",
75    default=os.path.dirname(__file__),
76)
77parser.add_argument(
78    "--no-piglit", dest="piglit", help="Disable piglit tests", action="store_false"
79)
80parser.add_argument(
81    "--no-glcts", dest="glcts", help="Disable GLCTS tests", action="store_false"
82)
83parser.add_argument(
84    "--no-escts", dest="escts", help="Disable GLES CTS tests", action="store_false"
85)
86parser.add_argument(
87    "--no-deqp", dest="deqp", help="Disable dEQP tests", action="store_false"
88)
89parser.add_argument(
90    "--slow", dest="slow", help="Include slowest glcts tests", action="store_true"
91)
92parser.add_argument(
93    "--no-deqp-egl",
94    dest="deqp_egl",
95    help="Disable dEQP-EGL tests",
96    action="store_false",
97)
98parser.add_argument(
99    "--no-deqp-gles2",
100    dest="deqp_gles2",
101    help="Disable dEQP-gles2 tests",
102    action="store_false",
103)
104parser.add_argument(
105    "--no-deqp-gles3",
106    dest="deqp_gles3",
107    help="Disable dEQP-gles3 tests",
108    action="store_false",
109)
110parser.add_argument(
111    "--no-deqp-gles31",
112    dest="deqp_gles31",
113    help="Disable dEQP-gles31 tests",
114    action="store_false",
115)
116parser.set_defaults(piglit=True)
117parser.set_defaults(glcts=True)
118parser.set_defaults(escts=True)
119parser.set_defaults(deqp=True)
120parser.set_defaults(deqp_egl=True)
121parser.set_defaults(deqp_gles2=True)
122parser.set_defaults(deqp_gles3=True)
123parser.set_defaults(deqp_gles31=True)
124parser.set_defaults(slow=False)
125
126parser.add_argument(
127    "output_folder",
128    nargs="?",
129    help="Output folder (logs, etc)",
130    default=os.path.join(
131        # Default is ../../../../../../test-results/datetime
132        os.path.join(path_above_mesa, 'test-results',
133                     datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))
134    ),
135)
136
137available_gpus = []
138for f in os.listdir("/dev/dri/by-path"):
139    idx = f.find("-render")
140    if idx < 0:
141        continue
142    # gbm name is the full path, but DRI_PRIME expects a different
143    # format
144    available_gpus += [
145        (
146            os.path.join("/dev/dri/by-path", f),
147            f[:idx].replace(":", "_").replace(".", "_"),
148        )
149    ]
150
151parser.add_argument(
152    "--gpu",
153    type=int,
154    dest="gpu",
155    default=0,
156    help="Select GPU (0..{})".format(len(available_gpus) - 1),
157)
158
159args = parser.parse_args(sys.argv[1:])
160piglit_path = args.piglit_path
161glcts_path = args.glcts_path
162
163if args.parent_path:
164    if args.piglit_path or args.glcts_path:
165        parser.print_help()
166        sys.exit(0)
167    piglit_path = os.path.join(args.parent_path, "piglit")
168    glcts_path = os.path.join(args.parent_path, "glcts")
169else:
170    if not args.piglit_path or not args.glcts_path:
171        parser.print_help()
172        sys.exit(0)
173
174base = args.baseline
175skips = os.path.join(os.path.dirname(__file__), "skips.csv")
176
177env = os.environ.copy()
178
179if "DISPLAY" not in env:
180    print_red("DISPLAY environment variable missing.")
181    sys.exit(1)
182p = subprocess.run(
183    ["deqp-runner", "--version"], capture_output="True", check=True, env=env
184)
185for line in p.stdout.decode().split("\n"):
186    if line.find("deqp-runner") >= 0:
187        s = line.split(" ")[1].split(".")
188        if args.verbose > 1:
189            print("Checking deqp-version ({})".format(s))
190        # We want at least 0.9.0
191        if not (int(s[0]) > 0 or int(s[1]) >= 9):
192            print("Expecting deqp-runner 0.9.0+ version (got {})".format(".".join(s)))
193            sys.exit(1)
194
195env["PIGLIT_PLATFORM"] = "gbm"
196
197if "DRI_PRIME" in env:
198    print("Don't use DRI_PRIME. Instead use --gpu N")
199    del env["DRI_PRIME"]
200
201assert "gpu" in args, "--gpu defaults to 0"
202
203gpu_device = available_gpus[args.gpu][1]
204env["DRI_PRIME"] = gpu_device
205env["WAFFLE_GBM_DEVICE"] = available_gpus[args.gpu][0]
206
207# Use piglit's glinfo to determine the GPU name
208gpu_name = "unknown"
209gpu_name_full = ""
210gfx_level = -1
211
212amd_debug = env["AMD_DEBUG"] if "AMD_DEBUG" in env else ""
213env["AMD_DEBUG"] = "info"
214p = subprocess.run(
215    ["./glinfo"],
216    capture_output="True",
217    cwd=os.path.join(piglit_path, "bin"),
218    check=True,
219    env=env,
220)
221del env["AMD_DEBUG"]
222env["AMD_DEBUG"] = amd_debug
223
224for line in p.stdout.decode().split("\n"):
225    if "GL_RENDER" in line:
226        line = line.split("=")[1]
227        gpu_name_full = "(".join(line.split("(")[:-1]).strip()
228        gpu_name = line.replace("(TM)", "").split("(")[1].split(",")[1].lower().strip()
229        break
230    elif "gfx_level" in line:
231        gfx_level = int(line.split("=")[1])
232
233output_folder = args.output_folder
234print_green("Tested GPU: '{}' ({}) {}".format(gpu_name_full, gpu_name, gpu_device))
235print_green("Output folder: '{}'".format(output_folder))
236
237count = 1
238while os.path.exists(output_folder):
239    output_folder = "{}.{}".format(os.path.abspath(args.output_folder), count)
240    count += 1
241
242os.makedirs(output_folder, exist_ok=True)
243
244logfile = open(os.path.join(output_folder, "{}-run-tests.log".format(gpu_name)), "w")
245
246spin = itertools.cycle("-\\|/")
247
248shutil.copy(skips, output_folder)
249skips = os.path.join(output_folder, "skips.csv")
250if not args.slow:
251    # Exclude these 4 tests slow tests
252    with open(skips, "a") as f:
253        print("KHR-GL46.copy_image.functional", file=f)
254        print("KHR-GL46.texture_swizzle.smoke", file=f)
255        print(
256            "KHR-GL46.tessellation_shader.tessellation_control_to_tessellation_evaluation.gl_MaxPatchVertices_Position_PointSize",
257            file=f,
258        )
259        print("KHR-Single-GL46.arrays_of_arrays_gl.AtomicUsage", file=f)
260
261
262def gfx_level_to_str(cl):
263    supported = ["gfx6", "gfx7", "gfx8", "gfx9", "gfx10", "gfx10_3", "gfx11", "gfx12"]
264    if 8 <= cl and cl < 8 + len(supported):
265        return supported[cl - 8]
266    return supported[-1]
267
268
269def run_cmd(args, verbosity):
270    if verbosity > 1:
271        print_yellow(
272            "| Command line argument '"
273            + " ".join(['"{}"'.format(a) for a in args])
274            + "'"
275        )
276    start = datetime.now()
277    proc = subprocess.Popen(
278        args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env
279    )
280    while True:
281        line = proc.stdout.readline().decode()
282        if verbosity > 0:
283            if "ERROR" in line:
284                print_red(line.strip(), prefix="| ")
285            else:
286                print("| " + line.strip())
287        else:
288            sys.stdout.write(next(spin))
289            sys.stdout.flush()
290            sys.stdout.write("\b")
291
292        logfile.write(line)
293
294        if proc.poll() is not None:
295            break
296    proc.wait()
297    end = datetime.now()
298
299    if verbosity == 0:
300        sys.stdout.write(" ... ")
301
302    print_yellow(
303        "Completed in {} seconds".format(int((end - start).total_seconds())),
304        prefix="└ " if verbosity > 0 else None,
305    )
306
307
308def verify_results(results):
309    with open(results) as file:
310        lines = file.readlines()
311        if len(lines) == 0:
312            return True
313        print("{} new result{}:".format(len(lines), 's' if len(lines) > 1 else ''))
314        for i in range(min(10, len(lines))):
315            print("  * ", end='')
316            if "Pass" in lines[i]:
317                print_green(lines[i][:-1])
318            else:
319                print_red(lines[i][:-1])
320        if len(lines) > 10:
321            print_yellow("...")
322        print("Full results: {}".format(results))
323
324    return False
325
326
327def parse_test_filters(include_tests, baseline):
328    cmd = []
329    for t in include_tests:
330        if t == 'baseline':
331            t = baseline
332
333        if os.path.exists(t):
334            with open(t, "r") as file:
335                for row in csv.reader(file, delimiter=","):
336                    if not row or row[0][0] == "#":
337                        continue
338                    cmd += ["-t", row[0]]
339        else:
340            cmd += ["-t", t]
341    return cmd
342
343
344def select_baseline(basepath, gfx_level, gpu_name):
345    gfx_level_str = gfx_level_to_str(gfx_level)
346
347    # select the best baseline we can find
348    # 1. exact match
349    exact = os.path.join(base, "{}-{}-fail.csv".format(gfx_level_str, gpu_name))
350    if os.path.exists(exact):
351        return exact
352    # 2. any baseline with the same gfx_level
353    while gfx_level >= 8:
354        gfx_level_str += '-'
355        for subdir, dirs, files in os.walk(basepath):
356            for file in files:
357                if file.find(gfx_level_str) == 0 and file.endswith("-fail.csv"):
358                    return os.path.join(base, file)
359        # No match. Try an earlier class
360        gfx_level = gfx_level - 1
361        gfx_level_str = gfx_level_to_str(gfx_level)
362
363    return exact
364
365
366success = True
367baseline = select_baseline(base, gfx_level, gpu_name)
368filters_args = parse_test_filters(args.include_tests, baseline)
369flakes = [
370    f
371    for f in (
372        os.path.join(base, g)
373        for g in [
374            "radeonsi-flakes.csv",
375            "{}-{}-flakes.csv".format(gfx_level_to_str(gfx_level), gpu_name),
376        ]
377    )
378    if os.path.exists(f)
379]
380flakes_args = []
381for f in flakes:
382    flakes_args += ["--flakes", f]
383
384if os.path.exists(baseline):
385    print_yellow("Baseline: {}".format(baseline))
386if flakes_args:
387    print_yellow("Flakes: {}".format(flakes_args))
388
389# piglit test
390if args.piglit:
391    out = os.path.join(output_folder, "piglit")
392    print_yellow("Running piglit tests", args.verbose > 0)
393    cmd = [
394        "piglit-runner",
395        "run",
396        "--piglit-folder",
397        piglit_path,
398        "--profile",
399        "quick",
400        "--output",
401        out,
402        "--process-isolation",
403        "--timeout",
404        "300",
405        "--jobs",
406        str(args.jobs),
407        "--skips",
408        skips,
409        "--skips",
410        os.path.join(path_above_mesa, "mesa", ".gitlab-ci", "gbm-skips.txt")
411    ] + filters_args + flakes_args
412
413    if os.path.exists(baseline):
414        cmd += ["--baseline", baseline]
415
416    run_cmd(cmd, args.verbose)
417
418    if not verify_results(os.path.join(out, "failures.csv")):
419        success = False
420
421deqp_args = "-- --deqp-surface-width=256 --deqp-surface-height=256 --deqp-gl-config-name=rgba8888d24s8ms0 --deqp-visibility=hidden".split(
422    " "
423)
424
425# glcts test
426if args.glcts:
427    out = os.path.join(output_folder, "glcts")
428    print_yellow("Running  GLCTS tests", args.verbose > 0)
429    os.mkdir(os.path.join(output_folder, "glcts"))
430
431    cmd = [
432        "deqp-runner",
433        "run",
434        "--tests-per-group",
435        "100",
436        "--deqp",
437        "{}/build/external/openglcts/modules/glcts".format(glcts_path),
438        "--caselist",
439        "{}/external/openglcts/data/gl_cts/data/mustpass/gl/khronos_mustpass/4.6.1.x/gl46-main.txt".format(
440            glcts_path
441        ),
442        "--caselist",
443        "{}/external/openglcts/data/gl_cts/data/mustpass/gl/khronos_mustpass_single/4.6.1.x/gl46-khr-single.txt".format(
444            glcts_path
445        ),
446        "--caselist",
447        "{}/external/openglcts/data/gl_cts/data/mustpass/gl/khronos_mustpass/4.6.1.x/gl46-gtf-main.txt".format(
448            glcts_path
449        ),
450        "--output",
451        out,
452        "--skips",
453        skips,
454        "--jobs",
455        str(args.jobs),
456        "--timeout",
457        "1000"
458    ] + filters_args + flakes_args
459
460    if os.path.exists(baseline):
461        cmd += ["--baseline", baseline]
462    cmd += deqp_args
463
464    run_cmd(cmd, args.verbose)
465
466    if not verify_results(os.path.join(out, "failures.csv")):
467        success = False
468
469# escts test
470if args.escts:
471    out = os.path.join(output_folder, "escts")
472    print_yellow("Running  ESCTS tests", args.verbose > 0)
473    os.mkdir(out)
474
475    cmd = [
476        "deqp-runner",
477        "run",
478        "--tests-per-group",
479        "100",
480        "--deqp",
481        "{}/build_es/external/openglcts/modules/glcts".format(glcts_path),
482        "--caselist",
483        "{}/external/openglcts/data/gl_cts/data/mustpass/gles/khronos_mustpass/3.2.6.x/gles2-khr-main.txt".format(
484            glcts_path
485        ),
486        "--caselist",
487        "{}/external/openglcts/data/gl_cts/data/mustpass/gles/khronos_mustpass/3.2.6.x/gles3-khr-main.txt".format(
488            glcts_path
489        ),
490        "--caselist",
491        "{}/external/openglcts/data/gl_cts/data/mustpass/gles/khronos_mustpass/3.2.6.x/gles31-khr-main.txt".format(
492            glcts_path
493        ),
494        "--caselist",
495        "{}/external/openglcts/data/gl_cts/data/mustpass/gles/khronos_mustpass/3.2.6.x/gles32-khr-main.txt".format(
496            glcts_path
497        ),
498        "--output",
499        out,
500        "--skips",
501        skips,
502        "--jobs",
503        str(args.jobs),
504        "--timeout",
505        "1000"
506    ] + filters_args + flakes_args
507
508    if os.path.exists(baseline):
509        cmd += ["--baseline", baseline]
510    cmd += deqp_args
511
512    run_cmd(cmd, args.verbose)
513
514    if not verify_results(os.path.join(out, "failures.csv")):
515        success = False
516
517if args.deqp:
518    print_yellow("Running   dEQP tests", args.verbose > 0)
519
520    # Generate a test-suite file
521    out = os.path.join(output_folder, "deqp")
522    suite_filename = os.path.join(output_folder, "deqp-suite.toml")
523    suite = open(suite_filename, "w")
524    os.mkdir(out)
525
526    deqp_tests = {
527        "egl": args.deqp_egl,
528        "gles2": args.deqp_gles2,
529        "gles3": args.deqp_gles3,
530        "gles31": args.deqp_gles31,
531    }
532
533    for k in deqp_tests:
534        if not deqp_tests[k]:
535            continue
536
537        suite.write("[[deqp]]\n")
538        suite.write(
539            'deqp = "{}"\n'.format(
540                "{}/build/modules/{subtest}/deqp-{subtest}".format(glcts_path, subtest=k)
541            )
542        )
543        suite.write(
544            'caselists = ["{}"]\n'.format(
545                "{}/external/openglcts/data/gl_cts/data/mustpass/{}/aosp_mustpass/3.2.6.x/{}-main.txt".format(glcts_path, "egl" if k == "egl" else "gles", k)
546            )
547        )
548        if os.path.exists(baseline):
549            suite.write('baseline = "{}"\n'.format(baseline))
550        suite.write('skips = ["{}"]\n'.format(skips))
551        suite.write("deqp_args = [\n")
552        for a in deqp_args[1:-1]:
553            suite.write('    "{}",\n'.format(a))
554        suite.write('    "{}"\n'.format(deqp_args[-1]))
555        suite.write("]\n")
556
557    suite.close()
558
559    cmd = [
560        "deqp-runner",
561        "suite",
562        "--jobs",
563        str(args.jobs),
564        "--output",
565        os.path.join(output_folder, "deqp"),
566        "--suite",
567        suite_filename,
568    ] + filters_args + flakes_args
569
570    run_cmd(cmd, args.verbose)
571
572    if not verify_results(os.path.join(out, "failures.csv")):
573        success = False
574
575sys.exit(0 if success else 1)
576