1#!/usr/bin/env python3
2#
3# Copyright (C) 2018 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17#
18# This test script to be used by the build server.
19# It is supposed to be executed from trusty root directory
20# and expects the following environment variables:
21#
22"""Parse trusty build and test configuration files."""
23
24import argparse
25import os
26import re
27from enum import StrEnum, auto
28from typing import List, Dict, Optional
29
30script_dir = os.path.dirname(os.path.abspath(__file__))
31
32class PortType(StrEnum):
33    TEST = auto()
34    BENCHMARK = auto()
35
36
37class TrustyBuildConfigProject(object):
38    """Stores build enabled status and test lists for a project
39
40    Attributes:
41        build: A boolean indicating if project should be built be default.
42        tests: A list of commands to run to test this project.
43        also_build: Set of project to also build if building this one.
44    """
45
46    def __init__(self):
47        """Inits TrustyBuildConfigProject with an empty test list and no
48           build."""
49        self.build = False
50        self.tests = []
51        self.also_build = {}
52        self.signing_keys = None
53
54
55class TrustyPortTestFlags(object):
56    """Stores need flags for a test or provide flags for a test environment."""
57
58    ALLOWED_FLAGS = {"android", "storage_boot", "storage_full",
59                     "smp4", "abl", "tablet"}
60
61    def __init__(self, **flags):
62        self.flags = set()
63        self.set(**flags)
64
65    def set(self, **flags):
66        """Set flags."""
67        for name, arg in flags.items():
68            if name in self.ALLOWED_FLAGS:
69                if arg:
70                    self.flags.add(name)
71                else:
72                    self.flags.discard(name)
73            else:
74                raise TypeError("Unexpected flag: " + name)
75
76    def match_provide(self, provide):
77        return self.flags.issubset(provide.flags)
78
79
80class TrustyArchiveBuildFile(object):
81    """Copy a file to archive directory after a build."""
82    def __init__(self, src, dest, optional):
83        self.src = src
84        self.dest = dest
85        self.optional = optional
86
87
88class TrustyTest(object):
89    """Stores a pair of a test name and a command to run"""
90    def __init__(self, name, command, enabled, port_type = PortType.TEST):
91        self.name = name
92        self.command = command
93        self.enabled = enabled
94        self.port_type = port_type
95
96    def type(self, port_type):
97        self.port_type = PortType(port_type)  # ensure we have a valid port type
98        return self
99
100class TrustyHostTest(TrustyTest):
101    """Stores a pair of a test name and a command to run on host."""
102
103    class TrustyHostTestFlags:
104        """Enable needs to be matched with provides without special casing"""
105
106        @staticmethod
107        def match_provide(_):
108            # cause host test to be filtered out if they appear in a boottests
109            # or androidportests environment which provides a set of features.
110            return False
111
112    need = TrustyHostTestFlags()
113
114
115class TrustyAndroidTest(TrustyTest):
116    """Stores a test name and command to run inside Android"""
117
118    def __init__(self, name, command, need=None,
119                 port_type=PortType.TEST, enabled=True, nameprefix="",
120                 runargs=(), timeout=None):
121        nameprefix = nameprefix + "android-test:"
122        cmd = ["run", "--headless", "--shell-command", command]
123        if timeout:
124            cmd += ['--timeout', str(timeout)]
125        if runargs:
126            cmd += list(runargs)
127        super().__init__(nameprefix + name, cmd, enabled, port_type)
128        self.shell_command = command
129        if need:
130            self.need = need
131        else:
132            self.need = TrustyPortTestFlags()
133
134    def needs(self, **need):
135        self.need.set(**need)
136        return self
137
138
139class TrustyPortTest(TrustyTest):
140    """Stores a trusty port name for a test to run."""
141
142    def __init__(self, port, port_type=PortType.TEST, enabled=True,
143                 timeout=None):
144        super().__init__(port, None, enabled, port_type)
145        self.port = port
146        self.need = TrustyPortTestFlags()
147        self.timeout = timeout
148
149    def needs(self, **need):
150        self.need.set(**need)
151        return self
152
153    def into_androidporttest(self, cmdargs, **kwargs):
154        cmdargs = list(cmdargs)
155        cmd = " ".join(["/vendor/bin/trusty-ut-ctrl", self.port] + cmdargs)
156        return TrustyAndroidTest(self.name, cmd, self.need, self.port_type,
157                                 self.enabled, timeout=self.timeout, **kwargs)
158
159    def into_bootporttest(self) -> TrustyTest:
160        cmd = ["run", "--headless", "--boot-test", self.port]
161        cmd += ['--timeout', str(self.timeout)] if self.timeout else []
162        return TrustyTest("boot-test:" + self.port, cmd, self.enabled,
163                          self.port_type)
164
165
166class TrustyCommand:
167    """Base class for all types of commands that are *not* tests"""
168
169    def __init__(self, name):
170        self.name = name
171        self.enabled = True
172        # avoids special cases in list_config
173        self.command = []
174        # avoids special cases in porttest_match
175        self.need = TrustyPortTestFlags()
176
177    def needs(self, **_):
178        """Allows commands to be used inside a needs block."""
179        return self
180
181    def into_androidporttest(self, **_):
182        return self
183
184    def into_bootporttest(self):
185        return self
186
187
188class RebootMode(StrEnum):
189    REGULAR = "reboot"
190    FACTORY_RESET = "reboot (with factory reset)"
191    FULL_WIPE = "reboot (with full wipe)"
192
193    def factory_reset(self) -> bool:
194        """Whether this reboot includes a factory reset.
195        This function exists because we can't make the test runner module depend
196        on types defined here, so its function args have to be builtins.
197        """
198        match self:
199            case RebootMode.REGULAR:
200                return False
201            case RebootMode.FACTORY_RESET:
202                return True
203            case RebootMode.FULL_WIPE:
204                return True
205
206    def full_wipe(self) -> bool:
207        """Whether this reboot includes a an RPMB wipe.
208        This function exists because we can't make the test runner module depend
209        on types defined here, so its function args have to be builtins.
210        """
211        match self:
212            case RebootMode.REGULAR:
213                return False
214            case RebootMode.FACTORY_RESET:
215                return False
216            case RebootMode.FULL_WIPE:
217                return True
218
219
220class TrustyRebootCommand(TrustyCommand):
221    """Marker object which causes the test environment to be rebooted before the
222       next test is run. Used to reset the test environment and to test storage.
223    """
224    def __init__(self, mode: RebootMode = RebootMode.FACTORY_RESET):
225        super().__init__(mode)
226        self.mode = mode
227
228class TrustyPrintCommand(TrustyCommand):
229
230    def msg(self) -> str:
231        return self.name
232
233class TrustyCompositeTest(TrustyTest):
234    """Stores a sequence of tests that must execute in order"""
235
236    def __init__(self, name: str,
237                 sequence: List[TrustyPortTest | TrustyCommand],
238                 enabled=True):
239        super().__init__(name, [], enabled)
240        self.sequence = sequence
241        flags = set()
242        for subtest in sequence:
243            flags.update(subtest.need.flags)
244        self.need = TrustyPortTestFlags(**{flag: True for flag in flags})
245
246    def needs(self, **need):
247        self.need.set(**need)
248        return self
249
250    def into_androidporttest(self, **kwargs):
251        # because the needs of the composite test is the union of the needs of
252        # its subtests, we do not need to filter out any subtests; all needs met
253        self.sequence = [subtest.into_androidporttest(**kwargs)
254                         for subtest in self.sequence]
255        return self
256
257    def into_bootporttest(self):
258        # similarly to into_androidporttest, we do not need to filter out tests
259        self.sequence = [subtest.into_bootporttest()
260                         for subtest in self.sequence]
261        return self
262
263
264class TrustyBuildConfig(object):
265    """Trusty build and test configuration file parser."""
266
267    def __init__(self, config_file=None, debug=False, android=None):
268        """Inits TrustyBuildConfig.
269
270        Args:
271            config_file: Optional config file path. If omitted config file is
272                found relative to script directory.
273            debug: Optional boolean value. Set to True to enable debug messages.
274        """
275        self.debug = debug
276        self.android = android
277        self.projects = {}
278        self.dist = []
279        self.default_signing_keys = []
280        self.doc_files = []
281
282        if config_file is None:
283            config_file = os.path.join(script_dir, "build-config")
284        self.read_config_file(config_file)
285
286    def read_config_file(self, path, optional=False):
287        """Main parser function called constructor or recursively by itself."""
288        if optional and not os.path.exists(path):
289            if self.debug:
290                print("Skipping optional config file:", path)
291            return []
292
293        if self.debug:
294            print("Reading config file:", path)
295
296        config_dir = os.path.dirname(path)
297
298        def _flatten_list(inp, out):
299            for obj in inp:
300                if isinstance(obj, list):
301                    _flatten_list(obj, out)
302                else:
303                    out.append(obj)
304
305        def flatten_list(inp):
306            out = []
307            _flatten_list(inp, out)
308            return out
309
310        def include(path, optional=False):
311            """Process include statement in config file."""
312            if self.debug:
313                print("include", path, "optional", optional)
314            if path.startswith("."):
315                path = os.path.join(config_dir, path)
316            return self.read_config_file(path=path, optional=optional)
317
318        def build(projects, enabled=True, dist=None):
319            """Process build statement in config file."""
320            for project_name in projects:
321                if self.debug:
322                    print("build", project_name, "enabled", enabled)
323                project = self.get_project(project_name)
324                project.build = bool(enabled)
325            if dist:
326                for item in dist:
327                    assert isinstance(item, TrustyArchiveBuildFile), item
328                    self.dist.append(item)
329
330        def builddep(projects, needs):
331            """Process build statement in config file."""
332            for project_name in projects:
333                project = self.get_project(project_name)
334                for project_dep_name in needs:
335                    project_dep = self.get_project(project_dep_name)
336                    if self.debug:
337                        print("build", project_name, "needs", project_dep_name)
338                    project.also_build[project_dep_name] = project_dep
339
340        def archive(src, dest=None, optional=False):
341            return TrustyArchiveBuildFile(src, dest, optional)
342
343        def testmap(projects, tests=()):
344            """Process testmap statement in config file."""
345            for project_name in projects:
346                if self.debug:
347                    print("testmap", project_name, "build", build)
348                    for test in tests:
349                        print(test)
350                project = self.get_project(project_name)
351                project.tests += flatten_list(tests)
352
353        def hosttest(host_cmd, enabled=True, repeat=1):
354            cmd = ["host_tests/" + host_cmd]
355            # TODO: assumes that host test is always a googletest
356            if repeat > 1:
357                cmd.append(f"--gtest_repeat={repeat}")
358            return TrustyHostTest("host-test:" + host_cmd, cmd, enabled)
359
360        def hosttests(tests):
361            return [test for test in flatten_list(tests)
362                    if isinstance(test, TrustyHostTest)]
363
364        def porttest_match(test, provides):
365            return test.need.match_provide(provides)
366
367        def porttests_filter(tests, provides):
368            return [test for test in flatten_list(tests)
369                    if porttest_match(test, provides)]
370
371        def boottests(port_tests, provides=None):
372            if provides is None:
373                provides = TrustyPortTestFlags(storage_boot=True,
374                                               smp4=True)
375            return [test.into_bootporttest()
376                    for test in porttests_filter(port_tests, provides)]
377
378        def androidporttests(port_tests, provides=None, nameprefix="",
379                             cmdargs=(), runargs=()):
380            nameprefix = nameprefix + "android-port-test:"
381            if provides is None:
382                provides = TrustyPortTestFlags(android=True,
383                                               storage_boot=True,
384                                               storage_full=True,
385                                               smp4=True,
386                                               abl=True,
387                                               tablet=True)
388
389            return [test.into_androidporttest(nameprefix=nameprefix,
390                                              cmdargs=cmdargs,
391                                              runargs=runargs)
392                    for test in porttests_filter(port_tests, provides)]
393
394        def needs(tests, *args, **kwargs):
395            return [
396                test.needs(*args, **kwargs)
397                for test in flatten_list(tests)
398            ]
399
400        def devsigningkeys(
401            default_key_paths: List[str],
402            project_overrides: Optional[Dict[str, List[str]]] = None):
403            self.default_signing_keys.extend(default_key_paths)
404            if project_overrides is None:
405                return
406
407            for project_name, overrides in project_overrides.items():
408                project = self.get_project(project_name)
409                if project.signing_keys is None:
410                    project.signing_keys = []
411                project.signing_keys.extend(overrides)
412
413        def docs(doc_files: List[str]):
414            self.doc_files.extend(doc_files)
415
416        file_format = {
417            "BENCHMARK": PortType.BENCHMARK,
418            "TEST": PortType.TEST,
419            "include": include,
420            "build": build,
421            "builddep": builddep,
422            "archive": archive,
423            "testmap": testmap,
424            "hosttest": hosttest,
425            "porttest": TrustyPortTest,
426            "compositetest": TrustyCompositeTest,
427            "porttestflags": TrustyPortTestFlags,
428            "hosttests": hosttests,
429            "boottests": boottests,
430            "androidtest": TrustyAndroidTest,
431            "androidporttests": androidporttests,
432            "needs": needs,
433            "reboot": TrustyRebootCommand,
434            "RebootMode": RebootMode,
435            "devsigningkeys": devsigningkeys,
436            "print": TrustyPrintCommand,
437            "docs": docs,
438        }
439
440        with open(path, encoding="utf8") as f:
441            code = compile(f.read(), path, "eval")
442            config = eval(code, file_format)  # pylint: disable=eval-used
443            return flatten_list(config)
444
445    def get_project(self, project):
446        """Return TrustyBuildConfigProject entry for a project."""
447        if project not in self.projects:
448            self.projects[project] = TrustyBuildConfigProject()
449        return self.projects[project]
450
451    def get_projects(self, build=None, have_tests=None):
452        """Return a list of projects.
453
454        Args:
455            build: If True only return projects that should be built. If False
456                only return projects that should not be built. If None return
457                both projects that should be built and not be built. (default
458                None).
459            have_tests: If True only return projects that have tests. If False
460                only return projects that don't have tests. If None return
461                projects regardless if they have tests. (default None).
462        """
463
464        def match(item):
465            """filter function for get_projects."""
466            project = self.projects[item]
467
468            return ((build is None or build == project.build) and
469                    (have_tests is None or
470                     have_tests == bool(project.tests)))
471
472        return (project for project in sorted(self.projects.keys())
473                if match(project))
474
475    def signing_keys(self, project_name: str):
476        project_specific_keys = self.get_project(project_name).signing_keys
477        if project_specific_keys is None:
478            return self.default_signing_keys
479        return project_specific_keys
480
481
482def list_projects(args):
483    """Read config file and print a list of projects.
484
485    See TrustyBuildConfig.get_projects for filtering options.
486
487    Args:
488        args: Program arguments.
489    """
490    config = TrustyBuildConfig(config_file=args.file, debug=args.debug)
491    for project in sorted(config.get_projects(**dict(args.filter))):
492        print(project)
493
494
495def list_config(args):
496    """Read config file and print all project and tests."""
497    config = TrustyBuildConfig(config_file=args.file, debug=args.debug)
498    print("Projects:")
499    for project_name, project in sorted(config.projects.items()):
500        print("  " + project_name + ":")
501        print("    Build:", project.build)
502        print("    Tests:")
503        for test in project.tests:
504            print("      " + test.name + ":")
505            print("        " + str(test.command))
506
507    for build in [True, False]:
508        print()
509        print("Build:" if build else "Don't build:")
510        for tested in [True, False]:
511            projects = config.get_projects(build=build, have_tests=tested)
512            for project in sorted(projects):
513                print("  " + project + ":")
514                project_config = config.get_project(project)
515                for test in project_config.tests:
516                    print("    " + test.name)
517            if projects and not tested:
518                print("    No tests")
519
520
521def any_test_name(regex, tests):
522    """Checks the name of all tests in a list for a regex.
523
524    This is intended only as part of the selftest facility, do not use it
525    to decide how to consider actual tests.
526
527    Args:
528        tests: List of tests to check the names of
529        regex: Regular expression to check them for (as a string)
530    """
531
532    return any(re.match(regex, test.name) is not None for test in tests)
533
534
535def has_host(tests):
536    """Checks for a host test in the provided tests by name.
537
538    This is intended only as part of the selftest facility, do not use it
539    to decide how to consider actual tests.
540
541    Args:
542        tests: List of tests to check for host tests
543    """
544    return any_test_name("host-test:", tests)
545
546
547def has_unit(tests):
548    """Checks for a unit test in the provided tests by name.
549
550    This is intended only as part of the selftest facility, do not use it
551    to decide how to consider actual tests.
552
553    Args:
554        tests: List of tests to check for unit tests
555    """
556    return any_test_name("boot-test:", tests)
557
558
559def test_config(args):
560    """Test config file parser.
561
562    Uses a test config file where all projects have names that describe if they
563    should be built and if they have tests.
564
565    Args:
566        args: Program arguments.
567    """
568    config_file = os.path.join(script_dir, "trusty_build_config_self_test_main")
569    config = TrustyBuildConfig(config_file=config_file, debug=args.debug)
570
571    projects_build = {}
572
573    project_regex = re.compile(
574        r"self_test\.build_(yes|no)\.tests_(none|host|unit|both)\..*")
575
576    for build in [None, True, False]:
577        projects_build[build] = {}
578        for tested in [None, True, False]:
579            projects = list(config.get_projects(build=build, have_tests=tested))
580            projects_build[build][tested] = projects
581            if args.debug:
582                print("Build", build, "tested", tested, "count", len(projects))
583            assert projects
584            for project in projects:
585                if args.debug:
586                    print("-", project)
587                m = project_regex.match(project)
588                assert m
589                if build is not None:
590                    assert m.group(1) == ("yes" if build else "no")
591                if tested is not None:
592                    if tested:
593                        assert (m.group(2) == "host" or
594                                m.group(2) == "unit" or
595                                m.group(2) == "both")
596                    else:
597                        assert m.group(2) == "none"
598
599        assert(projects_build[build][None] ==
600               sorted(projects_build[build][True] +
601                      projects_build[build][False]))
602    for tested in [None, True, False]:
603        assert(projects_build[None][tested] ==
604               sorted(projects_build[True][tested] +
605                      projects_build[False][tested]))
606
607    print("get_projects test passed")
608
609    reboot_seen = False
610
611    def check_test(i, test):
612        match test:
613            case TrustyTest():
614                host_m = re.match(r"host-test:self_test.*\.(\d+)",
615                                  test.name)
616                unit_m = re.match(r"boot-test:self_test.*\.(\d+)",
617                                  test.name)
618                if args.debug:
619                    print(project, i, test.name)
620                m = host_m or unit_m
621                assert m
622                assert m.group(1) == str(i + 1)
623            case TrustyRebootCommand():
624                assert False, "Reboot outside composite command"
625            case _:
626                assert False, "Unexpected test type"
627
628    def check_subtest(i, test):
629        nonlocal reboot_seen
630        match test:
631            case TrustyRebootCommand():
632                reboot_seen = True
633            case _:
634                check_test(i, test)
635
636    for project_name in config.get_projects():
637        project = config.get_project(project_name)
638        if args.debug:
639            print(project_name, project)
640        m = project_regex.match(project_name)
641        assert m
642        kind = m.group(2)
643        if kind == "both":
644            assert has_host(project.tests)
645            assert has_unit(project.tests)
646        elif kind == "unit":
647            assert not has_host(project.tests)
648            assert has_unit(project.tests)
649        elif kind == "host":
650            assert has_host(project.tests)
651            assert not has_unit(project.tests)
652        elif kind == "none":
653            assert not has_host(project.tests)
654            assert not has_unit(project.tests)
655        else:
656            assert False, "Unknown project kind"
657
658        for i, test in enumerate(project.tests):
659            match test:
660                case TrustyCompositeTest():
661                    # because one of its subtest needs storage_boot,
662                    # the composite test should similarly need it
663                    assert "storage_boot" in test.need.flags
664                    for subtest in test.sequence:
665                        check_subtest(i, subtest)
666                case _:
667                    check_test(i, test)
668
669    assert reboot_seen
670
671    print("get_tests test passed")
672
673
674def main():
675    top = os.path.abspath(os.path.join(script_dir, "../../../../.."))
676    os.chdir(top)
677
678    parser = argparse.ArgumentParser()
679    parser.add_argument("-d", "--debug", action="store_true")
680    parser.add_argument("--file")
681    # work around for https://bugs.python.org/issue16308
682    parser.set_defaults(func=lambda args: parser.print_help())
683    subparsers = parser.add_subparsers()
684
685    parser_projects = subparsers.add_parser("projects",
686                                            help="list project names")
687
688    group = parser_projects.add_mutually_exclusive_group()
689    group.add_argument("--with-tests", action="append_const",
690                       dest="filter", const=("have_tests", True),
691                       help="list projects that have tests")
692    group.add_argument("--without-tests", action="append_const",
693                       dest="filter", const=("have_tests", False),
694                       help="list projects that don't have tests")
695
696    group = parser_projects.add_mutually_exclusive_group()
697    group.add_argument("--all", action="append_const",
698                       dest="filter", const=("build", None),
699                       help="include disabled projects")
700    group.add_argument("--disabled", action="append_const",
701                       dest="filter", const=("build", False),
702                       help="only list disabled projects")
703    parser_projects.set_defaults(func=list_projects, filter=[("build", True)])
704
705    parser_config = subparsers.add_parser("config", help="dump config")
706    parser_config.set_defaults(func=list_config)
707
708    parser_config = subparsers.add_parser("selftest", help="test config parser")
709    parser_config.set_defaults(func=test_config)
710
711    args = parser.parse_args()
712    args.func(args)
713
714
715if __name__ == "__main__":
716    main()
717