1#!/usr/bin/env python3 2"""Run the PSA Crypto API compliance test suite. 3Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF, 4then compile and run the test suite. The clone is stored at <repository root>/psa-arch-tests. 5Known defects in either the test suite or mbedtls / psa-crypto - identified by their test 6number - are ignored, while unexpected failures AND successes are reported as errors, to help 7keep the list of known defects as up to date as possible. 8""" 9 10# Copyright The Mbed TLS Contributors 11# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 12 13import argparse 14import os 15import re 16import shutil 17import subprocess 18import sys 19from typing import List 20 21#pylint: disable=unused-import 22import scripts_path 23from mbedtls_dev import build_tree 24 25# PSA Compliance tests we expect to fail due to known defects in Mbed TLS / PSA Crypto 26# (or the test suite). 27# The test numbers correspond to the numbers used by the console output of the test suite. 28# Test number 2xx corresponds to the files in the folder 29# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx 30EXPECTED_FAILURES = { 31 # psa_hash_suspend() and psa_hash_resume() are not supported. 32 # - Tracked in issue #3274 33 262, 263 34} 35 36# We currently use a fork of ARM-software/psa-arch-tests, with a couple of downstream patches 37# that allow it to build with MbedTLS 3, and fixes a couple of issues in the compliance test suite. 38# These fixes allow the tests numbered 216, 248 and 249 to complete successfully. 39# 40# Once all the fixes are upstreamed, this fork should be replaced with an upstream commit/tag. 41# - Tracked in issue #5145 42# 43# Web URL: https://github.com/bensze01/psa-arch-tests/tree/fixes-for-mbedtls-3 44PSA_ARCH_TESTS_REPO = 'https://github.com/bensze01/psa-arch-tests.git' 45PSA_ARCH_TESTS_REF = 'fix-pr-5736' 46 47#pylint: disable=too-many-branches,too-many-statements,too-many-locals 48def main(library_build_dir: str): 49 root_dir = os.getcwd() 50 51 in_psa_crypto_repo = build_tree.looks_like_psa_crypto_root(root_dir) 52 53 if in_psa_crypto_repo: 54 crypto_name = 'psacrypto' 55 library_subdir = 'core' 56 else: 57 crypto_name = 'mbedcrypto' 58 library_subdir = 'library' 59 60 crypto_lib_filename = (library_build_dir + '/' + 61 library_subdir + '/' + 62 'lib' + crypto_name + '.a') 63 64 if not os.path.exists(crypto_lib_filename): 65 #pylint: disable=bad-continuation 66 subprocess.check_call([ 67 'cmake', '.', 68 '-GUnix Makefiles', 69 '-B' + library_build_dir 70 ]) 71 subprocess.check_call(['cmake', '--build', library_build_dir, 72 '--target', crypto_name]) 73 74 psa_arch_tests_dir = 'psa-arch-tests' 75 os.makedirs(psa_arch_tests_dir, exist_ok=True) 76 try: 77 os.chdir(psa_arch_tests_dir) 78 79 # Reuse existing local clone 80 subprocess.check_call(['git', 'init']) 81 subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) 82 subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) 83 84 build_dir = 'api-tests/build' 85 try: 86 shutil.rmtree(build_dir) 87 except FileNotFoundError: 88 pass 89 os.mkdir(build_dir) 90 os.chdir(build_dir) 91 92 extra_includes = (';{}/drivers/builtin/include'.format(root_dir) 93 if in_psa_crypto_repo else '') 94 95 #pylint: disable=bad-continuation 96 subprocess.check_call([ 97 'cmake', '..', 98 '-GUnix Makefiles', 99 '-DTARGET=tgt_dev_apis_stdc', 100 '-DTOOLCHAIN=HOST_GCC', 101 '-DSUITE=CRYPTO', 102 '-DPSA_CRYPTO_LIB_FILENAME={}/{}'.format(root_dir, 103 crypto_lib_filename), 104 ('-DPSA_INCLUDE_PATHS={}/include' + extra_includes).format(root_dir) 105 ]) 106 subprocess.check_call(['cmake', '--build', '.']) 107 108 proc = subprocess.Popen(['./psa-arch-tests-crypto'], 109 bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) 110 111 test_re = re.compile( 112 '^TEST: (?P<test_num>[0-9]*)|' 113 '^TEST RESULT: (?P<test_result>FAILED|PASSED)' 114 ) 115 test = -1 116 unexpected_successes = set(EXPECTED_FAILURES) 117 expected_failures = [] # type: List[int] 118 unexpected_failures = [] # type: List[int] 119 if proc.stdout is None: 120 return 1 121 122 for line in proc.stdout: 123 print(line, end='') 124 match = test_re.match(line) 125 if match is not None: 126 groupdict = match.groupdict() 127 test_num = groupdict['test_num'] 128 if test_num is not None: 129 test = int(test_num) 130 elif groupdict['test_result'] == 'FAILED': 131 try: 132 unexpected_successes.remove(test) 133 expected_failures.append(test) 134 print('Expected failure, ignoring') 135 except KeyError: 136 unexpected_failures.append(test) 137 print('ERROR: Unexpected failure') 138 elif test in unexpected_successes: 139 print('ERROR: Unexpected success') 140 proc.wait() 141 142 print() 143 print('***** test_psa_compliance.py report ******') 144 print() 145 print('Expected failures:', ', '.join(str(i) for i in expected_failures)) 146 print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) 147 print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) 148 print() 149 if unexpected_successes or unexpected_failures: 150 if unexpected_successes: 151 print('Unexpected successes encountered.') 152 print('Please remove the corresponding tests from ' 153 'EXPECTED_FAILURES in tests/scripts/compliance_test.py') 154 print() 155 print('FAILED') 156 return 1 157 else: 158 print('SUCCESS') 159 return 0 160 finally: 161 os.chdir(root_dir) 162 163if __name__ == '__main__': 164 BUILD_DIR = 'out_of_source_build' 165 166 # pylint: disable=invalid-name 167 parser = argparse.ArgumentParser() 168 parser.add_argument('--build-dir', nargs=1, 169 help='path to Mbed TLS / PSA Crypto build directory') 170 args = parser.parse_args() 171 172 if args.build_dir is not None: 173 BUILD_DIR = args.build_dir[0] 174 175 sys.exit(main(BUILD_DIR)) 176