1#!/usr/bin/env python3 2 3"""Analyze the test outcomes from a full CI run. 4 5This script can also run on outcomes from a partial run, but the results are 6less likely to be useful. 7""" 8 9import argparse 10import sys 11import traceback 12import re 13import subprocess 14import os 15 16import check_test_cases 17 18class Results: 19 """Process analysis results.""" 20 21 def __init__(self): 22 self.error_count = 0 23 self.warning_count = 0 24 25 @staticmethod 26 def log(fmt, *args, **kwargs): 27 sys.stderr.write((fmt + '\n').format(*args, **kwargs)) 28 29 def error(self, fmt, *args, **kwargs): 30 self.log('Error: ' + fmt, *args, **kwargs) 31 self.error_count += 1 32 33 def warning(self, fmt, *args, **kwargs): 34 self.log('Warning: ' + fmt, *args, **kwargs) 35 self.warning_count += 1 36 37class TestCaseOutcomes: 38 """The outcomes of one test case across many configurations.""" 39 # pylint: disable=too-few-public-methods 40 41 def __init__(self): 42 # Collect a list of witnesses of the test case succeeding or failing. 43 # Currently we don't do anything with witnesses except count them. 44 # The format of a witness is determined by the read_outcome_file 45 # function; it's the platform and configuration joined by ';'. 46 self.successes = [] 47 self.failures = [] 48 49 def hits(self): 50 """Return the number of times a test case has been run. 51 52 This includes passes and failures, but not skips. 53 """ 54 return len(self.successes) + len(self.failures) 55 56def execute_reference_driver_tests(ref_component, driver_component, outcome_file): 57 """Run the tests specified in ref_component and driver_component. Results 58 are stored in the output_file and they will be used for the following 59 coverage analysis""" 60 # If the outcome file already exists, we assume that the user wants to 61 # perform the comparison analysis again without repeating the tests. 62 if os.path.exists(outcome_file): 63 Results.log("Outcome file (" + outcome_file + ") already exists. " + \ 64 "Tests will be skipped.") 65 return 66 67 shell_command = "tests/scripts/all.sh --outcome-file " + outcome_file + \ 68 " " + ref_component + " " + driver_component 69 Results.log("Running: " + shell_command) 70 ret_val = subprocess.run(shell_command.split(), check=False).returncode 71 72 if ret_val != 0: 73 Results.log("Error: failed to run reference/driver components") 74 sys.exit(ret_val) 75 76def analyze_coverage(results, outcomes, allow_list, full_coverage): 77 """Check that all available test cases are executed at least once.""" 78 available = check_test_cases.collect_available_test_cases() 79 for key in available: 80 hits = outcomes[key].hits() if key in outcomes else 0 81 if hits == 0 and key not in allow_list: 82 if full_coverage: 83 results.error('Test case not executed: {}', key) 84 else: 85 results.warning('Test case not executed: {}', key) 86 elif hits != 0 and key in allow_list: 87 # Test Case should be removed from the allow list. 88 if full_coverage: 89 results.error('Allow listed test case was executed: {}', key) 90 else: 91 results.warning('Allow listed test case was executed: {}', key) 92 93def analyze_driver_vs_reference(outcomes, component_ref, component_driver, 94 ignored_suites, ignored_test=None): 95 """Check that all tests executed in the reference component are also 96 executed in the corresponding driver component. 97 Skip: 98 - full test suites provided in ignored_suites list 99 - only some specific test inside a test suite, for which the corresponding 100 output string is provided 101 """ 102 available = check_test_cases.collect_available_test_cases() 103 result = True 104 105 for key in available: 106 # Continue if test was not executed by any component 107 hits = outcomes[key].hits() if key in outcomes else 0 108 if hits == 0: 109 continue 110 # Skip ignored test suites 111 full_test_suite = key.split(';')[0] # retrieve full test suite name 112 test_string = key.split(';')[1] # retrieve the text string of this test 113 test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name 114 if test_suite in ignored_suites or full_test_suite in ignored_suites: 115 continue 116 if ((full_test_suite in ignored_test) and 117 (test_string in ignored_test[full_test_suite])): 118 continue 119 # Search for tests that run in reference component and not in driver component 120 driver_test_passed = False 121 reference_test_passed = False 122 for entry in outcomes[key].successes: 123 if component_driver in entry: 124 driver_test_passed = True 125 if component_ref in entry: 126 reference_test_passed = True 127 if(reference_test_passed and not driver_test_passed): 128 Results.log(key) 129 result = False 130 return result 131 132def analyze_outcomes(outcomes, args): 133 """Run all analyses on the given outcome collection.""" 134 results = Results() 135 analyze_coverage(results, outcomes, args['allow_list'], 136 args['full_coverage']) 137 return results 138 139def read_outcome_file(outcome_file): 140 """Parse an outcome file and return an outcome collection. 141 142An outcome collection is a dictionary mapping keys to TestCaseOutcomes objects. 143The keys are the test suite name and the test case description, separated 144by a semicolon. 145""" 146 outcomes = {} 147 with open(outcome_file, 'r', encoding='utf-8') as input_file: 148 for line in input_file: 149 (platform, config, suite, case, result, _cause) = line.split(';') 150 key = ';'.join([suite, case]) 151 setup = ';'.join([platform, config]) 152 if key not in outcomes: 153 outcomes[key] = TestCaseOutcomes() 154 if result == 'PASS': 155 outcomes[key].successes.append(setup) 156 elif result == 'FAIL': 157 outcomes[key].failures.append(setup) 158 return outcomes 159 160def do_analyze_coverage(outcome_file, args): 161 """Perform coverage analysis.""" 162 outcomes = read_outcome_file(outcome_file) 163 Results.log("\n*** Analyze coverage ***\n") 164 results = analyze_outcomes(outcomes, args) 165 return results.error_count == 0 166 167def do_analyze_driver_vs_reference(outcome_file, args): 168 """Perform driver vs reference analyze.""" 169 execute_reference_driver_tests(args['component_ref'], \ 170 args['component_driver'], outcome_file) 171 172 ignored_suites = ['test_suite_' + x for x in args['ignored_suites']] 173 174 outcomes = read_outcome_file(outcome_file) 175 Results.log("\n*** Analyze driver {} vs reference {} ***\n".format( 176 args['component_driver'], args['component_ref'])) 177 return analyze_driver_vs_reference(outcomes, args['component_ref'], 178 args['component_driver'], ignored_suites, 179 args['ignored_tests']) 180 181# List of tasks with a function that can handle this task and additional arguments if required 182TASKS = { 183 'analyze_coverage': { 184 'test_function': do_analyze_coverage, 185 'args': { 186 'allow_list': [ 187 # Algorithm not supported yet 188 'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA', 189 # Algorithm not supported yet 190 'test_suite_psa_crypto_metadata;Cipher: XTS', 191 ], 192 'full_coverage': False, 193 } 194 }, 195 # There are 2 options to use analyze_driver_vs_reference_xxx locally: 196 # 1. Run tests and then analysis: 197 # - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver> 198 # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx 199 # 2. Let this script run both automatically: 200 # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx 201 'analyze_driver_vs_reference_hash': { 202 'test_function': do_analyze_driver_vs_reference, 203 'args': { 204 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa', 205 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa', 206 'ignored_suites': [ 207 'shax', 'mdx', # the software implementations that are being excluded 208 'md.psa', # purposefully depends on whether drivers are present 209 'psa_crypto_low_hash.generated', # testing the builtins 210 ], 211 'ignored_tests': { 212 } 213 } 214 }, 215 'analyze_driver_vs_reference_ecp_light_only': { 216 'test_function': do_analyze_driver_vs_reference, 217 'args': { 218 'component_ref': 'test_psa_crypto_config_reference_ecc_ecp_light_only', 219 'component_driver': 'test_psa_crypto_config_accel_ecc_ecp_light_only', 220 'ignored_suites': [ 221 'ecdsa', 222 'ecdh', 223 'ecjpake', 224 ], 225 'ignored_tests': { 226 'test_suite_random': [ 227 'PSA classic wrapper: ECDSA signature (SECP256R1)', 228 ], 229 # In the accelerated test ECP_C is not set (only ECP_LIGHT is) 230 # so we must ignore disparities in the tests for which ECP_C 231 # is required. 232 'test_suite_ecp': [ 233 'ECP check public-private #1 (OK)', 234 'ECP check public-private #2 (group none)', 235 'ECP check public-private #3 (group mismatch)', 236 'ECP check public-private #4 (Qx mismatch)', 237 'ECP check public-private #5 (Qy mismatch)', 238 'ECP check public-private #6 (wrong Qx)', 239 'ECP check public-private #7 (wrong Qy)', 240 'ECP gen keypair [#1]', 241 'ECP gen keypair [#2]', 242 'ECP gen keypair [#3]', 243 'ECP gen keypair wrapper', 244 'ECP point muladd secp256r1 #1', 245 'ECP point muladd secp256r1 #2', 246 'ECP point multiplication Curve25519 (element of order 2: origin) #3', 247 'ECP point multiplication Curve25519 (element of order 4: 1) #4', 248 'ECP point multiplication Curve25519 (element of order 8) #5', 249 'ECP point multiplication Curve25519 (normalized) #1', 250 'ECP point multiplication Curve25519 (not normalized) #2', 251 'ECP point multiplication rng fail Curve25519', 252 'ECP point multiplication rng fail secp256r1', 253 'ECP test vectors Curve25519', 254 'ECP test vectors Curve448 (RFC 7748 6.2, after decodeUCoordinate)', 255 'ECP test vectors brainpoolP256r1 rfc 7027', 256 'ECP test vectors brainpoolP384r1 rfc 7027', 257 'ECP test vectors brainpoolP512r1 rfc 7027', 258 'ECP test vectors secp192k1', 259 'ECP test vectors secp192r1 rfc 5114', 260 'ECP test vectors secp224k1', 261 'ECP test vectors secp224r1 rfc 5114', 262 'ECP test vectors secp256k1', 263 'ECP test vectors secp256r1 rfc 5114', 264 'ECP test vectors secp384r1 rfc 5114', 265 'ECP test vectors secp521r1 rfc 5114', 266 ], 267 'test_suite_psa_crypto': [ 268 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', 269 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', 270 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', 271 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1', 272 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0', 273 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1', 274 ], 275 'test_suite_ssl': [ 276 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', 277 ], 278 } 279 } 280 }, 281 'analyze_driver_vs_reference_no_ecp_at_all': { 282 'test_function': do_analyze_driver_vs_reference, 283 'args': { 284 'component_ref': 'test_psa_crypto_config_reference_ecc_no_ecp_at_all', 285 'component_driver': 'test_psa_crypto_config_accel_ecc_no_ecp_at_all', 286 'ignored_suites': [ 287 # Ignore test suites for the modules that are disabled in the 288 # accelerated test case. 289 'ecp', 290 'ecdsa', 291 'ecdh', 292 'ecjpake', 293 ], 294 'ignored_tests': { 295 'test_suite_random': [ 296 'PSA classic wrapper: ECDSA signature (SECP256R1)', 297 ], 298 'test_suite_psa_crypto': [ 299 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', 300 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', 301 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', 302 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1', 303 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0', 304 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1', 305 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)', 306 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)', 307 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)', 308 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)', 309 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)', 310 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)', 311 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)', 312 ], 313 'test_suite_pkparse': [ 314 # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED 315 # is automatically enabled in build_info.h (backward compatibility) 316 # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a 317 # consequence compressed points are supported in the reference 318 # component but not in the accelerated one, so they should be skipped 319 # while checking driver's coverage. 320 'Parse EC Key #10a (SEC1 PEM, secp384r1, compressed)', 321 'Parse EC Key #11a (SEC1 PEM, secp521r1, compressed)', 322 'Parse EC Key #12a (SEC1 PEM, bp256r1, compressed)', 323 'Parse EC Key #13a (SEC1 PEM, bp384r1, compressed)', 324 'Parse EC Key #14a (SEC1 PEM, bp512r1, compressed)', 325 'Parse EC Key #2a (SEC1 PEM, secp192r1, compressed)', 326 'Parse EC Key #8a (SEC1 PEM, secp224r1, compressed)', 327 'Parse EC Key #9a (SEC1 PEM, secp256r1, compressed)', 328 'Parse Public EC Key #2a (RFC 5480, PEM, secp192r1, compressed)', 329 'Parse Public EC Key #3a (RFC 5480, secp224r1, compressed)', 330 'Parse Public EC Key #4a (RFC 5480, secp256r1, compressed)', 331 'Parse Public EC Key #5a (RFC 5480, secp384r1, compressed)', 332 'Parse Public EC Key #6a (RFC 5480, secp521r1, compressed)', 333 'Parse Public EC Key #7a (RFC 5480, brainpoolP256r1, compressed)', 334 'Parse Public EC Key #8a (RFC 5480, brainpoolP384r1, compressed)', 335 'Parse Public EC Key #9a (RFC 5480, brainpoolP512r1, compressed)', 336 ], 337 'test_suite_ssl': [ 338 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', 339 ], 340 } 341 } 342 }, 343 'analyze_driver_vs_reference_ecc_no_bignum': { 344 'test_function': do_analyze_driver_vs_reference, 345 'args': { 346 'component_ref': 'test_psa_crypto_config_reference_ecc_no_bignum', 347 'component_driver': 'test_psa_crypto_config_accel_ecc_no_bignum', 348 'ignored_suites': [ 349 # Ignore test suites for the modules that are disabled in the 350 # accelerated test case. 351 'ecp', 352 'ecdsa', 353 'ecdh', 354 'ecjpake', 355 'bignum_core', 356 'bignum_random', 357 'bignum_mod', 358 'bignum_mod_raw', 359 'bignum.generated', 360 'bignum.misc', 361 ], 362 'ignored_tests': { 363 'test_suite_random': [ 364 'PSA classic wrapper: ECDSA signature (SECP256R1)', 365 ], 366 'test_suite_psa_crypto': [ 367 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', 368 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', 369 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', 370 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1', 371 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0', 372 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1', 373 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)', 374 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)', 375 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)', 376 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)', 377 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)', 378 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)', 379 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)', 380 ], 381 'test_suite_pkparse': [ 382 # See the description provided above in the 383 # analyze_driver_vs_reference_no_ecp_at_all component. 384 'Parse EC Key #10a (SEC1 PEM, secp384r1, compressed)', 385 'Parse EC Key #11a (SEC1 PEM, secp521r1, compressed)', 386 'Parse EC Key #12a (SEC1 PEM, bp256r1, compressed)', 387 'Parse EC Key #13a (SEC1 PEM, bp384r1, compressed)', 388 'Parse EC Key #14a (SEC1 PEM, bp512r1, compressed)', 389 'Parse EC Key #2a (SEC1 PEM, secp192r1, compressed)', 390 'Parse EC Key #8a (SEC1 PEM, secp224r1, compressed)', 391 'Parse EC Key #9a (SEC1 PEM, secp256r1, compressed)', 392 'Parse Public EC Key #2a (RFC 5480, PEM, secp192r1, compressed)', 393 'Parse Public EC Key #3a (RFC 5480, secp224r1, compressed)', 394 'Parse Public EC Key #4a (RFC 5480, secp256r1, compressed)', 395 'Parse Public EC Key #5a (RFC 5480, secp384r1, compressed)', 396 'Parse Public EC Key #6a (RFC 5480, secp521r1, compressed)', 397 'Parse Public EC Key #7a (RFC 5480, brainpoolP256r1, compressed)', 398 'Parse Public EC Key #8a (RFC 5480, brainpoolP384r1, compressed)', 399 'Parse Public EC Key #9a (RFC 5480, brainpoolP512r1, compressed)', 400 ], 401 'test_suite_asn1parse': [ 402 # This test depends on BIGNUM_C 403 'INTEGER too large for mpi', 404 ], 405 'test_suite_asn1write': [ 406 # Following tests depends on BIGNUM_C 407 'ASN.1 Write mpi 0 (1 limb)', 408 'ASN.1 Write mpi 0 (null)', 409 'ASN.1 Write mpi 0x100', 410 'ASN.1 Write mpi 0x7f', 411 'ASN.1 Write mpi 0x7f with leading 0 limb', 412 'ASN.1 Write mpi 0x80', 413 'ASN.1 Write mpi 0x80 with leading 0 limb', 414 'ASN.1 Write mpi 0xff', 415 'ASN.1 Write mpi 1', 416 'ASN.1 Write mpi, 127*8 bits', 417 'ASN.1 Write mpi, 127*8+1 bits', 418 'ASN.1 Write mpi, 127*8-1 bits', 419 'ASN.1 Write mpi, 255*8 bits', 420 'ASN.1 Write mpi, 255*8-1 bits', 421 'ASN.1 Write mpi, 256*8-1 bits', 422 ], 423 'test_suite_debug': [ 424 # Following tests depends on BIGNUM_C 425 'Debug print mbedtls_mpi #2: 3 bits', 426 'Debug print mbedtls_mpi: 0 (empty representation)', 427 'Debug print mbedtls_mpi: 0 (non-empty representation)', 428 'Debug print mbedtls_mpi: 49 bits', 429 'Debug print mbedtls_mpi: 759 bits', 430 'Debug print mbedtls_mpi: 764 bits #1', 431 'Debug print mbedtls_mpi: 764 bits #2', 432 ], 433 'test_suite_ssl': [ 434 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', 435 ], 436 } 437 } 438 }, 439 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': { 440 'test_function': do_analyze_driver_vs_reference, 441 'args': { 442 'component_ref': 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum', 443 'component_driver': 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum', 444 'ignored_suites': [ 445 # Ignore test suites for the modules that are disabled in the 446 # accelerated test case. 447 'ecp', 448 'ecdsa', 449 'ecdh', 450 'ecjpake', 451 'bignum_core', 452 'bignum_random', 453 'bignum_mod', 454 'bignum_mod_raw', 455 'bignum.generated', 456 'bignum.misc', 457 'dhm', 458 ], 459 'ignored_tests': { 460 'test_suite_random': [ 461 'PSA classic wrapper: ECDSA signature (SECP256R1)', 462 ], 463 'test_suite_psa_crypto': [ 464 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', 465 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', 466 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', 467 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1', 468 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0', 469 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1', 470 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)', 471 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)', 472 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)', 473 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)', 474 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)', 475 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)', 476 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)', 477 ], 478 'test_suite_pkparse': [ 479 # See the description provided above in the 480 # analyze_driver_vs_reference_no_ecp_at_all component. 481 'Parse EC Key #10a (SEC1 PEM, secp384r1, compressed)', 482 'Parse EC Key #11a (SEC1 PEM, secp521r1, compressed)', 483 'Parse EC Key #12a (SEC1 PEM, bp256r1, compressed)', 484 'Parse EC Key #13a (SEC1 PEM, bp384r1, compressed)', 485 'Parse EC Key #14a (SEC1 PEM, bp512r1, compressed)', 486 'Parse EC Key #2a (SEC1 PEM, secp192r1, compressed)', 487 'Parse EC Key #8a (SEC1 PEM, secp224r1, compressed)', 488 'Parse EC Key #9a (SEC1 PEM, secp256r1, compressed)', 489 'Parse Public EC Key #2a (RFC 5480, PEM, secp192r1, compressed)', 490 'Parse Public EC Key #3a (RFC 5480, secp224r1, compressed)', 491 'Parse Public EC Key #4a (RFC 5480, secp256r1, compressed)', 492 'Parse Public EC Key #5a (RFC 5480, secp384r1, compressed)', 493 'Parse Public EC Key #6a (RFC 5480, secp521r1, compressed)', 494 'Parse Public EC Key #7a (RFC 5480, brainpoolP256r1, compressed)', 495 'Parse Public EC Key #8a (RFC 5480, brainpoolP384r1, compressed)', 496 'Parse Public EC Key #9a (RFC 5480, brainpoolP512r1, compressed)', 497 ], 498 'test_suite_asn1parse': [ 499 # This test depends on BIGNUM_C 500 'INTEGER too large for mpi', 501 ], 502 'test_suite_asn1write': [ 503 # Following tests depends on BIGNUM_C 504 'ASN.1 Write mpi 0 (1 limb)', 505 'ASN.1 Write mpi 0 (null)', 506 'ASN.1 Write mpi 0x100', 507 'ASN.1 Write mpi 0x7f', 508 'ASN.1 Write mpi 0x7f with leading 0 limb', 509 'ASN.1 Write mpi 0x80', 510 'ASN.1 Write mpi 0x80 with leading 0 limb', 511 'ASN.1 Write mpi 0xff', 512 'ASN.1 Write mpi 1', 513 'ASN.1 Write mpi, 127*8 bits', 514 'ASN.1 Write mpi, 127*8+1 bits', 515 'ASN.1 Write mpi, 127*8-1 bits', 516 'ASN.1 Write mpi, 255*8 bits', 517 'ASN.1 Write mpi, 255*8-1 bits', 518 'ASN.1 Write mpi, 256*8-1 bits', 519 ], 520 'test_suite_debug': [ 521 # Following tests depends on BIGNUM_C 522 'Debug print mbedtls_mpi #2: 3 bits', 523 'Debug print mbedtls_mpi: 0 (empty representation)', 524 'Debug print mbedtls_mpi: 0 (non-empty representation)', 525 'Debug print mbedtls_mpi: 49 bits', 526 'Debug print mbedtls_mpi: 759 bits', 527 'Debug print mbedtls_mpi: 764 bits #1', 528 'Debug print mbedtls_mpi: 764 bits #2', 529 ], 530 'test_suite_ssl': [ 531 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', 532 ], 533 } 534 } 535 }, 536 'analyze_driver_vs_reference_ffdh_alg': { 537 'test_function': do_analyze_driver_vs_reference, 538 'args': { 539 'component_ref': 'test_psa_crypto_config_reference_ffdh', 540 'component_driver': 'test_psa_crypto_config_accel_ffdh', 541 'ignored_suites': ['dhm'], 542 'ignored_tests': {} 543 } 544 }, 545 'analyze_driver_vs_reference_tfm_config': { 546 'test_function': do_analyze_driver_vs_reference, 547 'args': { 548 'component_ref': 'test_tfm_config', 549 'component_driver': 'test_tfm_config_p256m_driver_accel_ec', 550 'ignored_suites': [ 551 # Ignore test suites for the modules that are disabled in the 552 # accelerated test case. 553 'ecp', 554 'ecdsa', 555 'ecdh', 556 'ecjpake', 557 'bignum_core', 558 'bignum_random', 559 'bignum_mod', 560 'bignum_mod_raw', 561 'bignum.generated', 562 'bignum.misc', 563 ], 564 'ignored_tests': { 565 # Ignore all tests that require DERIVE support which is disabled 566 # in the driver version 567 'test_suite_psa_crypto': [ 568 'PSA key agreement setup: ECDH + HKDF-SHA-256: good', 569 ('PSA key agreement setup: ECDH + HKDF-SHA-256: good, key algorithm broader ' 570 'than required'), 571 'PSA key agreement setup: ECDH + HKDF-SHA-256: public key not on curve', 572 'PSA key agreement setup: KDF instead of a key agreement algorithm', 573 'PSA key agreement setup: bad key agreement algorithm', 574 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: capacity=8160', 575 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 0+32', 576 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 1+31', 577 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 31+1', 578 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 32+0', 579 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 32+32', 580 'PSA key agreement: ECDH SECP256R1 (RFC 5903) + HKDF-SHA-256: read 64+0', 581 'PSA key derivation: ECDH on P256 with HKDF-SHA256, info first', 582 'PSA key derivation: ECDH on P256 with HKDF-SHA256, key output', 583 'PSA key derivation: ECDH on P256 with HKDF-SHA256, missing info', 584 'PSA key derivation: ECDH on P256 with HKDF-SHA256, omitted salt', 585 'PSA key derivation: ECDH on P256 with HKDF-SHA256, raw output', 586 'PSA key derivation: ECDH on P256 with HKDF-SHA256, salt after secret', 587 'PSA key derivation: ECDH with TLS 1.2 PRF SHA-256, good case', 588 'PSA key derivation: ECDH with TLS 1.2 PRF SHA-256, missing label', 589 'PSA key derivation: ECDH with TLS 1.2 PRF SHA-256, missing label and secret', 590 'PSA key derivation: ECDH with TLS 1.2 PRF SHA-256, no inputs', 591 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1', 592 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)', 593 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA', 594 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, SHA-256, 0+48, ka', 595 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, SHA-256, 24+24, ka', 596 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, SHA-256, 48+0, ka', 597 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, bad state #1, ka', 598 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, bad state #3, ka', 599 'PSA key derivation: TLS 1.2 Mix-PSK-to-MS, bad state #4, ka', 600 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)', 601 'PSA key derivation: bits=7 invalid for ECC MONTGOMERY (ECC enabled)', 602 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)', 603 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)', 604 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)', 605 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)', 606 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)', 607 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)', 608 'PSA raw key agreement: ECDH SECP256R1 (RFC 5903)', 609 ], 610 'test_suite_random': [ 611 'PSA classic wrapper: ECDSA signature (SECP256R1)', 612 ], 613 'test_suite_psa_crypto_pake': [ 614 'PSA PAKE: ecjpake size macros', 615 ], 616 'test_suite_asn1parse': [ 617 # This test depends on BIGNUM_C 618 'INTEGER too large for mpi', 619 ], 620 'test_suite_asn1write': [ 621 # Following tests depends on BIGNUM_C 622 'ASN.1 Write mpi 0 (1 limb)', 623 'ASN.1 Write mpi 0 (null)', 624 'ASN.1 Write mpi 0x100', 625 'ASN.1 Write mpi 0x7f', 626 'ASN.1 Write mpi 0x7f with leading 0 limb', 627 'ASN.1 Write mpi 0x80', 628 'ASN.1 Write mpi 0x80 with leading 0 limb', 629 'ASN.1 Write mpi 0xff', 630 'ASN.1 Write mpi 1', 631 'ASN.1 Write mpi, 127*8 bits', 632 'ASN.1 Write mpi, 127*8+1 bits', 633 'ASN.1 Write mpi, 127*8-1 bits', 634 'ASN.1 Write mpi, 255*8 bits', 635 'ASN.1 Write mpi, 255*8-1 bits', 636 'ASN.1 Write mpi, 256*8-1 bits', 637 ], 638 } 639 } 640 } 641} 642 643def main(): 644 try: 645 parser = argparse.ArgumentParser(description=__doc__) 646 parser.add_argument('outcomes', metavar='OUTCOMES.CSV', 647 help='Outcome file to analyze') 648 parser.add_argument('task', default='all', nargs='?', 649 help='Analysis to be done. By default, run all tasks. ' 650 'With one or more TASK, run only those. ' 651 'TASK can be the name of a single task or ' 652 'comma/space-separated list of tasks. ') 653 parser.add_argument('--list', action='store_true', 654 help='List all available tasks and exit.') 655 parser.add_argument('--require-full-coverage', action='store_true', 656 dest='full_coverage', help="Require all available " 657 "test cases to be executed and issue an error " 658 "otherwise. This flag is ignored if 'task' is " 659 "neither 'all' nor 'analyze_coverage'") 660 options = parser.parse_args() 661 662 if options.list: 663 for task in TASKS: 664 Results.log(task) 665 sys.exit(0) 666 667 result = True 668 669 if options.task == 'all': 670 tasks = TASKS.keys() 671 else: 672 tasks = re.split(r'[, ]+', options.task) 673 674 for task in tasks: 675 if task not in TASKS: 676 Results.log('Error: invalid task: {}'.format(task)) 677 sys.exit(1) 678 679 TASKS['analyze_coverage']['args']['full_coverage'] = \ 680 options.full_coverage 681 682 for task in TASKS: 683 if task in tasks: 684 if not TASKS[task]['test_function'](options.outcomes, TASKS[task]['args']): 685 result = False 686 687 if result is False: 688 sys.exit(1) 689 Results.log("SUCCESS :-)") 690 except Exception: # pylint: disable=broad-except 691 # Print the backtrace and exit explicitly with our chosen status. 692 traceback.print_exc() 693 sys.exit(120) 694 695if __name__ == '__main__': 696 main() 697