1#!/bin/sh 2 3# This script runs tests before and after a PR and analyzes the results in 4# order to highlight any difference in the set of tests skipped. 5# 6# It can be used to check the first testing criterion mentioned in strategy.md, 7# end of section "Supporting builds with drivers without the software 8# implementation", namely: the sets of tests skipped in the default config and 9# the full config must be the same before and after the PR. 10# 11# USAGE: 12# - First, commit any uncommited changes. (Also, see warning below.) 13# - Then launch --> [SKIP_SSL_OPT=1] docs/architecture/psa-migration/outcome-analysis.sh 14# - SKIP_SSL_OPT=1 can optionally be set to skip ssl-opt.sh tests 15# 16# WARNING: this script checks out a commit other than the head of the current 17# branch; it checks out the current branch again when running successfully, 18# but while the script is running, or if it terminates early in error, you 19# should be aware that you might be at a different commit than expected. 20# 21# NOTE: you can comment out parts that don't need to be re-done when 22# re-running this script (for example "get numbers before this PR"). 23 24set -eu 25 26: ${SKIP_SSL_OPT:=0} 27 28cleanup() { 29 make clean 30 git checkout -- include/mbedtls/mbedtls_config.h include/psa/crypto_config.h 31} 32 33record() { 34 export MBEDTLS_TEST_OUTCOME_FILE="$PWD/outcome-$1.csv" 35 rm -f $MBEDTLS_TEST_OUTCOME_FILE 36 37 make check 38 39 if [ $SKIP_SSL_OPT -eq 0 ]; then 40 make -C programs ssl/ssl_server2 ssl/ssl_client2 \ 41 test/udp_proxy test/query_compile_time_config 42 tests/ssl-opt.sh 43 fi 44} 45 46# save current HEAD. 47# Note: this can optionally be updated to 48# HEAD=$(git branch --show-current) 49# when using a Git version above 2.22 50HEAD=$(git rev-parse --abbrev-ref HEAD) 51 52# get the numbers before this PR for default and full 53cleanup 54git checkout $(git merge-base HEAD development) 55 56record "before-default" 57 58cleanup 59 60scripts/config.py full 61record "before-full" 62 63# get the numbers now for default and full 64cleanup 65git checkout $HEAD 66 67record "after-default" 68 69cleanup 70 71scripts/config.py full 72record "after-full" 73 74cleanup 75 76# analysis 77 78populate_suites () { 79 SUITES='' 80 make generated_files >/dev/null 81 data_files=$(cd tests/suites && echo *.data) 82 for data in $data_files; do 83 suite=${data%.data} 84 SUITES="$SUITES $suite" 85 done 86 make neat 87 88 if [ $SKIP_SSL_OPT -eq 0 ]; then 89 SUITES="$SUITES ssl-opt" 90 extra_files=$(cd tests/opt-testcases && echo *.sh) 91 for extra in $extra_files; do 92 suite=${extra%.sh} 93 SUITES="$SUITES $suite" 94 done 95 fi 96} 97 98compare_suite () { 99 ref="outcome-$1.csv" 100 new="outcome-$2.csv" 101 suite="$3" 102 103 pattern_suite=";$suite;" 104 total=$(grep -c "$pattern_suite" "$ref") 105 sed_cmd="s/^.*$pattern_suite\(.*\);SKIP.*/\1/p" 106 sed -n "$sed_cmd" "$ref" > skipped-ref 107 sed -n "$sed_cmd" "$new" > skipped-new 108 nb_ref=$(wc -l <skipped-ref) 109 nb_new=$(wc -l <skipped-new) 110 111 name=${suite#test_suite_} 112 printf "%40s: total %4d; skipped %4d -> %4d\n" \ 113 $name $total $nb_ref $nb_new 114 if diff skipped-ref skipped-new | grep '^> '; then 115 ret=1 116 else 117 ret=0 118 fi 119 rm skipped-ref skipped-new 120 return $ret 121} 122 123compare_builds () { 124 printf "\n*** Comparing $1 -> $2 ***\n" 125 failed='' 126 for suite in $SUITES; do 127 if compare_suite "$1" "$2" "$suite"; then :; else 128 failed="$failed $suite" 129 fi 130 done 131 if [ -z "$failed" ]; then 132 printf "No coverage gap found.\n" 133 else 134 printf "Suites with less coverage:%s\n" "$failed" 135 fi 136} 137 138populate_suites 139compare_builds before-default after-default 140compare_builds before-full after-full 141