1#! /usr/bin/python3 -B 2# 3# SPDX-License-Identifier: BSD-2-Clause 4# 5# Copyright (c) 2018-2024 Gavin D. Howard and contributors. 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions are met: 9# 10# * Redistributions of source code must retain the above copyright notice, this 11# list of conditions and the following disclaimer. 12# 13# * Redistributions in binary form must reproduce the above copyright notice, 14# this list of conditions and the following disclaimer in the documentation 15# and/or other materials provided with the distribution. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 21# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27# POSSIBILITY OF SUCH DAMAGE. 28# 29 30import os 31import sys 32import shutil 33import subprocess 34 35 36# Print the usage and exit with an error. 37def usage(): 38 print("usage: {} [--asan] dir [results_dir [exe options...]]".format(script)) 39 print(" The valid values for dir are: 'bc1', 'bc2', and 'dc'.") 40 sys.exit(1) 41 42 43# Check for a crash. 44# @param exebase The calculator that crashed. 45# @param out The file to copy the crash file to. 46# @param error The error code (negative). 47# @param file The crash file. 48# @param type The type of run that caused the crash. This is just a string 49# that would make sense to the user. 50# @param test The contents of the crash file, or which line caused the crash 51# for a run through stdin. 52def check_crash(exebase, out, error, file, type, test): 53 if error < 0: 54 print("\n{} crashed ({}) on {}:\n".format(exebase, -error, type)) 55 print(" {}".format(test)) 56 print("\nCopying to \"{}\"".format(out)) 57 shutil.copy2(file, out) 58 print("\nexiting...") 59 sys.exit(error) 60 61 62# Runs a test. This function is used to ensure that if a test times out, it is 63# discarded. Otherwise, some tests result in incredibly long runtimes. We need 64# to ignore those. 65# 66# @param cmd The command to run. 67# @param exebase The calculator to test. 68# @param tout The timeout to use. 69# @param indata The data to push through stdin for the test. 70# @param out The file to copy the test file to if it causes a crash. 71# @param file The test file. 72# @param type The type of test. This is just a string that would make sense 73# to the user. 74# @param test The test. It could be an entire file, or just one line. 75# @param environ The environment to run the command under. 76def run_test(cmd, exebase, tout, indata, out, file, type, test, environ=None): 77 try: 78 p = subprocess.run(cmd, timeout=tout, input=indata, stdout=subprocess.PIPE, 79 stderr=subprocess.PIPE, env=environ) 80 check_crash(exebase, out, p.returncode, file, type, test) 81 except subprocess.TimeoutExpired: 82 print("\n {} timed out. Continuing...\n".format(exebase)) 83 84 85# Creates and runs a test. This basically just takes a file, runs it through the 86# appropriate calculator as a whole file, then runs it through the calculator 87# using stdin. 88# @param file The file to test. 89# @param tout The timeout to use. 90# @param environ The environment to run under. 91def create_test(file, tout, environ=None): 92 93 print(" {}".format(file)) 94 95 base = os.path.basename(file) 96 97 if base == "README.txt": 98 return 99 100 with open(file, "rb") as f: 101 lines = f.readlines() 102 103 print(" Running whole file...") 104 105 run_test(exe + [ file ], exebase, tout, halt.encode(), out, file, "file", file, environ) 106 107 print(" Running file through stdin...") 108 109 with open(file, "rb") as f: 110 content = f.read() 111 112 run_test(exe, exebase, tout, content, out, file, 113 "running {} through stdin".format(file), file, environ) 114 115 116# Get the children of a directory. 117# @param dir The directory to get the children of. 118# @param get_files True if files should be gotten, false if directories should 119# be gotten. 120def get_children(dir, get_files): 121 dirs = [] 122 with os.scandir(dir) as it: 123 for entry in it: 124 if not entry.name.startswith('.') and \ 125 ((entry.is_dir() and not get_files) or \ 126 (entry.is_file() and get_files)): 127 dirs.append(entry.name) 128 dirs.sort() 129 return dirs 130 131 132# Returns the correct executable name for the directory under test. 133# @param d The directory under test. 134def exe_name(d): 135 return "bc" if d == "bc1" or d == "bc2" else "dc" 136 137 138# Housekeeping. 139script = sys.argv[0] 140scriptdir = os.path.dirname(script) 141 142# Must run this script alone. 143if __name__ != "__main__": 144 usage() 145 146timeout = 2.5 147 148if len(sys.argv) < 2: 149 usage() 150 151idx = 1 152 153exedir = sys.argv[idx] 154 155asan = (exedir == "--asan") 156 157# We could possibly run under ASan. See later for what that means. 158if asan: 159 idx += 1 160 if len(sys.argv) < idx + 1: 161 usage() 162 exedir = sys.argv[idx] 163 164print("exedir: {}".format(exedir)) 165 166# Grab the correct directory of AFL++ results. 167if len(sys.argv) >= idx + 2: 168 resultsdir = sys.argv[idx + 1] 169else: 170 if exedir == "bc1": 171 resultsdir = scriptdir + "/../tests/fuzzing/bc_outputs1" 172 elif exedir == "bc2": 173 resultsdir = scriptdir + "/../tests/fuzzing/bc_outputs2" 174 elif exedir == "dc": 175 resultsdir = scriptdir + "/../tests/fuzzing/dc_outputs" 176 else: 177 raise ValueError("exedir must be either bc1, bc2, or dc"); 178 179print("resultsdir: {}".format(resultsdir)) 180 181# More command-line processing. 182if len(sys.argv) >= idx + 3: 183 exe = sys.argv[idx + 2] 184else: 185 exe = scriptdir + "/../bin/" + exe_name(exedir) 186 187exebase = os.path.basename(exe) 188 189 190# Use the correct options. 191if exebase == "bc": 192 halt = "halt\n" 193 options = "-lq" 194 seed = ["-e", "seed = 1280937142.20981723890730892738902938071028973408912703984712093", "-f-" ] 195else: 196 halt = "q\n" 197 options = "-x" 198 seed = ["-e", "1280937142.20981723890730892738902938071028973408912703984712093j", "-f-" ] 199 200# More command-line processing. 201if len(sys.argv) >= idx + 4: 202 exe = [ exe, sys.argv[idx + 3:], options ] + seed 203else: 204 exe = [ exe, options ] + seed 205for i in range(4, len(sys.argv)): 206 exe.append(sys.argv[i]) 207 208out = scriptdir + "/../.test.txt" 209 210print(os.path.realpath(os.getcwd())) 211 212dirs = get_children(resultsdir, False) 213 214# Set the correct ASAN_OPTIONS. 215if asan: 216 env = os.environ.copy() 217 env['ASAN_OPTIONS'] = 'abort_on_error=1:allocator_may_return_null=1' 218 219for d in dirs: 220 221 d = resultsdir + "/" + d 222 223 print(d) 224 225 # Check the crash files. 226 files = get_children(d + "/crashes/", True) 227 228 for file in files: 229 file = d + "/crashes/" + file 230 create_test(file, timeout) 231 232 # If we are running under ASan, we want to check all files. Otherwise, skip. 233 if not asan: 234 continue 235 236 # Check all of the test cases found by AFL++. 237 files = get_children(d + "/queue/", True) 238 239 for file in files: 240 file = d + "/queue/" + file 241 create_test(file, timeout * 2, env) 242 243print("Done") 244