1#! /usr/bin/python3 -B 2# 3# SPDX-License-Identifier: BSD-2-Clause 4# 5# Copyright (c) 2018-2024 Gavin D. Howard and contributors. 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions are met: 9# 10# * Redistributions of source code must retain the above copyright notice, this 11# list of conditions and the following disclaimer. 12# 13# * Redistributions in binary form must reproduce the above copyright notice, 14# this list of conditions and the following disclaimer in the documentation 15# and/or other materials provided with the distribution. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 21# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27# POSSIBILITY OF SUCH DAMAGE. 28# 29 30import os, errno 31import random 32import sys 33import subprocess 34 35# I want line length to *not* affect differences between the two, so I set it 36# as high as possible. 37env = { 38 "BC_LINE_LENGTH": "65535", 39 "DC_LINE_LENGTH": "65535" 40} 41 42 43# Generate a random integer between 0 and 2^limit. 44# @param limit The power of two for the upper limit. 45def gen(limit=4): 46 return random.randint(0, 2 ** (8 * limit)) 47 48 49# Returns a random boolean for whether a number should be negative or not. 50def negative(): 51 return random.randint(0, 1) == 1 52 53 54# Returns a random boolean for whether a number should be 0 or not. I decided to 55# have it be 0 every 2^4 times since sometimes it is used to make a number less 56# than 1. 57def zero(): 58 return random.randint(0, 2 ** (4) - 1) == 0 59 60 61# Generate a real portion of a number. 62def gen_real(): 63 64 # Figure out if we should have a real portion. If so generate it. 65 if negative(): 66 n = str(gen(25)) 67 length = gen(7 / 8) 68 if len(n) < length: 69 n = ("0" * (length - len(n))) + n 70 else: 71 n = "0" 72 73 return n 74 75 76# Generates a number (as a string) based on the parameters. 77# @param op The operation under test. 78# @param neg Whether the number can be negative. 79# @param real Whether the number can be a non-integer. 80# @param z Whether the number can be zero. 81# @param limit The power of 2 upper limit for the number. 82def num(op, neg, real, z, limit=4): 83 84 # Handle zero first. 85 if z: 86 z = zero() 87 else: 88 z = False 89 90 if z: 91 # Generate a real portion maybe 92 if real: 93 n = gen_real() 94 if n != "0": 95 return "0." + n 96 return "0" 97 98 # Figure out if we should be negative. 99 if neg: 100 neg = negative() 101 102 # Generate the integer portion. 103 g = gen(limit) 104 105 # Figure out if we should have a real number. negative() is used to give a 106 # 50/50 chance of getting a negative number. 107 if real: 108 n = gen_real() 109 else: 110 n = "0" 111 112 # Generate the string. 113 g = str(g) 114 if n != "0": 115 g = g + "." + n 116 117 # Make sure to use the right negative sign. 118 if neg and g != "0": 119 if op != modexp: 120 g = "-" + g 121 else: 122 g = "_" + g 123 124 return g 125 126 127# Add a failed test to the list. 128# @param test The test that failed. 129# @param op The operation for the test. 130def add(test, op): 131 tests.append(test) 132 gen_ops.append(op) 133 134 135# Compare the output between the two. 136# @param exe The executable under test. 137# @param options The command-line options. 138# @param p The object returned from subprocess.run() for the calculator 139# under test. 140# @param test The test. 141# @param halt The halt string for the calculator under test. 142# @param expected The expected result. 143# @param op The operation under test. 144# @param do_add If true, add a failing test to the list, otherwise, don't. 145def compare(exe, options, p, test, halt, expected, op, do_add=True): 146 147 # Check for error from the calculator under test. 148 if p.returncode != 0: 149 150 print(" {} returned an error ({})".format(exe, p.returncode)) 151 152 if do_add: 153 print(" adding to checklist...") 154 add(test, op) 155 156 return 157 158 actual = p.stdout.decode() 159 160 # Check for a difference in output. 161 if actual != expected: 162 163 if op >= exponent: 164 165 # This is here because GNU bc, like mine can be flaky on the 166 # functions in the math library. This is basically testing if adding 167 # 10 to the scale works to make them match. If so, the difference is 168 # only because of that. 169 indata = "scale += 10; {}; {}".format(test, halt) 170 args = [ exe, options ] 171 p2 = subprocess.run(args, input=indata.encode(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) 172 expected = p2.stdout[:-10].decode() 173 174 if actual == expected: 175 print(" failed because of bug in other {}".format(exe)) 176 print(" continuing...") 177 return 178 179 # Do the correct output for the situation. 180 if do_add: 181 print(" failed; adding to checklist...") 182 add(test, op) 183 else: 184 print(" failed {}".format(test)) 185 print(" expected:") 186 print(" {}".format(expected)) 187 print(" actual:") 188 print(" {}".format(actual)) 189 190 191# Generates a test for op. I made sure that there was no clashing between 192# calculators. Each calculator is responsible for certain ops. 193# @param op The operation to test. 194def gen_test(op): 195 196 # First, figure out how big the scale should be. 197 scale = num(op, False, False, True, 5 / 8) 198 199 # Do the right thing for each op. Generate the test based on the format 200 # string and the constraints of each op. For example, some ops can't accept 201 # 0 in some arguments, and some must have integers in some arguments. 202 if op < div: 203 s = fmts[op].format(scale, num(op, True, True, True), num(op, True, True, True)) 204 elif op == div or op == mod: 205 s = fmts[op].format(scale, num(op, True, True, True), num(op, True, True, False)) 206 elif op == power: 207 s = fmts[op].format(scale, num(op, True, True, True, 7 / 8), num(op, True, False, True, 6 / 8)) 208 elif op == modexp: 209 s = fmts[op].format(scale, num(op, True, False, True), num(op, True, False, True), 210 num(op, True, False, False)) 211 elif op == sqrt: 212 s = "1" 213 while s == "1": 214 s = num(op, False, True, True, 1) 215 s = fmts[op].format(scale, s) 216 else: 217 218 if op == exponent: 219 first = num(op, True, True, True, 6 / 8) 220 elif op == bessel: 221 first = num(op, False, True, True, 6 / 8) 222 else: 223 first = num(op, True, True, True) 224 225 if op != bessel: 226 s = fmts[op].format(scale, first) 227 else: 228 s = fmts[op].format(scale, first, 6 / 8) 229 230 return s 231 232 233# Runs a test with number t. 234# @param t The number of the test. 235def run_test(t): 236 237 # Randomly select the operation. 238 op = random.randrange(bessel + 1) 239 240 # Select the right calculator. 241 if op != modexp: 242 exe = "bc" 243 halt = "halt" 244 options = "-lq" 245 else: 246 exe = "dc" 247 halt = "q" 248 options = "" 249 250 # Generate the test. 251 test = gen_test(op) 252 253 # These don't work very well for some reason. 254 if "c(0)" in test or "scale = 4; j(4" in test: 255 return 256 257 # Make sure the calculator will halt. 258 bcexe = exedir + "/" + exe 259 indata = test + "\n" + halt 260 261 print("Test {}: {}".format(t, test)) 262 263 # Only bc has options. 264 if exe == "bc": 265 args = [ exe, options ] 266 else: 267 args = [ exe ] 268 269 # Run the GNU bc. 270 p = subprocess.run(args, input=indata.encode(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) 271 272 output1 = p.stdout.decode() 273 274 # Error checking for GNU. 275 if p.returncode != 0 or output1 == "": 276 print(" other {} returned an error ({}); continuing...".format(exe, p.returncode)) 277 return 278 279 if output1 == "\n": 280 print(" other {} has a bug; continuing...".format(exe)) 281 return 282 283 # Don't know why GNU has this problem... 284 if output1 == "-0\n": 285 output1 = "0\n" 286 elif output1 == "-0": 287 output1 = "0" 288 289 args = [ bcexe, options ] 290 291 # Run this bc/dc and compare. 292 p = subprocess.run(args, input=indata.encode(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) 293 compare(exe, options, p, test, halt, output1, op) 294 295 296# This script must be run by itself. 297if __name__ != "__main__": 298 sys.exit(1) 299 300script = sys.argv[0] 301testdir = os.path.dirname(script) 302 303exedir = testdir + "/../bin" 304 305# The following are tables used to generate numbers. 306 307# The operations to test. 308ops = [ '+', '-', '*', '/', '%', '^', '|' ] 309 310# The functions that can be tested. 311funcs = [ "sqrt", "e", "l", "a", "s", "c", "j" ] 312 313# The files (corresponding to the operations with the functions appended) to add 314# tests to if they fail. 315files = [ "add", "subtract", "multiply", "divide", "modulus", "power", "modexp", 316 "sqrt", "exponent", "log", "arctangent", "sine", "cosine", "bessel" ] 317 318# The format strings corresponding to each operation and then each function. 319fmts = [ "scale = {}; {} + {}", "scale = {}; {} - {}", "scale = {}; {} * {}", 320 "scale = {}; {} / {}", "scale = {}; {} % {}", "scale = {}; {} ^ {}", 321 "{}k {} {} {}|pR", "scale = {}; sqrt({})", "scale = {}; e({})", 322 "scale = {}; l({})", "scale = {}; a({})", "scale = {}; s({})", 323 "scale = {}; c({})", "scale = {}; j({}, {})" ] 324 325# Constants to make some code easier later. 326div = 3 327mod = 4 328power = 5 329modexp = 6 330sqrt = 7 331exponent = 8 332bessel = 13 333 334gen_ops = [] 335tests = [] 336 337# Infinite loop until the user sends SIGINT. 338try: 339 i = 0 340 while True: 341 run_test(i) 342 i = i + 1 343except KeyboardInterrupt: 344 pass 345 346# This is where we start processing the checklist of possible failures. Why only 347# possible failures? Because some operations, specifically the functions in the 348# math library, are not guaranteed to be exactly correct. Because of that, we 349# need to present every failed test to the user for a final check before we 350# add them as test cases. 351 352# No items, just exit. 353if len(tests) == 0: 354 print("\nNo items in checklist.") 355 print("Exiting") 356 sys.exit(0) 357 358print("\nGoing through the checklist...\n") 359 360# Just do some error checking. If this fails here, it's a bug in this script. 361if len(tests) != len(gen_ops): 362 print("Corrupted checklist!") 363 print("Exiting...") 364 sys.exit(1) 365 366# Go through each item in the checklist. 367for i in range(0, len(tests)): 368 369 # Yes, there's some code duplication. Sue me. 370 371 print("\n{}".format(tests[i])) 372 373 op = int(gen_ops[i]) 374 375 if op != modexp: 376 exe = "bc" 377 halt = "halt" 378 options = "-lq" 379 else: 380 exe = "dc" 381 halt = "q" 382 options = "" 383 384 # We want to run the test again to show the user the difference. 385 indata = tests[i] + "\n" + halt 386 387 args = [ exe, options ] 388 389 p = subprocess.run(args, input=indata.encode(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) 390 391 expected = p.stdout.decode() 392 393 bcexe = exedir + "/" + exe 394 args = [ bcexe, options ] 395 396 p = subprocess.run(args, input=indata.encode(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) 397 398 compare(exe, options, p, tests[i], halt, expected, op, False) 399 400 # Ask the user to make a decision on the failed test. 401 answer = input("\nAdd test ({}/{}) to test suite? [y/N]: ".format(i + 1, len(tests))) 402 403 # Quick and dirty answer parsing. 404 if 'Y' in answer or 'y' in answer: 405 406 print("Yes") 407 408 name = testdir + "/" + exe + "/" + files[op] 409 410 # Write the test to the test file and the expected result to the 411 # results file. 412 with open(name + ".txt", "a") as f: 413 f.write(tests[i] + "\n") 414 415 with open(name + "_results.txt", "a") as f: 416 f.write(expected) 417 418 else: 419 print("No") 420 421print("Done!") 422