1#!/usr/bin/env python3 2"""Test LZ4 interoperability between versions""" 3 4# 5# Copyright (C) 2011-present, Takayuki Matsuoka 6# All rights reserved. 7# GPL v2 License 8# 9 10import argparse 11import glob 12import subprocess 13import filecmp 14import os 15import shutil 16import sys 17import hashlib 18 19repo_url = 'https://github.com/lz4/lz4.git' 20tmp_dir_name = 'tests/versionsTest' 21make_cmd = 'make' 22git_cmd = 'git' 23test_dat_src = 'README.md' 24test_dat = 'test_dat' 25head = 'v999' 26 27parser = argparse.ArgumentParser() 28parser.add_argument("--verbose", action="store_true", help="increase output verbosity") 29args = parser.parse_args() 30 31def debug_message(msg): 32 if args.verbose: 33 print(msg) 34 35def env_or_empty(env, key): 36 if key in env: 37 return " " + env[key] 38 return "" 39 40def proc(cmd_args, pipe=True, env=False): 41 if env == False: 42 env = os.environ.copy() 43 debug_message("Executing command {} with env {}".format(cmd_args, env)) 44 if pipe: 45 s = subprocess.Popen(cmd_args, 46 stdout=subprocess.PIPE, 47 stderr=subprocess.PIPE, 48 env = env) 49 else: 50 s = subprocess.Popen(cmd_args, env = env) 51 stdout_data, stderr_data = s.communicate() 52 if s.poll() != 0: 53 print('Error Code:', s.poll()) 54 print('Standard Error:', stderr_data.decode()) 55 sys.exit(1) 56 return stdout_data, stderr_data 57 58def make(args, pipe=True, env=False): 59 if env == False: 60 env = os.environ.copy() 61 # favor compilation speed for faster total test time, actual runtime is very short 62 env["CFLAGS"] = env_or_empty(env, 'CFLAGS') + " -O0" 63 # old versions of lz4 may require MOREFLAGS 64 env["MOREFLAGS"] = env_or_empty(env, 'MOREFLAGS') + env_or_empty(env, 'CFLAGS') + env_or_empty(env, 'CPPFLAGS') + env_or_empty(env, 'LDFLAGS') 65 return proc([make_cmd] + args, pipe, env) 66 67def git(args, pipe=True): 68 return proc([git_cmd] + args, pipe) 69 70def get_git_tags(): 71 stdout, stderr = git(['tag', '-l', 'r[0-9][0-9][0-9]']) 72 tags = stdout.decode('utf-8').split() 73 stdout, stderr = git(['tag', '-l', 'v[1-9].[0-9].[0-9]']) 74 tags += stdout.decode('utf-8').split() 75 return tags 76 77# https://stackoverflow.com/a/19711609/2132223 78def sha1_of_file(filepath): 79 with open(filepath, 'rb') as f: 80 return hashlib.sha1(f.read()).hexdigest() 81 82if __name__ == '__main__': 83 error_code = 0 84 base_dir = os.getcwd() + '/..' # /path/to/lz4 85 tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/lz4/tests/versionsTest 86 clone_dir = tmp_dir + '/' + 'lz4' # /path/to/lz4/tests/versionsTest/lz4 87 programs_dir = base_dir + '/programs' # /path/to/lz4/programs 88 os.makedirs(tmp_dir, exist_ok=True) 89 90 # since Travis clones limited depth, we should clone full repository 91 if not os.path.isdir(clone_dir): 92 git(['clone', repo_url, clone_dir]) 93 94 shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat) 95 96 # Retrieve all release tags 97 print('Retrieve all release tags :') 98 os.chdir(clone_dir) 99 tags = [head] + get_git_tags() 100 print(tags); 101 102 # Build all release lz4c and lz4c32 103 for tag in tags: 104 print("processing tag " + tag) 105 os.chdir(base_dir) 106 dst_lz4c = '{}/lz4c.{}' .format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c.<TAG> 107 dst_lz4c32 = '{}/lz4c32.{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c32.<TAG> 108 if not os.path.isfile(dst_lz4c) or not os.path.isfile(dst_lz4c32) or tag == head: 109 if tag != head: 110 r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/<TAG> 111 os.makedirs(r_dir, exist_ok=True) 112 os.chdir(clone_dir) 113 git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False) 114 os.chdir(r_dir + '/programs') # /path/to/lz4/lz4test/<TAG>/programs 115 else: 116 os.chdir(programs_dir) 117 make(['clean'], False) 118 make(['lz4c'], False) 119 shutil.copy2('lz4c', dst_lz4c) 120 make(['clean'], False) 121 make(['lz4c32'], False) 122 shutil.copy2('lz4c32', dst_lz4c32) 123 124 # Compress test.dat by all released lz4c and lz4c32 125 print('Compress test.dat by all released lz4c and lz4c32') 126 os.chdir(tmp_dir) 127 for lz4 in glob.glob("*.lz4"): 128 os.remove(lz4) 129 for tag in tags: 130 proc(['./lz4c.' + tag, '-1fz', test_dat, test_dat + '_1_64_' + tag + '.lz4']) 131 proc(['./lz4c.' + tag, '-9fz', test_dat, test_dat + '_9_64_' + tag + '.lz4']) 132 proc(['./lz4c32.' + tag, '-1fz', test_dat, test_dat + '_1_32_' + tag + '.lz4']) 133 proc(['./lz4c32.' + tag, '-9fz', test_dat, test_dat + '_9_32_' + tag + '.lz4']) 134 135 print('Full list of compressed files') 136 lz4s = sorted(glob.glob('*.lz4')) 137 for lz4 in lz4s: 138 print(lz4 + ' : ' + repr(os.path.getsize(lz4))) 139 140 # Remove duplicated .lz4 files 141 print('') 142 print('Duplicated files') 143 lz4s = sorted(glob.glob('*.lz4')) 144 for i, lz4 in enumerate(lz4s): 145 if not os.path.isfile(lz4): 146 continue 147 for j in range(i+1, len(lz4s)): 148 lz4t = lz4s[j] 149 if not os.path.isfile(lz4t): 150 continue 151 if filecmp.cmp(lz4, lz4t): 152 os.remove(lz4t) 153 print('{} == {}'.format(lz4, lz4t)) 154 155 print('Enumerate only different compressed files') 156 lz4s = sorted(glob.glob('*.lz4')) 157 for lz4 in lz4s: 158 print(lz4 + ' : ' + repr(os.path.getsize(lz4)) + ', ' + sha1_of_file(lz4)) 159 160 # Decompress remained .lz4 files by all released lz4c and lz4c32 161 print('Decompression tests and verifications') 162 lz4s = sorted(glob.glob('*.lz4')) 163 for dec in glob.glob("*.dec"): 164 os.remove(dec) 165 for lz4 in lz4s: 166 print(lz4, end=" ") 167 for tag in tags: 168 print(tag, end=" ") 169 proc(['./lz4c.' + tag, '-df', lz4, lz4 + '_d64_' + tag + '.dec']) 170 proc(['./lz4c32.' + tag, '-df', lz4, lz4 + '_d32_' + tag + '.dec']) 171 print(' OK') # well, here, decompression has worked; but file is not yet verified 172 173 # Compare all '.dec' files with test_dat 174 decs = glob.glob('*.dec') 175 for dec in decs: 176 if not filecmp.cmp(dec, test_dat): 177 print('ERR : ' + dec) 178 error_code = 1 179 else: 180 print('OK : ' + dec) 181 os.remove(dec) 182 183 if error_code != 0: 184 print('ERROR') 185 186 sys.exit(error_code) 187