1# ################################################################ 2# Copyright (c) Meta Platforms, Inc. and affiliates. 3# All rights reserved. 4# 5# This source code is licensed under both the BSD-style license (found in the 6# LICENSE file in the root directory of this source tree) and the GPLv2 (found 7# in the COPYING file in the root directory of this source tree). 8# You may select, at your option, one of the above-listed licenses. 9# ################################################################ 10 11# verbose mode (print commands) on V=1 or VERBOSE=1 12Q = $(if $(filter 1,$(V) $(VERBOSE)),,@) 13 14PRGDIR = programs 15ZSTDDIR = lib 16BUILDIR = build 17ZWRAPDIR = zlibWrapper 18TESTDIR = tests 19FUZZDIR = $(TESTDIR)/fuzz 20 21# Define nul output 22VOID = /dev/null 23 24# When cross-compiling from linux to windows, you might 25# need to specify this as "Windows." Fedora build fails 26# without it. 27# 28# Note: mingw-w64 build from linux to windows does not 29# fail on other tested distros (ubuntu, debian) even 30# without manually specifying the TARGET_SYSTEM. 31TARGET_SYSTEM ?= $(OS) 32CP ?= cp 33 34ifneq (,$(filter Windows%,$(TARGET_SYSTEM))) 35 EXT =.exe 36else 37 EXT = 38endif 39 40## default: Build lib-release and zstd-release 41.PHONY: default 42default: lib-release zstd-release 43 44.PHONY: all 45all: allmost examples manual contrib 46 47.PHONY: allmost 48allmost: allzstd zlibwrapper 49 50# skip zwrapper, can't build that on alternate architectures without the proper zlib installed 51.PHONY: allzstd 52allzstd: lib 53 $(Q)$(MAKE) -C $(PRGDIR) all 54 $(Q)$(MAKE) -C $(TESTDIR) all 55 56.PHONY: all32 57all32: 58 $(MAKE) -C $(PRGDIR) zstd32 59 $(MAKE) -C $(TESTDIR) all32 60 61.PHONY: lib lib-release lib-mt lib-nomt 62lib lib-release lib-mt lib-nomt: 63 $(Q)$(MAKE) -C $(ZSTDDIR) $@ 64 65.PHONY: zstd zstd-release 66zstd zstd-release: 67 $(Q)$(MAKE) -C $(PRGDIR) $@ 68 $(Q)ln -sf $(PRGDIR)/zstd$(EXT) zstd$(EXT) 69 70.PHONY: zstdmt 71zstdmt: 72 $(Q)$(MAKE) -C $(PRGDIR) $@ 73 $(Q)$(CP) $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT) 74 75.PHONY: zlibwrapper 76zlibwrapper: lib 77 $(MAKE) -C $(ZWRAPDIR) all 78 79## test: run long-duration tests 80.PHONY: test 81DEBUGLEVEL ?= 1 82test: MOREFLAGS += -g -Werror 83test: 84 DEBUGLEVEL=$(DEBUGLEVEL) MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants 85 $(MAKE) -C $(TESTDIR) $@ 86 ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@ 87 88## shortest: same as `make check` 89.PHONY: shortest 90shortest: 91 $(Q)$(MAKE) -C $(TESTDIR) $@ 92 93## check: run basic tests for `zstd` cli 94.PHONY: check 95check: shortest 96 97.PHONY: automated_benchmarking 98automated_benchmarking: 99 $(MAKE) -C $(TESTDIR) $@ 100 101.PHONY: benchmarking 102benchmarking: automated_benchmarking 103 104## examples: build all examples in `examples/` directory 105.PHONY: examples 106examples: lib 107 $(MAKE) -C examples all 108 109## manual: generate API documentation in html format 110.PHONY: manual 111manual: 112 $(MAKE) -C contrib/gen_html $@ 113 114## man: generate man page 115.PHONY: man 116man: 117 $(MAKE) -C programs $@ 118 119## contrib: build all supported projects in `/contrib` directory 120.PHONY: contrib 121contrib: lib 122 $(MAKE) -C contrib/pzstd all 123 $(MAKE) -C contrib/seekable_format/examples all 124 $(MAKE) -C contrib/seekable_format/tests test 125 $(MAKE) -C contrib/largeNbDicts all 126 $(MAKE) -C contrib/externalSequenceProducer all 127 cd build/single_file_libs/ ; ./build_decoder_test.sh 128 cd build/single_file_libs/ ; ./build_library_test.sh 129 130.PHONY: cleanTabs 131cleanTabs: 132 cd contrib; ./cleanTabs 133 134.PHONY: clean 135clean: 136 $(Q)$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) 137 $(Q)$(MAKE) -C $(PRGDIR) $@ > $(VOID) 138 $(Q)$(MAKE) -C $(TESTDIR) $@ > $(VOID) 139 $(Q)$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) 140 $(Q)$(MAKE) -C examples/ $@ > $(VOID) 141 $(Q)$(MAKE) -C contrib/gen_html $@ > $(VOID) 142 $(Q)$(MAKE) -C contrib/pzstd $@ > $(VOID) 143 $(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID) 144 $(Q)$(MAKE) -C contrib/seekable_format/tests $@ > $(VOID) 145 $(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID) 146 $(Q)$(MAKE) -C contrib/externalSequenceProducer $@ > $(VOID) 147 $(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp* 148 $(Q)$(RM) -r lz4 cmakebuild install 149 @echo Cleaning completed 150 151#------------------------------------------------------------------------------ 152# make install is validated only for Linux, macOS, Hurd and some BSD targets 153#------------------------------------------------------------------------------ 154ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT CYGWIN_NT Haiku AIX)) 155 156HOST_OS = POSIX 157 158MKDIR ?= mkdir -p 159 160HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) 161EGREP_OPTIONS ?= 162ifeq ($(HAVE_COLORNEVER), 1) 163EGREP_OPTIONS += --color=never 164endif 165EGREP = egrep $(EGREP_OPTIONS) 166 167# Print a two column output of targets and their description. To add a target description, put a 168# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>". For example: 169# 170## list: Print all targets and their descriptions (if provided) 171.PHONY: list 172list: 173 $(Q)TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \ 174 | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \ 175 | $(EGREP) -v -e '^[^[:alnum:]]' | sort); \ 176 { \ 177 printf "Target Name\tDescription\n"; \ 178 printf "%0.s-" {1..16}; printf "\t"; printf "%0.s-" {1..40}; printf "\n"; \ 179 for target in $$TARGETS; do \ 180 line=$$($(EGREP) "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \ 181 description=$$(echo $$line | awk '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \ 182 printf "$$target\t$$description\n"; \ 183 done \ 184 } | column -t -s $$'\t' 185 186.PHONY: install armtest usan asan uasan msan asan32 187install: 188 $(Q)$(MAKE) -C $(ZSTDDIR) $@ 189 $(Q)$(MAKE) -C $(PRGDIR) $@ 190 191.PHONY: uninstall 192uninstall: 193 $(Q)$(MAKE) -C $(ZSTDDIR) $@ 194 $(Q)$(MAKE) -C $(PRGDIR) $@ 195 196.PHONY: travis-install 197travis-install: 198 $(MAKE) install PREFIX=~/install_test_dir 199 200.PHONY: clangbuild-darwin-fat 201clangbuild-darwin-fat: clean 202 clang -v 203 CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch arm64" $(MAKE) zstd-release 204 mv programs/zstd programs/zstd_arm64 205 CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch x86_64" $(MAKE) zstd-release 206 mv programs/zstd programs/zstd_x64 207 lipo -create programs/zstd_x64 programs/zstd_arm64 -output programs/zstd 208 209.PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build 210gcc5build: clean 211 gcc-5 -v 212 CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)" 213 214gcc6build: clean 215 gcc-6 -v 216 CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)" 217 218gcc7build: clean 219 gcc-7 -v 220 CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)" 221 222clangbuild: clean 223 clang -v 224 CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" $(MAKE) all 225 226m32build: clean 227 gcc -v 228 $(MAKE) all32 229 230armbuild: clean 231 CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allzstd 232 233aarch64build: clean 234 CC=aarch64-linux-gnu-gcc CFLAGS="-Werror -O0" $(MAKE) allzstd 235 236ppcbuild: clean 237 CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd 238 239ppc64build: clean 240 CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) -j allzstd 241 242.PHONY: armfuzz aarch64fuzz ppcfuzz ppc64fuzz 243armfuzz: clean 244 CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest 245 246aarch64fuzz: clean 247 ld -v 248 CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest 249 250ppcfuzz: clean 251 CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest 252 253ppc64fuzz: clean 254 CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest 255 256.PHONY: cxxtest gcc5test gcc6test armtest aarch64test ppctest ppc64test 257cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror 258cxxtest: clean 259 $(MAKE) -C $(PRGDIR) all CC="$(CXX) -Wno-deprecated" CFLAGS="$(CXXFLAGS)" # adding -Wno-deprecated to avoid clang++ warning on dealing with C files directly 260 261gcc5test: clean 262 gcc-5 -v 263 $(MAKE) all CC=gcc-5 MOREFLAGS="-Werror $(MOREFLAGS)" 264 265gcc6test: clean 266 gcc-6 -v 267 $(MAKE) all CC=gcc-6 MOREFLAGS="-Werror $(MOREFLAGS)" 268 269armtest: clean 270 $(MAKE) -C $(TESTDIR) datagen # use native, faster 271 $(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" 272 273aarch64test: 274 $(MAKE) -C $(TESTDIR) datagen # use native, faster 275 $(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" 276 277ppctest: clean 278 $(MAKE) -C $(TESTDIR) datagen # use native, faster 279 $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" 280 281ppc64test: clean 282 $(MAKE) -C $(TESTDIR) datagen # use native, faster 283 $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" 284 285.PHONY: arm-ppc-compilation 286arm-ppc-compilation: 287 $(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" 288 $(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" 289 $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)" 290 $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)" 291 292regressiontest: 293 $(MAKE) -C $(FUZZDIR) regressiontest 294 295uasanregressiontest: 296 $(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined" CXXFLAGS="-O3 -fsanitize=address,undefined" 297 298msanregressiontest: 299 $(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory" 300 301update_regressionResults : REGRESS_RESULTS_DIR := /tmp/regress_results_dir/ 302update_regressionResults: 303 $(MAKE) -C programs zstd 304 $(MAKE) -C tests/regression test 305 $(RM) -rf $(REGRESS_RESULTS_DIR) 306 $(MKDIR) $(REGRESS_RESULTS_DIR) 307 ./tests/regression/test \ 308 --cache tests/regression/cache \ 309 --output $(REGRESS_RESULTS_DIR)/results.csv \ 310 --zstd programs/zstd 311 echo "Showing results differences" 312 ! diff tests/regression/results.csv $(REGRESS_RESULTS_DIR)/results.csv 313 echo "Updating results.csv" 314 $(CP) $(REGRESS_RESULTS_DIR)/results.csv tests/regression/results.csv 315 316 317# run UBsan with -fsanitize-recover=pointer-overflow 318# this only works with recent compilers such as gcc 8+ 319usan: clean 320 $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=undefined -Werror $(MOREFLAGS)" 321 322asan: clean 323 $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror $(MOREFLAGS)" 324 325asan-%: clean 326 LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $* 327 328msan: clean 329 $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason 330 331msan-%: 332 $(MAKE) clean 333 LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -j -C $(TESTDIR) HAVE_LZMA=0 $* 334 335asan32: clean 336 $(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address $(MOREFLAGS)" 337 338uasan: clean 339 $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address,undefined -Werror $(MOREFLAGS)" 340 341uasan-%: clean 342 LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address,undefined -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $* 343 344tsan-%: clean 345 LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" 346 347.PHONY: apt-install 348apt-install: 349 # TODO: uncomment once issue 3011 is resolved and remove hack from Github Actions .yml 350 # sudo apt-get update 351 sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES) 352 353.PHONY: apt-add-repo 354apt-add-repo: 355 sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test 356 sudo apt-get update -y -qq 357 358.PHONY: ppcinstall arminstall valgrindinstall libc6install gcc6install gcc7install gcc8install gpp6install clang38install lz4install 359ppcinstall: 360 APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install 361 362arminstall: 363 APT_PACKAGES="qemu-system-arm qemu-user-static gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install 364 365valgrindinstall: 366 APT_PACKAGES="valgrind" $(MAKE) apt-install 367 368libc6install: 369 APT_PACKAGES="libc6-dev-i386 gcc-multilib" $(MAKE) apt-install 370 371gcc6install: apt-add-repo 372 APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install 373 374gcc7install: apt-add-repo 375 APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-7 gcc-7-multilib" $(MAKE) apt-install 376 377gcc8install: apt-add-repo 378 APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-8 gcc-8-multilib" $(MAKE) apt-install 379 380gpp6install: apt-add-repo 381 APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install 382 383clang38install: 384 APT_PACKAGES="clang-3.8" $(MAKE) apt-install 385 386# Ubuntu 14.04 ships a too-old lz4 387lz4install: 388 [ -e lz4 ] || git clone https://github.com/lz4/lz4 && sudo $(MAKE) -C lz4 install 389 390endif 391 392 393ifneq (,$(filter MSYS%,$(shell uname))) 394HOST_OS = MSYS 395endif 396 397#------------------------------------------------------------------------ 398# target specific tests 399#------------------------------------------------------------------------ 400ifneq (,$(filter $(HOST_OS),MSYS POSIX)) 401 402CMAKE ?= cmake 403CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON 404 405ifneq (,$(filter MSYS%,$(shell uname))) 406CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON 407endif 408 409.PHONY: cmakebuild 410cmakebuild: 411 $(CMAKE) --version 412 $(RM) -r cmakebuild install 413 $(MKDIR) cmakebuild install 414 cd cmakebuild; $(CMAKE) -Wdev -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS="-Werror -O0" -DCMAKE_INSTALL_PREFIX=install $(CMAKE_PARAMS) ../build/cmake 415 $(CMAKE) --build cmakebuild --target install -- -j V=1 416 cd cmakebuild; ctest -V -L Medium 417 418.PHONY: c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze 419c89build: clean 420 $(CC) -v 421 CFLAGS="-std=c89 -Werror -Wno-attributes -Wpedantic -Wno-long-long -Wno-variadic-macros -O0" $(MAKE) lib zstd 422 423gnu90build: clean 424 $(CC) -v 425 CFLAGS="-std=gnu90 -Werror -O0" $(MAKE) allmost 426 427c99build: clean 428 $(CC) -v 429 CFLAGS="-std=c99 -Werror -O0" $(MAKE) allmost 430 431gnu99build: clean 432 $(CC) -v 433 CFLAGS="-std=gnu99 -Werror -O0" $(MAKE) allmost 434 435c11build: clean 436 $(CC) -v 437 CFLAGS="-std=c11 -Werror -O0" $(MAKE) allmost 438 439bmix64build: clean 440 $(CC) -v 441 CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test 442 443bmix32build: clean 444 $(CC) -v 445 CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test 446 447bmi32build: clean 448 $(CC) -v 449 CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test 450 451# static analyzer test uses clang's scan-build 452# does not analyze zlibWrapper, due to detected issues in zlib source code 453staticAnalyze: SCANBUILD ?= scan-build 454staticAnalyze: 455 $(CC) -v 456 CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) zstd 457endif 458