1#!/bin/bash -ex 2 3# Copyright 2020 Google Inc. All rights reserved. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17: "${OUT_DIR:?Must set OUT_DIR}" 18TOP=$(pwd) 19 20UNAME="$(uname)" 21case "$UNAME" in 22Linux) 23 OS='linux' 24 ;; 25Darwin) 26 OS='darwin' 27 ;; 28*) 29 exit 1 30 ;; 31esac 32 33build_soong=1 34build_asan=1 35[[ ! -d ${TOP}/toolchain/go ]] || build_go=1 36 37use_musl=false 38clean=true 39while getopts ":-:" opt; do 40 case "$opt" in 41 -) 42 case "${OPTARG}" in 43 resume) clean= ;; 44 musl) use_musl=true ;; 45 skip-go) unset build_go ;; 46 skip-soong) unset build_soong ;; 47 skip-soong-tests) skip_soong_tests=--skip-soong-tests ;; 48 skip-asan) unset build_asan ;; 49 *) echo "Unknown option --${OPTARG}"; exit 1 ;; 50 esac;; 51 *) echo "'${opt}' '${OPTARG}'" 52 esac 53done 54 55secondary_arch="" 56if [[ ${OS} = linux ]]; then 57 secondary_arch="\"HostSecondaryArch\":\"x86\"," 58fi 59 60cross_compile="" 61if [[ ${use_musl} = "true" ]]; then 62 cross_compile=$(cat <<EOF 63 "CrossHost": "linux_musl", 64 "CrossHostArch": "arm64", 65 "CrossHostSecondaryArch": "arm", 66EOF 67 ) 68elif [[ ${OS} = darwin ]]; then 69 cross_compile=$(cat <<EOF 70 "CrossHost": "darwin", 71 "CrossHostArch": "arm64", 72EOF 73 ) 74fi 75 76# Use toybox and other prebuilts even outside of the build (test running, go, etc) 77export PATH=${TOP}/prebuilts/build-tools/path/${OS}-x86:$PATH 78 79if [ -n "${build_soong}" ]; then 80 SOONG_OUT=${OUT_DIR}/soong 81 SOONG_HOST_OUT=${OUT_DIR}/soong/host/${OS}-x86 82 [[ -z "${clean}" ]] || rm -rf ${SOONG_OUT} 83 mkdir -p ${SOONG_OUT} 84 rm -rf ${SOONG_OUT}/dist ${SOONG_OUT}/dist-common ${SOONG_OUT}/dist-arm64 85 cat > ${SOONG_OUT}/soong.variables.tmp << EOF 86{ 87 "Allow_missing_dependencies": true, 88 "HostArch":"x86_64", 89 ${secondary_arch} 90 ${cross_compile} 91 "HostMusl": $use_musl, 92 "VendorVars": { 93 "cpython3": { 94 "force_build_host": "true" 95 }, 96 "art_module": { 97 "source_build": "true" 98 } 99 }, 100 "VendorVarTypes": { 101 "cpython3": { 102 "force_build_host": "bool" 103 } 104 } 105} 106EOF 107 if cmp -s ${SOONG_OUT}/soong.variables.tmp ${SOONG_OUT}/soong.variables; then 108 rm ${SOONG_OUT}/soong.variables.tmp 109 else 110 mv -f ${SOONG_OUT}/soong.variables.tmp ${SOONG_OUT}/soong.variables 111 fi 112 SOONG_GO_BINARIES=( 113 bpfmt 114 go_extractor 115 kotlinc_extractor 116 merge_zips 117 soong_zip 118 runextractor 119 rust_extractor 120 zip2zip 121 ) 122 SOONG_BINARIES=( 123 aconfig 124 acp 125 aidl 126 bison 127 bloaty 128 brotli 129 bzip2 130 ckati 131 edit_monitor 132 flex 133 gavinhoward-bc 134 hidl-gen 135 hidl-lint 136 m4 137 make 138 n2 139 ninja 140 one-true-awk 141 openssl 142 py3-cmd 143 py3-launcher64 144 py3-launcher-autorun64 145 tool_event_logger 146 toybox 147 xz 148 zipalign 149 ziptime 150 ziptool 151 ) 152 SOONG_MUSL_BINARIES=( 153 py3-launcher-static64 154 py3-launcher-autorun-static64 155 ) 156 SOONG_ASAN_BINARIES=( 157 acp 158 aidl 159 ckati 160 gavinhoward-bc 161 ninja 162 toybox 163 zipalign 164 ziptime 165 ziptool 166 ) 167 SOONG_JAVA_LIBRARIES=( 168 desugar.jar 169 dx.jar 170 javac_extractor.jar 171 ktfmt.jar 172 turbine.jar 173 ) 174 SOONG_JAVA_WRAPPERS=( 175 dx 176 ) 177 if [[ $OS == "linux" ]]; then 178 SOONG_BINARIES+=( 179 create_minidebuginfo 180 nsjail 181 py2-cmd 182 ) 183 fi 184 185 go_binaries="${SOONG_GO_BINARIES[@]/#/${SOONG_HOST_OUT}/bin/}" 186 binaries="${SOONG_BINARIES[@]/#/${SOONG_HOST_OUT}/bin/}" 187 asan_binaries="${SOONG_ASAN_BINARIES[@]/#/${SOONG_HOST_OUT}/bin/}" 188 jars="${SOONG_JAVA_LIBRARIES[@]/#/${SOONG_HOST_OUT}/framework/}" 189 wrappers="${SOONG_JAVA_WRAPPERS[@]/#/${SOONG_HOST_OUT}/bin/}" 190 191 # TODO: When we have a better method of extracting zips from Soong, use that. 192 py3_stdlib_zip="${SOONG_OUT}/.intermediates/external/python/cpython3/Lib/py3-stdlib-zip/gen/py3-stdlib.zip" 193 194 musl_x86_sysroot="" 195 musl_x86_64_sysroot="" 196 musl_arm_sysroot="" 197 musl_arm64_sysroot="" 198 cross_binaries="" 199 if [[ ${use_musl} = "true" ]]; then 200 binaries="${binaries} ${SOONG_MUSL_BINARIES[@]/#/${SOONG_HOST_OUT}/bin/}" 201 musl_x86_sysroot="${SOONG_OUT}/.intermediates/external/musl/libc_musl_sysroot/linux_musl_x86/gen/libc_musl_sysroot.zip" 202 musl_x86_64_sysroot="${SOONG_OUT}/.intermediates/external/musl/libc_musl_sysroot/linux_musl_x86_64/gen/libc_musl_sysroot.zip" 203 204 # Build cross-compiled musl arm64 binaries, except for go binaries 205 # since Blueprint doesn't have cross compilation support for them. 206 SOONG_HOST_ARM_OUT=${OUT_DIR}/soong/host/linux-arm64 207 cross_binaries="${SOONG_BINARIES[@]/#/${SOONG_HOST_ARM_OUT}/bin/}" 208 cross_binaries="${cross_binaries} ${SOONG_MUSL_BINARIES[@]/#/${SOONG_HOST_ARM_OUT}/bin/}" 209 210 musl_arm_sysroot="${SOONG_OUT}/.intermediates/external/musl/libc_musl_sysroot/linux_musl_arm/gen/libc_musl_sysroot.zip" 211 musl_arm64_sysroot="${SOONG_OUT}/.intermediates/external/musl/libc_musl_sysroot/linux_musl_arm64/gen/libc_musl_sysroot.zip" 212 fi 213 214 # Build everything 215 build/soong/soong_ui.bash --make-mode --soong-only --skip-config ${skip_soong_tests} \ 216 ${go_binaries} \ 217 ${binaries} \ 218 ${cross_binaries} \ 219 ${wrappers} \ 220 ${jars} \ 221 ${py3_stdlib_zip} \ 222 ${musl_x86_sysroot} \ 223 ${musl_x86_64_sysroot} \ 224 ${musl_arm_sysroot} \ 225 ${musl_arm64_sysroot} \ 226 ${SOONG_HOST_OUT}/nativetest64/n2_e2e_tests/n2_e2e_tests \ 227 ${SOONG_HOST_OUT}/nativetest64/n2_unit_tests/n2_unit_tests \ 228 ${SOONG_HOST_OUT}/nativetest64/ninja_test/ninja_test \ 229 ${SOONG_HOST_OUT}/nativetest64/ckati_find_test/ckati_find_test \ 230 ${SOONG_HOST_OUT}/nativetest64/par_test/par_test \ 231 soong_docs 232 233 # Run ninja tests 234 ${SOONG_HOST_OUT}/nativetest64/ninja_test/ninja_test 235 236 # Run n2 tests 237 timeout -v -k 10 300 ${SOONG_HOST_OUT}/nativetest64/n2_unit_tests/n2_unit_tests 238 N2_PATH=${SOONG_HOST_OUT}/bin/n2 timeout -v -k 10 300 ${SOONG_HOST_OUT}/nativetest64/n2_e2e_tests/n2_e2e_tests 239 240 # Run ckati tests 241 ${SOONG_HOST_OUT}/nativetest64/ckati_find_test/ckati_find_test 242 243 # Run python par/py*-cmd tests 244 ANDROID_HOST_OUT=${PWD}/${SOONG_HOST_OUT} build/soong/python/tests/runtest.sh 245 246 # Copy arch-specific binaries 247 mkdir -p ${SOONG_OUT}/dist/bin 248 cp ${go_binaries} ${binaries} ${SOONG_OUT}/dist/bin/ 249 cp -R ${SOONG_HOST_OUT}/lib* ${SOONG_OUT}/dist/ 250 251 # Copy cross-compiled binaries 252 if [[ ${use_musl} = "true" ]]; then 253 mkdir -p ${SOONG_OUT}/dist-arm64/bin 254 cp ${cross_binaries} ${SOONG_OUT}/dist-arm64/bin/ 255 cp -R ${SOONG_HOST_ARM_OUT}/lib* ${SOONG_OUT}/dist-arm64/ 256 fi 257 258 # Copy jars and wrappers 259 mkdir -p ${SOONG_OUT}/dist-common/{bin,flex,framework,py3-stdlib} 260 cp ${wrappers} ${SOONG_OUT}/dist-common/bin 261 cp ${jars} ${SOONG_OUT}/dist-common/framework 262 263 cp -r external/bison/data ${SOONG_OUT}/dist-common/bison 264 cp external/bison/NOTICE ${SOONG_OUT}/dist-common/bison/ 265 cp -r external/flex/src/FlexLexer.h ${SOONG_OUT}/dist-common/flex/ 266 cp external/flex/NOTICE ${SOONG_OUT}/dist-common/flex/ 267 268 unzip -q -d ${SOONG_OUT}/dist-common/py3-stdlib ${py3_stdlib_zip} 269 cp external/python/cpython3/LICENSE ${SOONG_OUT}/dist-common/py3-stdlib/ 270 271 if [[ ${use_musl} = "true" ]]; then 272 cp ${musl_x86_64_sysroot} ${SOONG_OUT}/musl-sysroot-x86_64-unknown-linux-musl.zip 273 cp ${musl_x86_sysroot} ${SOONG_OUT}/musl-sysroot-i686-unknown-linux-musl.zip 274 cp ${musl_arm_sysroot} ${SOONG_OUT}/musl-sysroot-arm-unknown-linux-musleabihf.zip 275 cp ${musl_arm64_sysroot} ${SOONG_OUT}/musl-sysroot-aarch64-unknown-linux-musl.zip 276 fi 277 278 279 if [[ $OS == "linux" && -n "${build_asan}" ]]; then 280 # Build ASAN versions 281 export ASAN_OPTIONS=detect_leaks=0 282 cat > ${SOONG_OUT}/soong.variables << EOF 283{ 284 "Allow_missing_dependencies": true, 285 "HostArch":"x86_64", 286 ${secondary_arch} 287 "SanitizeHost": ["address"], 288 "VendorVars": { 289 "art_module": { 290 "source_build": "true" 291 } 292 } 293} 294EOF 295 296 export ASAN_SYMBOLIZER_PATH=${PWD}/prebuilts/clang/host/linux-x86/llvm-binutils-stable/llvm-symbolizer 297 298 # Clean up non-ASAN installed versions 299 rm -rf ${SOONG_HOST_OUT} 300 301 # Build everything with ASAN 302 build/soong/soong_ui.bash --make-mode --soong-only --skip-config ${skip_soong_tests} \ 303 ${asan_binaries} \ 304 ${SOONG_HOST_OUT}/nativetest64/ninja_test/ninja_test \ 305 ${SOONG_HOST_OUT}/nativetest64/ckati_find_test/ckati_find_test 306 307 # Run ninja tests 308 ${SOONG_HOST_OUT}/nativetest64/ninja_test/ninja_test 309 310 # Run ckati tests 311 ${SOONG_HOST_OUT}/nativetest64/ckati_find_test/ckati_find_test 312 313 # Copy arch-specific binaries 314 mkdir -p ${SOONG_OUT}/dist/asan/bin 315 cp ${asan_binaries} ${SOONG_OUT}/dist/asan/bin/ 316 cp -R ${SOONG_HOST_OUT}/lib* ${SOONG_OUT}/dist/asan/ 317 fi 318 319 # Package arch-specific prebuilts 320 ( 321 cd ${SOONG_OUT}/dist 322 zip -qryX build-prebuilts.zip * 323 ) 324 325 if [[ ${use_musl} = "true" ]]; then 326 # Package cross-compiled prebuilts 327 ( 328 cd ${SOONG_OUT}/dist-arm64 329 zip -qryX build-arm64-prebuilts.zip * 330 ) 331 fi 332 333 # Package common prebuilts 334 ( 335 cd ${SOONG_OUT}/dist-common 336 zip -qryX build-common-prebuilts.zip * 337 ) 338fi 339 340if [ -z "${skip_soong_tests}" ]; then 341 build/soong/scripts/run-soong-tests-with-go-tools.sh 342fi 343 344# Go 345if [ -n "${build_go}" ]; then 346 GO_OUT=${OUT_DIR}/obj/go 347 rm -rf ${GO_OUT} 348 mkdir -p ${GO_OUT} 349 cp -a ${TOP}/toolchain/go/* ${GO_OUT}/ 350 ( 351 cd ${GO_OUT}/src 352 export GOROOT_BOOTSTRAP=${TOP}/prebuilts/go/${OS}-x86 353 export GOROOT_FINAL=./prebuilts/go/${OS}-x86 354 export GO_TEST_TIMEOUT_SCALE=100 355 export GODEBUG=installgoroot=all 356 ./make.bash 357 rm -rf ../pkg/bootstrap 358 rm -rf ../pkg/obj 359 GOROOT=$(pwd)/.. ../bin/go install -race std 360 ) 361 ( 362 cd ${GO_OUT} 363 zip -qryX go.zip * --exclude update_prebuilts.sh 364 ) 365fi 366 367if [ -n "${DIST_DIR}" ]; then 368 mkdir -p ${DIST_DIR} || true 369 370 if [ -n "${build_soong}" ]; then 371 cp ${SOONG_OUT}/dist/build-prebuilts.zip ${DIST_DIR}/ 372 cp ${SOONG_OUT}/dist-common/build-common-prebuilts.zip ${DIST_DIR}/ 373 cp ${SOONG_OUT}/docs/*.html ${DIST_DIR}/ 374 if [ ${use_musl} = "true" ]; then 375 cp ${SOONG_OUT}/dist-arm64/build-arm64-prebuilts.zip ${DIST_DIR}/ 376 377 cp ${SOONG_OUT}/musl-sysroot-x86_64-unknown-linux-musl.zip ${DIST_DIR}/ 378 cp ${SOONG_OUT}/musl-sysroot-i686-unknown-linux-musl.zip ${DIST_DIR}/ 379 cp ${SOONG_OUT}/musl-sysroot-arm-unknown-linux-musleabihf.zip ${DIST_DIR}/ 380 cp ${SOONG_OUT}/musl-sysroot-aarch64-unknown-linux-musl.zip ${DIST_DIR}/ 381 fi 382 fi 383 if [ -n "${build_go}" ]; then 384 cp ${GO_OUT}/go.zip ${DIST_DIR}/ 385 fi 386fi 387