1#!/bin/bash
2
3# Note: not intended to be invoked directly, see rebuild.sh.
4#
5# Rebuilds Crosvm and its dependencies from a clean state.
6
7: ${TOOLS_DIR:="$(pwd)/tools"}
8
9# Stable is usually too old for crosvm, but make sure you bump this
10# up as far as you can each time this script is touched..
11RUST_TOOLCHAIN_VER=1.65.0
12
13setup_env() {
14  : ${SOURCE_DIR:="$(pwd)/source"}
15  : ${WORKING_DIR:="$(pwd)/working"}
16  : ${CUSTOM_MANIFEST:=""}
17
18  ARCH="$(uname -m)"
19  : ${OUTPUT_DIR:="$(pwd)/${ARCH}-linux-gnu"}
20  OUTPUT_BIN_DIR="${OUTPUT_DIR}/bin"
21  OUTPUT_ETC_DIR="${OUTPUT_DIR}/etc"
22  OUTPUT_SECCOMP_DIR="${OUTPUT_ETC_DIR}/seccomp"
23
24  export PATH="${PATH}:${TOOLS_DIR}:${HOME}/.local/bin"
25  export PKG_CONFIG_PATH="${WORKING_DIR}/usr/lib/pkgconfig"
26}
27
28set -e
29set -x
30
31fatal_echo() {
32  echo "$@"
33  exit 1
34}
35
36prepare_cargo() {
37  echo Setting up cargo...
38  cd
39  rm -rf .cargo
40  # Sometimes curl hangs. When it does, retry
41  retry curl -L \
42    --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rustup-init
43  chmod +x rustup-init
44  ./rustup-init -y --no-modify-path --default-toolchain ${RUST_TOOLCHAIN_VER}
45  source $HOME/.cargo/env
46  if [[ -n "$1" ]]; then
47    rustup target add "$1"
48  fi
49  rm -f rustup-init
50
51  if [[ -n "$1" ]]; then
52  cat >>~/.cargo/config <<EOF
53[target.$1]
54linker = "${1/-unknown-/-}"
55EOF
56  fi
57}
58
59install_packages() {
60  echo Installing packages...
61
62  sudo dpkg --add-architecture arm64
63  sudo apt-get update
64  sudo apt-get install -y \
65      "$@" \
66      autoconf \
67      automake \
68      build-essential \
69      clang \
70      curl \
71      doxygen \
72      g++ \
73      gcc \
74      git \
75      graphviz \
76      libcap-dev \
77      libegl1-mesa-dev \
78      libfdt-dev \
79      libgl1-mesa-dev \
80      libgles2-mesa-dev \
81      libpciaccess-dev \
82      libssl-dev \
83      libtool \
84      libusb-1.0-0-dev \
85      libwayland-bin \
86      libwayland-dev \
87      libxml2-dev \
88      make \
89      nasm \
90      ninja-build \
91      pkg-config \
92      protobuf-compiler \
93      python3 \
94      python3-pip \
95      texinfo \
96      wayland-protocols \
97      xmlto \
98      xutils-dev # Needed to pacify autogen.sh for libepoxy
99  mkdir -p "${TOOLS_DIR}"
100
101  # Repo needs python3 but python-is-python3 package not available:
102  sudo ln -s -f /usr/bin/python3 /usr/bin/python
103
104  curl https://storage.googleapis.com/git-repo-downloads/repo > "${TOOLS_DIR}/repo"
105  chmod a+x "${TOOLS_DIR}/repo"
106
107  # Gfxstream needs a new-ish version of CMake
108  mkdir -p "${TOOLS_DIR}/cmake"
109  cd "${TOOLS_DIR}/cmake"
110  curl -O -L https://cmake.org/files/v3.22/cmake-3.22.1-linux-$(uname -m).sh
111  chmod +x cmake-3.22.1-linux-$(uname -m).sh
112  sudo ./cmake-3.22.1-linux-$(uname -m).sh --skip-license --exclude-subdir --prefix=/usr/local
113  cmake --version
114
115  # Meson getting started guide mentions that the distro version is frequently
116  # outdated and recommends installing via pip.
117  pip3 install --no-warn-script-location meson
118
119  # Tools for building gfxstream
120  pip3 install absl-py
121  pip3 install urlfetch
122
123  case "$(uname -m)" in
124    aarch64)
125      prepare_cargo
126      ;;
127    x86_64)
128      # Cross-compilation is x86_64 specific
129      sudo apt install -y crossbuild-essential-arm64
130      prepare_cargo aarch64-unknown-linux-gnu
131      ;;
132  esac
133}
134
135retry() {
136  for i in $(seq 5); do
137    "$@" && return 0
138    sleep 1
139  done
140  return 1
141}
142
143fetch_source() {
144  echo "Fetching source..."
145
146  mkdir -p "${SOURCE_DIR}"
147  cd "${SOURCE_DIR}"
148
149  if ! git config user.name; then
150    git config --global user.name "AOSP Crosvm Builder"
151    git config --global user.email "[email protected]"
152    git config --global color.ui false
153  fi
154
155  if [[ -z "${CUSTOM_MANIFEST}" ]]; then
156    # Building Crosvm currently depends using Chromium's directory scheme for subproject
157    # directories ('third_party' vs 'external').
158    fatal_echo "CUSTOM_MANIFEST must be provided. You most likely want to provide a full path to" \
159               "a copy of device/google/cuttlefish_vmm/manifest.xml."
160  fi
161
162  cp ${CUSTOM_MANIFEST} manifest.xml
163  repo init --depth=1 -q -u https://android.googlesource.com/platform/manifest -m $PWD/manifest.xml
164  repo sync
165}
166
167prepare_source() {
168  if [ "$(ls -A $SOURCE_DIR)" ]; then
169    echo "${SOURCE_DIR} is non empty. Run this from an empty directory if you wish to fetch the source." 1>&2
170    exit 2
171  fi
172  fetch_source
173}
174
175resync_source() {
176  echo "Deleting source directory..."
177  rm -rf "${SOURCE_DIR}/.*"
178  rm -rf "${SOURCE_DIR}/*"
179  fetch_source
180}
181
182# $1 = installed library filename
183debuglink() {
184  objcopy --only-keep-debug "${OUTPUT_BIN_DIR}/$1" "${OUTPUT_BIN_DIR}/$1.debug"
185  strip --strip-debug "${OUTPUT_BIN_DIR}/$1"
186  cd "${OUTPUT_BIN_DIR}"
187  objcopy --add-gnu-debuglink="$1.debug" "$1"
188  cd -
189}
190
191compile_libdrm() {
192  cd "${SOURCE_DIR}/external/libdrm"
193
194  # Ensure pkg-config file supplies rpath to dependent libraries
195  grep "install_rpath" meson.build || \
196    sed -i "s|install : true,|install : true, install_rpath : '\$ORIGIN',|" meson.build
197
198  meson build \
199    --libdir="${WORKING_DIR}/usr/lib" \
200    --prefix="${WORKING_DIR}/usr" \
201    -Damdgpu=false \
202    -Dfreedreno=false \
203    -Dintel=false \
204    -Dlibkms=false \
205    -Dnouveau=false \
206    -Dradeon=false \
207    -Dvc4=false \
208    -Dvmwgfx=false
209
210  cd build
211  ninja install
212
213  cp -a "${WORKING_DIR}"/usr/lib/libdrm.so* "${OUTPUT_BIN_DIR}"
214  debuglink libdrm.so.2.4.0
215}
216
217compile_minijail() {
218  echo "Compiling Minijail..."
219
220  cd "${SOURCE_DIR}/platform/minijail"
221
222  if ! grep '^# cuttlefish_vmm-rebuild-mark' Makefile; then
223    # Link minijail-sys rust crate dynamically to minijail
224    sed -i '/BUILD_STATIC_LIBS/d' rust/minijail-sys/build.rs
225    sed -i 's,static=minijail.pic,dylib=minijail,' rust/minijail-sys/build.rs
226
227    # Use Android prebuilt C files instead of generating them
228    sed -i 's,\(.*\.gen\.c: \),DISABLED_\1,' Makefile
229    cat >>Makefile <<EOF
230libconstants.gen.c: \$(SRC)/linux-x86/libconstants.gen.c
231	@cp \$< \$@
232libsyscalls.gen.c: \$(SRC)/linux-x86/libsyscalls.gen.c
233	@cp \$< \$@
234# cuttlefish_vmm-rebuild-mark
235EOF
236  fi
237
238  make -j OUT="${WORKING_DIR}"
239  cp "${WORKING_DIR}"/libminijail.so "${WORKING_DIR}"/usr/lib
240
241  cp -a "${WORKING_DIR}"/usr/lib/libminijail.so "${OUTPUT_BIN_DIR}"
242  debuglink libminijail.so
243}
244
245compile_minigbm() {
246  echo "Compiling Minigbm..."
247
248  cd "${SOURCE_DIR}/platform/minigbm"
249
250  # Minigbm's package config file has a default hard-coded path. Update here so
251  # that dependent packages can find the files.
252  sed -i "s|prefix=/usr\$|prefix=${WORKING_DIR}/usr|" gbm.pc
253
254  # The gbm used by upstream linux distros is not compatible with crosvm, which must use Chrome OS's
255  # minigbm.
256  local cpp_flags=(-I/working/usr/include -I/working/usr/include/libdrm)
257  local ld_flags=(-Wl,-soname,libgbm.so.1 -Wl,-rpath,\\\$\$ORIGIN -L/working/usr/lib)
258  local make_flags=()
259  local minigbm_drv=(${MINIGBM_DRV})
260  for drv in "${minigbm_drv[@]}"; do
261    cpp_flags+=(-D"DRV_${drv}")
262    make_flags+=("DRV_${drv}"=1)
263  done
264
265  make -j install \
266    "${make_flags[@]}" \
267    CPPFLAGS="${cpp_flags[*]}" \
268    DESTDIR="${WORKING_DIR}" \
269    LDFLAGS="${ld_flags[*]}" \
270    OUT="${WORKING_DIR}"
271
272  cp -a "${WORKING_DIR}"/usr/lib/libminigbm.so* "${OUTPUT_BIN_DIR}"
273  cp -a "${WORKING_DIR}"/usr/lib/libgbm.so* "${OUTPUT_BIN_DIR}"
274  debuglink libminigbm.so.1.0.0
275}
276
277compile_epoxy() {
278  cd "${SOURCE_DIR}/third_party/libepoxy"
279
280  meson build \
281    --libdir="${WORKING_DIR}/usr/lib" \
282    --prefix="${WORKING_DIR}/usr" \
283    -Dglx=no \
284    -Dx11=false \
285    -Degl=yes
286
287  cd build
288  ninja install
289
290  cp -a "${WORKING_DIR}"/usr/lib/libepoxy.so* "${OUTPUT_BIN_DIR}"
291  debuglink libepoxy.so.0.0.0
292}
293
294compile_virglrenderer() {
295  echo "Compiling VirglRenderer..."
296
297  # Note: depends on libepoxy
298  cd "${SOURCE_DIR}/third_party/virglrenderer"
299
300  # Meson needs to have dependency information for header lookup.
301  sed -i "s|cc.has_header('epoxy/egl.h')|cc.has_header('epoxy/egl.h', dependencies: epoxy_dep)|" meson.build
302
303  # Ensure pkg-config file supplies rpath to dependent libraries
304  grep "install_rpath" src/meson.build || \
305    sed -i "s|install : true|install : true, install_rpath : '\$ORIGIN'|" src/meson.build
306
307  meson build \
308    --libdir="${WORKING_DIR}/usr/lib" \
309    --prefix="${WORKING_DIR}/usr" \
310    -Dplatforms=egl \
311    -Dminigbm_allocation=false
312
313  cd build
314  ninja install
315
316  cp -a "${WORKING_DIR}"/usr/lib/libvirglrenderer.so* "${OUTPUT_BIN_DIR}"
317  debuglink libvirglrenderer.so.1.7.7
318}
319
320compile_libffi() {
321  cd "${SOURCE_DIR}/third_party/libffi"
322
323  ./autogen.sh
324  ./configure \
325    --prefix="${WORKING_DIR}/usr" \
326    --libdir="${WORKING_DIR}/usr/lib"
327  make && make check && make install
328
329  cp -a "${WORKING_DIR}"/usr/lib/libffi.so* "${OUTPUT_BIN_DIR}"
330  debuglink libffi.so.7.1.0
331}
332
333compile_wayland() {
334  cd "${SOURCE_DIR}/third_party/wayland"
335
336  # Need to figure out the right way to pass this down...
337  sed -i "s|install: true\$|install: true, install_rpath : '\$ORIGIN'|" src/meson.build
338
339  meson build \
340    --libdir="${WORKING_DIR}/usr/lib" \
341    --prefix="${WORKING_DIR}/usr"
342  ninja -C build/ install
343
344  cp -a "${WORKING_DIR}"/usr/lib/libwayland-client.so* "${OUTPUT_BIN_DIR}"
345  debuglink libwayland-client.so.0.3.0
346}
347
348compile_gfxstream() {
349  echo "Compiling gfxstream..."
350
351  local dist_dir="${SOURCE_DIR}/hardware/google/gfxstream/build"
352  [ -d "${dist_dir}" ] && rm -rf "${dist_dir}"
353  mkdir "${dist_dir}"
354  cd "${dist_dir}"
355
356  cmake .. \
357    -G Ninja \
358    -DBUILD_GRAPHICS_DETECTOR=ON \
359    -DDEPENDENCY_RESOLUTION=AOSP
360
361  ninja
362
363  chmod +x "${dist_dir}"/gfxstream_graphics_detector
364  cp -a "${dist_dir}"/gfxstream_graphics_detector "${OUTPUT_BIN_DIR}"
365  debuglink gfxstream_graphics_detector
366
367  chmod +x "${dist_dir}"/libgfxstream_backend.so
368  cp -a "${dist_dir}"/libgfxstream_backend.so "${WORKING_DIR}"/usr/lib
369  cp -a "${WORKING_DIR}"/usr/lib/libgfxstream_backend.so "${OUTPUT_BIN_DIR}"
370  debuglink libgfxstream_backend.so
371}
372
373compile_swiftshader() {
374  echo "Compiling SwiftShader..."
375
376  local dist_dir="${SOURCE_DIR}/external/swiftshader/build"
377  [ -d "${dist_dir}" ] && rm -rf "${dist_dir}"
378  mkdir -p "${dist_dir}"
379  cd "${dist_dir}"
380
381  CC=/usr/bin/clang \
382  CXX=/usr/bin/clang++ \
383  cmake .. \
384    -G Ninja \
385    -DSWIFTSHADER_BUILD_TESTS=FALSE \
386    -DSWIFTSHADER_LLVM_VERSION=16.0
387
388  ninja
389
390  cp -a "${dist_dir}"/libvk_swiftshader.so "${OUTPUT_BIN_DIR}"
391  debuglink libvk_swiftshader.so
392
393  cp -a "${dist_dir}"/Linux/vk_swiftshader_icd.json "${OUTPUT_BIN_DIR}"
394}
395
396compile_crosvm() {
397  echo "Compiling Crosvm..."
398
399  source "${HOME}/.cargo/env"
400
401  # Workaround for aosp/1412815
402  cd "${SOURCE_DIR}/platform/crosvm/protos/src"
403  if ! grep '^mod generated {$' lib.rs; then
404    cat >>lib.rs <<EOF
405mod generated {
406    include!(concat!(env!("OUT_DIR"), "/generated.rs"));
407}
408EOF
409  fi
410  sed -i "s/pub use cdisk_spec_proto::cdisk_spec/pub use generated::cdisk_spec/" lib.rs
411
412  cd "${SOURCE_DIR}/platform/crosvm"
413
414  # Workaround for minijail-sys prepending -L/usr/lib/$arch dir
415  # which breaks the preferred search path for libdrm.so
416  sed -i '0,/pkg_config::Config::new().probe("libdrm")?;/{/pkg_config::Config::new().probe("libdrm")?;/d;}' rutabaga_gfx/build.rs
417
418  # Workaround rutabaga build thinking it needs at later version of virglrenderer.
419  sed -i 's/atleast_version("1.0.0")/atleast_version("0.10.0")/g' rutabaga_gfx/build.rs
420
421  local crosvm_features=audio,gdb,gpu,composite-disk,usb,virgl_renderer
422  if [[ $BUILD_GFXSTREAM -eq 1 ]]; then
423      crosvm_features+=,gfxstream
424  fi
425
426  CROSVM_USE_SYSTEM_MINIGBM=1 \
427  CROSVM_USE_SYSTEM_VIRGLRENDERER=1 \
428  GFXSTREAM_PATH="${WORKING_DIR}/usr/lib" \
429  PKG_CONFIG_PATH="${WORKING_DIR}/usr/lib/pkgconfig" \
430  RUSTFLAGS="-C link-arg=-Wl,-rpath,\$ORIGIN -C link-arg=${WORKING_DIR}/usr/lib/libdrm.so" \
431    cargo build --features ${crosvm_features}
432
433  # Save the outputs
434  cp Cargo.lock "${OUTPUT_DIR}"
435  cp target/debug/crosvm "${OUTPUT_BIN_DIR}"
436  debuglink crosvm
437
438  cargo --version --verbose > "${OUTPUT_DIR}/cargo_version.txt"
439  rustup show > "${OUTPUT_DIR}/rustup_show.txt"
440}
441
442compile_crosvm_seccomp() {
443  echo "Processing Crosvm Seccomp..."
444
445  cd "${SOURCE_DIR}/platform/crosvm"
446  case ${ARCH} in
447    x86_64) subdir="${ARCH}" ;;
448    amd64) subdir="x86_64" ;;
449    arm64) subdir="aarch64" ;;
450    aarch64) subdir="${ARCH}" ;;
451    *)
452      echo "${ARCH} is not supported"
453      exit 15
454  esac
455
456  inlined_policy_list="\
457    jail/seccomp/$subdir/common_device.policy \
458    jail/seccomp/$subdir/gpu_common.policy \
459    jail/seccomp/$subdir/serial.policy \
460    jail/seccomp/$subdir/net.policy \
461    jail/seccomp/$subdir/block.policy \
462    jail/seccomp/$subdir/vvu.policy \
463    jail/seccomp/$subdir/vhost_user.policy \
464    jail/seccomp/$subdir/vhost_vsock.policy"
465  for policy_file in "jail/seccomp/$subdir/"*.policy; do
466    [[ "$inlined_policy_list" = *"$policy_file"* ]] && continue
467    jail/seccomp/policy-inliner.sh $inlined_policy_list <"$policy_file" | \
468      grep -ve "^@frequency" \
469      >"${OUTPUT_SECCOMP_DIR}"/$(basename "$policy_file")
470  done
471}
472
473compile() {
474  echo "Compiling..."
475  mkdir -p \
476    "${WORKING_DIR}" \
477    "${OUTPUT_DIR}" \
478    "${OUTPUT_BIN_DIR}" \
479    "${OUTPUT_ETC_DIR}" \
480    "${OUTPUT_SECCOMP_DIR}"
481
482  if [[ $BUILD_CROSVM -eq 1 ]]; then
483    compile_libdrm
484    compile_minijail
485    compile_minigbm
486    compile_epoxy
487    compile_virglrenderer
488    compile_libffi # wayland depends on it
489    compile_wayland
490  fi
491
492  if [[ $BUILD_GFXSTREAM -eq 1 ]]; then
493    compile_gfxstream
494  fi
495
496  compile_crosvm
497  compile_crosvm_seccomp
498
499  if [[ $BUILD_SWIFTSHADER -eq 1 ]]; then
500    compile_swiftshader
501  fi
502
503  dpkg-query -W > "${OUTPUT_DIR}/builder-packages.txt"
504  echo "Results in ${OUTPUT_DIR}"
505}
506
507aarch64_retry() {
508  BUILD_CROSVM=1 BUILD_GFXSTREAM=1 BUILD_SWIFTSHADER=1 compile
509}
510
511aarch64_build() {
512  rm -rf "${WORKING_DIR}/*"
513  aarch64_retry
514}
515
516x86_64_retry() {
517  MINIGBM_DRV="I915 RADEON VC4" BUILD_CROSVM=1 BUILD_GFXSTREAM=1 BUILD_SWIFTSHADER=1 compile
518}
519
520x86_64_build() {
521  rm -rf "${WORKING_DIR}/*"
522  x86_64_retry
523}
524
525if [[ $# -lt 1 ]]; then
526  echo Choosing default config
527  set setup_env prepare_source x86_64_build
528fi
529
530echo Steps: "$@"
531
532for i in "$@"; do
533  echo $i
534  case "$i" in
535    ARCH=*) ARCH="${i/ARCH=/}" ;;
536    CUSTOM_MANIFEST=*) CUSTOM_MANIFEST="${i/CUSTOM_MANIFEST=/}" ;;
537    aarch64_build) $i ;;
538    aarch64_retry) $i ;;
539    setup_env) $i ;;
540    install_packages) $i ;;
541    fetch_source) $i ;;
542    resync_source) $i ;;
543    prepare_source) $i ;;
544    x86_64_build) $i ;;
545    x86_64_retry) $i ;;
546    *) echo $i unknown 1>&2
547      echo usage: $0 'install_packages|prepare_source|resync_source|fetch_source|$(uname -m)_build|$(uname -m)_retry' 1>&2
548       exit 2
549       ;;
550  esac
551done
552