xref: /aosp_15_r20/external/cronet/third_party/libc++/src/utils/ci/Dockerfile (revision 6777b5387eb2ff775bb5750e3f5d96f37fb7352b) 
1#===----------------------------------------------------------------------===##
2#
3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4# See https://llvm.org/LICENSE.txt for license information.
5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6#
7#===----------------------------------------------------------------------===##
8#
9# This file defines the buildkite and github actions builder images.
10# You can build both images using:
11#
12#   docker compose build
13#
14# Or you can select a single image to build
15#
16#  docker compose build buildkite-builder
17#
18# The final images can be found at
19#
20#  ghcr.io/libcxx/buildkite-builder
21#  ghcr.io/libcxx/actions-builder
22#  ghcr.io/libcxx/android-buildkite-builder
23#
24# Members of the github.com/libcxx/ organizations can push new images to the CI.
25# This is done by GitHub actions in the https://github.com/libcxx/builders repo.
26#
27# ===----------------------------------------------------------------------===##
28#                     Running the buildkite image
29# ===----------------------------------------------------------------------===##
30#
31# To start a Buildkite Agent, run it as:
32#   $ docker run --env-file <secrets> -it $(docker build -q libcxx/utils/ci)
33#
34# The environment variables in `<secrets>` should be the ones necessary
35# to run a BuildKite agent:
36#
37#   BUILDKITE_AGENT_TOKEN=<token>
38#
39# If you're only looking to run the Docker image locally for debugging a
40# build bot, see the `run-buildbot-container` script located in this directory.
41
42
43# HACK: We set the base image in the docker-compose file depending on the final target (buildkite vs github actions).
44# This means we have a much slower container build, but we can use the same Dockerfile for both targets.
45ARG BASE_IMAGE
46FROM $BASE_IMAGE AS builder-base
47
48# Make sure apt-get doesn't try to prompt for stuff like our time zone, etc.
49ENV DEBIAN_FRONTEND=noninteractive
50
51# populated in the docker-compose file
52ARG GCC_LATEST_VERSION
53ENV GCC_LATEST_VERSION=${GCC_LATEST_VERSION}
54
55# populated in the docker-compose file
56ARG LLVM_HEAD_VERSION
57ENV LLVM_HEAD_VERSION=${LLVM_HEAD_VERSION}
58
59# HACK: The github actions runner image already has sudo and requires its use. The buildkite base image does not.
60# Reconcile this.
61RUN <<EOF
62  apt-get update || true
63  apt-get install -y sudo || true
64  echo "ALL ALL = (ALL) NOPASSWD: ALL" | tee /etc/sudoers || true
65EOF
66
67# Installing tzdata before other packages avoids the time zone prompts.
68# These prompts seem to ignore DEBIAN_FRONTEND=noninteractive.
69RUN sudo apt-get update \
70    && sudo apt-get install -y \
71        tzdata
72
73RUN sudo apt-get update \
74    && sudo apt-get install -y \
75        python3 \
76        python3-distutils \
77        python3-psutil \
78        git \
79        gdb \
80        ccache \
81        gpg \
82        wget \
83        bash \
84        curl \
85        python3 \
86        python3-dev \
87        libpython3-dev \
88        uuid-dev \
89        libncurses5-dev \
90        swig3.0 \
91        libxml2-dev \
92        libedit-dev \
93        language-pack-en \
94        language-pack-fr \
95        language-pack-ja \
96        language-pack-ru \
97        language-pack-zh-hans \
98        lsb-release \
99        wget \
100        unzip \
101        software-properties-common \
102    && sudo rm -rf /var/lib/apt/lists/*
103
104
105# Install various tools used by the build or the test suite
106#RUN apt-get update && apt-get install -y ninja-build python3 python3-distutils python3-psutil git gdb ccache
107# TODO add ninja-build once 1.11 is available in Ubuntu, also remove the manual installation.
108RUN <<EOF
109  wget -qO /tmp/ninja.gz https://github.com/ninja-build/ninja/releases/latest/download/ninja-linux.zip
110  gunzip /tmp/ninja.gz
111  chmod a+x /tmp/ninja
112  sudo mv /tmp/ninja /usr/local/bin/ninja
113EOF
114
115
116# These two locales are not enabled by default so generate them
117RUN <<EOF
118  printf "fr_CA ISO-8859-1\ncs_CZ ISO-8859-2" | sudo tee -a /etc/locale.gen
119  sudo mkdir /usr/local/share/i1en/
120  printf "fr_CA ISO-8859-1\ncs_CZ ISO-8859-2" | sudo tee -a /usr/local/share/i1en/SUPPORTED
121  sudo locale-gen
122EOF
123
124# Install Clang <latest>, <latest-1> and ToT, which are the ones we support.
125# We also install <latest-2> because we need to support the "latest-1" of the
126# current LLVM release branch, which is effectively the <latest-2> of the
127# tip-of-trunk LLVM. For example, after branching LLVM 14 but before branching
128# LLVM 15, we still need to have Clang 12 in this Docker image because the LLVM
129# 14 release branch CI uses it. The tip-of-trunk CI will never use Clang 12,
130# though.
131RUN <<EOF
132  sudo apt-get update
133  wget https://apt.llvm.org/llvm.sh -O /tmp/llvm.sh
134  chmod +x /tmp/llvm.sh
135  sudo /tmp/llvm.sh $(($LLVM_HEAD_VERSION - 3)) all  # for CI transitions
136  sudo /tmp/llvm.sh $(($LLVM_HEAD_VERSION - 2)) all  # previous release
137  sudo /tmp/llvm.sh $(($LLVM_HEAD_VERSION - 1)) all  # latest release
138  sudo /tmp/llvm.sh $LLVM_HEAD_VERSION          all  # current ToT
139  sudo apt-get install -y libomp5-$LLVM_HEAD_VERSION
140  sudo rm -rf /var/lib/apt/lists/*
141EOF
142
143# Install the most recent GCC, like clang install the previous version as a transition.
144RUN <<EOF
145  sudo add-apt-repository ppa:ubuntu-toolchain-r/test
146  sudo apt-get update
147  sudo apt-get install -y \
148    gcc-$((GCC_LATEST_VERSION - 1)) \
149    g++-$((GCC_LATEST_VERSION - 1)) \
150    gcc-$GCC_LATEST_VERSION \
151    g++-$GCC_LATEST_VERSION
152  sudo rm -rf /var/lib/apt/lists/*
153EOF
154
155RUN <<EOF
156    # Install a recent CMake
157    wget https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-linux-x86_64.sh -O /tmp/install-cmake.sh
158    sudo bash /tmp/install-cmake.sh --prefix=/usr --exclude-subdir --skip-license
159    rm /tmp/install-cmake.sh
160EOF
161
162# ===----------------------------------------------------------------------===##
163#                       Android Buildkite Image
164# ===----------------------------------------------------------------------===##
165
166FROM ubuntu:jammy AS android-builder-base
167
168ARG ANDROID_CLANG_VERSION
169ARG ANDROID_CLANG_PREBUILTS_COMMIT
170ARG ANDROID_SYSROOT_BID
171
172RUN  apt-get update && apt-get install -y curl unzip git
173
174# Install the Android platform tools (e.g. adb) into /opt/android/sdk.
175RUN <<EOF
176  mkdir -p /opt/android/sdk
177  cd /opt/android/sdk
178  curl -LO https://dl.google.com/android/repository/platform-tools-latest-linux.zip
179  unzip platform-tools-latest-linux.zip
180  rm platform-tools-latest-linux.zip
181EOF
182
183# Install the current Android compiler. Specify the prebuilts commit to retrieve
184# this compiler version even after it's removed from HEAD.
185
186ENV ANDROID_CLANG_VERSION=$ANDROID_CLANG_VERSION
187ENV ANDROID_CLANG_PREBUILTS_COMMIT=$ANDROID_CLANG_PREBUILTS_COMMIT
188RUN <<EOF
189    git clone --filter=blob:none --sparse \
190        https://android.googlesource.com/platform/prebuilts/clang/host/linux-x86 \
191        /opt/android/clang
192    git -C /opt/android/clang checkout ${ANDROID_CLANG_PREBUILTS_COMMIT}
193    git -C /opt/android/clang sparse-checkout add clang-${ANDROID_CLANG_VERSION}
194    rm -fr /opt/android/clang/.git
195    ln -sf /opt/android/clang/clang-${ANDROID_CLANG_VERSION} /opt/android/clang/clang-current
196    # The "git sparse-checkout" and "ln" commands succeed even if nothing was
197    # checked out, so use this "ls" command to fix that.
198    ls /opt/android/clang/clang-current/bin/clang
199EOF
200
201# Install an Android sysroot. New AOSP sysroots are available at
202# https://ci.android.com/builds/branches/aosp-main/grid, the "ndk" target. The
203# NDK also makes its sysroot prebuilt available at
204# https://android.googlesource.com/platform/prebuilts/ndk/+/refs/heads/dev/platform/sysroot.
205
206ENV ANDROID_SYSROOT_BID=$ANDROID_SYSROOT_BID
207RUN <<EOF
208  cd /opt/android
209  curl -L -o ndk_platform.tar.bz2 \
210      https://androidbuildinternal.googleapis.com/android/internal/build/v3/builds/${ANDROID_SYSROOT_BID}/ndk/attempts/latest/artifacts/ndk_platform.tar.bz2/url
211  tar xf ndk_platform.tar.bz2
212  rm ndk_platform.tar.bz2
213EOF
214
215# Install Docker
216RUN <<EOF
217  curl -fsSL https://get.docker.com -o /tmp/get-docker.sh
218  sh /tmp/get-docker.sh
219  rm /tmp/get-docker.sh
220
221  # Install Docker. Mark the binary setuid so it can be run without prefixing it
222  # with sudo. Adding the container user to the docker group doesn't work because
223  # /var/run/docker.sock is owned by the host's docker GID, not the container's
224  # docker GID.
225  chmod u+s /usr/bin/docker
226EOF
227
228# ===----------------------------------------------------------------------===##
229#                    Buildkite Builder Image
230# ===----------------------------------------------------------------------===##
231#
232# IMAGE: ghcr.io/libcxx/buildkite-builder.
233#
234FROM builder-base AS buildkite-builder
235
236# Create the libcxx-builder user, regardless of if we use it or not
237RUN sudo useradd --create-home libcxx-builder
238
239USER libcxx-builder
240WORKDIR /home/libcxx-builder
241
242# Install the Buildkite agent and dependencies. This must be done as non-root
243# for the Buildkite agent to be installed in a path where we can find it.
244RUN <<EOF
245  cd /home/libcxx-builder
246  curl -sL https://raw.githubusercontent.com/buildkite/agent/main/install.sh -o /tmp/install-agent.sh
247  bash /tmp/install-agent.sh
248  rm /tmp/install-agent.sh
249  echo "tags=\"queue=libcxx-builders,arch=$(uname -m),os=linux\"" \
250    >> /home/libcxx-builder/.buildkite-agent/buildkite-agent.cfg
251EOF
252
253USER libcxx-builder
254WORKDIR /home/libcxx-builder
255
256ENV PATH="${PATH}:/home/libcxx-builder/.buildkite-agent/bin"
257
258CMD ["buildkite-agent", "start"]
259
260# ===----------------------------------------------------------------------===##
261#                    Android Buildkite Builder Image
262# ===----------------------------------------------------------------------===##
263#
264# IMAGE: ghcr.io/libcxx/android-buildkite-builder.
265#
266FROM buildkite-builder AS android-buildkite-builder
267
268COPY --from=android-builder-base /opt/android /opt/android
269COPY ./vendor/android/container-setup.sh /opt/android/container-setup.sh
270
271ENV PATH="/opt/android/sdk/platform-tools:${PATH}"
272
273USER libcxx-builder
274WORKDIR /home/libcxx-builder
275
276# Reset the configuration, we pass the configuration via the environment.
277RUN cp /home/libcxx-builder/.buildkite-agent/buildkite-agent.dist.cfg \
278       /home/libcxx-builder/.buildkite-agent/buildkite-agent.cfg
279
280# Modify the Buildkite agent cmdline to do Android setup stuff first.
281CMD /opt/android/container-setup.sh && buildkite-agent start
282
283# ===----------------------------------------------------------------------===##
284#                    Github Actions Builder Image
285# ===----------------------------------------------------------------------===##
286#
287# IMAGE: ghcr.io/libcxx/actions-builder.
288#
289FROM builder-base AS actions-builder
290
291# Install 'act' for running github actions locally. This provides an alternative to the run-buildbot script
292# while still providing reproducability.
293RUN curl -s https://raw.githubusercontent.com/nektos/act/master/install.sh | sudo bash
294
295WORKDIR /home/runner
296USER runner
297
298
299
300