1#!/usr/bin/env python3 2# Copyright 2016 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Definition of targets to build artifacts.""" 16 17import os.path 18import random 19import string 20import sys 21 22sys.path.insert(0, os.path.abspath("..")) 23import python_utils.jobset as jobset 24 25_LATEST_MANYLINUX = "manylinux2014" 26 27 28def create_docker_jobspec( 29 name, 30 dockerfile_dir, 31 shell_command, 32 environ={}, 33 flake_retries=0, 34 timeout_retries=0, 35 timeout_seconds=30 * 60, 36 extra_docker_args=None, 37 verbose_success=False, 38): 39 """Creates jobspec for a task running under docker.""" 40 environ = environ.copy() 41 environ["ARTIFACTS_OUT"] = "artifacts/%s" % name 42 43 docker_args = [] 44 for k, v in list(environ.items()): 45 docker_args += ["-e", "%s=%s" % (k, v)] 46 docker_env = { 47 "DOCKERFILE_DIR": dockerfile_dir, 48 "DOCKER_RUN_SCRIPT": "tools/run_tests/dockerize/docker_run.sh", 49 "DOCKER_RUN_SCRIPT_COMMAND": shell_command, 50 "OUTPUT_DIR": "artifacts", 51 } 52 if extra_docker_args is not None: 53 docker_env["EXTRA_DOCKER_ARGS"] = extra_docker_args 54 jobspec = jobset.JobSpec( 55 cmdline=["tools/run_tests/dockerize/build_and_run_docker.sh"] 56 + docker_args, 57 environ=docker_env, 58 shortname="build_artifact.%s" % (name), 59 timeout_seconds=timeout_seconds, 60 flake_retries=flake_retries, 61 timeout_retries=timeout_retries, 62 verbose_success=verbose_success, 63 ) 64 return jobspec 65 66 67def create_jobspec( 68 name, 69 cmdline, 70 environ={}, 71 shell=False, 72 flake_retries=0, 73 timeout_retries=0, 74 timeout_seconds=30 * 60, 75 use_workspace=False, 76 cpu_cost=1.0, 77 verbose_success=False, 78): 79 """Creates jobspec.""" 80 environ = environ.copy() 81 if use_workspace: 82 environ["WORKSPACE_NAME"] = "workspace_%s" % name 83 environ["ARTIFACTS_OUT"] = os.path.join("..", "artifacts", name) 84 cmdline = [ 85 "bash", 86 "tools/run_tests/artifacts/run_in_workspace.sh", 87 ] + cmdline 88 else: 89 environ["ARTIFACTS_OUT"] = os.path.join("artifacts", name) 90 91 jobspec = jobset.JobSpec( 92 cmdline=cmdline, 93 environ=environ, 94 shortname="build_artifact.%s" % (name), 95 timeout_seconds=timeout_seconds, 96 flake_retries=flake_retries, 97 timeout_retries=timeout_retries, 98 shell=shell, 99 cpu_cost=cpu_cost, 100 verbose_success=verbose_success, 101 ) 102 return jobspec 103 104 105_MACOS_COMPAT_FLAG = "-mmacosx-version-min=10.10" 106 107_ARCH_FLAG_MAP = {"x86": "-m32", "x64": "-m64"} 108 109 110class PythonArtifact: 111 """Builds Python artifacts.""" 112 113 def __init__(self, platform, arch, py_version, presubmit=False): 114 self.name = "python_%s_%s_%s" % (platform, arch, py_version) 115 self.platform = platform 116 self.arch = arch 117 self.labels = ["artifact", "python", platform, arch, py_version] 118 if presubmit: 119 self.labels.append("presubmit") 120 self.py_version = py_version 121 if platform == _LATEST_MANYLINUX: 122 self.labels.append("latest-manylinux") 123 if "manylinux" in platform: 124 self.labels.append("linux") 125 if "linux_extra" in platform: 126 # linux_extra wheels used to be built by a separate kokoro job. 127 # Their build is now much faster, so they can be included 128 # in the regular artifact build. 129 self.labels.append("linux") 130 if "musllinux" in platform: 131 self.labels.append("linux") 132 133 def pre_build_jobspecs(self): 134 return [] 135 136 def build_jobspec(self, inner_jobs=None): 137 environ = {} 138 if inner_jobs is not None: 139 # set number of parallel jobs when building native extension 140 # building the native extension is the most time-consuming part of the build 141 environ["GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS"] = str(inner_jobs) 142 143 if self.platform == "macos": 144 environ["ARCHFLAGS"] = "-arch arm64 -arch x86_64" 145 environ["GRPC_BUILD_MAC"] = "true" 146 147 if self.platform == "linux_extra": 148 # Crosscompilation build for armv7 (e.g. Raspberry Pi) 149 environ["PYTHON"] = "/opt/python/{}/bin/python3".format( 150 self.py_version 151 ) 152 environ["PIP"] = "/opt/python/{}/bin/pip3".format(self.py_version) 153 environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" 154 environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" 155 environ["LDFLAGS"] = "-s" 156 return create_docker_jobspec( 157 self.name, 158 "tools/dockerfile/grpc_artifact_python_linux_{}".format( 159 self.arch 160 ), 161 "tools/run_tests/artifacts/build_artifact_python.sh", 162 environ=environ, 163 timeout_seconds=60 * 60, 164 ) 165 elif "manylinux" in self.platform: 166 if self.arch == "x86": 167 environ["SETARCH_CMD"] = "linux32" 168 # Inside the manylinux container, the python installations are located in 169 # special places... 170 environ["PYTHON"] = "/opt/python/{}/bin/python".format( 171 self.py_version 172 ) 173 environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version) 174 environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" 175 if self.arch == "aarch64": 176 environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" 177 # As we won't strip the binary with auditwheel (see below), strip 178 # it at link time. 179 environ["LDFLAGS"] = "-s" 180 else: 181 # only run auditwheel if we're not crosscompiling 182 environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE" 183 # only build the packages that depend on grpcio-tools 184 # if we're not crosscompiling. 185 # - they require protoc to run on current architecture 186 # - they only have sdist packages anyway, so it's useless to build them again 187 environ["GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS"] = "TRUE" 188 return create_docker_jobspec( 189 self.name, 190 "tools/dockerfile/grpc_artifact_python_%s_%s" 191 % (self.platform, self.arch), 192 "tools/run_tests/artifacts/build_artifact_python.sh", 193 environ=environ, 194 timeout_seconds=60 * 60 * 2, 195 ) 196 elif "musllinux" in self.platform: 197 environ["PYTHON"] = "/opt/python/{}/bin/python".format( 198 self.py_version 199 ) 200 environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version) 201 environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" 202 environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE" 203 environ["GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX"] = "TRUE" 204 return create_docker_jobspec( 205 self.name, 206 "tools/dockerfile/grpc_artifact_python_%s_%s" 207 % (self.platform, self.arch), 208 "tools/run_tests/artifacts/build_artifact_python.sh", 209 environ=environ, 210 timeout_seconds=60 * 60 * 2, 211 ) 212 elif self.platform == "windows": 213 environ["EXT_COMPILER"] = "msvc" 214 # For some reason, the batch script %random% always runs with the same 215 # seed. We create a random temp-dir here 216 dir = "".join( 217 random.choice(string.ascii_uppercase) for _ in range(10) 218 ) 219 return create_jobspec( 220 self.name, 221 [ 222 "tools\\run_tests\\artifacts\\build_artifact_python.bat", 223 self.py_version, 224 "32" if self.arch == "x86" else "64", 225 ], 226 environ=environ, 227 timeout_seconds=60 * 60 * 2, 228 use_workspace=True, 229 ) 230 else: 231 environ["PYTHON"] = self.py_version 232 environ["SKIP_PIP_INSTALL"] = "TRUE" 233 return create_jobspec( 234 self.name, 235 ["tools/run_tests/artifacts/build_artifact_python.sh"], 236 environ=environ, 237 timeout_seconds=60 * 60 * 2, 238 use_workspace=True, 239 ) 240 241 def __str__(self): 242 return self.name 243 244 245class RubyArtifact: 246 """Builds ruby native gem.""" 247 248 def __init__(self, platform, gem_platform, presubmit=False): 249 self.name = "ruby_native_gem_%s_%s" % (platform, gem_platform) 250 self.platform = platform 251 self.gem_platform = gem_platform 252 self.labels = ["artifact", "ruby", platform, gem_platform] 253 if presubmit: 254 self.labels.append("presubmit") 255 256 def pre_build_jobspecs(self): 257 return [] 258 259 def build_jobspec(self, inner_jobs=None): 260 environ = {} 261 if inner_jobs is not None: 262 # set number of parallel jobs when building native extension 263 environ["GRPC_RUBY_BUILD_PROCS"] = str(inner_jobs) 264 # Ruby build uses docker internally and docker cannot be nested. 265 # We are using a custom workspace instead. 266 return create_jobspec( 267 self.name, 268 [ 269 "tools/run_tests/artifacts/build_artifact_ruby.sh", 270 self.gem_platform, 271 ], 272 use_workspace=True, 273 timeout_seconds=90 * 60, 274 environ=environ, 275 ) 276 277 278class PHPArtifact: 279 """Builds PHP PECL package""" 280 281 def __init__(self, platform, arch, presubmit=False): 282 self.name = "php_pecl_package_{0}_{1}".format(platform, arch) 283 self.platform = platform 284 self.arch = arch 285 self.labels = ["artifact", "php", platform, arch] 286 if presubmit: 287 self.labels.append("presubmit") 288 289 def pre_build_jobspecs(self): 290 return [] 291 292 def build_jobspec(self, inner_jobs=None): 293 del inner_jobs # arg unused as PHP artifact build is basically just packing an archive 294 if self.platform == "linux": 295 return create_docker_jobspec( 296 self.name, 297 "tools/dockerfile/test/php73_zts_debian11_{}".format(self.arch), 298 "tools/run_tests/artifacts/build_artifact_php.sh", 299 ) 300 else: 301 return create_jobspec( 302 self.name, 303 ["tools/run_tests/artifacts/build_artifact_php.sh"], 304 use_workspace=True, 305 ) 306 307 308class ProtocArtifact: 309 """Builds protoc and protoc-plugin artifacts""" 310 311 def __init__(self, platform, arch, presubmit=False): 312 self.name = "protoc_%s_%s" % (platform, arch) 313 self.platform = platform 314 self.arch = arch 315 self.labels = ["artifact", "protoc", platform, arch] 316 if presubmit: 317 self.labels.append("presubmit") 318 319 def pre_build_jobspecs(self): 320 return [] 321 322 def build_jobspec(self, inner_jobs=None): 323 environ = {} 324 if inner_jobs is not None: 325 # set number of parallel jobs when building protoc 326 environ["GRPC_PROTOC_BUILD_COMPILER_JOBS"] = str(inner_jobs) 327 328 if self.platform != "windows": 329 environ["CXXFLAGS"] = "" 330 environ["LDFLAGS"] = "" 331 if self.platform == "linux": 332 dockerfile_dir = ( 333 "tools/dockerfile/grpc_artifact_centos6_{}".format( 334 self.arch 335 ) 336 ) 337 if self.arch == "aarch64": 338 # for aarch64, use a dockcross manylinux image that will 339 # give us both ready to use crosscompiler and sufficient backward compatibility 340 dockerfile_dir = ( 341 "tools/dockerfile/grpc_artifact_protoc_aarch64" 342 ) 343 environ["LDFLAGS"] += " -static-libgcc -static-libstdc++ -s" 344 return create_docker_jobspec( 345 self.name, 346 dockerfile_dir, 347 "tools/run_tests/artifacts/build_artifact_protoc.sh", 348 environ=environ, 349 ) 350 else: 351 environ["CXXFLAGS"] += ( 352 " -std=c++14 -stdlib=libc++ %s" % _MACOS_COMPAT_FLAG 353 ) 354 return create_jobspec( 355 self.name, 356 ["tools/run_tests/artifacts/build_artifact_protoc.sh"], 357 environ=environ, 358 timeout_seconds=60 * 60, 359 use_workspace=True, 360 ) 361 else: 362 vs_tools_architecture = ( 363 self.arch 364 ) # architecture selector passed to vcvarsall.bat 365 environ["ARCHITECTURE"] = vs_tools_architecture 366 return create_jobspec( 367 self.name, 368 ["tools\\run_tests\\artifacts\\build_artifact_protoc.bat"], 369 environ=environ, 370 use_workspace=True, 371 ) 372 373 def __str__(self): 374 return self.name 375 376 377def _reorder_targets_for_build_speed(targets): 378 """Reorder targets to achieve optimal build speed""" 379 # ruby artifact build builds multiple artifacts at once, so make sure 380 # we start building ruby artifacts first, so that they don't end up 381 # being a long tail once everything else finishes. 382 return list( 383 sorted( 384 targets, 385 key=lambda target: 0 if target.name.startswith("ruby_") else 1, 386 ) 387 ) 388 389 390def targets(): 391 """Gets list of supported targets""" 392 return _reorder_targets_for_build_speed( 393 [ 394 ProtocArtifact("linux", "x64", presubmit=True), 395 ProtocArtifact("linux", "x86", presubmit=True), 396 ProtocArtifact("linux", "aarch64", presubmit=True), 397 ProtocArtifact("macos", "x64", presubmit=True), 398 ProtocArtifact("windows", "x64", presubmit=True), 399 ProtocArtifact("windows", "x86", presubmit=True), 400 PythonArtifact("manylinux2014", "x64", "cp38-cp38", presubmit=True), 401 PythonArtifact("manylinux2014", "x64", "cp39-cp39", presubmit=True), 402 PythonArtifact("manylinux2014", "x64", "cp310-cp310"), 403 PythonArtifact("manylinux2014", "x64", "cp311-cp311"), 404 PythonArtifact( 405 "manylinux2014", "x64", "cp312-cp312", presubmit=True 406 ), 407 PythonArtifact("manylinux2014", "x86", "cp38-cp38", presubmit=True), 408 PythonArtifact("manylinux2014", "x86", "cp39-cp39", presubmit=True), 409 PythonArtifact("manylinux2014", "x86", "cp310-cp310"), 410 PythonArtifact("manylinux2014", "x86", "cp311-cp311"), 411 PythonArtifact( 412 "manylinux2014", "x86", "cp312-cp312", presubmit=True 413 ), 414 PythonArtifact( 415 "manylinux2014", "aarch64", "cp38-cp38", presubmit=True 416 ), 417 PythonArtifact("manylinux2014", "aarch64", "cp39-cp39"), 418 PythonArtifact("manylinux2014", "aarch64", "cp310-cp310"), 419 PythonArtifact("manylinux2014", "aarch64", "cp311-cp311"), 420 PythonArtifact( 421 "manylinux2014", "aarch64", "cp312-cp312", presubmit=True 422 ), 423 PythonArtifact("linux_extra", "armv7", "cp38-cp38", presubmit=True), 424 PythonArtifact("linux_extra", "armv7", "cp39-cp39"), 425 PythonArtifact("linux_extra", "armv7", "cp310-cp310"), 426 PythonArtifact("linux_extra", "armv7", "cp311-cp311"), 427 PythonArtifact( 428 "linux_extra", "armv7", "cp312-cp312", presubmit=True 429 ), 430 PythonArtifact("musllinux_1_1", "x64", "cp38-cp38", presubmit=True), 431 PythonArtifact("musllinux_1_1", "x64", "cp39-cp39"), 432 PythonArtifact("musllinux_1_1", "x64", "cp310-cp310"), 433 PythonArtifact("musllinux_1_1", "x64", "cp311-cp311"), 434 PythonArtifact( 435 "musllinux_1_1", "x64", "cp312-cp312", presubmit=True 436 ), 437 PythonArtifact("musllinux_1_1", "x86", "cp38-cp38", presubmit=True), 438 PythonArtifact("musllinux_1_1", "x86", "cp39-cp39"), 439 PythonArtifact("musllinux_1_1", "x86", "cp310-cp310"), 440 PythonArtifact("musllinux_1_1", "x86", "cp311-cp311"), 441 PythonArtifact( 442 "musllinux_1_1", "x86", "cp312-cp312", presubmit=True 443 ), 444 PythonArtifact("macos", "x64", "python3.8", presubmit=True), 445 PythonArtifact("macos", "x64", "python3.9"), 446 PythonArtifact("macos", "x64", "python3.10"), 447 PythonArtifact("macos", "x64", "python3.11"), 448 PythonArtifact("macos", "x64", "python3.12", presubmit=True), 449 PythonArtifact("windows", "x86", "Python38_32bit", presubmit=True), 450 PythonArtifact("windows", "x86", "Python39_32bit"), 451 PythonArtifact("windows", "x86", "Python310_32bit"), 452 PythonArtifact("windows", "x86", "Python311_32bit"), 453 PythonArtifact("windows", "x86", "Python312_32bit", presubmit=True), 454 PythonArtifact("windows", "x64", "Python38", presubmit=True), 455 PythonArtifact("windows", "x64", "Python39"), 456 PythonArtifact("windows", "x64", "Python310"), 457 PythonArtifact("windows", "x64", "Python311"), 458 PythonArtifact("windows", "x64", "Python312", presubmit=True), 459 RubyArtifact("linux", "x86-mingw32", presubmit=True), 460 RubyArtifact("linux", "x64-mingw32", presubmit=True), 461 RubyArtifact("linux", "x64-mingw-ucrt", presubmit=True), 462 RubyArtifact("linux", "x86_64-linux", presubmit=True), 463 RubyArtifact("linux", "x86-linux", presubmit=True), 464 RubyArtifact("linux", "aarch64-linux", presubmit=True), 465 RubyArtifact("linux", "x86_64-darwin", presubmit=True), 466 RubyArtifact("linux", "arm64-darwin", presubmit=True), 467 PHPArtifact("linux", "x64", presubmit=True), 468 PHPArtifact("macos", "x64", presubmit=True), 469 ] 470 ) 471