1#!/usr/bin/env python3 2# Copyright 2015 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run interop (cross-language) tests in parallel.""" 16 17from __future__ import print_function 18 19import argparse 20import atexit 21import itertools 22import json 23import multiprocessing 24import os 25import re 26import subprocess 27import sys 28import tempfile 29import time 30import traceback 31import uuid 32 33import six 34 35import python_utils.dockerjob as dockerjob 36import python_utils.jobset as jobset 37import python_utils.report_utils as report_utils 38 39# It's ok to not import because this is only necessary to upload results to BQ. 40try: 41 from python_utils.upload_test_results import upload_interop_results_to_bq 42except ImportError as e: 43 print(e) 44 45# Docker doesn't clean up after itself, so we do it on exit. 46atexit.register(lambda: subprocess.call(["stty", "echo"])) 47 48ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../..")) 49os.chdir(ROOT) 50 51_DEFAULT_SERVER_PORT = 8080 52 53_SKIP_CLIENT_COMPRESSION = [ 54 "client_compressed_unary", 55 "client_compressed_streaming", 56] 57 58_SKIP_SERVER_COMPRESSION = [ 59 "server_compressed_unary", 60 "server_compressed_streaming", 61] 62 63_SKIP_COMPRESSION = _SKIP_CLIENT_COMPRESSION + _SKIP_SERVER_COMPRESSION 64 65_SKIP_ADVANCED = [ 66 "status_code_and_message", 67 "custom_metadata", 68 "unimplemented_method", 69 "unimplemented_service", 70] 71 72_SKIP_SPECIAL_STATUS_MESSAGE = ["special_status_message"] 73 74_ORCA_TEST_CASES = ["orca_per_rpc", "orca_oob"] 75 76_GOOGLE_DEFAULT_CREDS_TEST_CASE = "google_default_credentials" 77 78_SKIP_GOOGLE_DEFAULT_CREDS = [ 79 _GOOGLE_DEFAULT_CREDS_TEST_CASE, 80] 81 82_COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE = "compute_engine_channel_credentials" 83 84_SKIP_COMPUTE_ENGINE_CHANNEL_CREDS = [ 85 _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE, 86] 87 88_TEST_TIMEOUT = 3 * 60 89 90# disable this test on core-based languages, 91# see https://github.com/grpc/grpc/issues/9779 92_SKIP_DATA_FRAME_PADDING = ["data_frame_padding"] 93 94# report suffix "sponge_log.xml" is important for reports to get picked up by internal CI 95_DOCKER_BUILD_XML_REPORT = "interop_docker_build/sponge_log.xml" 96_TESTS_XML_REPORT = "interop_test/sponge_log.xml" 97 98 99class CXXLanguage: 100 def __init__(self): 101 self.client_cwd = None 102 self.server_cwd = None 103 self.http2_cwd = None 104 self.safename = "cxx" 105 106 def client_cmd(self, args): 107 return ["cmake/build/interop_client"] + args 108 109 def client_cmd_http2interop(self, args): 110 return ["cmake/build/http2_client"] + args 111 112 def cloud_to_prod_env(self): 113 return {} 114 115 def server_cmd(self, args): 116 return ["cmake/build/interop_server"] + args 117 118 def global_env(self): 119 return {} 120 121 def unimplemented_test_cases(self): 122 return ( 123 _SKIP_DATA_FRAME_PADDING 124 + _SKIP_SPECIAL_STATUS_MESSAGE 125 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 126 ) 127 128 def unimplemented_test_cases_server(self): 129 return [] 130 131 def __str__(self): 132 return "c++" 133 134 135class AspNetCoreLanguage: 136 def __init__(self): 137 self.client_cwd = "../grpc-dotnet/output/InteropTestsClient" 138 self.server_cwd = "../grpc-dotnet/output/InteropTestsWebsite" 139 self.safename = str(self) 140 141 def cloud_to_prod_env(self): 142 return {} 143 144 def client_cmd(self, args): 145 return ["dotnet", "exec", "InteropTestsClient.dll"] + args 146 147 def server_cmd(self, args): 148 return ["dotnet", "exec", "InteropTestsWebsite.dll"] + args 149 150 def global_env(self): 151 return {} 152 153 def unimplemented_test_cases(self): 154 return ( 155 _SKIP_GOOGLE_DEFAULT_CREDS 156 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 157 + _ORCA_TEST_CASES 158 ) 159 160 def unimplemented_test_cases_server(self): 161 return _ORCA_TEST_CASES 162 163 def __str__(self): 164 return "aspnetcore" 165 166 167class DartLanguage: 168 def __init__(self): 169 self.client_cwd = "../grpc-dart/interop" 170 self.server_cwd = "../grpc-dart/interop" 171 self.http2_cwd = "../grpc-dart/interop" 172 self.safename = str(self) 173 174 def client_cmd(self, args): 175 return ["dart", "bin/client.dart"] + args 176 177 def cloud_to_prod_env(self): 178 return {} 179 180 def server_cmd(self, args): 181 return ["dart", "bin/server.dart"] + args 182 183 def global_env(self): 184 return {} 185 186 def unimplemented_test_cases(self): 187 return ( 188 _SKIP_COMPRESSION 189 + _SKIP_SPECIAL_STATUS_MESSAGE 190 + _SKIP_GOOGLE_DEFAULT_CREDS 191 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 192 + _ORCA_TEST_CASES 193 ) 194 195 def unimplemented_test_cases_server(self): 196 return ( 197 _SKIP_COMPRESSION + _SKIP_SPECIAL_STATUS_MESSAGE + _ORCA_TEST_CASES 198 ) 199 200 def __str__(self): 201 return "dart" 202 203 204class JavaLanguage: 205 def __init__(self): 206 self.client_cwd = "../grpc-java" 207 self.server_cwd = "../grpc-java" 208 self.http2_cwd = "../grpc-java" 209 self.safename = str(self) 210 211 def client_cmd(self, args): 212 return ["./run-test-client.sh"] + args 213 214 def client_cmd_http2interop(self, args): 215 return [ 216 "./interop-testing/build/install/grpc-interop-testing/bin/http2-client" 217 ] + args 218 219 def cloud_to_prod_env(self): 220 return {} 221 222 def server_cmd(self, args): 223 return ["./run-test-server.sh"] + args 224 225 def global_env(self): 226 return {} 227 228 def unimplemented_test_cases(self): 229 return [] 230 231 def unimplemented_test_cases_server(self): 232 # Does not support CompressedRequest feature. 233 # Only supports CompressedResponse feature for unary. 234 return _SKIP_CLIENT_COMPRESSION + ["server_compressed_streaming"] 235 236 def __str__(self): 237 return "java" 238 239 240class JavaOkHttpClient: 241 def __init__(self): 242 self.client_cwd = "../grpc-java" 243 self.safename = "java" 244 245 def client_cmd(self, args): 246 return ["./run-test-client.sh", "--use_okhttp=true"] + args 247 248 def cloud_to_prod_env(self): 249 return {} 250 251 def global_env(self): 252 return {} 253 254 def unimplemented_test_cases(self): 255 return _SKIP_DATA_FRAME_PADDING 256 257 def __str__(self): 258 return "javaokhttp" 259 260 261class GoLanguage: 262 def __init__(self): 263 # TODO: this relies on running inside docker 264 self.client_cwd = "/go/src/google.golang.org/grpc/interop/client" 265 self.server_cwd = "/go/src/google.golang.org/grpc/interop/server" 266 self.http2_cwd = "/go/src/google.golang.org/grpc/interop/http2" 267 self.safename = str(self) 268 269 def client_cmd(self, args): 270 return ["go", "run", "client.go"] + args 271 272 def client_cmd_http2interop(self, args): 273 return ["go", "run", "negative_http2_client.go"] + args 274 275 def cloud_to_prod_env(self): 276 return {} 277 278 def server_cmd(self, args): 279 return ["go", "run", "server.go"] + args 280 281 def global_env(self): 282 return {"GO111MODULE": "on"} 283 284 def unimplemented_test_cases(self): 285 return _SKIP_COMPRESSION 286 287 def unimplemented_test_cases_server(self): 288 return _SKIP_COMPRESSION 289 290 def __str__(self): 291 return "go" 292 293 294class Http2Server: 295 """Represents the HTTP/2 Interop Test server 296 297 This pretends to be a language in order to be built and run, but really it 298 isn't. 299 """ 300 301 def __init__(self): 302 self.server_cwd = None 303 self.safename = str(self) 304 305 def server_cmd(self, args): 306 return ["python test/http2_test/http2_test_server.py"] 307 308 def cloud_to_prod_env(self): 309 return {} 310 311 def global_env(self): 312 return {} 313 314 def unimplemented_test_cases(self): 315 return ( 316 _TEST_CASES 317 + _SKIP_DATA_FRAME_PADDING 318 + _SKIP_SPECIAL_STATUS_MESSAGE 319 + _SKIP_GOOGLE_DEFAULT_CREDS 320 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 321 ) 322 323 def unimplemented_test_cases_server(self): 324 return _TEST_CASES 325 326 def __str__(self): 327 return "http2" 328 329 330class Http2Client: 331 """Represents the HTTP/2 Interop Test 332 333 This pretends to be a language in order to be built and run, but really it 334 isn't. 335 """ 336 337 def __init__(self): 338 self.client_cwd = None 339 self.safename = str(self) 340 341 def client_cmd(self, args): 342 return ["tools/http2_interop/http2_interop.test", "-test.v"] + args 343 344 def cloud_to_prod_env(self): 345 return {} 346 347 def global_env(self): 348 return {} 349 350 def unimplemented_test_cases(self): 351 return ( 352 _TEST_CASES 353 + _SKIP_SPECIAL_STATUS_MESSAGE 354 + _SKIP_GOOGLE_DEFAULT_CREDS 355 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 356 ) 357 358 def unimplemented_test_cases_server(self): 359 return _TEST_CASES 360 361 def __str__(self): 362 return "http2" 363 364 365class NodeLanguage: 366 def __init__(self): 367 self.client_cwd = "../../../../home/appuser/grpc-node" 368 self.server_cwd = "../../../../home/appuser/grpc-node" 369 self.safename = str(self) 370 371 def client_cmd(self, args): 372 return [ 373 "packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh", 374 "node", 375 "--require", 376 "./test/fixtures/native_native", 377 "test/interop/interop_client.js", 378 ] + args 379 380 def cloud_to_prod_env(self): 381 return {} 382 383 def server_cmd(self, args): 384 return [ 385 "packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh", 386 "node", 387 "--require", 388 "./test/fixtures/native_native", 389 "test/interop/interop_server.js", 390 ] + args 391 392 def global_env(self): 393 return {} 394 395 def unimplemented_test_cases(self): 396 return ( 397 _SKIP_COMPRESSION 398 + _SKIP_DATA_FRAME_PADDING 399 + _SKIP_GOOGLE_DEFAULT_CREDS 400 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 401 + _ORCA_TEST_CASES 402 ) 403 404 def unimplemented_test_cases_server(self): 405 return _SKIP_COMPRESSION + _ORCA_TEST_CASES 406 407 def __str__(self): 408 return "node" 409 410 411class NodePureJSLanguage: 412 def __init__(self): 413 self.client_cwd = "../../../../home/appuser/grpc-node" 414 self.server_cwd = "../../../../home/appuser/grpc-node" 415 self.safename = str(self) 416 417 def client_cmd(self, args): 418 return [ 419 "packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh", 420 "node", 421 "--require", 422 "./test/fixtures/js_js", 423 "test/interop/interop_client.js", 424 ] + args 425 426 def cloud_to_prod_env(self): 427 return {} 428 429 def global_env(self): 430 return {} 431 432 def unimplemented_test_cases(self): 433 return ( 434 _SKIP_COMPRESSION 435 + _SKIP_DATA_FRAME_PADDING 436 + _SKIP_GOOGLE_DEFAULT_CREDS 437 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 438 + _ORCA_TEST_CASES 439 ) 440 441 def unimplemented_test_cases_server(self): 442 return _ORCA_TEST_CASES 443 444 def __str__(self): 445 return "nodepurejs" 446 447 448class PHP7Language: 449 def __init__(self): 450 self.client_cwd = None 451 self.server_cwd = None 452 self.safename = str(self) 453 454 def client_cmd(self, args): 455 return ["src/php/bin/interop_client.sh"] + args 456 457 def cloud_to_prod_env(self): 458 return {} 459 460 def server_cmd(self, args): 461 return ["src/php/bin/interop_server.sh"] + args 462 463 def global_env(self): 464 return {} 465 466 def unimplemented_test_cases(self): 467 return ( 468 _SKIP_SERVER_COMPRESSION 469 + _SKIP_DATA_FRAME_PADDING 470 + _SKIP_GOOGLE_DEFAULT_CREDS 471 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 472 + _ORCA_TEST_CASES 473 ) 474 475 def unimplemented_test_cases_server(self): 476 return _SKIP_COMPRESSION + _ORCA_TEST_CASES 477 478 def __str__(self): 479 return "php7" 480 481 482class ObjcLanguage: 483 def __init__(self): 484 self.client_cwd = "src/objective-c/tests" 485 self.safename = str(self) 486 487 def client_cmd(self, args): 488 # from args, extract the server port and craft xcodebuild command out of it 489 for arg in args: 490 port = re.search("--server_port=(\d+)", arg) 491 if port: 492 portnum = port.group(1) 493 cmdline = ( 494 "pod install && xcodebuild -workspace Tests.xcworkspace" 495 ' -scheme InteropTestsLocalSSL -destination name="iPhone 6"' 496 " HOST_PORT_LOCALSSL=localhost:%s test" % portnum 497 ) 498 return [cmdline] 499 500 def cloud_to_prod_env(self): 501 return {} 502 503 def global_env(self): 504 return {} 505 506 def unimplemented_test_cases(self): 507 # ObjC test runs all cases with the same command. It ignores the testcase 508 # cmdline argument. Here we return all but one test cases as unimplemented, 509 # and depend upon ObjC test's behavior that it runs all cases even when 510 # we tell it to run just one. 511 return ( 512 _TEST_CASES[1:] 513 + _SKIP_COMPRESSION 514 + _SKIP_DATA_FRAME_PADDING 515 + _SKIP_SPECIAL_STATUS_MESSAGE 516 + _SKIP_GOOGLE_DEFAULT_CREDS 517 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 518 + _ORCA_TEST_CASES 519 ) 520 521 def unimplemented_test_cases_server(self): 522 return _SKIP_COMPRESSION + _ORCA_TEST_CASES 523 524 def __str__(self): 525 return "objc" 526 527 528class RubyLanguage: 529 def __init__(self): 530 self.client_cwd = None 531 self.server_cwd = None 532 self.safename = str(self) 533 534 def client_cmd(self, args): 535 return [ 536 "tools/run_tests/interop/with_rvm.sh", 537 "ruby", 538 "src/ruby/pb/test/client.rb", 539 ] + args 540 541 def cloud_to_prod_env(self): 542 return {} 543 544 def server_cmd(self, args): 545 return [ 546 "tools/run_tests/interop/with_rvm.sh", 547 "ruby", 548 "src/ruby/pb/test/server.rb", 549 ] + args 550 551 def global_env(self): 552 return {} 553 554 def unimplemented_test_cases(self): 555 return ( 556 _SKIP_SERVER_COMPRESSION 557 + _SKIP_DATA_FRAME_PADDING 558 + _SKIP_GOOGLE_DEFAULT_CREDS 559 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 560 + _ORCA_TEST_CASES 561 ) 562 563 def unimplemented_test_cases_server(self): 564 return _SKIP_COMPRESSION + _ORCA_TEST_CASES 565 566 def __str__(self): 567 return "ruby" 568 569 570_PYTHON_BINARY = "py39/bin/python" 571 572 573class PythonLanguage: 574 def __init__(self): 575 self.client_cwd = None 576 self.server_cwd = None 577 self.http2_cwd = None 578 self.safename = str(self) 579 580 def client_cmd(self, args): 581 return [ 582 _PYTHON_BINARY, 583 "src/python/grpcio_tests/setup.py", 584 "run_interop", 585 "--client", 586 '--args="{}"'.format(" ".join(args)), 587 ] 588 589 def client_cmd_http2interop(self, args): 590 return [ 591 _PYTHON_BINARY, 592 "src/python/grpcio_tests/tests/http2/negative_http2_client.py", 593 ] + args 594 595 def cloud_to_prod_env(self): 596 return {} 597 598 def server_cmd(self, args): 599 return [ 600 _PYTHON_BINARY, 601 "src/python/grpcio_tests/setup.py", 602 "run_interop", 603 "--server", 604 '--args="{}"'.format(" ".join(args)), 605 ] 606 607 def global_env(self): 608 return { 609 "LD_LIBRARY_PATH": "{}/libs/opt".format(DOCKER_WORKDIR_ROOT), 610 "PYTHONPATH": "{}/src/python/gens".format(DOCKER_WORKDIR_ROOT), 611 } 612 613 def unimplemented_test_cases(self): 614 return ( 615 _SKIP_COMPRESSION 616 + _SKIP_DATA_FRAME_PADDING 617 + _SKIP_GOOGLE_DEFAULT_CREDS 618 + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS 619 + _ORCA_TEST_CASES 620 ) 621 622 def unimplemented_test_cases_server(self): 623 return _SKIP_COMPRESSION + _ORCA_TEST_CASES 624 625 def __str__(self): 626 return "python" 627 628 629class PythonAsyncIOLanguage: 630 def __init__(self): 631 self.client_cwd = None 632 self.server_cwd = None 633 self.http2_cwd = None 634 self.safename = str(self) 635 636 def client_cmd(self, args): 637 return [ 638 _PYTHON_BINARY, 639 "src/python/grpcio_tests/setup.py", 640 "run_interop", 641 "--use-asyncio", 642 "--client", 643 '--args="{}"'.format(" ".join(args)), 644 ] 645 646 def client_cmd_http2interop(self, args): 647 return [ 648 _PYTHON_BINARY, 649 "src/python/grpcio_tests/tests/http2/negative_http2_client.py", 650 ] + args 651 652 def cloud_to_prod_env(self): 653 return {} 654 655 def server_cmd(self, args): 656 return [ 657 _PYTHON_BINARY, 658 "src/python/grpcio_tests/setup.py", 659 "run_interop", 660 "--use-asyncio", 661 "--server", 662 '--args="{}"'.format(" ".join(args)), 663 ] 664 665 def global_env(self): 666 return { 667 "LD_LIBRARY_PATH": "{}/libs/opt".format(DOCKER_WORKDIR_ROOT), 668 "PYTHONPATH": "{}/src/python/gens".format(DOCKER_WORKDIR_ROOT), 669 } 670 671 def unimplemented_test_cases(self): 672 # TODO(https://github.com/grpc/grpc/issues/21707) 673 return ( 674 _SKIP_COMPRESSION 675 + _SKIP_DATA_FRAME_PADDING 676 + _AUTH_TEST_CASES 677 + ["timeout_on_sleeping_server"] 678 + _ORCA_TEST_CASES 679 ) 680 681 def unimplemented_test_cases_server(self): 682 # TODO(https://github.com/grpc/grpc/issues/21749) 683 return ( 684 _TEST_CASES 685 + _AUTH_TEST_CASES 686 + _HTTP2_TEST_CASES 687 + _HTTP2_SERVER_TEST_CASES 688 ) 689 690 def __str__(self): 691 return "pythonasyncio" 692 693 694_LANGUAGES = { 695 "c++": CXXLanguage(), 696 "aspnetcore": AspNetCoreLanguage(), 697 "dart": DartLanguage(), 698 "go": GoLanguage(), 699 "java": JavaLanguage(), 700 "javaokhttp": JavaOkHttpClient(), 701 "node": NodeLanguage(), 702 "nodepurejs": NodePureJSLanguage(), 703 "php7": PHP7Language(), 704 "objc": ObjcLanguage(), 705 "ruby": RubyLanguage(), 706 "python": PythonLanguage(), 707 "pythonasyncio": PythonAsyncIOLanguage(), 708} 709 710# languages supported as cloud_to_cloud servers 711_SERVERS = [ 712 "c++", 713 "node", 714 "aspnetcore", 715 "java", 716 "go", 717 "ruby", 718 "python", 719 "dart", 720 "pythonasyncio", 721 "php7", 722] 723 724_TEST_CASES = [ 725 "large_unary", 726 "empty_unary", 727 "ping_pong", 728 "empty_stream", 729 "client_streaming", 730 "server_streaming", 731 "cancel_after_begin", 732 "cancel_after_first_response", 733 "timeout_on_sleeping_server", 734 "custom_metadata", 735 "status_code_and_message", 736 "unimplemented_method", 737 "client_compressed_unary", 738 "server_compressed_unary", 739 "client_compressed_streaming", 740 "server_compressed_streaming", 741 "unimplemented_service", 742 "special_status_message", 743 "orca_per_rpc", 744 "orca_oob", 745] 746 747_AUTH_TEST_CASES = [ 748 "compute_engine_creds", 749 "jwt_token_creds", 750 "oauth2_auth_token", 751 "per_rpc_creds", 752 _GOOGLE_DEFAULT_CREDS_TEST_CASE, 753 _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE, 754] 755 756_HTTP2_TEST_CASES = ["tls", "framing"] 757 758_HTTP2_SERVER_TEST_CASES = [ 759 "rst_after_header", 760 "rst_after_data", 761 "rst_during_data", 762 "goaway", 763 "ping", 764 "max_streams", 765 "data_frame_padding", 766 "no_df_padding_sanity_test", 767] 768 769_GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES = { 770 "data_frame_padding": "large_unary", 771 "no_df_padding_sanity_test": "large_unary", 772} 773 774_HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS = list( 775 _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES.keys() 776) 777 778_LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ 779 "java", 780 "go", 781 "python", 782 "c++", 783] 784 785_LANGUAGES_FOR_ALTS_TEST_CASES = ["java", "go", "c++", "python"] 786 787_SERVERS_FOR_ALTS_TEST_CASES = ["java", "go", "c++", "python"] 788 789_TRANSPORT_SECURITY_OPTIONS = ["tls", "alts", "insecure"] 790 791_CUSTOM_CREDENTIALS_TYPE_OPTIONS = [ 792 "tls", 793 "google_default_credentials", 794 "compute_engine_channel_creds", 795] 796 797DOCKER_WORKDIR_ROOT = "/var/local/git/grpc" 798 799 800def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): 801 """Wraps given cmdline array to create 'docker run' cmdline from it.""" 802 803 # don't use '-t' even when TTY is available, since that would break 804 # the testcases generated by tools/interop_matrix/create_testcases.sh 805 docker_cmdline = ["docker", "run", "-i", "--rm=true"] 806 807 # turn environ into -e docker args 808 if environ: 809 for k, v in list(environ.items()): 810 docker_cmdline += ["-e", "%s=%s" % (k, v)] 811 812 # set working directory 813 workdir = DOCKER_WORKDIR_ROOT 814 if cwd: 815 workdir = os.path.join(workdir, cwd) 816 docker_cmdline += ["-w", workdir] 817 818 docker_cmdline += docker_args + [image] + cmdline 819 return docker_cmdline 820 821 822def manual_cmdline(docker_cmdline, docker_image): 823 """Returns docker cmdline adjusted for manual invocation.""" 824 print_cmdline = [] 825 for item in docker_cmdline: 826 if item.startswith("--name="): 827 continue 828 if item == docker_image: 829 item = "$docker_image" 830 item = item.replace('"', '\\"') 831 # add quotes when necessary 832 if any(character.isspace() for character in item): 833 item = '"%s"' % item 834 print_cmdline.append(item) 835 return " ".join(print_cmdline) 836 837 838def write_cmdlog_maybe(cmdlog, filename): 839 """Returns docker cmdline adjusted for manual invocation.""" 840 if cmdlog: 841 with open(filename, "w") as logfile: 842 logfile.write("#!/bin/bash\n") 843 logfile.write("# DO NOT MODIFY\n") 844 logfile.write( 845 "# This file is generated by" 846 " run_interop_tests.py/create_testcases.sh\n" 847 ) 848 logfile.writelines("%s\n" % line for line in cmdlog) 849 print("Command log written to file %s" % filename) 850 851 852def bash_cmdline(cmdline): 853 """Creates bash -c cmdline from args list.""" 854 # Use login shell: 855 # * makes error messages clearer if executables are missing 856 return ["bash", "-c", " ".join(cmdline)] 857 858 859def compute_engine_creds_required(language, test_case): 860 """Returns True if given test requires access to compute engine creds.""" 861 language = str(language) 862 if test_case == "compute_engine_creds": 863 return True 864 if test_case == "oauth2_auth_token" and language == "c++": 865 # C++ oauth2 test uses GCE creds because C++ only supports JWT 866 return True 867 return False 868 869 870def auth_options( 871 language, 872 test_case, 873 google_default_creds_use_key_file, 874 service_account_key_file, 875 default_service_account, 876): 877 """Returns (cmdline, env) tuple with cloud_to_prod_auth test options.""" 878 879 language = str(language) 880 cmdargs = [] 881 env = {} 882 883 oauth_scope_arg = "--oauth_scope=https://www.googleapis.com/auth/xapi.zoo" 884 key_file_arg = "--service_account_key_file=%s" % service_account_key_file 885 default_account_arg = ( 886 "--default_service_account=%s" % default_service_account 887 ) 888 889 if test_case in ["jwt_token_creds", "per_rpc_creds", "oauth2_auth_token"]: 890 if language in [ 891 "aspnetcore", 892 "node", 893 "php7", 894 "python", 895 "ruby", 896 "nodepurejs", 897 ]: 898 env["GOOGLE_APPLICATION_CREDENTIALS"] = service_account_key_file 899 else: 900 cmdargs += [key_file_arg] 901 902 if test_case in ["per_rpc_creds", "oauth2_auth_token"]: 903 cmdargs += [oauth_scope_arg] 904 905 if test_case == "oauth2_auth_token" and language == "c++": 906 # C++ oauth2 test uses GCE creds and thus needs to know the default account 907 cmdargs += [default_account_arg] 908 909 if test_case == "compute_engine_creds": 910 cmdargs += [oauth_scope_arg, default_account_arg] 911 912 if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE: 913 if google_default_creds_use_key_file: 914 env["GOOGLE_APPLICATION_CREDENTIALS"] = service_account_key_file 915 cmdargs += [default_account_arg] 916 917 if test_case == _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE: 918 cmdargs += [default_account_arg] 919 920 return (cmdargs, env) 921 922 923def _job_kill_handler(job): 924 if job._spec.container_name: 925 dockerjob.docker_kill(job._spec.container_name) 926 # When the job times out and we decide to kill it, 927 # we need to wait a before restarting the job 928 # to prevent "container name already in use" error. 929 # TODO(jtattermusch): figure out a cleaner way to this. 930 time.sleep(2) 931 932 933def cloud_to_prod_jobspec( 934 language, 935 test_case, 936 server_host_nickname, 937 server_host, 938 google_default_creds_use_key_file, 939 docker_image=None, 940 auth=False, 941 manual_cmd_log=None, 942 service_account_key_file=None, 943 default_service_account=None, 944 transport_security="tls", 945): 946 """Creates jobspec for cloud-to-prod interop test""" 947 container_name = None 948 cmdargs = [ 949 "--server_host=%s" % server_host, 950 "--server_port=443", 951 "--test_case=%s" % test_case, 952 ] 953 if transport_security == "tls": 954 transport_security_options = ["--use_tls=true"] 955 elif transport_security == "google_default_credentials" and str( 956 language 957 ) in ["c++", "go", "java", "javaokhttp"]: 958 transport_security_options = [ 959 "--custom_credentials_type=google_default_credentials" 960 ] 961 elif transport_security == "compute_engine_channel_creds" and str( 962 language 963 ) in ["go", "java", "javaokhttp"]: 964 transport_security_options = [ 965 "--custom_credentials_type=compute_engine_channel_creds" 966 ] 967 else: 968 print( 969 "Invalid transport security option %s in cloud_to_prod_jobspec." 970 " Lang: %s" % (str(language), transport_security) 971 ) 972 sys.exit(1) 973 cmdargs = cmdargs + transport_security_options 974 environ = dict(language.cloud_to_prod_env(), **language.global_env()) 975 if auth: 976 auth_cmdargs, auth_env = auth_options( 977 language, 978 test_case, 979 google_default_creds_use_key_file, 980 service_account_key_file, 981 default_service_account, 982 ) 983 cmdargs += auth_cmdargs 984 environ.update(auth_env) 985 cmdline = bash_cmdline(language.client_cmd(cmdargs)) 986 cwd = language.client_cwd 987 988 if docker_image: 989 container_name = dockerjob.random_name( 990 "interop_client_%s" % language.safename 991 ) 992 cmdline = docker_run_cmdline( 993 cmdline, 994 image=docker_image, 995 cwd=cwd, 996 environ=environ, 997 docker_args=["--net=host", "--name=%s" % container_name], 998 ) 999 if manual_cmd_log is not None: 1000 if manual_cmd_log == []: 1001 manual_cmd_log.append( 1002 'echo "Testing ${docker_image:=%s}"' % docker_image 1003 ) 1004 manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) 1005 cwd = None 1006 environ = None 1007 1008 suite_name = "cloud_to_prod_auth" if auth else "cloud_to_prod" 1009 test_job = jobset.JobSpec( 1010 cmdline=cmdline, 1011 cwd=cwd, 1012 environ=environ, 1013 shortname="%s:%s:%s:%s:%s" 1014 % ( 1015 suite_name, 1016 language, 1017 server_host_nickname, 1018 test_case, 1019 transport_security, 1020 ), 1021 timeout_seconds=_TEST_TIMEOUT, 1022 flake_retries=4 if args.allow_flakes else 0, 1023 timeout_retries=2 if args.allow_flakes else 0, 1024 kill_handler=_job_kill_handler, 1025 ) 1026 if docker_image: 1027 test_job.container_name = container_name 1028 return test_job 1029 1030 1031def cloud_to_cloud_jobspec( 1032 language, 1033 test_case, 1034 server_name, 1035 server_host, 1036 server_port, 1037 docker_image=None, 1038 transport_security="tls", 1039 manual_cmd_log=None, 1040): 1041 """Creates jobspec for cloud-to-cloud interop test""" 1042 interop_only_options = [ 1043 "--server_host_override=foo.test.google.fr", 1044 "--use_test_ca=true", 1045 ] 1046 if transport_security == "tls": 1047 interop_only_options += ["--use_tls=true"] 1048 elif transport_security == "alts": 1049 interop_only_options += ["--use_tls=false", "--use_alts=true"] 1050 elif transport_security == "insecure": 1051 interop_only_options += ["--use_tls=false"] 1052 else: 1053 print( 1054 "Invalid transport security option %s in cloud_to_cloud_jobspec." 1055 % transport_security 1056 ) 1057 sys.exit(1) 1058 1059 client_test_case = test_case 1060 if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 1061 client_test_case = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES[ 1062 test_case 1063 ] 1064 if client_test_case in language.unimplemented_test_cases(): 1065 print( 1066 "asking client %s to run unimplemented test case %s" 1067 % (repr(language), client_test_case) 1068 ) 1069 sys.exit(1) 1070 1071 if test_case in _ORCA_TEST_CASES: 1072 interop_only_options += [ 1073 '--service_config_json=\'{"loadBalancingConfig":[{"test_backend_metrics_load_balancer":{}}]}\'' 1074 ] 1075 1076 common_options = [ 1077 "--test_case=%s" % client_test_case, 1078 "--server_host=%s" % server_host, 1079 "--server_port=%s" % server_port, 1080 ] 1081 1082 if test_case in _HTTP2_SERVER_TEST_CASES: 1083 if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 1084 client_options = interop_only_options + common_options 1085 cmdline = bash_cmdline(language.client_cmd(client_options)) 1086 cwd = language.client_cwd 1087 else: 1088 cmdline = bash_cmdline( 1089 language.client_cmd_http2interop(common_options) 1090 ) 1091 cwd = language.http2_cwd 1092 else: 1093 cmdline = bash_cmdline( 1094 language.client_cmd(common_options + interop_only_options) 1095 ) 1096 cwd = language.client_cwd 1097 1098 environ = language.global_env() 1099 if docker_image and language.safename != "objc": 1100 # we can't run client in docker for objc. 1101 container_name = dockerjob.random_name( 1102 "interop_client_%s" % language.safename 1103 ) 1104 cmdline = docker_run_cmdline( 1105 cmdline, 1106 image=docker_image, 1107 environ=environ, 1108 cwd=cwd, 1109 docker_args=["--net=host", "--name=%s" % container_name], 1110 ) 1111 if manual_cmd_log is not None: 1112 if manual_cmd_log == []: 1113 manual_cmd_log.append( 1114 'echo "Testing ${docker_image:=%s}"' % docker_image 1115 ) 1116 manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) 1117 cwd = None 1118 1119 test_job = jobset.JobSpec( 1120 cmdline=cmdline, 1121 cwd=cwd, 1122 environ=environ, 1123 shortname="cloud_to_cloud:%s:%s_server:%s:%s" 1124 % (language, server_name, test_case, transport_security), 1125 timeout_seconds=_TEST_TIMEOUT, 1126 flake_retries=4 if args.allow_flakes else 0, 1127 timeout_retries=2 if args.allow_flakes else 0, 1128 kill_handler=_job_kill_handler, 1129 ) 1130 if docker_image: 1131 test_job.container_name = container_name 1132 return test_job 1133 1134 1135def server_jobspec( 1136 language, docker_image, transport_security="tls", manual_cmd_log=None 1137): 1138 """Create jobspec for running a server""" 1139 container_name = dockerjob.random_name( 1140 "interop_server_%s" % language.safename 1141 ) 1142 server_cmd = ["--port=%s" % _DEFAULT_SERVER_PORT] 1143 if transport_security == "tls": 1144 server_cmd += ["--use_tls=true"] 1145 elif transport_security == "alts": 1146 server_cmd += ["--use_tls=false", "--use_alts=true"] 1147 elif transport_security == "insecure": 1148 server_cmd += ["--use_tls=false"] 1149 else: 1150 print( 1151 "Invalid transport security option %s in server_jobspec." 1152 % transport_security 1153 ) 1154 sys.exit(1) 1155 cmdline = bash_cmdline(language.server_cmd(server_cmd)) 1156 environ = language.global_env() 1157 docker_args = ["--name=%s" % container_name] 1158 if language.safename == "http2": 1159 # we are running the http2 interop server. Open next N ports beginning 1160 # with the server port. These ports are used for http2 interop test 1161 # (one test case per port). 1162 docker_args += list( 1163 itertools.chain.from_iterable( 1164 ("-p", str(_DEFAULT_SERVER_PORT + i)) 1165 for i in range(len(_HTTP2_SERVER_TEST_CASES)) 1166 ) 1167 ) 1168 # Enable docker's healthcheck mechanism. 1169 # This runs a Python script inside the container every second. The script 1170 # pings the http2 server to verify it is ready. The 'health-retries' flag 1171 # specifies the number of consecutive failures before docker will report 1172 # the container's status as 'unhealthy'. Prior to the first 'health_retries' 1173 # failures or the first success, the status will be 'starting'. 'docker ps' 1174 # or 'docker inspect' can be used to see the health of the container on the 1175 # command line. 1176 docker_args += [ 1177 "--health-cmd=python test/http2_test/http2_server_health_check.py " 1178 "--server_host=%s --server_port=%d" 1179 % ("localhost", _DEFAULT_SERVER_PORT), 1180 "--health-interval=1s", 1181 "--health-retries=5", 1182 "--health-timeout=10s", 1183 ] 1184 1185 else: 1186 docker_args += ["-p", str(_DEFAULT_SERVER_PORT)] 1187 1188 docker_cmdline = docker_run_cmdline( 1189 cmdline, 1190 image=docker_image, 1191 cwd=language.server_cwd, 1192 environ=environ, 1193 docker_args=docker_args, 1194 ) 1195 if manual_cmd_log is not None: 1196 if manual_cmd_log == []: 1197 manual_cmd_log.append( 1198 'echo "Testing ${docker_image:=%s}"' % docker_image 1199 ) 1200 manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image)) 1201 server_job = jobset.JobSpec( 1202 cmdline=docker_cmdline, 1203 environ=environ, 1204 shortname="interop_server_%s" % language, 1205 timeout_seconds=30 * 60, 1206 ) 1207 server_job.container_name = container_name 1208 return server_job 1209 1210 1211def build_interop_image_jobspec(language, tag=None): 1212 """Creates jobspec for building interop docker image for a language""" 1213 if not tag: 1214 tag = "grpc_interop_%s:%s" % (language.safename, uuid.uuid4()) 1215 env = { 1216 "INTEROP_IMAGE": tag, 1217 "BASE_NAME": "grpc_interop_%s" % language.safename, 1218 } 1219 build_job = jobset.JobSpec( 1220 cmdline=["tools/run_tests/dockerize/build_interop_image.sh"], 1221 environ=env, 1222 shortname="build_docker_%s" % (language), 1223 timeout_seconds=30 * 60, 1224 ) 1225 build_job.tag = tag 1226 return build_job 1227 1228 1229def aggregate_http2_results(stdout): 1230 match = re.search(r'\{"cases[^\]]*\]\}', stdout) 1231 if not match: 1232 return None 1233 1234 results = json.loads(match.group(0)) 1235 skipped = 0 1236 passed = 0 1237 failed = 0 1238 failed_cases = [] 1239 for case in results["cases"]: 1240 if case.get("skipped", False): 1241 skipped += 1 1242 else: 1243 if case.get("passed", False): 1244 passed += 1 1245 else: 1246 failed += 1 1247 failed_cases.append(case.get("name", "NONAME")) 1248 return { 1249 "passed": passed, 1250 "failed": failed, 1251 "skipped": skipped, 1252 "failed_cases": ", ".join(failed_cases), 1253 "percent": 1.0 * passed / (passed + failed), 1254 } 1255 1256 1257# A dictionary of prod servers to test against. 1258# See go/grpc-interop-tests (internal-only) for details. 1259prod_servers = { 1260 "default": "grpc-test.sandbox.googleapis.com", 1261 "gateway_v4": "grpc-test4.sandbox.googleapis.com", 1262} 1263 1264argp = argparse.ArgumentParser(description="Run interop tests.") 1265argp.add_argument( 1266 "-l", 1267 "--language", 1268 choices=["all"] + sorted(_LANGUAGES), 1269 nargs="+", 1270 default=["all"], 1271 help="Clients to run. Objc client can be only run on OSX.", 1272) 1273argp.add_argument("-j", "--jobs", default=multiprocessing.cpu_count(), type=int) 1274argp.add_argument( 1275 "--cloud_to_prod", 1276 default=False, 1277 action="store_const", 1278 const=True, 1279 help="Run cloud_to_prod tests.", 1280) 1281argp.add_argument( 1282 "--cloud_to_prod_auth", 1283 default=False, 1284 action="store_const", 1285 const=True, 1286 help="Run cloud_to_prod_auth tests.", 1287) 1288argp.add_argument( 1289 "--google_default_creds_use_key_file", 1290 default=False, 1291 action="store_const", 1292 const=True, 1293 help=( 1294 "Whether or not we should use a key file for the " 1295 "google_default_credentials test case, e.g. by " 1296 "setting env var GOOGLE_APPLICATION_CREDENTIALS." 1297 ), 1298) 1299argp.add_argument( 1300 "--prod_servers", 1301 choices=list(prod_servers.keys()), 1302 default=["default"], 1303 nargs="+", 1304 help=( 1305 "The servers to run cloud_to_prod and cloud_to_prod_auth tests against." 1306 ), 1307) 1308argp.add_argument( 1309 "-s", 1310 "--server", 1311 choices=["all"] + sorted(_SERVERS), 1312 nargs="+", 1313 help="Run cloud_to_cloud servers in a separate docker " 1314 + "image. Servers can only be started automatically if " 1315 + "--use_docker option is enabled.", 1316 default=[], 1317) 1318argp.add_argument( 1319 "--override_server", 1320 action="append", 1321 type=lambda kv: kv.split("="), 1322 help=( 1323 "Use servername=HOST:PORT to explicitly specify a server. E.g." 1324 " csharp=localhost:50000" 1325 ), 1326 default=[], 1327) 1328# TODO(jtattermusch): the default service_account_key_file only works when --use_docker is used. 1329argp.add_argument( 1330 "--service_account_key_file", 1331 type=str, 1332 help="The service account key file to use for some auth interop tests.", 1333 default="/root/service_account/grpc-testing-ebe7c1ac7381.json", 1334) 1335argp.add_argument( 1336 "--default_service_account", 1337 type=str, 1338 help=( 1339 "Default GCE service account email to use for some auth interop tests." 1340 ), 1341 default="[email protected]", 1342) 1343argp.add_argument( 1344 "-t", 1345 "--travis", 1346 default=False, 1347 action="store_const", 1348 const=True, 1349 help=( 1350 "When set, indicates that the script is running on CI (= not locally)." 1351 ), 1352) 1353argp.add_argument( 1354 "-v", "--verbose", default=False, action="store_const", const=True 1355) 1356argp.add_argument( 1357 "--use_docker", 1358 default=False, 1359 action="store_const", 1360 const=True, 1361 help="Run all the interop tests under docker. That provides " 1362 + "additional isolation and prevents the need to install " 1363 + "language specific prerequisites. Only available on Linux.", 1364) 1365argp.add_argument( 1366 "--allow_flakes", 1367 default=False, 1368 action="store_const", 1369 const=True, 1370 help=( 1371 "Allow flaky tests to show as passing (re-runs failed tests up to five" 1372 " times)" 1373 ), 1374) 1375argp.add_argument( 1376 "--manual_run", 1377 default=False, 1378 action="store_const", 1379 const=True, 1380 help="Prepare things for running interop tests manually. " 1381 + "Preserve docker images after building them and skip " 1382 "actually running the tests. Only print commands to run by " + "hand.", 1383) 1384argp.add_argument( 1385 "--http2_interop", 1386 default=False, 1387 action="store_const", 1388 const=True, 1389 help="Enable HTTP/2 client edge case testing. (Bad client, good server)", 1390) 1391argp.add_argument( 1392 "--http2_server_interop", 1393 default=False, 1394 action="store_const", 1395 const=True, 1396 help=( 1397 "Enable HTTP/2 server edge case testing. (Includes positive and" 1398 " negative tests" 1399 ), 1400) 1401argp.add_argument( 1402 "--transport_security", 1403 choices=_TRANSPORT_SECURITY_OPTIONS, 1404 default="tls", 1405 type=str, 1406 nargs="?", 1407 const=True, 1408 help="Which transport security mechanism to use.", 1409) 1410argp.add_argument( 1411 "--custom_credentials_type", 1412 choices=_CUSTOM_CREDENTIALS_TYPE_OPTIONS, 1413 default=_CUSTOM_CREDENTIALS_TYPE_OPTIONS, 1414 nargs="+", 1415 help=( 1416 "Credential types to test in the cloud_to_prod setup. Default is to" 1417 " test with all creds types possible." 1418 ), 1419) 1420argp.add_argument( 1421 "--skip_compute_engine_creds", 1422 default=False, 1423 action="store_const", 1424 const=True, 1425 help="Skip auth tests requiring access to compute engine credentials.", 1426) 1427argp.add_argument( 1428 "--internal_ci", 1429 default=False, 1430 action="store_const", 1431 const=True, 1432 help=( 1433 "(Deprecated, has no effect) Put reports into subdirectories to improve" 1434 " presentation of results by Internal CI." 1435 ), 1436) 1437argp.add_argument( 1438 "--bq_result_table", 1439 default="", 1440 type=str, 1441 nargs="?", 1442 help="Upload test results to a specified BQ table.", 1443) 1444args = argp.parse_args() 1445 1446servers = set( 1447 s 1448 for s in itertools.chain.from_iterable( 1449 _SERVERS if x == "all" else [x] for x in args.server 1450 ) 1451) 1452# ALTS servers are only available for certain languages. 1453if args.transport_security == "alts": 1454 servers = servers.intersection(_SERVERS_FOR_ALTS_TEST_CASES) 1455 1456if args.use_docker: 1457 if not args.travis: 1458 print("Seen --use_docker flag, will run interop tests under docker.") 1459 print("") 1460 print( 1461 "IMPORTANT: The changes you are testing need to be locally" 1462 " committed" 1463 ) 1464 print( 1465 "because only the committed changes in the current branch will be" 1466 ) 1467 print("copied to the docker environment.") 1468 time.sleep(5) 1469 1470if args.manual_run and not args.use_docker: 1471 print("--manual_run is only supported with --use_docker option enabled.") 1472 sys.exit(1) 1473 1474if not args.use_docker and servers: 1475 print( 1476 "Running interop servers is only supported with --use_docker option" 1477 " enabled." 1478 ) 1479 sys.exit(1) 1480 1481# we want to include everything but objc in 'all' 1482# because objc won't run on non-mac platforms 1483all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(["objc"]) 1484languages = set( 1485 _LANGUAGES[l] 1486 for l in itertools.chain.from_iterable( 1487 all_but_objc if x == "all" else [x] for x in args.language 1488 ) 1489) 1490# ALTS interop clients are only available for certain languages. 1491if args.transport_security == "alts": 1492 alts_languages = set(_LANGUAGES[l] for l in _LANGUAGES_FOR_ALTS_TEST_CASES) 1493 languages = languages.intersection(alts_languages) 1494 1495languages_http2_clients_for_http2_server_interop = set() 1496if args.http2_server_interop: 1497 languages_http2_clients_for_http2_server_interop = set( 1498 _LANGUAGES[l] 1499 for l in _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES 1500 if "all" in args.language or l in args.language 1501 ) 1502 1503http2Interop = Http2Client() if args.http2_interop else None 1504http2InteropServer = Http2Server() if args.http2_server_interop else None 1505 1506docker_images = {} 1507if args.use_docker: 1508 # languages for which to build docker images 1509 languages_to_build = set( 1510 _LANGUAGES[k] 1511 for k in set([str(l) for l in languages] + [s for s in servers]) 1512 ) 1513 languages_to_build = ( 1514 languages_to_build | languages_http2_clients_for_http2_server_interop 1515 ) 1516 1517 if args.http2_interop: 1518 languages_to_build.add(http2Interop) 1519 1520 if args.http2_server_interop: 1521 languages_to_build.add(http2InteropServer) 1522 1523 build_jobs = [] 1524 for l in languages_to_build: 1525 if str(l) == "objc": 1526 # we don't need to build a docker image for objc 1527 continue 1528 job = build_interop_image_jobspec(l) 1529 docker_images[str(l)] = job.tag 1530 build_jobs.append(job) 1531 1532 if build_jobs: 1533 jobset.message( 1534 "START", "Building interop docker images.", do_newline=True 1535 ) 1536 if args.verbose: 1537 print("Jobs to run: \n%s\n" % "\n".join(str(j) for j in build_jobs)) 1538 1539 num_failures, build_resultset = jobset.run( 1540 build_jobs, newline_on_success=True, maxjobs=args.jobs 1541 ) 1542 1543 report_utils.render_junit_xml_report( 1544 build_resultset, _DOCKER_BUILD_XML_REPORT 1545 ) 1546 1547 if num_failures == 0: 1548 jobset.message( 1549 "SUCCESS", 1550 "All docker images built successfully.", 1551 do_newline=True, 1552 ) 1553 else: 1554 jobset.message( 1555 "FAILED", 1556 "Failed to build interop docker images.", 1557 do_newline=True, 1558 ) 1559 for image in six.itervalues(docker_images): 1560 dockerjob.remove_image(image, skip_nonexistent=True) 1561 sys.exit(1) 1562 1563server_manual_cmd_log = [] if args.manual_run else None 1564client_manual_cmd_log = [] if args.manual_run else None 1565 1566# Start interop servers. 1567server_jobs = {} 1568server_addresses = {} 1569try: 1570 for s in servers: 1571 lang = str(s) 1572 spec = server_jobspec( 1573 _LANGUAGES[lang], 1574 docker_images.get(lang), 1575 args.transport_security, 1576 manual_cmd_log=server_manual_cmd_log, 1577 ) 1578 if not args.manual_run: 1579 job = dockerjob.DockerJob(spec) 1580 server_jobs[lang] = job 1581 server_addresses[lang] = ( 1582 "localhost", 1583 job.mapped_port(_DEFAULT_SERVER_PORT), 1584 ) 1585 else: 1586 # don't run the server, set server port to a placeholder value 1587 server_addresses[lang] = ("localhost", "${SERVER_PORT}") 1588 1589 http2_server_job = None 1590 if args.http2_server_interop: 1591 # launch a HTTP2 server emulator that creates edge cases 1592 lang = str(http2InteropServer) 1593 spec = server_jobspec( 1594 http2InteropServer, 1595 docker_images.get(lang), 1596 manual_cmd_log=server_manual_cmd_log, 1597 ) 1598 if not args.manual_run: 1599 http2_server_job = dockerjob.DockerJob(spec) 1600 server_jobs[lang] = http2_server_job 1601 else: 1602 # don't run the server, set server port to a placeholder value 1603 server_addresses[lang] = ("localhost", "${SERVER_PORT}") 1604 1605 jobs = [] 1606 if args.cloud_to_prod: 1607 if args.transport_security not in ["tls"]: 1608 print("TLS is always enabled for cloud_to_prod scenarios.") 1609 for server_host_nickname in args.prod_servers: 1610 for language in languages: 1611 for test_case in _TEST_CASES: 1612 if not test_case in language.unimplemented_test_cases(): 1613 if ( 1614 not test_case 1615 in _SKIP_ADVANCED 1616 + _SKIP_COMPRESSION 1617 + _SKIP_SPECIAL_STATUS_MESSAGE 1618 + _ORCA_TEST_CASES 1619 ): 1620 for ( 1621 transport_security 1622 ) in args.custom_credentials_type: 1623 # google_default_credentials not yet supported by all languages 1624 if ( 1625 transport_security 1626 == "google_default_credentials" 1627 and str(language) 1628 not in ["c++", "go", "java", "javaokhttp"] 1629 ): 1630 continue 1631 # compute_engine_channel_creds not yet supported by all languages 1632 if ( 1633 transport_security 1634 == "compute_engine_channel_creds" 1635 and str(language) 1636 not in ["go", "java", "javaokhttp"] 1637 ): 1638 continue 1639 test_job = cloud_to_prod_jobspec( 1640 language, 1641 test_case, 1642 server_host_nickname, 1643 prod_servers[server_host_nickname], 1644 google_default_creds_use_key_file=args.google_default_creds_use_key_file, 1645 docker_image=docker_images.get( 1646 str(language) 1647 ), 1648 manual_cmd_log=client_manual_cmd_log, 1649 service_account_key_file=args.service_account_key_file, 1650 default_service_account=args.default_service_account, 1651 transport_security=transport_security, 1652 ) 1653 jobs.append(test_job) 1654 if args.http2_interop: 1655 for test_case in _HTTP2_TEST_CASES: 1656 test_job = cloud_to_prod_jobspec( 1657 http2Interop, 1658 test_case, 1659 server_host_nickname, 1660 prod_servers[server_host_nickname], 1661 google_default_creds_use_key_file=args.google_default_creds_use_key_file, 1662 docker_image=docker_images.get(str(http2Interop)), 1663 manual_cmd_log=client_manual_cmd_log, 1664 service_account_key_file=args.service_account_key_file, 1665 default_service_account=args.default_service_account, 1666 transport_security=args.transport_security, 1667 ) 1668 jobs.append(test_job) 1669 1670 if args.cloud_to_prod_auth: 1671 if args.transport_security not in ["tls"]: 1672 print("TLS is always enabled for cloud_to_prod scenarios.") 1673 for server_host_nickname in args.prod_servers: 1674 for language in languages: 1675 for test_case in _AUTH_TEST_CASES: 1676 if ( 1677 not args.skip_compute_engine_creds 1678 or not compute_engine_creds_required( 1679 language, test_case 1680 ) 1681 ): 1682 if not test_case in language.unimplemented_test_cases(): 1683 if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE: 1684 transport_security = ( 1685 "google_default_credentials" 1686 ) 1687 elif ( 1688 test_case 1689 == _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE 1690 ): 1691 transport_security = ( 1692 "compute_engine_channel_creds" 1693 ) 1694 else: 1695 transport_security = "tls" 1696 if ( 1697 transport_security 1698 not in args.custom_credentials_type 1699 ): 1700 continue 1701 test_job = cloud_to_prod_jobspec( 1702 language, 1703 test_case, 1704 server_host_nickname, 1705 prod_servers[server_host_nickname], 1706 google_default_creds_use_key_file=args.google_default_creds_use_key_file, 1707 docker_image=docker_images.get(str(language)), 1708 auth=True, 1709 manual_cmd_log=client_manual_cmd_log, 1710 service_account_key_file=args.service_account_key_file, 1711 default_service_account=args.default_service_account, 1712 transport_security=transport_security, 1713 ) 1714 jobs.append(test_job) 1715 for server in args.override_server: 1716 server_name = server[0] 1717 (server_host, server_port) = server[1].split(":") 1718 server_addresses[server_name] = (server_host, server_port) 1719 1720 for server_name, server_address in list(server_addresses.items()): 1721 (server_host, server_port) = server_address 1722 server_language = _LANGUAGES.get(server_name, None) 1723 skip_server = [] # test cases unimplemented by server 1724 if server_language: 1725 skip_server = server_language.unimplemented_test_cases_server() 1726 for language in languages: 1727 for test_case in _TEST_CASES: 1728 if not test_case in language.unimplemented_test_cases(): 1729 if not test_case in skip_server: 1730 test_job = cloud_to_cloud_jobspec( 1731 language, 1732 test_case, 1733 server_name, 1734 server_host, 1735 server_port, 1736 docker_image=docker_images.get(str(language)), 1737 transport_security=args.transport_security, 1738 manual_cmd_log=client_manual_cmd_log, 1739 ) 1740 jobs.append(test_job) 1741 1742 if args.http2_interop: 1743 for test_case in _HTTP2_TEST_CASES: 1744 if server_name == "go": 1745 # TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434 1746 continue 1747 test_job = cloud_to_cloud_jobspec( 1748 http2Interop, 1749 test_case, 1750 server_name, 1751 server_host, 1752 server_port, 1753 docker_image=docker_images.get(str(http2Interop)), 1754 transport_security=args.transport_security, 1755 manual_cmd_log=client_manual_cmd_log, 1756 ) 1757 jobs.append(test_job) 1758 1759 if args.http2_server_interop: 1760 if not args.manual_run: 1761 http2_server_job.wait_for_healthy(timeout_seconds=600) 1762 for language in languages_http2_clients_for_http2_server_interop: 1763 for test_case in set(_HTTP2_SERVER_TEST_CASES) - set( 1764 _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS 1765 ): 1766 offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) 1767 server_port = _DEFAULT_SERVER_PORT + offset 1768 if not args.manual_run: 1769 server_port = http2_server_job.mapped_port(server_port) 1770 test_job = cloud_to_cloud_jobspec( 1771 language, 1772 test_case, 1773 str(http2InteropServer), 1774 "localhost", 1775 server_port, 1776 docker_image=docker_images.get(str(language)), 1777 manual_cmd_log=client_manual_cmd_log, 1778 ) 1779 jobs.append(test_job) 1780 for language in languages: 1781 # HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS is a subset of 1782 # HTTP_SERVER_TEST_CASES, in which clients use their gRPC interop clients rather 1783 # than specialized http2 clients, reusing existing test implementations. 1784 # For example, in the "data_frame_padding" test, use language's gRPC 1785 # interop clients and make them think that they're running "large_unary" 1786 # test case. This avoids implementing a new test case in each language. 1787 for test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: 1788 if test_case not in language.unimplemented_test_cases(): 1789 offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) 1790 server_port = _DEFAULT_SERVER_PORT + offset 1791 if not args.manual_run: 1792 server_port = http2_server_job.mapped_port(server_port) 1793 if args.transport_security != "insecure": 1794 print( 1795 "Creating grpc client to http2 server test case " 1796 "with insecure connection, even though " 1797 "args.transport_security is not insecure. Http2 " 1798 "test server only supports insecure connections." 1799 ) 1800 test_job = cloud_to_cloud_jobspec( 1801 language, 1802 test_case, 1803 str(http2InteropServer), 1804 "localhost", 1805 server_port, 1806 docker_image=docker_images.get(str(language)), 1807 transport_security="insecure", 1808 manual_cmd_log=client_manual_cmd_log, 1809 ) 1810 jobs.append(test_job) 1811 1812 if not jobs: 1813 print("No jobs to run.") 1814 for image in six.itervalues(docker_images): 1815 dockerjob.remove_image(image, skip_nonexistent=True) 1816 sys.exit(1) 1817 1818 if args.manual_run: 1819 print("All tests will skipped --manual_run option is active.") 1820 1821 if args.verbose: 1822 print("Jobs to run: \n%s\n" % "\n".join(str(job) for job in jobs)) 1823 1824 num_failures, resultset = jobset.run( 1825 jobs, 1826 newline_on_success=True, 1827 maxjobs=args.jobs, 1828 skip_jobs=args.manual_run, 1829 ) 1830 if args.bq_result_table and resultset: 1831 upload_interop_results_to_bq(resultset, args.bq_result_table) 1832 if num_failures: 1833 jobset.message("FAILED", "Some tests failed", do_newline=True) 1834 else: 1835 jobset.message("SUCCESS", "All tests passed", do_newline=True) 1836 1837 write_cmdlog_maybe(server_manual_cmd_log, "interop_server_cmds.sh") 1838 write_cmdlog_maybe(client_manual_cmd_log, "interop_client_cmds.sh") 1839 1840 report_utils.render_junit_xml_report(resultset, _TESTS_XML_REPORT) 1841 1842 for name, job in list(resultset.items()): 1843 if "http2" in name: 1844 job[0].http2results = aggregate_http2_results(job[0].message) 1845 1846 http2_server_test_cases = ( 1847 _HTTP2_SERVER_TEST_CASES if args.http2_server_interop else [] 1848 ) 1849 1850 if num_failures: 1851 sys.exit(1) 1852 else: 1853 sys.exit(0) 1854finally: 1855 # Check if servers are still running. 1856 for server, job in list(server_jobs.items()): 1857 if not job.is_running(): 1858 print('Server "%s" has exited prematurely.' % server) 1859 1860 dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)]) 1861 1862 for image in six.itervalues(docker_images): 1863 if not args.manual_run: 1864 print("Removing docker image %s" % image) 1865 dockerjob.remove_image(image) 1866 else: 1867 print("Preserving docker image: %s" % image) 1868