1*800a58d9SAndroid Build Coastguard Worker# Copyright 2018 - The Android Open Source Project 2*800a58d9SAndroid Build Coastguard Worker# 3*800a58d9SAndroid Build Coastguard Worker# Licensed under the Apache License, Version 2.0 (the "License"); 4*800a58d9SAndroid Build Coastguard Worker# you may not use this file except in compliance with the License. 5*800a58d9SAndroid Build Coastguard Worker# You may obtain a copy of the License at 6*800a58d9SAndroid Build Coastguard Worker# 7*800a58d9SAndroid Build Coastguard Worker# http://www.apache.org/licenses/LICENSE-2.0 8*800a58d9SAndroid Build Coastguard Worker# 9*800a58d9SAndroid Build Coastguard Worker# Unless required by applicable law or agreed to in writing, software 10*800a58d9SAndroid Build Coastguard Worker# distributed under the License is distributed on an "AS IS" BASIS, 11*800a58d9SAndroid Build Coastguard Worker# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12*800a58d9SAndroid Build Coastguard Worker# See the License for the specific language governing permissions and 13*800a58d9SAndroid Build Coastguard Worker# limitations under the License. 14*800a58d9SAndroid Build Coastguard Workerr"""Reconnect entry point. 15*800a58d9SAndroid Build Coastguard Worker 16*800a58d9SAndroid Build Coastguard WorkerReconnect will: 17*800a58d9SAndroid Build Coastguard Worker - re-establish ssh tunnels for adb/vnc port forwarding for a remote instance 18*800a58d9SAndroid Build Coastguard Worker - adb connect to forwarded ssh port for remote instance 19*800a58d9SAndroid Build Coastguard Worker - restart vnc for remote/local instances 20*800a58d9SAndroid Build Coastguard Worker""" 21*800a58d9SAndroid Build Coastguard Worker 22*800a58d9SAndroid Build Coastguard Workerimport logging 23*800a58d9SAndroid Build Coastguard Workerimport os 24*800a58d9SAndroid Build Coastguard Workerimport re 25*800a58d9SAndroid Build Coastguard Worker 26*800a58d9SAndroid Build Coastguard Workerfrom acloud import errors 27*800a58d9SAndroid Build Coastguard Workerfrom acloud.internal import constants 28*800a58d9SAndroid Build Coastguard Workerfrom acloud.internal.lib import auth 29*800a58d9SAndroid Build Coastguard Workerfrom acloud.internal.lib import android_compute_client 30*800a58d9SAndroid Build Coastguard Workerfrom acloud.internal.lib import cvd_runtime_config 31*800a58d9SAndroid Build Coastguard Workerfrom acloud.internal.lib import gcompute_client 32*800a58d9SAndroid Build Coastguard Workerfrom acloud.internal.lib import utils 33*800a58d9SAndroid Build Coastguard Workerfrom acloud.internal.lib import ssh as ssh_object 34*800a58d9SAndroid Build Coastguard Workerfrom acloud.internal.lib.adb_tools import AdbTools 35*800a58d9SAndroid Build Coastguard Workerfrom acloud.list import list as list_instance 36*800a58d9SAndroid Build Coastguard Workerfrom acloud.public import config 37*800a58d9SAndroid Build Coastguard Workerfrom acloud.public import report 38*800a58d9SAndroid Build Coastguard Worker 39*800a58d9SAndroid Build Coastguard Worker 40*800a58d9SAndroid Build Coastguard Workerlogger = logging.getLogger(__name__) 41*800a58d9SAndroid Build Coastguard Worker 42*800a58d9SAndroid Build Coastguard Worker_RE_DISPLAY = re.compile(r"([\d]+)x([\d]+)\s.*") 43*800a58d9SAndroid Build Coastguard Worker_VNC_STARTED_PATTERN = "ssvnc vnc://127.0.0.1:%(vnc_port)d" 44*800a58d9SAndroid Build Coastguard Worker 45*800a58d9SAndroid Build Coastguard Worker 46*800a58d9SAndroid Build Coastguard Workerdef _IsWebrtcEnable(instance, host_user, host_ssh_private_key_path, 47*800a58d9SAndroid Build Coastguard Worker extra_args_ssh_tunnel): 48*800a58d9SAndroid Build Coastguard Worker """Check local/remote instance webRTC is enable. 49*800a58d9SAndroid Build Coastguard Worker 50*800a58d9SAndroid Build Coastguard Worker Args: 51*800a58d9SAndroid Build Coastguard Worker instance: Local/Remote Instance object. 52*800a58d9SAndroid Build Coastguard Worker host_user: String of user login into the instance. 53*800a58d9SAndroid Build Coastguard Worker host_ssh_private_key_path: String of host key for logging in to the 54*800a58d9SAndroid Build Coastguard Worker host. 55*800a58d9SAndroid Build Coastguard Worker extra_args_ssh_tunnel: String, extra args for ssh tunnel connection. 56*800a58d9SAndroid Build Coastguard Worker 57*800a58d9SAndroid Build Coastguard Worker Returns: 58*800a58d9SAndroid Build Coastguard Worker Boolean: True if cf_runtime_cfg.enable_webrtc is True. 59*800a58d9SAndroid Build Coastguard Worker """ 60*800a58d9SAndroid Build Coastguard Worker if instance.islocal: 61*800a58d9SAndroid Build Coastguard Worker return instance.cf_runtime_cfg.enable_webrtc 62*800a58d9SAndroid Build Coastguard Worker ssh = ssh_object.Ssh(ip=ssh_object.IP(ip=instance.ip), user=host_user, 63*800a58d9SAndroid Build Coastguard Worker ssh_private_key_path=host_ssh_private_key_path, 64*800a58d9SAndroid Build Coastguard Worker extra_args_ssh_tunnel=extra_args_ssh_tunnel) 65*800a58d9SAndroid Build Coastguard Worker remote_cuttlefish_config = os.path.join(constants.REMOTE_LOG_FOLDER, 66*800a58d9SAndroid Build Coastguard Worker constants.CUTTLEFISH_CONFIG_FILE) 67*800a58d9SAndroid Build Coastguard Worker raw_data = ssh.GetCmdOutput("cat " + remote_cuttlefish_config) 68*800a58d9SAndroid Build Coastguard Worker try: 69*800a58d9SAndroid Build Coastguard Worker cf_runtime_cfg = cvd_runtime_config.CvdRuntimeConfig( 70*800a58d9SAndroid Build Coastguard Worker raw_data=raw_data.strip()) 71*800a58d9SAndroid Build Coastguard Worker return cf_runtime_cfg.enable_webrtc 72*800a58d9SAndroid Build Coastguard Worker except errors.ConfigError: 73*800a58d9SAndroid Build Coastguard Worker logger.debug("No cuttlefish config[%s] found!", 74*800a58d9SAndroid Build Coastguard Worker remote_cuttlefish_config) 75*800a58d9SAndroid Build Coastguard Worker return False 76*800a58d9SAndroid Build Coastguard Worker 77*800a58d9SAndroid Build Coastguard Worker 78*800a58d9SAndroid Build Coastguard Workerdef StartVnc(vnc_port, display): 79*800a58d9SAndroid Build Coastguard Worker """Start vnc connect to AVD. 80*800a58d9SAndroid Build Coastguard Worker 81*800a58d9SAndroid Build Coastguard Worker Confirm whether there is already a connection before VNC connection. 82*800a58d9SAndroid Build Coastguard Worker If there is a connection, it will not be connected. If not, connect it. 83*800a58d9SAndroid Build Coastguard Worker Before reconnecting, clear old disconnect ssvnc viewer. 84*800a58d9SAndroid Build Coastguard Worker 85*800a58d9SAndroid Build Coastguard Worker Args: 86*800a58d9SAndroid Build Coastguard Worker vnc_port: Integer of vnc port number. 87*800a58d9SAndroid Build Coastguard Worker display: String, vnc connection resolution. e.g., 1080x720 (240) 88*800a58d9SAndroid Build Coastguard Worker """ 89*800a58d9SAndroid Build Coastguard Worker vnc_started_pattern = _VNC_STARTED_PATTERN % {"vnc_port": vnc_port} 90*800a58d9SAndroid Build Coastguard Worker if not utils.IsCommandRunning(vnc_started_pattern): 91*800a58d9SAndroid Build Coastguard Worker #clean old disconnect ssvnc viewer. 92*800a58d9SAndroid Build Coastguard Worker utils.CleanupSSVncviewer(vnc_port) 93*800a58d9SAndroid Build Coastguard Worker 94*800a58d9SAndroid Build Coastguard Worker match = _RE_DISPLAY.match(display) 95*800a58d9SAndroid Build Coastguard Worker if match: 96*800a58d9SAndroid Build Coastguard Worker utils.LaunchVncClient(vnc_port, match.group(1), match.group(2)) 97*800a58d9SAndroid Build Coastguard Worker else: 98*800a58d9SAndroid Build Coastguard Worker utils.LaunchVncClient(vnc_port) 99*800a58d9SAndroid Build Coastguard Worker 100*800a58d9SAndroid Build Coastguard Worker 101*800a58d9SAndroid Build Coastguard Workerdef AddPublicSshRsaToInstance(cfg, user, instance_name): 102*800a58d9SAndroid Build Coastguard Worker """Add the public rsa key to the instance's metadata. 103*800a58d9SAndroid Build Coastguard Worker 104*800a58d9SAndroid Build Coastguard Worker When the public key doesn't exist in the metadata, it will add it. 105*800a58d9SAndroid Build Coastguard Worker 106*800a58d9SAndroid Build Coastguard Worker Args: 107*800a58d9SAndroid Build Coastguard Worker cfg: An AcloudConfig instance. 108*800a58d9SAndroid Build Coastguard Worker user: String, the ssh username to access instance. 109*800a58d9SAndroid Build Coastguard Worker instance_name: String, instance name. 110*800a58d9SAndroid Build Coastguard Worker """ 111*800a58d9SAndroid Build Coastguard Worker credentials = auth.CreateCredentials(cfg) 112*800a58d9SAndroid Build Coastguard Worker compute_client = android_compute_client.AndroidComputeClient( 113*800a58d9SAndroid Build Coastguard Worker cfg, credentials) 114*800a58d9SAndroid Build Coastguard Worker compute_client.AddSshRsaInstanceMetadata( 115*800a58d9SAndroid Build Coastguard Worker user, 116*800a58d9SAndroid Build Coastguard Worker cfg.ssh_public_key_path, 117*800a58d9SAndroid Build Coastguard Worker instance_name) 118*800a58d9SAndroid Build Coastguard Worker 119*800a58d9SAndroid Build Coastguard Worker 120*800a58d9SAndroid Build Coastguard Worker@utils.TimeExecute(function_description="Reconnect instances") 121*800a58d9SAndroid Build Coastguard Workerdef ReconnectInstance(ssh_private_key_path, 122*800a58d9SAndroid Build Coastguard Worker instance, 123*800a58d9SAndroid Build Coastguard Worker reconnect_report, 124*800a58d9SAndroid Build Coastguard Worker extra_args_ssh_tunnel=None, 125*800a58d9SAndroid Build Coastguard Worker autoconnect=None, 126*800a58d9SAndroid Build Coastguard Worker connect_hostname=None): 127*800a58d9SAndroid Build Coastguard Worker """Reconnect to the specified instance. 128*800a58d9SAndroid Build Coastguard Worker 129*800a58d9SAndroid Build Coastguard Worker It will: 130*800a58d9SAndroid Build Coastguard Worker - re-establish ssh tunnels for adb/vnc port forwarding 131*800a58d9SAndroid Build Coastguard Worker - re-establish adb connection 132*800a58d9SAndroid Build Coastguard Worker - restart vnc client 133*800a58d9SAndroid Build Coastguard Worker - update device information in reconnect_report 134*800a58d9SAndroid Build Coastguard Worker 135*800a58d9SAndroid Build Coastguard Worker Args: 136*800a58d9SAndroid Build Coastguard Worker ssh_private_key_path: Path to the private key file. 137*800a58d9SAndroid Build Coastguard Worker e.g. ~/.ssh/acloud_rsa 138*800a58d9SAndroid Build Coastguard Worker instance: list.Instance() object. 139*800a58d9SAndroid Build Coastguard Worker reconnect_report: Report object. 140*800a58d9SAndroid Build Coastguard Worker extra_args_ssh_tunnel: String, extra args for ssh tunnel connection. 141*800a58d9SAndroid Build Coastguard Worker autoconnect: String, for decide whether to launch vnc/browser or not. 142*800a58d9SAndroid Build Coastguard Worker connect_hostname: String, the hostname for ssh connect. 143*800a58d9SAndroid Build Coastguard Worker 144*800a58d9SAndroid Build Coastguard Worker Raises: 145*800a58d9SAndroid Build Coastguard Worker errors.UnknownAvdType: Unable to reconnect to instance of unknown avd 146*800a58d9SAndroid Build Coastguard Worker type. 147*800a58d9SAndroid Build Coastguard Worker """ 148*800a58d9SAndroid Build Coastguard Worker if instance.avd_type not in utils.AVD_PORT_DICT: 149*800a58d9SAndroid Build Coastguard Worker raise errors.UnknownAvdType("Unable to reconnect to instance (%s) of " 150*800a58d9SAndroid Build Coastguard Worker "unknown avd type: %s" % 151*800a58d9SAndroid Build Coastguard Worker (instance.name, instance.avd_type)) 152*800a58d9SAndroid Build Coastguard Worker # Ignore extra ssh tunnel to connect with hostname. 153*800a58d9SAndroid Build Coastguard Worker if connect_hostname: 154*800a58d9SAndroid Build Coastguard Worker extra_args_ssh_tunnel = None 155*800a58d9SAndroid Build Coastguard Worker 156*800a58d9SAndroid Build Coastguard Worker adb_cmd = AdbTools(instance.adb_port) 157*800a58d9SAndroid Build Coastguard Worker vnc_port = instance.vnc_port 158*800a58d9SAndroid Build Coastguard Worker adb_port = instance.adb_port 159*800a58d9SAndroid Build Coastguard Worker webrtc_port = instance.webrtc_port 160*800a58d9SAndroid Build Coastguard Worker # ssh tunnel is up but device is disconnected on adb 161*800a58d9SAndroid Build Coastguard Worker if instance.ssh_tunnel_is_connected and not adb_cmd.IsAdbConnectionAlive(): 162*800a58d9SAndroid Build Coastguard Worker adb_cmd.DisconnectAdb() 163*800a58d9SAndroid Build Coastguard Worker adb_cmd.ConnectAdb() 164*800a58d9SAndroid Build Coastguard Worker # ssh tunnel is down and it's a remote instance 165*800a58d9SAndroid Build Coastguard Worker elif not instance.ssh_tunnel_is_connected and not instance.islocal: 166*800a58d9SAndroid Build Coastguard Worker adb_cmd.DisconnectAdb() 167*800a58d9SAndroid Build Coastguard Worker forwarded_ports = utils.AutoConnect( 168*800a58d9SAndroid Build Coastguard Worker ip_addr=connect_hostname or instance.ip, 169*800a58d9SAndroid Build Coastguard Worker rsa_key_file=ssh_private_key_path, 170*800a58d9SAndroid Build Coastguard Worker target_vnc_port=utils.AVD_PORT_DICT[instance.avd_type].vnc_port, 171*800a58d9SAndroid Build Coastguard Worker target_adb_port=utils.AVD_PORT_DICT[instance.avd_type].adb_port, 172*800a58d9SAndroid Build Coastguard Worker ssh_user=constants.GCE_USER, 173*800a58d9SAndroid Build Coastguard Worker extra_args_ssh_tunnel=extra_args_ssh_tunnel) 174*800a58d9SAndroid Build Coastguard Worker vnc_port = forwarded_ports.vnc_port 175*800a58d9SAndroid Build Coastguard Worker adb_port = forwarded_ports.adb_port 176*800a58d9SAndroid Build Coastguard Worker if autoconnect is constants.INS_KEY_WEBRTC: 177*800a58d9SAndroid Build Coastguard Worker if not instance.islocal: 178*800a58d9SAndroid Build Coastguard Worker webrtc_port = utils.GetWebrtcPortFromSSHTunnel(instance.ip) 179*800a58d9SAndroid Build Coastguard Worker if not webrtc_port: 180*800a58d9SAndroid Build Coastguard Worker webrtc_port = utils.PickFreePort() 181*800a58d9SAndroid Build Coastguard Worker utils.EstablishWebRTCSshTunnel( 182*800a58d9SAndroid Build Coastguard Worker ip_addr=connect_hostname or instance.ip, 183*800a58d9SAndroid Build Coastguard Worker webrtc_local_port=webrtc_port, 184*800a58d9SAndroid Build Coastguard Worker rsa_key_file=ssh_private_key_path, 185*800a58d9SAndroid Build Coastguard Worker ssh_user=constants.GCE_USER, 186*800a58d9SAndroid Build Coastguard Worker extra_args_ssh_tunnel=extra_args_ssh_tunnel) 187*800a58d9SAndroid Build Coastguard Worker utils.LaunchBrowser(constants.WEBRTC_LOCAL_HOST, 188*800a58d9SAndroid Build Coastguard Worker webrtc_port) 189*800a58d9SAndroid Build Coastguard Worker elif vnc_port and autoconnect is constants.INS_KEY_VNC: 190*800a58d9SAndroid Build Coastguard Worker StartVnc(vnc_port, instance.display) 191*800a58d9SAndroid Build Coastguard Worker 192*800a58d9SAndroid Build Coastguard Worker device_dict = { 193*800a58d9SAndroid Build Coastguard Worker constants.IP: instance.ip, 194*800a58d9SAndroid Build Coastguard Worker constants.INSTANCE_NAME: instance.name, 195*800a58d9SAndroid Build Coastguard Worker constants.VNC_PORT: vnc_port, 196*800a58d9SAndroid Build Coastguard Worker constants.ADB_PORT: adb_port 197*800a58d9SAndroid Build Coastguard Worker } 198*800a58d9SAndroid Build Coastguard Worker if adb_port and not instance.islocal: 199*800a58d9SAndroid Build Coastguard Worker device_dict[constants.DEVICE_SERIAL] = ( 200*800a58d9SAndroid Build Coastguard Worker constants.REMOTE_INSTANCE_ADB_SERIAL % adb_port) 201*800a58d9SAndroid Build Coastguard Worker 202*800a58d9SAndroid Build Coastguard Worker if (vnc_port or webrtc_port) and adb_port: 203*800a58d9SAndroid Build Coastguard Worker reconnect_report.AddData(key="devices", value=device_dict) 204*800a58d9SAndroid Build Coastguard Worker else: 205*800a58d9SAndroid Build Coastguard Worker # We use 'ps aux' to grep adb/vnc fowarding port from ssh tunnel 206*800a58d9SAndroid Build Coastguard Worker # command. Therefore we report failure here if no vnc_port and 207*800a58d9SAndroid Build Coastguard Worker # adb_port found. 208*800a58d9SAndroid Build Coastguard Worker reconnect_report.AddData(key="device_failing_reconnect", value=device_dict) 209*800a58d9SAndroid Build Coastguard Worker reconnect_report.AddError(instance.name) 210*800a58d9SAndroid Build Coastguard Worker 211*800a58d9SAndroid Build Coastguard Worker 212*800a58d9SAndroid Build Coastguard Workerdef GetSshConnectHostname(cfg, instance): 213*800a58d9SAndroid Build Coastguard Worker """Get ssh connect hostname. 214*800a58d9SAndroid Build Coastguard Worker 215*800a58d9SAndroid Build Coastguard Worker Get GCE hostname with specific rule for cloudtop users. 216*800a58d9SAndroid Build Coastguard Worker 217*800a58d9SAndroid Build Coastguard Worker Args: 218*800a58d9SAndroid Build Coastguard Worker cfg: AcloudConfig object. 219*800a58d9SAndroid Build Coastguard Worker instance: list.Instance() object. 220*800a58d9SAndroid Build Coastguard Worker 221*800a58d9SAndroid Build Coastguard Worker Returns: 222*800a58d9SAndroid Build Coastguard Worker String of hostname for ssh connect. None is for not connect with 223*800a58d9SAndroid Build Coastguard Worker hostname such as local instance mode. 224*800a58d9SAndroid Build Coastguard Worker """ 225*800a58d9SAndroid Build Coastguard Worker if instance.islocal: 226*800a58d9SAndroid Build Coastguard Worker return None 227*800a58d9SAndroid Build Coastguard Worker if cfg.connect_hostname: 228*800a58d9SAndroid Build Coastguard Worker return gcompute_client.GetGCEHostName( 229*800a58d9SAndroid Build Coastguard Worker cfg.project, instance.name, cfg.zone) 230*800a58d9SAndroid Build Coastguard Worker return None 231*800a58d9SAndroid Build Coastguard Worker 232*800a58d9SAndroid Build Coastguard Worker 233*800a58d9SAndroid Build Coastguard Workerdef Run(args): 234*800a58d9SAndroid Build Coastguard Worker """Run reconnect. 235*800a58d9SAndroid Build Coastguard Worker 236*800a58d9SAndroid Build Coastguard Worker Args: 237*800a58d9SAndroid Build Coastguard Worker args: Namespace object from argparse.parse_args. 238*800a58d9SAndroid Build Coastguard Worker """ 239*800a58d9SAndroid Build Coastguard Worker cfg = config.GetAcloudConfig(args) 240*800a58d9SAndroid Build Coastguard Worker instances_to_reconnect = [] 241*800a58d9SAndroid Build Coastguard Worker if args.instance_names is not None: 242*800a58d9SAndroid Build Coastguard Worker # user input instance name to get instance object. 243*800a58d9SAndroid Build Coastguard Worker instances_to_reconnect = list_instance.GetInstancesFromInstanceNames( 244*800a58d9SAndroid Build Coastguard Worker cfg, args.instance_names) 245*800a58d9SAndroid Build Coastguard Worker if not instances_to_reconnect: 246*800a58d9SAndroid Build Coastguard Worker instances_to_reconnect = list_instance.ChooseInstances(cfg, args.all) 247*800a58d9SAndroid Build Coastguard Worker 248*800a58d9SAndroid Build Coastguard Worker reconnect_report = report.Report(command="reconnect") 249*800a58d9SAndroid Build Coastguard Worker for instance in instances_to_reconnect: 250*800a58d9SAndroid Build Coastguard Worker if instance.avd_type not in utils.AVD_PORT_DICT: 251*800a58d9SAndroid Build Coastguard Worker utils.PrintColorString("Skipping reconnect of instance %s due to " 252*800a58d9SAndroid Build Coastguard Worker "unknown avd type (%s)." % 253*800a58d9SAndroid Build Coastguard Worker (instance.name, instance.avd_type), 254*800a58d9SAndroid Build Coastguard Worker utils.TextColors.WARNING) 255*800a58d9SAndroid Build Coastguard Worker continue 256*800a58d9SAndroid Build Coastguard Worker if not instance.islocal: 257*800a58d9SAndroid Build Coastguard Worker AddPublicSshRsaToInstance(cfg, constants.GCE_USER, instance.name) 258*800a58d9SAndroid Build Coastguard Worker ReconnectInstance(cfg.ssh_private_key_path, 259*800a58d9SAndroid Build Coastguard Worker instance, 260*800a58d9SAndroid Build Coastguard Worker reconnect_report, 261*800a58d9SAndroid Build Coastguard Worker cfg.extra_args_ssh_tunnel, 262*800a58d9SAndroid Build Coastguard Worker autoconnect=(args.autoconnect or instance.autoconnect), 263*800a58d9SAndroid Build Coastguard Worker connect_hostname=GetSshConnectHostname(cfg, instance)) 264*800a58d9SAndroid Build Coastguard Worker 265*800a58d9SAndroid Build Coastguard Worker utils.PrintDeviceSummary(reconnect_report) 266