1*8975f5c5SAndroid Build Coastguard Worker#!/usr/bin/env vpython3 2*8975f5c5SAndroid Build Coastguard Worker# 3*8975f5c5SAndroid Build Coastguard Worker# Copyright 2013 The Chromium Authors 4*8975f5c5SAndroid Build Coastguard Worker# Use of this source code is governed by a BSD-style license that can be 5*8975f5c5SAndroid Build Coastguard Worker# found in the LICENSE file. 6*8975f5c5SAndroid Build Coastguard Worker# 7*8975f5c5SAndroid Build Coastguard Worker# Find the most recent tombstone file(s) on all connected devices 8*8975f5c5SAndroid Build Coastguard Worker# and prints their stacks. 9*8975f5c5SAndroid Build Coastguard Worker# 10*8975f5c5SAndroid Build Coastguard Worker# Assumes tombstone file was created with current symbols. 11*8975f5c5SAndroid Build Coastguard Worker 12*8975f5c5SAndroid Build Coastguard Workerimport argparse 13*8975f5c5SAndroid Build Coastguard Workerimport datetime 14*8975f5c5SAndroid Build Coastguard Workerimport logging 15*8975f5c5SAndroid Build Coastguard Workerimport os 16*8975f5c5SAndroid Build Coastguard Workerimport sys 17*8975f5c5SAndroid Build Coastguard Worker 18*8975f5c5SAndroid Build Coastguard Workerfrom multiprocessing.pool import ThreadPool 19*8975f5c5SAndroid Build Coastguard Worker 20*8975f5c5SAndroid Build Coastguard Workerimport devil_chromium 21*8975f5c5SAndroid Build Coastguard Worker 22*8975f5c5SAndroid Build Coastguard Workerfrom devil.android import device_denylist 23*8975f5c5SAndroid Build Coastguard Workerfrom devil.android import device_errors 24*8975f5c5SAndroid Build Coastguard Workerfrom devil.android import device_utils 25*8975f5c5SAndroid Build Coastguard Workerfrom devil.utils import run_tests_helper 26*8975f5c5SAndroid Build Coastguard Workerfrom pylib import constants 27*8975f5c5SAndroid Build Coastguard Workerfrom pylib.symbols import stack_symbolizer 28*8975f5c5SAndroid Build Coastguard Worker 29*8975f5c5SAndroid Build Coastguard Worker 30*8975f5c5SAndroid Build Coastguard Worker_TZ_UTC = {'TZ': 'UTC'} 31*8975f5c5SAndroid Build Coastguard Worker 32*8975f5c5SAndroid Build Coastguard Worker 33*8975f5c5SAndroid Build Coastguard Workerdef _ListTombstones(device): 34*8975f5c5SAndroid Build Coastguard Worker """List the tombstone files on the device. 35*8975f5c5SAndroid Build Coastguard Worker 36*8975f5c5SAndroid Build Coastguard Worker Args: 37*8975f5c5SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 38*8975f5c5SAndroid Build Coastguard Worker 39*8975f5c5SAndroid Build Coastguard Worker Yields: 40*8975f5c5SAndroid Build Coastguard Worker Tuples of (tombstone filename, date time of file on device). 41*8975f5c5SAndroid Build Coastguard Worker """ 42*8975f5c5SAndroid Build Coastguard Worker try: 43*8975f5c5SAndroid Build Coastguard Worker if not device.PathExists('/data/tombstones', as_root=True): 44*8975f5c5SAndroid Build Coastguard Worker return 45*8975f5c5SAndroid Build Coastguard Worker entries = device.StatDirectory('/data/tombstones', as_root=True) 46*8975f5c5SAndroid Build Coastguard Worker for entry in entries: 47*8975f5c5SAndroid Build Coastguard Worker if 'tombstone' in entry['filename']: 48*8975f5c5SAndroid Build Coastguard Worker yield (entry['filename'], 49*8975f5c5SAndroid Build Coastguard Worker datetime.datetime.fromtimestamp(entry['st_mtime'])) 50*8975f5c5SAndroid Build Coastguard Worker except device_errors.CommandFailedError: 51*8975f5c5SAndroid Build Coastguard Worker logging.exception('Could not retrieve tombstones.') 52*8975f5c5SAndroid Build Coastguard Worker except device_errors.DeviceUnreachableError: 53*8975f5c5SAndroid Build Coastguard Worker logging.exception('Device unreachable retrieving tombstones.') 54*8975f5c5SAndroid Build Coastguard Worker except device_errors.CommandTimeoutError: 55*8975f5c5SAndroid Build Coastguard Worker logging.exception('Timed out retrieving tombstones.') 56*8975f5c5SAndroid Build Coastguard Worker 57*8975f5c5SAndroid Build Coastguard Worker 58*8975f5c5SAndroid Build Coastguard Workerdef _GetDeviceDateTime(device): 59*8975f5c5SAndroid Build Coastguard Worker """Determine the date time on the device. 60*8975f5c5SAndroid Build Coastguard Worker 61*8975f5c5SAndroid Build Coastguard Worker Args: 62*8975f5c5SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 63*8975f5c5SAndroid Build Coastguard Worker 64*8975f5c5SAndroid Build Coastguard Worker Returns: 65*8975f5c5SAndroid Build Coastguard Worker A datetime instance. 66*8975f5c5SAndroid Build Coastguard Worker """ 67*8975f5c5SAndroid Build Coastguard Worker device_now_string = device.RunShellCommand( 68*8975f5c5SAndroid Build Coastguard Worker ['date'], check_return=True, env=_TZ_UTC) 69*8975f5c5SAndroid Build Coastguard Worker return datetime.datetime.strptime( 70*8975f5c5SAndroid Build Coastguard Worker device_now_string[0], '%a %b %d %H:%M:%S %Z %Y') 71*8975f5c5SAndroid Build Coastguard Worker 72*8975f5c5SAndroid Build Coastguard Worker 73*8975f5c5SAndroid Build Coastguard Workerdef _GetTombstoneData(device, tombstone_file): 74*8975f5c5SAndroid Build Coastguard Worker """Retrieve the tombstone data from the device 75*8975f5c5SAndroid Build Coastguard Worker 76*8975f5c5SAndroid Build Coastguard Worker Args: 77*8975f5c5SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 78*8975f5c5SAndroid Build Coastguard Worker tombstone_file: the tombstone to retrieve 79*8975f5c5SAndroid Build Coastguard Worker 80*8975f5c5SAndroid Build Coastguard Worker Returns: 81*8975f5c5SAndroid Build Coastguard Worker A list of lines 82*8975f5c5SAndroid Build Coastguard Worker """ 83*8975f5c5SAndroid Build Coastguard Worker return device.ReadFile( 84*8975f5c5SAndroid Build Coastguard Worker '/data/tombstones/' + tombstone_file, as_root=True).splitlines() 85*8975f5c5SAndroid Build Coastguard Worker 86*8975f5c5SAndroid Build Coastguard Worker 87*8975f5c5SAndroid Build Coastguard Workerdef _EraseTombstone(device, tombstone_file): 88*8975f5c5SAndroid Build Coastguard Worker """Deletes a tombstone from the device. 89*8975f5c5SAndroid Build Coastguard Worker 90*8975f5c5SAndroid Build Coastguard Worker Args: 91*8975f5c5SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 92*8975f5c5SAndroid Build Coastguard Worker tombstone_file: the tombstone to delete. 93*8975f5c5SAndroid Build Coastguard Worker """ 94*8975f5c5SAndroid Build Coastguard Worker return device.RunShellCommand( 95*8975f5c5SAndroid Build Coastguard Worker ['rm', '/data/tombstones/' + tombstone_file], 96*8975f5c5SAndroid Build Coastguard Worker as_root=True, check_return=True) 97*8975f5c5SAndroid Build Coastguard Worker 98*8975f5c5SAndroid Build Coastguard Worker 99*8975f5c5SAndroid Build Coastguard Workerdef _ResolveTombstone(args): 100*8975f5c5SAndroid Build Coastguard Worker tombstone = args[0] 101*8975f5c5SAndroid Build Coastguard Worker tombstone_symbolizer = args[1] 102*8975f5c5SAndroid Build Coastguard Worker lines = [] 103*8975f5c5SAndroid Build Coastguard Worker lines += [tombstone['file'] + ' created on ' + str(tombstone['time']) + 104*8975f5c5SAndroid Build Coastguard Worker ', about this long ago: ' + 105*8975f5c5SAndroid Build Coastguard Worker (str(tombstone['device_now'] - tombstone['time']) + 106*8975f5c5SAndroid Build Coastguard Worker ' Device: ' + tombstone['serial'])] 107*8975f5c5SAndroid Build Coastguard Worker logging.info('\n'.join(lines)) 108*8975f5c5SAndroid Build Coastguard Worker logging.info('Resolving...') 109*8975f5c5SAndroid Build Coastguard Worker lines += tombstone_symbolizer.ExtractAndResolveNativeStackTraces( 110*8975f5c5SAndroid Build Coastguard Worker tombstone['data'], 111*8975f5c5SAndroid Build Coastguard Worker tombstone['device_abi'], 112*8975f5c5SAndroid Build Coastguard Worker tombstone['stack']) 113*8975f5c5SAndroid Build Coastguard Worker return lines 114*8975f5c5SAndroid Build Coastguard Worker 115*8975f5c5SAndroid Build Coastguard Worker 116*8975f5c5SAndroid Build Coastguard Workerdef _ResolveTombstones(jobs, tombstones, tombstone_symbolizer): 117*8975f5c5SAndroid Build Coastguard Worker """Resolve a list of tombstones. 118*8975f5c5SAndroid Build Coastguard Worker 119*8975f5c5SAndroid Build Coastguard Worker Args: 120*8975f5c5SAndroid Build Coastguard Worker jobs: the number of jobs to use with multithread. 121*8975f5c5SAndroid Build Coastguard Worker tombstones: a list of tombstones. 122*8975f5c5SAndroid Build Coastguard Worker """ 123*8975f5c5SAndroid Build Coastguard Worker if not tombstones: 124*8975f5c5SAndroid Build Coastguard Worker logging.warning('No tombstones to resolve.') 125*8975f5c5SAndroid Build Coastguard Worker return [] 126*8975f5c5SAndroid Build Coastguard Worker if len(tombstones) == 1: 127*8975f5c5SAndroid Build Coastguard Worker data = [_ResolveTombstone([tombstones[0], tombstone_symbolizer])] 128*8975f5c5SAndroid Build Coastguard Worker else: 129*8975f5c5SAndroid Build Coastguard Worker pool = ThreadPool(jobs) 130*8975f5c5SAndroid Build Coastguard Worker data = pool.map( 131*8975f5c5SAndroid Build Coastguard Worker _ResolveTombstone, 132*8975f5c5SAndroid Build Coastguard Worker [[tombstone, tombstone_symbolizer] for tombstone in tombstones]) 133*8975f5c5SAndroid Build Coastguard Worker pool.close() 134*8975f5c5SAndroid Build Coastguard Worker pool.join() 135*8975f5c5SAndroid Build Coastguard Worker resolved_tombstones = [] 136*8975f5c5SAndroid Build Coastguard Worker for tombstone in data: 137*8975f5c5SAndroid Build Coastguard Worker resolved_tombstones.extend(tombstone) 138*8975f5c5SAndroid Build Coastguard Worker return resolved_tombstones 139*8975f5c5SAndroid Build Coastguard Worker 140*8975f5c5SAndroid Build Coastguard Worker 141*8975f5c5SAndroid Build Coastguard Workerdef _GetTombstonesForDevice(device, resolve_all_tombstones, 142*8975f5c5SAndroid Build Coastguard Worker include_stack_symbols, 143*8975f5c5SAndroid Build Coastguard Worker wipe_tombstones): 144*8975f5c5SAndroid Build Coastguard Worker """Returns a list of tombstones on a given device. 145*8975f5c5SAndroid Build Coastguard Worker 146*8975f5c5SAndroid Build Coastguard Worker Args: 147*8975f5c5SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 148*8975f5c5SAndroid Build Coastguard Worker resolve_all_tombstone: Whether to resolve every tombstone. 149*8975f5c5SAndroid Build Coastguard Worker include_stack_symbols: Whether to include symbols for stack data. 150*8975f5c5SAndroid Build Coastguard Worker wipe_tombstones: Whether to wipe tombstones. 151*8975f5c5SAndroid Build Coastguard Worker """ 152*8975f5c5SAndroid Build Coastguard Worker ret = [] 153*8975f5c5SAndroid Build Coastguard Worker all_tombstones = list(_ListTombstones(device)) 154*8975f5c5SAndroid Build Coastguard Worker if not all_tombstones: 155*8975f5c5SAndroid Build Coastguard Worker logging.warning('No tombstones.') 156*8975f5c5SAndroid Build Coastguard Worker return ret 157*8975f5c5SAndroid Build Coastguard Worker 158*8975f5c5SAndroid Build Coastguard Worker # Sort the tombstones in date order, descending 159*8975f5c5SAndroid Build Coastguard Worker all_tombstones.sort(key=lambda a: a[1], reverse=True) 160*8975f5c5SAndroid Build Coastguard Worker 161*8975f5c5SAndroid Build Coastguard Worker # Only resolve the most recent unless --all-tombstones given. 162*8975f5c5SAndroid Build Coastguard Worker tombstones = all_tombstones if resolve_all_tombstones else [all_tombstones[0]] 163*8975f5c5SAndroid Build Coastguard Worker 164*8975f5c5SAndroid Build Coastguard Worker device_now = _GetDeviceDateTime(device) 165*8975f5c5SAndroid Build Coastguard Worker try: 166*8975f5c5SAndroid Build Coastguard Worker for tombstone_file, tombstone_time in tombstones: 167*8975f5c5SAndroid Build Coastguard Worker ret += [{'serial': str(device), 168*8975f5c5SAndroid Build Coastguard Worker 'device_abi': device.product_cpu_abi, 169*8975f5c5SAndroid Build Coastguard Worker 'device_now': device_now, 170*8975f5c5SAndroid Build Coastguard Worker 'time': tombstone_time, 171*8975f5c5SAndroid Build Coastguard Worker 'file': tombstone_file, 172*8975f5c5SAndroid Build Coastguard Worker 'stack': include_stack_symbols, 173*8975f5c5SAndroid Build Coastguard Worker 'data': _GetTombstoneData(device, tombstone_file)}] 174*8975f5c5SAndroid Build Coastguard Worker except device_errors.CommandFailedError: 175*8975f5c5SAndroid Build Coastguard Worker for entry in device.StatDirectory( 176*8975f5c5SAndroid Build Coastguard Worker '/data/tombstones', as_root=True, timeout=60): 177*8975f5c5SAndroid Build Coastguard Worker logging.info('%s: %s', str(device), entry) 178*8975f5c5SAndroid Build Coastguard Worker raise 179*8975f5c5SAndroid Build Coastguard Worker 180*8975f5c5SAndroid Build Coastguard Worker # Erase all the tombstones if desired. 181*8975f5c5SAndroid Build Coastguard Worker if wipe_tombstones: 182*8975f5c5SAndroid Build Coastguard Worker for tombstone_file, _ in all_tombstones: 183*8975f5c5SAndroid Build Coastguard Worker _EraseTombstone(device, tombstone_file) 184*8975f5c5SAndroid Build Coastguard Worker 185*8975f5c5SAndroid Build Coastguard Worker return ret 186*8975f5c5SAndroid Build Coastguard Worker 187*8975f5c5SAndroid Build Coastguard Worker 188*8975f5c5SAndroid Build Coastguard Workerdef ClearAllTombstones(device): 189*8975f5c5SAndroid Build Coastguard Worker """Clear all tombstones in the device. 190*8975f5c5SAndroid Build Coastguard Worker 191*8975f5c5SAndroid Build Coastguard Worker Args: 192*8975f5c5SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 193*8975f5c5SAndroid Build Coastguard Worker """ 194*8975f5c5SAndroid Build Coastguard Worker all_tombstones = list(_ListTombstones(device)) 195*8975f5c5SAndroid Build Coastguard Worker if not all_tombstones: 196*8975f5c5SAndroid Build Coastguard Worker logging.warning('No tombstones to clear.') 197*8975f5c5SAndroid Build Coastguard Worker 198*8975f5c5SAndroid Build Coastguard Worker for tombstone_file, _ in all_tombstones: 199*8975f5c5SAndroid Build Coastguard Worker _EraseTombstone(device, tombstone_file) 200*8975f5c5SAndroid Build Coastguard Worker 201*8975f5c5SAndroid Build Coastguard Worker 202*8975f5c5SAndroid Build Coastguard Workerdef ResolveTombstones(device, resolve_all_tombstones, include_stack_symbols, 203*8975f5c5SAndroid Build Coastguard Worker wipe_tombstones, jobs=4, apk_under_test=None, 204*8975f5c5SAndroid Build Coastguard Worker tombstone_symbolizer=None): 205*8975f5c5SAndroid Build Coastguard Worker """Resolve tombstones in the device. 206*8975f5c5SAndroid Build Coastguard Worker 207*8975f5c5SAndroid Build Coastguard Worker Args: 208*8975f5c5SAndroid Build Coastguard Worker device: An instance of DeviceUtils. 209*8975f5c5SAndroid Build Coastguard Worker resolve_all_tombstone: Whether to resolve every tombstone. 210*8975f5c5SAndroid Build Coastguard Worker include_stack_symbols: Whether to include symbols for stack data. 211*8975f5c5SAndroid Build Coastguard Worker wipe_tombstones: Whether to wipe tombstones. 212*8975f5c5SAndroid Build Coastguard Worker jobs: Number of jobs to use when processing multiple crash stacks. 213*8975f5c5SAndroid Build Coastguard Worker 214*8975f5c5SAndroid Build Coastguard Worker Returns: 215*8975f5c5SAndroid Build Coastguard Worker A list of resolved tombstones. 216*8975f5c5SAndroid Build Coastguard Worker """ 217*8975f5c5SAndroid Build Coastguard Worker return _ResolveTombstones(jobs, 218*8975f5c5SAndroid Build Coastguard Worker _GetTombstonesForDevice(device, 219*8975f5c5SAndroid Build Coastguard Worker resolve_all_tombstones, 220*8975f5c5SAndroid Build Coastguard Worker include_stack_symbols, 221*8975f5c5SAndroid Build Coastguard Worker wipe_tombstones), 222*8975f5c5SAndroid Build Coastguard Worker (tombstone_symbolizer 223*8975f5c5SAndroid Build Coastguard Worker or stack_symbolizer.Symbolizer(apk_under_test))) 224*8975f5c5SAndroid Build Coastguard Worker 225*8975f5c5SAndroid Build Coastguard Worker 226*8975f5c5SAndroid Build Coastguard Workerdef main(): 227*8975f5c5SAndroid Build Coastguard Worker custom_handler = logging.StreamHandler(sys.stdout) 228*8975f5c5SAndroid Build Coastguard Worker custom_handler.setFormatter(run_tests_helper.CustomFormatter()) 229*8975f5c5SAndroid Build Coastguard Worker logging.getLogger().addHandler(custom_handler) 230*8975f5c5SAndroid Build Coastguard Worker logging.getLogger().setLevel(logging.INFO) 231*8975f5c5SAndroid Build Coastguard Worker 232*8975f5c5SAndroid Build Coastguard Worker parser = argparse.ArgumentParser() 233*8975f5c5SAndroid Build Coastguard Worker parser.add_argument('--device', 234*8975f5c5SAndroid Build Coastguard Worker help='The serial number of the device. If not specified ' 235*8975f5c5SAndroid Build Coastguard Worker 'will use all devices.') 236*8975f5c5SAndroid Build Coastguard Worker parser.add_argument('--denylist-file', help='Device denylist JSON file.') 237*8975f5c5SAndroid Build Coastguard Worker parser.add_argument('-a', '--all-tombstones', action='store_true', 238*8975f5c5SAndroid Build Coastguard Worker help='Resolve symbols for all tombstones, rather than ' 239*8975f5c5SAndroid Build Coastguard Worker 'just the most recent.') 240*8975f5c5SAndroid Build Coastguard Worker parser.add_argument('-s', '--stack', action='store_true', 241*8975f5c5SAndroid Build Coastguard Worker help='Also include symbols for stack data') 242*8975f5c5SAndroid Build Coastguard Worker parser.add_argument('-w', '--wipe-tombstones', action='store_true', 243*8975f5c5SAndroid Build Coastguard Worker help='Erase all tombstones from device after processing') 244*8975f5c5SAndroid Build Coastguard Worker parser.add_argument('-j', '--jobs', type=int, 245*8975f5c5SAndroid Build Coastguard Worker default=4, 246*8975f5c5SAndroid Build Coastguard Worker help='Number of jobs to use when processing multiple ' 247*8975f5c5SAndroid Build Coastguard Worker 'crash stacks.') 248*8975f5c5SAndroid Build Coastguard Worker parser.add_argument('--output-directory', 249*8975f5c5SAndroid Build Coastguard Worker help='Path to the root build directory.') 250*8975f5c5SAndroid Build Coastguard Worker parser.add_argument('--adb-path', type=os.path.abspath, 251*8975f5c5SAndroid Build Coastguard Worker help='Path to the adb binary.') 252*8975f5c5SAndroid Build Coastguard Worker args = parser.parse_args() 253*8975f5c5SAndroid Build Coastguard Worker 254*8975f5c5SAndroid Build Coastguard Worker if args.output_directory: 255*8975f5c5SAndroid Build Coastguard Worker constants.SetOutputDirectory(args.output_directory) 256*8975f5c5SAndroid Build Coastguard Worker 257*8975f5c5SAndroid Build Coastguard Worker devil_chromium.Initialize(output_directory=constants.GetOutDirectory(), 258*8975f5c5SAndroid Build Coastguard Worker adb_path=args.adb_path) 259*8975f5c5SAndroid Build Coastguard Worker 260*8975f5c5SAndroid Build Coastguard Worker denylist = (device_denylist.Denylist(args.denylist_file) 261*8975f5c5SAndroid Build Coastguard Worker if args.denylist_file else None) 262*8975f5c5SAndroid Build Coastguard Worker 263*8975f5c5SAndroid Build Coastguard Worker if args.device: 264*8975f5c5SAndroid Build Coastguard Worker devices = [device_utils.DeviceUtils(args.device)] 265*8975f5c5SAndroid Build Coastguard Worker else: 266*8975f5c5SAndroid Build Coastguard Worker devices = device_utils.DeviceUtils.HealthyDevices(denylist) 267*8975f5c5SAndroid Build Coastguard Worker 268*8975f5c5SAndroid Build Coastguard Worker # This must be done serially because strptime can hit a race condition if 269*8975f5c5SAndroid Build Coastguard Worker # used for the first time in a multithreaded environment. 270*8975f5c5SAndroid Build Coastguard Worker # http://bugs.python.org/issue7980 271*8975f5c5SAndroid Build Coastguard Worker for device in devices: 272*8975f5c5SAndroid Build Coastguard Worker resolved_tombstones = ResolveTombstones( 273*8975f5c5SAndroid Build Coastguard Worker device, args.all_tombstones, 274*8975f5c5SAndroid Build Coastguard Worker args.stack, args.wipe_tombstones, args.jobs) 275*8975f5c5SAndroid Build Coastguard Worker for line in resolved_tombstones: 276*8975f5c5SAndroid Build Coastguard Worker logging.info(line) 277*8975f5c5SAndroid Build Coastguard Worker 278*8975f5c5SAndroid Build Coastguard Worker 279*8975f5c5SAndroid Build Coastguard Workerif __name__ == '__main__': 280*8975f5c5SAndroid Build Coastguard Worker sys.exit(main()) 281