1*387f9dfdSAndroid Build Coastguard Worker#!/usr/bin/env python 2*387f9dfdSAndroid Build Coastguard Worker# @lint-avoid-python-3-compatibility-imports 3*387f9dfdSAndroid Build Coastguard Workerfrom __future__ import print_function 4*387f9dfdSAndroid Build Coastguard Worker 5*387f9dfdSAndroid Build Coastguard Workerimport argparse 6*387f9dfdSAndroid Build Coastguard Workerimport os 7*387f9dfdSAndroid Build Coastguard Workerimport platform 8*387f9dfdSAndroid Build Coastguard Workerimport re 9*387f9dfdSAndroid Build Coastguard Workerimport signal 10*387f9dfdSAndroid Build Coastguard Workerimport sys 11*387f9dfdSAndroid Build Coastguard Worker 12*387f9dfdSAndroid Build Coastguard Workerfrom bcc import BPF 13*387f9dfdSAndroid Build Coastguard Workerfrom datetime import datetime 14*387f9dfdSAndroid Build Coastguard Workerfrom time import strftime 15*387f9dfdSAndroid Build Coastguard Worker 16*387f9dfdSAndroid Build Coastguard Worker# 17*387f9dfdSAndroid Build Coastguard Worker# exitsnoop Trace all process termination (exit, fatal signal) 18*387f9dfdSAndroid Build Coastguard Worker# For Linux, uses BCC, eBPF. Embedded C. 19*387f9dfdSAndroid Build Coastguard Worker# 20*387f9dfdSAndroid Build Coastguard Worker# USAGE: exitsnoop [-h] [-x] [-t] [--utc] [--label[=LABEL]] [-p PID] 21*387f9dfdSAndroid Build Coastguard Worker# 22*387f9dfdSAndroid Build Coastguard Worker_examples = """examples: 23*387f9dfdSAndroid Build Coastguard Worker exitsnoop # trace all process termination 24*387f9dfdSAndroid Build Coastguard Worker exitsnoop -x # trace only fails, exclude exit(0) 25*387f9dfdSAndroid Build Coastguard Worker exitsnoop -t # include timestamps (local time) 26*387f9dfdSAndroid Build Coastguard Worker exitsnoop --utc # include timestamps (UTC) 27*387f9dfdSAndroid Build Coastguard Worker exitsnoop -p 181 # only trace PID 181 28*387f9dfdSAndroid Build Coastguard Worker exitsnoop --label=exit # label each output line with 'exit' 29*387f9dfdSAndroid Build Coastguard Worker exitsnoop --per-thread # trace per thread termination 30*387f9dfdSAndroid Build Coastguard Worker""" 31*387f9dfdSAndroid Build Coastguard Worker""" 32*387f9dfdSAndroid Build Coastguard Worker Exit status (from <include/sysexits.h>): 33*387f9dfdSAndroid Build Coastguard Worker 34*387f9dfdSAndroid Build Coastguard Worker 0 EX_OK Success 35*387f9dfdSAndroid Build Coastguard Worker 2 argparse error 36*387f9dfdSAndroid Build Coastguard Worker 70 EX_SOFTWARE syntax error detected by compiler, or 37*387f9dfdSAndroid Build Coastguard Worker verifier error from kernel 38*387f9dfdSAndroid Build Coastguard Worker 77 EX_NOPERM Need sudo (CAP_SYS_ADMIN) for BPF() system call 39*387f9dfdSAndroid Build Coastguard Worker 40*387f9dfdSAndroid Build Coastguard Worker The template for this script was Brendan Gregg's execsnoop 41*387f9dfdSAndroid Build Coastguard Worker https://github.com/iovisor/bcc/blob/master/tools/execsnoop.py 42*387f9dfdSAndroid Build Coastguard Worker 43*387f9dfdSAndroid Build Coastguard Worker More information about this script is in bcc/tools/exitsnoop_example.txt 44*387f9dfdSAndroid Build Coastguard Worker 45*387f9dfdSAndroid Build Coastguard Worker Copyright 2016 Netflix, Inc. 46*387f9dfdSAndroid Build Coastguard Worker Copyright 2019 Instana, Inc. 47*387f9dfdSAndroid Build Coastguard Worker Licensed under the Apache License, Version 2.0 (the "License") 48*387f9dfdSAndroid Build Coastguard Worker 49*387f9dfdSAndroid Build Coastguard Worker 07-Feb-2016 Brendan Gregg (Netflix) Created execsnoop 50*387f9dfdSAndroid Build Coastguard Worker 04-May-2019 Arturo Martin-de-Nicolas (Instana) Created exitsnoop 51*387f9dfdSAndroid Build Coastguard Worker 13-May-2019 Jeroen Soeters (Instana) Refactor to import as module 52*387f9dfdSAndroid Build Coastguard Worker""" 53*387f9dfdSAndroid Build Coastguard Worker 54*387f9dfdSAndroid Build Coastguard Workerdef _getParser(): 55*387f9dfdSAndroid Build Coastguard Worker parser = argparse.ArgumentParser( 56*387f9dfdSAndroid Build Coastguard Worker description="Trace all process termination (exit, fatal signal)", 57*387f9dfdSAndroid Build Coastguard Worker formatter_class=argparse.RawDescriptionHelpFormatter, 58*387f9dfdSAndroid Build Coastguard Worker epilog=_examples) 59*387f9dfdSAndroid Build Coastguard Worker a=parser.add_argument 60*387f9dfdSAndroid Build Coastguard Worker a("-t", "--timestamp", action="store_true", help="include timestamp (local time default)") 61*387f9dfdSAndroid Build Coastguard Worker a("--utc", action="store_true", help="include timestamp in UTC (-t implied)") 62*387f9dfdSAndroid Build Coastguard Worker a("-p", "--pid", help="trace this PID only") 63*387f9dfdSAndroid Build Coastguard Worker a("--label", help="label each line") 64*387f9dfdSAndroid Build Coastguard Worker a("-x", "--failed", action="store_true", help="trace only fails, exclude exit(0)") 65*387f9dfdSAndroid Build Coastguard Worker a("--per-thread", action="store_true", help="trace per thread termination") 66*387f9dfdSAndroid Build Coastguard Worker # print the embedded C program and exit, for debugging 67*387f9dfdSAndroid Build Coastguard Worker a("--ebpf", action="store_true", help=argparse.SUPPRESS) 68*387f9dfdSAndroid Build Coastguard Worker # RHEL 7.6 keeps task->start_time as struct timespec, convert to u64 nanoseconds 69*387f9dfdSAndroid Build Coastguard Worker a("--timespec", action="store_true", help=argparse.SUPPRESS) 70*387f9dfdSAndroid Build Coastguard Worker return parser.parse_args 71*387f9dfdSAndroid Build Coastguard Worker 72*387f9dfdSAndroid Build Coastguard Worker 73*387f9dfdSAndroid Build Coastguard Workerclass Global(): 74*387f9dfdSAndroid Build Coastguard Worker parse_args = _getParser() 75*387f9dfdSAndroid Build Coastguard Worker args = None 76*387f9dfdSAndroid Build Coastguard Worker argv = None 77*387f9dfdSAndroid Build Coastguard Worker SIGNUM_TO_SIGNAME = dict((v, re.sub("^SIG", "", k)) 78*387f9dfdSAndroid Build Coastguard Worker for k,v in signal.__dict__.items() if re.match("^SIG[A-Z]+$", k)) 79*387f9dfdSAndroid Build Coastguard Worker 80*387f9dfdSAndroid Build Coastguard Workerdef _embedded_c(args): 81*387f9dfdSAndroid Build Coastguard Worker """Generate C program for sched_process_exit tracepoint in kernel/exit.c.""" 82*387f9dfdSAndroid Build Coastguard Worker c = """ 83*387f9dfdSAndroid Build Coastguard Worker EBPF_COMMENT 84*387f9dfdSAndroid Build Coastguard Worker #include <linux/sched.h> 85*387f9dfdSAndroid Build Coastguard Worker 86*387f9dfdSAndroid Build Coastguard Worker struct data_t { 87*387f9dfdSAndroid Build Coastguard Worker u64 start_time; 88*387f9dfdSAndroid Build Coastguard Worker u64 exit_time; 89*387f9dfdSAndroid Build Coastguard Worker u32 pid; 90*387f9dfdSAndroid Build Coastguard Worker u32 tid; 91*387f9dfdSAndroid Build Coastguard Worker u32 ppid; 92*387f9dfdSAndroid Build Coastguard Worker int exit_code; 93*387f9dfdSAndroid Build Coastguard Worker u32 sig_info; 94*387f9dfdSAndroid Build Coastguard Worker char task[TASK_COMM_LEN]; 95*387f9dfdSAndroid Build Coastguard Worker }; 96*387f9dfdSAndroid Build Coastguard Worker 97*387f9dfdSAndroid Build Coastguard Worker BPF_PERF_OUTPUT(events); 98*387f9dfdSAndroid Build Coastguard Worker 99*387f9dfdSAndroid Build Coastguard Worker TRACEPOINT_PROBE(sched, sched_process_exit) 100*387f9dfdSAndroid Build Coastguard Worker { 101*387f9dfdSAndroid Build Coastguard Worker struct task_struct *task = (typeof(task))bpf_get_current_task(); 102*387f9dfdSAndroid Build Coastguard Worker if (FILTER_PID || FILTER_EXIT_CODE) { return 0; } 103*387f9dfdSAndroid Build Coastguard Worker 104*387f9dfdSAndroid Build Coastguard Worker struct data_t data = {}; 105*387f9dfdSAndroid Build Coastguard Worker 106*387f9dfdSAndroid Build Coastguard Worker data.start_time = PROCESS_START_TIME_NS, 107*387f9dfdSAndroid Build Coastguard Worker data.exit_time = bpf_ktime_get_ns(), 108*387f9dfdSAndroid Build Coastguard Worker data.pid = task->tgid, 109*387f9dfdSAndroid Build Coastguard Worker data.tid = task->pid, 110*387f9dfdSAndroid Build Coastguard Worker data.ppid = task->real_parent->tgid, 111*387f9dfdSAndroid Build Coastguard Worker data.exit_code = task->exit_code >> 8, 112*387f9dfdSAndroid Build Coastguard Worker data.sig_info = task->exit_code & 0xFF, 113*387f9dfdSAndroid Build Coastguard Worker bpf_get_current_comm(&data.task, sizeof(data.task)); 114*387f9dfdSAndroid Build Coastguard Worker 115*387f9dfdSAndroid Build Coastguard Worker events.perf_submit(args, &data, sizeof(data)); 116*387f9dfdSAndroid Build Coastguard Worker return 0; 117*387f9dfdSAndroid Build Coastguard Worker } 118*387f9dfdSAndroid Build Coastguard Worker """ 119*387f9dfdSAndroid Build Coastguard Worker 120*387f9dfdSAndroid Build Coastguard Worker if Global.args.pid: 121*387f9dfdSAndroid Build Coastguard Worker if Global.args.per_thread: 122*387f9dfdSAndroid Build Coastguard Worker filter_pid = "task->tgid != %s" % Global.args.pid 123*387f9dfdSAndroid Build Coastguard Worker else: 124*387f9dfdSAndroid Build Coastguard Worker filter_pid = "!(task->tgid == %s && task->pid == task->tgid)" % Global.args.pid 125*387f9dfdSAndroid Build Coastguard Worker else: 126*387f9dfdSAndroid Build Coastguard Worker filter_pid = '0' if Global.args.per_thread else 'task->pid != task->tgid' 127*387f9dfdSAndroid Build Coastguard Worker 128*387f9dfdSAndroid Build Coastguard Worker code_substitutions = [ 129*387f9dfdSAndroid Build Coastguard Worker ('EBPF_COMMENT', '' if not Global.args.ebpf else _ebpf_comment()), 130*387f9dfdSAndroid Build Coastguard Worker ('FILTER_PID', filter_pid), 131*387f9dfdSAndroid Build Coastguard Worker ('FILTER_EXIT_CODE', '0' if not Global.args.failed else 'task->exit_code == 0'), 132*387f9dfdSAndroid Build Coastguard Worker ('PROCESS_START_TIME_NS', 'task->start_time' if not Global.args.timespec else 133*387f9dfdSAndroid Build Coastguard Worker '(task->start_time.tv_sec * 1000000000L) + task->start_time.tv_nsec'), 134*387f9dfdSAndroid Build Coastguard Worker ] 135*387f9dfdSAndroid Build Coastguard Worker for old,new in code_substitutions: 136*387f9dfdSAndroid Build Coastguard Worker c = c.replace(old, new) 137*387f9dfdSAndroid Build Coastguard Worker return c 138*387f9dfdSAndroid Build Coastguard Worker 139*387f9dfdSAndroid Build Coastguard Workerdef _ebpf_comment(): 140*387f9dfdSAndroid Build Coastguard Worker """Return a C-style comment with information about the generated code.""" 141*387f9dfdSAndroid Build Coastguard Worker comment=('Created by %s at %s:\n\t%s' % 142*387f9dfdSAndroid Build Coastguard Worker (sys.argv[0], strftime("%Y-%m-%d %H:%M:%S %Z"), _embedded_c.__doc__)) 143*387f9dfdSAndroid Build Coastguard Worker args = str(vars(Global.args)).replace('{','{\n\t').replace(', ',',\n\t').replace('}',',\n }\n\n') 144*387f9dfdSAndroid Build Coastguard Worker return ("\n /*" + ("\n %s\n\n ARGV = %s\n\n ARGS = %s/" % 145*387f9dfdSAndroid Build Coastguard Worker (comment, ' '.join(Global.argv), args)) 146*387f9dfdSAndroid Build Coastguard Worker .replace('\n','\n\t*').replace('\t',' ')) 147*387f9dfdSAndroid Build Coastguard Worker 148*387f9dfdSAndroid Build Coastguard Workerdef _print_header(): 149*387f9dfdSAndroid Build Coastguard Worker if Global.args.timestamp: 150*387f9dfdSAndroid Build Coastguard Worker title = 'TIME-' + ('UTC' if Global.args.utc else strftime("%Z")) 151*387f9dfdSAndroid Build Coastguard Worker print("%-13s" % title, end="") 152*387f9dfdSAndroid Build Coastguard Worker if Global.args.label is not None: 153*387f9dfdSAndroid Build Coastguard Worker print("%-6s" % "LABEL", end="") 154*387f9dfdSAndroid Build Coastguard Worker print("%-16s %-7s %-7s %-7s %-7s %-10s" % 155*387f9dfdSAndroid Build Coastguard Worker ("PCOMM", "PID", "PPID", "TID", "AGE(s)", "EXIT_CODE")) 156*387f9dfdSAndroid Build Coastguard Worker 157*387f9dfdSAndroid Build Coastguard Workerbuffer = None 158*387f9dfdSAndroid Build Coastguard Worker 159*387f9dfdSAndroid Build Coastguard Workerdef _print_event(cpu, data, size): # callback 160*387f9dfdSAndroid Build Coastguard Worker """Print the exit event.""" 161*387f9dfdSAndroid Build Coastguard Worker global buffer 162*387f9dfdSAndroid Build Coastguard Worker e = buffer["events"].event(data) 163*387f9dfdSAndroid Build Coastguard Worker if Global.args.timestamp: 164*387f9dfdSAndroid Build Coastguard Worker now = datetime.utcnow() if Global.args.utc else datetime.now() 165*387f9dfdSAndroid Build Coastguard Worker print("%-13s" % (now.strftime("%H:%M:%S.%f")[:-3]), end="") 166*387f9dfdSAndroid Build Coastguard Worker if Global.args.label is not None: 167*387f9dfdSAndroid Build Coastguard Worker label = Global.args.label if len(Global.args.label) else 'exit' 168*387f9dfdSAndroid Build Coastguard Worker print("%-6s" % label, end="") 169*387f9dfdSAndroid Build Coastguard Worker age = (e.exit_time - e.start_time) / 1e9 170*387f9dfdSAndroid Build Coastguard Worker print("%-16s %-7d %-7d %-7d %-7.2f " % 171*387f9dfdSAndroid Build Coastguard Worker (e.task.decode(), e.pid, e.ppid, e.tid, age), end="") 172*387f9dfdSAndroid Build Coastguard Worker if e.sig_info == 0: 173*387f9dfdSAndroid Build Coastguard Worker print("0" if e.exit_code == 0 else "code %d" % e.exit_code) 174*387f9dfdSAndroid Build Coastguard Worker else: 175*387f9dfdSAndroid Build Coastguard Worker sig = e.sig_info & 0x7F 176*387f9dfdSAndroid Build Coastguard Worker if sig: 177*387f9dfdSAndroid Build Coastguard Worker print("signal %d (%s)" % (sig, signum_to_signame(sig)), end="") 178*387f9dfdSAndroid Build Coastguard Worker if e.sig_info & 0x80: 179*387f9dfdSAndroid Build Coastguard Worker print(", core dumped ", end="") 180*387f9dfdSAndroid Build Coastguard Worker print() 181*387f9dfdSAndroid Build Coastguard Worker 182*387f9dfdSAndroid Build Coastguard Worker# ============================= 183*387f9dfdSAndroid Build Coastguard Worker# Module: These functions are available for import 184*387f9dfdSAndroid Build Coastguard Worker# ============================= 185*387f9dfdSAndroid Build Coastguard Workerdef initialize(arg_list = sys.argv[1:]): 186*387f9dfdSAndroid Build Coastguard Worker """Trace all process termination. 187*387f9dfdSAndroid Build Coastguard Worker 188*387f9dfdSAndroid Build Coastguard Worker arg_list - list of args, if omitted then uses command line args 189*387f9dfdSAndroid Build Coastguard Worker arg_list is passed to argparse.ArgumentParser.parse_args() 190*387f9dfdSAndroid Build Coastguard Worker 191*387f9dfdSAndroid Build Coastguard Worker For example, if arg_list = [ '-x', '-t' ] 192*387f9dfdSAndroid Build Coastguard Worker args.failed == True 193*387f9dfdSAndroid Build Coastguard Worker args.timestamp == True 194*387f9dfdSAndroid Build Coastguard Worker 195*387f9dfdSAndroid Build Coastguard Worker Returns a tuple (return_code, result) 196*387f9dfdSAndroid Build Coastguard Worker 0 = Ok, result is the return value from BPF() 197*387f9dfdSAndroid Build Coastguard Worker 1 = args.ebpf is requested, result is the generated C code 198*387f9dfdSAndroid Build Coastguard Worker os.EX_NOPERM: need CAP_SYS_ADMIN, result is error message 199*387f9dfdSAndroid Build Coastguard Worker os.EX_SOFTWARE: internal software error, result is error message 200*387f9dfdSAndroid Build Coastguard Worker """ 201*387f9dfdSAndroid Build Coastguard Worker Global.argv = arg_list 202*387f9dfdSAndroid Build Coastguard Worker Global.args = Global.parse_args(arg_list) 203*387f9dfdSAndroid Build Coastguard Worker if Global.args.utc and not Global.args.timestamp: 204*387f9dfdSAndroid Build Coastguard Worker Global.args.timestamp = True 205*387f9dfdSAndroid Build Coastguard Worker if not Global.args.ebpf and os.geteuid() != 0: 206*387f9dfdSAndroid Build Coastguard Worker return (os.EX_NOPERM, "Need sudo (CAP_SYS_ADMIN) for BPF() system call") 207*387f9dfdSAndroid Build Coastguard Worker if re.match('^3\.10\..*el7.*$', platform.release()): # Centos/Red Hat 208*387f9dfdSAndroid Build Coastguard Worker Global.args.timespec = True 209*387f9dfdSAndroid Build Coastguard Worker for _ in range(2): 210*387f9dfdSAndroid Build Coastguard Worker c = _embedded_c(Global.args) 211*387f9dfdSAndroid Build Coastguard Worker if Global.args.ebpf: 212*387f9dfdSAndroid Build Coastguard Worker return (1, c) 213*387f9dfdSAndroid Build Coastguard Worker try: 214*387f9dfdSAndroid Build Coastguard Worker return (os.EX_OK, BPF(text=c)) 215*387f9dfdSAndroid Build Coastguard Worker except Exception as e: 216*387f9dfdSAndroid Build Coastguard Worker error = format(e) 217*387f9dfdSAndroid Build Coastguard Worker if (not Global.args.timespec 218*387f9dfdSAndroid Build Coastguard Worker and error.find('struct timespec') 219*387f9dfdSAndroid Build Coastguard Worker and error.find('start_time')): 220*387f9dfdSAndroid Build Coastguard Worker print('This kernel keeps task->start_time in a struct timespec.\n' + 221*387f9dfdSAndroid Build Coastguard Worker 'Retrying with --timespec') 222*387f9dfdSAndroid Build Coastguard Worker Global.args.timespec = True 223*387f9dfdSAndroid Build Coastguard Worker continue 224*387f9dfdSAndroid Build Coastguard Worker return (os.EX_SOFTWARE, "BPF error: " + error) 225*387f9dfdSAndroid Build Coastguard Worker except: 226*387f9dfdSAndroid Build Coastguard Worker return (os.EX_SOFTWARE, "Unexpected error: {0}".format(sys.exc_info()[0])) 227*387f9dfdSAndroid Build Coastguard Worker 228*387f9dfdSAndroid Build Coastguard Workerdef snoop(bpf, event_handler): 229*387f9dfdSAndroid Build Coastguard Worker """Call event_handler for process termination events. 230*387f9dfdSAndroid Build Coastguard Worker 231*387f9dfdSAndroid Build Coastguard Worker bpf - result returned by successful initialize() 232*387f9dfdSAndroid Build Coastguard Worker event_handler - callback function to handle termination event 233*387f9dfdSAndroid Build Coastguard Worker args.pid - Return after event_handler is called, only monitoring this pid 234*387f9dfdSAndroid Build Coastguard Worker """ 235*387f9dfdSAndroid Build Coastguard Worker bpf["events"].open_perf_buffer(event_handler) 236*387f9dfdSAndroid Build Coastguard Worker while True: 237*387f9dfdSAndroid Build Coastguard Worker bpf.perf_buffer_poll() 238*387f9dfdSAndroid Build Coastguard Worker if Global.args.pid: 239*387f9dfdSAndroid Build Coastguard Worker return 240*387f9dfdSAndroid Build Coastguard Worker 241*387f9dfdSAndroid Build Coastguard Workerdef signum_to_signame(signum): 242*387f9dfdSAndroid Build Coastguard Worker """Return the name of the signal corresponding to signum.""" 243*387f9dfdSAndroid Build Coastguard Worker return Global.SIGNUM_TO_SIGNAME.get(signum, "unknown") 244*387f9dfdSAndroid Build Coastguard Worker 245*387f9dfdSAndroid Build Coastguard Worker# ============================= 246*387f9dfdSAndroid Build Coastguard Worker# Script: invoked as a script 247*387f9dfdSAndroid Build Coastguard Worker# ============================= 248*387f9dfdSAndroid Build Coastguard Workerdef main(): 249*387f9dfdSAndroid Build Coastguard Worker global buffer 250*387f9dfdSAndroid Build Coastguard Worker try: 251*387f9dfdSAndroid Build Coastguard Worker rc, buffer = initialize() 252*387f9dfdSAndroid Build Coastguard Worker if rc: 253*387f9dfdSAndroid Build Coastguard Worker print(buffer) 254*387f9dfdSAndroid Build Coastguard Worker sys.exit(0 if Global.args.ebpf else rc) 255*387f9dfdSAndroid Build Coastguard Worker _print_header() 256*387f9dfdSAndroid Build Coastguard Worker snoop(buffer, _print_event) 257*387f9dfdSAndroid Build Coastguard Worker except KeyboardInterrupt: 258*387f9dfdSAndroid Build Coastguard Worker print() 259*387f9dfdSAndroid Build Coastguard Worker sys.exit() 260*387f9dfdSAndroid Build Coastguard Worker 261*387f9dfdSAndroid Build Coastguard Worker return 0 262*387f9dfdSAndroid Build Coastguard Worker 263*387f9dfdSAndroid Build Coastguard Workerif __name__ == '__main__': 264*387f9dfdSAndroid Build Coastguard Worker main() 265