1#!/usr/bin/env python 2# @lint-avoid-python-3-compatibility-imports 3from __future__ import print_function 4 5import argparse 6import os 7import platform 8import re 9import signal 10import sys 11 12from bcc import BPF 13from datetime import datetime 14from time import strftime 15 16# 17# exitsnoop Trace all process termination (exit, fatal signal) 18# For Linux, uses BCC, eBPF. Embedded C. 19# 20# USAGE: exitsnoop [-h] [-x] [-t] [--utc] [--label[=LABEL]] [-p PID] 21# 22_examples = """examples: 23 exitsnoop # trace all process termination 24 exitsnoop -x # trace only fails, exclude exit(0) 25 exitsnoop -t # include timestamps (local time) 26 exitsnoop --utc # include timestamps (UTC) 27 exitsnoop -p 181 # only trace PID 181 28 exitsnoop --label=exit # label each output line with 'exit' 29 exitsnoop --per-thread # trace per thread termination 30""" 31""" 32 Exit status (from <include/sysexits.h>): 33 34 0 EX_OK Success 35 2 argparse error 36 70 EX_SOFTWARE syntax error detected by compiler, or 37 verifier error from kernel 38 77 EX_NOPERM Need sudo (CAP_SYS_ADMIN) for BPF() system call 39 40 The template for this script was Brendan Gregg's execsnoop 41 https://github.com/iovisor/bcc/blob/master/tools/execsnoop.py 42 43 More information about this script is in bcc/tools/exitsnoop_example.txt 44 45 Copyright 2016 Netflix, Inc. 46 Copyright 2019 Instana, Inc. 47 Licensed under the Apache License, Version 2.0 (the "License") 48 49 07-Feb-2016 Brendan Gregg (Netflix) Created execsnoop 50 04-May-2019 Arturo Martin-de-Nicolas (Instana) Created exitsnoop 51 13-May-2019 Jeroen Soeters (Instana) Refactor to import as module 52""" 53 54def _getParser(): 55 parser = argparse.ArgumentParser( 56 description="Trace all process termination (exit, fatal signal)", 57 formatter_class=argparse.RawDescriptionHelpFormatter, 58 epilog=_examples) 59 a=parser.add_argument 60 a("-t", "--timestamp", action="store_true", help="include timestamp (local time default)") 61 a("--utc", action="store_true", help="include timestamp in UTC (-t implied)") 62 a("-p", "--pid", help="trace this PID only") 63 a("--label", help="label each line") 64 a("-x", "--failed", action="store_true", help="trace only fails, exclude exit(0)") 65 a("--per-thread", action="store_true", help="trace per thread termination") 66 # print the embedded C program and exit, for debugging 67 a("--ebpf", action="store_true", help=argparse.SUPPRESS) 68 # RHEL 7.6 keeps task->start_time as struct timespec, convert to u64 nanoseconds 69 a("--timespec", action="store_true", help=argparse.SUPPRESS) 70 return parser.parse_args 71 72 73class Global(): 74 parse_args = _getParser() 75 args = None 76 argv = None 77 SIGNUM_TO_SIGNAME = dict((v, re.sub("^SIG", "", k)) 78 for k,v in signal.__dict__.items() if re.match("^SIG[A-Z]+$", k)) 79 80def _embedded_c(args): 81 """Generate C program for sched_process_exit tracepoint in kernel/exit.c.""" 82 c = """ 83 EBPF_COMMENT 84 #include <linux/sched.h> 85 86 struct data_t { 87 u64 start_time; 88 u64 exit_time; 89 u32 pid; 90 u32 tid; 91 u32 ppid; 92 int exit_code; 93 u32 sig_info; 94 char task[TASK_COMM_LEN]; 95 }; 96 97 BPF_PERF_OUTPUT(events); 98 99 TRACEPOINT_PROBE(sched, sched_process_exit) 100 { 101 struct task_struct *task = (typeof(task))bpf_get_current_task(); 102 if (FILTER_PID || FILTER_EXIT_CODE) { return 0; } 103 104 struct data_t data = {}; 105 106 data.start_time = PROCESS_START_TIME_NS, 107 data.exit_time = bpf_ktime_get_ns(), 108 data.pid = task->tgid, 109 data.tid = task->pid, 110 data.ppid = task->real_parent->tgid, 111 data.exit_code = task->exit_code >> 8, 112 data.sig_info = task->exit_code & 0xFF, 113 bpf_get_current_comm(&data.task, sizeof(data.task)); 114 115 events.perf_submit(args, &data, sizeof(data)); 116 return 0; 117 } 118 """ 119 120 if Global.args.pid: 121 if Global.args.per_thread: 122 filter_pid = "task->tgid != %s" % Global.args.pid 123 else: 124 filter_pid = "!(task->tgid == %s && task->pid == task->tgid)" % Global.args.pid 125 else: 126 filter_pid = '0' if Global.args.per_thread else 'task->pid != task->tgid' 127 128 code_substitutions = [ 129 ('EBPF_COMMENT', '' if not Global.args.ebpf else _ebpf_comment()), 130 ('FILTER_PID', filter_pid), 131 ('FILTER_EXIT_CODE', '0' if not Global.args.failed else 'task->exit_code == 0'), 132 ('PROCESS_START_TIME_NS', 'task->start_time' if not Global.args.timespec else 133 '(task->start_time.tv_sec * 1000000000L) + task->start_time.tv_nsec'), 134 ] 135 for old,new in code_substitutions: 136 c = c.replace(old, new) 137 return c 138 139def _ebpf_comment(): 140 """Return a C-style comment with information about the generated code.""" 141 comment=('Created by %s at %s:\n\t%s' % 142 (sys.argv[0], strftime("%Y-%m-%d %H:%M:%S %Z"), _embedded_c.__doc__)) 143 args = str(vars(Global.args)).replace('{','{\n\t').replace(', ',',\n\t').replace('}',',\n }\n\n') 144 return ("\n /*" + ("\n %s\n\n ARGV = %s\n\n ARGS = %s/" % 145 (comment, ' '.join(Global.argv), args)) 146 .replace('\n','\n\t*').replace('\t',' ')) 147 148def _print_header(): 149 if Global.args.timestamp: 150 title = 'TIME-' + ('UTC' if Global.args.utc else strftime("%Z")) 151 print("%-13s" % title, end="") 152 if Global.args.label is not None: 153 print("%-6s" % "LABEL", end="") 154 print("%-16s %-7s %-7s %-7s %-7s %-10s" % 155 ("PCOMM", "PID", "PPID", "TID", "AGE(s)", "EXIT_CODE")) 156 157buffer = None 158 159def _print_event(cpu, data, size): # callback 160 """Print the exit event.""" 161 global buffer 162 e = buffer["events"].event(data) 163 if Global.args.timestamp: 164 now = datetime.utcnow() if Global.args.utc else datetime.now() 165 print("%-13s" % (now.strftime("%H:%M:%S.%f")[:-3]), end="") 166 if Global.args.label is not None: 167 label = Global.args.label if len(Global.args.label) else 'exit' 168 print("%-6s" % label, end="") 169 age = (e.exit_time - e.start_time) / 1e9 170 print("%-16s %-7d %-7d %-7d %-7.2f " % 171 (e.task.decode(), e.pid, e.ppid, e.tid, age), end="") 172 if e.sig_info == 0: 173 print("0" if e.exit_code == 0 else "code %d" % e.exit_code) 174 else: 175 sig = e.sig_info & 0x7F 176 if sig: 177 print("signal %d (%s)" % (sig, signum_to_signame(sig)), end="") 178 if e.sig_info & 0x80: 179 print(", core dumped ", end="") 180 print() 181 182# ============================= 183# Module: These functions are available for import 184# ============================= 185def initialize(arg_list = sys.argv[1:]): 186 """Trace all process termination. 187 188 arg_list - list of args, if omitted then uses command line args 189 arg_list is passed to argparse.ArgumentParser.parse_args() 190 191 For example, if arg_list = [ '-x', '-t' ] 192 args.failed == True 193 args.timestamp == True 194 195 Returns a tuple (return_code, result) 196 0 = Ok, result is the return value from BPF() 197 1 = args.ebpf is requested, result is the generated C code 198 os.EX_NOPERM: need CAP_SYS_ADMIN, result is error message 199 os.EX_SOFTWARE: internal software error, result is error message 200 """ 201 Global.argv = arg_list 202 Global.args = Global.parse_args(arg_list) 203 if Global.args.utc and not Global.args.timestamp: 204 Global.args.timestamp = True 205 if not Global.args.ebpf and os.geteuid() != 0: 206 return (os.EX_NOPERM, "Need sudo (CAP_SYS_ADMIN) for BPF() system call") 207 if re.match('^3\.10\..*el7.*$', platform.release()): # Centos/Red Hat 208 Global.args.timespec = True 209 for _ in range(2): 210 c = _embedded_c(Global.args) 211 if Global.args.ebpf: 212 return (1, c) 213 try: 214 return (os.EX_OK, BPF(text=c)) 215 except Exception as e: 216 error = format(e) 217 if (not Global.args.timespec 218 and error.find('struct timespec') 219 and error.find('start_time')): 220 print('This kernel keeps task->start_time in a struct timespec.\n' + 221 'Retrying with --timespec') 222 Global.args.timespec = True 223 continue 224 return (os.EX_SOFTWARE, "BPF error: " + error) 225 except: 226 return (os.EX_SOFTWARE, "Unexpected error: {0}".format(sys.exc_info()[0])) 227 228def snoop(bpf, event_handler): 229 """Call event_handler for process termination events. 230 231 bpf - result returned by successful initialize() 232 event_handler - callback function to handle termination event 233 args.pid - Return after event_handler is called, only monitoring this pid 234 """ 235 bpf["events"].open_perf_buffer(event_handler) 236 while True: 237 bpf.perf_buffer_poll() 238 if Global.args.pid: 239 return 240 241def signum_to_signame(signum): 242 """Return the name of the signal corresponding to signum.""" 243 return Global.SIGNUM_TO_SIGNAME.get(signum, "unknown") 244 245# ============================= 246# Script: invoked as a script 247# ============================= 248def main(): 249 global buffer 250 try: 251 rc, buffer = initialize() 252 if rc: 253 print(buffer) 254 sys.exit(0 if Global.args.ebpf else rc) 255 _print_header() 256 snoop(buffer, _print_event) 257 except KeyboardInterrupt: 258 print() 259 sys.exit() 260 261 return 0 262 263if __name__ == '__main__': 264 main() 265