1#!/usr/bin/env python 2# @lint-avoid-python-3-compatibility-imports 3# 4# hardirqs Summarize hard IRQ (interrupt) event time. 5# For Linux, uses BCC, eBPF. 6# 7# USAGE: hardirqs [-h] [-T] [-N] [-C] [-d] [-c CPU] [interval] [outputs] 8# 9# Thanks Amer Ather for help understanding irq behavior. 10# 11# Copyright (c) 2015 Brendan Gregg. 12# Licensed under the Apache License, Version 2.0 (the "License") 13# 14# 19-Oct-2015 Brendan Gregg Created this. 15# 22-May-2021 Hengqi Chen Migrated to kernel tracepoints. 16# 07-Mar-2022 Rocky Xing Added CPU filter support. 17 18from __future__ import print_function 19from bcc import BPF 20from time import sleep, strftime 21import argparse 22import sys 23 24# arguments 25examples = """examples: 26 ./hardirqs # sum hard irq event time 27 ./hardirqs -d # show hard irq event time as histograms 28 ./hardirqs 1 10 # print 1 second summaries, 10 times 29 ./hardirqs -NT 1 # 1s summaries, nanoseconds, and timestamps 30 ./hardirqs -c 1 # sum hard irq event time on CPU 1 only 31""" 32parser = argparse.ArgumentParser( 33 description="Summarize hard irq event time as histograms", 34 formatter_class=argparse.RawDescriptionHelpFormatter, 35 epilog=examples) 36parser.add_argument("-T", "--timestamp", action="store_true", 37 help="include timestamp on output") 38parser.add_argument("-N", "--nanoseconds", action="store_true", 39 help="output in nanoseconds") 40parser.add_argument("-C", "--count", action="store_true", 41 help="show event counts instead of timing") 42parser.add_argument("-d", "--dist", action="store_true", 43 help="show distributions as histograms") 44parser.add_argument("-c", "--cpu", type=int, 45 help="trace this CPU only") 46parser.add_argument("interval", nargs="?", default=99999999, 47 help="output interval, in seconds") 48parser.add_argument("outputs", nargs="?", default=99999999, 49 help="number of outputs") 50parser.add_argument("--ebpf", action="store_true", 51 help=argparse.SUPPRESS) 52args = parser.parse_args() 53countdown = int(args.outputs) 54if args.count and (args.dist or args.nanoseconds): 55 print("The --count option can't be used with time-based options") 56 exit() 57if args.count: 58 factor = 1 59 label = "count" 60elif args.nanoseconds: 61 factor = 1 62 label = "nsecs" 63else: 64 factor = 1000 65 label = "usecs" 66debug = 0 67 68# define BPF program 69bpf_text = """ 70#include <uapi/linux/ptrace.h> 71#include <linux/irq.h> 72#include <linux/irqdesc.h> 73#include <linux/interrupt.h> 74 75// Add cpu_id as part of key for irq entry event to handle the case which irq 76// is triggered while idle thread(swapper/x, tid=0) for each cpu core. 77// Please see more detail at pull request #2804, #3733. 78typedef struct entry_key { 79 u32 tid; 80 u32 cpu_id; 81} entry_key_t; 82 83typedef struct irq_key { 84 char name[32]; 85 u64 slot; 86} irq_key_t; 87 88typedef struct irq_name { 89 char name[32]; 90} irq_name_t; 91 92BPF_HASH(start, entry_key_t); 93BPF_HASH(irqnames, entry_key_t, irq_name_t); 94BPF_HISTOGRAM(dist, irq_key_t); 95""" 96 97bpf_text_count = """ 98TRACEPOINT_PROBE(irq, irq_handler_entry) 99{ 100 struct entry_key key = {}; 101 irq_name_t name = {}; 102 u32 cpu = bpf_get_smp_processor_id(); 103 104 FILTER_CPU 105 106 key.tid = bpf_get_current_pid_tgid(); 107 key.cpu_id = cpu; 108 109 TP_DATA_LOC_READ_STR(&name.name, name, sizeof(name)); 110 irqnames.update(&key, &name); 111 return 0; 112} 113 114TRACEPOINT_PROBE(irq, irq_handler_exit) 115{ 116 struct entry_key key = {}; 117 u32 cpu = bpf_get_smp_processor_id(); 118 119 FILTER_CPU 120 121 key.tid = bpf_get_current_pid_tgid(); 122 key.cpu_id = cpu; 123 124 // check ret value of irq handler is not IRQ_NONE to make sure 125 // the current event belong to this irq handler 126 if (args->ret != IRQ_NONE) { 127 irq_name_t *namep; 128 129 namep = irqnames.lookup(&key); 130 if (namep == 0) { 131 return 0; // missed irq name 132 } 133 char *name = (char *)namep->name; 134 irq_key_t key = {.slot = 0 /* ignore */}; 135 136 bpf_probe_read_kernel(&key.name, sizeof(key.name), name); 137 dist.atomic_increment(key); 138 } 139 140 irqnames.delete(&key); 141 return 0; 142} 143""" 144 145bpf_text_time = """ 146TRACEPOINT_PROBE(irq, irq_handler_entry) 147{ 148 u64 ts = bpf_ktime_get_ns(); 149 irq_name_t name = {}; 150 struct entry_key key = {}; 151 u32 cpu = bpf_get_smp_processor_id(); 152 153 FILTER_CPU 154 155 key.tid = bpf_get_current_pid_tgid(); 156 key.cpu_id = cpu; 157 158 TP_DATA_LOC_READ_STR(&name.name, name, sizeof(name)); 159 irqnames.update(&key, &name); 160 start.update(&key, &ts); 161 return 0; 162} 163 164TRACEPOINT_PROBE(irq, irq_handler_exit) 165{ 166 u64 *tsp, delta; 167 irq_name_t *namep; 168 struct entry_key key = {}; 169 u32 cpu = bpf_get_smp_processor_id(); 170 171 key.tid = bpf_get_current_pid_tgid(); 172 key.cpu_id = cpu; 173 174 // check ret value of irq handler is not IRQ_NONE to make sure 175 // the current event belong to this irq handler 176 if (args->ret != IRQ_NONE) { 177 // fetch timestamp and calculate delta 178 tsp = start.lookup(&key); 179 namep = irqnames.lookup(&key); 180 if (tsp == 0 || namep == 0) { 181 return 0; // missed start 182 } 183 184 char *name = (char *)namep->name; 185 delta = bpf_ktime_get_ns() - *tsp; 186 187 // store as sum or histogram 188 STORE 189 } 190 191 start.delete(&key); 192 irqnames.delete(&key); 193 return 0; 194} 195""" 196 197if args.count: 198 bpf_text += bpf_text_count 199else: 200 bpf_text += bpf_text_time 201 202# code substitutions 203if args.dist: 204 bpf_text = bpf_text.replace('STORE', 205 'irq_key_t key = {.slot = bpf_log2l(delta / %d)};' % factor + 206 'bpf_probe_read_kernel(&key.name, sizeof(key.name), name);' + 207 'dist.atomic_increment(key);') 208else: 209 bpf_text = bpf_text.replace('STORE', 210 'irq_key_t key = {.slot = 0 /* ignore */};' + 211 'bpf_probe_read_kernel(&key.name, sizeof(key.name), name);' + 212 'dist.atomic_increment(key, delta);') 213if args.cpu is not None: 214 bpf_text = bpf_text.replace('FILTER_CPU', 215 'if (cpu != %d) { return 0; }' % int(args.cpu)) 216else: 217 bpf_text = bpf_text.replace('FILTER_CPU', '') 218if debug or args.ebpf: 219 print(bpf_text) 220 if args.ebpf: 221 exit() 222 223# load BPF program 224b = BPF(text=bpf_text) 225 226if args.count: 227 print("Tracing hard irq events... Hit Ctrl-C to end.") 228else: 229 print("Tracing hard irq event time... Hit Ctrl-C to end.") 230 231# output 232exiting = 0 if args.interval else 1 233dist = b.get_table("dist") 234while (1): 235 try: 236 sleep(int(args.interval)) 237 except KeyboardInterrupt: 238 exiting = 1 239 240 print() 241 if args.timestamp: 242 print("%-8s\n" % strftime("%H:%M:%S"), end="") 243 244 if args.dist: 245 dist.print_log2_hist(label, "hardirq", section_print_fn=bytes.decode) 246 else: 247 print("%-26s %11s" % ("HARDIRQ", "TOTAL_" + label)) 248 for k, v in sorted(dist.items(), key=lambda dist: dist[1].value): 249 print("%-26s %11d" % (k.name.decode('utf-8', 'replace'), v.value / factor)) 250 dist.clear() 251 252 sys.stdout.flush() 253 254 countdown -= 1 255 if exiting or countdown == 0: 256 exit() 257