1*387f9dfdSAndroid Build Coastguard Worker#!/usr/bin/env python 2*387f9dfdSAndroid Build Coastguard Worker# @lint-avoid-python-3-compatibility-imports 3*387f9dfdSAndroid Build Coastguard Worker# 4*387f9dfdSAndroid Build Coastguard Worker# btrfsdist Summarize btrfs operation latency. 5*387f9dfdSAndroid Build Coastguard Worker# For Linux, uses BCC, eBPF. 6*387f9dfdSAndroid Build Coastguard Worker# 7*387f9dfdSAndroid Build Coastguard Worker# USAGE: btrfsdist [-h] [-T] [-m] [-p PID] [interval] [count] 8*387f9dfdSAndroid Build Coastguard Worker# 9*387f9dfdSAndroid Build Coastguard Worker# Copyright 2016 Netflix, Inc. 10*387f9dfdSAndroid Build Coastguard Worker# Licensed under the Apache License, Version 2.0 (the "License") 11*387f9dfdSAndroid Build Coastguard Worker# 12*387f9dfdSAndroid Build Coastguard Worker# 15-Feb-2016 Brendan Gregg Created this. 13*387f9dfdSAndroid Build Coastguard Worker 14*387f9dfdSAndroid Build Coastguard Workerfrom __future__ import print_function 15*387f9dfdSAndroid Build Coastguard Workerfrom bcc import BPF 16*387f9dfdSAndroid Build Coastguard Workerfrom time import sleep, strftime 17*387f9dfdSAndroid Build Coastguard Workerimport argparse 18*387f9dfdSAndroid Build Coastguard Worker 19*387f9dfdSAndroid Build Coastguard Worker# symbols 20*387f9dfdSAndroid Build Coastguard Workerkallsyms = "/proc/kallsyms" 21*387f9dfdSAndroid Build Coastguard Worker 22*387f9dfdSAndroid Build Coastguard Worker# arguments 23*387f9dfdSAndroid Build Coastguard Workerexamples = """examples: 24*387f9dfdSAndroid Build Coastguard Worker ./btrfsdist # show operation latency as a histogram 25*387f9dfdSAndroid Build Coastguard Worker ./btrfsdist -p 181 # trace PID 181 only 26*387f9dfdSAndroid Build Coastguard Worker ./btrfsdist 1 10 # print 1 second summaries, 10 times 27*387f9dfdSAndroid Build Coastguard Worker ./btrfsdist -m 5 # 5s summaries, milliseconds 28*387f9dfdSAndroid Build Coastguard Worker""" 29*387f9dfdSAndroid Build Coastguard Workerparser = argparse.ArgumentParser( 30*387f9dfdSAndroid Build Coastguard Worker description="Summarize btrfs operation latency", 31*387f9dfdSAndroid Build Coastguard Worker formatter_class=argparse.RawDescriptionHelpFormatter, 32*387f9dfdSAndroid Build Coastguard Worker epilog=examples) 33*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-T", "--notimestamp", action="store_true", 34*387f9dfdSAndroid Build Coastguard Worker help="don't include timestamp on interval output") 35*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-m", "--milliseconds", action="store_true", 36*387f9dfdSAndroid Build Coastguard Worker help="output in milliseconds") 37*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-p", "--pid", 38*387f9dfdSAndroid Build Coastguard Worker help="trace this PID only") 39*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("interval", nargs="?", 40*387f9dfdSAndroid Build Coastguard Worker help="output interval, in seconds") 41*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("count", nargs="?", default=99999999, 42*387f9dfdSAndroid Build Coastguard Worker help="number of outputs") 43*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("--ebpf", action="store_true", 44*387f9dfdSAndroid Build Coastguard Worker help=argparse.SUPPRESS) 45*387f9dfdSAndroid Build Coastguard Workerargs = parser.parse_args() 46*387f9dfdSAndroid Build Coastguard Workerpid = args.pid 47*387f9dfdSAndroid Build Coastguard Workercountdown = int(args.count) 48*387f9dfdSAndroid Build Coastguard Workerif args.milliseconds: 49*387f9dfdSAndroid Build Coastguard Worker factor = 1000000 50*387f9dfdSAndroid Build Coastguard Worker label = "msecs" 51*387f9dfdSAndroid Build Coastguard Workerelse: 52*387f9dfdSAndroid Build Coastguard Worker factor = 1000 53*387f9dfdSAndroid Build Coastguard Worker label = "usecs" 54*387f9dfdSAndroid Build Coastguard Workerif args.interval and int(args.interval) == 0: 55*387f9dfdSAndroid Build Coastguard Worker print("ERROR: interval 0. Exiting.") 56*387f9dfdSAndroid Build Coastguard Worker exit() 57*387f9dfdSAndroid Build Coastguard Workerdebug = 0 58*387f9dfdSAndroid Build Coastguard Worker 59*387f9dfdSAndroid Build Coastguard Worker# define BPF program 60*387f9dfdSAndroid Build Coastguard Workerbpf_text = """ 61*387f9dfdSAndroid Build Coastguard Worker#include <uapi/linux/ptrace.h> 62*387f9dfdSAndroid Build Coastguard Worker#include <linux/fs.h> 63*387f9dfdSAndroid Build Coastguard Worker#include <linux/sched.h> 64*387f9dfdSAndroid Build Coastguard Worker 65*387f9dfdSAndroid Build Coastguard Worker#define OP_NAME_LEN 8 66*387f9dfdSAndroid Build Coastguard Workertypedef struct dist_key { 67*387f9dfdSAndroid Build Coastguard Worker char op[OP_NAME_LEN]; 68*387f9dfdSAndroid Build Coastguard Worker u64 slot; 69*387f9dfdSAndroid Build Coastguard Worker} dist_key_t; 70*387f9dfdSAndroid Build Coastguard WorkerBPF_HASH(start, u32); 71*387f9dfdSAndroid Build Coastguard WorkerBPF_HISTOGRAM(dist, dist_key_t); 72*387f9dfdSAndroid Build Coastguard Worker 73*387f9dfdSAndroid Build Coastguard Worker// time operation 74*387f9dfdSAndroid Build Coastguard Workerint trace_entry(struct pt_regs *ctx) 75*387f9dfdSAndroid Build Coastguard Worker{ 76*387f9dfdSAndroid Build Coastguard Worker u64 pid_tgid = bpf_get_current_pid_tgid(); 77*387f9dfdSAndroid Build Coastguard Worker u32 pid = pid_tgid >> 32; 78*387f9dfdSAndroid Build Coastguard Worker u32 tid = (u32)pid_tgid; 79*387f9dfdSAndroid Build Coastguard Worker 80*387f9dfdSAndroid Build Coastguard Worker if (FILTER_PID) 81*387f9dfdSAndroid Build Coastguard Worker return 0; 82*387f9dfdSAndroid Build Coastguard Worker u64 ts = bpf_ktime_get_ns(); 83*387f9dfdSAndroid Build Coastguard Worker start.update(&tid, &ts); 84*387f9dfdSAndroid Build Coastguard Worker return 0; 85*387f9dfdSAndroid Build Coastguard Worker} 86*387f9dfdSAndroid Build Coastguard Worker 87*387f9dfdSAndroid Build Coastguard Worker// The current btrfs (Linux 4.5) uses generic_file_read_iter() instead of it's 88*387f9dfdSAndroid Build Coastguard Worker// own read function. So we need to trace that and then filter on btrfs, which 89*387f9dfdSAndroid Build Coastguard Worker// I do by checking file->f_op. 90*387f9dfdSAndroid Build Coastguard Workerint trace_read_entry(struct pt_regs *ctx, struct kiocb *iocb) 91*387f9dfdSAndroid Build Coastguard Worker{ 92*387f9dfdSAndroid Build Coastguard Worker u64 pid_tgid = bpf_get_current_pid_tgid(); 93*387f9dfdSAndroid Build Coastguard Worker u32 pid = pid_tgid >> 32; 94*387f9dfdSAndroid Build Coastguard Worker u32 tid = (u32)pid_tgid; 95*387f9dfdSAndroid Build Coastguard Worker 96*387f9dfdSAndroid Build Coastguard Worker if (FILTER_PID) 97*387f9dfdSAndroid Build Coastguard Worker return 0; 98*387f9dfdSAndroid Build Coastguard Worker 99*387f9dfdSAndroid Build Coastguard Worker // btrfs filter on file->f_op == btrfs_file_operations 100*387f9dfdSAndroid Build Coastguard Worker struct file *fp = iocb->ki_filp; 101*387f9dfdSAndroid Build Coastguard Worker if ((u64)fp->f_op != BTRFS_FILE_OPERATIONS) 102*387f9dfdSAndroid Build Coastguard Worker return 0; 103*387f9dfdSAndroid Build Coastguard Worker 104*387f9dfdSAndroid Build Coastguard Worker u64 ts = bpf_ktime_get_ns(); 105*387f9dfdSAndroid Build Coastguard Worker start.update(&tid, &ts); 106*387f9dfdSAndroid Build Coastguard Worker return 0; 107*387f9dfdSAndroid Build Coastguard Worker} 108*387f9dfdSAndroid Build Coastguard Worker 109*387f9dfdSAndroid Build Coastguard Worker// The current btrfs (Linux 4.5) uses generic_file_open(), instead of it's own 110*387f9dfdSAndroid Build Coastguard Worker// function. Same as with reads. Trace the generic path and filter: 111*387f9dfdSAndroid Build Coastguard Workerint trace_open_entry(struct pt_regs *ctx, struct inode *inode, 112*387f9dfdSAndroid Build Coastguard Worker struct file *file) 113*387f9dfdSAndroid Build Coastguard Worker{ 114*387f9dfdSAndroid Build Coastguard Worker u64 pid_tgid = bpf_get_current_pid_tgid(); 115*387f9dfdSAndroid Build Coastguard Worker u32 pid = pid_tgid >> 32; 116*387f9dfdSAndroid Build Coastguard Worker u32 tid = (u32)pid_tgid; 117*387f9dfdSAndroid Build Coastguard Worker 118*387f9dfdSAndroid Build Coastguard Worker if (FILTER_PID) 119*387f9dfdSAndroid Build Coastguard Worker return 0; 120*387f9dfdSAndroid Build Coastguard Worker 121*387f9dfdSAndroid Build Coastguard Worker // btrfs filter on file->f_op == btrfs_file_operations 122*387f9dfdSAndroid Build Coastguard Worker if ((u64)file->f_op != BTRFS_FILE_OPERATIONS) 123*387f9dfdSAndroid Build Coastguard Worker return 0; 124*387f9dfdSAndroid Build Coastguard Worker 125*387f9dfdSAndroid Build Coastguard Worker u64 ts = bpf_ktime_get_ns(); 126*387f9dfdSAndroid Build Coastguard Worker start.update(&tid, &ts); 127*387f9dfdSAndroid Build Coastguard Worker return 0; 128*387f9dfdSAndroid Build Coastguard Worker} 129*387f9dfdSAndroid Build Coastguard Worker 130*387f9dfdSAndroid Build Coastguard Workerstatic int trace_return(struct pt_regs *ctx, const char *op) 131*387f9dfdSAndroid Build Coastguard Worker{ 132*387f9dfdSAndroid Build Coastguard Worker u64 *tsp; 133*387f9dfdSAndroid Build Coastguard Worker u64 pid_tgid = bpf_get_current_pid_tgid(); 134*387f9dfdSAndroid Build Coastguard Worker u32 pid = pid_tgid >> 32; 135*387f9dfdSAndroid Build Coastguard Worker u32 tid = (u32)pid_tgid; 136*387f9dfdSAndroid Build Coastguard Worker 137*387f9dfdSAndroid Build Coastguard Worker // fetch timestamp and calculate delta 138*387f9dfdSAndroid Build Coastguard Worker tsp = start.lookup(&tid); 139*387f9dfdSAndroid Build Coastguard Worker if (tsp == 0) { 140*387f9dfdSAndroid Build Coastguard Worker return 0; // missed start or filtered 141*387f9dfdSAndroid Build Coastguard Worker } 142*387f9dfdSAndroid Build Coastguard Worker u64 delta = (bpf_ktime_get_ns() - *tsp) / FACTOR; 143*387f9dfdSAndroid Build Coastguard Worker 144*387f9dfdSAndroid Build Coastguard Worker // store as histogram 145*387f9dfdSAndroid Build Coastguard Worker dist_key_t key = {.slot = bpf_log2l(delta)}; 146*387f9dfdSAndroid Build Coastguard Worker __builtin_memcpy(&key.op, op, sizeof(key.op)); 147*387f9dfdSAndroid Build Coastguard Worker dist.atomic_increment(key); 148*387f9dfdSAndroid Build Coastguard Worker 149*387f9dfdSAndroid Build Coastguard Worker start.delete(&tid); 150*387f9dfdSAndroid Build Coastguard Worker return 0; 151*387f9dfdSAndroid Build Coastguard Worker} 152*387f9dfdSAndroid Build Coastguard Worker 153*387f9dfdSAndroid Build Coastguard Workerint trace_read_return(struct pt_regs *ctx) 154*387f9dfdSAndroid Build Coastguard Worker{ 155*387f9dfdSAndroid Build Coastguard Worker char *op = "read"; 156*387f9dfdSAndroid Build Coastguard Worker return trace_return(ctx, op); 157*387f9dfdSAndroid Build Coastguard Worker} 158*387f9dfdSAndroid Build Coastguard Worker 159*387f9dfdSAndroid Build Coastguard Workerint trace_write_return(struct pt_regs *ctx) 160*387f9dfdSAndroid Build Coastguard Worker{ 161*387f9dfdSAndroid Build Coastguard Worker char *op = "write"; 162*387f9dfdSAndroid Build Coastguard Worker return trace_return(ctx, op); 163*387f9dfdSAndroid Build Coastguard Worker} 164*387f9dfdSAndroid Build Coastguard Worker 165*387f9dfdSAndroid Build Coastguard Workerint trace_open_return(struct pt_regs *ctx) 166*387f9dfdSAndroid Build Coastguard Worker{ 167*387f9dfdSAndroid Build Coastguard Worker char *op = "open"; 168*387f9dfdSAndroid Build Coastguard Worker return trace_return(ctx, op); 169*387f9dfdSAndroid Build Coastguard Worker} 170*387f9dfdSAndroid Build Coastguard Worker 171*387f9dfdSAndroid Build Coastguard Workerint trace_fsync_return(struct pt_regs *ctx) 172*387f9dfdSAndroid Build Coastguard Worker{ 173*387f9dfdSAndroid Build Coastguard Worker char *op = "fsync"; 174*387f9dfdSAndroid Build Coastguard Worker return trace_return(ctx, op); 175*387f9dfdSAndroid Build Coastguard Worker} 176*387f9dfdSAndroid Build Coastguard Worker""" 177*387f9dfdSAndroid Build Coastguard Worker 178*387f9dfdSAndroid Build Coastguard Worker# code replacements 179*387f9dfdSAndroid Build Coastguard Workerwith open(kallsyms) as syms: 180*387f9dfdSAndroid Build Coastguard Worker ops = '' 181*387f9dfdSAndroid Build Coastguard Worker for line in syms: 182*387f9dfdSAndroid Build Coastguard Worker a = line.rstrip().split() 183*387f9dfdSAndroid Build Coastguard Worker (addr, name) = (a[0], a[2]) 184*387f9dfdSAndroid Build Coastguard Worker name = name.split("\t")[0] 185*387f9dfdSAndroid Build Coastguard Worker if name == "btrfs_file_operations": 186*387f9dfdSAndroid Build Coastguard Worker ops = "0x" + addr 187*387f9dfdSAndroid Build Coastguard Worker break 188*387f9dfdSAndroid Build Coastguard Worker if ops == '': 189*387f9dfdSAndroid Build Coastguard Worker print("ERROR: no btrfs_file_operations in /proc/kallsyms. Exiting.") 190*387f9dfdSAndroid Build Coastguard Worker print("HINT: the kernel should be built with CONFIG_KALLSYMS_ALL.") 191*387f9dfdSAndroid Build Coastguard Worker exit() 192*387f9dfdSAndroid Build Coastguard Worker bpf_text = bpf_text.replace('BTRFS_FILE_OPERATIONS', ops) 193*387f9dfdSAndroid Build Coastguard Workerbpf_text = bpf_text.replace('FACTOR', str(factor)) 194*387f9dfdSAndroid Build Coastguard Workerif args.pid: 195*387f9dfdSAndroid Build Coastguard Worker bpf_text = bpf_text.replace('FILTER_PID', 'pid != %s' % pid) 196*387f9dfdSAndroid Build Coastguard Workerelse: 197*387f9dfdSAndroid Build Coastguard Worker bpf_text = bpf_text.replace('FILTER_PID', '0') 198*387f9dfdSAndroid Build Coastguard Workerif debug or args.ebpf: 199*387f9dfdSAndroid Build Coastguard Worker print(bpf_text) 200*387f9dfdSAndroid Build Coastguard Worker if args.ebpf: 201*387f9dfdSAndroid Build Coastguard Worker exit() 202*387f9dfdSAndroid Build Coastguard Worker 203*387f9dfdSAndroid Build Coastguard Worker# load BPF program 204*387f9dfdSAndroid Build Coastguard Workerb = BPF(text=bpf_text) 205*387f9dfdSAndroid Build Coastguard Worker 206*387f9dfdSAndroid Build Coastguard Worker# Common file functions. See earlier comment about generic_file_read_iter(). 207*387f9dfdSAndroid Build Coastguard Workerb.attach_kprobe(event="generic_file_read_iter", fn_name="trace_read_entry") 208*387f9dfdSAndroid Build Coastguard Workerb.attach_kprobe(event="btrfs_file_write_iter", fn_name="trace_entry") 209*387f9dfdSAndroid Build Coastguard Workerb.attach_kprobe(event="generic_file_open", fn_name="trace_open_entry") 210*387f9dfdSAndroid Build Coastguard Workerb.attach_kprobe(event="btrfs_sync_file", fn_name="trace_entry") 211*387f9dfdSAndroid Build Coastguard Workerb.attach_kretprobe(event="generic_file_read_iter", fn_name="trace_read_return") 212*387f9dfdSAndroid Build Coastguard Workerb.attach_kretprobe(event="btrfs_file_write_iter", fn_name="trace_write_return") 213*387f9dfdSAndroid Build Coastguard Workerb.attach_kretprobe(event="generic_file_open", fn_name="trace_open_return") 214*387f9dfdSAndroid Build Coastguard Workerb.attach_kretprobe(event="btrfs_sync_file", fn_name="trace_fsync_return") 215*387f9dfdSAndroid Build Coastguard Worker 216*387f9dfdSAndroid Build Coastguard Workerprint("Tracing btrfs operation latency... Hit Ctrl-C to end.") 217*387f9dfdSAndroid Build Coastguard Worker 218*387f9dfdSAndroid Build Coastguard Worker# output 219*387f9dfdSAndroid Build Coastguard Workerexiting = 0 220*387f9dfdSAndroid Build Coastguard Workerdist = b.get_table("dist") 221*387f9dfdSAndroid Build Coastguard Workerwhile (1): 222*387f9dfdSAndroid Build Coastguard Worker try: 223*387f9dfdSAndroid Build Coastguard Worker if args.interval: 224*387f9dfdSAndroid Build Coastguard Worker sleep(int(args.interval)) 225*387f9dfdSAndroid Build Coastguard Worker else: 226*387f9dfdSAndroid Build Coastguard Worker sleep(99999999) 227*387f9dfdSAndroid Build Coastguard Worker except KeyboardInterrupt: 228*387f9dfdSAndroid Build Coastguard Worker exiting = 1 229*387f9dfdSAndroid Build Coastguard Worker 230*387f9dfdSAndroid Build Coastguard Worker print() 231*387f9dfdSAndroid Build Coastguard Worker if args.interval and (not args.notimestamp): 232*387f9dfdSAndroid Build Coastguard Worker print(strftime("%H:%M:%S:")) 233*387f9dfdSAndroid Build Coastguard Worker 234*387f9dfdSAndroid Build Coastguard Worker dist.print_log2_hist(label, "operation", section_print_fn=bytes.decode) 235*387f9dfdSAndroid Build Coastguard Worker dist.clear() 236*387f9dfdSAndroid Build Coastguard Worker 237*387f9dfdSAndroid Build Coastguard Worker countdown -= 1 238*387f9dfdSAndroid Build Coastguard Worker if exiting or countdown == 0: 239*387f9dfdSAndroid Build Coastguard Worker exit() 240