1*387f9dfdSAndroid Build Coastguard Worker#!/usr/bin/env python 2*387f9dfdSAndroid Build Coastguard Worker# @lint-avoid-python-3-compatibility-imports 3*387f9dfdSAndroid Build Coastguard Worker# 4*387f9dfdSAndroid Build Coastguard Worker# zfsdist Summarize ZFS operation latency. 5*387f9dfdSAndroid Build Coastguard Worker# For Linux, uses BCC, eBPF. 6*387f9dfdSAndroid Build Coastguard Worker# 7*387f9dfdSAndroid Build Coastguard Worker# USAGE: zfsdist [-h] [-T] [-m] [-p PID] [interval] [count] 8*387f9dfdSAndroid Build Coastguard Worker# 9*387f9dfdSAndroid Build Coastguard Worker# Copyright 2016 Netflix, Inc. 10*387f9dfdSAndroid Build Coastguard Worker# Licensed under the Apache License, Version 2.0 (the "License") 11*387f9dfdSAndroid Build Coastguard Worker# 12*387f9dfdSAndroid Build Coastguard Worker# 14-Feb-2016 Brendan Gregg Created this. 13*387f9dfdSAndroid Build Coastguard Worker 14*387f9dfdSAndroid Build Coastguard Workerfrom __future__ import print_function 15*387f9dfdSAndroid Build Coastguard Workerfrom bcc import BPF 16*387f9dfdSAndroid Build Coastguard Workerfrom time import sleep, strftime 17*387f9dfdSAndroid Build Coastguard Workerimport argparse 18*387f9dfdSAndroid Build Coastguard Worker 19*387f9dfdSAndroid Build Coastguard Worker# arguments 20*387f9dfdSAndroid Build Coastguard Workerexamples = """examples: 21*387f9dfdSAndroid Build Coastguard Worker ./zfsdist # show operation latency as a histogram 22*387f9dfdSAndroid Build Coastguard Worker ./zfsdist -p 181 # trace PID 181 only 23*387f9dfdSAndroid Build Coastguard Worker ./zfsdist 1 10 # print 1 second summaries, 10 times 24*387f9dfdSAndroid Build Coastguard Worker ./zfsdist -m 5 # 5s summaries, milliseconds 25*387f9dfdSAndroid Build Coastguard Worker""" 26*387f9dfdSAndroid Build Coastguard Workerparser = argparse.ArgumentParser( 27*387f9dfdSAndroid Build Coastguard Worker description="Summarize ZFS operation latency", 28*387f9dfdSAndroid Build Coastguard Worker formatter_class=argparse.RawDescriptionHelpFormatter, 29*387f9dfdSAndroid Build Coastguard Worker epilog=examples) 30*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-T", "--notimestamp", action="store_true", 31*387f9dfdSAndroid Build Coastguard Worker help="don't include timestamp on interval output") 32*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-m", "--milliseconds", action="store_true", 33*387f9dfdSAndroid Build Coastguard Worker help="output in milliseconds") 34*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("-p", "--pid", 35*387f9dfdSAndroid Build Coastguard Worker help="trace this PID only") 36*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("interval", nargs="?", 37*387f9dfdSAndroid Build Coastguard Worker help="output interval, in seconds") 38*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("count", nargs="?", default=99999999, 39*387f9dfdSAndroid Build Coastguard Worker help="number of outputs") 40*387f9dfdSAndroid Build Coastguard Workerparser.add_argument("--ebpf", action="store_true", 41*387f9dfdSAndroid Build Coastguard Worker help=argparse.SUPPRESS) 42*387f9dfdSAndroid Build Coastguard Workerargs = parser.parse_args() 43*387f9dfdSAndroid Build Coastguard Workerpid = args.pid 44*387f9dfdSAndroid Build Coastguard Workercountdown = int(args.count) 45*387f9dfdSAndroid Build Coastguard Workerif args.milliseconds: 46*387f9dfdSAndroid Build Coastguard Worker factor = 1000000 47*387f9dfdSAndroid Build Coastguard Worker label = "msecs" 48*387f9dfdSAndroid Build Coastguard Workerelse: 49*387f9dfdSAndroid Build Coastguard Worker factor = 1000 50*387f9dfdSAndroid Build Coastguard Worker label = "usecs" 51*387f9dfdSAndroid Build Coastguard Workerif args.interval and int(args.interval) == 0: 52*387f9dfdSAndroid Build Coastguard Worker print("ERROR: interval 0. Exiting.") 53*387f9dfdSAndroid Build Coastguard Worker exit() 54*387f9dfdSAndroid Build Coastguard Workerdebug = 0 55*387f9dfdSAndroid Build Coastguard Worker 56*387f9dfdSAndroid Build Coastguard Worker# define BPF program 57*387f9dfdSAndroid Build Coastguard Workerbpf_text = """ 58*387f9dfdSAndroid Build Coastguard Worker#include <uapi/linux/ptrace.h> 59*387f9dfdSAndroid Build Coastguard Worker#include <linux/fs.h> 60*387f9dfdSAndroid Build Coastguard Worker#include <linux/sched.h> 61*387f9dfdSAndroid Build Coastguard Worker 62*387f9dfdSAndroid Build Coastguard Worker#define OP_NAME_LEN 8 63*387f9dfdSAndroid Build Coastguard Workertypedef struct dist_key { 64*387f9dfdSAndroid Build Coastguard Worker char op[OP_NAME_LEN]; 65*387f9dfdSAndroid Build Coastguard Worker u64 slot; 66*387f9dfdSAndroid Build Coastguard Worker} dist_key_t; 67*387f9dfdSAndroid Build Coastguard WorkerBPF_HASH(start, u32); 68*387f9dfdSAndroid Build Coastguard WorkerBPF_HISTOGRAM(dist, dist_key_t); 69*387f9dfdSAndroid Build Coastguard Worker 70*387f9dfdSAndroid Build Coastguard Worker// time operation 71*387f9dfdSAndroid Build Coastguard Workerint trace_entry(struct pt_regs *ctx) 72*387f9dfdSAndroid Build Coastguard Worker{ 73*387f9dfdSAndroid Build Coastguard Worker u64 pid_tgid = bpf_get_current_pid_tgid(); 74*387f9dfdSAndroid Build Coastguard Worker u32 pid = pid_tgid >> 32; 75*387f9dfdSAndroid Build Coastguard Worker u32 tid = (u32)pid_tgid; 76*387f9dfdSAndroid Build Coastguard Worker 77*387f9dfdSAndroid Build Coastguard Worker if (FILTER_PID) 78*387f9dfdSAndroid Build Coastguard Worker return 0; 79*387f9dfdSAndroid Build Coastguard Worker u64 ts = bpf_ktime_get_ns(); 80*387f9dfdSAndroid Build Coastguard Worker start.update(&tid, &ts); 81*387f9dfdSAndroid Build Coastguard Worker return 0; 82*387f9dfdSAndroid Build Coastguard Worker} 83*387f9dfdSAndroid Build Coastguard Worker 84*387f9dfdSAndroid Build Coastguard Workerstatic int trace_return(struct pt_regs *ctx, const char *op) 85*387f9dfdSAndroid Build Coastguard Worker{ 86*387f9dfdSAndroid Build Coastguard Worker u64 *tsp; 87*387f9dfdSAndroid Build Coastguard Worker u64 pid_tgid = bpf_get_current_pid_tgid(); 88*387f9dfdSAndroid Build Coastguard Worker u32 pid = pid_tgid >> 32; 89*387f9dfdSAndroid Build Coastguard Worker u32 tid = (u32)pid_tgid; 90*387f9dfdSAndroid Build Coastguard Worker 91*387f9dfdSAndroid Build Coastguard Worker // fetch timestamp and calculate delta 92*387f9dfdSAndroid Build Coastguard Worker tsp = start.lookup(&tid); 93*387f9dfdSAndroid Build Coastguard Worker if (tsp == 0) { 94*387f9dfdSAndroid Build Coastguard Worker return 0; // missed start or filtered 95*387f9dfdSAndroid Build Coastguard Worker } 96*387f9dfdSAndroid Build Coastguard Worker u64 delta = (bpf_ktime_get_ns() - *tsp) / FACTOR; 97*387f9dfdSAndroid Build Coastguard Worker 98*387f9dfdSAndroid Build Coastguard Worker // store as histogram 99*387f9dfdSAndroid Build Coastguard Worker dist_key_t key = {.slot = bpf_log2l(delta)}; 100*387f9dfdSAndroid Build Coastguard Worker __builtin_memcpy(&key.op, op, sizeof(key.op)); 101*387f9dfdSAndroid Build Coastguard Worker dist.atomic_increment(key); 102*387f9dfdSAndroid Build Coastguard Worker 103*387f9dfdSAndroid Build Coastguard Worker start.delete(&tid); 104*387f9dfdSAndroid Build Coastguard Worker return 0; 105*387f9dfdSAndroid Build Coastguard Worker} 106*387f9dfdSAndroid Build Coastguard Worker 107*387f9dfdSAndroid Build Coastguard Workerint trace_read_return(struct pt_regs *ctx) 108*387f9dfdSAndroid Build Coastguard Worker{ 109*387f9dfdSAndroid Build Coastguard Worker char *op = "read"; 110*387f9dfdSAndroid Build Coastguard Worker return trace_return(ctx, op); 111*387f9dfdSAndroid Build Coastguard Worker} 112*387f9dfdSAndroid Build Coastguard Worker 113*387f9dfdSAndroid Build Coastguard Workerint trace_write_return(struct pt_regs *ctx) 114*387f9dfdSAndroid Build Coastguard Worker{ 115*387f9dfdSAndroid Build Coastguard Worker char *op = "write"; 116*387f9dfdSAndroid Build Coastguard Worker return trace_return(ctx, op); 117*387f9dfdSAndroid Build Coastguard Worker} 118*387f9dfdSAndroid Build Coastguard Worker 119*387f9dfdSAndroid Build Coastguard Workerint trace_open_return(struct pt_regs *ctx) 120*387f9dfdSAndroid Build Coastguard Worker{ 121*387f9dfdSAndroid Build Coastguard Worker char *op = "open"; 122*387f9dfdSAndroid Build Coastguard Worker return trace_return(ctx, op); 123*387f9dfdSAndroid Build Coastguard Worker} 124*387f9dfdSAndroid Build Coastguard Worker 125*387f9dfdSAndroid Build Coastguard Workerint trace_fsync_return(struct pt_regs *ctx) 126*387f9dfdSAndroid Build Coastguard Worker{ 127*387f9dfdSAndroid Build Coastguard Worker char *op = "fsync"; 128*387f9dfdSAndroid Build Coastguard Worker return trace_return(ctx, op); 129*387f9dfdSAndroid Build Coastguard Worker} 130*387f9dfdSAndroid Build Coastguard Worker""" 131*387f9dfdSAndroid Build Coastguard Workerbpf_text = bpf_text.replace('FACTOR', str(factor)) 132*387f9dfdSAndroid Build Coastguard Workerif args.pid: 133*387f9dfdSAndroid Build Coastguard Worker bpf_text = bpf_text.replace('FILTER_PID', 'pid != %s' % pid) 134*387f9dfdSAndroid Build Coastguard Workerelse: 135*387f9dfdSAndroid Build Coastguard Worker bpf_text = bpf_text.replace('FILTER_PID', '0') 136*387f9dfdSAndroid Build Coastguard Workerif debug or args.ebpf: 137*387f9dfdSAndroid Build Coastguard Worker print(bpf_text) 138*387f9dfdSAndroid Build Coastguard Worker if args.ebpf: 139*387f9dfdSAndroid Build Coastguard Worker exit() 140*387f9dfdSAndroid Build Coastguard Worker 141*387f9dfdSAndroid Build Coastguard Worker# load BPF program 142*387f9dfdSAndroid Build Coastguard Workerb = BPF(text=bpf_text) 143*387f9dfdSAndroid Build Coastguard Worker 144*387f9dfdSAndroid Build Coastguard Worker# common file functions 145*387f9dfdSAndroid Build Coastguard Workerif BPF.get_kprobe_functions(b'zpl_iter'): 146*387f9dfdSAndroid Build Coastguard Worker b.attach_kprobe(event="zpl_iter_read", fn_name="trace_entry") 147*387f9dfdSAndroid Build Coastguard Worker b.attach_kprobe(event="zpl_iter_write", fn_name="trace_entry") 148*387f9dfdSAndroid Build Coastguard Workerelif BPF.get_kprobe_functions(b'zpl_aio'): 149*387f9dfdSAndroid Build Coastguard Worker b.attach_kprobe(event="zpl_aio_read", fn_name="trace_entry") 150*387f9dfdSAndroid Build Coastguard Worker b.attach_kprobe(event="zpl_aio_write", fn_name="trace_entry") 151*387f9dfdSAndroid Build Coastguard Workerelse: 152*387f9dfdSAndroid Build Coastguard Worker b.attach_kprobe(event="zpl_read", fn_name="trace_entry") 153*387f9dfdSAndroid Build Coastguard Worker b.attach_kprobe(event="zpl_write", fn_name="trace_entry") 154*387f9dfdSAndroid Build Coastguard Workerb.attach_kprobe(event="zpl_open", fn_name="trace_entry") 155*387f9dfdSAndroid Build Coastguard Workerb.attach_kprobe(event="zpl_fsync", fn_name="trace_entry") 156*387f9dfdSAndroid Build Coastguard Workerif BPF.get_kprobe_functions(b'zpl_iter'): 157*387f9dfdSAndroid Build Coastguard Worker b.attach_kretprobe(event="zpl_iter_read", fn_name="trace_read_return") 158*387f9dfdSAndroid Build Coastguard Worker b.attach_kretprobe(event="zpl_iter_write", fn_name="trace_write_return") 159*387f9dfdSAndroid Build Coastguard Workerelif BPF.get_kprobe_functions(b'zpl_aio'): 160*387f9dfdSAndroid Build Coastguard Worker b.attach_kretprobe(event="zpl_aio_read", fn_name="trace_read_return") 161*387f9dfdSAndroid Build Coastguard Worker b.attach_kretprobe(event="zpl_aio_write", fn_name="trace_write_return") 162*387f9dfdSAndroid Build Coastguard Workerelse: 163*387f9dfdSAndroid Build Coastguard Worker b.attach_kretprobe(event="zpl_read", fn_name="trace_read_return") 164*387f9dfdSAndroid Build Coastguard Worker b.attach_kretprobe(event="zpl_write", fn_name="trace_write_return") 165*387f9dfdSAndroid Build Coastguard Workerb.attach_kretprobe(event="zpl_open", fn_name="trace_open_return") 166*387f9dfdSAndroid Build Coastguard Workerb.attach_kretprobe(event="zpl_fsync", fn_name="trace_fsync_return") 167*387f9dfdSAndroid Build Coastguard Worker 168*387f9dfdSAndroid Build Coastguard Workerprint("Tracing ZFS operation latency... Hit Ctrl-C to end.") 169*387f9dfdSAndroid Build Coastguard Worker 170*387f9dfdSAndroid Build Coastguard Worker# output 171*387f9dfdSAndroid Build Coastguard Workerexiting = 0 172*387f9dfdSAndroid Build Coastguard Workerdist = b.get_table("dist") 173*387f9dfdSAndroid Build Coastguard Workerwhile (1): 174*387f9dfdSAndroid Build Coastguard Worker try: 175*387f9dfdSAndroid Build Coastguard Worker if args.interval: 176*387f9dfdSAndroid Build Coastguard Worker sleep(int(args.interval)) 177*387f9dfdSAndroid Build Coastguard Worker else: 178*387f9dfdSAndroid Build Coastguard Worker sleep(99999999) 179*387f9dfdSAndroid Build Coastguard Worker except KeyboardInterrupt: 180*387f9dfdSAndroid Build Coastguard Worker exiting = 1 181*387f9dfdSAndroid Build Coastguard Worker 182*387f9dfdSAndroid Build Coastguard Worker print() 183*387f9dfdSAndroid Build Coastguard Worker if args.interval and (not args.notimestamp): 184*387f9dfdSAndroid Build Coastguard Worker print(strftime("%H:%M:%S:")) 185*387f9dfdSAndroid Build Coastguard Worker 186*387f9dfdSAndroid Build Coastguard Worker dist.print_log2_hist(label, "operation", section_print_fn=bytes.decode) 187*387f9dfdSAndroid Build Coastguard Worker dist.clear() 188*387f9dfdSAndroid Build Coastguard Worker 189*387f9dfdSAndroid Build Coastguard Worker countdown -= 1 190*387f9dfdSAndroid Build Coastguard Worker if exiting or countdown == 0: 191*387f9dfdSAndroid Build Coastguard Worker exit() 192