1#!/usr/bin/env python 2 3import argparse 4import numpy as np 5import pandas as pd 6 7from bokeh.layouts import column 8from bokeh.models import ColumnDataSource 9from bokeh.models import CustomJSHover 10from bokeh.models import HoverTool 11from bokeh.models import LabelSet 12from bokeh.models import TapTool 13from bokeh.models import VSpan 14from bokeh.plotting import figure 15from bokeh.plotting import output_file 16from bokeh.plotting import save 17from bokeh.plotting import show 18from typing import Tuple 19 20SHADES_OF_GREEN = ['GreenYellow', 'LimeGreen', 'LightGreen', 'SeaGreen', 'Green', 'DarkGreen', 21 'YellowGreen', 'DarkOliveGreen', 'MediumAquaMarine', 'Teal'] 22PLOT_HEIGHT = 600 23PLOT_WIDTH = 1500 24X_AXIS_FIELD = 'monitor_start_relative_millis' 25X_AXIS_LABEL = 'Time since start of PSI monitor (millis)' 26MILLIS_TO_SECONDS = CustomJSHover(code=""" 27 if (value < 1000) 28 return value + 'ms' 29 return (value / 1000) +'s';""") 30 31def add_events_to_plot(plot, events_df, vspan_opts, fixed_y_values=0, color_list=None) -> figure: 32 vspan_source = dict(x_values=[], color=[], y_values=[], desc=[]) 33 for index, row in events_df.iterrows(): 34 vspan_source['x_values'].append(row[X_AXIS_FIELD]) 35 vspan_source['y_values'].append(fixed_y_values) 36 vspan_source['desc'].append(row['event_description']) 37 if 'color' in row: 38 vspan_source['color'].append(row.color) 39 elif color_list: 40 vspan_source['color'].append(color_list[index % len(color_list)]) 41 else: 42 vspan_source['color'].append('black') 43 line_width = 4 if 'is_high_latency_event' in row and row['is_high_latency_event'] else 2 44 45 glyph = VSpan(x='x_values', line_width=line_width, line_dash='dashed', line_color='color') 46 plot.add_glyph(ColumnDataSource(data=vspan_source), glyph) 47 plot.add_layout(LabelSet(**vspan_opts, source=ColumnDataSource(data=vspan_source))) 48 return plot 49 50 51# Bokeh cannot show tooltip for multi line plot. The workaround for this is to plot one scatter plot 52# per plot in a multiline plot and add the tooltips for these scatter plots. Then overlay 53# the scatter plots on the multi line plot. 54def plot_tooltips(x_values, multi_line_plot_y_values, multi_line_plot_desc, plot) -> figure: 55 for index, line_plot_y_values in enumerate(multi_line_plot_y_values): 56 source = ColumnDataSource({'X': x_values, 'Y': line_plot_y_values, 57 'Description': [multi_line_plot_desc[index]] * x_values.size}) 58 r = plot.scatter('X', 'Y', source = source, fill_alpha=0, line_alpha=0.3, line_color="grey") 59 hover = HoverTool(tooltips=[("X", "@X{custom}"), ("PSI", "@Y"), ("Desc", "@Description")], 60 formatters={'@X': MILLIS_TO_SECONDS}, renderers=[r]) 61 plot.add_tools(hover) 62 return plot 63 64 65def get_psi_plot(cuj_name, psi_df, x_range_list) -> figure: 66 if psi_df.empty: 67 return None 68 69 plot = figure(title="PSI metrics during CUJ '" + cuj_name + "'", width=PLOT_WIDTH, 70 height=PLOT_HEIGHT, x_axis_label=X_AXIS_LABEL, 71 y_axis_label='PSI %', x_range=x_range_list, y_range=[0, 100], 72 x_axis_type='datetime') 73 74 psi_source_dict = dict( 75 x_values=[], 76 y_values=[psi_df.memory_full_avg10.values, psi_df.io_full_avg10.values, 77 psi_df.cpu_some_avg10.values, psi_df.memory_some_avg10.values, 78 psi_df.io_some_avg10.values], 79 color=['red', 'blue', 'sandybrown', 'yellow', 'purple'], 80 desc=['Memory full avg10', 'IO full avg10', 'CPU some avg10', 'Memory some avg10', 81 'IO some avg10']) 82 if 'cpu_full_avg10' in psi_df.columns: 83 psi_source_dict['y_values'].append(psi_df.cpu_full_avg10.values) 84 psi_source_dict['color'].append('black') 85 psi_source_dict['desc'].append('CPU full avg10') 86 87 if 'irq_full_avg10' in psi_df.columns: 88 psi_source_dict['y_values'].append(psi_df.irq_full_avg10.values) 89 psi_source_dict['color'].append('pink') 90 psi_source_dict['desc'].append('IRQ full avg10') 91 92 x_values = pd.to_timedelta(pd.Series(psi_df[X_AXIS_FIELD].values), unit='ms') 93 psi_source_dict['x_values'] = [x_values for _ in psi_source_dict['y_values']] 94 95 source=ColumnDataSource(data=psi_source_dict) 96 plot.multi_line(xs='x_values', ys='y_values', source=source, 97 color='color', legend_field='desc', line_width=3) 98 plot = plot_tooltips(x_values, psi_source_dict['y_values'], psi_source_dict['desc'], plot) 99 100 vspan_opts = dict(x='x_values', y='y_values', text='desc', x_units='data', y_units='data', 101 level='annotation', angle=90, angle_units='deg', x_offset=-15, y_offset=-200, 102 text_font_size='10pt', text_color='black', text_alpha=1.0, text_align='center', 103 text_baseline='middle') 104 105 psi_events_df = psi_df[psi_df['event_description'].notna()].reset_index() 106 if not psi_events_df.empty: 107 plot = add_events_to_plot(plot, psi_events_df, vspan_opts, 100, SHADES_OF_GREEN) 108 109 return plot 110 111 112def get_events_plot(cuj_name, events_df, x_range_obj) -> figure: 113 if events_df.empty: 114 return None 115 116 plot = figure(title="Events during CUJ '" + cuj_name + "'", width=PLOT_WIDTH, height=PLOT_HEIGHT, 117 x_axis_label=X_AXIS_LABEL, y_axis_label='Event', x_range=x_range_obj, 118 y_range=[0, 100], x_axis_type='datetime') 119 hover = HoverTool(tooltips=[("X", "@x_values{custom}"), ("Event", "@desc")], 120 formatters={'@x_values': MILLIS_TO_SECONDS}) 121 plot.add_tools(hover) 122 123 vspan_opts = dict(x='x_values', y='y_values', x_units='data', y_units='data') 124 125 plot = add_events_to_plot(plot, events_df, vspan_opts) 126 127 return plot 128 129 130def save_and_show_plot(plot, should_show_plot, out_file): 131 output_file(out_file) 132 save(plot) 133 if should_show_plot: 134 show(plot) 135 136 137def get_x_range_list(psi_df, events_df) -> Tuple[int, int]: 138 if events_df is not None and not events_df.empty: 139 return (min(events_df[X_AXIS_FIELD].min(), psi_df[X_AXIS_FIELD].min()), 140 max(events_df[X_AXIS_FIELD].max(), psi_df[X_AXIS_FIELD].max())) 141 return (psi_df[X_AXIS_FIELD].min(), psi_df[X_AXIS_FIELD].max()) 142 143 144def generate_plot(cuj_name, psi_df, events_df, should_show_plot, out_file): 145 # psi_df has event_descriptions (which is a list of all events happened on a polling event) 146 # while events_df has event_description (which is a single event). To keep the dfs consistent, 147 # concatenate all events from event_descriptions in psi_df as a single event string. 148 psi_df['event_description'] = psi_df['event_descriptions'].apply( 149 lambda x: ','.join(x) if len(x) > 0 else np.nan) 150 psi_plot = get_psi_plot(cuj_name, psi_df, get_x_range_list(psi_df, events_df)) 151 events_plot = None 152 153 if events_df is not None: 154 events_plot = get_events_plot(cuj_name, events_df, psi_plot.x_range) 155 156 if events_plot: 157 plot = column([psi_plot, events_plot], sizing_mode='stretch_both') 158 else: 159 plot = psi_plot 160 161 save_and_show_plot(plot, should_show_plot, out_file) 162 163 164def main(): 165 parser = argparse.ArgumentParser( 166 description='Plot PSI csv dump from PSI monitor.', 167 ) 168 parser.add_argument('--psi_csv', action='store', type=str, required=True, dest='psi_csv', 169 help='PSI csv dump from psi_monitor.sh') 170 parser.add_argument('--events_csv', action='store', type=str, dest='events_csv', 171 help='Events csv dump from run_cuj.sh') 172 parser.add_argument('--cuj_name', action='store', type=str, dest='cuj_name', 173 default='Unknown CUJ', help='Name of the CUJ') 174 parser.add_argument('--out_file', action='store', type=str, dest='out_file', 175 default='psi_plot.html', help='Output HTML file') 176 parser.add_argument('--show_plot', action='store_true', dest='show_plot', 177 default=False, help='Show the plot') 178 args = parser.parse_args() 179 180 print('Plotting PSI csv dump {}'.format(args.psi_csv)) 181 psi_df = pd.read_csv(args.psi_csv, converters={'event_descriptions': pd.eval}) 182 events_df = None 183 if args.events_csv: 184 print('Plotting PSI events csv dump {}'.format(args.events_csv)) 185 events_df = pd.read_csv(args.events_csv).drop_duplicates() 186 events_df = events_df[events_df['should_plot']] 187 188 generate_plot(args.cuj_name, psi_df, events_df, args.show_plot, args.out_file) 189 190if __name__ == '__main__': 191 main() 192