1#!/usr/bin/env python
2
3import argparse
4import numpy as np
5import pandas as pd
6
7from bokeh.layouts import column
8from bokeh.models import ColumnDataSource
9from bokeh.models import CustomJSHover
10from bokeh.models import HoverTool
11from bokeh.models import LabelSet
12from bokeh.models import TapTool
13from bokeh.models import VSpan
14from bokeh.plotting import figure
15from bokeh.plotting import output_file
16from bokeh.plotting import save
17from bokeh.plotting import show
18from typing import Tuple
19
20SHADES_OF_GREEN = ['GreenYellow', 'LimeGreen', 'LightGreen', 'SeaGreen', 'Green', 'DarkGreen',
21                 'YellowGreen', 'DarkOliveGreen', 'MediumAquaMarine', 'Teal']
22PLOT_HEIGHT = 600
23PLOT_WIDTH = 1500
24X_AXIS_FIELD = 'monitor_start_relative_millis'
25X_AXIS_LABEL = 'Time since start of PSI monitor (millis)'
26MILLIS_TO_SECONDS = CustomJSHover(code="""
27  if (value < 1000)
28    return value + 'ms'
29  return (value / 1000) +'s';""")
30
31def add_events_to_plot(plot, events_df, vspan_opts, fixed_y_values=0, color_list=None) -> figure:
32  vspan_source = dict(x_values=[], color=[], y_values=[], desc=[])
33  for index, row in events_df.iterrows():
34    vspan_source['x_values'].append(row[X_AXIS_FIELD])
35    vspan_source['y_values'].append(fixed_y_values)
36    vspan_source['desc'].append(row['event_description'])
37    if 'color' in row:
38      vspan_source['color'].append(row.color)
39    elif color_list:
40      vspan_source['color'].append(color_list[index % len(color_list)])
41    else:
42      vspan_source['color'].append('black')
43    line_width = 4 if 'is_high_latency_event' in row and row['is_high_latency_event'] else 2
44
45  glyph = VSpan(x='x_values', line_width=line_width, line_dash='dashed', line_color='color')
46  plot.add_glyph(ColumnDataSource(data=vspan_source), glyph)
47  plot.add_layout(LabelSet(**vspan_opts, source=ColumnDataSource(data=vspan_source)))
48  return plot
49
50
51# Bokeh cannot show tooltip for multi line plot. The workaround for this is to plot one scatter plot
52# per plot in a multiline plot and add the tooltips for these scatter plots. Then overlay
53# the scatter plots on the multi line plot.
54def plot_tooltips(x_values, multi_line_plot_y_values, multi_line_plot_desc, plot) -> figure:
55  for index, line_plot_y_values in enumerate(multi_line_plot_y_values):
56    source = ColumnDataSource({'X': x_values, 'Y': line_plot_y_values,
57      'Description': [multi_line_plot_desc[index]] * x_values.size})
58    r = plot.scatter('X', 'Y', source = source, fill_alpha=0, line_alpha=0.3, line_color="grey")
59    hover = HoverTool(tooltips=[("X", "@X{custom}"), ("PSI", "@Y"), ("Desc", "@Description")],
60      formatters={'@X': MILLIS_TO_SECONDS}, renderers=[r])
61    plot.add_tools(hover)
62  return plot
63
64
65def get_psi_plot(cuj_name, psi_df, x_range_list) -> figure:
66  if psi_df.empty:
67    return None
68
69  plot = figure(title="PSI metrics during CUJ '" + cuj_name + "'", width=PLOT_WIDTH,
70                height=PLOT_HEIGHT, x_axis_label=X_AXIS_LABEL,
71                y_axis_label='PSI %', x_range=x_range_list, y_range=[0, 100],
72                x_axis_type='datetime')
73
74  psi_source_dict = dict(
75      x_values=[],
76      y_values=[psi_df.memory_full_avg10.values, psi_df.io_full_avg10.values,
77           psi_df.cpu_some_avg10.values, psi_df.memory_some_avg10.values,
78           psi_df.io_some_avg10.values],
79      color=['red', 'blue', 'sandybrown', 'yellow', 'purple'],
80      desc=['Memory full avg10', 'IO full avg10', 'CPU some avg10', 'Memory some avg10',
81                    'IO some avg10'])
82  if 'cpu_full_avg10' in psi_df.columns:
83    psi_source_dict['y_values'].append(psi_df.cpu_full_avg10.values)
84    psi_source_dict['color'].append('black')
85    psi_source_dict['desc'].append('CPU full avg10')
86
87  if 'irq_full_avg10' in psi_df.columns:
88    psi_source_dict['y_values'].append(psi_df.irq_full_avg10.values)
89    psi_source_dict['color'].append('pink')
90    psi_source_dict['desc'].append('IRQ full avg10')
91
92  x_values = pd.to_timedelta(pd.Series(psi_df[X_AXIS_FIELD].values), unit='ms')
93  psi_source_dict['x_values'] = [x_values for _ in psi_source_dict['y_values']]
94
95  source=ColumnDataSource(data=psi_source_dict)
96  plot.multi_line(xs='x_values', ys='y_values', source=source,
97                  color='color', legend_field='desc', line_width=3)
98  plot = plot_tooltips(x_values, psi_source_dict['y_values'], psi_source_dict['desc'], plot)
99
100  vspan_opts = dict(x='x_values', y='y_values', text='desc', x_units='data', y_units='data',
101                    level='annotation', angle=90, angle_units='deg', x_offset=-15, y_offset=-200,
102                    text_font_size='10pt', text_color='black', text_alpha=1.0, text_align='center',
103                    text_baseline='middle')
104
105  psi_events_df = psi_df[psi_df['event_description'].notna()].reset_index()
106  if not psi_events_df.empty:
107    plot = add_events_to_plot(plot, psi_events_df, vspan_opts, 100, SHADES_OF_GREEN)
108
109  return plot
110
111
112def get_events_plot(cuj_name, events_df, x_range_obj) -> figure:
113  if events_df.empty:
114    return None
115
116  plot = figure(title="Events during CUJ '" + cuj_name + "'", width=PLOT_WIDTH, height=PLOT_HEIGHT,
117                x_axis_label=X_AXIS_LABEL, y_axis_label='Event', x_range=x_range_obj,
118                y_range=[0, 100], x_axis_type='datetime')
119  hover = HoverTool(tooltips=[("X", "@x_values{custom}"), ("Event", "@desc")],
120    formatters={'@x_values': MILLIS_TO_SECONDS})
121  plot.add_tools(hover)
122
123  vspan_opts = dict(x='x_values', y='y_values', x_units='data', y_units='data')
124
125  plot = add_events_to_plot(plot, events_df, vspan_opts)
126
127  return plot
128
129
130def save_and_show_plot(plot, should_show_plot, out_file):
131  output_file(out_file)
132  save(plot)
133  if should_show_plot:
134    show(plot)
135
136
137def get_x_range_list(psi_df, events_df) -> Tuple[int, int]:
138  if events_df is not None and not events_df.empty:
139    return (min(events_df[X_AXIS_FIELD].min(), psi_df[X_AXIS_FIELD].min()),
140            max(events_df[X_AXIS_FIELD].max(), psi_df[X_AXIS_FIELD].max()))
141  return (psi_df[X_AXIS_FIELD].min(), psi_df[X_AXIS_FIELD].max())
142
143
144def generate_plot(cuj_name, psi_df, events_df, should_show_plot, out_file):
145  # psi_df has event_descriptions (which is a list of all events happened on a polling event)
146  # while events_df has event_description (which is a single event). To keep the dfs consistent,
147  # concatenate all events from event_descriptions in psi_df as a single event string.
148  psi_df['event_description'] = psi_df['event_descriptions'].apply(
149    lambda x: ','.join(x) if len(x) > 0 else np.nan)
150  psi_plot = get_psi_plot(cuj_name, psi_df, get_x_range_list(psi_df, events_df))
151  events_plot = None
152
153  if events_df is not None:
154    events_plot = get_events_plot(cuj_name, events_df, psi_plot.x_range)
155
156  if events_plot:
157    plot = column([psi_plot, events_plot], sizing_mode='stretch_both')
158  else:
159    plot = psi_plot
160
161  save_and_show_plot(plot, should_show_plot, out_file)
162
163
164def main():
165  parser = argparse.ArgumentParser(
166      description='Plot PSI csv dump from PSI monitor.',
167  )
168  parser.add_argument('--psi_csv', action='store', type=str, required=True, dest='psi_csv',
169                      help='PSI csv dump from psi_monitor.sh')
170  parser.add_argument('--events_csv', action='store', type=str, dest='events_csv',
171                      help='Events csv dump from run_cuj.sh')
172  parser.add_argument('--cuj_name', action='store', type=str, dest='cuj_name',
173                      default='Unknown CUJ', help='Name of the CUJ')
174  parser.add_argument('--out_file', action='store', type=str, dest='out_file',
175                      default='psi_plot.html', help='Output HTML file')
176  parser.add_argument('--show_plot', action='store_true', dest='show_plot',
177                      default=False, help='Show the plot')
178  args = parser.parse_args()
179
180  print('Plotting PSI csv dump {}'.format(args.psi_csv))
181  psi_df = pd.read_csv(args.psi_csv, converters={'event_descriptions': pd.eval})
182  events_df = None
183  if args.events_csv:
184    print('Plotting PSI events csv dump {}'.format(args.events_csv))
185    events_df = pd.read_csv(args.events_csv).drop_duplicates()
186    events_df = events_df[events_df['should_plot']]
187
188  generate_plot(args.cuj_name, psi_df, events_df, args.show_plot, args.out_file)
189
190if __name__ == '__main__':
191  main()
192