1#!/usr/local/bin/python3.8 2 3# 4# Copyright (c) 2021, Chad Miller chad.org 5# All rights reserved. 6# 7 8import curses 9import subprocess 10from contextlib import suppress 11from base64 import encodebytes as b64encodebytes 12import argparse 13from itertools import groupby 14from time import time 15from random import choice as randomchoice 16import sys 17 18 19EMPTY_COLORS = (238, 8) 20EMPTY_CHAR = "|" 21PALETTES = list(range(start, end+(second-start), second-start) for second, start, end in ((63, 27, 207), (87, 51, 231), (116, 123, 88), (220, 226, 196), (224, 231, 196), (80, 51, 196), (77, 40, 225), (225, 231, 201), (243, 240, 255), (227, 226, 231), (27, 21, 51))) 22DISPLAY_CHARS_LETTERS = ".abcdefghijklmnopqrstuvwxyz^" 23DISPLAY_CHARS_DIGITS = ".0123456789#" 24DISPLAY_CHARS_SYMBOLS = " .:;*#" 25DISPLAY_CHARS = DISPLAY_CHARS_LETTERS 26DIFFL_CLOCK_CHARS = "╷╴╵╶" 27 28DIFFL_STAT_MEMORY = 5 29 30raw = None 31 32def short_time_and_color(ns): 33 """Given a time in nanoseconds, return a pleasant, legible string 34 approximating that time, and an integer representing an appropriate color for 35 that number of nanoseconds of waiting. Bigger is worse.""" 36 if ns < 30000: 37 color = TIME_ANSI_COLORS[0] 38 elif ns < 600000: 39 color = TIME_ANSI_COLORS[1] 40 elif ns < 8500000: 41 color = TIME_ANSI_COLORS[2] 42 elif ns < 15000000: 43 color = TIME_ANSI_COLORS[3] 44 elif ns < 155000000: 45 color = TIME_ANSI_COLORS[4] 46 else: 47 color = TIME_ANSI_COLORS[5] 48 49 if ns < 800: 50 return "{: 7.0f}ns".format(ns), color 51 elif ns < 800000: 52 return "{: 7.2f}µs".format(ns/1000), color 53 elif ns < 1000000000: 54 return "{: 7.2f}ms".format(ns/1000/1000), color 55 else: 56 return "{: 7.2f}s".format(ns/1000/1000/1000), color 57 58 59def get_stats(pool_names, filename=None): 60 """Populate a "raw" global variable of the last thing we read, and return a 61 structure -- a list of pairs of vdev-name and vdev-timings, where a vdev-timing 62 is a list of rows, each as a ColsIoStat named tuple.""" 63 global raw 64 if filename: 65 with open(filename, encoding="UTF-8") as f: 66 raw = f.read() 67 else: 68 zpool_cmd = subprocess.run(["zpool", "iostat", "-wvHp", "--"] + (pool_names if pool_names else []), check=True, stdout=subprocess.PIPE, encoding="UTF-8") 69 raw = zpool_cmd.stdout 70 71 ## TODO: Get this header list from zpool somehow. Only it is authoratative. 72 headers = "total wait read/total wait write/disk wait read/disk wait write/syncq wait read (through txg)/syncq wait write (through txg)/asyncq wait read (from zil)/asyncq wait write (to zil)/scrub/trim".split("/") 73 74 stats = [] 75 for line in raw.split("\n"): 76 if not line: 77 continue 78 elif "\t" in line: 79 row_number += 1 80 values = tuple(int(s) for s in line.split("\t")) 81 stats[-1][1].append((values[0], tuple(zip(headers, values[1:])))) 82 else: 83 row_number = -1 84 stats.append([line, []]) 85 86 return stats 87 88 89def scaled_to_fraction(range_minimum, subject_value, range_maximum): 90 """Take a number in a range and return a fraction of how far it is into 91 that range.""" 92 if range_minimum == range_maximum: 93 return 0 94 assert range_minimum <= subject_value <= range_maximum, (range_minimum, subject_value, range_maximum) 95 return (subject_value-range_minimum) / (range_maximum-range_minimum) 96 97 98def stats_as_device_centric(stats): 99 """ 100 >>> stats_as_device_centric([['dev1', [(1, (('mes1', 2), ('mes2', 3))), (4, (('mes1', 5), ('mes2', 6)))]], ['dev2', [(1, (('mes1', 7), ('mes2', 8))), (4, ('mes1', 9), ('mes2', 10))]]]) 101 [['dev1', [(1, (('mes1', 2), ('mes2', 3))), (4, (('mes1', 5), ('mes2', 6)))]], ['dev2', [(1, (('mes1', 7), ('mes2', 8))), (4, ('mes1', 9), ('mes2', 10))]]] 102 """ 103 return stats 104 105 106def stats_as_measurement_centric(stats): 107 """ 108 >>> stats_as_measurement_centric([['dev1', [(1, (('mes1', 2), ('mes2', 3))), (4, (('mes1', 5), ('mes2', 6)))]], ['dev2', [(1, (('mes1', 7), ('mes2', 8))), (4, (('mes1', 9), ('mes2', 10)))]]]) 109 [['mes1', [(1, (('dev1', 2), ('dev2', 7))), (4, (('dev1', 5), ('dev2', 9)))]], ['mes2', [(1, (('dev1', 3), ('dev2', 8))), (4, (('dev1', 6), ('dev2', 10)))]]] 110 >>> stats_as_measurement_centric([['devx', [(1, (('mes1', 2), ('mes2', 3))), (4, (('mes1', 5), ('mes2', 6 )))]], ['devx', [(1, (('mes1', 7), ('mes2', 8))), (4, (('mes1', 9), ('mes2', 10)))]]]) 111 [['mes1', [(1, (('devx', 2), ('devx', 7))), (4, (('devx', 5), ('devx', 9)))]], ['mes2', [(1, (('devx', 3), ('devx', 8))), (4, (('devx', 6), ('devx', 10)))]]] 112 >>> stats_as_measurement_centric([['dev2', [(1, (('mes2', 2), ('mes1', 3))), (4, (('mes2', 5), ('mes1', 6)))]], ['dev1', [(1, (('mes2', 7), ('mes1', 8))), (4, (('mes2', 9), ('mes1', 10)))]]]) 113 [['mes2', [(1, (('dev2', 2), ('dev1', 7))), (4, (('dev2', 5), ('dev1', 9)))]], ['mes1', [(1, (('dev2', 3), ('dev1', 8))), (4, (('dev2', 6), ('dev1', 10)))]]] 114 """ 115 116 return list([mes, list((timing, tuple(devicefortiming[4:] for devicefortiming in alldatafortiming)) for timing, alldatafortiming in groupby(sorted(v0, key=lambda six: six[3]), key=lambda six: six[3]))] for (_, mes), v0 in groupby(sorted(("m"+str(measurementnumber), "d"+str(devicenumber), measurementname, tns, device, bucketsize) for devicenumber, (device, timingsperdevice) in enumerate(stats) for tns, devicesandbuckets in timingsperdevice for measurementnumber, (measurementname, bucketsize) in enumerate(devicesandbuckets)), key=lambda six: (six[0], six[2]))) 117 118 119def render_stats(window, transform, should_show_differential, pool, filename=None): 120 read_count = 0 121 stats = None 122 stats_history = [] 123 current = 0 124 load_time = None 125 diffl_stat_intervals = (2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 30, 60, 120, 180, 300, 600) #seconds 126 diffl_stat_interval_index = 2 127 diffl_title = "(&){:d}s↕ ".format(diffl_stat_intervals[diffl_stat_interval_index]) 128 while True: 129 if not load_time or load_time + diffl_stat_intervals[diffl_stat_interval_index] < time(): 130 if stats: 131 stats_history.append(stats) 132 while len(stats_history) > DIFFL_STAT_MEMORY: 133 stats_history.pop(0) 134 135 stats = transform(get_stats(pool, filename)) 136 load_time = time() 137 read_count += 1 138 139 name, rows = stats[current] 140 141 if should_show_differential: 142 rows_containing_data = list(row_number for row_number in range(len(rows))) 143 else: 144 rows_containing_data = list(row_number for row_number, (_, data) in enumerate(rows) if any(colval > 0 for colhead, colval in data)) 145 146 if should_show_differential and stats_history: 147 assert stats is not None 148 assert stats_history is not None 149 assert stats_history[0] is not None 150 rows = list((stats[current][1][rn][0], tuple((newk, newv-oldv) for (newk,newv),(oldk,oldv) in zip(stats[current][1][rn][1], stats_history[0][current][1][rn][1]))) for rn in range(len(rows))) 151 152 max_per_column = tuple(max(column) for column in zip(*tuple([v for k, v in row] for _, row in rows))) 153 154 155 window.clear() 156 if should_show_differential: 157 if stats_history: 158 diffl_title = "({}){:d}s↕ ".format(DIFFL_CLOCK_CHARS[read_count%4], diffl_stat_intervals[diffl_stat_interval_index]) 159 with suppress(curses.error): 160 window.addstr(0, 0, diffl_title) 161 with suppress(curses.error): 162 window.addstr(0, 10 if should_show_differential else 0, "{:>2d}/{}↔ Histogram for {}".format(current+1, len(stats), name)) 163 164 printed_row_number = 0 165 for row_number, (time_ns, data) in enumerate(rows): 166 if not rows_containing_data: continue 167 if not min(rows_containing_data)-1 <= row_number <= max(rows_containing_data)+1: continue 168 printed_row_number += 1 169 170 with suppress(curses.error): 171 t, col = short_time_and_color(time_ns) 172 window.addstr(printed_row_number, 0, t, curses.color_pair(col)) # write legend 173 174 for col_number, (_, value) in enumerate(data): 175 176 scaled_0_to_1 = scaled_to_fraction(0, value, max_per_column[col_number]) 177 if scaled_0_to_1 > 0.001: 178 glyph = randomchoice(DISPLAY_CHARS[int(scaled_0_to_1 * (len(DISPLAY_CHARS)-1))]) 179 color = HISTOGRAM_ANSI_COLORS[int(scaled_0_to_1 * (len(HISTOGRAM_ANSI_COLORS)-1))] 180 else: 181 glyph = EMPTY_CHAR 182 color = EMPTY_COLORS[col_number % len(EMPTY_COLORS)] 183 184 with suppress(curses.error): 185 window.addstr(printed_row_number, 12+(col_number*2), glyph, curses.color_pair(color)) 186 187 if rows_containing_data: 188 device_names = list(reversed([k for k, _ in rows[0][1]])) 189 glyph = EMPTY_CHAR 190 while device_names: 191 printed_row_number += 1 192 with suppress(curses.error): 193 for col_number in range(len(device_names)): 194 color = EMPTY_COLORS[col_number % len(EMPTY_COLORS)] 195 window.addstr(printed_row_number, 12+(2*col_number), glyph, curses.color_pair(color)) 196 device_name = device_names.pop(0) 197 with suppress(curses.error): 198 window.addstr(printed_row_number, 12+(2*col_number), "`{}".format(device_name)) 199 else: 200 with suppress(curses.error): 201 window.addstr(4, 16, "(no data)") 202 203 height, width = window.getmaxyx() 204 message = " Population of histogram buckets shown with .a-z^ and colors" 205 with suppress(curses.error): 206 window.addstr(height-1, width-len(message)-1, message) 207 for i, (ch, color) in enumerate(zip(reversed(message), reversed(HISTOGRAM_ANSI_COLORS))): 208 window.addstr(height-1, width-1-i-1, ch, curses.color_pair(color)) 209 210 window.refresh() 211 212 in_key = window.getch() 213 if in_key == -1: 214 pass 215 elif in_key == curses.KEY_RIGHT: 216 current += 1 217 elif in_key == curses.KEY_LEFT: 218 current -= 1 219 elif in_key == curses.KEY_UP: 220 if diffl_stat_interval_index < len(diffl_stat_intervals) - 1: 221 diffl_stat_interval_index += 1 222 elif in_key == curses.KEY_DOWN: 223 if diffl_stat_interval_index > 0: 224 diffl_stat_interval_index -= 1 225 elif in_key == ord('d'): 226 should_show_differential = not should_show_differential 227 elif in_key == ord('m'): 228 load_time = None 229 current = 0 230 stats = None 231 stats_history = [] 232 if transform == stats_as_device_centric: 233 transform = stats_as_measurement_centric 234 else: 235 transform = stats_as_device_centric 236 elif in_key == ord('q') or in_key == ord('x') or in_key == 27: 237 return 238 if stats: 239 current += len(stats) 240 current %= len(stats) 241 242 243def main(window, should_show_differential, pool, filename, views): 244 window.timeout(1000) 245 246 curses.use_default_colors() 247 curses.curs_set(0) 248 for i in range(0, curses.COLORS): 249 curses.init_pair(i, i, -1) 250 251 transformation_function = stats_as_measurement_centric 252 for view in views: 253 if view == "d": transformation_function = stats_as_device_centric 254 if view == "m": transformation_function = stats_as_measurement_centric 255 256 render_stats(window, transformation_function, should_show_differential, pool, filename) 257 258 259if __name__ == "__main__": 260 import doctest 261 if doctest.testmod().failed: 262 sys.exit(1) 263 264 try: 265 arg_parser = argparse.ArgumentParser("zpool-iostat-viz", description="Display ZFS pool statistics, by device and by measurement") 266 267 arg_parser.add_argument("--diff", "-d", dest="diff", action="store_true", help="show changes while running") 268 arg_parser.add_argument("--by", dest="by", choices="dm", action="store", default="s", help="slice data by device or measurement") 269 arg_parser.add_argument("--from-file", "-f", dest="file", action="store") 270 arg_parser.add_argument("--pal-time", "--pt", action="store", metavar="P", default="3", help="palette for time buckets") 271 arg_parser.add_argument("--pal-count", "--pc", action="store", metavar="P", default="0", help="palette for bucket populations") 272 arg_parser.add_argument("parts", metavar="pool/vdev", nargs="*", help="Pools or vdevs to display") 273 arg_parser.add_argument("--help-colors", action="store_true", help="see color palettes available") 274 arg_parser.add_argument("--digits", action="store_true", help="use digits instead of letters") 275 arg_parser.add_argument("--symbols", action="store_true", help="use digits instead of letters") 276 277 parsed_args = vars(arg_parser.parse_args()) 278 279 help_see_colors = parsed_args["help_colors"] 280 try: 281 HISTOGRAM_ANSI_COLORS = PALETTES[int(parsed_args["pal_count"], 16)] 282 TIME_ANSI_COLORS = PALETTES[int(parsed_args["pal_time"], 16)] 283 except (IndexError, ValueError): 284 help_see_colors = True 285 286 if help_see_colors: 287 print("color palettes (P) for use with --pal-time P or --pal-count P") 288 for pi, palette in enumerate(PALETTES): 289 rainbow = ["\033[38;5;{0}m {0:03d}\033[m".format(color) for color in palette] 290 print(" {0:x} {1}".format(pi, "".join(rainbow)), end="") 291 if hex(pi)[2:] == parsed_args["pal_count"]: print(" (count)", end="") 292 if hex(pi)[2:] == parsed_args["pal_time"]: print(" (time)", end="") 293 print() 294 sys.exit(0) 295 296 if parsed_args["digits"]: 297 DISPLAY_CHARS = DISPLAY_CHARS_DIGITS 298 if parsed_args["symbols"]: 299 DISPLAY_CHARS = DISPLAY_CHARS_SYMBOLS 300 301 curses.wrapper(lambda window: main(window, parsed_args["diff"], parsed_args["parts"], parsed_args["file"], parsed_args["by"] or "m")) 302 except subprocess.CalledProcessError as exc: 303 print("I couldn't get your pool information. Make sure you have 'zpool' program and specify your pool correctly.") 304 print(exc) 305 except Exception as exc: 306 print("CRASH! Sorry!") 307 print("Please report this error at \nhttps://github.com/chadmiller/zpool-iostat-viz/issues/new") 308 print() 309 print("Paste the following:") 310 if raw is not None: 311 print(b64encodebytes(raw.encode("UTF-8")).decode("ASCII"), end="and also include ") 312 raise exc 313