1#!/usr/local/bin/python3.8
2
3#
4# Copyright (c) 2021, Chad Miller  chad.org
5# All rights reserved.
6#
7
8import curses
9import subprocess
10from contextlib import suppress
11from base64 import encodebytes as b64encodebytes
12import argparse
13from itertools import groupby
14from time import time
15from random import choice as randomchoice
16import sys
17
18
19EMPTY_COLORS = (238, 8)
20EMPTY_CHAR = "|"
21PALETTES = list(range(start, end+(second-start), second-start) for second, start, end in ((63, 27, 207), (87, 51, 231), (116, 123, 88), (220, 226, 196), (224, 231, 196), (80, 51, 196), (77, 40, 225), (225, 231, 201), (243, 240, 255), (227, 226, 231), (27, 21, 51)))
22DISPLAY_CHARS_LETTERS = ".abcdefghijklmnopqrstuvwxyz^"
23DISPLAY_CHARS_DIGITS = ".0123456789#"
24DISPLAY_CHARS_SYMBOLS = " .:;*#"
25DISPLAY_CHARS = DISPLAY_CHARS_LETTERS
26DIFFL_CLOCK_CHARS = "╷╴╵╶"
27
28DIFFL_STAT_MEMORY = 5
29
30raw = None
31
32def short_time_and_color(ns):
33    """Given a time in nanoseconds, return a pleasant, legible string
34    approximating that time, and an integer representing an appropriate color for
35    that number of nanoseconds of waiting. Bigger is worse."""
36    if ns < 30000:
37        color = TIME_ANSI_COLORS[0]
38    elif ns < 600000:
39        color = TIME_ANSI_COLORS[1]
40    elif ns < 8500000:
41        color = TIME_ANSI_COLORS[2]
42    elif ns < 15000000:
43        color = TIME_ANSI_COLORS[3]
44    elif ns < 155000000:
45        color = TIME_ANSI_COLORS[4]
46    else:
47        color = TIME_ANSI_COLORS[5]
48
49    if ns < 800:
50        return "{: 7.0f}ns".format(ns), color
51    elif ns < 800000:
52        return "{: 7.2f}µs".format(ns/1000), color
53    elif ns < 1000000000:
54        return "{: 7.2f}ms".format(ns/1000/1000), color
55    else:
56        return "{: 7.2f}s".format(ns/1000/1000/1000), color
57
58
59def get_stats(pool_names, filename=None):
60    """Populate a "raw" global variable of the last thing we read, and return a
61    structure -- a list of pairs of vdev-name and vdev-timings, where a vdev-timing
62    is a list of rows, each as a ColsIoStat named tuple."""
63    global raw
64    if filename:
65        with open(filename, encoding="UTF-8") as f:
66            raw = f.read()
67    else:
68        zpool_cmd = subprocess.run(["zpool", "iostat", "-wvHp", "--"] + (pool_names if pool_names else []), check=True, stdout=subprocess.PIPE, encoding="UTF-8")
69        raw = zpool_cmd.stdout
70
71    ## TODO: Get this header list from zpool somehow. Only it is authoratative.
72    headers = "total wait read/total wait write/disk wait read/disk wait write/syncq wait read (through txg)/syncq wait write (through txg)/asyncq wait read (from zil)/asyncq wait write (to zil)/scrub/trim".split("/")
73
74    stats = []
75    for line in raw.split("\n"):
76        if not line:
77            continue
78        elif "\t" in line:
79            row_number += 1
80            values = tuple(int(s) for s in line.split("\t"))
81            stats[-1][1].append((values[0], tuple(zip(headers, values[1:]))))
82        else:
83            row_number = -1
84            stats.append([line, []])
85
86    return stats
87
88
89def scaled_to_fraction(range_minimum, subject_value, range_maximum):
90    """Take a number in a range and return a fraction of how far it is into
91    that range."""
92    if range_minimum == range_maximum:
93        return 0
94    assert range_minimum <= subject_value <= range_maximum, (range_minimum, subject_value, range_maximum)
95    return (subject_value-range_minimum) / (range_maximum-range_minimum)
96
97
98def stats_as_device_centric(stats):
99    """
100    >>> stats_as_device_centric([['dev1', [(1, (('mes1', 2), ('mes2', 3))), (4, (('mes1', 5), ('mes2', 6)))]], ['dev2', [(1, (('mes1', 7), ('mes2', 8))), (4, ('mes1', 9), ('mes2', 10))]]])
101    [['dev1', [(1, (('mes1', 2), ('mes2', 3))), (4, (('mes1', 5), ('mes2', 6)))]], ['dev2', [(1, (('mes1', 7), ('mes2', 8))), (4, ('mes1', 9), ('mes2', 10))]]]
102    """
103    return stats
104
105
106def stats_as_measurement_centric(stats):
107    """
108    >>> stats_as_measurement_centric([['dev1', [(1, (('mes1', 2), ('mes2', 3))), (4, (('mes1', 5), ('mes2', 6)))]], ['dev2', [(1, (('mes1', 7), ('mes2', 8))), (4, (('mes1', 9), ('mes2', 10)))]]])
109    [['mes1', [(1, (('dev1', 2), ('dev2', 7))), (4, (('dev1', 5), ('dev2', 9)))]], ['mes2', [(1, (('dev1', 3), ('dev2', 8))), (4, (('dev1', 6), ('dev2', 10)))]]]
110    >>> stats_as_measurement_centric([['devx', [(1, (('mes1', 2), ('mes2', 3))), (4, (('mes1', 5), ('mes2', 6 )))]], ['devx', [(1, (('mes1', 7), ('mes2', 8))), (4, (('mes1', 9), ('mes2', 10)))]]])
111    [['mes1', [(1, (('devx', 2), ('devx', 7))), (4, (('devx', 5), ('devx', 9)))]], ['mes2', [(1, (('devx', 3), ('devx', 8))), (4, (('devx', 6), ('devx', 10)))]]]
112    >>> stats_as_measurement_centric([['dev2', [(1, (('mes2', 2), ('mes1', 3))), (4, (('mes2', 5), ('mes1', 6)))]], ['dev1', [(1, (('mes2', 7), ('mes1', 8))), (4, (('mes2', 9), ('mes1', 10)))]]])
113    [['mes2', [(1, (('dev2', 2), ('dev1', 7))), (4, (('dev2', 5), ('dev1', 9)))]], ['mes1', [(1, (('dev2', 3), ('dev1', 8))), (4, (('dev2', 6), ('dev1', 10)))]]]
114    """
115
116    return list([mes, list((timing, tuple(devicefortiming[4:] for devicefortiming in alldatafortiming)) for timing, alldatafortiming in groupby(sorted(v0, key=lambda six: six[3]), key=lambda six: six[3]))] for (_, mes), v0 in groupby(sorted(("m"+str(measurementnumber), "d"+str(devicenumber), measurementname, tns, device, bucketsize) for devicenumber, (device, timingsperdevice) in enumerate(stats) for tns, devicesandbuckets in timingsperdevice for measurementnumber, (measurementname, bucketsize) in enumerate(devicesandbuckets)), key=lambda six: (six[0], six[2])))
117
118
119def render_stats(window, transform, should_show_differential, pool, filename=None):
120    read_count = 0
121    stats = None
122    stats_history = []
123    current = 0
124    load_time = None
125    diffl_stat_intervals = (2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 30, 60, 120, 180, 300, 600) #seconds
126    diffl_stat_interval_index = 2
127    diffl_title = "(&){:d}s↕ ".format(diffl_stat_intervals[diffl_stat_interval_index])
128    while True:
129        if not load_time or load_time + diffl_stat_intervals[diffl_stat_interval_index] < time():
130            if stats:
131                stats_history.append(stats)
132            while len(stats_history) > DIFFL_STAT_MEMORY:
133                stats_history.pop(0)
134
135            stats = transform(get_stats(pool, filename))
136            load_time = time()
137            read_count += 1
138
139        name, rows  = stats[current]
140
141        if should_show_differential:
142            rows_containing_data = list(row_number for row_number in range(len(rows)))
143        else:
144            rows_containing_data = list(row_number for row_number, (_, data) in enumerate(rows) if any(colval > 0 for colhead, colval in data))
145
146        if should_show_differential and stats_history:
147            assert stats is not None
148            assert stats_history is not None
149            assert stats_history[0] is not None
150            rows = list((stats[current][1][rn][0], tuple((newk, newv-oldv) for (newk,newv),(oldk,oldv) in zip(stats[current][1][rn][1], stats_history[0][current][1][rn][1]))) for rn in range(len(rows)))
151
152        max_per_column = tuple(max(column) for column in zip(*tuple([v for k, v in row] for _, row in rows)))
153
154
155        window.clear()
156        if should_show_differential:
157            if stats_history:
158                diffl_title = "({}){:d}s↕ ".format(DIFFL_CLOCK_CHARS[read_count%4], diffl_stat_intervals[diffl_stat_interval_index])
159            with suppress(curses.error):
160                window.addstr(0, 0, diffl_title)
161        with suppress(curses.error):
162            window.addstr(0, 10 if should_show_differential else 0, "{:>2d}/{}↔  Histogram for {}".format(current+1, len(stats), name))
163
164        printed_row_number = 0
165        for row_number, (time_ns, data) in enumerate(rows):
166            if not rows_containing_data: continue
167            if not min(rows_containing_data)-1 <= row_number <= max(rows_containing_data)+1: continue
168            printed_row_number += 1
169
170            with suppress(curses.error):
171                t, col = short_time_and_color(time_ns)
172                window.addstr(printed_row_number, 0, t, curses.color_pair(col))  # write legend
173
174            for col_number, (_, value) in enumerate(data):
175
176                scaled_0_to_1 = scaled_to_fraction(0, value, max_per_column[col_number])
177                if scaled_0_to_1 > 0.001:
178                    glyph = randomchoice(DISPLAY_CHARS[int(scaled_0_to_1 * (len(DISPLAY_CHARS)-1))])
179                    color = HISTOGRAM_ANSI_COLORS[int(scaled_0_to_1 * (len(HISTOGRAM_ANSI_COLORS)-1))]
180                else:
181                    glyph = EMPTY_CHAR
182                    color = EMPTY_COLORS[col_number % len(EMPTY_COLORS)]
183
184                with suppress(curses.error):
185                    window.addstr(printed_row_number, 12+(col_number*2), glyph, curses.color_pair(color))
186
187        if rows_containing_data:
188            device_names = list(reversed([k for k, _ in rows[0][1]]))
189            glyph = EMPTY_CHAR
190            while device_names:
191                printed_row_number += 1
192                with suppress(curses.error):
193                    for col_number in range(len(device_names)):
194                        color = EMPTY_COLORS[col_number % len(EMPTY_COLORS)]
195                        window.addstr(printed_row_number, 12+(2*col_number), glyph, curses.color_pair(color))
196                device_name = device_names.pop(0)
197                with suppress(curses.error):
198                    window.addstr(printed_row_number, 12+(2*col_number), "`{}".format(device_name))
199        else:
200            with suppress(curses.error):
201                window.addstr(4, 16, "(no data)")
202
203        height, width = window.getmaxyx()
204        message = "  Population of histogram buckets shown with .a-z^ and colors"
205        with suppress(curses.error):
206            window.addstr(height-1, width-len(message)-1, message)
207            for i, (ch, color) in enumerate(zip(reversed(message), reversed(HISTOGRAM_ANSI_COLORS))):
208                window.addstr(height-1, width-1-i-1, ch, curses.color_pair(color))
209
210        window.refresh()
211
212        in_key = window.getch()
213        if in_key == -1:
214            pass
215        elif in_key == curses.KEY_RIGHT:
216            current += 1
217        elif in_key == curses.KEY_LEFT:
218            current -= 1
219        elif in_key == curses.KEY_UP:
220            if diffl_stat_interval_index < len(diffl_stat_intervals) - 1:
221                diffl_stat_interval_index += 1
222        elif in_key == curses.KEY_DOWN:
223            if diffl_stat_interval_index > 0:
224                diffl_stat_interval_index -= 1
225        elif in_key == ord('d'):
226            should_show_differential = not should_show_differential
227        elif in_key == ord('m'):
228            load_time = None
229            current = 0
230            stats = None
231            stats_history = []
232            if transform == stats_as_device_centric:
233                transform = stats_as_measurement_centric
234            else:
235                transform = stats_as_device_centric
236        elif in_key == ord('q') or in_key == ord('x') or in_key == 27:
237            return
238        if stats:
239            current += len(stats)
240            current %= len(stats)
241
242
243def main(window, should_show_differential, pool, filename, views):
244    window.timeout(1000)
245
246    curses.use_default_colors()
247    curses.curs_set(0)
248    for i in range(0, curses.COLORS):
249        curses.init_pair(i, i, -1)
250
251    transformation_function = stats_as_measurement_centric
252    for view in views:
253        if view == "d": transformation_function = stats_as_device_centric
254        if view == "m": transformation_function = stats_as_measurement_centric
255
256    render_stats(window, transformation_function, should_show_differential, pool, filename)
257
258
259if __name__ == "__main__":
260    import doctest
261    if doctest.testmod().failed:
262        sys.exit(1)
263
264    try:
265        arg_parser = argparse.ArgumentParser("zpool-iostat-viz", description="Display ZFS pool statistics, by device and by measurement")
266
267        arg_parser.add_argument("--diff", "-d", dest="diff", action="store_true", help="show changes while running")
268        arg_parser.add_argument("--by", dest="by", choices="dm", action="store", default="s", help="slice data by device or measurement")
269        arg_parser.add_argument("--from-file", "-f", dest="file", action="store")
270        arg_parser.add_argument("--pal-time", "--pt", action="store", metavar="P", default="3", help="palette for time buckets")
271        arg_parser.add_argument("--pal-count", "--pc", action="store", metavar="P", default="0", help="palette for bucket populations")
272        arg_parser.add_argument("parts", metavar="pool/vdev", nargs="*", help="Pools or vdevs to display")
273        arg_parser.add_argument("--help-colors", action="store_true", help="see color palettes available")
274        arg_parser.add_argument("--digits", action="store_true", help="use digits instead of letters")
275        arg_parser.add_argument("--symbols", action="store_true", help="use digits instead of letters")
276
277        parsed_args = vars(arg_parser.parse_args())
278
279        help_see_colors = parsed_args["help_colors"]
280        try:
281            HISTOGRAM_ANSI_COLORS = PALETTES[int(parsed_args["pal_count"], 16)]
282            TIME_ANSI_COLORS = PALETTES[int(parsed_args["pal_time"], 16)]
283        except (IndexError, ValueError):
284            help_see_colors = True
285
286        if help_see_colors:
287            print("color palettes (P) for use with --pal-time P or --pal-count P")
288            for pi, palette in enumerate(PALETTES):
289                rainbow = ["\033[38;5;{0}m {0:03d}\033[m".format(color) for color in palette]
290                print("   {0:x}   {1}".format(pi, "".join(rainbow)), end="")
291                if hex(pi)[2:] == parsed_args["pal_count"]: print("  (count)", end="")
292                if hex(pi)[2:] == parsed_args["pal_time"]: print("  (time)", end="")
293                print()
294            sys.exit(0)
295
296        if parsed_args["digits"]:
297            DISPLAY_CHARS = DISPLAY_CHARS_DIGITS
298        if parsed_args["symbols"]:
299            DISPLAY_CHARS = DISPLAY_CHARS_SYMBOLS
300
301        curses.wrapper(lambda window: main(window, parsed_args["diff"], parsed_args["parts"], parsed_args["file"], parsed_args["by"] or "m"))
302    except subprocess.CalledProcessError as exc:
303        print("I couldn't get your pool information. Make sure you have 'zpool' program and specify your pool correctly.")
304        print(exc)
305    except Exception as exc:
306        print("CRASH! Sorry!")
307        print("Please report this error at \nhttps://github.com/chadmiller/zpool-iostat-viz/issues/new")
308        print()
309        print("Paste the following:")
310        if raw is not None:
311            print(b64encodebytes(raw.encode("UTF-8")).decode("ASCII"), end="and also include ")
312        raise exc
313