1#!/usr/bin/env @PYTHON_SHEBANG@ 2# 3# Print out statistics for all zil stats. This information is 4# available through the zil kstat. 5# 6# CDDL HEADER START 7# 8# The contents of this file are subject to the terms of the 9# Common Development and Distribution License, Version 1.0 only 10# (the "License"). You may not use this file except in compliance 11# with the License. 12# 13# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 14# or https://opensource.org/licenses/CDDL-1.0. 15# See the License for the specific language governing permissions 16# and limitations under the License. 17# 18# When distributing Covered Code, include this CDDL HEADER in each 19# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 20# If applicable, add the following below this CDDL HEADER, with the 21# fields enclosed by brackets "[]" replaced with your own identifying 22# information: Portions Copyright [yyyy] [name of copyright owner] 23# 24# This script must remain compatible with Python 3.6+. 25# 26 27import sys 28import subprocess 29import time 30import copy 31import os 32import re 33import signal 34from collections import defaultdict 35import argparse 36from argparse import RawTextHelpFormatter 37 38cols = { 39 # hdr: [size, scale, kstat name] 40 "time": [8, -1, "time"], 41 "pool": [12, -1, "pool"], 42 "ds": [12, -1, "dataset_name"], 43 "obj": [12, -1, "objset"], 44 "zcc": [10, 1000, "zil_commit_count"], 45 "zcwc": [10, 1000, "zil_commit_writer_count"], 46 "ziic": [10, 1000, "zil_itx_indirect_count"], 47 "zic": [10, 1000, "zil_itx_count"], 48 "ziib": [10, 1024, "zil_itx_indirect_bytes"], 49 "zicc": [10, 1000, "zil_itx_copied_count"], 50 "zicb": [10, 1024, "zil_itx_copied_bytes"], 51 "zinc": [10, 1000, "zil_itx_needcopy_count"], 52 "zinb": [10, 1024, "zil_itx_needcopy_bytes"], 53 "zimnc": [10, 1000, "zil_itx_metaslab_normal_count"], 54 "zimnb": [10, 1024, "zil_itx_metaslab_normal_bytes"], 55 "zimsc": [10, 1000, "zil_itx_metaslab_slog_count"], 56 "zimsb": [10, 1024, "zil_itx_metaslab_slog_bytes"], 57} 58 59hdr = ["time", "pool", "ds", "obj", "zcc", "zcwc", "ziic", "zic", "ziib", \ 60 "zicc", "zicb", "zinc", "zinb", "zimnc", "zimnb", "zimsc", "zimsb"] 61 62ghdr = ["time", "zcc", "zcwc", "ziic", "zic", "ziib", "zicc", "zicb", 63 "zinc", "zinb", "zimnc", "zimnb", "zimsc", "zimsb"] 64 65cmd = ("Usage: zilstat [-hgdv] [-i interval] [-p pool_name]") 66 67curr = {} 68diff = {} 69kstat = {} 70ds_pairs = {} 71pool_name = None 72dataset_name = None 73interval = 0 74sep = " " 75gFlag = True 76dsFlag = False 77 78def prettynum(sz, scale, num=0): 79 suffix = [' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z'] 80 index = 0 81 save = 0 82 83 if scale == -1: 84 return "%*s" % (sz, num) 85 86 # Rounding error, return 0 87 elif 0 < num < 1: 88 num = 0 89 90 while num > scale and index < 5: 91 save = num 92 num = num / scale 93 index += 1 94 95 if index == 0: 96 return "%*d" % (sz, num) 97 98 if (save / scale) < 10: 99 return "%*.1f%s" % (sz - 1, num, suffix[index]) 100 else: 101 return "%*d%s" % (sz - 1, num, suffix[index]) 102 103def print_header(): 104 global hdr 105 global sep 106 for col in hdr: 107 new_col = col 108 if interval > 0 and col not in ['time', 'pool', 'ds', 'obj']: 109 new_col += "/s" 110 sys.stdout.write("%*s%s" % (cols[col][0], new_col, sep)) 111 sys.stdout.write("\n") 112 113def print_values(v): 114 global hdr 115 global sep 116 for col in hdr: 117 val = v[cols[col][2]] 118 if col not in ['time', 'pool', 'ds', 'obj'] and interval > 0: 119 val = v[cols[col][2]] // interval 120 sys.stdout.write("%s%s" % ( 121 prettynum(cols[col][0], cols[col][1], val), sep)) 122 sys.stdout.write("\n") 123 124def print_dict(d): 125 for pool in d: 126 for objset in d[pool]: 127 print_values(d[pool][objset]) 128 129def detailed_usage(): 130 sys.stderr.write("%s\n" % cmd) 131 sys.stderr.write("Field definitions are as follows:\n") 132 for key in cols: 133 sys.stderr.write("%11s : %s\n" % (key, cols[key][2])) 134 sys.stderr.write("\n") 135 sys.exit(0) 136 137def init(): 138 global pool_name 139 global dataset_name 140 global interval 141 global hdr 142 global curr 143 global gFlag 144 global sep 145 146 curr = dict() 147 148 parser = argparse.ArgumentParser(description='Program to print zilstats', 149 add_help=True, 150 formatter_class=RawTextHelpFormatter, 151 epilog="\nUsage Examples\n"\ 152 "Note: Global zilstats is shown by default,"\ 153 " if none of a|p|d option is not provided\n"\ 154 "\tzilstat -a\n"\ 155 '\tzilstat -v\n'\ 156 '\tzilstat -p tank\n'\ 157 '\tzilstat -d tank/d1,tank/d2,tank/zv1\n'\ 158 '\tzilstat -i 1\n'\ 159 '\tzilstat -s \"***\"\n'\ 160 '\tzilstat -f zcwc,zimnb,zimsb\n') 161 162 parser.add_argument( 163 "-v", "--verbose", 164 action="store_true", 165 help="List field headers and definitions" 166 ) 167 168 pool_grp = parser.add_mutually_exclusive_group() 169 170 pool_grp.add_argument( 171 "-a", "--all", 172 action="store_true", 173 dest="all", 174 help="Print all dataset stats" 175 ) 176 177 pool_grp.add_argument( 178 "-p", "--pool", 179 type=str, 180 help="Print stats for all datasets of a speicfied pool" 181 ) 182 183 pool_grp.add_argument( 184 "-d", "--dataset", 185 type=str, 186 help="Print given dataset(s) (Comma separated)" 187 ) 188 189 parser.add_argument( 190 "-f", "--columns", 191 type=str, 192 help="Specify specific fields to print (see -v)" 193 ) 194 195 parser.add_argument( 196 "-s", "--separator", 197 type=str, 198 help="Override default field separator with custom " 199 "character or string" 200 ) 201 202 parser.add_argument( 203 "-i", "--interval", 204 type=int, 205 dest="interval", 206 help="Print stats between specified interval" 207 " (in seconds)" 208 ) 209 210 parsed_args = parser.parse_args() 211 212 if parsed_args.verbose: 213 detailed_usage() 214 215 if parsed_args.all: 216 gFlag = False 217 218 if parsed_args.interval: 219 interval = parsed_args.interval 220 221 if parsed_args.pool: 222 pool_name = parsed_args.pool 223 gFlag = False 224 225 if parsed_args.dataset: 226 dataset_name = parsed_args.dataset 227 gFlag = False 228 229 if parsed_args.separator: 230 sep = parsed_args.separator 231 232 if gFlag: 233 hdr = ghdr 234 235 if parsed_args.columns: 236 hdr = parsed_args.columns.split(",") 237 238 invalid = [] 239 for ele in hdr: 240 if gFlag and ele not in ghdr: 241 invalid.append(ele) 242 elif ele not in cols: 243 invalid.append(ele) 244 245 if len(invalid) > 0: 246 sys.stderr.write("Invalid column definition! -- %s\n" % invalid) 247 sys.exit(1) 248 249 if pool_name and dataset_name: 250 print ("Error: Can not filter both dataset and pool") 251 sys.exit(1) 252 253def FileCheck(fname): 254 try: 255 return (open(fname)) 256 except IOError: 257 print ("Unable to open zilstat proc file: " + fname) 258 sys.exit(1) 259 260if sys.platform.startswith('freebsd'): 261 # Requires py-sysctl on FreeBSD 262 import sysctl 263 264 def kstat_update(pool = None, objid = None): 265 global kstat 266 kstat = {} 267 if not pool: 268 file = "kstat.zfs.misc.zil" 269 k = [ctl for ctl in sysctl.filter(file) \ 270 if ctl.type != sysctl.CTLTYPE_NODE] 271 kstat_process_str(k, file, "GLOBAL", len(file + ".")) 272 elif objid: 273 file = "kstat.zfs." + pool + ".dataset.objset-" + objid 274 k = [ctl for ctl in sysctl.filter(file) if ctl.type \ 275 != sysctl.CTLTYPE_NODE] 276 kstat_process_str(k, file, objid, len(file + ".")) 277 else: 278 file = "kstat.zfs." + pool + ".dataset" 279 zil_start = len(file + ".") 280 obj_start = len("kstat.zfs." + pool + ".") 281 k = [ctl for ctl in sysctl.filter(file) 282 if ctl.type != sysctl.CTLTYPE_NODE] 283 for s in k: 284 if not s or (s.name.find("zil") == -1 and \ 285 s.name.find("dataset_name") == -1): 286 continue 287 name, value = s.name, s.value 288 objid = re.findall(r'0x[0-9A-F]+', \ 289 name[obj_start:], re.I)[0] 290 if objid not in kstat: 291 kstat[objid] = dict() 292 zil_start = len(file + ".objset-" + \ 293 objid + ".") 294 kstat[objid][name[zil_start:]] = value \ 295 if (name.find("dataset_name")) \ 296 else int(value) 297 298 def kstat_process_str(k, file, objset = "GLOBAL", zil_start = 0): 299 global kstat 300 if not k: 301 print("Unable to process kstat for: " + file) 302 sys.exit(1) 303 kstat[objset] = dict() 304 for s in k: 305 if not s or (s.name.find("zil") == -1 and \ 306 s.name.find("dataset_name") == -1): 307 continue 308 name, value = s.name, s.value 309 kstat[objset][name[zil_start:]] = value \ 310 if (name.find("dataset_name")) else int(value) 311 312elif sys.platform.startswith('linux'): 313 def kstat_update(pool = None, objid = None): 314 global kstat 315 kstat = {} 316 if not pool: 317 k = [line.strip() for line in \ 318 FileCheck("/proc/spl/kstat/zfs/zil")] 319 kstat_process_str(k, "/proc/spl/kstat/zfs/zil") 320 elif objid: 321 file = "/proc/spl/kstat/zfs/" + pool + "/objset-" + objid 322 k = [line.strip() for line in FileCheck(file)] 323 kstat_process_str(k, file, objid) 324 else: 325 if not os.path.exists(f"/proc/spl/kstat/zfs/{pool}"): 326 print("Pool \"" + pool + "\" does not exist, Exitting") 327 sys.exit(1) 328 objsets = os.listdir(f'/proc/spl/kstat/zfs/{pool}') 329 for objid in objsets: 330 if objid.find("objset-") == -1: 331 continue 332 file = "/proc/spl/kstat/zfs/" + pool + "/" + objid 333 k = [line.strip() for line in FileCheck(file)] 334 kstat_process_str(k, file, objid.replace("objset-", "")) 335 336 def kstat_process_str(k, file, objset = "GLOBAL", zil_start = 0): 337 global kstat 338 if not k: 339 print("Unable to process kstat for: " + file) 340 sys.exit(1) 341 342 kstat[objset] = dict() 343 for s in k: 344 if not s or (s.find("zil") == -1 and \ 345 s.find("dataset_name") == -1): 346 continue 347 name, unused, value = s.split() 348 kstat[objset][name] = value \ 349 if (name == "dataset_name") else int(value) 350 351def zil_process_kstat(): 352 global curr, pool_name, dataset_name, dsFlag, ds_pairs 353 curr.clear() 354 if gFlag == True: 355 kstat_update() 356 zil_build_dict() 357 else: 358 if pool_name: 359 kstat_update(pool_name) 360 zil_build_dict(pool_name) 361 elif dataset_name: 362 if dsFlag == False: 363 dsFlag = True 364 datasets = dataset_name.split(',') 365 ds_pairs = defaultdict(list) 366 for ds in datasets: 367 try: 368 objid = subprocess.check_output(['zfs', 369 'list', '-Hpo', 'objsetid', ds], \ 370 stderr=subprocess.DEVNULL) \ 371 .decode('utf-8').strip() 372 except subprocess.CalledProcessError as e: 373 print("Command: \"zfs list -Hpo objset "\ 374 + str(ds) + "\" failed with error code:"\ 375 + str(e.returncode)) 376 print("Please make sure that dataset \""\ 377 + str(ds) + "\" exists") 378 sys.exit(1) 379 if not objid: 380 continue 381 ds_pairs[ds.split('/')[0]]. \ 382 append(hex(int(objid))) 383 for pool, objids in ds_pairs.items(): 384 for objid in objids: 385 kstat_update(pool, objid) 386 zil_build_dict(pool) 387 else: 388 try: 389 pools = subprocess.check_output(['zpool', 'list', '-Hpo',\ 390 'name']).decode('utf-8').split() 391 except subprocess.CalledProcessError as e: 392 print("Command: \"zpool list -Hpo name\" failed with error"\ 393 "code: " + str(e.returncode)) 394 sys.exit(1) 395 for pool in pools: 396 kstat_update(pool) 397 zil_build_dict(pool) 398 399def calculate_diff(): 400 global curr, diff 401 prev = copy.deepcopy(curr) 402 zil_process_kstat() 403 diff = copy.deepcopy(curr) 404 for pool in curr: 405 for objset in curr[pool]: 406 for col in hdr: 407 if col not in ['time', 'pool', 'ds', 'obj']: 408 key = cols[col][2] 409 # If prev is NULL, this is the 410 # first time we are here 411 if not prev: 412 diff[pool][objset][key] = 0 413 else: 414 diff[pool][objset][key] \ 415 = curr[pool][objset][key] \ 416 - prev[pool][objset][key] 417 418def zil_build_dict(pool = "GLOBAL"): 419 global kstat 420 for objset in kstat: 421 for key in kstat[objset]: 422 val = kstat[objset][key] 423 if pool not in curr: 424 curr[pool] = dict() 425 if objset not in curr[pool]: 426 curr[pool][objset] = dict() 427 curr[pool][objset][key] = val 428 curr[pool][objset]["pool"] = pool 429 curr[pool][objset]["objset"] = objset 430 curr[pool][objset]["time"] = time.strftime("%H:%M:%S", \ 431 time.localtime()) 432 433def sign_handler_epipe(sig, frame): 434 print("Caught EPIPE signal: " + str(frame)) 435 print("Exitting...") 436 sys.exit(0) 437 438def main(): 439 global interval 440 global curr 441 hprint = False 442 init() 443 signal.signal(signal.SIGINT, signal.SIG_DFL) 444 signal.signal(signal.SIGPIPE, sign_handler_epipe) 445 446 if interval > 0: 447 while True: 448 calculate_diff() 449 if not diff: 450 print ("Error: No stats to show") 451 sys.exit(0) 452 if hprint == False: 453 print_header() 454 hprint = True 455 print_dict(diff) 456 time.sleep(interval) 457 else: 458 zil_process_kstat() 459 if not curr: 460 print ("Error: No stats to show") 461 sys.exit(0) 462 print_header() 463 print_dict(curr) 464 465if __name__ == '__main__': 466 main() 467 468