1#!/usr/bin/env python 2 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this 5# file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7 8# Firefox about:memory log parser. 9 10from __future__ import absolute_import, print_function 11 12import argparse 13from collections import defaultdict 14import gzip 15import json 16 17# This value comes from nsIMemoryReporter.idl. 18KIND_HEAP = 1 19 20 21def path_total(data, path): 22 """ 23 Calculates the sum for the given data point path and its children. If 24 path does not end with a '/' then only the value for the exact path is 25 returned. 26 """ 27 path_totals = defaultdict(int) 28 29 # Bookkeeping for calculating the heap-unclassified measurement. 30 explicit_heap = defaultdict(int) 31 heap_allocated = defaultdict(int) 32 33 discrete = not path.endswith("/") 34 35 def match(value): 36 """ 37 Helper that performs either an explicit match or a prefix match 38 depending on the format of the path passed in. 39 """ 40 if discrete: 41 return value == path 42 else: 43 return value.startswith(path) 44 45 def update_bookkeeping(report): 46 """ 47 Adds the value to the heap total if this an explicit entry that is a 48 heap measurement and updates the heap allocated value if necessary. 49 """ 50 if report["kind"] == KIND_HEAP and report["path"].startswith("explicit/"): 51 explicit_heap[report["process"]] += report["amount"] 52 elif report["path"] == "heap-allocated": 53 heap_allocated[report["process"]] = report["amount"] 54 55 def heap_unclassified(process): 56 """ 57 Calculates the heap-unclassified value for the given process. This is 58 simply the difference between all values reported as heap allocated 59 under the explicit/ tree and the value reported for heap-allocated by 60 the allocator. 61 """ 62 # Memory reports should always include heap-allocated. If it's missing 63 # just assert. 64 assert process in heap_allocated 65 66 unclassified = heap_allocated[process] - explicit_heap[process] 67 68 # Make sure the value is sane. A misbehaving reporter could lead to 69 # negative values. 70 assert unclassified >= 0, "heap-unclassified was negative: %d" % unclassified 71 72 return unclassified 73 74 needs_bookkeeping = path in ("explicit/", "explicit/heap-unclassified") 75 76 # Process all the reports. 77 for report in data["reports"]: 78 if needs_bookkeeping: 79 update_bookkeeping(report) 80 81 if match(report["path"]): 82 path_totals[report["process"]] += report["amount"] 83 84 # Handle special processing for explicit and heap-unclassified. 85 if path == "explicit/": 86 # If 'explicit/' is requested we need to add the 'explicit/heap-unclassified' 87 # node that is generated by about:memory. 88 for k, v in explicit_heap.items(): 89 path_totals[k] += heap_unclassified(k) 90 elif path == "explicit/heap-unclassified": 91 # If 'explicit/heap-unclassified' is requested we need to calculate the 92 # value as it's generated by about:memory, not explicitly reported. 93 for k, v in explicit_heap.items(): 94 path_totals[k] = heap_unclassified(k) 95 96 return path_totals 97 98 99def calculate_memory_report_values( 100 memory_report_path, data_point_path, process_names=None 101): 102 """ 103 Opens the given memory report file and calculates the value for the given 104 data point. 105 106 :param memory_report_path: Path to the memory report file to parse. 107 :param data_point_path: Path of the data point to calculate in the memory 108 report, ie: 'explicit/heap-unclassified'. 109 :param process_name: Name of processes to limit reports to. ie 'Main' 110 """ 111 try: 112 with open(memory_report_path) as f: 113 data = json.load(f) 114 except ValueError: 115 # Check if the file is gzipped. 116 with gzip.open(memory_report_path, "rb") as f: 117 data = json.load(f) 118 119 totals = path_total(data, data_point_path) 120 121 # If a process name is provided, restricted output to processes matching 122 # that name. 123 if process_names is not None: 124 for k in list(totals.keys()): 125 if not any([process_name in k for process_name in process_names]): 126 del totals[k] 127 128 return totals 129 130 131if __name__ == "__main__": 132 parser = argparse.ArgumentParser( 133 description="Extract data points from about:memory reports" 134 ) 135 parser.add_argument("report", action="store", help="Path to a memory report file.") 136 parser.add_argument( 137 "prefix", 138 action="store", 139 help="Prefix of data point to measure. " 140 "If the prefix does not end in a '/' " 141 "then an exact match is made.", 142 ) 143 parser.add_argument( 144 "--proc-filter", 145 action="store", 146 nargs="*", 147 default=None, 148 help="Process name filter. " "If not provided all processes will be included.", 149 ) 150 parser.add_argument( 151 "--mebi", 152 action="store_true", 153 help="Output values as mebibytes (instead of bytes)" " to match about:memory.", 154 ) 155 156 args = parser.parse_args() 157 totals = calculate_memory_report_values(args.report, args.prefix, args.proc_filter) 158 159 sorted_totals = sorted(totals.items(), key=lambda item: (-item[1], item[0])) 160 for (k, v) in sorted_totals: 161 if v: 162 print("{0}\t".format(k)), 163 print("") 164 165 bytes_per_mebibyte = 1024.0 * 1024.0 166 for (k, v) in sorted_totals: 167 if v: 168 if args.mebi: 169 print("{0:.2f} MiB".format(v / bytes_per_mebibyte)), 170 else: 171 print("{0} bytes".format(v)), 172 print("\t"), 173 print("") 174