1#!/usr/bin/env python
2
3# This Source Code Form is subject to the terms of the Mozilla Public
4# License, v. 2.0. If a copy of the MPL was not distributed with this
5# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
7
8# Firefox about:memory log parser.
9
10from __future__ import absolute_import, print_function
11
12import argparse
13from collections import defaultdict
14import gzip
15import json
16
17# This value comes from nsIMemoryReporter.idl.
18KIND_HEAP = 1
19
20
21def path_total(data, path):
22    """
23    Calculates the sum for the given data point path and its children. If
24    path does not end with a '/' then only the value for the exact path is
25    returned.
26    """
27    path_totals = defaultdict(int)
28
29    # Bookkeeping for calculating the heap-unclassified measurement.
30    explicit_heap = defaultdict(int)
31    heap_allocated = defaultdict(int)
32
33    discrete = not path.endswith("/")
34
35    def match(value):
36        """
37        Helper that performs either an explicit match or a prefix match
38        depending on the format of the path passed in.
39        """
40        if discrete:
41            return value == path
42        else:
43            return value.startswith(path)
44
45    def update_bookkeeping(report):
46        """
47        Adds the value to the heap total if this an explicit entry that is a
48        heap measurement and updates the heap allocated value if necessary.
49        """
50        if report["kind"] == KIND_HEAP and report["path"].startswith("explicit/"):
51            explicit_heap[report["process"]] += report["amount"]
52        elif report["path"] == "heap-allocated":
53            heap_allocated[report["process"]] = report["amount"]
54
55    def heap_unclassified(process):
56        """
57        Calculates the heap-unclassified value for the given process. This is
58        simply the difference between all values reported as heap allocated
59        under the explicit/ tree and the value reported for heap-allocated by
60        the allocator.
61        """
62        # Memory reports should always include heap-allocated. If it's missing
63        # just assert.
64        assert process in heap_allocated
65
66        unclassified = heap_allocated[process] - explicit_heap[process]
67
68        # Make sure the value is sane. A misbehaving reporter could lead to
69        # negative values.
70        assert unclassified >= 0, "heap-unclassified was negative: %d" % unclassified
71
72        return unclassified
73
74    needs_bookkeeping = path in ("explicit/", "explicit/heap-unclassified")
75
76    # Process all the reports.
77    for report in data["reports"]:
78        if needs_bookkeeping:
79            update_bookkeeping(report)
80
81        if match(report["path"]):
82            path_totals[report["process"]] += report["amount"]
83
84    # Handle special processing for explicit and heap-unclassified.
85    if path == "explicit/":
86        # If 'explicit/' is requested we need to add the 'explicit/heap-unclassified'
87        # node that is generated by about:memory.
88        for k, v in explicit_heap.items():
89            path_totals[k] += heap_unclassified(k)
90    elif path == "explicit/heap-unclassified":
91        # If 'explicit/heap-unclassified' is requested we need to calculate the
92        # value as it's generated by about:memory, not explicitly reported.
93        for k, v in explicit_heap.items():
94            path_totals[k] = heap_unclassified(k)
95
96    return path_totals
97
98
99def calculate_memory_report_values(
100    memory_report_path, data_point_path, process_names=None
101):
102    """
103    Opens the given memory report file and calculates the value for the given
104    data point.
105
106    :param memory_report_path: Path to the memory report file to parse.
107    :param data_point_path: Path of the data point to calculate in the memory
108     report, ie: 'explicit/heap-unclassified'.
109    :param process_name: Name of processes to limit reports to. ie 'Main'
110    """
111    try:
112        with open(memory_report_path) as f:
113            data = json.load(f)
114    except ValueError:
115        # Check if the file is gzipped.
116        with gzip.open(memory_report_path, "rb") as f:
117            data = json.load(f)
118
119    totals = path_total(data, data_point_path)
120
121    # If a process name is provided, restricted output to processes matching
122    # that name.
123    if process_names is not None:
124        for k in list(totals.keys()):
125            if not any([process_name in k for process_name in process_names]):
126                del totals[k]
127
128    return totals
129
130
131if __name__ == "__main__":
132    parser = argparse.ArgumentParser(
133        description="Extract data points from about:memory reports"
134    )
135    parser.add_argument("report", action="store", help="Path to a memory report file.")
136    parser.add_argument(
137        "prefix",
138        action="store",
139        help="Prefix of data point to measure. "
140        "If the prefix does not end in a '/' "
141        "then an exact match is made.",
142    )
143    parser.add_argument(
144        "--proc-filter",
145        action="store",
146        nargs="*",
147        default=None,
148        help="Process name filter. " "If not provided all processes will be included.",
149    )
150    parser.add_argument(
151        "--mebi",
152        action="store_true",
153        help="Output values as mebibytes (instead of bytes)" " to match about:memory.",
154    )
155
156    args = parser.parse_args()
157    totals = calculate_memory_report_values(args.report, args.prefix, args.proc_filter)
158
159    sorted_totals = sorted(totals.items(), key=lambda item: (-item[1], item[0]))
160    for (k, v) in sorted_totals:
161        if v:
162            print("{0}\t".format(k)),
163    print("")
164
165    bytes_per_mebibyte = 1024.0 * 1024.0
166    for (k, v) in sorted_totals:
167        if v:
168            if args.mebi:
169                print("{0:.2f} MiB".format(v / bytes_per_mebibyte)),
170            else:
171                print("{0} bytes".format(v)),
172            print("\t"),
173    print("")
174