1#!/usr/bin/env python
2
3# This Source Code Form is subject to the terms of the Mozilla Public
4# License, v. 2.0. If a copy of the MPL was not distributed with this
5# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
7
8# Firefox about:memory log parser.
9
10from __future__ import absolute_import, print_function
11
12import argparse
13from collections import defaultdict
14import gzip
15import json
16
17# This value comes from nsIMemoryReporter.idl.
18KIND_HEAP = 1
19
20
21def path_total(data, path):
22    """
23    Calculates the sum for the given data point path and its children. If
24    path does not end with a '/' then only the value for the exact path is
25    returned.
26    """
27    path_totals = defaultdict(int)
28
29    # Bookkeeping for calculating the heap-unclassified measurement.
30    explicit_heap = defaultdict(int)
31    heap_allocated = defaultdict(int)
32
33    discrete = not path.endswith('/')
34
35    def match(value):
36        """
37        Helper that performs either an explicit match or a prefix match
38        depending on the format of the path passed in.
39        """
40        if discrete:
41            return value == path
42        else:
43            return value.startswith(path)
44
45    def update_bookkeeping(report):
46        """
47        Adds the value to the heap total if this an explicit entry that is a
48        heap measurement and updates the heap allocated value if necessary.
49        """
50        if report["kind"] == KIND_HEAP and report["path"].startswith("explicit/"):
51            explicit_heap[report["process"]] += report["amount"]
52        elif report["path"] == "heap-allocated":
53            heap_allocated[report["process"]] = report["amount"]
54
55    def heap_unclassified(process):
56        """
57        Calculates the heap-unclassified value for the given process. This is
58        simply the difference between all values reported as heap allocated
59        under the explicit/ tree and the value reported for heap-allocated by
60        the allocator.
61        """
62        # Memory reports should always include heap-allocated. If it's missing
63        # just assert.
64        assert process in heap_allocated
65
66        unclassified = heap_allocated[process] - explicit_heap[process]
67
68        # Make sure the value is sane. A misbehaving reporter could lead to
69        # negative values.
70        assert unclassified >= 0, "heap-unclassified was negative: %d" % unclassified
71
72        return unclassified
73
74    needs_bookkeeping = path in ("explicit/", "explicit/heap-unclassified")
75
76    # Process all the reports.
77    for report in data["reports"]:
78        if needs_bookkeeping:
79            update_bookkeeping(report)
80
81        if match(report["path"]):
82            path_totals[report["process"]] += report["amount"]
83
84    # Handle special processing for explicit and heap-unclassified.
85    if path == "explicit/":
86        # If 'explicit/' is requested we need to add the 'explicit/heap-unclassified'
87        # node that is generated by about:memory.
88        for k, v in explicit_heap.items():
89            path_totals[k] += heap_unclassified(k)
90    elif path == "explicit/heap-unclassified":
91        # If 'explicit/heap-unclassified' is requested we need to calculate the
92        # value as it's generated by about:memory, not explicitly reported.
93        for k, v in explicit_heap.items():
94            path_totals[k] = heap_unclassified(k)
95
96    return path_totals
97
98
99def calculate_memory_report_values(memory_report_path, data_point_path,
100                                   process_name=None):
101    """
102    Opens the given memory report file and calculates the value for the given
103    data point.
104
105    :param memory_report_path: Path to the memory report file to parse.
106    :param data_point_path: Path of the data point to calculate in the memory
107     report, ie: 'explicit/heap-unclassified'.
108    :param process_name: Name of process to limit reports to. ie 'Main'
109    """
110    try:
111        with open(memory_report_path) as f:
112            data = json.load(f)
113    except ValueError:
114        # Check if the file is gzipped.
115        with gzip.open(memory_report_path, 'rb') as f:
116            data = json.load(f)
117
118    totals = path_total(data, data_point_path)
119
120    # If a process name is provided, restricted output to processes matching
121    # that name.
122    if process_name:
123        for k in totals.keys():
124            if process_name not in k:
125                del totals[k]
126
127    return totals
128
129
130if __name__ == "__main__":
131    parser = argparse.ArgumentParser(
132            description='Extract data points from about:memory reports')
133    parser.add_argument('report', action='store',
134                        help='Path to a memory report file.')
135    parser.add_argument('prefix', action='store',
136                        help='Prefix of data point to measure. '
137                        'If the prefix does not end in a \'/\' '
138                        'then an exact match is made.')
139    parser.add_argument('--proc-filter', action='store', default=None,
140                        help='Process name filter. '
141                             'If not provided all processes will be included.')
142    parser.add_argument('--mebi', action='store_true',
143                        help='Output values as mebibytes (instead of bytes)'
144                        ' to match about:memory.')
145
146    args = parser.parse_args()
147    totals = calculate_memory_report_values(
148                    args.report, args.prefix, args.proc_filter)
149
150    sorted_totals = sorted(totals.items(), key=lambda item: (-item[1], item[0]))
151    for (k, v) in sorted_totals:
152        if v:
153            print("{0}\t".format(k)),
154    print("")
155
156    bytes_per_mebibyte = 1024.0 * 1024.0
157    for (k, v) in sorted_totals:
158        if v:
159            if args.mebi:
160                print("{0:.2f} MiB".format(v / bytes_per_mebibyte)),
161            else:
162                print("{0} bytes".format(v)),
163            print("\t"),
164    print("")
165