1#!/usr/local/bin/python3.8
2#
3# Find holes in structures, so that we can pack them and improve our memory density.
4#
5# In order to make this work, you need to
6# (1) Be operating in a workspace where you have a __NON-DEBUG__ build of LibreOffice, but __WITH SYMBOLS__.
7#     (A debug build has different sizes for some things in the standard library.)
8# (2) First run the unusedfields loplugin to generate a log file
9# (3) Install the pahole stuff into your gdb, I used this one:
10#     https://github.com/PhilArmstrong/pahole-gdb
11# (4) Run the script
12#     ./compilerplugins/clang/pahole-all-classes.py
13#
14
15import _thread
16import io
17import os
18import subprocess
19import time
20import re
21
22# search for all the class names in the file produced by the unusedfields loplugin
23#a = subprocess.Popen("grep 'definition:' workdir/loplugin.unusedfields.log | sort -u", stdout=subprocess.PIPE, shell=True)
24a = subprocess.Popen("cat n1", stdout=subprocess.PIPE, shell=True)
25
26classSet = set()
27classSourceLocDict = dict()
28locToClassDict = dict()
29with a.stdout as txt:
30    for line in txt:
31        tokens = line.decode('utf8').strip().split("\t")
32        className = tokens[2].strip()
33        srcLoc = tokens[5].strip()
34        # ignore things like unions
35        if "anonymous" in className: continue
36        # ignore duplicates
37        if className in classSet: continue
38        classSet.add(className)
39        classSourceLocDict[className] = srcLoc
40        locToClassDict[srcLoc] = className
41a.terminate()
42
43# Some of the pahole commands are going to fail, and I cannot read the error stream and the input stream
44# together because python has no way of (easily) doing a non-blocking read.
45# So I have to write the commands out using a background thread, and then read the entire resulting
46# stream out below.
47def write_pahole_commands(classes):
48    for className in classes:
49        stdin.write("echo " + className + " " + classSourceLocDict[className] + "\n")
50        stdin.write("pahole " + className + "\n")
51        stdin.flush()
52    stdin.write("echo all-done\n")
53    stdin.flush()
54    stdin.close() # only way to make it flush the last echo command
55
56# Use generator because lines often end up merged together in gdb's output, and we need
57# to split them up, and that creates a mess in the parsing logic.
58def read_generator(gdbOutput):
59    while True:
60        line = gdbOutput.readline();
61        if line == "": return # end of file
62        line = line.decode('utf8').strip()
63        print("gdb: " + line)
64        for split in line.split("(gdb)"):
65            split = split.strip()
66            if len(split) == 0: continue
67            if "all-done" in split: return
68            yield split
69
70# build list of classes sorted by source location to increase the chances of
71# processing stuff stored in the same DSO together
72sortedLocs = sorted(locToClassDict.keys())
73classList = list()
74for src in sortedLocs:
75    if "/inc/" in src or "include/" in src:
76        classList.append(locToClassDict[src])
77
78with open("compilerplugins/clang/pahole.results", "wt") as f:
79    # Process 400 classes at a time, otherwise gdb's memory usage blows up and kills the machine
80    # This number is chosen to make gdb peak at around 8G.
81    while len(classList) > 0:
82
83        currClassList = classList[0:500];
84        classList = classList[500:]
85
86        gdbProc = subprocess.Popen("gdb", stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
87
88        stdin = io.TextIOWrapper(gdbProc.stdin, 'utf-8')
89
90        # make gdb load all the debugging info
91        stdin.write("set confirm off\n")
92        # make gdb not wrap output and mess up my parsing
93        stdin.write("set width unlimited\n")
94        for filename in sorted(os.listdir('instdir/program')):
95            if filename.endswith(".so"):
96                stdin.write("add-symbol-file instdir/program/" + filename + "\n")
97        stdin.flush()
98
99
100        _thread.start_new_thread( write_pahole_commands, (currClassList,) )
101
102        firstLineRegex = re.compile(r"/\*\s+(\d+)\s+\*/ struct") # /* 16 */ struct Foo
103        fieldLineRegex = re.compile(r"/\*\s+(\d+)\s+(\d+)\s+\*/ ") # /* 12 8 */ class rtl::OUString aName
104        holeLineRegex = re.compile(r"/\* XXX (\d+) bit hole, try to pack \*/")
105        # sometimes pahole can't determine the size of a sub-struct, and then it returns bad data
106        bogusLineRegex = re.compile(r"/\*\s+\d+\s+0\s+\*/")
107        structLines = list()
108        foundHole = False
109        cumulativeHoleBits = 0
110        alignedStructSize = 0
111        foundBogusLine = False
112        # pahole doesn't report space at the end of the structure, so work it out myself
113        sizeOfStructWithoutPadding = 0
114        for line in read_generator(gdbProc.stdout):
115            structLines.append(line)
116            firstLineMatch = firstLineRegex.match(line)
117            if firstLineMatch:
118                alignedStructSize = int(firstLineMatch.group(1))
119                structLines.clear()
120                structLines.append(line)
121            holeLineMatch = holeLineRegex.match(line)
122            if holeLineMatch:
123                foundHole = True
124                cumulativeHoleBits += int(holeLineMatch.group(1))
125            fieldLineMatch = fieldLineRegex.match(line)
126            if fieldLineMatch:
127                fieldPosInBytes = int(fieldLineMatch.group(1))
128                fieldSizeInBytes = int(fieldLineMatch.group(2))
129                sizeOfStructWithoutPadding = fieldPosInBytes + fieldSizeInBytes
130            if bogusLineRegex.match(line):
131                foundBogusLine = True
132            if line == "}":
133                # Ignore very large structs, packing those is not going to help much, and
134                # re-organising them can make them much less readable.
135                if foundHole and len(structLines) < 16 and alignedStructSize < 100 and not foundBogusLine:
136                    # Verify that, after packing, and compiler alignment, the new structure will be actually smaller.
137                    # Sometimes, we can save space, but the compiler will align the structure such that we don't
138                    # actually save any space.
139                    # TODO improve detection of the required alignment for a structure
140                    holeAtEnd = alignedStructSize - sizeOfStructWithoutPadding
141                    potentialSpace = (cumulativeHoleBits / 8) + holeAtEnd
142                    if potentialSpace >= 8:
143                        for line in structLines:
144                            f.write(line + "\n")
145                        if holeAtEnd > 0:
146                            f.write("hole at end of struct: " + str(holeAtEnd) + "\n")
147                        f.write("\n")
148                #  reset state
149                structLines.clear()
150                foundHole = False
151                cumulativeHoleBits = 0
152                structSize = 0
153                foundBogusLine = False
154                actualStructSize = 0
155
156        gdbProc.terminate()
157