1#!/usr/local/bin/python3.8
2#
3# Find holes in structures, so that we can pack them and improve our memory density.
4#
5# In order to make this work, you need to
6# (1) Be operating in a workspace where you have a __NON-DEBUG__ build of LibreOffice, but __WITH SYMBOLS__.
7#     (A debug build has different sizes for some things in the standard library.)
8# (2) First run the unusedfields loplugin to generate a log file
9# (3) Install the pahole stuff into your gdb, I used this one:
10#     https://github.com/PhilArmstrong/pahole-gdb
11# (4) Edit the loop near the top of the script to only produce results for one of our modules.
12#     Note that this will make GDB soak up about 8G of RAM, which is why I don't do more than one module at a time
13# (5) Run the script
14#     ./compilerplugins/clang/pahole-all-classes.py > ./compilerplugins/clang/pahole.results
15#
16
17import _thread
18import io
19import os
20import subprocess
21import time
22import re
23
24# search for all the class names in the file produced by the unusedfields loplugin
25#a = subprocess.Popen("grep 'definition:' workdir/loplugin.unusedfields.log | sort -u", stdout=subprocess.PIPE, shell=True)
26a = subprocess.Popen("cat n1", stdout=subprocess.PIPE, shell=True)
27
28classSet = set()
29classSourceLocDict = dict()
30with a.stdout as txt:
31    for line in txt:
32        tokens = line.decode('utf8').strip().split("\t")
33        className = tokens[2].strip()
34        srcLoc = tokens[5].strip()
35        # ignore things like unions
36        if "anonymous" in className: continue
37        # ignore duplicates
38        if className in classSet: continue
39        classSet.add(className)
40        classSourceLocDict[className] = srcLoc
41a.terminate()
42
43# Some of the pahole commands are going to fail, and I cannot read the error stream and the input stream
44# together because python has no way of (easily) doing a non-blocking read.
45# So I have to write the commands out using a background thread, and then read the entire resulting
46# stream out below.
47def write_pahole_commands(classes):
48    for className in classes:
49        stdin.write("echo " + className + " " + classSourceLocDict[className] + "\n")
50        stdin.write("pahole " + className + "\n")
51        stdin.flush()
52    stdin.write("echo all-done\n")
53    stdin.flush()
54    stdin.close() # only way to make it flush the last echo command
55
56# Use generator because lines often end up merged together in gdb's output, and we need
57# to split them up, and that creates a mess in the parsing logic.
58def read_generator(gdbOutput):
59    while True:
60        line = gdbOutput.readline().decode('utf8').strip()
61        for split in line.split("(gdb)"):
62            split = split.strip()
63            if len(split) == 0: continue
64            if "all-done" in split: return
65            yield split
66
67classList = sorted(classSet)
68
69# Process 200 classes at a time, otherwise gdb's memory usage blows up and kills the machine
70#
71while len(classList) > 0:
72
73    currClassList = classList[1:200];
74    classList = classList[200:]
75
76    gdbProc = subprocess.Popen("gdb", stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
77
78    stdin = io.TextIOWrapper(gdbProc.stdin, 'utf-8')
79
80    # make gdb load all the debugging info
81    stdin.write("set confirm off\n")
82    for filename in sorted(os.listdir('instdir/program')):
83        if filename.endswith(".so"):
84            stdin.write("add-symbol-file instdir/program/" + filename + "\n")
85    stdin.flush()
86
87
88    _thread.start_new_thread( write_pahole_commands, (currClassList,) )
89
90    firstLineRegex = re.compile("/\*\s+(\d+)\s+\*/ struct")
91    fieldLineRegex = re.compile("/\*\s+(\d+)\s+(\d+)\s+\*/ ")
92    holeLineRegex = re.compile("/\* XXX (\d+) bit hole, try to pack \*/")
93    # sometimes pahole can't determine the size of a sub-struct, and then it returns bad data
94    bogusLineRegex = re.compile("/\*\s+\d+\s+0\s+\*/")
95    structLines = list()
96    foundHole = False
97    cumulativeHoleBits = 0
98    structSize = 0
99    foundBogusLine = False
100    # pahole doesn't report space at the end of the structure, so work it out myself
101    sizeOfFields = 0
102    for line in read_generator(gdbProc.stdout):
103        structLines.append(line)
104        firstLineMatch = firstLineRegex.match(line)
105        if firstLineMatch:
106            structSize = int(firstLineMatch.group(1))
107        holeLineMatch = holeLineRegex.match(line)
108        if holeLineMatch:
109            foundHole = True
110            cumulativeHoleBits += int(holeLineMatch.group(1))
111        fieldLineMatch = fieldLineRegex.match(line)
112        if fieldLineMatch:
113            fieldSize = int(fieldLineMatch.group(2))
114            sizeOfFields = int(fieldLineMatch.group(1)) + fieldSize
115        if bogusLineRegex.match(line):
116            foundBogusLine = True
117        if line == "}":
118            # Ignore very large structs, packing those is not going to help much, and
119            # re-organising them can make them much less readable.
120            if foundHole and len(structLines) < 12 and structSize < 100 and not foundBogusLine:
121                # Verify that we have enough hole-space that removing it will result in a structure
122                # that still satisfies alignment requirements, otherwise the compiler will just put empty
123                # space at the end of the struct.
124                # TODO improve detection of the required alignment for a structure
125                potentialSpace = (cumulativeHoleBits / 8) + (sizeOfFields - structSize)
126                if potentialSpace >= 8:
127                    for line in structLines:
128                        print(line)
129                    if (sizeOfFields - structSize) > 0:
130                        print("hole at end of struct: " + str(sizeOfFields - structSize))
131            #  reset state
132            structLines.clear()
133            foundHole = False
134            cumulativeHoleBits = 0
135            structSize = 0
136            foundBogusLine = False
137            actualStructSize = 0
138
139    gdbProc.terminate()
140