1#!/usr/bin/env python
2
3import memcache
4import struct
5import os
6import hashlib
7
8"""
9/* blob format for storing:
10
11    char magic[4]; # 'CCH1', might change for other version of ccache
12                   # ccache will erase the blob in memcached if wrong magic
13    uint32_t obj_len; # network endian
14    char *obj[obj_len];
15    uint32_t stderr_len; # network endian
16    char *stderr[stderr_len];
17    uint32_t dia_len; # network endian
18    char *dia[dia_len];
19    uint32_t dep_len; # network endian
20    char *dep[dep_len];
21
22*/
23"""
24MEMCCACHE_MAGIC = 'CCH1'
25
26def set_blob(data):
27    return struct.pack('!I', len(data)) + str(data)
28MEMCCACHE_BIG = 'CCBM'
29
30"""
31/* blob format for big values:
32
33    char magic[4]; # 'CCBM'
34    uint32_t numkeys; # network endian
35    uint32_t hash_size; # network endian
36    uint32_t reserved; # network endian
37    uint32_t value_length; # network endian
38
39    <hash[0]>       hash of include file                (<hash_size> bytes)
40    <size[0]>       size of include file                (4 bytes unsigned int)
41    ...
42    <hash[n-1]>
43    <size[n-1]>
44
45*/
46"""
47MEMCCACHE_BIG = 'CCBM'
48
49MAX_VALUE_SIZE = 1000 << 10 # 1M with memcached overhead
50SPLIT_VALUE_SIZE = MAX_VALUE_SIZE
51
52server = os.getenv("MEMCACHED_SERVERS", "localhost")
53mc = memcache.Client(server.split(','), debug=1)
54
55ccache = os.getenv("CCACHE_DIR", os.path.expanduser("~/.ccache"))
56filelist = []
57for dirpath, dirnames, filenames in os.walk(ccache):
58    # sort by modification time, most recently used last
59    for filename in filenames:
60        stat = os.stat(os.path.join(dirpath, filename))
61        filelist.append((stat.st_mtime, dirpath, filename))
62filelist.sort()
63files = blobs = chunks = objects = manifest = 0
64for mtime, dirpath, filename in filelist:
65    dirname = dirpath.replace(ccache + os.path.sep, "")
66    if filename == "CACHEDIR.TAG":
67        # ignore these
68        files = files + 1
69    else:
70        (base, ext) = os.path.splitext(filename)
71        if ext == '.o':
72            objects = objects + 1
73            key = "".join(list(os.path.split(dirname)) + [base])
74            def read_file(path):
75                return os.path.exists(path) and open(path).read() or ""
76            obj = read_file(os.path.join(dirpath, filename))
77            stderr = read_file(os.path.join(dirpath, base) + '.stderr')
78            dia = read_file(os.path.join(dirpath, base) + '.dia')
79            dep = read_file(os.path.join(dirpath, base) + '.d')
80
81            print "%s: %d %d %d %d" % (key, len(obj), len(stderr), len(dia), len(dep))
82            val = MEMCCACHE_MAGIC
83            val += set_blob(obj)
84            val += set_blob(stderr)
85            val += set_blob(dia)
86            val += set_blob(dep)
87            if len(val) > MAX_VALUE_SIZE:
88                numkeys = (len(val) + SPLIT_VALUE_SIZE - 1) / SPLIT_VALUE_SIZE
89                buf = MEMCCACHE_BIG
90                buf += struct.pack('!I', numkeys)
91                buf += struct.pack('!I', 16)
92                buf += struct.pack('!I', 0)
93                buf += struct.pack('!I', len(val))
94                def splitchunks(s, n):
95                    """Produce `n`-character chunks from `s`."""
96                    for start in range(0, len(s), n):
97                        yield s[start:start+n]
98                valmap = {}
99                for subval in splitchunks(val, SPLIT_VALUE_SIZE):
100                    subhash = hashlib.new('md4')
101                    subhash.update(subval)
102                    buf += subhash.digest() + struct.pack('!I', len(subval))
103                    subkey = "%s-%d" % (subhash.hexdigest(), len(subval))
104                    print "# %s: chunk %d" % (subkey, len(subval))
105                    #mc.set(subkey, subval)
106                    valmap[subkey] = subval
107                    chunks = chunks + 1
108                mc.set_multi(valmap)
109                mc.set(key, buf)
110            else:
111                mc.set(key, val)
112            files = files + 1
113            blobs = blobs + 1
114        elif ext == '.stderr' or ext == '.d' or ext == '.dia':
115            # was added above
116            files = files + 1
117        elif ext == '.manifest':
118            manifest = manifest + 1
119            key = "".join(list(os.path.split(dirname)) + [base])
120            val = open(os.path.join(dirpath, filename)).read() or None
121            if val:
122                print "%s: manifest %d" % (key, len(val))
123                mc.set(key, val, 0, 0)
124            files = files + 1
125            blobs = blobs + 1
126print "%d files, %d objects (%d manifest) = %d blobs (%d chunks)" % (files, objects, manifest, blobs, chunks)
127