1#!/usr/bin/env python3
2
3# ***** BEGIN GPL LICENSE BLOCK *****
4#
5# This program is free software; you can redistribute it and/or
6# modify it under the terms of the GNU General Public License
7# as published by the Free Software Foundation; either version 2
8# of the License, or (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License
16# along with this program; if not, write to the Free Software Foundation,
17# Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
18#
19# ***** END GPL LICENCE BLOCK *****
20#
21# (c) 2015, Blender Foundation - Bastien Montagne
22
23# <pep8 compliant>
24
25
26"""
27This is a tool for generating a JSon version of a blender file (only its structure, or all its data included).
28
29It can also run some simple validity checks over a .blend file.
30
31WARNING! This is still WIP tool!
32
33Example usage:
34
35   ./blend2json.py foo.blend
36
37To output also all 'name' fields from data:
38
39   ./blend2json.py --filter-data="name" foo.blend
40
41To output complete DNA struct info:
42
43   ./blend2json.py --full-dna foo.blend
44
45To avoid getting all 'uid' old addresses (those will change really often even when data itself does not change,
46making diff pretty noisy):
47
48   ./blend2json.py --no-old-addresses foo.blend
49
50To check a .blend file instead of outputting its JSon version (use explicit -o option to do both at the same time):
51
52   ./blend2json.py -c foo.blend
53
54"""
55
56FILTER_DOC = """
57Each generic filter is made of three arguments, the include/exclude toggle ('+'/'-'), a regex to match against the name
58of the field to check (either one of the 'meta-data' generated by json exporter, or actual data field from DNA structs),
59and some regex to match against the data of this field (JSON-ified representation of the data, hence always a string).
60
61Filters are evaluated in the order they are given, that is, if a block does not pass the first filter,
62it is immediately rejected and no further check is done on it.
63
64You can add some recursivity to a filter (that is, if an 'include' filter is successful over a 'pointer' property,
65it will also automatically include pointed data, with a level of recusivity), by adding either
66'*' (for infinite recursion) or a number (to specify the maximum level of recursion) to the include/exclude toggle.
67Note that it only makes sense in 'include' case, and gets ignored for 'exclude' one.
68
69Examples:
70
71To include only MESH blocks:
72
73   ./blend2json.py --filter-block "+" "code" "ME" foo.blend
74
75To include only MESH or CURVE blocks and all data used by them:
76
77   ./blend2json.py --filter-block "+" "code" "(ME)|(CU)" --filter-block "+*" ".*" ".*" foo.blend
78
79"""
80
81import os
82import json
83import re
84
85# Avoid maintaining multiple blendfile modules
86import sys
87sys.path.append(os.path.join(os.path.dirname(__file__), "..", "modules"))
88del sys
89
90import blendfile
91
92
93##### Utils (own json formatting) #####
94
95
96def json_default(o):
97    if isinstance(o, bytes):
98        return repr(o)[2:-1]
99    elif i is ...:
100        return "<...>"
101    return o
102
103
104def json_dumps(i):
105    return json.dumps(i, default=json_default)
106
107
108def keyval_to_json(kvs, indent, indent_step, compact_output=False):
109    if compact_output:
110        return ('{' + ', '.join('"%s": %s' % (k, v) for k, v in kvs) + '}')
111    else:
112        return ('{%s' % indent_step[:-1] +
113                (',\n%s%s' % (indent, indent_step)).join(
114                    ('"%s":\n%s%s%s' % (k, indent, indent_step, v) if (v[0] in {'[', '{'}) else
115                     '"%s": %s' % (k, v)) for k, v in kvs) +
116                '\n%s}' % indent)
117
118
119def list_to_json(lst, indent, indent_step, compact_output=False):
120    if compact_output:
121        return ('[' + ', '.join(l for l in lst) + ']')
122    else:
123        return ('[%s' % indent_step[:-1] +
124                ((',\n%s%s' % (indent, indent_step)).join(
125                    ('\n%s%s%s' % (indent, indent_step, l) if (i == 0 and l[0] in {'[', '{'}) else l)
126                    for i, l in enumerate(lst))
127                 ) +
128                '\n%s]' % indent)
129
130
131##### Main 'struct' writers #####
132
133def gen_fake_addresses(args, blend):
134    if args.use_fake_address:
135        hashes = set()
136        ret = {}
137        for block in blend.blocks:
138            if not block.addr_old:
139                continue
140            hsh = block.get_data_hash()
141            while hsh in hashes:
142                hsh += 1
143            hashes.add(hsh)
144            ret[block.addr_old] = hsh
145        return ret
146
147    return {}
148
149
150def bheader_to_json(args, fw, blend, indent, indent_step):
151    fw('%s"%s": [\n' % (indent, "HEADER"))
152    indent = indent + indent_step
153
154    keyval = (
155        ("magic", json_dumps(blend.header.magic)),
156        ("pointer_size", json_dumps(blend.header.pointer_size)),
157        ("is_little_endian", json_dumps(blend.header.is_little_endian)),
158        ("version", json_dumps(blend.header.version)),
159    )
160    keyval = keyval_to_json(keyval, indent, indent_step)
161    fw('%s%s' % (indent, keyval))
162
163    indent = indent[:-len(indent_step)]
164    fw('\n%s]' % indent)
165
166
167def do_bblock_filter(filters, blend, block, meta_keyval, data_keyval):
168    def do_bblock_filter_data_recursive(blend, block, rec_lvl, rec_iter, key=None):
169        fields = (blend.structs[block.sdna_index].fields if key is None else
170                  [blend.structs[block.sdna_index].field_from_name.get(key[1:-1].encode())])
171        for fld in fields:
172            if fld is None:
173                continue
174            if fld.dna_name.is_pointer:
175                paths = ([(fld.dna_name.name_only, i) for i in range(fld.dna_name.array_size)]
176                         if fld.dna_name.array_size > 1 else [fld.dna_name.name_only])
177                for p in paths:
178                    child_block = block.get_pointer(p)
179                    if child_block is not None:
180                        child_block.user_data = max(block.user_data, rec_iter)
181                        if rec_lvl != 0:
182                            do_bblock_filter_data_recursive(blend, child_block, rec_lvl - 1, rec_iter + 1)
183
184    has_include = False
185    do_break = False
186    rec_iter = 1
187    if block.user_data is None:
188        block.user_data = 0
189    for include, rec_lvl, key, val in filters:
190        if rec_lvl < 0:
191            rec_lvl = 100
192        has_include = has_include or include
193        # Skip exclude filters if block was already processed some way.
194        if not include and block.user_data is not None:
195            continue
196        has_match = False
197        for k, v in meta_keyval:
198            if key.search(k) and val.search(v):
199                has_match = True
200                if include:
201                    block.user_data = max(block.user_data, rec_iter)
202                    # Note that in include cases, we have to keep checking filters, since some 'include recursive'
203                    # ones may still have to be processed...
204                else:
205                    block.user_data = min(block.user_data, -rec_iter)
206                    do_break = True  # No need to check more filters in exclude case...
207                    break
208        for k, v in data_keyval:
209            if key.search(k) and val.search(v):
210                has_match = True
211                if include:
212                    block.user_data = max(block.user_data, rec_iter)
213                    if rec_lvl != 0:
214                        do_bblock_filter_data_recursive(blend, block, rec_lvl - 1, rec_iter + 1, k)
215                    # Note that in include cases, we have to keep checking filters, since some 'include recursive'
216                    # ones may still have to be processed...
217                else:
218                    block.user_data = min(block.user_data, -rec_iter)
219                    do_break = True  # No need to check more filters in exclude case...
220                    break
221        if include and not has_match:  # Include check failed, implies exclusion.
222            block.user_data = min(block.user_data, -rec_iter)
223            do_break = True  # No need to check more filters in exclude case...
224        if do_break:
225            break
226    # Implicit 'include all' in case no include filter is specified...
227    if block.user_data == 0 and not has_include:
228        block.user_data = max(block.user_data, rec_iter)
229
230
231def bblocks_to_json(args, fw, blend, address_map, indent, indent_step):
232    no_address = args.no_address
233    full_data = args.full_data
234    filter_data = args.filter_data
235
236    def gen_meta_keyval(blend, block):
237        keyval = [
238            ("code", json_dumps(block.code)),
239            ("size", json_dumps(block.size)),
240        ]
241        if not no_address:
242            keyval += [("addr_old", json_dumps(address_map.get(block.addr_old, block.addr_old)))]
243        keyval += [
244            ("dna_type_id", json_dumps(blend.structs[block.sdna_index].dna_type_id)),
245            ("count", json_dumps(block.count)),
246        ]
247        return keyval
248
249    def gen_data_keyval(blend, block, key_filter=None):
250        def _is_pointer(k):
251            return blend.structs[block.sdna_index].field_from_path(blend.header, blend.handle, k).dna_name.is_pointer
252        if key_filter is not None:
253            return [(json_dumps(k)[1:-1], json_dumps(address_map.get(v, v) if _is_pointer(k) else v))
254                    for k, v in block.items_recursive_iter() if k in key_filter]
255        return [(json_dumps(k)[1:-1], json_dumps(address_map.get(v, v) if _is_pointer(k) else v))
256                for k, v in block.items_recursive_iter()]
257
258    if args.block_filters:
259        for block in blend.blocks:
260            meta_keyval = gen_meta_keyval(blend, block)
261            data_keyval = gen_data_keyval(blend, block)
262            do_bblock_filter(args.block_filters, blend, block, meta_keyval, data_keyval)
263
264    fw('%s"%s": [\n' % (indent, "DATA"))
265    indent = indent + indent_step
266
267    is_first = True
268    for i, block in enumerate(blend.blocks):
269        if block.user_data is None or block.user_data > 0:
270            meta_keyval = gen_meta_keyval(blend, block)
271            if full_data:
272                meta_keyval.append(("data", keyval_to_json(gen_data_keyval(blend, block),
273                                                           indent + indent_step, indent_step, args.compact_output)))
274            elif filter_data:
275                meta_keyval.append(("data", keyval_to_json(gen_data_keyval(blend, block, filter_data),
276                                                           indent + indent_step, indent_step, args.compact_output)))
277            keyval = keyval_to_json(meta_keyval, indent, indent_step, args.compact_output)
278            fw('%s%s%s' % ('' if is_first else ',\n', indent, keyval))
279            is_first = False
280
281    indent = indent[:-len(indent_step)]
282    fw('\n%s]' % indent)
283
284
285def bdna_to_json(args, fw, blend, indent, indent_step):
286    full_dna = args.full_dna and not args.compact_output
287
288    def bdna_fields_to_json(blend, dna, indent, indent_step):
289        lst = []
290        for i, field in enumerate(dna.fields):
291            keyval = (
292                ("dna_name", json_dumps(field.dna_name.name_only)),
293                ("dna_type_id", json_dumps(field.dna_type.dna_type_id)),
294                ("is_pointer", json_dumps(field.dna_name.is_pointer)),
295                ("is_method_pointer", json_dumps(field.dna_name.is_method_pointer)),
296                ("array_size", json_dumps(field.dna_name.array_size)),
297            )
298            lst.append(keyval_to_json(keyval, indent + indent_step, indent_step))
299        return list_to_json(lst, indent, indent_step)
300
301    fw('%s"%s": [\n' % (indent, "DNA_STRUCT"))
302    indent = indent + indent_step
303
304    is_first = True
305    for dna in blend.structs:
306        keyval = [
307            ("dna_type_id", json_dumps(dna.dna_type_id)),
308            ("size", json_dumps(dna.size)),
309        ]
310        if full_dna:
311            keyval += [("fields", bdna_fields_to_json(blend, dna, indent + indent_step, indent_step))]
312        else:
313            keyval += [("nbr_fields", json_dumps(len(dna.fields)))]
314        keyval = keyval_to_json(keyval, indent, indent_step, args.compact_output)
315        fw('%s%s%s' % ('' if is_first else ',\n', indent, keyval))
316        is_first = False
317
318    indent = indent[:-len(indent_step)]
319    fw('\n%s]' % indent)
320
321
322def blend_to_json(args, f, blend, address_map):
323    fw = f.write
324    fw('{\n')
325    indent = indent_step = "  "
326    bheader_to_json(args, fw, blend, indent, indent_step)
327    fw(',\n')
328    bblocks_to_json(args, fw, blend, address_map, indent, indent_step)
329    fw(',\n')
330    bdna_to_json(args, fw, blend, indent, indent_step)
331    fw('\n}\n')
332
333
334##### Checks #####
335
336def check_file(args, blend):
337    addr_old = set()
338    for block in blend.blocks:
339        if block.addr_old in addr_old:
340            print("ERROR! Several data blocks share same 'addr_old' uuid %d, "
341                  "this should never happen!" % block.addr_old)
342            continue
343        addr_old.add(block.addr_old)
344
345
346##### Main #####
347
348def argparse_create():
349    import argparse
350    global __doc__
351
352    # When --help or no args are given, print this help
353    usage_text = __doc__
354
355    epilog = "This script is typically used to check differences between .blend files, or to check their validity."
356
357    parser = argparse.ArgumentParser(description=usage_text, epilog=epilog,
358                                     formatter_class=argparse.RawDescriptionHelpFormatter)
359
360    parser.add_argument(
361        dest="input", nargs="+", metavar='PATH',
362        help="Input .blend file(s)")
363    parser.add_argument(
364        "-o", "--output", dest="output", action="append", metavar='PATH', required=False,
365        help="Output .json file(s) (same path/name as input file(s) if not specified)")
366    parser.add_argument(
367        "-c", "--check-file", dest="check_file", default=False, action='store_true', required=False,
368        help=("Perform some basic validation checks over the .blend file"))
369    parser.add_argument(
370        "--compact-output", dest="compact_output", default=False, action='store_true', required=False,
371        help=("Output a very compact representation of blendfile (one line per block/DNAStruct)"))
372    parser.add_argument(
373        "--no-old-addresses", dest="no_address", default=False, action='store_true', required=False,
374        help=("Do not output old memory address of each block of data "
375              "(used as 'uuid' in .blend files, but change pretty noisily)"))
376    parser.add_argument(
377        "--no-fake-old-addresses", dest="use_fake_address", default=True, action='store_false',
378        required=False,
379        help=("Do not 'rewrite' old memory address of each block of data "
380              "(they are rewritten by default to some hash of their content, "
381              "to try to avoid too much diff noise between different but similar files)"))
382    parser.add_argument(
383        "--full-data", dest="full_data",
384        default=False, action='store_true', required=False,
385        help=("Also put in JSon file data itself "
386              "(WARNING! will generate *huge* verbose files - and is far from complete yet)"))
387    parser.add_argument(
388        "--filter-data", dest="filter_data",
389        default=None, required=False,
390        help=("Only put in JSon file data fields which names match given comma-separated list "
391              "(ignored if --full-data is set)"))
392    parser.add_argument(
393        "--full-dna", dest="full_dna", default=False, action='store_true', required=False,
394        help=("Also put in JSon file dna properties description (ignored when --compact-output is used)"))
395
396    group = parser.add_argument_group("Filters", FILTER_DOC)
397    group.add_argument(
398        "--filter-block", dest="block_filters", nargs=3, action='append',
399        help=("Filter to apply to BLOCKS (a.k.a. data itself)"))
400
401    return parser
402
403
404def main():
405    # ----------
406    # Parse Args
407
408    args = argparse_create().parse_args()
409
410    if not args.output:
411        if args.check_file:
412            args.output = [None] * len(args.input)
413        else:
414            args.output = [os.path.splitext(infile)[0] + ".json" for infile in args.input]
415
416    if args.block_filters:
417        args.block_filters = [(True if m[0] == "+" else False,
418                               0 if len(m) == 1 else (-1 if m[1] == "*" else int(m[1:])),
419                               re.compile(f), re.compile(d))
420                              for m, f, d in args.block_filters]
421
422    if args.filter_data:
423        if args.full_data:
424            args.filter_data = None
425        else:
426            args.filter_data = {n.encode() for n in args.filter_data.split(',')}
427
428    for infile, outfile in zip(args.input, args.output):
429        with blendfile.open_blend(infile) as blend:
430            address_map = gen_fake_addresses(args, blend)
431
432            if args.check_file:
433                check_file(args, blend)
434
435            if outfile:
436                with open(outfile, 'w', encoding="ascii", errors='xmlcharrefreplace') as f:
437                    blend_to_json(args, f, blend, address_map)
438
439
440if __name__ == "__main__":
441    main()
442