1#!/usr/bin/env python3 2 3# ***** BEGIN GPL LICENSE BLOCK ***** 4# 5# This program is free software; you can redistribute it and/or 6# modify it under the terms of the GNU General Public License 7# as published by the Free Software Foundation; either version 2 8# of the License, or (at your option) any later version. 9# 10# This program is distributed in the hope that it will be useful, 11# but WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13# GNU General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with this program; if not, write to the Free Software Foundation, 17# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18# 19# ***** END GPL LICENCE BLOCK ***** 20# 21# (c) 2015, Blender Foundation - Bastien Montagne 22 23# <pep8 compliant> 24 25 26""" 27This is a tool for generating a JSon version of a blender file (only its structure, or all its data included). 28 29It can also run some simple validity checks over a .blend file. 30 31WARNING! This is still WIP tool! 32 33Example usage: 34 35 ./blend2json.py foo.blend 36 37To output also all 'name' fields from data: 38 39 ./blend2json.py --filter-data="name" foo.blend 40 41To output complete DNA struct info: 42 43 ./blend2json.py --full-dna foo.blend 44 45To avoid getting all 'uid' old addresses (those will change really often even when data itself does not change, 46making diff pretty noisy): 47 48 ./blend2json.py --no-old-addresses foo.blend 49 50To check a .blend file instead of outputting its JSon version (use explicit -o option to do both at the same time): 51 52 ./blend2json.py -c foo.blend 53 54""" 55 56FILTER_DOC = """ 57Each generic filter is made of three arguments, the include/exclude toggle ('+'/'-'), a regex to match against the name 58of the field to check (either one of the 'meta-data' generated by json exporter, or actual data field from DNA structs), 59and some regex to match against the data of this field (JSON-ified representation of the data, hence always a string). 60 61Filters are evaluated in the order they are given, that is, if a block does not pass the first filter, 62it is immediately rejected and no further check is done on it. 63 64You can add some recursivity to a filter (that is, if an 'include' filter is successful over a 'pointer' property, 65it will also automatically include pointed data, with a level of recusivity), by adding either 66'*' (for infinite recursion) or a number (to specify the maximum level of recursion) to the include/exclude toggle. 67Note that it only makes sense in 'include' case, and gets ignored for 'exclude' one. 68 69Examples: 70 71To include only MESH blocks: 72 73 ./blend2json.py --filter-block "+" "code" "ME" foo.blend 74 75To include only MESH or CURVE blocks and all data used by them: 76 77 ./blend2json.py --filter-block "+" "code" "(ME)|(CU)" --filter-block "+*" ".*" ".*" foo.blend 78 79""" 80 81import os 82import json 83import re 84 85# Avoid maintaining multiple blendfile modules 86import sys 87sys.path.append(os.path.join(os.path.dirname(__file__), "..", "modules")) 88del sys 89 90import blendfile 91 92 93##### Utils (own json formatting) ##### 94 95 96def json_default(o): 97 if isinstance(o, bytes): 98 return repr(o)[2:-1] 99 elif i is ...: 100 return "<...>" 101 return o 102 103 104def json_dumps(i): 105 return json.dumps(i, default=json_default) 106 107 108def keyval_to_json(kvs, indent, indent_step, compact_output=False): 109 if compact_output: 110 return ('{' + ', '.join('"%s": %s' % (k, v) for k, v in kvs) + '}') 111 else: 112 return ('{%s' % indent_step[:-1] + 113 (',\n%s%s' % (indent, indent_step)).join( 114 ('"%s":\n%s%s%s' % (k, indent, indent_step, v) if (v[0] in {'[', '{'}) else 115 '"%s": %s' % (k, v)) for k, v in kvs) + 116 '\n%s}' % indent) 117 118 119def list_to_json(lst, indent, indent_step, compact_output=False): 120 if compact_output: 121 return ('[' + ', '.join(l for l in lst) + ']') 122 else: 123 return ('[%s' % indent_step[:-1] + 124 ((',\n%s%s' % (indent, indent_step)).join( 125 ('\n%s%s%s' % (indent, indent_step, l) if (i == 0 and l[0] in {'[', '{'}) else l) 126 for i, l in enumerate(lst)) 127 ) + 128 '\n%s]' % indent) 129 130 131##### Main 'struct' writers ##### 132 133def gen_fake_addresses(args, blend): 134 if args.use_fake_address: 135 hashes = set() 136 ret = {} 137 for block in blend.blocks: 138 if not block.addr_old: 139 continue 140 hsh = block.get_data_hash() 141 while hsh in hashes: 142 hsh += 1 143 hashes.add(hsh) 144 ret[block.addr_old] = hsh 145 return ret 146 147 return {} 148 149 150def bheader_to_json(args, fw, blend, indent, indent_step): 151 fw('%s"%s": [\n' % (indent, "HEADER")) 152 indent = indent + indent_step 153 154 keyval = ( 155 ("magic", json_dumps(blend.header.magic)), 156 ("pointer_size", json_dumps(blend.header.pointer_size)), 157 ("is_little_endian", json_dumps(blend.header.is_little_endian)), 158 ("version", json_dumps(blend.header.version)), 159 ) 160 keyval = keyval_to_json(keyval, indent, indent_step) 161 fw('%s%s' % (indent, keyval)) 162 163 indent = indent[:-len(indent_step)] 164 fw('\n%s]' % indent) 165 166 167def do_bblock_filter(filters, blend, block, meta_keyval, data_keyval): 168 def do_bblock_filter_data_recursive(blend, block, rec_lvl, rec_iter, key=None): 169 fields = (blend.structs[block.sdna_index].fields if key is None else 170 [blend.structs[block.sdna_index].field_from_name.get(key[1:-1].encode())]) 171 for fld in fields: 172 if fld is None: 173 continue 174 if fld.dna_name.is_pointer: 175 paths = ([(fld.dna_name.name_only, i) for i in range(fld.dna_name.array_size)] 176 if fld.dna_name.array_size > 1 else [fld.dna_name.name_only]) 177 for p in paths: 178 child_block = block.get_pointer(p) 179 if child_block is not None: 180 child_block.user_data = max(block.user_data, rec_iter) 181 if rec_lvl != 0: 182 do_bblock_filter_data_recursive(blend, child_block, rec_lvl - 1, rec_iter + 1) 183 184 has_include = False 185 do_break = False 186 rec_iter = 1 187 if block.user_data is None: 188 block.user_data = 0 189 for include, rec_lvl, key, val in filters: 190 if rec_lvl < 0: 191 rec_lvl = 100 192 has_include = has_include or include 193 # Skip exclude filters if block was already processed some way. 194 if not include and block.user_data is not None: 195 continue 196 has_match = False 197 for k, v in meta_keyval: 198 if key.search(k) and val.search(v): 199 has_match = True 200 if include: 201 block.user_data = max(block.user_data, rec_iter) 202 # Note that in include cases, we have to keep checking filters, since some 'include recursive' 203 # ones may still have to be processed... 204 else: 205 block.user_data = min(block.user_data, -rec_iter) 206 do_break = True # No need to check more filters in exclude case... 207 break 208 for k, v in data_keyval: 209 if key.search(k) and val.search(v): 210 has_match = True 211 if include: 212 block.user_data = max(block.user_data, rec_iter) 213 if rec_lvl != 0: 214 do_bblock_filter_data_recursive(blend, block, rec_lvl - 1, rec_iter + 1, k) 215 # Note that in include cases, we have to keep checking filters, since some 'include recursive' 216 # ones may still have to be processed... 217 else: 218 block.user_data = min(block.user_data, -rec_iter) 219 do_break = True # No need to check more filters in exclude case... 220 break 221 if include and not has_match: # Include check failed, implies exclusion. 222 block.user_data = min(block.user_data, -rec_iter) 223 do_break = True # No need to check more filters in exclude case... 224 if do_break: 225 break 226 # Implicit 'include all' in case no include filter is specified... 227 if block.user_data == 0 and not has_include: 228 block.user_data = max(block.user_data, rec_iter) 229 230 231def bblocks_to_json(args, fw, blend, address_map, indent, indent_step): 232 no_address = args.no_address 233 full_data = args.full_data 234 filter_data = args.filter_data 235 236 def gen_meta_keyval(blend, block): 237 keyval = [ 238 ("code", json_dumps(block.code)), 239 ("size", json_dumps(block.size)), 240 ] 241 if not no_address: 242 keyval += [("addr_old", json_dumps(address_map.get(block.addr_old, block.addr_old)))] 243 keyval += [ 244 ("dna_type_id", json_dumps(blend.structs[block.sdna_index].dna_type_id)), 245 ("count", json_dumps(block.count)), 246 ] 247 return keyval 248 249 def gen_data_keyval(blend, block, key_filter=None): 250 def _is_pointer(k): 251 return blend.structs[block.sdna_index].field_from_path(blend.header, blend.handle, k).dna_name.is_pointer 252 if key_filter is not None: 253 return [(json_dumps(k)[1:-1], json_dumps(address_map.get(v, v) if _is_pointer(k) else v)) 254 for k, v in block.items_recursive_iter() if k in key_filter] 255 return [(json_dumps(k)[1:-1], json_dumps(address_map.get(v, v) if _is_pointer(k) else v)) 256 for k, v in block.items_recursive_iter()] 257 258 if args.block_filters: 259 for block in blend.blocks: 260 meta_keyval = gen_meta_keyval(blend, block) 261 data_keyval = gen_data_keyval(blend, block) 262 do_bblock_filter(args.block_filters, blend, block, meta_keyval, data_keyval) 263 264 fw('%s"%s": [\n' % (indent, "DATA")) 265 indent = indent + indent_step 266 267 is_first = True 268 for i, block in enumerate(blend.blocks): 269 if block.user_data is None or block.user_data > 0: 270 meta_keyval = gen_meta_keyval(blend, block) 271 if full_data: 272 meta_keyval.append(("data", keyval_to_json(gen_data_keyval(blend, block), 273 indent + indent_step, indent_step, args.compact_output))) 274 elif filter_data: 275 meta_keyval.append(("data", keyval_to_json(gen_data_keyval(blend, block, filter_data), 276 indent + indent_step, indent_step, args.compact_output))) 277 keyval = keyval_to_json(meta_keyval, indent, indent_step, args.compact_output) 278 fw('%s%s%s' % ('' if is_first else ',\n', indent, keyval)) 279 is_first = False 280 281 indent = indent[:-len(indent_step)] 282 fw('\n%s]' % indent) 283 284 285def bdna_to_json(args, fw, blend, indent, indent_step): 286 full_dna = args.full_dna and not args.compact_output 287 288 def bdna_fields_to_json(blend, dna, indent, indent_step): 289 lst = [] 290 for i, field in enumerate(dna.fields): 291 keyval = ( 292 ("dna_name", json_dumps(field.dna_name.name_only)), 293 ("dna_type_id", json_dumps(field.dna_type.dna_type_id)), 294 ("is_pointer", json_dumps(field.dna_name.is_pointer)), 295 ("is_method_pointer", json_dumps(field.dna_name.is_method_pointer)), 296 ("array_size", json_dumps(field.dna_name.array_size)), 297 ) 298 lst.append(keyval_to_json(keyval, indent + indent_step, indent_step)) 299 return list_to_json(lst, indent, indent_step) 300 301 fw('%s"%s": [\n' % (indent, "DNA_STRUCT")) 302 indent = indent + indent_step 303 304 is_first = True 305 for dna in blend.structs: 306 keyval = [ 307 ("dna_type_id", json_dumps(dna.dna_type_id)), 308 ("size", json_dumps(dna.size)), 309 ] 310 if full_dna: 311 keyval += [("fields", bdna_fields_to_json(blend, dna, indent + indent_step, indent_step))] 312 else: 313 keyval += [("nbr_fields", json_dumps(len(dna.fields)))] 314 keyval = keyval_to_json(keyval, indent, indent_step, args.compact_output) 315 fw('%s%s%s' % ('' if is_first else ',\n', indent, keyval)) 316 is_first = False 317 318 indent = indent[:-len(indent_step)] 319 fw('\n%s]' % indent) 320 321 322def blend_to_json(args, f, blend, address_map): 323 fw = f.write 324 fw('{\n') 325 indent = indent_step = " " 326 bheader_to_json(args, fw, blend, indent, indent_step) 327 fw(',\n') 328 bblocks_to_json(args, fw, blend, address_map, indent, indent_step) 329 fw(',\n') 330 bdna_to_json(args, fw, blend, indent, indent_step) 331 fw('\n}\n') 332 333 334##### Checks ##### 335 336def check_file(args, blend): 337 addr_old = set() 338 for block in blend.blocks: 339 if block.addr_old in addr_old: 340 print("ERROR! Several data blocks share same 'addr_old' uuid %d, " 341 "this should never happen!" % block.addr_old) 342 continue 343 addr_old.add(block.addr_old) 344 345 346##### Main ##### 347 348def argparse_create(): 349 import argparse 350 global __doc__ 351 352 # When --help or no args are given, print this help 353 usage_text = __doc__ 354 355 epilog = "This script is typically used to check differences between .blend files, or to check their validity." 356 357 parser = argparse.ArgumentParser(description=usage_text, epilog=epilog, 358 formatter_class=argparse.RawDescriptionHelpFormatter) 359 360 parser.add_argument( 361 dest="input", nargs="+", metavar='PATH', 362 help="Input .blend file(s)") 363 parser.add_argument( 364 "-o", "--output", dest="output", action="append", metavar='PATH', required=False, 365 help="Output .json file(s) (same path/name as input file(s) if not specified)") 366 parser.add_argument( 367 "-c", "--check-file", dest="check_file", default=False, action='store_true', required=False, 368 help=("Perform some basic validation checks over the .blend file")) 369 parser.add_argument( 370 "--compact-output", dest="compact_output", default=False, action='store_true', required=False, 371 help=("Output a very compact representation of blendfile (one line per block/DNAStruct)")) 372 parser.add_argument( 373 "--no-old-addresses", dest="no_address", default=False, action='store_true', required=False, 374 help=("Do not output old memory address of each block of data " 375 "(used as 'uuid' in .blend files, but change pretty noisily)")) 376 parser.add_argument( 377 "--no-fake-old-addresses", dest="use_fake_address", default=True, action='store_false', 378 required=False, 379 help=("Do not 'rewrite' old memory address of each block of data " 380 "(they are rewritten by default to some hash of their content, " 381 "to try to avoid too much diff noise between different but similar files)")) 382 parser.add_argument( 383 "--full-data", dest="full_data", 384 default=False, action='store_true', required=False, 385 help=("Also put in JSon file data itself " 386 "(WARNING! will generate *huge* verbose files - and is far from complete yet)")) 387 parser.add_argument( 388 "--filter-data", dest="filter_data", 389 default=None, required=False, 390 help=("Only put in JSon file data fields which names match given comma-separated list " 391 "(ignored if --full-data is set)")) 392 parser.add_argument( 393 "--full-dna", dest="full_dna", default=False, action='store_true', required=False, 394 help=("Also put in JSon file dna properties description (ignored when --compact-output is used)")) 395 396 group = parser.add_argument_group("Filters", FILTER_DOC) 397 group.add_argument( 398 "--filter-block", dest="block_filters", nargs=3, action='append', 399 help=("Filter to apply to BLOCKS (a.k.a. data itself)")) 400 401 return parser 402 403 404def main(): 405 # ---------- 406 # Parse Args 407 408 args = argparse_create().parse_args() 409 410 if not args.output: 411 if args.check_file: 412 args.output = [None] * len(args.input) 413 else: 414 args.output = [os.path.splitext(infile)[0] + ".json" for infile in args.input] 415 416 if args.block_filters: 417 args.block_filters = [(True if m[0] == "+" else False, 418 0 if len(m) == 1 else (-1 if m[1] == "*" else int(m[1:])), 419 re.compile(f), re.compile(d)) 420 for m, f, d in args.block_filters] 421 422 if args.filter_data: 423 if args.full_data: 424 args.filter_data = None 425 else: 426 args.filter_data = {n.encode() for n in args.filter_data.split(',')} 427 428 for infile, outfile in zip(args.input, args.output): 429 with blendfile.open_blend(infile) as blend: 430 address_map = gen_fake_addresses(args, blend) 431 432 if args.check_file: 433 check_file(args, blend) 434 435 if outfile: 436 with open(outfile, 'w', encoding="ascii", errors='xmlcharrefreplace') as f: 437 blend_to_json(args, f, blend, address_map) 438 439 440if __name__ == "__main__": 441 main() 442