1 2from gzip import GzipFile 3from io import BytesIO 4from json import loads as json_loads 5from os import fsync 6from sys import exc_info, version 7from .utils import NoNumpyException # keep 'unused' imports 8from .comment import strip_comment_line_with_symbol, strip_comments # keep 'unused' imports 9from .encoders import TricksEncoder, json_date_time_encode, class_instance_encode, ClassInstanceEncoder, \ 10 json_complex_encode, json_set_encode, numeric_types_encode, numpy_encode, nonumpy_encode, NoNumpyEncoder, \ 11 nopandas_encode, pandas_encode # keep 'unused' imports 12from .decoders import DuplicateJsonKeyException, TricksPairHook, json_date_time_hook, ClassInstanceHook, \ 13 json_complex_hook, json_set_hook, numeric_types_hook, json_numpy_obj_hook, json_nonumpy_obj_hook, \ 14 nopandas_hook, pandas_hook # keep 'unused' imports 15from json import JSONEncoder 16 17 18is_py3 = (version[:2] == '3.') 19str_type = str if is_py3 else (basestring, unicode,) 20ENCODING = 'UTF-8' 21 22 23_cih_instance = ClassInstanceHook() 24DEFAULT_ENCODERS = [json_date_time_encode, class_instance_encode, json_complex_encode, json_set_encode, numeric_types_encode,] 25DEFAULT_HOOKS = [json_date_time_hook, _cih_instance, json_complex_hook, json_set_hook, numeric_types_hook,] 26 27try: 28 import numpy 29except ImportError: 30 DEFAULT_ENCODERS = [nonumpy_encode,] + DEFAULT_ENCODERS 31 DEFAULT_HOOKS = [json_nonumpy_obj_hook,] + DEFAULT_HOOKS 32else: 33 # numpy encode needs to be before complex 34 DEFAULT_ENCODERS = [numpy_encode,] + DEFAULT_ENCODERS 35 DEFAULT_HOOKS = [json_numpy_obj_hook,] + DEFAULT_HOOKS 36 37try: 38 import pandas 39except ImportError: 40 DEFAULT_ENCODERS = [nopandas_encode,] + DEFAULT_ENCODERS 41 DEFAULT_HOOKS = [nopandas_hook,] + DEFAULT_HOOKS 42else: 43 DEFAULT_ENCODERS = [pandas_encode,] + DEFAULT_ENCODERS 44 DEFAULT_HOOKS = [pandas_hook,] + DEFAULT_HOOKS 45 46 47DEFAULT_NONP_ENCODERS = [nonumpy_encode,] + DEFAULT_ENCODERS # DEPRECATED 48DEFAULT_NONP_HOOKS = [json_nonumpy_obj_hook,] + DEFAULT_HOOKS # DEPRECATED 49 50 51def dumps(obj, sort_keys=None, cls=TricksEncoder, obj_encoders=DEFAULT_ENCODERS, extra_obj_encoders=(), 52 primitives=False, compression=None, allow_nan=False, conv_str_byte=False, **jsonkwargs): 53 """ 54 Convert a nested data structure to a json string. 55 56 :param obj: The Python object to convert. 57 :param sort_keys: Keep this False if you want order to be preserved. 58 :param cls: The json encoder class to use, defaults to NoNumpyEncoder which gives a warning for numpy arrays. 59 :param obj_encoders: Iterable of encoders to use to convert arbitrary objects into json-able promitives. 60 :param extra_obj_encoders: Like `obj_encoders` but on top of them: use this to add encoders without replacing defaults. Since v3.5 these happen before default encoders. 61 :param allow_nan: Allow NaN and Infinity values, which is a (useful) violation of the JSON standard (default False). 62 :param conv_str_byte: Try to automatically convert between strings and bytes (assuming utf-8) (default False). 63 :return: The string containing the json-encoded version of obj. 64 65 Other arguments are passed on to `cls`. Note that `sort_keys` should be false if you want to preserve order. 66 """ 67 if not hasattr(extra_obj_encoders, '__iter__'): 68 raise TypeError('`extra_obj_encoders` should be a tuple in `json_tricks.dump(s)`') 69 encoders = tuple(extra_obj_encoders) + tuple(obj_encoders) 70 txt = cls(sort_keys=sort_keys, obj_encoders=encoders, allow_nan=allow_nan, 71 primitives=primitives, **jsonkwargs).encode(obj) 72 if not is_py3 and isinstance(txt, str): 73 txt = unicode(txt, ENCODING) 74 if not compression: 75 return txt 76 if compression is True: 77 compression = 5 78 txt = txt.encode(ENCODING) 79 sh = BytesIO() 80 with GzipFile(mode='wb', fileobj=sh, compresslevel=compression) as zh: 81 zh.write(txt) 82 gzstring = sh.getvalue() 83 return gzstring 84 85 86def dump(obj, fp, sort_keys=None, cls=TricksEncoder, obj_encoders=DEFAULT_ENCODERS, extra_obj_encoders=(), 87 primitives=False, compression=None, force_flush=False, allow_nan=False, conv_str_byte=False, **jsonkwargs): 88 """ 89 Convert a nested data structure to a json string. 90 91 :param fp: File handle or path to write to. 92 :param compression: The gzip compression level, or None for no compression. 93 :param force_flush: If True, flush the file handle used, when possibly also in the operating system (default False). 94 95 The other arguments are identical to `dumps`. 96 """ 97 txt = dumps(obj, sort_keys=sort_keys, cls=cls, obj_encoders=obj_encoders, extra_obj_encoders=extra_obj_encoders, 98 primitives=primitives, compression=compression, allow_nan=allow_nan, conv_str_byte=conv_str_byte, **jsonkwargs) 99 if isinstance(fp, str_type): 100 fh = open(fp, 'wb+') 101 else: 102 fh = fp 103 if conv_str_byte: 104 try: 105 fh.write(b'') 106 except TypeError: 107 pass 108 # if not isinstance(txt, str_type): 109 # # Cannot write bytes, so must be in text mode, but we didn't get a text 110 # if not compression: 111 # txt = txt.decode(ENCODING) 112 else: 113 try: 114 fh.write(u'') 115 except TypeError: 116 if isinstance(txt, str_type): 117 txt = txt.encode(ENCODING) 118 try: 119 if 'b' not in getattr(fh, 'mode', 'b?') and not isinstance(txt, str_type) and compression: 120 raise IOError('If compression is enabled, the file must be opened in binary mode.') 121 try: 122 fh.write(txt) 123 except TypeError as err: 124 err.args = (err.args[0] + '. A possible reason is that the file is not opened in binary mode; ' 125 'be sure to set file mode to something like "wb".',) 126 raise 127 finally: 128 if force_flush: 129 fh.flush() 130 try: 131 if fh.fileno() is not None: 132 fsync(fh.fileno()) 133 except (ValueError,): 134 pass 135 if isinstance(fp, str_type): 136 fh.close() 137 return txt 138 139 140def loads(string, preserve_order=True, ignore_comments=True, decompression=None, obj_pairs_hooks=DEFAULT_HOOKS, 141 extra_obj_pairs_hooks=(), cls_lookup_map=None, allow_duplicates=True, conv_str_byte=False, **jsonkwargs): 142 """ 143 Convert a nested data structure to a json string. 144 145 :param string: The string containing a json encoded data structure. 146 :param decode_cls_instances: True to attempt to decode class instances (requires the environment to be similar the the encoding one). 147 :param preserve_order: Whether to preserve order by using OrderedDicts or not. 148 :param ignore_comments: Remove comments (starting with # or //). 149 :param decompression: True to use gzip decompression, False to use raw data, None to automatically determine (default). Assumes utf-8 encoding! 150 :param obj_pairs_hooks: A list of dictionary hooks to apply. 151 :param extra_obj_pairs_hooks: Like `obj_pairs_hooks` but on top of them: use this to add hooks without replacing defaults. Since v3.5 these happen before default hooks. 152 :param cls_lookup_map: If set to a dict, for example ``globals()``, then classes encoded from __main__ are looked up this dict. 153 :param allow_duplicates: If set to False, an error will be raised when loading a json-map that contains duplicate keys. 154 :param parse_float: A function to parse strings to integers (e.g. Decimal). There is also `parse_int`. 155 :param conv_str_byte: Try to automatically convert between strings and bytes (assuming utf-8) (default False). 156 :return: The string containing the json-encoded version of obj. 157 158 Other arguments are passed on to json_func. 159 """ 160 if not hasattr(extra_obj_pairs_hooks, '__iter__'): 161 raise TypeError('`extra_obj_pairs_hooks` should be a tuple in `json_tricks.load(s)`') 162 if decompression is None: 163 decompression = string[:2] == b'\x1f\x8b' 164 if decompression: 165 with GzipFile(fileobj=BytesIO(string), mode='rb') as zh: 166 string = zh.read() 167 string = string.decode(ENCODING) 168 if not isinstance(string, str_type): 169 if conv_str_byte: 170 string = string.decode(ENCODING) 171 else: 172 raise TypeError(('Cannot automatically encode object of type "{0:}" in `json_tricks.load(s)` since ' 173 'the encoding is not known. You should instead encode the bytes to a string and pass that ' 174 'string to `load(s)`, for example bytevar.encode("utf-8") if utf-8 is the encoding.').format(type(string))) 175 if ignore_comments: 176 string = strip_comments(string) 177 obj_pairs_hooks = tuple(obj_pairs_hooks) 178 _cih_instance.cls_lookup_map = cls_lookup_map or {} 179 hooks = tuple(extra_obj_pairs_hooks) + obj_pairs_hooks 180 hook = TricksPairHook(ordered=preserve_order, obj_pairs_hooks=hooks, allow_duplicates=allow_duplicates) 181 return json_loads(string, object_pairs_hook=hook, **jsonkwargs) 182 183 184def load(fp, preserve_order=True, ignore_comments=True, decompression=None, obj_pairs_hooks=DEFAULT_HOOKS, 185 extra_obj_pairs_hooks=(), cls_lookup_map=None, allow_duplicates=True, conv_str_byte=False, **jsonkwargs): 186 """ 187 Convert a nested data structure to a json string. 188 189 :param fp: File handle or path to load from. 190 191 The other arguments are identical to loads. 192 """ 193 try: 194 if isinstance(fp, str_type): 195 with open(fp, 'rb') as fh: 196 string = fh.read() 197 else: 198 string = fp.read() 199 except UnicodeDecodeError as err: 200 # todo: not covered in tests, is it relevant? 201 raise Exception('There was a problem decoding the file content. A possible reason is that the file is not ' + 202 'opened in binary mode; be sure to set file mode to something like "rb".').with_traceback(exc_info()[2]) 203 return loads(string, preserve_order=preserve_order, ignore_comments=ignore_comments, decompression=decompression, 204 obj_pairs_hooks=obj_pairs_hooks, extra_obj_pairs_hooks=extra_obj_pairs_hooks, cls_lookup_map=cls_lookup_map, 205 allow_duplicates=allow_duplicates, conv_str_byte=conv_str_byte, **jsonkwargs) 206 207 208