1# Copyright (c) 2019 Ultimaker B.V. 2# Uranium is released under the terms of the LGPLv3 or higher. 3 4import json 5import collections 6import copy 7 8from PyQt5.QtCore import QObject, pyqtProperty 9from PyQt5.QtQml import QQmlEngine 10 11from UM.i18n import i18nCatalog #For typing. 12from UM.Logger import Logger 13from UM.MimeTypeDatabase import MimeTypeDatabase, MimeType 14from UM.PluginObject import PluginObject 15from UM.Resources import Resources 16from UM.Settings.Interfaces import DefinitionContainerInterface 17from UM.Settings.PropertyEvaluationContext import PropertyEvaluationContext 18from UM.Settings.SettingDefinition import SettingDefinition 19from UM.Settings.SettingDefinition import DefinitionPropertyType 20from UM.Settings.SettingRelation import SettingRelation 21from UM.Settings.SettingRelation import RelationType 22from UM.Settings.SettingFunction import SettingFunction 23from UM.Signal import Signal 24 25from typing import Dict, Any, List, Optional, Set, Tuple 26 27class InvalidDefinitionError(Exception): 28 pass 29 30 31class IncorrectDefinitionVersionError(Exception): 32 pass 33 34 35class InvalidOverrideError(Exception): 36 pass 37 38MimeTypeDatabase.addMimeType( 39 MimeType( 40 name = "application/x-uranium-definitioncontainer", 41 comment = "Uranium Definition Container", 42 suffixes = ["def.json"] 43 ) 44) 45 46 47class DefinitionContainer(QObject, DefinitionContainerInterface, PluginObject): 48 """A container for SettingDefinition objects.""" 49 50 Version = 2 51 52 def __init__(self, container_id: str, i18n_catalog: i18nCatalog = None, parent: QObject = None, *args, **kwargs) -> None: 53 """Constructor 54 55 :param container_id: A unique, machine readable/writable ID for this container. 56 """ 57 58 super().__init__() 59 QQmlEngine.setObjectOwnership(self, QQmlEngine.CppOwnership) 60 61 self._metadata = {"id": container_id, 62 "name": container_id, 63 "container_type": DefinitionContainer, 64 "version": self.Version} # type: Dict[str, Any] 65 self._definitions = [] # type: List[SettingDefinition] 66 self._inherited_files = [] # type: List[str] 67 self._i18n_catalog = i18n_catalog # type: Optional[i18nCatalog] 68 69 self._definition_cache = {} # type: Dict[str, SettingDefinition] 70 self._path = "" 71 72 def __setattr__(self, name: str, value: Any) -> None: 73 """Reimplement __setattr__ so we can make sure the definition remains unchanged after creation.""" 74 75 super().__setattr__(name, value) 76 #raise NotImplementedError() 77 78 def __getnewargs__(self) -> Tuple[str, Optional[i18nCatalog]]: 79 """For pickle support""" 80 81 return (self.getId(), self._i18n_catalog) 82 83 def __getstate__(self) -> Dict[str, Any]: 84 """For pickle support""" 85 86 return self.__dict__ 87 88 def __setstate__(self, state: Dict[str, Any]) -> None: 89 """For pickle support""" 90 91 # We need to call QObject.__init__() in order to initialize the underlying C++ object. 92 # pickle doesn't do that so we have to do this here. 93 QObject.__init__(self, parent = None) 94 self.__dict__.update(state) 95 96 def getId(self) -> str: 97 """:copydoc ContainerInterface::getId 98 99 Reimplemented from ContainerInterface 100 """ 101 102 return self._metadata["id"] 103 104 id = pyqtProperty(str, fget = getId, constant = True) 105 106 def getName(self) -> str: 107 """:copydoc ContainerInterface::getName 108 109 Reimplemented from ContainerInterface 110 """ 111 112 return self._metadata["name"] 113 114 name = pyqtProperty(str, fget = getName, constant = True) 115 116 def isReadOnly(self) -> bool: 117 """:copydoc ContainerInterface::isReadOnly 118 119 Reimplemented from ContainerInterface 120 """ 121 122 return True 123 124 def setReadOnly(self, read_only: bool) -> None: 125 pass 126 127 readOnly = pyqtProperty(bool, fget = isReadOnly, constant = True) 128 129 def getPath(self) -> str: 130 """:copydoc ContainerInterface::getPath. 131 132 Reimplemented from ContainerInterface 133 """ 134 135 return self._path 136 137 def setPath(self, path: str) -> None: 138 """:copydoc ContainerInterface::setPath 139 140 Reimplemented from ContainerInterface 141 """ 142 143 self._path = path 144 145 def getMetaData(self) -> Dict[str, Any]: 146 """:copydoc ContainerInterface::getMetaData 147 148 Reimplemented from ContainerInterface 149 """ 150 151 return self._metadata 152 153 metaData = pyqtProperty("QVariantMap", fget = getMetaData, constant = True) 154 155 @property 156 def definitions(self) -> List[SettingDefinition]: 157 return self._definitions 158 159 def getInheritedFiles(self) -> List[str]: 160 """Gets all ancestors of this definition container. 161 162 This returns the definition in the "inherits" property of this 163 container, and the definition in its "inherits" property, and so on. The 164 ancestors are returned in order from parent to 165 grand-grand-grand-...-grandparent, normally ending in a "root" 166 container. 167 168 :return: A list of ancestors, in order from near ancestor to the root. 169 """ 170 171 return self._inherited_files 172 173 def getAllKeys(self) -> Set[str]: 174 """:copydoc DefinitionContainerInterface::getAllKeys 175 176 :return: A set of all keys of settings in this container. 177 """ 178 179 keys = set() # type: Set[str] 180 for definition in self.definitions: 181 keys |= definition.getAllKeys() 182 return keys 183 184 def getMetaDataEntry(self, entry: str, default: Any = None) -> Any: 185 """:copydoc ContainerInterface::getMetaDataEntry 186 187 Reimplemented from ContainerInterface 188 """ 189 190 return self._metadata.get(entry, default) 191 192 def getProperty(self, key: str, property_name: str, context: PropertyEvaluationContext = None) -> Any: 193 """:copydoc ContainerInterface::getProperty 194 195 Reimplemented from ContainerInterface. 196 """ 197 198 definition = self._getDefinition(key) 199 if not definition: 200 return None 201 202 try: 203 value = getattr(definition, property_name) 204 if value is None and property_name == "value": 205 value = getattr(definition, "default_value") 206 return value 207 except AttributeError: 208 return None 209 210 def hasProperty(self, key: str, property_name: str, ignore_inherited: bool = False) -> Any: 211 """:copydoc ContainerInterface::hasProperty 212 213 Reimplemented from ContainerInterface 214 """ 215 216 definition = self._getDefinition(key) 217 if not definition: 218 return False 219 if definition.parent is not None and ignore_inherited: 220 return False 221 return hasattr(definition, property_name) 222 223 propertyChanged = Signal() 224 """This signal is unused since the definition container is immutable, but is provided for API consistency.""" 225 226 metaDataChanged = Signal() 227 228 def serialize(self, ignored_metadata_keys: Optional[Set[str]] = None) -> str: 229 """:copydoc ContainerInterface::serialize 230 231 TODO: This implementation flattens the definition container, since the 232 data about inheritance and overrides was lost when deserialising. 233 234 Reimplemented from ContainerInterface 235 """ 236 237 data = {} # type: Dict[str, Any] # The data to write to a JSON file. 238 data["name"] = self.getName() 239 data["version"] = DefinitionContainer.Version 240 data["metadata"] = self.getMetaData().copy() 241 242 # Remove the keys that we want to ignore in the metadata 243 if not ignored_metadata_keys: 244 ignored_metadata_keys = set() 245 ignored_metadata_keys |= {"name", "version", "id", "container_type"} 246 for key in ignored_metadata_keys: 247 if key in data["metadata"]: 248 del data["metadata"][key] 249 250 data["settings"] = {} 251 for definition in self.definitions: 252 data["settings"][definition.key] = definition.serialize_to_dict() 253 254 return json.dumps(data, separators = (", ", ": "), indent = 4) # Pretty print the JSON. 255 256 @classmethod 257 def getConfigurationTypeFromSerialized(cls, serialized: str) -> Optional[str]: 258 configuration_type = None 259 try: 260 parsed = json.loads(serialized, object_pairs_hook = collections.OrderedDict) 261 configuration_type = parsed.get("metadata", {}).get("type", "machine") #TODO: Not all definitions have a type. They get this via inheritance but that requires an instance. 262 except InvalidDefinitionError as ide: 263 raise ide 264 except Exception as e: 265 Logger.log("d", "Could not get configuration type: %s", e) 266 return configuration_type 267 268 def readAndValidateSerialized(self, serialized: str) -> Tuple[Dict[str, Any], bool]: 269 parsed = json.loads(serialized, object_pairs_hook = collections.OrderedDict) 270 271 if "inherits" in parsed: 272 inherited = self._resolveInheritance(parsed["inherits"]) 273 parsed = self._mergeDicts(inherited, parsed) 274 275 self._verifyJson(parsed) 276 277 is_valid = self._preprocessParsedJson(parsed) 278 279 return parsed, is_valid 280 281 @classmethod 282 def getVersionFromSerialized(cls, serialized: str) -> Optional[int]: 283 version = None 284 parsed = json.loads(serialized, object_pairs_hook = collections.OrderedDict) 285 try: 286 version = int(parsed["version"]) 287 except Exception as e: 288 Logger.log("d", "Could not get version from serialized: %s", e) 289 return version 290 291 # Returns whether the parsed JSON is valid. 292 def _preprocessParsedJson(self, parsed: Dict[str, Any]) -> bool: 293 # Pre-process the JSON data to include the overrides. 294 is_valid = True 295 if "overrides" in parsed: 296 for key, value in parsed["overrides"].items(): 297 setting = self._findInDict(parsed["settings"], key) 298 if setting is None: 299 Logger.log("w", "Unable to override setting %s", key) 300 is_valid = False 301 else: 302 setting.update(value) 303 304 return is_valid 305 306 def addDefinition(self, definition: SettingDefinition) -> None: 307 """Add a setting definition instance if it doesn't exist yet. 308 309 Warning: this might not work when there are relationships higher up in the stack. 310 """ 311 312 if definition.key not in [d.key for d in self._definitions]: 313 self._definitions.append(definition) 314 self._definition_cache[definition.key] = definition 315 self._updateRelations(definition) 316 317 def deserialize(self, serialized: str, file_name: Optional[str] = None) -> str: 318 """:copydoc ContainerInterface::deserialize 319 320 Reimplemented from ContainerInterface 321 """ 322 323 # update the serialized data first 324 serialized = super().deserialize(serialized, file_name) 325 parsed, is_valid = self.readAndValidateSerialized(serialized) 326 327 # Update properties with the data from the JSON 328 old_id = self.getId() #The ID must be set via the constructor. Retain it. 329 self._metadata = parsed["metadata"] 330 self._metadata["id"] = old_id 331 self._metadata["name"] = parsed["name"] 332 self._metadata["version"] = self.Version #Guaranteed to be equal to what's in the parsed data by the validation. 333 self._metadata["container_type"] = DefinitionContainer 334 335 for key, value in parsed["settings"].items(): 336 definition = SettingDefinition(key, self, None, self._i18n_catalog) 337 self._definition_cache[key] = definition 338 definition.deserialize(value) 339 self._definitions.append(definition) 340 341 for definition in self._definitions: 342 self._updateRelations(definition) 343 344 return serialized 345 346 @classmethod 347 def deserializeMetadata(cls, serialized: str, container_id: str) -> List[Dict[str, Any]]: 348 """Gets the metadata of a definition container from a serialised format. 349 350 This parses the entire JSON document and only extracts the metadata from 351 it. 352 353 :param serialized: A JSON document, serialised as a string. 354 :param container_id: The ID of the container (as obtained from the file name). 355 356 :return: A dictionary of metadata that was in the JSON document in a 357 singleton list. If anything went wrong, the list will be empty. 358 """ 359 360 serialized = cls._updateSerialized(serialized) #Update to most recent version. 361 try: 362 parsed = json.loads(serialized, object_pairs_hook = collections.OrderedDict) #TODO: Load only part of this JSON until we find the metadata. We need an external library for this though. 363 except json.JSONDecodeError as e: 364 Logger.log("d", "Could not parse definition: %s", e) 365 return [] 366 metadata = {} #type: Dict[str, Any] 367 if "inherits" in parsed: 368 import UM.Settings.ContainerRegistry #To find the definitions we're inheriting from. 369 parent_metadata = UM.Settings.ContainerRegistry.ContainerRegistry.getInstance().findDefinitionContainersMetadata(id = parsed["inherits"]) 370 if not parent_metadata: 371 Logger.log("e", "Could not load parent definition container {parent} of child {child}".format(parent = parsed["inherits"], child = container_id)) 372 #Ignore the parent then. 373 else: 374 metadata.update(parent_metadata[0]) 375 metadata["inherits"] = parsed["inherits"] 376 377 metadata["container_type"] = DefinitionContainer 378 metadata["id"] = container_id 379 try: # Move required fields to metadata. 380 metadata["name"] = parsed["name"] 381 metadata["version"] = parsed["version"] 382 except KeyError as e: # Required fields not present! 383 raise InvalidDefinitionError("Missing required fields: {error_msg}".format(error_msg = str(e))) 384 if "metadata" in parsed: 385 metadata.update(parsed["metadata"]) 386 return [metadata] 387 388 def findDefinitions(self, **kwargs: Any) -> List[SettingDefinition]: 389 """Find definitions matching certain criteria. 390 391 :param kwargs: A dictionary of keyword arguments containing key-value pairs which should match properties of 392 the definition. 393 """ 394 395 if len(kwargs) == 1 and "key" in kwargs: 396 # If we are searching for a single definition by exact key, we can speed up things by retrieving from the cache. 397 key = kwargs["key"] 398 if key in self._definition_cache: 399 return [self._definition_cache[key]] 400 401 definitions = [] 402 for definition in self._definitions: 403 definitions.extend(definition.findDefinitions(**kwargs)) 404 405 if len(kwargs) == 1 and "key" in kwargs: 406 # Ensure that next time round, the definition is in the cache! 407 if definitions: 408 self._definition_cache[kwargs["key"]] = definitions[0] 409 410 return definitions 411 412 @classmethod 413 def getLoadingPriority(cls) -> int: 414 return 0 415 416 # protected: 417 418 # Load a file from disk, used to handle inheritance and includes 419 def _loadFile(self, file_name: str) -> Dict[str, Any]: 420 path = Resources.getPath(Resources.DefinitionContainers, file_name + ".def.json") 421 with open(path, encoding = "utf-8") as f: 422 contents = json.load(f, object_pairs_hook=collections.OrderedDict) 423 424 self._inherited_files.append(path) 425 return contents 426 427 # Recursively resolve loading inherited files 428 def _resolveInheritance(self, file_name: str) -> Dict[str, Any]: 429 json_dict = self._loadFile(file_name) 430 431 if "inherits" in json_dict: 432 inherited = self._resolveInheritance(json_dict["inherits"]) 433 json_dict = self._mergeDicts(inherited, json_dict) 434 435 self._verifyJson(json_dict) 436 437 return json_dict 438 439 # Verify that a loaded json matches our basic expectations. 440 def _verifyJson(self, json_dict: Dict[str, Any]): 441 required_fields = {"version", "name", "settings", "metadata"} 442 missing_fields = required_fields - json_dict.keys() 443 if missing_fields: 444 raise InvalidDefinitionError("Missing required properties: {properties}".format(properties = ", ".join(missing_fields))) 445 446 if json_dict["version"] != self.Version: 447 raise IncorrectDefinitionVersionError("Definition uses version {0} but expected version {1}".format(json_dict["version"], self.Version)) 448 449 # Recursively find a key in a dictionary 450 def _findInDict(self, dictionary: Dict[str, Any], key: str) -> Any: 451 if key in dictionary: 452 return dictionary[key] 453 for v in dictionary.values(): 454 if isinstance(v, dict): 455 item = self._findInDict(v, key) 456 if item is not None: 457 return item 458 459 # Recursively merge two dictionaries, returning a new dictionary 460 def _mergeDicts(self, first: Dict[Any, Any], second: Dict[Any, Any]) -> Dict[Any, Any]: 461 result = copy.deepcopy(first) 462 for key, value in second.items(): 463 if key in result: 464 if isinstance(value, dict): 465 result[key] = self._mergeDicts(result[key], value) 466 else: 467 result[key] = value 468 else: 469 result[key] = value 470 471 return result 472 473 # Recursively update relations of settings 474 def _updateRelations(self, definition: SettingDefinition) -> None: 475 for property_name in SettingDefinition.getPropertyNames(DefinitionPropertyType.Function): 476 self._processFunction(definition, property_name) 477 478 for child in definition.children: 479 self._updateRelations(child) 480 481 # Create relation objects for all settings used by a certain function 482 def _processFunction(self, definition: SettingDefinition, property_name: str) -> None: 483 try: 484 function = getattr(definition, property_name) 485 except AttributeError: 486 return 487 488 if not isinstance(function, SettingFunction): 489 return 490 491 for setting in function.getUsedSettingKeys(): 492 # Prevent circular relations between the same setting and the same property 493 # Note that the only property used by SettingFunction is the "value" property, which 494 # is why this is hard coded here. 495 if setting == definition.key and property_name == "value": 496 Logger.log("w", "Found circular relation for property 'value' between {0} and {1}", definition.key, setting) 497 continue 498 499 other = self._getDefinition(setting) 500 if not other: 501 other = SettingDefinition(setting) 502 503 relation = SettingRelation(definition, other, RelationType.RequiresTarget, property_name) 504 definition.relations.append(relation) 505 506 relation = SettingRelation(other, definition, RelationType.RequiredByTarget, property_name) 507 other.relations.append(relation) 508 509 def _getDefinition(self, key: str) -> Optional[SettingDefinition]: 510 definition = None 511 if key in self._definition_cache: 512 definition = self._definition_cache[key] 513 else: 514 definitions = self.findDefinitions(key = key) 515 if definitions: 516 definition = definitions[0] 517 self._definition_cache[key] = definition 518 519 return definition 520 521 def isDirty(self) -> bool: 522 return False 523 524 def __str__(self) -> str: 525 """Simple short string representation for debugging purposes.""" 526 return "<DefContainer '{definition_id}'>".format(definition_id = self.getId()) 527 528 def __repr__(self) -> str: 529 return str(self)