1# -*- coding: utf-8 -*- 2# Copyright 2009-2013, Peter A. Bigot 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may 5# not use this file except in compliance with the License. You may obtain a 6# copy of the License at: 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations 14# under the License. 15 16"""Classes and global objects related to archiving U{XML 17Namespaces<http://www.w3.org/TR/2006/REC-xml-names-20060816/index.html>}.""" 18 19import logging 20import os 21import os.path 22import pyxb 23import pyxb.utils.utility 24from pyxb.utils import six 25 26_log = logging.getLogger(__name__) 27 28PathEnvironmentVariable = 'PYXB_ARCHIVE_PATH' 29"""Environment variable from which default path to pre-loaded namespaces is 30read. The value should be a colon-separated list of absolute paths. The 31character C{&} at the start of a member of the list is replaced by the path to 32the directory where the %{pyxb} modules are found, including a trailing C{/}. 33For example, use C{&pyxb/bundles//} to enable search of any archive bundled 34with PyXB. 35 36@note: If you put a path separator between C{&} and the following path, this 37will cause the substitution to be ignored.""" 38 39DefaultArchivePrefix = os.path.realpath(os.path.join(os.path.dirname( __file__), '../..')) 40"""The default archive prefix, substituted for C{&} in C{PYXB_ARCHIVE_PATH}.""" 41 42def GetArchivePath (): 43 """Return the archive path as defined by the L{PathEnvironmentVariable}, 44 or C{None} if that variable is not defined.""" 45 return os.environ.get(PathEnvironmentVariable) 46 47# Stuff required for pickling 48from pyxb.utils.six.moves import cPickle as pickle 49import re 50 51class NamespaceArchive (object): 52 """Represent a file from which one or more namespaces can be read, or to 53 which they will be written.""" 54 55 # A code used to identify the format of the archive, so we don't 56 # mis-interpret its contents. 57 # YYYYMMDDHHMM 58 __PickleFormat = '200907190858' 59 60 @classmethod 61 def _AnonymousCategory (cls): 62 """The category name to use when storing references to anonymous type 63 definitions. For example, attribute definitions defined within an 64 attribute use in a model group definition.that can be referenced frojm 65 ax different namespace.""" 66 return cls.__AnonymousCategory 67 __AnonymousCategory = '_anonymousTypeDefinition' 68 69 @classmethod 70 def PicklingArchive (cls): 71 """Return a reference to a set specifying the namespace instances that 72 are being archived. 73 74 This is needed to determine whether a component must be serialized as 75 aa reference.""" 76 # NB: Use root class explicitly. If we use cls, when this is invoked 77 # by subclasses it gets mangled using the subclass name so the one 78 # defined in this class is not found 79 return NamespaceArchive.__PicklingArchive 80 # Class variable recording the namespace that is currently being 81 # pickled. Used to prevent storing components that belong to 82 # other namespaces. Should be None unless within an invocation of 83 # SaveToFile. 84 __PicklingArchive = None 85 86 __NamespaceArchives = None 87 """A mapping from generation UID to NamespaceArchive instances.""" 88 89 def discard (self): 90 """Remove this archive from the set of available archives. 91 92 This is invoked when an archive contains a namespace that the user has 93 specified should not be loaded.""" 94 del self.__NamespaceArchives[self.generationUID()] 95 for ns in self.__namespaces: 96 ns._removeArchive(self) 97 98 @classmethod 99 def __GetArchiveInstance (cls, archive_file, stage=None): 100 """Return a L{NamespaceArchive} instance associated with the given file. 101 102 To the extent possible, the same file accessed through different paths 103 returns the same L{NamespaceArchive} instance. 104 """ 105 106 nsa = NamespaceArchive(archive_path=archive_file, stage=cls._STAGE_uid) 107 rv = cls.__NamespaceArchives.get(nsa.generationUID(), nsa) 108 if rv == nsa: 109 cls.__NamespaceArchives[rv.generationUID()] = rv 110 rv._readToStage(stage) 111 return rv 112 113 __ArchivePattern_re = re.compile('\.wxs$') 114 115 @classmethod 116 def PreLoadArchives (cls, archive_path=None, reset=False): 117 """Scan for available archives, associating them with namespaces. 118 119 This only validates potential archive contents; it does not load 120 namespace data from the archives. 121 122 @keyword archive_path: A list of files or directories in which 123 namespace archives can be found. The entries are separated by 124 os.pathsep, which is a colon on POSIX platforms and a semi-colon on 125 Windows. See L{PathEnvironmentVariable}. Defaults to 126 L{GetArchivePath()}. If not defaulted, C{reset} will be forced to 127 C{True}. For any directory in the path, all files ending with 128 C{.wxs} are examined. 129 130 @keyword reset: If C{False} (default), the most recently read set of 131 archives is returned; if C{True}, the archive path is re-scanned and the 132 namespace associations validated. 133 """ 134 135 from pyxb.namespace import builtin 136 137 reset = reset or (archive_path is not None) or (cls.__NamespaceArchives is None) 138 if reset: 139 # Get a list of pre-existing archives, initializing the map if 140 # this is the first time through. 141 if cls.__NamespaceArchives is None: 142 cls.__NamespaceArchives = { } 143 existing_archives = set(six.itervalues(cls.__NamespaceArchives)) 144 archive_set = set() 145 146 # Ensure we have an archive path. If not, don't do anything. 147 if archive_path is None: 148 archive_path = GetArchivePath() 149 if archive_path is not None: 150 151 # Get archive instances for everything in the archive path 152 candidate_files = pyxb.utils.utility.GetMatchingFiles(archive_path, cls.__ArchivePattern_re, 153 default_path_wildcard='+', default_path=GetArchivePath(), 154 prefix_pattern='&', prefix_substituend=DefaultArchivePrefix) 155 for afn in candidate_files: 156 try: 157 nsa = cls.__GetArchiveInstance(afn, stage=cls._STAGE_readModules) 158 archive_set.add(nsa) 159 except pickle.UnpicklingError: 160 _log.exception('Cannot unpickle archive %s', afn) 161 except pyxb.NamespaceArchiveError: 162 _log.exception('Cannot process archive %s', afn) 163 164 # Do this for two reasons: first, to get an iterable that won't 165 # cause problems when we remove unresolvable archives from 166 # archive_set; and second to aid with forced dependency inversion 167 # testing 168 ordered_archives = sorted(list(archive_set), key=lambda _a: _a.archivePath()) 169 ordered_archives.reverse() 170 171 # Create a graph that identifies dependencies between the archives 172 archive_map = { } 173 for a in archive_set: 174 archive_map[a.generationUID()] = a 175 archive_graph = pyxb.utils.utility.Graph() 176 for a in ordered_archives: 177 prereqs = a._unsatisfiedModulePrerequisites() 178 if 0 < len(prereqs): 179 for p in prereqs: 180 if builtin.BuiltInObjectUID == p: 181 continue 182 da = archive_map.get(p) 183 if da is None: 184 _log.warning('%s depends on unavailable archive %s', a, p) 185 archive_set.remove(a) 186 else: 187 archive_graph.addEdge(a, da) 188 else: 189 archive_graph.addRoot(a) 190 191 # Verify that there are no dependency loops. 192 archive_scc = archive_graph.sccOrder() 193 for scc in archive_scc: 194 if 1 < len(scc): 195 raise pyxb.LogicError("Cycle in archive dependencies. How'd you do that?\n " + "\n ".join([ _a.archivePath() for _a in scc ])) 196 archive = scc[0] 197 if not (archive in archive_set): 198 archive.discard() 199 existing_archives.remove(archive) 200 continue 201 #archive._readToStage(cls._STAGE_COMPLETE) 202 203 # Discard any archives that we used to know about but now aren't 204 # supposed to. @todo make this friendlier in the case of archives 205 # we've already incorporated. 206 for archive in existing_archives.difference(archive_set): 207 _log.info('Discarding excluded archive %s', archive) 208 archive.discard() 209 210 def archivePath (self): 211 """Path to the file in which this namespace archive is stored.""" 212 return self.__archivePath 213 __archivePath = None 214 215 def generationUID (self): 216 """The unique identifier for the generation that produced this archive.""" 217 return self.__generationUID 218 __generationUID = None 219 220 def isLoadable (self): 221 """Return C{True} iff it is permissible to load the archive. 222 Archives created for output cannot be loaded.""" 223 return self.__isLoadable 224 __isLoadable = None 225 226 def __locateModuleRecords (self): 227 self.__moduleRecords = set() 228 namespaces = set() 229 for ns in pyxb.namespace.utility.AvailableNamespaces(): 230 # @todo allow these; right now it's usually the XML 231 # namespace and we're not prepared to reconcile 232 # redefinitions of those components. 233 if ns.isUndeclaredNamespace(): 234 continue 235 mr = ns.lookupModuleRecordByUID(self.generationUID()) 236 if mr is not None: 237 namespaces.add(ns) 238 mr.prepareForArchive(self) 239 self.__moduleRecords.add(mr) 240 self.__namespaces.update(namespaces) 241 def moduleRecords (self): 242 """Return the set of L{module records <ModuleRecord>} stored in this 243 archive. 244 245 Each module record represents""" 246 return self.__moduleRecords 247 __moduleRecords = None 248 249 @classmethod 250 def ForPath (cls, archive_file): 251 """Return the L{NamespaceArchive} instance that can be found at the 252 given path.""" 253 return cls.__GetArchiveInstance(archive_file) 254 255 # States in the finite automaton that is used to read archive contents. 256 _STAGE_UNOPENED = 0 # Haven't even checked for existence 257 _STAGE_uid = 1 # Verified archive exists, obtained generation UID from it 258 _STAGE_readModules = 2 # Read module records from archive, which includes UID dependences 259 _STAGE_validateModules = 3 # Verified pre-requisites for module loading 260 _STAGE_readComponents = 4 # Extracted components from archive and integrated into namespaces 261 _STAGE_COMPLETE = _STAGE_readComponents 262 263 def _stage (self): 264 return self.__stage 265 __stage = None 266 267 def __init__ (self, archive_path=None, generation_uid=None, loadable=True, stage=None): 268 """Create a new namespace archive. 269 270 If C{namespaces} is given, this is an output archive. 271 272 If C{namespaces} is absent, this is an input archive. 273 274 @raise IOError: error attempting to read the archive file 275 @raise pickle.UnpicklingError: something is wrong with the format of the library 276 """ 277 self.__namespaces = set() 278 if generation_uid is not None: 279 if archive_path: 280 raise pyxb.LogicError('NamespaceArchive: cannot define both namespaces and archive_path') 281 self.__generationUID = generation_uid 282 self.__locateModuleRecords() 283 elif archive_path is not None: 284 if generation_uid is not None: 285 raise pyxb.LogicError('NamespaceArchive: cannot provide generation_uid with archive_path') 286 self.__archivePath = archive_path 287 self.__stage = self._STAGE_UNOPENED 288 self.__isLoadable = loadable 289 if self.__isLoadable: 290 if stage is None: 291 stage = self._STAGE_moduleRecords 292 self._readToStage(stage) 293 else: 294 pass 295 296 def add (self, namespace): 297 """Add the given namespace to the set that is to be stored in this archive.""" 298 if namespace.isAbsentNamespace(): 299 raise pyxb.NamespaceArchiveError('Cannot archive absent namespace') 300 self.__namespaces.add(namespace) 301 302 def update (self, namespace_set): 303 """Add the given namespaces to the set that is to be stored in this archive.""" 304 [ self.add(_ns) for _ns in namespace_set ] 305 306 def namespaces (self): 307 """Set of namespaces that can be read from this archive.""" 308 return self.__namespaces 309 __namespaces = None 310 311 def __createPickler (self, output): 312 if isinstance(output, six.string_types): 313 output = open(output, 'wb') 314 pickler = pickle.Pickler(output, -1) 315 316 # The format of the archive 317 pickler.dump(NamespaceArchive.__PickleFormat) 318 319 # The UID for the set 320 assert self.generationUID() is not None 321 pickler.dump(self.generationUID()) 322 323 return pickler 324 325 def __createUnpickler (self): 326 unpickler = pickle.Unpickler(open(self.__archivePath, 'rb')) 327 328 fmt = unpickler.load() 329 if self.__PickleFormat != fmt: 330 raise pyxb.NamespaceArchiveError('Archive format is %s, require %s' % (fmt, self.__PickleFormat)) 331 332 self.__generationUID = unpickler.load() 333 334 return unpickler 335 336 def __readModules (self, unpickler): 337 mrs = unpickler.load() 338 assert isinstance(mrs, set), 'Expected set got %s from %s' % (type(mrs), self.archivePath()) 339 if self.__moduleRecords is None: 340 for mr in mrs.copy(): 341 mr2 = mr.namespace().lookupModuleRecordByUID(mr.generationUID()) 342 if mr2 is not None: 343 mr2._setFromOther(mr, self) 344 mrs.remove(mr) 345 self.__moduleRecords = set() 346 assert 0 == len(self.__namespaces) 347 for mr in mrs: 348 mr._setArchive(self) 349 ns = mr.namespace() 350 ns.addModuleRecord(mr) 351 self.__namespaces.add(ns) 352 self.__moduleRecords.add(mr) 353 else: 354 # Verify the archive still has what was in it when we created this. 355 for mr in mrs: 356 mr2 = mr.namespace().lookupModuleRecordByUID(mr.generationUID()) 357 if not (mr2 in self.__moduleRecords): 358 raise pyxb.NamespaceArchiveError('Lost module record %s %s from %s' % (mr.namespace(), mr.generationUID(), self.archivePath())) 359 360 def _unsatisfiedModulePrerequisites (self): 361 prereq_uids = set() 362 for mr in self.__moduleRecords: 363 prereq_uids.update(mr.dependsOnExternal()) 364 return prereq_uids 365 366 def __validatePrerequisites (self, stage): 367 from pyxb.namespace import builtin 368 prereq_uids = self._unsatisfiedModulePrerequisites() 369 for uid in prereq_uids: 370 if builtin.BuiltInObjectUID == uid: 371 continue 372 depends_on = self.__NamespaceArchives.get(uid) 373 if depends_on is None: 374 raise pyxb.NamespaceArchiveError('%s: archive depends on unavailable archive %s' % (self.archivePath(), uid)) 375 depends_on._readToStage(stage) 376 377 def __validateModules (self): 378 self.__validatePrerequisites(self._STAGE_validateModules) 379 for mr in self.__moduleRecords: 380 ns = mr.namespace() 381 for base_uid in mr.dependsOnExternal(): 382 xmr = ns.lookupModuleRecordByUID(base_uid) 383 if xmr is None: 384 raise pyxb.NamespaceArchiveError('Module %s depends on external module %s, not available in archive path' % (mr.generationUID(), base_uid)) 385 if not xmr.isIncorporated(): 386 _log.info('Need to incorporate data from %s', xmr) 387 else: 388 _log.info('Have required base data %s', xmr) 389 390 for origin in mr.origins(): 391 for (cat, names) in six.iteritems(origin.categoryMembers()): 392 if not (cat in ns.categories()): 393 continue 394 cross_objects = names.intersection(six.iterkeys(ns.categoryMap(cat))) 395 if 0 < len(cross_objects): 396 raise pyxb.NamespaceArchiveError('Archive %s namespace %s module %s origin %s archive/active conflict on category %s: %s' % (self.__archivePath, ns, mr, origin, cat, " ".join(cross_objects))) 397 _log.info('%s no conflicts on %d names', cat, len(names)) 398 399 def __readComponentSet (self, unpickler): 400 self.__validatePrerequisites(self._STAGE_readComponents) 401 for n in range(len(self.__moduleRecords)): 402 ns = unpickler.load() 403 mr = ns.lookupModuleRecordByUID(self.generationUID()) 404 assert mr in self.__moduleRecords 405 assert not mr.isIncorporated() 406 objects = unpickler.load() 407 mr._loadCategoryObjects(objects) 408 409 __unpickler = None 410 def _readToStage (self, stage): 411 if self.__stage is None: 412 raise pyxb.NamespaceArchiveError('Attempt to read from invalid archive %s' % (self,)) 413 try: 414 while self.__stage < stage: 415 if self.__stage < self._STAGE_uid: 416 self.__unpickler = self.__createUnpickler() 417 self.__stage = self._STAGE_uid 418 continue 419 if self.__stage < self._STAGE_readModules: 420 assert self.__unpickler is not None 421 self.__readModules(self.__unpickler) 422 self.__stage = self._STAGE_readModules 423 continue 424 if self.__stage < self._STAGE_validateModules: 425 self.__validateModules() 426 self.__stage = self._STAGE_validateModules 427 continue 428 if self.__stage < self._STAGE_readComponents: 429 assert self.__unpickler is not None 430 self.__stage = self._STAGE_readComponents 431 self.__readComponentSet(self.__unpickler) 432 self.__unpickler = None 433 continue 434 raise pyxb.LogicError('Too many stages (at %s, want %s)' % (self.__stage, stage)) 435 except: 436 self.__stage = None 437 self.__unpickler = None 438 raise 439 440 def readNamespaces (self): 441 """Read all the components from this archive, integrating them into 442 their respective namespaces.""" 443 self._readToStage(self._STAGE_COMPLETE) 444 445 def writeNamespaces (self, output): 446 """Store the namespaces into the archive. 447 448 @param output: An instance substitutable for a writable file, or the 449 name of a file to write to. 450 """ 451 import sys 452 453 assert NamespaceArchive.__PicklingArchive is None 454 NamespaceArchive.__PicklingArchive = self 455 assert self.__moduleRecords is not None 456 457 # Recalculate the record/object associations: we didn't assign 458 # anonymous names to the indeterminate scope objects because they 459 # weren't needed for bindings, but they are needed in the archive. 460 for mr in self.__moduleRecords: 461 mr.namespace()._associateOrigins(mr) 462 463 try: 464 # See http://bugs.python.org/issue3338 465 recursion_limit = sys.getrecursionlimit() 466 sys.setrecursionlimit(10 * recursion_limit) 467 468 pickler = self.__createPickler(output) 469 470 assert isinstance(self.__moduleRecords, set) 471 pickler.dump(self.__moduleRecords) 472 473 for mr in self.__moduleRecords: 474 pickler.dump(mr.namespace()) 475 pickler.dump(mr.categoryObjects()) 476 finally: 477 sys.setrecursionlimit(recursion_limit) 478 NamespaceArchive.__PicklingArchive = None 479 480 def __str__ (self): 481 archive_path = self.__archivePath 482 if archive_path is None: 483 archive_path = '??' 484 return 'NSArchive@%s' % (archive_path,) 485 486class _ArchivableObject_mixin (pyxb.cscRoot): 487 """Mix-in to any object that can be stored in a namespace within an archive.""" 488 489 # Need to set this per category item 490 __objectOrigin = None 491 def _objectOrigin (self): 492 return self.__objectOrigin 493 def _setObjectOrigin (self, object_origin, override=False): 494 if (self.__objectOrigin is not None) and (not override): 495 if self.__objectOrigin != object_origin: 496 raise pyxb.LogicError('Inconsistent origins for object %s: %s %s' % (self, self.__objectOrigin, object_origin)) 497 else: 498 self.__objectOrigin = object_origin 499 500 def _prepareForArchive (self, archive): 501 #assert self.__objectOrigin is not None 502 if self._objectOrigin() is not None: 503 return getattr(super(_ArchivableObject_mixin, self), '_prepareForArchive_csc', lambda *_args,**_kw: self)(self._objectOrigin().moduleRecord()) 504 assert not isinstance(self, pyxb.xmlschema.structures._NamedComponent_mixin) 505 506 def _updateFromOther_csc (self, other): 507 return getattr(super(_ArchivableObject_mixin, self), '_updateFromOther_csc', lambda *_args,**_kw: self)(other) 508 509 def _updateFromOther (self, other): 510 """Update this instance with additional information provided by the other instance. 511 512 This is used, for example, when a built-in type is already registered 513 in the namespace, but we've processed the corresponding schema and 514 have obtained more details.""" 515 assert self != other 516 return self._updateFromOther_csc(other) 517 518 def _allowUpdateFromOther (self, other): 519 from pyxb.namespace import builtin 520 assert self._objectOrigin() 521 return builtin.BuiltInObjectUID == self._objectOrigin().generationUID() 522 523class _NamespaceArchivable_mixin (pyxb.cscRoot): 524 """Encapsulate the operations and data relevant to archiving namespaces. 525 526 This class mixes-in to L{pyxb.namespace.Namespace}""" 527 528 def _reset (self): 529 """CSC extension to reset fields of a Namespace. 530 531 This one handles category-related data.""" 532 getattr(super(_NamespaceArchivable_mixin, self), '_reset', lambda *args, **kw: None)() 533 self.__loadedFromArchive = None 534 self.__wroteToArchive = None 535 self.__active = False 536 self.__moduleRecordMap = {} 537 538 def _loadedFromArchive (self): 539 return self.__loadedFromArchive 540 541 __wroteToArchive = None 542 __loadedFromArchive = None 543 544 def isActive (self, empty_inactive=False): 545 if self.__isActive and empty_inactive: 546 for (ct, cm) in six.iteritems(self._categoryMap()): 547 if 0 < len(cm): 548 return True 549 return False 550 return self.__isActive 551 552 def _activate (self): 553 self.__isActive = True 554 __isActive = None 555 556 def __init__ (self, *args, **kw): 557 super(_NamespaceArchivable_mixin, self).__init__(*args, **kw) 558 559 def _setLoadedFromArchive (self, archive): 560 self.__loadedFromArchive = archive 561 self._activate() 562 def _setWroteToArchive (self, archive): 563 self.__wroteToArchive = archive 564 565 def _removeArchive (self, archive): 566 # Yes, I do want this to raise KeyError if the archive is not present 567 mr = self.__moduleRecordMap[archive.generationUID()] 568 assert not mr.isIncorporated(), 'Removing archive %s after incorporation' % (archive.archivePath(),) 569 del self.__moduleRecordMap[archive.generationUID()] 570 571 def isLoadable (self): 572 """Return C{True} iff the component model for this namespace can be 573 loaded from a namespace archive.""" 574 for mr in self.moduleRecords(): 575 if mr.isLoadable(): 576 return True 577 return False 578 579 def isImportAugmentable (self): 580 """Return C{True} iff the component model for this namespace may be 581 extended by import directives. 582 583 This is the case if the namespace has been marked with 584 L{setImportAugmentable}, or if there is no archive or built-in that 585 provides a component model for the namespace.""" 586 if self.__isImportAugmentable: 587 return True 588 for mr in self.moduleRecords(): 589 if mr.isLoadable() or mr.isIncorporated(): 590 return False 591 return True 592 def setImportAugmentable (self, value=True): 593 self.__isImportAugmentable = value 594 __isImportAugmentable = False 595 596 def loadableFrom (self): 597 """Return the list of archives from which components for this 598 namespace can be loaded.""" 599 rv = [] 600 for mr in self.moduleRecords(): 601 if mr.isLoadable(): 602 rv.append(mr.archive()) 603 return rv 604 605 def moduleRecords (self): 606 return list(six.itervalues(self.__moduleRecordMap)) 607 __moduleRecordMap = None 608 609 def addModuleRecord (self, module_record): 610 assert isinstance(module_record, ModuleRecord) 611 assert not (module_record.generationUID() in self.__moduleRecordMap) 612 self.__moduleRecordMap[module_record.generationUID()] = module_record 613 return module_record 614 def lookupModuleRecordByUID (self, generation_uid, create_if_missing=False, *args, **kw): 615 rv = self.__moduleRecordMap.get(generation_uid) 616 if (rv is None) and create_if_missing: 617 rv = self.addModuleRecord(ModuleRecord(self, generation_uid, *args, **kw)) 618 return rv 619 620 def _setState_csc (self, kw): 621 #assert not self.__isActive, 'ERROR: State set for active namespace %s' % (self,) 622 return getattr(super(_NamespaceArchivable_mixin, self), '_getState_csc', lambda _kw: _kw)(kw) 623 624 def markNotLoadable (self): 625 """Prevent loading this namespace from an archive. 626 627 This marks all archives in which the namespace appears, whether 628 publically or privately, as not loadable.""" 629 if self._loadedFromArchive(): 630 raise pyxb.NamespaceError(self, 'cannot mark not loadable when already loaded') 631 for mr in self.moduleRecords(): 632 mr._setIsLoadable(False) 633 634class ModuleRecord (pyxb.utils.utility.PrivateTransient_mixin): 635 __PrivateTransient = set() 636 637 def namespace (self): 638 return self.__namespace 639 __namespace = None 640 641 def archive (self): 642 return self.__archive 643 def _setArchive (self, archive): 644 self.__archive = archive 645 return self 646 __archive = None 647 __PrivateTransient.add('archive') 648 649 def isPublic (self): 650 return self.__isPublic 651 def _setIsPublic (self, is_public): 652 self.__isPublic = is_public 653 return self 654 __isPublic = None 655 656 def isIncorporated (self): 657 return self.__isIncorporated or (self.archive() is None) 658 def markIncorporated (self): 659 assert self.__isLoadable 660 self.__isIncorporated = True 661 self.__isLoadable = False 662 return self 663 __isIncorporated = None 664 __PrivateTransient.add('isIncorporated') 665 666 def isLoadable (self): 667 return self.__isLoadable and (self.archive() is not None) 668 def _setIsLoadable (self, is_loadable): 669 self.__isLoadable = is_loadable 670 return self 671 __isLoadable = None 672 673 def generationUID (self): 674 return self.__generationUID 675 __generationUID = None 676 677 def origins (self): 678 return list(six.itervalues(self.__originMap)) 679 def addOrigin (self, origin): 680 assert isinstance(origin, _ObjectOrigin) 681 assert not (origin.signature() in self.__originMap) 682 self.__originMap[origin.signature()] = origin 683 return origin 684 def lookupOriginBySignature (self, signature): 685 return self.__originMap.get(signature) 686 def _setOrigins (self, origins): 687 if self.__originMap is None: 688 self.__originMap = {} 689 else: 690 self.__originMap.clear() 691 [ self.addOrigin(_o) for _o in origins ] 692 return self 693 __originMap = None 694 695 def hasMatchingOrigin (self, **kw): 696 for origin in self.origins(): 697 if origin.match(**kw): 698 return True 699 return False 700 701 def modulePath (self): 702 return self.__modulePath 703 def setModulePath (self, module_path): 704 if isinstance(module_path, six.string_types): 705 self.__modulePath = '.'.join(map(pyxb.utils.utility.MakeModuleElement, module_path.split('.'))) 706 else: 707 assert (module_path is None) 708 self.__modulePath = module_path 709 return self 710 __modulePath = None 711 712 def referencedNamespaces (self): 713 return self.__referencedNamespaces 714 def _setReferencedNamespaces (self, referenced_namespaces): 715 self.__referencedNamespaces.update(referenced_namespaces) 716 return self 717 def referenceNamespace (self, namespace): 718 self.__referencedNamespaces.add(namespace) 719 return namespace 720 __referencedNamespaces = None 721 722 __constructedLocally = False 723 __PrivateTransient.add('constructedLocally') 724 725 def __init__ (self, namespace, generation_uid, **kw): 726 from pyxb.namespace import builtin 727 728 super(ModuleRecord, self).__init__() 729 self.__namespace = namespace 730 assert (generation_uid != builtin.BuiltInObjectUID) or namespace.isBuiltinNamespace() 731 self.__isPublic = kw.get('is_public', False) 732 self.__isIncorporated = kw.get('is_incorporated', False) 733 self.__isLoadable = kw.get('is_loadable', True) 734 assert isinstance(generation_uid, pyxb.utils.utility.UniqueIdentifier) 735 self.__generationUID = generation_uid 736 self.__modulePath = kw.get('module_path') 737 self.__originMap = {} 738 self.__referencedNamespaces = set() 739 self.__categoryObjects = { } 740 self.__constructedLocally = True 741 self.__dependsOnExternal = set() 742 743 def _setFromOther (self, other, archive): 744 if (not self.__constructedLocally) or other.__constructedLocally: 745 raise pyxb.ImplementationError('Module record update requires local to be updated from archive') 746 assert self.__generationUID == other.__generationUID 747 assert self.__archive is None 748 self.__isPublic = other.__isPublic 749 assert not self.__isIncorporated 750 self.__isLoadable = other.__isLoadable 751 self.__modulePath = other.__modulePath 752 self.__originMap.update(other.__originMap) 753 self.__referencedNamespaces.update(other.__referencedNamespaces) 754 if not (other.__categoryObjects is None): 755 self.__categoryObjects.update(other.__categoryObjects) 756 self.__dependsOnExternal.update(other.__dependsOnExternal) 757 self._setArchive(archive) 758 759 def categoryObjects (self): 760 return self.__categoryObjects 761 def resetCategoryObjects (self): 762 self.__categoryObjects.clear() 763 for origin in self.origins(): 764 origin.resetCategoryMembers() 765 def _addCategoryObject (self, category, name, obj): 766 obj._prepareForArchive(self) 767 self.__categoryObjects.setdefault(category, {})[name] = obj 768 def _loadCategoryObjects (self, category_objects): 769 assert self.__categoryObjects is None 770 assert not self.__constructedLocally 771 ns = self.namespace() 772 ns.configureCategories(six.iterkeys(category_objects)) 773 for (cat, obj_map) in six.iteritems(category_objects): 774 current_map = ns.categoryMap(cat) 775 for (local_name, component) in six.iteritems(obj_map): 776 existing_component = current_map.get(local_name) 777 if existing_component is None: 778 current_map[local_name] = component 779 elif existing_component._allowUpdateFromOther(component): 780 existing_component._updateFromOther(component) 781 else: 782 raise pyxb.NamespaceError(self, 'Load attempted to override %s %s in %s' % (cat, local_name, self.namespace())) 783 self.markIncorporated() 784 __categoryObjects = None 785 __PrivateTransient.add('categoryObjects') 786 787 def dependsOnExternal (self): 788 return self.__dependsOnExternal 789 __dependsOnExternal = None 790 791 def prepareForArchive (self, archive): 792 assert self.archive() is None 793 self._setArchive(archive) 794 ns = self.namespace() 795 self.__dependsOnExternal.clear() 796 for mr in ns.moduleRecords(): 797 if mr != self: 798 _log.info('This gen depends on %s', mr) 799 self.__dependsOnExternal.add(mr.generationUID()) 800 for obj in ns._namedObjects().union(ns.components()): 801 if isinstance(obj, _ArchivableObject_mixin): 802 if obj._objectOrigin(): 803 obj._prepareForArchive(self) 804 805 def completeGenerationAssociations (self): 806 self.namespace()._transferReferencedNamespaces(self) 807 self.namespace()._associateOrigins(self) 808 809 def __str__ (self): 810 return 'MR[%s]@%s' % (self.generationUID(), self.namespace()) 811 812class _ObjectOrigin (pyxb.utils.utility.PrivateTransient_mixin, pyxb.cscRoot): 813 """Marker class for objects that can serve as an origin for an object in a 814 namespace.""" 815 __PrivateTransient = set() 816 817 def signature (self): 818 return self.__signature 819 __signature = None 820 821 def moduleRecord (self): 822 return self.__moduleRecord 823 __moduleRecord = None 824 825 def namespace (self): 826 return self.moduleRecord().namespace() 827 828 def generationUID (self): 829 return self.moduleRecord().generationUID() 830 831 def __init__ (self, namespace, generation_uid, **kw): 832 self.__signature = kw.pop('signature', None) 833 super(_ObjectOrigin, self).__init__(**kw) 834 self.__moduleRecord = namespace.lookupModuleRecordByUID(generation_uid, create_if_missing=True, **kw) 835 self.__moduleRecord.addOrigin(self) 836 self.__categoryMembers = { } 837 self.__categoryObjectMap = { } 838 839 def resetCategoryMembers (self): 840 self.__categoryMembers.clear() 841 self.__categoryObjectMap.clear() 842 self.__originatedComponents = None 843 def addCategoryMember (self, category, name, obj): 844 self.__categoryMembers.setdefault(category, set()).add(name) 845 self.__categoryObjectMap.setdefault(category, {})[name] = obj 846 self.__moduleRecord._addCategoryObject(category, name, obj) 847 def categoryMembers (self): 848 return self.__categoryMembers 849 def originatedObjects (self): 850 if self.__originatedObjects is None: 851 components = set() 852 [ components.update(six.itervalues(_v)) for _v in six.itervalues(self.__categoryObjectMap) ] 853 self.__originatedObjects = frozenset(components) 854 return self.__originatedObjects 855 856 # The set of category names associated with objects. Don't throw this 857 # away and use categoryObjectMap.keys() instead: that's transient, and we 858 # need this to have a value when read from an archive. 859 __categoryMembers = None 860 861 # Map from category name to a map from an object name to the object 862 __categoryObjectMap = None 863 __PrivateTransient.add('categoryObjectMap') 864 865 # The set of objects that originated at this origin 866 __originatedObjects = None 867 __PrivateTransient.add('originatedObjects') 868 869class _SchemaOrigin (_ObjectOrigin): 870 """Holds the data regarding components derived from a single schema. 871 872 Coupled to a particular namespace through the 873 L{_NamespaceComponentAssociation_mixin}. 874 """ 875 876 __PrivateTransient = set() 877 878 def __setDefaultKW (self, kw): 879 schema = kw.get('schema') 880 if schema is not None: 881 assert not ('location' in kw) 882 kw['location'] = schema.location() 883 assert not ('signature' in kw) 884 kw['signature'] = schema.signature() 885 assert not ('generation_uid' in kw) 886 kw['generation_uid'] = schema.generationUID() 887 assert not ('namespace' in kw) 888 kw['namespace'] = schema.targetNamespace() 889 assert not ('version' in kw) 890 kw['version'] = schema.schemaAttribute('version') 891 892 def match (self, **kw): 893 """Determine whether this record matches the parameters. 894 895 @keyword schema: a L{pyxb.xmlschema.structures.Schema} instance from 896 which the other parameters are obtained. 897 @keyword location: a schema location (URI) 898 @keyword signature: a schema signature 899 @return: C{True} iff I{either} C{location} or C{signature} matches.""" 900 self.__setDefaultKW(kw) 901 location = kw.get('location') 902 if (location is not None) and (self.location() == location): 903 return True 904 signature = kw.get('signature') 905 if (signature is not None) and (self.signature() == signature): 906 return True 907 return False 908 909 def location (self): 910 return self.__location 911 __location = None 912 913 def schema (self): 914 return self.__schema 915 __schema = None 916 __PrivateTransient.add('schema') 917 918 def version (self): 919 return self.__version 920 __version = None 921 922 def __init__ (self, **kw): 923 self.__setDefaultKW(kw) 924 self.__schema = kw.pop('schema', None) 925 self.__location = kw.pop('location', None) 926 self.__version = kw.pop('version', None) 927 super(_SchemaOrigin, self).__init__(kw.pop('namespace'), kw.pop('generation_uid'), **kw) 928 929 def __str__ (self): 930 rv = [ '_SchemaOrigin(%s@%s' % (self.namespace(), self.location()) ] 931 if self.version() is not None: 932 rv.append(',version=%s' % (self.version(),)) 933 rv.append(')') 934 return ''.join(rv) 935 936class NamespaceDependencies (object): 937 938 def rootNamespaces (self): 939 return self.__rootNamespaces 940 __rootNamespaces = None 941 942 def namespaceGraph (self, reset=False): 943 if reset or (self.__namespaceGraph is None): 944 self.__namespaceGraph = pyxb.utils.utility.Graph() 945 for ns in self.rootNamespaces(): 946 self.__namespaceGraph.addRoot(ns) 947 948 # Make sure all referenced namespaces have valid components 949 need_check = self.__rootNamespaces.copy() 950 done_check = set() 951 while 0 < len(need_check): 952 ns = need_check.pop() 953 ns.validateComponentModel() 954 self.__namespaceGraph.addNode(ns) 955 for rns in ns.referencedNamespaces().union(ns.importedNamespaces()): 956 self.__namespaceGraph.addEdge(ns, rns) 957 if not rns in done_check: 958 need_check.add(rns) 959 if not ns.hasSchemaComponents(): 960 _log.warning('Referenced %s has no schema components', ns.uri()) 961 done_check.add(ns) 962 assert done_check == self.__namespaceGraph.nodes() 963 964 return self.__namespaceGraph 965 __namespaceGraph = None 966 967 def namespaceOrder (self, reset=False): 968 return self.namespaceGraph(reset).sccOrder() 969 970 def siblingsFromGraph (self, reset=False): 971 siblings = set() 972 ns_graph = self.namespaceGraph(reset) 973 for ns in self.__rootNamespaces: 974 ns_siblings = ns_graph.sccMap().get(ns) 975 if ns_siblings is not None: 976 siblings.update(ns_siblings) 977 else: 978 siblings.add(ns) 979 return siblings 980 981 def siblingNamespaces (self): 982 if self.__siblingNamespaces is None: 983 self.__siblingNamespaces = self.siblingsFromGraph() 984 return self.__siblingNamespaces 985 986 def setSiblingNamespaces (self, sibling_namespaces): 987 self.__siblingNamespaces = sibling_namespaces 988 989 __siblingNamespaces = None 990 991 def dependentNamespaces (self, reset=False): 992 return self.namespaceGraph(reset).nodes() 993 994 def componentGraph (self, reset=False): 995 if reset or (self.__componentGraph is None): 996 self.__componentGraph = pyxb.utils.utility.Graph() 997 all_components = set() 998 for ns in self.siblingNamespaces(): 999 [ all_components.add(_c) for _c in ns.components() if _c.hasBinding() ] 1000 1001 need_visit = all_components.copy() 1002 while 0 < len(need_visit): 1003 c = need_visit.pop() 1004 self.__componentGraph.addNode(c) 1005 for cd in c.bindingRequires(include_lax=True): 1006 if cd in all_components: 1007 self.__componentGraph.addEdge(c, cd) 1008 return self.__componentGraph 1009 __componentGraph = None 1010 1011 def componentOrder (self, reset=False): 1012 return self.componentGraph(reset).sccOrder() 1013 1014 def __init__ (self, **kw): 1015 namespace_set = set(kw.get('namespace_set', [])) 1016 namespace = kw.get('namespace') 1017 if namespace is not None: 1018 namespace_set.add(namespace) 1019 if 0 == len(namespace_set): 1020 raise pyxb.LogicError('NamespaceDependencies requires at least one root namespace') 1021 self.__rootNamespaces = namespace_set 1022 1023 1024## Local Variables: 1025## fill-column:78 1026## End: 1027