1# -*- coding: utf-8 -*- 2 3######################################################################## 4# 5# License: BSD 6# Created: September 4, 2002 7# Author: Francesc Alted - faltet@pytables.com 8# 9# $Id$ 10# 11######################################################################## 12 13"""Here is defined the Group class.""" 14 15import os 16import re 17import weakref 18import warnings 19 20from .misc.proxydict import ProxyDict 21from . import hdf5extension 22from . import utilsextension 23from .registry import class_id_dict 24from .exceptions import NodeError, NoSuchNodeError, NaturalNameWarning, PerformanceWarning 25from .filters import Filters 26from .registry import get_class_by_name 27from .path import check_name_validity, join_path, isvisiblename 28from .node import Node, NotLoggedMixin 29from .leaf import Leaf 30from .unimplemented import UnImplemented, Unknown 31 32from .link import Link, SoftLink, ExternalLink 33 34 35obversion = "1.0" 36 37 38class _ChildrenDict(ProxyDict): 39 def _get_value_from_container(self, container, key): 40 return container._f_get_child(key) 41 42 43class Group(hdf5extension.Group, Node): 44 """Basic PyTables grouping structure. 45 46 Instances of this class are grouping structures containing *child* 47 instances of zero or more groups or leaves, together with 48 supporting metadata. Each group has exactly one *parent* group. 49 50 Working with groups and leaves is similar in many ways to working 51 with directories and files, respectively, in a Unix filesystem. 52 As with Unix directories and files, objects in the object tree are 53 often described by giving their full (or absolute) path names. 54 This full path can be specified either as a string (like in 55 '/group1/group2') or as a complete object path written in *natural 56 naming* schema (like in file.root.group1.group2). 57 58 A collateral effect of the *natural naming* schema is that the 59 names of members in the Group class and its instances must be 60 carefully chosen to avoid colliding with existing children node 61 names. For this reason and to avoid polluting the children 62 namespace all members in a Group start with some reserved prefix, 63 like _f_ (for public methods), _g_ (for private ones), _v_ (for 64 instance variables) or _c_ (for class variables). Any attempt to 65 create a new child node whose name starts with one of these 66 prefixes will raise a ValueError exception. 67 68 Another effect of natural naming is that children named after 69 Python keywords or having names not valid as Python identifiers 70 (e.g. class, $a or 44) can not be accessed using the node.child 71 syntax. You will be forced to use node._f_get_child(child) to 72 access them (which is recommended for programmatic accesses). 73 74 You will also need to use _f_get_child() to access an existing 75 child node if you set a Python attribute in the Group with the 76 same name as that node (you will get a NaturalNameWarning when 77 doing this). 78 79 Parameters 80 ---------- 81 parentnode 82 The parent :class:`Group` object. 83 name : str 84 The name of this node in its parent group. 85 title 86 The title for this group 87 new 88 If this group is new or has to be read from disk 89 filters : Filters 90 A Filters instance 91 92 93 .. versionchanged:: 3.0 94 *parentNode* renamed into *parentnode* 95 96 Notes 97 ----- 98 The following documentation includes methods that are automatically 99 called when a Group instance is accessed in a special way. 100 101 For instance, this class defines the __setattr__, __getattr__, 102 __delattr__ and __dir__ methods, and they set, get and delete 103 *ordinary Python attributes* as normally intended. In addition to that, 104 __getattr__ allows getting *child nodes* by their name for the sake of 105 easy interaction on the command line, as long as there is no Python 106 attribute with the same name. Groups also allow the interactive 107 completion (when using readline) of the names of child nodes. 108 For instance:: 109 110 # get a Python attribute 111 nchild = group._v_nchildren 112 113 # Add a Table child called 'table' under 'group'. 114 h5file.create_table(group, 'table', myDescription) 115 table = group.table # get the table child instance 116 group.table = 'foo' # set a Python attribute 117 118 # (PyTables warns you here about using the name of a child node.) 119 foo = group.table # get a Python attribute 120 del group.table # delete a Python attribute 121 table = group.table # get the table child instance again 122 123 Additionally, on interactive python sessions you may get autocompletions 124 of children named as *valid python identifiers* by pressing the `[Tab]` 125 key, or to use the dir() global function. 126 127 .. rubric:: Group attributes 128 129 The following instance variables are provided in addition to those 130 in Node (see :ref:`NodeClassDescr`): 131 132 .. attribute:: _v_children 133 134 Dictionary with all nodes hanging from this group. 135 136 .. attribute:: _v_groups 137 138 Dictionary with all groups hanging from this group. 139 140 .. attribute:: _v_hidden 141 142 Dictionary with all hidden nodes hanging from this group. 143 144 .. attribute:: _v_leaves 145 146 Dictionary with all leaves hanging from this group. 147 148 .. attribute:: _v_links 149 150 Dictionary with all links hanging from this group. 151 152 .. attribute:: _v_unknown 153 154 Dictionary with all unknown nodes hanging from this group. 155 156 """ 157 158 # Class identifier. 159 _c_classid = 'GROUP' 160 161 162 # Children containers that should be loaded only in a lazy way. 163 # These are documented in the ``Group._g_add_children_names`` method. 164 _c_lazy_children_attrs = ( 165 '__members__', '_v_children', '_v_groups', '_v_leaves', 166 '_v_links', '_v_unknown', '_v_hidden') 167 168 # `_v_nchildren` is a direct read-only shorthand 169 # for the number of *visible* children in a group. 170 def _g_getnchildren(self): 171 "The number of children hanging from this group." 172 return len(self._v_children) 173 174 _v_nchildren = property(_g_getnchildren) 175 176 # `_v_filters` is a direct read-write shorthand for the ``FILTERS`` 177 # attribute with the default `Filters` instance as a default value. 178 def _g_getfilters(self): 179 filters = getattr(self._v_attrs, 'FILTERS', None) 180 if filters is None: 181 filters = Filters() 182 return filters 183 184 def _g_setfilters(self, value): 185 if not isinstance(value, Filters): 186 raise TypeError( 187 "value is not an instance of `Filters`: %r" % (value,)) 188 self._v_attrs.FILTERS = value 189 190 def _g_delfilters(self): 191 del self._v_attrs.FILTERS 192 193 _v_filters = property( 194 _g_getfilters, _g_setfilters, _g_delfilters, 195 """Default filter properties for child nodes. 196 197 You can (and are encouraged to) use this property to get, set and 198 delete the FILTERS HDF5 attribute of the group, which stores a Filters 199 instance (see :ref:`FiltersClassDescr`). When the group has no such 200 attribute, a default Filters instance is used. 201 """) 202 203 204 def __init__(self, parentnode, name, 205 title="", new=False, filters=None, 206 _log=True): 207 208 # Remember to assign these values in the root group constructor 209 # if it does not use this one! 210 211 # First, set attributes belonging to group objects. 212 213 self._v_version = obversion 214 """The object version of this group.""" 215 216 self._v_new = new 217 """Is this the first time the node has been created?""" 218 219 self._v_new_title = title 220 """New title for this node.""" 221 222 self._v_new_filters = filters 223 """New default filter properties for child nodes.""" 224 225 self._v_max_group_width = parentnode._v_file.params['MAX_GROUP_WIDTH'] 226 """Maximum number of children on each group before warning the user. 227 228 .. versionchanged:: 3.0 229 The *_v_maxGroupWidth* attribute has been renamed into 230 *_v_max_group_width*. 231 232 """ 233 234 # Finally, set up this object as a node. 235 super(Group, self).__init__(parentnode, name, _log) 236 237 def _g_post_init_hook(self): 238 if self._v_new: 239 if self._v_file.params['PYTABLES_SYS_ATTRS']: 240 # Save some attributes for the new group on disk. 241 set_attr = self._v_attrs._g__setattr 242 # Set the title, class and version attributes. 243 set_attr('TITLE', self._v_new_title) 244 set_attr('CLASS', self._c_classid) 245 set_attr('VERSION', self._v_version) 246 247 # Set the default filter properties. 248 newfilters = self._v_new_filters 249 if newfilters is None: 250 # If no filters have been passed in the constructor, 251 # inherit them from the parent group, but only if they 252 # have been inherited or explicitly set. 253 newfilters = getattr( 254 self._v_parent._v_attrs, 'FILTERS', None) 255 if newfilters is not None: 256 set_attr('FILTERS', newfilters) 257 else: 258 # If the file has PyTables format, get the VERSION attr 259 if 'VERSION' in self._v_attrs._v_attrnamessys: 260 self._v_version = self._v_attrs.VERSION 261 else: 262 self._v_version = "0.0 (unknown)" 263 # We don't need to get more attributes from disk, 264 # since the most important ones are defined as properties. 265 266 267 def __del__(self): 268 if (self._v_isopen and 269 self._v_pathname in self._v_file._node_manager.registry and 270 '_v_children' in self.__dict__): 271 # The group is going to be killed. Rebuild weak references 272 # (that Python cancelled just before calling this method) so 273 # that they are still usable if the object is revived later. 274 selfref = weakref.ref(self) 275 self._v_children.containerref = selfref 276 self._v_groups.containerref = selfref 277 self._v_leaves.containerref = selfref 278 self._v_links.containerref = selfref 279 self._v_unknown.containerref = selfref 280 self._v_hidden.containerref = selfref 281 282 super(Group, self).__del__() 283 284 def _g_get_child_group_class(self, childname): 285 """Get the class of a not-yet-loaded group child. 286 287 `childname` must be the name of a *group* child. 288 289 """ 290 291 childCID = self._g_get_gchild_attr(childname, 'CLASS') 292 if childCID is not None and not isinstance(childCID, str): 293 childCID = childCID.decode('utf-8') 294 295 if childCID in class_id_dict: 296 return class_id_dict[childCID] # look up group class 297 else: 298 return Group # default group class 299 300 301 def _g_get_child_leaf_class(self, childname, warn=True): 302 """Get the class of a not-yet-loaded leaf child. 303 304 `childname` must be the name of a *leaf* child. If the child 305 belongs to an unknown kind of leaf, or if its kind can not be 306 guessed, `UnImplemented` will be returned and a warning will be 307 issued if `warn` is true. 308 309 """ 310 311 if self._v_file.params['PYTABLES_SYS_ATTRS']: 312 childCID = self._g_get_lchild_attr(childname, 'CLASS') 313 if childCID is not None and not isinstance(childCID, str): 314 childCID = childCID.decode('utf-8') 315 else: 316 childCID = None 317 318 if childCID in class_id_dict: 319 return class_id_dict[childCID] # look up leaf class 320 else: 321 # Unknown or no ``CLASS`` attribute, try a guess. 322 childCID2 = utilsextension.which_class(self._v_objectid, childname) 323 if childCID2 == 'UNSUPPORTED': 324 if warn: 325 if childCID is None: 326 warnings.warn( 327 "leaf ``%s`` is of an unsupported type; " 328 "it will become an ``UnImplemented`` node" 329 % self._g_join(childname)) 330 else: 331 warnings.warn( 332 ("leaf ``%s`` has an unknown class ID ``%s``; " 333 "it will become an ``UnImplemented`` node") 334 % (self._g_join(childname), childCID)) 335 return UnImplemented 336 assert childCID2 in class_id_dict 337 return class_id_dict[childCID2] # look up leaf class 338 339 340 def _g_add_children_names(self): 341 """Add children names to this group taking into account their 342 visibility and kind.""" 343 344 mydict = self.__dict__ 345 346 # The names of the lazy attributes 347 mydict['__members__'] = members = [] 348 """The names of visible children nodes for readline-style completion. 349 """ 350 mydict['_v_children'] = children = _ChildrenDict(self) 351 """The number of children hanging from this group.""" 352 mydict['_v_groups'] = groups = _ChildrenDict(self) 353 """Dictionary with all groups hanging from this group.""" 354 mydict['_v_leaves'] = leaves = _ChildrenDict(self) 355 """Dictionary with all leaves hanging from this group.""" 356 mydict['_v_links'] = links = _ChildrenDict(self) 357 """Dictionary with all links hanging from this group.""" 358 mydict['_v_unknown'] = unknown = _ChildrenDict(self) 359 """Dictionary with all unknown nodes hanging from this group.""" 360 mydict['_v_hidden'] = hidden = _ChildrenDict(self) 361 """Dictionary with all hidden nodes hanging from this group.""" 362 363 # Get the names of *all* child groups and leaves. 364 (group_names, leaf_names, link_names, unknown_names) = \ 365 self._g_list_group(self._v_parent) 366 367 # Separate groups into visible groups and hidden nodes, 368 # and leaves into visible leaves and hidden nodes. 369 for (childnames, childdict) in ((group_names, groups), 370 (leaf_names, leaves), 371 (link_names, links), 372 (unknown_names, unknown)): 373 374 for childname in childnames: 375 # See whether the name implies that the node is hidden. 376 # (Assigned values are entirely irrelevant.) 377 if isvisiblename(childname): 378 # Visible node. 379 members.insert(0, childname) 380 children[childname] = None 381 childdict[childname] = None 382 else: 383 # Hidden node. 384 hidden[childname] = None 385 386 387 def _g_check_has_child(self, name): 388 """Check whether 'name' is a children of 'self' and return its type.""" 389 390 # Get the HDF5 name matching the PyTables name. 391 node_type = self._g_get_objinfo(name) 392 if node_type == "NoSuchNode": 393 raise NoSuchNodeError( 394 "group ``%s`` does not have a child named ``%s``" 395 % (self._v_pathname, name)) 396 return node_type 397 398 399 def __iter__(self): 400 """Iterate over the child nodes hanging directly from the group. 401 402 This iterator is *not* recursive. 403 404 Examples 405 -------- 406 407 :: 408 409 # Non-recursively list all the nodes hanging from '/detector' 410 print("Nodes in '/detector' group:") 411 for node in h5file.root.detector: 412 print(node) 413 414 """ 415 416 return self._f_iter_nodes() 417 418 def __contains__(self, name): 419 """Is there a child with that `name`? 420 421 Returns a true value if the group has a child node (visible or 422 hidden) with the given `name` (a string), false otherwise. 423 424 """ 425 426 self._g_check_open() 427 try: 428 self._g_check_has_child(name) 429 except NoSuchNodeError: 430 return False 431 return True 432 433 def __getitem__(self, childname): 434 """Return the (visible or hidden) child with that `name` ( a string). 435 436 Raise IndexError if child not exist. 437 """ 438 try: 439 return self._f_get_child(childname) 440 except NoSuchNodeError: 441 raise IndexError(childname) 442 443 def _f_walknodes(self, classname=None): 444 """Iterate over descendant nodes. 445 446 This method recursively walks *self* top to bottom (preorder), 447 iterating over child groups in alphanumerical order, and yielding 448 nodes. If classname is supplied, only instances of the named class are 449 yielded. 450 451 If *classname* is Group, it behaves like :meth:`Group._f_walk_groups`, 452 yielding only groups. If you don't want a recursive behavior, 453 use :meth:`Group._f_iter_nodes` instead. 454 455 Examples 456 -------- 457 458 :: 459 460 # Recursively print all the arrays hanging from '/' 461 print("Arrays in the object tree '/':") 462 for array in h5file.root._f_walknodes('Array', recursive=True): 463 print(array) 464 465 """ 466 467 self._g_check_open() 468 469 # For compatibility with old default arguments. 470 if classname == '': 471 classname = None 472 473 if classname == "Group": 474 # Recursive algorithm 475 for group in self._f_walk_groups(): 476 yield group 477 else: 478 for group in self._f_walk_groups(): 479 for leaf in group._f_iter_nodes(classname): 480 yield leaf 481 482 483 def _g_join(self, name): 484 """Helper method to correctly concatenate a name child object with the 485 pathname of this group.""" 486 487 if name == "/": 488 # This case can happen when doing copies 489 return self._v_pathname 490 return join_path(self._v_pathname, name) 491 492 def _g_width_warning(self): 493 """Issue a :exc:`PerformanceWarning` on too many children.""" 494 495 warnings.warn("""\ 496group ``%s`` is exceeding the recommended maximum number of children (%d); \ 497be ready to see PyTables asking for *lots* of memory and possibly slow I/O.""" 498 % (self._v_pathname, self._v_max_group_width), 499 PerformanceWarning) 500 501 502 def _g_refnode(self, childnode, childname, validate=True): 503 """Insert references to a `childnode` via a `childname`. 504 505 Checks that the `childname` is valid and does not exist, then 506 creates references to the given `childnode` by that `childname`. 507 The validation of the name can be omitted by setting `validate` 508 to a false value (this may be useful for adding already existing 509 nodes to the tree). 510 511 """ 512 513 # Check for name validity. 514 if validate: 515 check_name_validity(childname) 516 childnode._g_check_name(childname) 517 518 # Check if there is already a child with the same name. 519 # This can be triggered because of the user 520 # (via node construction or renaming/movement). 521 # Links are not checked here because they are copied and referenced 522 # using ``File.get_node`` so they already exist in `self`. 523 if (not isinstance(childnode, Link)) and childname in self: 524 raise NodeError( 525 "group ``%s`` already has a child node named ``%s``" 526 % (self._v_pathname, childname)) 527 528 # Show a warning if there is an object attribute with that name. 529 if childname in self.__dict__: 530 warnings.warn( 531 "group ``%s`` already has an attribute named ``%s``; " 532 "you will not be able to use natural naming " 533 "to access the child node" 534 % (self._v_pathname, childname), NaturalNameWarning) 535 536 # Check group width limits. 537 if (len(self._v_children) + len(self._v_hidden) >= 538 self._v_max_group_width): 539 self._g_width_warning() 540 541 # Update members information. 542 # Insert references to the new child. 543 # (Assigned values are entirely irrelevant.) 544 if isvisiblename(childname): 545 # Visible node. 546 self.__members__.insert(0, childname) # enable completion 547 self._v_children[childname] = None # insert node 548 if isinstance(childnode, Unknown): 549 self._v_unknown[childname] = None 550 elif isinstance(childnode, Link): 551 self._v_links[childname] = None 552 elif isinstance(childnode, Leaf): 553 self._v_leaves[childname] = None 554 elif isinstance(childnode, Group): 555 self._v_groups[childname] = None 556 else: 557 # Hidden node. 558 self._v_hidden[childname] = None # insert node 559 560 561 def _g_unrefnode(self, childname): 562 """Remove references to a node. 563 564 Removes all references to the named node. 565 566 """ 567 568 # This can *not* be triggered because of the user. 569 assert childname in self, \ 570 ("group ``%s`` does not have a child node named ``%s``" 571 % (self._v_pathname, childname)) 572 573 # Update members information, if needed 574 if '_v_children' in self.__dict__: 575 if childname in self._v_children: 576 # Visible node. 577 members = self.__members__ 578 member_index = members.index(childname) 579 del members[member_index] # disables completion 580 581 del self._v_children[childname] # remove node 582 self._v_unknown.pop(childname, None) 583 self._v_links.pop(childname, None) 584 self._v_leaves.pop(childname, None) 585 self._v_groups.pop(childname, None) 586 else: 587 # Hidden node. 588 del self._v_hidden[childname] # remove node 589 590 591 def _g_move(self, newparent, newname): 592 # Move the node to the new location. 593 oldpath = self._v_pathname 594 super(Group, self)._g_move(newparent, newname) 595 newpath = self._v_pathname 596 597 # Update location information in children. This node shouldn't 598 # be affected since it has already been relocated. 599 self._v_file._update_node_locations(oldpath, newpath) 600 601 def _g_copy(self, newparent, newname, recursive, _log=True, **kwargs): 602 # Compute default arguments. 603 title = kwargs.get('title', self._v_title) 604 filters = kwargs.get('filters', None) 605 stats = kwargs.get('stats', None) 606 607 # Fix arguments with explicit None values for backwards compatibility. 608 if title is None: 609 title = self._v_title 610 # If no filters have been passed to the call, copy them from the 611 # source group, but only if inherited or explicitly set. 612 if filters is None: 613 filters = getattr(self._v_attrs, 'FILTERS', None) 614 615 # Create a copy of the object. 616 new_node = Group(newparent, newname, 617 title, new=True, filters=filters, _log=_log) 618 619 # Copy user attributes if needed. 620 if kwargs.get('copyuserattrs', True): 621 self._v_attrs._g_copy(new_node._v_attrs, copyclass=True) 622 623 # Update statistics if needed. 624 if stats is not None: 625 stats['groups'] += 1 626 627 if recursive: 628 # Copy child nodes if a recursive copy was requested. 629 # Some arguments should *not* be passed to children copy ops. 630 kwargs = kwargs.copy() 631 kwargs.pop('title', None) 632 self._g_copy_children(new_node, **kwargs) 633 634 return new_node 635 636 def _g_copy_children(self, newparent, **kwargs): 637 """Copy child nodes. 638 639 Copies all nodes descending from this one into the specified 640 `newparent`. If the new parent has a child node with the same 641 name as one of the nodes in this group, the copy fails with a 642 `NodeError`, maybe resulting in a partial copy. Nothing is 643 logged. 644 645 """ 646 647 # Recursive version of children copy. 648 # for srcchild in self._v_children.itervalues(): 649 ## srcchild._g_copy_as_child(newparent, **kwargs) 650 651 # Non-recursive version of children copy. 652 use_hardlinks = kwargs.get('use_hardlinks', False) 653 if use_hardlinks: 654 address_map = kwargs.setdefault('address_map', {}) 655 656 parentstack = [(self, newparent)] # [(source, destination), ...] 657 while parentstack: 658 (srcparent, dstparent) = parentstack.pop() 659 660 if use_hardlinks: 661 for srcchild in srcparent._v_children.values(): 662 addr, rc = srcchild._get_obj_info() 663 if rc > 1 and addr in address_map: 664 where, name = address_map[addr][0] 665 localsrc = os.path.join(where, name) 666 dstparent._v_file.create_hard_link(dstparent, 667 srcchild.name, 668 localsrc) 669 address_map[addr].append( 670 (dstparent._v_pathname, srcchild.name) 671 ) 672 673 # Update statistics if needed. 674 stats = kwargs.pop('stats', None) 675 if stats is not None: 676 stats['hardlinks'] += 1 677 else: 678 dstchild = srcchild._g_copy_as_child(dstparent, 679 **kwargs) 680 if isinstance(srcchild, Group): 681 parentstack.append((srcchild, dstchild)) 682 683 if rc > 1: 684 address_map[addr] = [ 685 (dstparent._v_pathname, srcchild.name) 686 ] 687 else: 688 for srcchild in srcparent._v_children.values(): 689 dstchild = srcchild._g_copy_as_child(dstparent, **kwargs) 690 if isinstance(srcchild, Group): 691 parentstack.append((srcchild, dstchild)) 692 693 694 def _f_get_child(self, childname): 695 """Get the child called childname of this group. 696 697 If the child exists (be it visible or not), it is returned. Else, a 698 NoSuchNodeError is raised. 699 700 Using this method is recommended over getattr() when doing programmatic 701 accesses to children if childname is unknown beforehand or when its 702 name is not a valid Python identifier. 703 704 """ 705 706 self._g_check_open() 707 708 self._g_check_has_child(childname) 709 710 childpath = join_path(self._v_pathname, childname) 711 return self._v_file._get_node(childpath) 712 713 714 def _f_list_nodes(self, classname=None): 715 """Return a *list* with children nodes. 716 717 This is a list-returning version of :meth:`Group._f_iter_nodes()`. 718 719 """ 720 721 return list(self._f_iter_nodes(classname)) 722 723 724 def _f_iter_nodes(self, classname=None): 725 """Iterate over children nodes. 726 727 Child nodes are yielded alphanumerically sorted by node name. If the 728 name of a class derived from Node (see :ref:`NodeClassDescr`) is 729 supplied in the classname parameter, only instances of that class (or 730 subclasses of it) will be returned. 731 732 This is an iterator version of :meth:`Group._f_list_nodes`. 733 734 """ 735 736 self._g_check_open() 737 738 if not classname: 739 # Returns all the children alphanumerically sorted 740 names = sorted(self._v_children.keys()) 741 for name in names: 742 yield self._v_children[name] 743 elif classname == 'Group': 744 # Returns all the groups alphanumerically sorted 745 names = sorted(self._v_groups.keys()) 746 for name in names: 747 yield self._v_groups[name] 748 elif classname == 'Leaf': 749 # Returns all the leaves alphanumerically sorted 750 names = sorted(self._v_leaves.keys()) 751 for name in names: 752 yield self._v_leaves[name] 753 elif classname == 'Link': 754 # Returns all the links alphanumerically sorted 755 names = sorted(self._v_links.keys()) 756 for name in names: 757 yield self._v_links[name] 758 elif classname == 'IndexArray': 759 raise TypeError( 760 "listing ``IndexArray`` nodes is not allowed") 761 else: 762 class_ = get_class_by_name(classname) 763 764 children = self._v_children 765 childnames = sorted(children.keys()) 766 767 for childname in childnames: 768 childnode = children[childname] 769 if isinstance(childnode, class_): 770 yield childnode 771 772 773 def _f_walk_groups(self): 774 """Recursively iterate over descendent groups (not leaves). 775 776 This method starts by yielding *self*, and then it goes on to 777 recursively iterate over all child groups in alphanumerical order, top 778 to bottom (preorder), following the same procedure. 779 780 """ 781 782 self._g_check_open() 783 784 stack = [self] 785 yield self 786 # Iterate over the descendants 787 while stack: 788 objgroup = stack.pop() 789 groupnames = sorted(objgroup._v_groups.keys()) 790 # Sort the groups before delivering. This uses the groups names 791 # for groups in tree (in order to sort() can classify them). 792 for groupname in groupnames: 793 stack.append(objgroup._v_groups[groupname]) 794 yield objgroup._v_groups[groupname] 795 796 797 def __delattr__(self, name): 798 """Delete a Python attribute called name. 799 800 This method only provides a extra warning in case the user 801 tries to delete a children node using __delattr__. 802 803 To remove a children node from this group use 804 :meth:`File.remove_node` or :meth:`Node._f_remove`. To delete 805 a PyTables node attribute use :meth:`File.del_node_attr`, 806 :meth:`Node._f_delattr` or :attr:`Node._v_attrs``. 807 808 If there is an attribute and a child node with the same name, 809 the child node will be made accessible again via natural naming. 810 811 """ 812 813 try: 814 super(Group, self).__delattr__(name) # nothing particular 815 except AttributeError as ae: 816 hint = " (use ``node._f_remove()`` if you want to remove a node)" 817 raise ae.__class__(str(ae) + hint) 818 819 def __dir__(self): 820 """Autocomplete only children named as valid python identifiers. 821 822 Only PY3 supports this special method. 823 """ 824 subnods = [c for c in self._v_children if c.isidentifier()] 825 return super(Group, self).__dir__() + subnods 826 827 def __getattr__(self, name): 828 """Get a Python attribute or child node called name. 829 If the node has a child node called name it is returned, 830 else an AttributeError is raised. 831 """ 832 833 if name in self._c_lazy_children_attrs: 834 self._g_add_children_names() 835 return self.__dict__[name] 836 return self._f_get_child(name) 837 838 def __setattr__(self, name, value): 839 """Set a Python attribute called name with the given value. 840 841 This method stores an *ordinary Python attribute* in the object. It 842 does *not* store new children nodes under this group; for that, use the 843 File.create*() methods (see the File class 844 in :ref:`FileClassDescr`). It does *neither* store a PyTables node 845 attribute; for that, 846 use :meth:`File.set_node_attr`, :meth`:Node._f_setattr` 847 or :attr:`Node._v_attrs`. 848 849 If there is already a child node with the same name, a 850 NaturalNameWarning will be issued and the child node will not be 851 accessible via natural naming nor getattr(). It will still be available 852 via :meth:`File.get_node`, :meth:`Group._f_get_child` and children 853 dictionaries in the group (if visible). 854 855 """ 856 857 # Show a warning if there is an child node with that name. 858 # 859 # ..note:: 860 # 861 # Using ``if name in self:`` is not right since that would 862 # require ``_v_children`` and ``_v_hidden`` to be already set 863 # when the very first attribute assignments are made. 864 # Moreover, this warning is only concerned about clashes with 865 # names used in natural naming, i.e. those in ``__members__``. 866 # 867 # ..note:: 868 # 869 # The check ``'__members__' in myDict`` allows attribute 870 # assignment to happen before calling `Group.__init__()`, by 871 # avoiding to look into the still not assigned ``__members__`` 872 # attribute. This allows subclasses to set up some attributes 873 # and then call the constructor of the superclass. If the 874 # check above is disabled, that results in Python entering an 875 # endless loop on exit! 876 877 mydict = self.__dict__ 878 if '__members__' in mydict and name in self.__members__: 879 warnings.warn( 880 "group ``%s`` already has a child node named ``%s``; " 881 "you will not be able to use natural naming " 882 "to access the child node" 883 % (self._v_pathname, name), NaturalNameWarning) 884 885 super(Group, self).__setattr__(name, value) 886 887 def _f_flush(self): 888 """Flush this Group.""" 889 890 self._g_check_open() 891 self._g_flush_group() 892 893 def _g_close_descendents(self): 894 """Close all the *loaded* descendent nodes of this group.""" 895 896 node_manager = self._v_file._node_manager 897 node_manager.close_subtree(self._v_pathname) 898 899 900 def _g_close(self): 901 """Close this (open) group.""" 902 903 if self._v_isopen: 904 # hdf5extension operations: 905 # Close HDF5 group. 906 self._g_close_group() 907 908 # Close myself as a node. 909 super(Group, self)._f_close() 910 911 def _f_close(self): 912 """Close this group and all its descendents. 913 914 This method has the behavior described in :meth:`Node._f_close`. 915 It should be noted that this operation closes all the nodes 916 descending from this group. 917 918 You should not need to close nodes manually because they are 919 automatically opened/closed when they are loaded/evicted from 920 the integrated LRU cache. 921 922 """ 923 924 # If the group is already closed, return immediately 925 if not self._v_isopen: 926 return 927 928 # First, close all the descendents of this group, unless a) the 929 # group is being deleted (evicted from LRU cache) or b) the node 930 # is being closed during an aborted creation, in which cases 931 # this is not an explicit close issued by the user. 932 if not (self._v__deleting or self._v_objectid is None): 933 self._g_close_descendents() 934 935 # When all the descendents have been closed, close this group. 936 # This is done at the end because some nodes may still need to 937 # be loaded during the closing process; thus this node must be 938 # open until the very end. 939 self._g_close() 940 941 def _g_remove(self, recursive=False, force=False): 942 """Remove (recursively if needed) the Group. 943 944 This version correctly handles both visible and hidden nodes. 945 946 """ 947 948 if self._v_nchildren > 0: 949 if not (recursive or force): 950 raise NodeError("group ``%s`` has child nodes; " 951 "please set `recursive` or `force` to true " 952 "to remove it" 953 % (self._v_pathname,)) 954 955 # First close all the descendents hanging from this group, 956 # so that it is not possible to use a node that no longer exists. 957 self._g_close_descendents() 958 959 # Remove the node itself from the hierarchy. 960 super(Group, self)._g_remove(recursive, force) 961 962 def _f_copy(self, newparent=None, newname=None, 963 overwrite=False, recursive=False, createparents=False, 964 **kwargs): 965 """Copy this node and return the new one. 966 967 This method has the behavior described in :meth:`Node._f_copy`. 968 In addition, it recognizes the following keyword arguments: 969 970 Parameters 971 ---------- 972 title 973 The new title for the destination. If omitted or None, the 974 original title is used. This only applies to the topmost 975 node in recursive copies. 976 filters : Filters 977 Specifying this parameter overrides the original filter 978 properties in the source node. If specified, it must be an 979 instance of the Filters class (see :ref:`FiltersClassDescr`). 980 The default is to copy the filter properties from the source 981 node. 982 copyuserattrs 983 You can prevent the user attributes from being copied by setting 984 thisparameter to False. The default is to copy them. 985 stats 986 This argument may be used to collect statistics on the copy 987 process. When used, it should be a dictionary with keys 'groups', 988 'leaves', 'links' and 'bytes' having a numeric value. Their values 989 willbe incremented to reflect the number of groups, leaves and 990 bytes, respectively, that have been copied during the operation. 991 992 """ 993 994 return super(Group, self)._f_copy( 995 newparent, newname, 996 overwrite, recursive, createparents, **kwargs) 997 998 def _f_copy_children(self, dstgroup, overwrite=False, recursive=False, 999 createparents=False, **kwargs): 1000 """Copy the children of this group into another group. 1001 1002 Children hanging directly from this group are copied into dstgroup, 1003 which can be a Group (see :ref:`GroupClassDescr`) object or its 1004 pathname in string form. If createparents is true, the needed groups 1005 for the given destination group path to exist will be created. 1006 1007 The operation will fail with a NodeError if there is a child node 1008 in the destination group with the same name as one of the copied 1009 children from this one, unless overwrite is true; in this case, 1010 the former child node is recursively removed before copying the 1011 later. 1012 1013 By default, nodes descending from children groups of this node 1014 are not copied. If the recursive argument is true, all descendant 1015 nodes of this node are recursively copied. 1016 1017 Additional keyword arguments may be passed to customize the 1018 copying process. For instance, title and filters may be changed, 1019 user attributes may be or may not be copied, data may be sub-sampled, 1020 stats may be collected, etc. Arguments unknown to nodes are simply 1021 ignored. Check the documentation for copying operations of nodes to 1022 see which options they support. 1023 1024 """ 1025 1026 self._g_check_open() 1027 1028 # `dstgroup` is used instead of its path to avoid accepting 1029 # `Node` objects when `createparents` is true. Also, note that 1030 # there is no risk of creating parent nodes and failing later 1031 # because of destination nodes already existing. 1032 dstparent = self._v_file._get_or_create_path(dstgroup, createparents) 1033 self._g_check_group(dstparent) # Is it a group? 1034 1035 if not overwrite: 1036 # Abort as early as possible when destination nodes exist 1037 # and overwriting is not enabled. 1038 for childname in self._v_children: 1039 if childname in dstparent: 1040 raise NodeError( 1041 "destination group ``%s`` already has " 1042 "a node named ``%s``; " 1043 "you may want to use the ``overwrite`` argument" 1044 % (dstparent._v_pathname, childname)) 1045 1046 use_hardlinks = kwargs.get('use_hardlinks', False) 1047 if use_hardlinks: 1048 address_map = kwargs.setdefault('address_map', {}) 1049 1050 for child in self._v_children.values(): 1051 addr, rc = child._get_obj_info() 1052 if rc > 1 and addr in address_map: 1053 where, name = address_map[addr][0] 1054 localsrc = os.path.join(where, name) 1055 dstparent._v_file.create_hard_link(dstparent, child.name, 1056 localsrc) 1057 address_map[addr].append( 1058 (dstparent._v_pathname, child.name) 1059 ) 1060 1061 # Update statistics if needed. 1062 stats = kwargs.pop('stats', None) 1063 if stats is not None: 1064 stats['hardlinks'] += 1 1065 else: 1066 child._f_copy(dstparent, None, overwrite, recursive, 1067 **kwargs) 1068 if rc > 1: 1069 address_map[addr] = [ 1070 (dstparent._v_pathname, child.name) 1071 ] 1072 else: 1073 for child in self._v_children.values(): 1074 child._f_copy(dstparent, None, overwrite, recursive, **kwargs) 1075 1076 1077 def __str__(self): 1078 """Return a short string representation of the group. 1079 1080 Examples 1081 -------- 1082 1083 :: 1084 1085 >>> f=tables.open_file('data/test.h5') 1086 >>> print(f.root.group0) 1087 /group0 (Group) 'First Group' 1088 1089 """ 1090 1091 pathname = self._v_pathname 1092 classname = self.__class__.__name__ 1093 title = self._v_title 1094 return "%s (%s) %r" % (pathname, classname, title) 1095 1096 def __repr__(self): 1097 """Return a detailed string representation of the group. 1098 1099 Examples 1100 -------- 1101 1102 :: 1103 1104 >>> f = tables.open_file('data/test.h5') 1105 >>> f.root.group0 1106 /group0 (Group) 'First Group' 1107 children := ['tuple1' (Table), 'group1' (Group)] 1108 1109 """ 1110 1111 rep = [ 1112 '%r (%s)' % (childname, child.__class__.__name__) 1113 for (childname, child) in self._v_children.items() 1114 ] 1115 childlist = '[%s]' % (', '.join(rep)) 1116 1117 return "%s\n children := %s" % (str(self), childlist) 1118 1119 1120# Special definition for group root 1121class RootGroup(Group): 1122 1123 1124 def __init__(self, ptfile, name, title, new, filters): 1125 mydict = self.__dict__ 1126 1127 # Set group attributes. 1128 self._v_version = obversion 1129 self._v_new = new 1130 if new: 1131 self._v_new_title = title 1132 self._v_new_filters = filters 1133 else: 1134 self._v_new_title = None 1135 self._v_new_filters = None 1136 1137 # Set node attributes. 1138 self._v_file = ptfile 1139 self._v_isopen = True # root is always open 1140 self._v_pathname = '/' 1141 self._v_name = '/' 1142 self._v_depth = 0 1143 self._v_max_group_width = ptfile.params['MAX_GROUP_WIDTH'] 1144 self._v__deleting = False 1145 self._v_objectid = None # later 1146 1147 # Only the root node has the file as a parent. 1148 # Bypass __setattr__ to avoid the ``Node._v_parent`` property. 1149 mydict['_v_parent'] = ptfile 1150 ptfile._node_manager.register_node(self, '/') 1151 1152 # hdf5extension operations (do before setting an AttributeSet): 1153 # Update node attributes. 1154 self._g_new(ptfile, name, init=True) 1155 # Open the node and get its object ID. 1156 self._v_objectid = self._g_open() 1157 1158 # Set disk attributes and read children names. 1159 # 1160 # This *must* be postponed because this method needs the root node 1161 # to be created and bound to ``File.root``. 1162 # This is an exception to the rule, handled by ``File.__init()__``. 1163 # 1164 # self._g_post_init_hook() 1165 1166 def _g_load_child(self, childname): 1167 """Load a child node from disk. 1168 1169 The child node `childname` is loaded from disk and an adequate 1170 `Node` object is created and returned. If there is no such 1171 child, a `NoSuchNodeError` is raised. 1172 1173 """ 1174 1175 if self._v_file.root_uep != "/": 1176 childname = join_path(self._v_file.root_uep, childname) 1177 # Is the node a group or a leaf? 1178 node_type = self._g_check_has_child(childname) 1179 1180 # Nodes that HDF5 report as H5G_UNKNOWN 1181 if node_type == 'Unknown': 1182 return Unknown(self, childname) 1183 1184 # Guess the PyTables class suited to the node, 1185 # build a PyTables node and return it. 1186 if node_type == "Group": 1187 if self._v_file.params['PYTABLES_SYS_ATTRS']: 1188 ChildClass = self._g_get_child_group_class(childname) 1189 else: 1190 # Default is a Group class 1191 ChildClass = Group 1192 return ChildClass(self, childname, new=False) 1193 elif node_type == "Leaf": 1194 ChildClass = self._g_get_child_leaf_class(childname, warn=True) 1195 # Building a leaf may still fail because of unsupported types 1196 # and other causes. 1197 # return ChildClass(self, childname) # uncomment for debugging 1198 try: 1199 return ChildClass(self, childname) 1200 except Exception as exc: # XXX 1201 warnings.warn( 1202 "problems loading leaf ``%s``::\n\n" 1203 " %s\n\n" 1204 "The leaf will become an ``UnImplemented`` node." 1205 % (self._g_join(childname), exc)) 1206 # If not, associate an UnImplemented object to it 1207 return UnImplemented(self, childname) 1208 elif node_type == "SoftLink": 1209 return SoftLink(self, childname) 1210 elif node_type == "ExternalLink": 1211 return ExternalLink(self, childname) 1212 else: 1213 return UnImplemented(self, childname) 1214 1215 1216 def _f_rename(self, newname): 1217 raise NodeError("the root node can not be renamed") 1218 1219 def _f_move(self, newparent=None, newname=None, createparents=False): 1220 raise NodeError("the root node can not be moved") 1221 1222 def _f_remove(self, recursive=False): 1223 raise NodeError("the root node can not be removed") 1224 1225 1226class TransactionGroupG(NotLoggedMixin, Group): 1227 _c_classid = 'TRANSGROUP' 1228 1229 1230 def _g_width_warning(self): 1231 warnings.warn("""\ 1232the number of transactions is exceeding the recommended maximum (%d);\ 1233be ready to see PyTables asking for *lots* of memory and possibly slow I/O""" 1234 % (self._v_max_group_width,), PerformanceWarning) 1235 1236 1237 1238class TransactionG(NotLoggedMixin, Group): 1239 _c_classid = 'TRANSG' 1240 1241 1242 def _g_width_warning(self): 1243 warnings.warn("""\ 1244transaction ``%s`` is exceeding the recommended maximum number of marks (%d);\ 1245be ready to see PyTables asking for *lots* of memory and possibly slow I/O""" 1246 % (self._v_pathname, self._v_max_group_width), 1247 PerformanceWarning) 1248 1249 1250 1251class MarkG(NotLoggedMixin, Group): 1252 # Class identifier. 1253 _c_classid = 'MARKG' 1254 1255 1256 import re 1257 _c_shadow_name_re = re.compile(r'^a[0-9]+$') 1258 1259 def _g_width_warning(self): 1260 warnings.warn("""\ 1261mark ``%s`` is exceeding the recommended maximum action storage (%d nodes);\ 1262be ready to see PyTables asking for *lots* of memory and possibly slow I/O""" 1263 % (self._v_pathname, self._v_max_group_width), 1264 PerformanceWarning) 1265 1266 1267 def _g_reset(self): 1268 """Empty action storage (nodes and attributes). 1269 1270 This method empties all action storage kept in this node: nodes 1271 and attributes. 1272 1273 """ 1274 1275 # Remove action storage nodes. 1276 for child in list(self._v_children.values()): 1277 child._g_remove(True, True) 1278 1279 # Remove action storage attributes. 1280 attrs = self._v_attrs 1281 shname = self._c_shadow_name_re 1282 for attrname in attrs._v_attrnamesuser[:]: 1283 if shname.match(attrname): 1284 attrs._g__delattr(attrname) 1285 1286 1287## Local Variables: 1288## mode: python 1289## py-indent-offset: 4 1290## tab-width: 4 1291## fill-column: 72 1292## End: 1293