1class Horizon: 2 def __init__(self, mod): 3 self.mod = mod 4 self._hiding_tag_ = mod._hiding_tag_ 5 # Make preallocations of things that will be needed for news() 6 self.retset = self.mod.retset 7 self.hv = mod.hv 8 self.exc_info = self.mod._root.sys.exc_info 9 self.iso = self.mod.iso 10 str(self.retset(self.iso(1, [], (), {}, self.__dict__)) - 11 self.iso(())) 12 mod.hv.heap 13 mod.enter 14 mod.gc.collect() 15 self.hv_horizon = mod.heapyc.Horizon(self.hv) 16 17 def news(self): 18 r = self.retset(self.hv_horizon.news(self.mod.enter(self.hv.heap))) 19 return r 20 21 22class ClearCallback(object): 23 __slots__ = 'callback', 24 25 def __init__(self, callback): 26 self.callback = callback 27 28 def __call__(self, wr): 29 if self.callback is not None: 30 self.callback(wr) 31 else: 32 print('No callback') 33 34 35class Gchook_type(object): 36 __slots__ = 'x', '__weakref__', 'cb' 37 38 def __init__(g): 39 g.x = g 40 41 42class ObservationList(list): 43 __slots__ = '_hiding_tag_', 44 45 def __init__(self, iterable, hiding_tag): 46 list.__init__(self, iterable) 47 self._hiding_tag_ = hiding_tag 48 49 50class _GLUECLAMP_: 51 _imports_ = ( 52 '_parent.ImpSet:immnodeset', 53 '_parent.ImpSet:immnodeset_union', 54 '_parent.ImpSet:mutnodeset', 55 '_parent.ImpSet:NodeSet', 56 '_parent.UniSet:nodeset_adapt', 57 '_parent.UniSet:retset', 58 '_parent.Use:idset', 59 '_parent.Use:iso', 60 '_parent.Use:Type', 61 '_root:ctypes', 62 '_root:gc', 63 '_root:types', 64 ) 65 66 _chgable_ = ('is_rg_update_all', 'referrers_lock', '_is_clear_drg_enabled', 67 'root',) 68 _setable_ = ('_hiding_tag_', 'target', 'is_hiding_calling_interpreter',) 69 70 is_hiding_calling_interpreter = False 71 is_rg_update_all = False 72 _is_clear_drg_enabled = 1 # Flag mainly for test, Note Apr 19 2005 73 _hiding_tag_ = [] 74 75 #opt_rg_update_all = True 76 77 _uniset_exports = ( 78 # 'dominos', 79 # 'domisize', 80 'imdom', 81 # 'indisize', 82 # 'referents', 83 # 'referrers', 84 'referrers_gc', 85 ) 86 87 def _get__clear_hook(self): 88 return self.mutnodeset() 89 90 def clear_check(self): 91 ch = self._clear_hook 92 try: 93 wr = list(ch)[0] 94 except IndexError: 95 self.clear_setup() 96 else: 97 # When tracing is enabled, each local variable gets two references. 98 # See Guppy 3 issue #3. 99 c = [wr()] 100 if c[0] is None: 101 self.clear_setup() 102 elif self._root.sys.getrefcount(c[0]) > 3: 103 print('GC hook object was referred to from somebody!') 104 self.clear_callback(wr) 105 c[0].cb.callback = None 106 107 def clear_callback(self, wr): 108 self._clear_hook.clear() 109 for m in self.clear_methods: 110 m() 111 self.clear_setup() 112 113 def clear_setup(self): 114 ch = self._clear_hook 115 ch.clear() 116 c = self.gchook_type() 117 cb = self.ClearCallback(self.clear_callback) 118 c.cb = cb 119 ch.add(self._root.weakref.ref(c, cb)) 120 121 def _get_clear_methods(self): 122 return [] 123 124 def clear_register_method(self, m): 125 self.clear_methods.append(m) 126 self.clear_check() 127 128 def _get_dict_ownership(self): 129 drg = self.nodegraph() 130 131 def clear_drg(): 132 if drg.is_sorted and self._is_clear_drg_enabled: 133 drg.clear() 134 else: 135 pass 136 self.clear_register_method(clear_drg) 137 return drg 138 139 def _get_gchook_type(self): 140 return Gchook_type 141 142 def _get_capsule_type(self): 143 return self.ctypes.cast( 144 self.ctypes.pythonapi.PyCapsule_Type, self.ctypes.py_object).value 145 146 def _get_heapdef_modules(self): 147 # We touch self.heapyc to import it & its dependent guppy.sets; 148 # this is kinda specialcase-hacky but see Notes Apr 8 2005. 149 self.heapyc 150 return list(self.target.sys.modules.items()) 151 152 def _get_heapdefs(self): 153 heapdefs = [] 154 for n, m in self.heapdef_modules: 155 try: 156 hd = getattr(m, '_NyHeapDefs_') 157 except AttributeError: 158 continue 159 if not hd or not isinstance(hd, self.capsule_type): 160 continue 161 heapdefs.append(hd) 162 return tuple(heapdefs) 163 164 def _get_heapyc(self): return self._parent.heapyc 165 166 def _get_hv(self): 167 hv = self.new_hv(_hiding_tag_=self._hiding_tag_, 168 is_hiding_calling_interpreter=self.is_hiding_calling_interpreter) 169 return hv 170 171 def _get_norefer(self): return self.mutnodeset() 172 173 def _get_referrers_targets(self): return [] 174 175 def _get_rg(self): 176 rg = self.nodegraph() 177 self.clear_register_method(self._clear_rg) 178 return rg 179 180 def _clear_rg(self): 181 if self.referrers_lock: 182 return 183 rg = self.rg 184 if rg.is_sorted: 185 rg.clear() 186 self.norefer.clear() 187 else: 188 pass 189 190 def _get_referrers_lock(self): return 0 191 192 def _get_root(self): return self.heapyc.RootState 193 def _get_target(self): return self._parent.Target.Target(self._hiding_tag_) 194 195 def _set_root(self, root): 196 self.clear_retainers() 197 self.hv.root = root 198 199 def call_with_referrers(self, X, f): 200 self.referrers_lock += 1 201 try: 202 self.update_referrers(X) 203 return f(X) 204 finally: 205 self.referrers_lock -= 1 206 207 def clear_retainers(self): 208 """G.clear_retainers() 209Clear the retainer graph V.rg. 210""" 211 self.rg.clear() 212 self.norefer.clear() 213 214 def dominos(self, X): 215 """dominos(X) -> idset 216Return the dominos of a set of objects X. The dominos of X is the set 217of objects that are dominated by X, which is the objects that will become 218deallocated, directly or indirectly, when the objects in X are deallocated.""" 219 return self.dominos_tuple((X,))[0] 220 221 def dominos_tuple(self, X): 222 """V.dominos_tuple(X) -> tuple of idsets 223Return a tuple of dominos for the tuple of sets of objects X.""" 224 D_ = [self.nodeset_adapt(x) 225 for x in X] # Convert to naming like in the appendix 226 T = self.hv.reachable 227 S = self.immnodeset([self.root]) 228 D = self.immnodeset_union(D_) 229 W = T(S, D) 230 return tuple([self.retset(T(Di, W) - T(D, W | Di)) for Di in D_]) 231 232 def domisize(self, X): 233 """domisize(X) -> int 234Return the dominated size of a set of objects X. The dominated size of X 235is the total size of memory that will become deallocated, directly or 236indirectly, when the objects in X are deallocated. See also: indisize.""" 237 238 return self.domisize_tuple((X,))[0] 239 240 def domisize_tuple(self, X): 241 """"V.domisize_tuple(X) -> tuple of ints 242Return a tuple of dominated sizes for the tuple of sets of objects X.""" 243 return tuple([self.indisize(dominos_i) 244 for dominos_i in self.dominos_tuple(X)]) 245 246 def enter(self, func): 247 if self.hv.is_hiding_calling_interpreter: 248 self.hv.limitframe = None 249 elif self.hv.limitframe is not None: 250 return func() 251 else: 252 import inspect 253 self.hv.limitframe = inspect.currentframe().f_back.f_back 254 255 try: 256 retval = func() 257 finally: 258 self.hv.limitframe = None 259 return retval 260 261 def gchook(self, func): 262 c = self.gchook_type() 263 ho = self.mutnodeset() 264 265 def cb(wr): 266 func() 267 ho.clear() 268 c = self.gchook_type() 269 ho.add(self._root.weakref.ref(c, cb)) 270 271 ho.add(self._root.weakref.ref(c, cb)) 272 return self.mutnodeset([ho]) 273 274 def heapg(self, rma=1): 275 # Almost the same as gc.get_objects(), 276 # except: 277 # 1. calls gc.collect() first (twice) 278 # 2. removes objects of type gchook 279 # 3. removes objects of type ClearCallback 280 # 4. removes all objects of type types.FrameType 281 # 5. removes all objects of weakref type 282 # 6. If rma = 1, 283 # removes all that is in the reachable heap 284 # except what is in the set itself. 285 286 # . wraps the result in an IdSet 287 288 self.gc.collect() 289 self.gc.collect() 290 objs = self.gc.get_objects() 291 cli = self.hv.cli_type() 292 objs = cli.select(objs, self.gchook_type, '!=') 293 objs = cli.select(objs, ClearCallback, '!=') 294 objs = cli.select(objs, self._root.types.FrameType, '!=') 295 objs = cli.select(objs, self._root.weakref.ReferenceType, '!=') 296 r = self.retset(objs) 297 del cli, objs 298 299 if rma: 300 r = (r - self.idset(self.heapyc.HeapView( 301 self.heapyc.RootState, 302 self.heapdefs 303 ).reachable_x( 304 self.immnodeset([self.heapyc.RootState]), 305 self.observation_containers() 306 )) 307 ) 308 309 return r 310 311 def heapu(self, rma=1): 312 self.gc.collect() 313 self.gc.collect() 314 r = self.gc.get_objects() 315 316 exclude = (self.Type(self.gchook_type) | 317 self.Type(ClearCallback) 318 ) 319 320 if rma: 321 exclude |= self.idset(self.heapyc.HeapView( 322 self.heapyc.RootState, 323 self.heapdefs 324 ).reachable_x( 325 self.immnodeset([self.heapyc.RootState]), 326 self.immnodeset([r]) 327 )) 328 329 r = self.retset(r) - exclude 330 ref = r.referents - exclude 331 while not ref <= r: 332 r |= ref 333 ref = ref.referents - exclude 334 335 del ref, exclude 336 337 r = r.bytype # Avoid memoizing for complicated classification 338 return r 339 340 def heap(self): 341 """V.heap() -> idset 342Return the set of objects in the visible heap. 343""" 344 global heap_one_time_initialized 345 # This is to make sure that the first time called 346 # the heap will contain things that may likely be loaded later 347 # because of common operations. 348 if not heap_one_time_initialized: 349 old_root = self.root 350 objs = [[], 'a', 1, 1.23, {'a': 'b'}, self] 351 352 self.root = objs 353 try: 354 repr(self.idset(objs)) 355 repr(self.iso(objs[0]).shpaths) 356 repr(self.iso(objs[0]).rp) 357 finally: 358 self.root = old_root 359 360 del objs 361 del old_root 362 heap_one_time_initialized = True 363 364 self.gc.collect() # Sealing a leak at particular usage ; Notes Apr 13 2005 365 # Exclude current frame by encapsulting in enter(). Note Apr 20 2005 366 return self.enter(lambda: 367 self.idset(self.hv.heap())) 368 369 def horizon(self): 370 return self.Horizon(self) 371 372 def imdom(self, X): 373 """imdom(X) -> idset 374Return the immediate dominators of a set of objects X. The immediate 375dominators is a subset of the referrers. It includes only those 376referrers that are reachable directly, avoiding any other referrer.""" 377 pred = self.nodeset_adapt(self.referrers(X)) 378 visit = self.hv.reachable_x(self.immnodeset([self.root]), pred) 379 return self.retset(pred & visit) 380 381 def indisize(self, X): 382 """indisize(X) -> int 383Return the sum of the individual sizes of the set of objects X. 384The individual size of an object is the size of memory that is 385allocated directly in the object, not including any externally 386visible subobjects. See also: domisize.""" 387 return self.hv.indisize_sum(self.nodeset_adapt(X)) 388 389 def new_hv(self, _hiding_tag_=None, is_hiding_calling_interpreter=False, 390 heapdefs=None, root=None, gchook_type=None): 391 if heapdefs is None: 392 heapdefs = self.heapdefs 393 if root is None: 394 root = self.root 395 if gchook_type is None: 396 gchook_type = self.gchook_type 397 hv = self.heapyc.HeapView(root, heapdefs) 398 hv._hiding_tag_ = _hiding_tag_ 399 hv.is_hiding_calling_interpreter = is_hiding_calling_interpreter 400 hv.register_hidden_exact_type(gchook_type) 401 # hv.register__hiding_tag__type(self._parent.UniSet.UniSet) 402 hv.register__hiding_tag__type(self._parent.UniSet.Kind) 403 hv.register__hiding_tag__type(self._parent.UniSet.IdentitySetMulti) 404 hv.register__hiding_tag__type(self._parent.UniSet.IdentitySetSingleton) 405 406 return hv 407 408 def nodegraph(self, iterable=None, is_mapping=False): 409 ng = self.heapyc.NodeGraph(iterable, is_mapping) 410 ng._hiding_tag_ = self._hiding_tag_ 411 return ng 412 413 def obj_at(self, addr): 414 try: 415 return self.immnodeset(self.hv.static_types).obj_at(addr) 416 except ValueError: 417 pass 418 try: 419 return self.immnodeset(self.gc.get_objects()).obj_at(addr) 420 except ValueError: 421 pass 422 try: 423 return self.immnodeset(self.hv.heap()).obj_at(addr) 424 except ValueError: 425 raise ValueError('No object found at address %s' % hex(addr)) 426 427 def observation_containers(self): 428 # Return the current set of 'observation containers' 429 # as discussed in Notes Oct 27 2005. 430 # returns a nodeset, not an idset, to avoid recursive referenes 431 432 objs = self.gc.get_objects() 433 cli = self.hv.cli_type() 434 objs = (cli.select(objs, self.NodeSet, '<=') + 435 cli.select(objs, ObservationList, '<=') + 436 cli.select( 437 objs, self._parent.UniSet.IdentitySetSingleton, '<=') 438 ) 439 r = self.immnodeset([x for x in objs if getattr( 440 x, '_hiding_tag_', None) is self._hiding_tag_]) 441 del cli, objs 442 return r 443 444 def observation_list(self, iterable=()): 445 # Return an ObservationList object with our _hiding_tag_ 446 return ObservationList(iterable, self._hiding_tag_) 447 448 def referents(self, X): 449 """V.referents(X) -> idset 450Return the set of objects that are directly referred to by 451any of the objects in the set X.""" 452 return self.retset(self.hv.relimg(self.nodeset_adapt(X))) 453 454 def referrers(self, X): 455 """V.referrers(X) -> idset 456Return the set of objects that directly refer to 457any of the objects in the set X.""" 458 459 X = self.nodeset_adapt(X) 460 if self.is_rg_update_all and self.root is self.heapyc.RootState: 461 if not (self.rg.domain_covers(X) or 462 self.rg.domain_covers(X - self.norefer)): 463 self.rg.clear() 464 import gc 465 gc.collect() 466 self.hv.update_referrers_completely(self.rg) 467 addnoref = X - self.rg.get_domain() 468 self.norefer |= addnoref 469 else: 470 Y = self.mutnodeset(X) 471 Y -= self.norefer 472 if not self.rg.domain_covers(Y): 473 for wt in self.referrers_targets: 474 t = wt() 475 if t is not None: 476 Y |= t.set.nodes 477 Y |= self.rg.get_domain() 478 self.rg.clear() 479 self.hv.update_referrers(self.rg, Y) 480 self.norefer.clear() 481 self.norefer |= (X | Y | self.rg.get_range()) 482 self.norefer -= self.rg.get_domain() 483 Y = self.mutnodeset(X) - self.norefer 484 if not self.rg.domain_covers(Y): 485 print('update_referrers failed') 486 print('Y - domain of rg:') 487 print(self.idset(Y - self.rg.get_domain())) 488 489 Y = None 490 491 X = self.rg.relimg(X) 492 X = self.immnodeset(X) - [None] 493 X = self.retset(X) 494 return X 495 496 def referrers_gc(self, X): 497 """V.referrers_gc(X) -> idset 498Return the set of objects that directly refer to 499any of the objects in the set X. 500This differs from referrers in that it uses the 501gc module's view of the referrers. This is more or less 502valid depending on viewpoint. 503 504""" 505 X = tuple(self.nodeset_adapt(X)) 506 return self.idset(self.gc.get_referrers(*X)) - self.iso(X) 507 508 def referrers_add_target(self, t): 509 def remove(wr): 510 self.referrers_targets.remove(wr) 511 wr = self._root.weakref.ref(t, remove) 512 self.referrers_targets.append(wr) 513 514 def update_referrers(self, X): 515 """V.update_referrers(X) 516Update the view V from the set X. X must be adaptable to NodeSet. V.rg is 517updated so that in addition to its previos mapping, it will also contain 518mappings for the elements of X to their referrers, from them to their 519referrers and so on. 520""" 521 self.referrers(X) 522 523 524def prime_builtin_types(): 525 # Make sure builtin types have been completely allocated 526 # with all method descriptors etc. 527 # so subsequent events will not give spurios confusing allocations. 528 # This should need to be done only once. 529 # (Or whenever a new (extension) module is imported??) 530 # The problem & solution is further discussed in Notes Nov 9 2005. 531 532 import types 533 import guppy.heapy.heapyc 534 import guppy.sets.setsc 535 import os 536 import sys 537 import warnings 538 import weakref 539 540 for mod in list(sys.modules.values()): 541 if mod is None or getattr(mod, '__dict__', None) is None: 542 continue 543 for t in list(mod.__dict__.values()): 544 if isinstance(t, type): 545 dir(t) 546 # Other type(s) 547 for t in [type(iter([])), type(iter(())), 548 ]: 549 dir(t) 550 551 # Ubuntu apport package installs a sys.excepthook that will import a lot 552 # of packages when there is an unhandled exception. This can cause a lot 553 # of irrelevant stuffs when we analyze a relative heap. 554 try: 555 import apport.fileutils 556 except ImportError: 557 pass 558 else: 559 import re 560 import traceback 561 562 if 'pythoncom' in sys.modules: 563 def get_pywin32_ver(): 564 try: 565 import pkg_resources 566 567 return pkg_resources.get_distribution('pywin32').version 568 except Exception: 569 pass 570 571 try: 572 import distutils.sysconfig 573 574 site_pkg = distutils.sysconfig.get_python_lib(plat_specific=1) 575 with open(os.path.join(site_pkg, 'pywin32.version.txt')) as f: 576 return f.read().strip() 577 except Exception: 578 pass 579 580 return None 581 582 pywin32_ver = get_pywin32_ver() 583 584 if pywin32_ver: 585 try: 586 pywin32_ver = int(pywin32_ver) 587 except ValueError: 588 pass 589 else: 590 if pywin32_ver < 300: 591 warnings.warn( 592 'pythoncom in pywin32 < 300 may cause crashes. See ' 593 'https://github.com/zhuyifei1999/guppy3/issues/25. ' 594 'You may want to upgrade to the newest version of ' 595 'pywin32 by running "pip install pywin32 --upgrade"') 596 597 598prime_builtin_types() 599 600# The following global variable is used by heap() 601# to do extra initializations the first time it is called. 602# having to do that we want to do import and init things 603# but only if heap is actually called 604 605heap_one_time_initialized = False 606