1""" 2Customisable progressbar decorator for iterators. 3Includes a default `range` iterator printing to `stderr`. 4 5Usage: 6>>> from tqdm import trange, tqdm 7>>> for i in trange(10): 8... ... 9""" 10from __future__ import absolute_import, division 11 12import sys 13from collections import OrderedDict, defaultdict 14from contextlib import contextmanager 15from datetime import datetime, timedelta 16from numbers import Number 17from time import time 18from warnings import warn 19from weakref import WeakSet 20 21from ._monitor import TMonitor 22from .utils import ( 23 CallbackIOWrapper, Comparable, DisableOnWriteError, FormatReplace, SimpleTextIOWrapper, 24 _basestring, _is_ascii, _range, _screen_shape_wrapper, _supports_unicode, _term_move_up, 25 _unich, _unicode, disp_len, disp_trim) 26 27__author__ = "https://github.com/tqdm/tqdm#contributions" 28__all__ = ['tqdm', 'trange', 29 'TqdmTypeError', 'TqdmKeyError', 'TqdmWarning', 30 'TqdmExperimentalWarning', 'TqdmDeprecationWarning', 31 'TqdmMonitorWarning'] 32 33 34class TqdmTypeError(TypeError): 35 pass 36 37 38class TqdmKeyError(KeyError): 39 pass 40 41 42class TqdmWarning(Warning): 43 """base class for all tqdm warnings. 44 45 Used for non-external-code-breaking errors, such as garbled printing. 46 """ 47 def __init__(self, msg, fp_write=None, *a, **k): 48 if fp_write is not None: 49 fp_write("\n" + self.__class__.__name__ + ": " + str(msg).rstrip() + '\n') 50 else: 51 super(TqdmWarning, self).__init__(msg, *a, **k) 52 53 54class TqdmExperimentalWarning(TqdmWarning, FutureWarning): 55 """beta feature, unstable API and behaviour""" 56 pass 57 58 59class TqdmDeprecationWarning(TqdmWarning, DeprecationWarning): 60 # not suppressed if raised 61 pass 62 63 64class TqdmMonitorWarning(TqdmWarning, RuntimeWarning): 65 """tqdm monitor errors which do not affect external functionality""" 66 pass 67 68 69def TRLock(*args, **kwargs): 70 """threading RLock""" 71 try: 72 from threading import RLock 73 return RLock(*args, **kwargs) 74 except (ImportError, OSError): # pragma: no cover 75 pass 76 77 78class TqdmDefaultWriteLock(object): 79 """ 80 Provide a default write lock for thread and multiprocessing safety. 81 Works only on platforms supporting `fork` (so Windows is excluded). 82 You must initialise a `tqdm` or `TqdmDefaultWriteLock` instance 83 before forking in order for the write lock to work. 84 On Windows, you need to supply the lock from the parent to the children as 85 an argument to joblib or the parallelism lib you use. 86 """ 87 # global thread lock so no setup required for multithreading. 88 # NB: Do not create multiprocessing lock as it sets the multiprocessing 89 # context, disallowing `spawn()`/`forkserver()` 90 th_lock = TRLock() 91 92 def __init__(self): 93 # Create global parallelism locks to avoid racing issues with parallel 94 # bars works only if fork available (Linux/MacOSX, but not Windows) 95 cls = type(self) 96 root_lock = cls.th_lock 97 if root_lock is not None: 98 root_lock.acquire() 99 cls.create_mp_lock() 100 self.locks = [lk for lk in [cls.mp_lock, cls.th_lock] if lk is not None] 101 if root_lock is not None: 102 root_lock.release() 103 104 def acquire(self, *a, **k): 105 for lock in self.locks: 106 lock.acquire(*a, **k) 107 108 def release(self): 109 for lock in self.locks[::-1]: # Release in inverse order of acquisition 110 lock.release() 111 112 def __enter__(self): 113 self.acquire() 114 115 def __exit__(self, *exc): 116 self.release() 117 118 @classmethod 119 def create_mp_lock(cls): 120 if not hasattr(cls, 'mp_lock'): 121 try: 122 from multiprocessing import RLock 123 cls.mp_lock = RLock() 124 except (ImportError, OSError): # pragma: no cover 125 cls.mp_lock = None 126 127 @classmethod 128 def create_th_lock(cls): 129 assert hasattr(cls, 'th_lock') 130 warn("create_th_lock not needed anymore", TqdmDeprecationWarning, stacklevel=2) 131 132 133class Bar(object): 134 """ 135 `str.format`-able bar with format specifiers: `[width][type]` 136 137 - `width` 138 + unspecified (default): use `self.default_len` 139 + `int >= 0`: overrides `self.default_len` 140 + `int < 0`: subtract from `self.default_len` 141 - `type` 142 + `a`: ascii (`charset=self.ASCII` override) 143 + `u`: unicode (`charset=self.UTF` override) 144 + `b`: blank (`charset=" "` override) 145 """ 146 ASCII = " 123456789#" 147 UTF = u" " + u''.join(map(_unich, range(0x258F, 0x2587, -1))) 148 BLANK = " " 149 COLOUR_RESET = '\x1b[0m' 150 COLOUR_RGB = '\x1b[38;2;%d;%d;%dm' 151 COLOURS = {'BLACK': '\x1b[30m', 'RED': '\x1b[31m', 'GREEN': '\x1b[32m', 152 'YELLOW': '\x1b[33m', 'BLUE': '\x1b[34m', 'MAGENTA': '\x1b[35m', 153 'CYAN': '\x1b[36m', 'WHITE': '\x1b[37m'} 154 155 def __init__(self, frac, default_len=10, charset=UTF, colour=None): 156 if not 0 <= frac <= 1: 157 warn("clamping frac to range [0, 1]", TqdmWarning, stacklevel=2) 158 frac = max(0, min(1, frac)) 159 assert default_len > 0 160 self.frac = frac 161 self.default_len = default_len 162 self.charset = charset 163 self.colour = colour 164 165 @property 166 def colour(self): 167 return self._colour 168 169 @colour.setter 170 def colour(self, value): 171 if not value: 172 self._colour = None 173 return 174 try: 175 if value.upper() in self.COLOURS: 176 self._colour = self.COLOURS[value.upper()] 177 elif value[0] == '#' and len(value) == 7: 178 self._colour = self.COLOUR_RGB % tuple( 179 int(i, 16) for i in (value[1:3], value[3:5], value[5:7])) 180 else: 181 raise KeyError 182 except (KeyError, AttributeError): 183 warn("Unknown colour (%s); valid choices: [hex (#00ff00), %s]" % ( 184 value, ", ".join(self.COLOURS)), 185 TqdmWarning, stacklevel=2) 186 self._colour = None 187 188 def __format__(self, format_spec): 189 if format_spec: 190 _type = format_spec[-1].lower() 191 try: 192 charset = {'a': self.ASCII, 'u': self.UTF, 'b': self.BLANK}[_type] 193 except KeyError: 194 charset = self.charset 195 else: 196 format_spec = format_spec[:-1] 197 if format_spec: 198 N_BARS = int(format_spec) 199 if N_BARS < 0: 200 N_BARS += self.default_len 201 else: 202 N_BARS = self.default_len 203 else: 204 charset = self.charset 205 N_BARS = self.default_len 206 207 nsyms = len(charset) - 1 208 bar_length, frac_bar_length = divmod(int(self.frac * N_BARS * nsyms), nsyms) 209 210 res = charset[-1] * bar_length 211 if bar_length < N_BARS: # whitespace padding 212 res = res + charset[frac_bar_length] + charset[0] * (N_BARS - bar_length - 1) 213 return self.colour + res + self.COLOUR_RESET if self.colour else res 214 215 216class EMA(object): 217 """ 218 Exponential moving average: smoothing to give progressively lower 219 weights to older values. 220 221 Parameters 222 ---------- 223 smoothing : float, optional 224 Smoothing factor in range [0, 1], [default: 0.3]. 225 Increase to give more weight to recent values. 226 Ranges from 0 (yields old value) to 1 (yields new value). 227 """ 228 def __init__(self, smoothing=0.3): 229 self.alpha = smoothing 230 self.last = 0 231 self.calls = 0 232 233 def __call__(self, x=None): 234 """ 235 Parameters 236 ---------- 237 x : float 238 New value to include in EMA. 239 """ 240 beta = 1 - self.alpha 241 if x is not None: 242 self.last = self.alpha * x + beta * self.last 243 self.calls += 1 244 return self.last / (1 - beta ** self.calls) if self.calls else self.last 245 246 247class tqdm(Comparable): 248 """ 249 Decorate an iterable object, returning an iterator which acts exactly 250 like the original iterable, but prints a dynamically updating 251 progressbar every time a value is requested. 252 """ 253 254 monitor_interval = 10 # set to 0 to disable the thread 255 monitor = None 256 _instances = WeakSet() 257 258 @staticmethod 259 def format_sizeof(num, suffix='', divisor=1000): 260 """ 261 Formats a number (greater than unity) with SI Order of Magnitude 262 prefixes. 263 264 Parameters 265 ---------- 266 num : float 267 Number ( >= 1) to format. 268 suffix : str, optional 269 Post-postfix [default: '']. 270 divisor : float, optional 271 Divisor between prefixes [default: 1000]. 272 273 Returns 274 ------- 275 out : str 276 Number with Order of Magnitude SI unit postfix. 277 """ 278 for unit in ['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z']: 279 if abs(num) < 999.5: 280 if abs(num) < 99.95: 281 if abs(num) < 9.995: 282 return '{0:1.2f}'.format(num) + unit + suffix 283 return '{0:2.1f}'.format(num) + unit + suffix 284 return '{0:3.0f}'.format(num) + unit + suffix 285 num /= divisor 286 return '{0:3.1f}Y'.format(num) + suffix 287 288 @staticmethod 289 def format_interval(t): 290 """ 291 Formats a number of seconds as a clock time, [H:]MM:SS 292 293 Parameters 294 ---------- 295 t : int 296 Number of seconds. 297 298 Returns 299 ------- 300 out : str 301 [H:]MM:SS 302 """ 303 mins, s = divmod(int(t), 60) 304 h, m = divmod(mins, 60) 305 if h: 306 return '{0:d}:{1:02d}:{2:02d}'.format(h, m, s) 307 else: 308 return '{0:02d}:{1:02d}'.format(m, s) 309 310 @staticmethod 311 def format_num(n): 312 """ 313 Intelligent scientific notation (.3g). 314 315 Parameters 316 ---------- 317 n : int or float or Numeric 318 A Number. 319 320 Returns 321 ------- 322 out : str 323 Formatted number. 324 """ 325 f = '{0:.3g}'.format(n).replace('+0', '+').replace('-0', '-') 326 n = str(n) 327 return f if len(f) < len(n) else n 328 329 @staticmethod 330 def status_printer(file): 331 """ 332 Manage the printing and in-place updating of a line of characters. 333 Note that if the string is longer than a line, then in-place 334 updating may not work (it will print a new line at each refresh). 335 """ 336 fp = file 337 fp_flush = getattr(fp, 'flush', lambda: None) # pragma: no cover 338 if fp in (sys.stderr, sys.stdout): 339 sys.stderr.flush() 340 sys.stdout.flush() 341 342 def fp_write(s): 343 fp.write(_unicode(s)) 344 fp_flush() 345 346 last_len = [0] 347 348 def print_status(s): 349 len_s = disp_len(s) 350 fp_write('\r' + s + (' ' * max(last_len[0] - len_s, 0))) 351 last_len[0] = len_s 352 353 return print_status 354 355 @staticmethod 356 def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, unit='it', 357 unit_scale=False, rate=None, bar_format=None, postfix=None, 358 unit_divisor=1000, initial=0, colour=None, **extra_kwargs): 359 """ 360 Return a string-based progress bar given some parameters 361 362 Parameters 363 ---------- 364 n : int or float 365 Number of finished iterations. 366 total : int or float 367 The expected total number of iterations. If meaningless (None), 368 only basic progress statistics are displayed (no ETA). 369 elapsed : float 370 Number of seconds passed since start. 371 ncols : int, optional 372 The width of the entire output message. If specified, 373 dynamically resizes `{bar}` to stay within this bound 374 [default: None]. If `0`, will not print any bar (only stats). 375 The fallback is `{bar:10}`. 376 prefix : str, optional 377 Prefix message (included in total width) [default: '']. 378 Use as {desc} in bar_format string. 379 ascii : bool, optional or str, optional 380 If not set, use unicode (smooth blocks) to fill the meter 381 [default: False]. The fallback is to use ASCII characters 382 " 123456789#". 383 unit : str, optional 384 The iteration unit [default: 'it']. 385 unit_scale : bool or int or float, optional 386 If 1 or True, the number of iterations will be printed with an 387 appropriate SI metric prefix (k = 10^3, M = 10^6, etc.) 388 [default: False]. If any other non-zero number, will scale 389 `total` and `n`. 390 rate : float, optional 391 Manual override for iteration rate. 392 If [default: None], uses n/elapsed. 393 bar_format : str, optional 394 Specify a custom bar string formatting. May impact performance. 395 [default: '{l_bar}{bar}{r_bar}'], where 396 l_bar='{desc}: {percentage:3.0f}%|' and 397 r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, ' 398 '{rate_fmt}{postfix}]' 399 Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, 400 percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, 401 rate, rate_fmt, rate_noinv, rate_noinv_fmt, 402 rate_inv, rate_inv_fmt, postfix, unit_divisor, 403 remaining, remaining_s, eta. 404 Note that a trailing ": " is automatically removed after {desc} 405 if the latter is empty. 406 postfix : *, optional 407 Similar to `prefix`, but placed at the end 408 (e.g. for additional stats). 409 Note: postfix is usually a string (not a dict) for this method, 410 and will if possible be set to postfix = ', ' + postfix. 411 However other types are supported (#382). 412 unit_divisor : float, optional 413 [default: 1000], ignored unless `unit_scale` is True. 414 initial : int or float, optional 415 The initial counter value [default: 0]. 416 colour : str, optional 417 Bar colour (e.g. 'green', '#00ff00'). 418 419 Returns 420 ------- 421 out : Formatted meter and stats, ready to display. 422 """ 423 424 # sanity check: total 425 if total and n >= (total + 0.5): # allow float imprecision (#849) 426 total = None 427 428 # apply custom scale if necessary 429 if unit_scale and unit_scale not in (True, 1): 430 if total: 431 total *= unit_scale 432 n *= unit_scale 433 if rate: 434 rate *= unit_scale # by default rate = self.avg_dn / self.avg_dt 435 unit_scale = False 436 437 elapsed_str = tqdm.format_interval(elapsed) 438 439 # if unspecified, attempt to use rate = average speed 440 # (we allow manual override since predicting time is an arcane art) 441 if rate is None and elapsed: 442 rate = (n - initial) / elapsed 443 inv_rate = 1 / rate if rate else None 444 format_sizeof = tqdm.format_sizeof 445 rate_noinv_fmt = ((format_sizeof(rate) if unit_scale else 446 '{0:5.2f}'.format(rate)) if rate else '?') + unit + '/s' 447 rate_inv_fmt = ( 448 (format_sizeof(inv_rate) if unit_scale else '{0:5.2f}'.format(inv_rate)) 449 if inv_rate else '?') + 's/' + unit 450 rate_fmt = rate_inv_fmt if inv_rate and inv_rate > 1 else rate_noinv_fmt 451 452 if unit_scale: 453 n_fmt = format_sizeof(n, divisor=unit_divisor) 454 total_fmt = format_sizeof(total, divisor=unit_divisor) if total is not None else '?' 455 else: 456 n_fmt = str(n) 457 total_fmt = str(total) if total is not None else '?' 458 459 try: 460 postfix = ', ' + postfix if postfix else '' 461 except TypeError: 462 pass 463 464 remaining = (total - n) / rate if rate and total else 0 465 remaining_str = tqdm.format_interval(remaining) if rate else '?' 466 try: 467 eta_dt = (datetime.now() + timedelta(seconds=remaining) 468 if rate and total else datetime.utcfromtimestamp(0)) 469 except OverflowError: 470 eta_dt = datetime.max 471 472 # format the stats displayed to the left and right sides of the bar 473 if prefix: 474 # old prefix setup work around 475 bool_prefix_colon_already = (prefix[-2:] == ": ") 476 l_bar = prefix if bool_prefix_colon_already else prefix + ": " 477 else: 478 l_bar = '' 479 480 r_bar = '| {0}/{1} [{2}<{3}, {4}{5}]'.format( 481 n_fmt, total_fmt, elapsed_str, remaining_str, rate_fmt, postfix) 482 483 # Custom bar formatting 484 # Populate a dict with all available progress indicators 485 format_dict = dict( 486 # slight extension of self.format_dict 487 n=n, n_fmt=n_fmt, total=total, total_fmt=total_fmt, 488 elapsed=elapsed_str, elapsed_s=elapsed, 489 ncols=ncols, desc=prefix or '', unit=unit, 490 rate=inv_rate if inv_rate and inv_rate > 1 else rate, 491 rate_fmt=rate_fmt, rate_noinv=rate, 492 rate_noinv_fmt=rate_noinv_fmt, rate_inv=inv_rate, 493 rate_inv_fmt=rate_inv_fmt, 494 postfix=postfix, unit_divisor=unit_divisor, 495 colour=colour, 496 # plus more useful definitions 497 remaining=remaining_str, remaining_s=remaining, 498 l_bar=l_bar, r_bar=r_bar, eta=eta_dt, 499 **extra_kwargs) 500 501 # total is known: we can predict some stats 502 if total: 503 # fractional and percentage progress 504 frac = n / total 505 percentage = frac * 100 506 507 l_bar += '{0:3.0f}%|'.format(percentage) 508 509 if ncols == 0: 510 return l_bar[:-1] + r_bar[1:] 511 512 format_dict.update(l_bar=l_bar) 513 if bar_format: 514 format_dict.update(percentage=percentage) 515 516 # auto-remove colon for empty `desc` 517 if not prefix: 518 bar_format = bar_format.replace("{desc}: ", '') 519 else: 520 bar_format = "{l_bar}{bar}{r_bar}" 521 522 full_bar = FormatReplace() 523 try: 524 nobar = bar_format.format(bar=full_bar, **format_dict) 525 except UnicodeEncodeError: 526 bar_format = _unicode(bar_format) 527 nobar = bar_format.format(bar=full_bar, **format_dict) 528 if not full_bar.format_called: 529 # no {bar}, we can just format and return 530 return nobar 531 532 # Formatting progress bar space available for bar's display 533 full_bar = Bar(frac, 534 max(1, ncols - disp_len(nobar)) if ncols else 10, 535 charset=Bar.ASCII if ascii is True else ascii or Bar.UTF, 536 colour=colour) 537 if not _is_ascii(full_bar.charset) and _is_ascii(bar_format): 538 bar_format = _unicode(bar_format) 539 res = bar_format.format(bar=full_bar, **format_dict) 540 return disp_trim(res, ncols) if ncols else res 541 542 elif bar_format: 543 # user-specified bar_format but no total 544 l_bar += '|' 545 format_dict.update(l_bar=l_bar, percentage=0) 546 full_bar = FormatReplace() 547 nobar = bar_format.format(bar=full_bar, **format_dict) 548 if not full_bar.format_called: 549 return nobar 550 full_bar = Bar(0, 551 max(1, ncols - disp_len(nobar)) if ncols else 10, 552 charset=Bar.BLANK, colour=colour) 553 res = bar_format.format(bar=full_bar, **format_dict) 554 return disp_trim(res, ncols) if ncols else res 555 else: 556 # no total: no progressbar, ETA, just progress stats 557 return '{0}{1}{2} [{3}, {4}{5}]'.format( 558 (prefix + ": ") if prefix else '', n_fmt, unit, elapsed_str, rate_fmt, postfix) 559 560 def __new__(cls, *_, **__): 561 instance = object.__new__(cls) 562 with cls.get_lock(): # also constructs lock if non-existent 563 cls._instances.add(instance) 564 # create monitoring thread 565 if cls.monitor_interval and (cls.monitor is None 566 or not cls.monitor.report()): 567 try: 568 cls.monitor = TMonitor(cls, cls.monitor_interval) 569 except Exception as e: # pragma: nocover 570 warn("tqdm:disabling monitor support" 571 " (monitor_interval = 0) due to:\n" + str(e), 572 TqdmMonitorWarning, stacklevel=2) 573 cls.monitor_interval = 0 574 return instance 575 576 @classmethod 577 def _get_free_pos(cls, instance=None): 578 """Skips specified instance.""" 579 positions = {abs(inst.pos) for inst in cls._instances 580 if inst is not instance and hasattr(inst, "pos")} 581 return min(set(range(len(positions) + 1)).difference(positions)) 582 583 @classmethod 584 def _decr_instances(cls, instance): 585 """ 586 Remove from list and reposition another unfixed bar 587 to fill the new gap. 588 589 This means that by default (where all nested bars are unfixed), 590 order is not maintained but screen flicker/blank space is minimised. 591 (tqdm<=4.44.1 moved ALL subsequent unfixed bars up.) 592 """ 593 with cls._lock: 594 try: 595 cls._instances.remove(instance) 596 except KeyError: 597 # if not instance.gui: # pragma: no cover 598 # raise 599 pass # py2: maybe magically removed already 600 # else: 601 if not instance.gui: 602 last = (instance.nrows or 20) - 1 603 # find unfixed (`pos >= 0`) overflow (`pos >= nrows - 1`) 604 instances = list(filter( 605 lambda i: hasattr(i, "pos") and last <= i.pos, 606 cls._instances)) 607 # set first found to current `pos` 608 if instances: 609 inst = min(instances, key=lambda i: i.pos) 610 inst.clear(nolock=True) 611 inst.pos = abs(instance.pos) 612 613 @classmethod 614 def write(cls, s, file=None, end="\n", nolock=False): 615 """Print a message via tqdm (without overlap with bars).""" 616 fp = file if file is not None else sys.stdout 617 with cls.external_write_mode(file=file, nolock=nolock): 618 # Write the message 619 fp.write(s) 620 fp.write(end) 621 622 @classmethod 623 @contextmanager 624 def external_write_mode(cls, file=None, nolock=False): 625 """ 626 Disable tqdm within context and refresh tqdm when exits. 627 Useful when writing to standard output stream 628 """ 629 fp = file if file is not None else sys.stdout 630 631 try: 632 if not nolock: 633 cls.get_lock().acquire() 634 # Clear all bars 635 inst_cleared = [] 636 for inst in getattr(cls, '_instances', []): 637 # Clear instance if in the target output file 638 # or if write output + tqdm output are both either 639 # sys.stdout or sys.stderr (because both are mixed in terminal) 640 if hasattr(inst, "start_t") and (inst.fp == fp or all( 641 f in (sys.stdout, sys.stderr) for f in (fp, inst.fp))): 642 inst.clear(nolock=True) 643 inst_cleared.append(inst) 644 yield 645 # Force refresh display of bars we cleared 646 for inst in inst_cleared: 647 inst.refresh(nolock=True) 648 finally: 649 if not nolock: 650 cls._lock.release() 651 652 @classmethod 653 def set_lock(cls, lock): 654 """Set the global lock.""" 655 cls._lock = lock 656 657 @classmethod 658 def get_lock(cls): 659 """Get the global lock. Construct it if it does not exist.""" 660 if not hasattr(cls, '_lock'): 661 cls._lock = TqdmDefaultWriteLock() 662 return cls._lock 663 664 @classmethod 665 def pandas(cls, **tqdm_kwargs): 666 """ 667 Registers the current `tqdm` class with 668 pandas.core. 669 ( frame.DataFrame 670 | series.Series 671 | groupby.(generic.)DataFrameGroupBy 672 | groupby.(generic.)SeriesGroupBy 673 ).progress_apply 674 675 A new instance will be create every time `progress_apply` is called, 676 and each instance will automatically `close()` upon completion. 677 678 Parameters 679 ---------- 680 tqdm_kwargs : arguments for the tqdm instance 681 682 Examples 683 -------- 684 >>> import pandas as pd 685 >>> import numpy as np 686 >>> from tqdm import tqdm 687 >>> from tqdm.gui import tqdm as tqdm_gui 688 >>> 689 >>> df = pd.DataFrame(np.random.randint(0, 100, (100000, 6))) 690 >>> tqdm.pandas(ncols=50) # can use tqdm_gui, optional kwargs, etc 691 >>> # Now you can use `progress_apply` instead of `apply` 692 >>> df.groupby(0).progress_apply(lambda x: x**2) 693 694 References 695 ---------- 696 <https://stackoverflow.com/questions/18603270/\ 697 progress-indicator-during-pandas-operations-python> 698 """ 699 from warnings import catch_warnings, simplefilter 700 701 from pandas.core.frame import DataFrame 702 from pandas.core.series import Series 703 try: 704 with catch_warnings(): 705 simplefilter("ignore", category=FutureWarning) 706 from pandas import Panel 707 except ImportError: # pandas>=1.2.0 708 Panel = None 709 Rolling, Expanding = None, None 710 try: # pandas>=1.0.0 711 from pandas.core.window.rolling import _Rolling_and_Expanding 712 except ImportError: 713 try: # pandas>=0.18.0 714 from pandas.core.window import _Rolling_and_Expanding 715 except ImportError: # pandas>=1.2.0 716 try: # pandas>=1.2.0 717 from pandas.core.window.expanding import Expanding 718 from pandas.core.window.rolling import Rolling 719 _Rolling_and_Expanding = Rolling, Expanding 720 except ImportError: # pragma: no cover 721 _Rolling_and_Expanding = None 722 try: # pandas>=0.25.0 723 from pandas.core.groupby.generic import SeriesGroupBy # , NDFrameGroupBy 724 from pandas.core.groupby.generic import DataFrameGroupBy 725 except ImportError: # pragma: no cover 726 try: # pandas>=0.23.0 727 from pandas.core.groupby.groupby import DataFrameGroupBy, SeriesGroupBy 728 except ImportError: 729 from pandas.core.groupby import DataFrameGroupBy, SeriesGroupBy 730 try: # pandas>=0.23.0 731 from pandas.core.groupby.groupby import GroupBy 732 except ImportError: # pragma: no cover 733 from pandas.core.groupby import GroupBy 734 735 try: # pandas>=0.23.0 736 from pandas.core.groupby.groupby import PanelGroupBy 737 except ImportError: 738 try: 739 from pandas.core.groupby import PanelGroupBy 740 except ImportError: # pandas>=0.25.0 741 PanelGroupBy = None 742 743 tqdm_kwargs = tqdm_kwargs.copy() 744 deprecated_t = [tqdm_kwargs.pop('deprecated_t', None)] 745 746 def inner_generator(df_function='apply'): 747 def inner(df, func, *args, **kwargs): 748 """ 749 Parameters 750 ---------- 751 df : (DataFrame|Series)[GroupBy] 752 Data (may be grouped). 753 func : function 754 To be applied on the (grouped) data. 755 **kwargs : optional 756 Transmitted to `df.apply()`. 757 """ 758 759 # Precompute total iterations 760 total = tqdm_kwargs.pop("total", getattr(df, 'ngroups', None)) 761 if total is None: # not grouped 762 if df_function == 'applymap': 763 total = df.size 764 elif isinstance(df, Series): 765 total = len(df) 766 elif (_Rolling_and_Expanding is None or 767 not isinstance(df, _Rolling_and_Expanding)): 768 # DataFrame or Panel 769 axis = kwargs.get('axis', 0) 770 if axis == 'index': 771 axis = 0 772 elif axis == 'columns': 773 axis = 1 774 # when axis=0, total is shape[axis1] 775 total = df.size // df.shape[axis] 776 777 # Init bar 778 if deprecated_t[0] is not None: 779 t = deprecated_t[0] 780 deprecated_t[0] = None 781 else: 782 t = cls(total=total, **tqdm_kwargs) 783 784 if len(args) > 0: 785 # *args intentionally not supported (see #244, #299) 786 TqdmDeprecationWarning( 787 "Except func, normal arguments are intentionally" + 788 " not supported by" + 789 " `(DataFrame|Series|GroupBy).progress_apply`." + 790 " Use keyword arguments instead.", 791 fp_write=getattr(t.fp, 'write', sys.stderr.write)) 792 793 try: # pandas>=1.3.0 794 from pandas.core.common import is_builtin_func 795 except ImportError: 796 is_builtin_func = df._is_builtin_func 797 try: 798 func = is_builtin_func(func) 799 except TypeError: 800 pass 801 802 # Define bar updating wrapper 803 def wrapper(*args, **kwargs): 804 # update tbar correctly 805 # it seems `pandas apply` calls `func` twice 806 # on the first column/row to decide whether it can 807 # take a fast or slow code path; so stop when t.total==t.n 808 t.update(n=1 if not t.total or t.n < t.total else 0) 809 return func(*args, **kwargs) 810 811 # Apply the provided function (in **kwargs) 812 # on the df using our wrapper (which provides bar updating) 813 try: 814 return getattr(df, df_function)(wrapper, **kwargs) 815 finally: 816 t.close() 817 818 return inner 819 820 # Monkeypatch pandas to provide easy methods 821 # Enable custom tqdm progress in pandas! 822 Series.progress_apply = inner_generator() 823 SeriesGroupBy.progress_apply = inner_generator() 824 Series.progress_map = inner_generator('map') 825 SeriesGroupBy.progress_map = inner_generator('map') 826 827 DataFrame.progress_apply = inner_generator() 828 DataFrameGroupBy.progress_apply = inner_generator() 829 DataFrame.progress_applymap = inner_generator('applymap') 830 831 if Panel is not None: 832 Panel.progress_apply = inner_generator() 833 if PanelGroupBy is not None: 834 PanelGroupBy.progress_apply = inner_generator() 835 836 GroupBy.progress_apply = inner_generator() 837 GroupBy.progress_aggregate = inner_generator('aggregate') 838 GroupBy.progress_transform = inner_generator('transform') 839 840 if Rolling is not None and Expanding is not None: 841 Rolling.progress_apply = inner_generator() 842 Expanding.progress_apply = inner_generator() 843 elif _Rolling_and_Expanding is not None: 844 _Rolling_and_Expanding.progress_apply = inner_generator() 845 846 def __init__(self, iterable=None, desc=None, total=None, leave=True, file=None, 847 ncols=None, mininterval=0.1, maxinterval=10.0, miniters=None, 848 ascii=None, disable=False, unit='it', unit_scale=False, 849 dynamic_ncols=False, smoothing=0.3, bar_format=None, initial=0, 850 position=None, postfix=None, unit_divisor=1000, write_bytes=None, 851 lock_args=None, nrows=None, colour=None, delay=0, gui=False, 852 **kwargs): 853 """ 854 Parameters 855 ---------- 856 iterable : iterable, optional 857 Iterable to decorate with a progressbar. 858 Leave blank to manually manage the updates. 859 desc : str, optional 860 Prefix for the progressbar. 861 total : int or float, optional 862 The number of expected iterations. If unspecified, 863 len(iterable) is used if possible. If float("inf") or as a last 864 resort, only basic progress statistics are displayed 865 (no ETA, no progressbar). 866 If `gui` is True and this parameter needs subsequent updating, 867 specify an initial arbitrary large positive number, 868 e.g. 9e9. 869 leave : bool, optional 870 If [default: True], keeps all traces of the progressbar 871 upon termination of iteration. 872 If `None`, will leave only if `position` is `0`. 873 file : `io.TextIOWrapper` or `io.StringIO`, optional 874 Specifies where to output the progress messages 875 (default: sys.stderr). Uses `file.write(str)` and `file.flush()` 876 methods. For encoding, see `write_bytes`. 877 ncols : int, optional 878 The width of the entire output message. If specified, 879 dynamically resizes the progressbar to stay within this bound. 880 If unspecified, attempts to use environment width. The 881 fallback is a meter width of 10 and no limit for the counter and 882 statistics. If 0, will not print any meter (only stats). 883 mininterval : float, optional 884 Minimum progress display update interval [default: 0.1] seconds. 885 maxinterval : float, optional 886 Maximum progress display update interval [default: 10] seconds. 887 Automatically adjusts `miniters` to correspond to `mininterval` 888 after long display update lag. Only works if `dynamic_miniters` 889 or monitor thread is enabled. 890 miniters : int or float, optional 891 Minimum progress display update interval, in iterations. 892 If 0 and `dynamic_miniters`, will automatically adjust to equal 893 `mininterval` (more CPU efficient, good for tight loops). 894 If > 0, will skip display of specified number of iterations. 895 Tweak this and `mininterval` to get very efficient loops. 896 If your progress is erratic with both fast and slow iterations 897 (network, skipping items, etc) you should set miniters=1. 898 ascii : bool or str, optional 899 If unspecified or False, use unicode (smooth blocks) to fill 900 the meter. The fallback is to use ASCII characters " 123456789#". 901 disable : bool, optional 902 Whether to disable the entire progressbar wrapper 903 [default: False]. If set to None, disable on non-TTY. 904 unit : str, optional 905 String that will be used to define the unit of each iteration 906 [default: it]. 907 unit_scale : bool or int or float, optional 908 If 1 or True, the number of iterations will be reduced/scaled 909 automatically and a metric prefix following the 910 International System of Units standard will be added 911 (kilo, mega, etc.) [default: False]. If any other non-zero 912 number, will scale `total` and `n`. 913 dynamic_ncols : bool, optional 914 If set, constantly alters `ncols` and `nrows` to the 915 environment (allowing for window resizes) [default: False]. 916 smoothing : float, optional 917 Exponential moving average smoothing factor for speed estimates 918 (ignored in GUI mode). Ranges from 0 (average speed) to 1 919 (current/instantaneous speed) [default: 0.3]. 920 bar_format : str, optional 921 Specify a custom bar string formatting. May impact performance. 922 [default: '{l_bar}{bar}{r_bar}'], where 923 l_bar='{desc}: {percentage:3.0f}%|' and 924 r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, ' 925 '{rate_fmt}{postfix}]' 926 Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, 927 percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, 928 rate, rate_fmt, rate_noinv, rate_noinv_fmt, 929 rate_inv, rate_inv_fmt, postfix, unit_divisor, 930 remaining, remaining_s, eta. 931 Note that a trailing ": " is automatically removed after {desc} 932 if the latter is empty. 933 initial : int or float, optional 934 The initial counter value. Useful when restarting a progress 935 bar [default: 0]. If using float, consider specifying `{n:.3f}` 936 or similar in `bar_format`, or specifying `unit_scale`. 937 position : int, optional 938 Specify the line offset to print this bar (starting from 0) 939 Automatic if unspecified. 940 Useful to manage multiple bars at once (eg, from threads). 941 postfix : dict or *, optional 942 Specify additional stats to display at the end of the bar. 943 Calls `set_postfix(**postfix)` if possible (dict). 944 unit_divisor : float, optional 945 [default: 1000], ignored unless `unit_scale` is True. 946 write_bytes : bool, optional 947 If (default: None) and `file` is unspecified, 948 bytes will be written in Python 2. If `True` will also write 949 bytes. In all other cases will default to unicode. 950 lock_args : tuple, optional 951 Passed to `refresh` for intermediate output 952 (initialisation, iterating, and updating). 953 nrows : int, optional 954 The screen height. If specified, hides nested bars outside this 955 bound. If unspecified, attempts to use environment height. 956 The fallback is 20. 957 colour : str, optional 958 Bar colour (e.g. 'green', '#00ff00'). 959 delay : float, optional 960 Don't display until [default: 0] seconds have elapsed. 961 gui : bool, optional 962 WARNING: internal parameter - do not use. 963 Use tqdm.gui.tqdm(...) instead. If set, will attempt to use 964 matplotlib animations for a graphical output [default: False]. 965 966 Returns 967 ------- 968 out : decorated iterator. 969 """ 970 if write_bytes is None: 971 write_bytes = file is None and sys.version_info < (3,) 972 973 if file is None: 974 file = sys.stderr 975 976 if write_bytes: 977 # Despite coercing unicode into bytes, py2 sys.std* streams 978 # should have bytes written to them. 979 file = SimpleTextIOWrapper( 980 file, encoding=getattr(file, 'encoding', None) or 'utf-8') 981 982 file = DisableOnWriteError(file, tqdm_instance=self) 983 984 if disable is None and hasattr(file, "isatty") and not file.isatty(): 985 disable = True 986 987 if total is None and iterable is not None: 988 try: 989 total = len(iterable) 990 except (TypeError, AttributeError): 991 total = None 992 if total == float("inf"): 993 # Infinite iterations, behave same as unknown 994 total = None 995 996 if disable: 997 self.iterable = iterable 998 self.disable = disable 999 with self._lock: 1000 self.pos = self._get_free_pos(self) 1001 self._instances.remove(self) 1002 self.n = initial 1003 self.total = total 1004 self.leave = leave 1005 return 1006 1007 if kwargs: 1008 self.disable = True 1009 with self._lock: 1010 self.pos = self._get_free_pos(self) 1011 self._instances.remove(self) 1012 raise ( 1013 TqdmDeprecationWarning( 1014 "`nested` is deprecated and automated.\n" 1015 "Use `position` instead for manual control.\n", 1016 fp_write=getattr(file, 'write', sys.stderr.write)) 1017 if "nested" in kwargs else 1018 TqdmKeyError("Unknown argument(s): " + str(kwargs))) 1019 1020 # Preprocess the arguments 1021 if ( 1022 (ncols is None or nrows is None) and (file in (sys.stderr, sys.stdout)) 1023 ) or dynamic_ncols: # pragma: no cover 1024 if dynamic_ncols: 1025 dynamic_ncols = _screen_shape_wrapper() 1026 if dynamic_ncols: 1027 ncols, nrows = dynamic_ncols(file) 1028 else: 1029 _dynamic_ncols = _screen_shape_wrapper() 1030 if _dynamic_ncols: 1031 _ncols, _nrows = _dynamic_ncols(file) 1032 if ncols is None: 1033 ncols = _ncols 1034 if nrows is None: 1035 nrows = _nrows 1036 1037 if miniters is None: 1038 miniters = 0 1039 dynamic_miniters = True 1040 else: 1041 dynamic_miniters = False 1042 1043 if mininterval is None: 1044 mininterval = 0 1045 1046 if maxinterval is None: 1047 maxinterval = 0 1048 1049 if ascii is None: 1050 ascii = not _supports_unicode(file) 1051 1052 if bar_format and ascii is not True and not _is_ascii(ascii): 1053 # Convert bar format into unicode since terminal uses unicode 1054 bar_format = _unicode(bar_format) 1055 1056 if smoothing is None: 1057 smoothing = 0 1058 1059 # Store the arguments 1060 self.iterable = iterable 1061 self.desc = desc or '' 1062 self.total = total 1063 self.leave = leave 1064 self.fp = file 1065 self.ncols = ncols 1066 self.nrows = nrows 1067 self.mininterval = mininterval 1068 self.maxinterval = maxinterval 1069 self.miniters = miniters 1070 self.dynamic_miniters = dynamic_miniters 1071 self.ascii = ascii 1072 self.disable = disable 1073 self.unit = unit 1074 self.unit_scale = unit_scale 1075 self.unit_divisor = unit_divisor 1076 self.initial = initial 1077 self.lock_args = lock_args 1078 self.delay = delay 1079 self.gui = gui 1080 self.dynamic_ncols = dynamic_ncols 1081 self.smoothing = smoothing 1082 self._ema_dn = EMA(smoothing) 1083 self._ema_dt = EMA(smoothing) 1084 self._ema_miniters = EMA(smoothing) 1085 self.bar_format = bar_format 1086 self.postfix = None 1087 self.colour = colour 1088 self._time = time 1089 if postfix: 1090 try: 1091 self.set_postfix(refresh=False, **postfix) 1092 except TypeError: 1093 self.postfix = postfix 1094 1095 # Init the iterations counters 1096 self.last_print_n = initial 1097 self.n = initial 1098 1099 # if nested, at initial sp() call we replace '\r' by '\n' to 1100 # not overwrite the outer progress bar 1101 with self._lock: 1102 # mark fixed positions as negative 1103 self.pos = self._get_free_pos(self) if position is None else -position 1104 1105 if not gui: 1106 # Initialize the screen printer 1107 self.sp = self.status_printer(self.fp) 1108 if delay <= 0: 1109 self.refresh(lock_args=self.lock_args) 1110 1111 # Init the time counter 1112 self.last_print_t = self._time() 1113 # NB: Avoid race conditions by setting start_t at the very end of init 1114 self.start_t = self.last_print_t 1115 1116 def __bool__(self): 1117 if self.total is not None: 1118 return self.total > 0 1119 if self.iterable is None: 1120 raise TypeError('bool() undefined when iterable == total == None') 1121 return bool(self.iterable) 1122 1123 def __nonzero__(self): 1124 return self.__bool__() 1125 1126 def __len__(self): 1127 return ( 1128 self.total if self.iterable is None 1129 else self.iterable.shape[0] if hasattr(self.iterable, "shape") 1130 else len(self.iterable) if hasattr(self.iterable, "__len__") 1131 else self.iterable.__length_hint__() if hasattr(self.iterable, "__length_hint__") 1132 else getattr(self, "total", None)) 1133 1134 def __enter__(self): 1135 return self 1136 1137 def __exit__(self, exc_type, exc_value, traceback): 1138 try: 1139 self.close() 1140 except AttributeError: 1141 # maybe eager thread cleanup upon external error 1142 if (exc_type, exc_value, traceback) == (None, None, None): 1143 raise 1144 warn("AttributeError ignored", TqdmWarning, stacklevel=2) 1145 1146 def __del__(self): 1147 self.close() 1148 1149 def __str__(self): 1150 return self.format_meter(**self.format_dict) 1151 1152 @property 1153 def _comparable(self): 1154 return abs(getattr(self, "pos", 1 << 31)) 1155 1156 def __hash__(self): 1157 return id(self) 1158 1159 def __iter__(self): 1160 """Backward-compatibility to use: for x in tqdm(iterable)""" 1161 1162 # Inlining instance variables as locals (speed optimisation) 1163 iterable = self.iterable 1164 1165 # If the bar is disabled, then just walk the iterable 1166 # (note: keep this check outside the loop for performance) 1167 if self.disable: 1168 for obj in iterable: 1169 yield obj 1170 return 1171 1172 mininterval = self.mininterval 1173 last_print_t = self.last_print_t 1174 last_print_n = self.last_print_n 1175 min_start_t = self.start_t + self.delay 1176 n = self.n 1177 time = self._time 1178 1179 try: 1180 for obj in iterable: 1181 yield obj 1182 # Update and possibly print the progressbar. 1183 # Note: does not call self.update(1) for speed optimisation. 1184 n += 1 1185 1186 if n - last_print_n >= self.miniters: 1187 cur_t = time() 1188 dt = cur_t - last_print_t 1189 if dt >= mininterval and cur_t >= min_start_t: 1190 self.update(n - last_print_n) 1191 last_print_n = self.last_print_n 1192 last_print_t = self.last_print_t 1193 finally: 1194 self.n = n 1195 self.close() 1196 1197 def update(self, n=1): 1198 """ 1199 Manually update the progress bar, useful for streams 1200 such as reading files. 1201 E.g.: 1202 >>> t = tqdm(total=filesize) # Initialise 1203 >>> for current_buffer in stream: 1204 ... ... 1205 ... t.update(len(current_buffer)) 1206 >>> t.close() 1207 The last line is highly recommended, but possibly not necessary if 1208 `t.update()` will be called in such a way that `filesize` will be 1209 exactly reached and printed. 1210 1211 Parameters 1212 ---------- 1213 n : int or float, optional 1214 Increment to add to the internal counter of iterations 1215 [default: 1]. If using float, consider specifying `{n:.3f}` 1216 or similar in `bar_format`, or specifying `unit_scale`. 1217 1218 Returns 1219 ------- 1220 out : bool or None 1221 True if a `display()` was triggered. 1222 """ 1223 if self.disable: 1224 return 1225 1226 if n < 0: 1227 self.last_print_n += n # for auto-refresh logic to work 1228 self.n += n 1229 1230 # check counter first to reduce calls to time() 1231 if self.n - self.last_print_n >= self.miniters: 1232 cur_t = self._time() 1233 dt = cur_t - self.last_print_t 1234 if dt >= self.mininterval and cur_t >= self.start_t + self.delay: 1235 cur_t = self._time() 1236 dn = self.n - self.last_print_n # >= n 1237 if self.smoothing and dt and dn: 1238 # EMA (not just overall average) 1239 self._ema_dn(dn) 1240 self._ema_dt(dt) 1241 self.refresh(lock_args=self.lock_args) 1242 if self.dynamic_miniters: 1243 # If no `miniters` was specified, adjust automatically to the 1244 # maximum iteration rate seen so far between two prints. 1245 # e.g.: After running `tqdm.update(5)`, subsequent 1246 # calls to `tqdm.update()` will only cause an update after 1247 # at least 5 more iterations. 1248 if self.maxinterval and dt >= self.maxinterval: 1249 self.miniters = dn * (self.mininterval or self.maxinterval) / dt 1250 elif self.smoothing: 1251 # EMA miniters update 1252 self.miniters = self._ema_miniters( 1253 dn * (self.mininterval / dt if self.mininterval and dt 1254 else 1)) 1255 else: 1256 # max iters between two prints 1257 self.miniters = max(self.miniters, dn) 1258 1259 # Store old values for next call 1260 self.last_print_n = self.n 1261 self.last_print_t = cur_t 1262 return True 1263 1264 def close(self): 1265 """Cleanup and (if leave=False) close the progressbar.""" 1266 if self.disable: 1267 return 1268 1269 # Prevent multiple closures 1270 self.disable = True 1271 1272 # decrement instance pos and remove from internal set 1273 pos = abs(self.pos) 1274 self._decr_instances(self) 1275 1276 if self.last_print_t < self.start_t + self.delay: 1277 # haven't ever displayed; nothing to clear 1278 return 1279 1280 # GUI mode 1281 if getattr(self, 'sp', None) is None: 1282 return 1283 1284 # annoyingly, _supports_unicode isn't good enough 1285 def fp_write(s): 1286 self.fp.write(_unicode(s)) 1287 1288 try: 1289 fp_write('') 1290 except ValueError as e: 1291 if 'closed' in str(e): 1292 return 1293 raise # pragma: no cover 1294 1295 leave = pos == 0 if self.leave is None else self.leave 1296 1297 with self._lock: 1298 if leave: 1299 # stats for overall rate (no weighted average) 1300 self._ema_dt = lambda: None 1301 self.display(pos=0) 1302 fp_write('\n') 1303 else: 1304 # clear previous display 1305 if self.display(msg='', pos=pos) and not pos: 1306 fp_write('\r') 1307 1308 def clear(self, nolock=False): 1309 """Clear current bar display.""" 1310 if self.disable: 1311 return 1312 1313 if not nolock: 1314 self._lock.acquire() 1315 pos = abs(self.pos) 1316 if pos < (self.nrows or 20): 1317 self.moveto(pos) 1318 self.sp('') 1319 self.fp.write('\r') # place cursor back at the beginning of line 1320 self.moveto(-pos) 1321 if not nolock: 1322 self._lock.release() 1323 1324 def refresh(self, nolock=False, lock_args=None): 1325 """ 1326 Force refresh the display of this bar. 1327 1328 Parameters 1329 ---------- 1330 nolock : bool, optional 1331 If `True`, does not lock. 1332 If [default: `False`]: calls `acquire()` on internal lock. 1333 lock_args : tuple, optional 1334 Passed to internal lock's `acquire()`. 1335 If specified, will only `display()` if `acquire()` returns `True`. 1336 """ 1337 if self.disable: 1338 return 1339 1340 if not nolock: 1341 if lock_args: 1342 if not self._lock.acquire(*lock_args): 1343 return False 1344 else: 1345 self._lock.acquire() 1346 self.display() 1347 if not nolock: 1348 self._lock.release() 1349 return True 1350 1351 def unpause(self): 1352 """Restart tqdm timer from last print time.""" 1353 if self.disable: 1354 return 1355 cur_t = self._time() 1356 self.start_t += cur_t - self.last_print_t 1357 self.last_print_t = cur_t 1358 1359 def reset(self, total=None): 1360 """ 1361 Resets to 0 iterations for repeated use. 1362 1363 Consider combining with `leave=True`. 1364 1365 Parameters 1366 ---------- 1367 total : int or float, optional. Total to use for the new bar. 1368 """ 1369 self.n = 0 1370 if total is not None: 1371 self.total = total 1372 if self.disable: 1373 return 1374 self.last_print_n = 0 1375 self.last_print_t = self.start_t = self._time() 1376 self._ema_dn = EMA(self.smoothing) 1377 self._ema_dt = EMA(self.smoothing) 1378 self._ema_miniters = EMA(self.smoothing) 1379 self.refresh() 1380 1381 def set_description(self, desc=None, refresh=True): 1382 """ 1383 Set/modify description of the progress bar. 1384 1385 Parameters 1386 ---------- 1387 desc : str, optional 1388 refresh : bool, optional 1389 Forces refresh [default: True]. 1390 """ 1391 self.desc = desc + ': ' if desc else '' 1392 if refresh: 1393 self.refresh() 1394 1395 def set_description_str(self, desc=None, refresh=True): 1396 """Set/modify description without ': ' appended.""" 1397 self.desc = desc or '' 1398 if refresh: 1399 self.refresh() 1400 1401 def set_postfix(self, ordered_dict=None, refresh=True, **kwargs): 1402 """ 1403 Set/modify postfix (additional stats) 1404 with automatic formatting based on datatype. 1405 1406 Parameters 1407 ---------- 1408 ordered_dict : dict or OrderedDict, optional 1409 refresh : bool, optional 1410 Forces refresh [default: True]. 1411 kwargs : dict, optional 1412 """ 1413 # Sort in alphabetical order to be more deterministic 1414 postfix = OrderedDict([] if ordered_dict is None else ordered_dict) 1415 for key in sorted(kwargs.keys()): 1416 postfix[key] = kwargs[key] 1417 # Preprocess stats according to datatype 1418 for key in postfix.keys(): 1419 # Number: limit the length of the string 1420 if isinstance(postfix[key], Number): 1421 postfix[key] = self.format_num(postfix[key]) 1422 # Else for any other type, try to get the string conversion 1423 elif not isinstance(postfix[key], _basestring): 1424 postfix[key] = str(postfix[key]) 1425 # Else if it's a string, don't need to preprocess anything 1426 # Stitch together to get the final postfix 1427 self.postfix = ', '.join(key + '=' + postfix[key].strip() 1428 for key in postfix.keys()) 1429 if refresh: 1430 self.refresh() 1431 1432 def set_postfix_str(self, s='', refresh=True): 1433 """ 1434 Postfix without dictionary expansion, similar to prefix handling. 1435 """ 1436 self.postfix = str(s) 1437 if refresh: 1438 self.refresh() 1439 1440 def moveto(self, n): 1441 # TODO: private method 1442 self.fp.write(_unicode('\n' * n + _term_move_up() * -n)) 1443 self.fp.flush() 1444 1445 @property 1446 def format_dict(self): 1447 """Public API for read-only member access.""" 1448 if self.disable and not hasattr(self, 'unit'): 1449 return defaultdict(lambda: None, { 1450 'n': self.n, 'total': self.total, 'elapsed': 0, 'unit': 'it'}) 1451 if self.dynamic_ncols: 1452 self.ncols, self.nrows = self.dynamic_ncols(self.fp) 1453 return { 1454 'n': self.n, 'total': self.total, 1455 'elapsed': self._time() - self.start_t if hasattr(self, 'start_t') else 0, 1456 'ncols': self.ncols, 'nrows': self.nrows, 'prefix': self.desc, 1457 'ascii': self.ascii, 'unit': self.unit, 'unit_scale': self.unit_scale, 1458 'rate': self._ema_dn() / self._ema_dt() if self._ema_dt() else None, 1459 'bar_format': self.bar_format, 'postfix': self.postfix, 1460 'unit_divisor': self.unit_divisor, 'initial': self.initial, 1461 'colour': self.colour} 1462 1463 def display(self, msg=None, pos=None): 1464 """ 1465 Use `self.sp` to display `msg` in the specified `pos`. 1466 1467 Consider overloading this function when inheriting to use e.g.: 1468 `self.some_frontend(**self.format_dict)` instead of `self.sp`. 1469 1470 Parameters 1471 ---------- 1472 msg : str, optional. What to display (default: `repr(self)`). 1473 pos : int, optional. Position to `moveto` 1474 (default: `abs(self.pos)`). 1475 """ 1476 if pos is None: 1477 pos = abs(self.pos) 1478 1479 nrows = self.nrows or 20 1480 if pos >= nrows - 1: 1481 if pos >= nrows: 1482 return False 1483 if msg or msg is None: # override at `nrows - 1` 1484 msg = " ... (more hidden) ..." 1485 1486 if not hasattr(self, "sp"): 1487 raise TqdmDeprecationWarning( 1488 "Please use `tqdm.gui.tqdm(...)`" 1489 " instead of `tqdm(..., gui=True)`\n", 1490 fp_write=getattr(self.fp, 'write', sys.stderr.write)) 1491 1492 if pos: 1493 self.moveto(pos) 1494 self.sp(self.__str__() if msg is None else msg) 1495 if pos: 1496 self.moveto(-pos) 1497 return True 1498 1499 @classmethod 1500 @contextmanager 1501 def wrapattr(cls, stream, method, total=None, bytes=True, **tqdm_kwargs): 1502 """ 1503 stream : file-like object. 1504 method : str, "read" or "write". The result of `read()` and 1505 the first argument of `write()` should have a `len()`. 1506 1507 >>> with tqdm.wrapattr(file_obj, "read", total=file_obj.size) as fobj: 1508 ... while True: 1509 ... chunk = fobj.read(chunk_size) 1510 ... if not chunk: 1511 ... break 1512 """ 1513 with cls(total=total, **tqdm_kwargs) as t: 1514 if bytes: 1515 t.unit = "B" 1516 t.unit_scale = True 1517 t.unit_divisor = 1024 1518 yield CallbackIOWrapper(t.update, stream, method) 1519 1520 1521def trange(*args, **kwargs): 1522 """ 1523 A shortcut for tqdm(xrange(*args), **kwargs). 1524 On Python3+ range is used instead of xrange. 1525 """ 1526 return tqdm(_range(*args), **kwargs) 1527