1# This file is part of Buildbot. Buildbot is free software: you can 2# redistribute it and/or modify it under the terms of the GNU General Public 3# License as published by the Free Software Foundation, version 2. 4# 5# This program is distributed in the hope that it will be useful, but WITHOUT 6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8# details. 9# 10# You should have received a copy of the GNU General Public License along with 11# this program; if not, write to the Free Software Foundation, Inc., 51 12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13# 14# Copyright Buildbot Team Members 15 16import calendar 17import datetime 18import itertools 19import json 20import locale 21import re 22import sys 23import textwrap 24import time 25from builtins import bytes 26from urllib.parse import urlsplit 27from urllib.parse import urlunsplit 28 29import dateutil.tz 30 31from twisted.python import reflect 32from twisted.python.deprecate import deprecatedModuleAttribute 33from twisted.python.versions import Version 34from zope.interface import implementer 35 36from buildbot.interfaces import IConfigured 37from buildbot.util.giturlparse import giturlparse 38from buildbot.util.misc import deferredLocked 39 40from ._notifier import Notifier 41 42 43def naturalSort(array): 44 array = array[:] 45 46 def try_int(s): 47 try: 48 return int(s) 49 except ValueError: 50 return s 51 52 def key_func(item): 53 return [try_int(s) for s in re.split(r'(\d+)', item)] 54 # prepend integer keys to each element, sort them, then strip the keys 55 keyed_array = sorted([(key_func(i), i) for i in array]) 56 array = [i[1] for i in keyed_array] 57 return array 58 59 60def flattened_iterator(l, types=(list, tuple)): 61 """ 62 Generator for a list/tuple that potentially contains nested/lists/tuples of arbitrary nesting 63 that returns every individual non-list/tuple element. In other words, 64 # [(5, 6, [8, 3]), 2, [2, 1, (3, 4)]] will yield 5, 6, 8, 3, 2, 2, 1, 3, 4 65 66 This is safe to call on something not a list/tuple - the original input is yielded. 67 """ 68 if not isinstance(l, types): 69 yield l 70 return 71 72 for element in l: 73 for sub_element in flattened_iterator(element, types): 74 yield sub_element 75 76 77def flatten(l, types=(list, )): 78 """ 79 Given a list/tuple that potentially contains nested lists/tuples of arbitrary nesting, 80 flatten into a single dimension. In other words, turn [(5, 6, [8, 3]), 2, [2, 1, (3, 4)]] 81 into [5, 6, 8, 3, 2, 2, 1, 3, 4] 82 83 This is safe to call on something not a list/tuple - the original input is returned as a list 84 """ 85 # For backwards compatibility, this returned a list, not an iterable. 86 # Changing to return an iterable could break things. 87 if not isinstance(l, types): 88 return l 89 return list(flattened_iterator(l, types)) 90 91 92def now(_reactor=None): 93 if _reactor and hasattr(_reactor, "seconds"): 94 return _reactor.seconds() 95 return time.time() 96 97 98def formatInterval(eta): 99 eta_parts = [] 100 if eta > 3600: 101 eta_parts.append("%d hrs" % (eta / 3600)) 102 eta %= 3600 103 if eta > 60: 104 eta_parts.append("%d mins" % (eta / 60)) 105 eta %= 60 106 eta_parts.append("%d secs" % eta) 107 return ", ".join(eta_parts) 108 109 110def fuzzyInterval(seconds): 111 """ 112 Convert time interval specified in seconds into fuzzy, human-readable form 113 """ 114 if seconds <= 1: 115 return "a moment" 116 if seconds < 20: 117 return "{:d} seconds".format(seconds) 118 if seconds < 55: 119 return "{:d} seconds".format(round(seconds / 10.) * 10) 120 minutes = round(seconds / 60.) 121 if minutes == 1: 122 return "a minute" 123 if minutes < 20: 124 return "{:d} minutes".format(minutes) 125 if minutes < 55: 126 return "{:d} minutes".format(round(minutes / 10.) * 10) 127 hours = round(minutes / 60.) 128 if hours == 1: 129 return "an hour" 130 if hours < 24: 131 return "{:d} hours".format(hours) 132 days = (hours + 6) // 24 133 if days == 1: 134 return "a day" 135 if days < 30: 136 return "{:d} days".format(days) 137 months = int((days + 10) / 30.5) 138 if months == 1: 139 return "a month" 140 if months < 12: 141 return "{} months".format(months) 142 years = round(days / 365.25) 143 if years == 1: 144 return "a year" 145 return "{} years".format(years) 146 147 148@implementer(IConfigured) 149class ComparableMixin: 150 compare_attrs = () 151 152 class _None: 153 pass 154 155 def __hash__(self): 156 compare_attrs = [] 157 reflect.accumulateClassList( 158 self.__class__, 'compare_attrs', compare_attrs) 159 160 alist = [self.__class__] + \ 161 [getattr(self, name, self._None) for name in compare_attrs] 162 return hash(tuple(map(str, alist))) 163 164 def _cmp_common(self, them): 165 if type(self) != type(them): 166 return (False, None, None) 167 168 if self.__class__ != them.__class__: 169 return (False, None, None) 170 171 compare_attrs = [] 172 reflect.accumulateClassList( 173 self.__class__, 'compare_attrs', compare_attrs) 174 175 self_list = [getattr(self, name, self._None) 176 for name in compare_attrs] 177 them_list = [getattr(them, name, self._None) 178 for name in compare_attrs] 179 return (True, self_list, them_list) 180 181 def __eq__(self, them): 182 (isComparable, self_list, them_list) = self._cmp_common(them) 183 if not isComparable: 184 return False 185 return self_list == them_list 186 187 @staticmethod 188 def isEquivalent(us, them): 189 if isinstance(them, ComparableMixin): 190 them, us = us, them 191 if isinstance(us, ComparableMixin): 192 (isComparable, us_list, them_list) = us._cmp_common(them) 193 if not isComparable: 194 return False 195 return all(ComparableMixin.isEquivalent(v, them_list[i]) for i, v in enumerate(us_list)) 196 return us == them 197 198 def __ne__(self, them): 199 (isComparable, self_list, them_list) = self._cmp_common(them) 200 if not isComparable: 201 return True 202 return self_list != them_list 203 204 def __lt__(self, them): 205 (isComparable, self_list, them_list) = self._cmp_common(them) 206 if not isComparable: 207 return False 208 return self_list < them_list 209 210 def __le__(self, them): 211 (isComparable, self_list, them_list) = self._cmp_common(them) 212 if not isComparable: 213 return False 214 return self_list <= them_list 215 216 def __gt__(self, them): 217 (isComparable, self_list, them_list) = self._cmp_common(them) 218 if not isComparable: 219 return False 220 return self_list > them_list 221 222 def __ge__(self, them): 223 (isComparable, self_list, them_list) = self._cmp_common(them) 224 if not isComparable: 225 return False 226 return self_list >= them_list 227 228 def getConfigDict(self): 229 compare_attrs = [] 230 reflect.accumulateClassList( 231 self.__class__, 'compare_attrs', compare_attrs) 232 return {k: getattr(self, k) 233 for k in compare_attrs 234 if hasattr(self, k) and k not in ("passwd", "password")} 235 236 237def diffSets(old, new): 238 if not isinstance(old, set): 239 old = set(old) 240 if not isinstance(new, set): 241 new = set(new) 242 return old - new, new - old 243 244 245# Remove potentially harmful characters from builder name if it is to be 246# used as the build dir. 247badchars_map = bytes.maketrans(b"\t !#$%&'()*+,./:;<=>?@[\\]^{|}~", 248 b"______________________________") 249 250 251def safeTranslate(s): 252 if isinstance(s, str): 253 s = s.encode('utf8') 254 return s.translate(badchars_map) 255 256 257def none_or_str(x): 258 if x is not None and not isinstance(x, str): 259 return str(x) 260 return x 261 262 263def unicode2bytes(x, encoding='utf-8', errors='strict'): 264 if isinstance(x, str): 265 x = x.encode(encoding, errors) 266 return x 267 268 269def bytes2unicode(x, encoding='utf-8', errors='strict'): 270 if isinstance(x, (str, type(None))): 271 return x 272 return str(x, encoding, errors) 273 274 275_hush_pyflakes = [json] 276 277deprecatedModuleAttribute( 278 Version("buildbot", 0, 9, 4), 279 message="Use json from the standard library instead.", 280 moduleName="buildbot.util", 281 name="json", 282) 283 284 285def toJson(obj): 286 if isinstance(obj, datetime.datetime): 287 return datetime2epoch(obj) 288 return None 289 290 291# changes and schedulers consider None to be a legitimate name for a branch, 292# which makes default function keyword arguments hard to handle. This value 293# is always false. 294 295 296class NotABranch: 297 298 def __bool__(self): 299 return False 300 301 302NotABranch = NotABranch() 303 304# time-handling methods 305 306# this used to be a custom class; now it's just an instance of dateutil's class 307UTC = dateutil.tz.tzutc() 308 309 310def epoch2datetime(epoch): 311 """Convert a UNIX epoch time to a datetime object, in the UTC timezone""" 312 if epoch is not None: 313 return datetime.datetime.fromtimestamp(epoch, tz=UTC) 314 return None 315 316 317def datetime2epoch(dt): 318 """Convert a non-naive datetime object to a UNIX epoch timestamp""" 319 if dt is not None: 320 return calendar.timegm(dt.utctimetuple()) 321 return None 322 323 324# TODO: maybe "merge" with formatInterval? 325def human_readable_delta(start, end): 326 """ 327 Return a string of human readable time delta. 328 """ 329 start_date = datetime.datetime.fromtimestamp(start) 330 end_date = datetime.datetime.fromtimestamp(end) 331 delta = end_date - start_date 332 333 result = [] 334 if delta.days > 0: 335 result.append('%d days' % (delta.days,)) 336 if delta.seconds > 0: 337 hours = int(delta.seconds / 3600) 338 if hours > 0: 339 result.append('%d hours' % (hours,)) 340 minutes = int((delta.seconds - hours * 3600) / 60) 341 if minutes: 342 result.append('%d minutes' % (minutes,)) 343 seconds = delta.seconds % 60 344 if seconds > 0: 345 result.append('%d seconds' % (seconds,)) 346 347 if result: 348 return ', '.join(result) 349 return 'super fast' 350 351 352def makeList(input): 353 if isinstance(input, str): 354 return [input] 355 elif input is None: 356 return [] 357 return list(input) 358 359 360def in_reactor(f): 361 """decorate a function by running it with maybeDeferred in a reactor""" 362 def wrap(*args, **kwargs): 363 from twisted.internet import reactor, defer 364 result = [] 365 366 def _async(): 367 d = defer.maybeDeferred(f, *args, **kwargs) 368 369 @d.addErrback 370 def eb(f): 371 f.printTraceback(file=sys.stderr) 372 373 @d.addBoth 374 def do_stop(r): 375 result.append(r) 376 reactor.stop() 377 reactor.callWhenRunning(_async) 378 reactor.run() 379 return result[0] 380 wrap.__doc__ = f.__doc__ 381 wrap.__name__ = f.__name__ 382 wrap._orig = f # for tests 383 return wrap 384 385 386def string2boolean(str): 387 return { 388 b'on': True, 389 b'true': True, 390 b'yes': True, 391 b'1': True, 392 b'off': False, 393 b'false': False, 394 b'no': False, 395 b'0': False, 396 }[str.lower()] 397 398 399def asyncSleep(delay, reactor=None): 400 from twisted.internet import defer 401 from twisted.internet import reactor as internet_reactor 402 if reactor is None: 403 reactor = internet_reactor 404 405 d = defer.Deferred() 406 reactor.callLater(delay, d.callback, None) 407 return d 408 409 410def check_functional_environment(config): 411 try: 412 locale.getdefaultlocale() 413 except (KeyError, ValueError) as e: 414 config.error("\n".join([ 415 "Your environment has incorrect locale settings. This means python cannot handle " 416 "strings safely.", 417 " Please check 'LANG', 'LC_CTYPE', 'LC_ALL' and 'LANGUAGE'" 418 " are either unset or set to a valid locale.", str(e) 419 ])) 420 421 422_netloc_url_re = re.compile(r':[^@]*@') 423 424 425def stripUrlPassword(url): 426 parts = list(urlsplit(url)) 427 parts[1] = _netloc_url_re.sub(':xxxx@', parts[1]) 428 return urlunsplit(parts) 429 430 431def join_list(maybeList): 432 if isinstance(maybeList, (list, tuple)): 433 return ' '.join(bytes2unicode(s) for s in maybeList) 434 return bytes2unicode(maybeList) 435 436 437def command_to_string(command): 438 words = command 439 if isinstance(words, (bytes, str)): 440 words = words.split() 441 442 try: 443 len(words) 444 except (AttributeError, TypeError): 445 # WithProperties and Property don't have __len__ 446 # For old-style classes instances AttributeError raised, 447 # for new-style classes instances - TypeError. 448 return None 449 450 # flatten any nested lists 451 words = flatten(words, (list, tuple)) 452 453 # strip instances and other detritus (which can happen if a 454 # description is requested before rendering) 455 stringWords = [] 456 for w in words: 457 if isinstance(w, (bytes, str)): 458 # If command was bytes, be gentle in 459 # trying to covert it. 460 w = bytes2unicode(w, errors="replace") 461 stringWords.append(w) 462 words = stringWords 463 464 if not words: 465 return None 466 if len(words) < 3: 467 rv = "'{}'".format(' '.join(words)) 468 else: 469 rv = "'{} ...'".format(' '.join(words[:2])) 470 471 return rv 472 473 474def rewrap(text, width=None): 475 """ 476 Rewrap text for output to the console. 477 478 Removes common indentation and rewraps paragraphs according to the console 479 width. 480 481 Line feeds between paragraphs preserved. 482 Formatting of paragraphs that starts with additional indentation 483 preserved. 484 """ 485 486 if width is None: 487 width = 80 488 489 # Remove common indentation. 490 text = textwrap.dedent(text) 491 492 def needs_wrapping(line): 493 # Line always non-empty. 494 return not line[0].isspace() 495 496 # Split text by lines and group lines that comprise paragraphs. 497 wrapped_text = "" 498 for do_wrap, lines in itertools.groupby(text.splitlines(True), 499 key=needs_wrapping): 500 paragraph = ''.join(lines) 501 502 if do_wrap: 503 paragraph = textwrap.fill(paragraph, width) 504 505 wrapped_text += paragraph 506 507 return wrapped_text 508 509 510def dictionary_merge(a, b): 511 """merges dictionary b into a 512 Like dict.update, but recursive 513 """ 514 for key, value in b.items(): 515 if key in a and isinstance(a[key], dict) and isinstance(value, dict): 516 dictionary_merge(a[key], b[key]) 517 continue 518 a[key] = b[key] 519 return a 520 521 522__all__ = [ 523 'naturalSort', 'now', 'formatInterval', 'ComparableMixin', 524 'safeTranslate', 'none_or_str', 525 'NotABranch', 'deferredLocked', 'UTC', 526 'diffSets', 'makeList', 'in_reactor', 'string2boolean', 527 'check_functional_environment', 'human_readable_delta', 528 'rewrap', 529 'Notifier', 530 "giturlparse", 531] 532