1# This file is part of Buildbot.  Buildbot is free software: you can
2# redistribute it and/or modify it under the terms of the GNU General Public
3# License as published by the Free Software Foundation, version 2.
4#
5# This program is distributed in the hope that it will be useful, but WITHOUT
6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8# details.
9#
10# You should have received a copy of the GNU General Public License along with
11# this program; if not, write to the Free Software Foundation, Inc., 51
12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13#
14# Copyright Buildbot Team Members
15
16import calendar
17import datetime
18import itertools
19import json
20import locale
21import re
22import sys
23import textwrap
24import time
25from builtins import bytes
26from urllib.parse import urlsplit
27from urllib.parse import urlunsplit
28
29import dateutil.tz
30
31from twisted.python import reflect
32from twisted.python.deprecate import deprecatedModuleAttribute
33from twisted.python.versions import Version
34from zope.interface import implementer
35
36from buildbot.interfaces import IConfigured
37from buildbot.util.giturlparse import giturlparse
38from buildbot.util.misc import deferredLocked
39
40from ._notifier import Notifier
41
42
43def naturalSort(array):
44    array = array[:]
45
46    def try_int(s):
47        try:
48            return int(s)
49        except ValueError:
50            return s
51
52    def key_func(item):
53        return [try_int(s) for s in re.split(r'(\d+)', item)]
54    # prepend integer keys to each element, sort them, then strip the keys
55    keyed_array = sorted([(key_func(i), i) for i in array])
56    array = [i[1] for i in keyed_array]
57    return array
58
59
60def flattened_iterator(l, types=(list, tuple)):
61    """
62    Generator for a list/tuple that potentially contains nested/lists/tuples of arbitrary nesting
63    that returns every individual non-list/tuple element.  In other words,
64    # [(5, 6, [8, 3]), 2, [2, 1, (3, 4)]] will yield 5, 6, 8, 3, 2, 2, 1, 3, 4
65
66    This is safe to call on something not a list/tuple - the original input is yielded.
67    """
68    if not isinstance(l, types):
69        yield l
70        return
71
72    for element in l:
73        for sub_element in flattened_iterator(element, types):
74            yield sub_element
75
76
77def flatten(l, types=(list, )):
78    """
79    Given a list/tuple that potentially contains nested lists/tuples of arbitrary nesting,
80    flatten into a single dimension.  In other words, turn [(5, 6, [8, 3]), 2, [2, 1, (3, 4)]]
81    into [5, 6, 8, 3, 2, 2, 1, 3, 4]
82
83    This is safe to call on something not a list/tuple - the original input is returned as a list
84    """
85    # For backwards compatibility, this returned a list, not an iterable.
86    # Changing to return an iterable could break things.
87    if not isinstance(l, types):
88        return l
89    return list(flattened_iterator(l, types))
90
91
92def now(_reactor=None):
93    if _reactor and hasattr(_reactor, "seconds"):
94        return _reactor.seconds()
95    return time.time()
96
97
98def formatInterval(eta):
99    eta_parts = []
100    if eta > 3600:
101        eta_parts.append("%d hrs" % (eta / 3600))
102        eta %= 3600
103    if eta > 60:
104        eta_parts.append("%d mins" % (eta / 60))
105        eta %= 60
106    eta_parts.append("%d secs" % eta)
107    return ", ".join(eta_parts)
108
109
110def fuzzyInterval(seconds):
111    """
112    Convert time interval specified in seconds into fuzzy, human-readable form
113    """
114    if seconds <= 1:
115        return "a moment"
116    if seconds < 20:
117        return "{:d} seconds".format(seconds)
118    if seconds < 55:
119        return "{:d} seconds".format(round(seconds / 10.) * 10)
120    minutes = round(seconds / 60.)
121    if minutes == 1:
122        return "a minute"
123    if minutes < 20:
124        return "{:d} minutes".format(minutes)
125    if minutes < 55:
126        return "{:d} minutes".format(round(minutes / 10.) * 10)
127    hours = round(minutes / 60.)
128    if hours == 1:
129        return "an hour"
130    if hours < 24:
131        return "{:d} hours".format(hours)
132    days = (hours + 6) // 24
133    if days == 1:
134        return "a day"
135    if days < 30:
136        return "{:d} days".format(days)
137    months = int((days + 10) / 30.5)
138    if months == 1:
139        return "a month"
140    if months < 12:
141        return "{} months".format(months)
142    years = round(days / 365.25)
143    if years == 1:
144        return "a year"
145    return "{} years".format(years)
146
147
148@implementer(IConfigured)
149class ComparableMixin:
150    compare_attrs = ()
151
152    class _None:
153        pass
154
155    def __hash__(self):
156        compare_attrs = []
157        reflect.accumulateClassList(
158            self.__class__, 'compare_attrs', compare_attrs)
159
160        alist = [self.__class__] + \
161                [getattr(self, name, self._None) for name in compare_attrs]
162        return hash(tuple(map(str, alist)))
163
164    def _cmp_common(self, them):
165        if type(self) != type(them):
166            return (False, None, None)
167
168        if self.__class__ != them.__class__:
169            return (False, None, None)
170
171        compare_attrs = []
172        reflect.accumulateClassList(
173            self.__class__, 'compare_attrs', compare_attrs)
174
175        self_list = [getattr(self, name, self._None)
176                     for name in compare_attrs]
177        them_list = [getattr(them, name, self._None)
178                     for name in compare_attrs]
179        return (True, self_list, them_list)
180
181    def __eq__(self, them):
182        (isComparable, self_list, them_list) = self._cmp_common(them)
183        if not isComparable:
184            return False
185        return self_list == them_list
186
187    @staticmethod
188    def isEquivalent(us, them):
189        if isinstance(them, ComparableMixin):
190            them, us = us, them
191        if isinstance(us, ComparableMixin):
192            (isComparable, us_list, them_list) = us._cmp_common(them)
193            if not isComparable:
194                return False
195            return all(ComparableMixin.isEquivalent(v, them_list[i]) for i, v in enumerate(us_list))
196        return us == them
197
198    def __ne__(self, them):
199        (isComparable, self_list, them_list) = self._cmp_common(them)
200        if not isComparable:
201            return True
202        return self_list != them_list
203
204    def __lt__(self, them):
205        (isComparable, self_list, them_list) = self._cmp_common(them)
206        if not isComparable:
207            return False
208        return self_list < them_list
209
210    def __le__(self, them):
211        (isComparable, self_list, them_list) = self._cmp_common(them)
212        if not isComparable:
213            return False
214        return self_list <= them_list
215
216    def __gt__(self, them):
217        (isComparable, self_list, them_list) = self._cmp_common(them)
218        if not isComparable:
219            return False
220        return self_list > them_list
221
222    def __ge__(self, them):
223        (isComparable, self_list, them_list) = self._cmp_common(them)
224        if not isComparable:
225            return False
226        return self_list >= them_list
227
228    def getConfigDict(self):
229        compare_attrs = []
230        reflect.accumulateClassList(
231            self.__class__, 'compare_attrs', compare_attrs)
232        return {k: getattr(self, k)
233                for k in compare_attrs
234                if hasattr(self, k) and k not in ("passwd", "password")}
235
236
237def diffSets(old, new):
238    if not isinstance(old, set):
239        old = set(old)
240    if not isinstance(new, set):
241        new = set(new)
242    return old - new, new - old
243
244
245# Remove potentially harmful characters from builder name if it is to be
246# used as the build dir.
247badchars_map = bytes.maketrans(b"\t !#$%&'()*+,./:;<=>?@[\\]^{|}~",
248                               b"______________________________")
249
250
251def safeTranslate(s):
252    if isinstance(s, str):
253        s = s.encode('utf8')
254    return s.translate(badchars_map)
255
256
257def none_or_str(x):
258    if x is not None and not isinstance(x, str):
259        return str(x)
260    return x
261
262
263def unicode2bytes(x, encoding='utf-8', errors='strict'):
264    if isinstance(x, str):
265        x = x.encode(encoding, errors)
266    return x
267
268
269def bytes2unicode(x, encoding='utf-8', errors='strict'):
270    if isinstance(x, (str, type(None))):
271        return x
272    return str(x, encoding, errors)
273
274
275_hush_pyflakes = [json]
276
277deprecatedModuleAttribute(
278    Version("buildbot", 0, 9, 4),
279    message="Use json from the standard library instead.",
280    moduleName="buildbot.util",
281    name="json",
282)
283
284
285def toJson(obj):
286    if isinstance(obj, datetime.datetime):
287        return datetime2epoch(obj)
288    return None
289
290
291# changes and schedulers consider None to be a legitimate name for a branch,
292# which makes default function keyword arguments hard to handle.  This value
293# is always false.
294
295
296class NotABranch:
297
298    def __bool__(self):
299        return False
300
301
302NotABranch = NotABranch()
303
304# time-handling methods
305
306# this used to be a custom class; now it's just an instance of dateutil's class
307UTC = dateutil.tz.tzutc()
308
309
310def epoch2datetime(epoch):
311    """Convert a UNIX epoch time to a datetime object, in the UTC timezone"""
312    if epoch is not None:
313        return datetime.datetime.fromtimestamp(epoch, tz=UTC)
314    return None
315
316
317def datetime2epoch(dt):
318    """Convert a non-naive datetime object to a UNIX epoch timestamp"""
319    if dt is not None:
320        return calendar.timegm(dt.utctimetuple())
321    return None
322
323
324# TODO: maybe "merge" with formatInterval?
325def human_readable_delta(start, end):
326    """
327    Return a string of human readable time delta.
328    """
329    start_date = datetime.datetime.fromtimestamp(start)
330    end_date = datetime.datetime.fromtimestamp(end)
331    delta = end_date - start_date
332
333    result = []
334    if delta.days > 0:
335        result.append('%d days' % (delta.days,))
336    if delta.seconds > 0:
337        hours = int(delta.seconds / 3600)
338        if hours > 0:
339            result.append('%d hours' % (hours,))
340        minutes = int((delta.seconds - hours * 3600) / 60)
341        if minutes:
342            result.append('%d minutes' % (minutes,))
343        seconds = delta.seconds % 60
344        if seconds > 0:
345            result.append('%d seconds' % (seconds,))
346
347    if result:
348        return ', '.join(result)
349    return 'super fast'
350
351
352def makeList(input):
353    if isinstance(input, str):
354        return [input]
355    elif input is None:
356        return []
357    return list(input)
358
359
360def in_reactor(f):
361    """decorate a function by running it with maybeDeferred in a reactor"""
362    def wrap(*args, **kwargs):
363        from twisted.internet import reactor, defer
364        result = []
365
366        def _async():
367            d = defer.maybeDeferred(f, *args, **kwargs)
368
369            @d.addErrback
370            def eb(f):
371                f.printTraceback(file=sys.stderr)
372
373            @d.addBoth
374            def do_stop(r):
375                result.append(r)
376                reactor.stop()
377        reactor.callWhenRunning(_async)
378        reactor.run()
379        return result[0]
380    wrap.__doc__ = f.__doc__
381    wrap.__name__ = f.__name__
382    wrap._orig = f  # for tests
383    return wrap
384
385
386def string2boolean(str):
387    return {
388        b'on': True,
389        b'true': True,
390        b'yes': True,
391        b'1': True,
392        b'off': False,
393        b'false': False,
394        b'no': False,
395        b'0': False,
396    }[str.lower()]
397
398
399def asyncSleep(delay, reactor=None):
400    from twisted.internet import defer
401    from twisted.internet import reactor as internet_reactor
402    if reactor is None:
403        reactor = internet_reactor
404
405    d = defer.Deferred()
406    reactor.callLater(delay, d.callback, None)
407    return d
408
409
410def check_functional_environment(config):
411    try:
412        locale.getdefaultlocale()
413    except (KeyError, ValueError) as e:
414        config.error("\n".join([
415            "Your environment has incorrect locale settings. This means python cannot handle "
416            "strings safely.",
417            " Please check 'LANG', 'LC_CTYPE', 'LC_ALL' and 'LANGUAGE'"
418            " are either unset or set to a valid locale.", str(e)
419        ]))
420
421
422_netloc_url_re = re.compile(r':[^@]*@')
423
424
425def stripUrlPassword(url):
426    parts = list(urlsplit(url))
427    parts[1] = _netloc_url_re.sub(':xxxx@', parts[1])
428    return urlunsplit(parts)
429
430
431def join_list(maybeList):
432    if isinstance(maybeList, (list, tuple)):
433        return ' '.join(bytes2unicode(s) for s in maybeList)
434    return bytes2unicode(maybeList)
435
436
437def command_to_string(command):
438    words = command
439    if isinstance(words, (bytes, str)):
440        words = words.split()
441
442    try:
443        len(words)
444    except (AttributeError, TypeError):
445        # WithProperties and Property don't have __len__
446        # For old-style classes instances AttributeError raised,
447        # for new-style classes instances - TypeError.
448        return None
449
450    # flatten any nested lists
451    words = flatten(words, (list, tuple))
452
453    # strip instances and other detritus (which can happen if a
454    # description is requested before rendering)
455    stringWords = []
456    for w in words:
457        if isinstance(w, (bytes, str)):
458            # If command was bytes, be gentle in
459            # trying to covert it.
460            w = bytes2unicode(w, errors="replace")
461            stringWords.append(w)
462    words = stringWords
463
464    if not words:
465        return None
466    if len(words) < 3:
467        rv = "'{}'".format(' '.join(words))
468    else:
469        rv = "'{} ...'".format(' '.join(words[:2]))
470
471    return rv
472
473
474def rewrap(text, width=None):
475    """
476    Rewrap text for output to the console.
477
478    Removes common indentation and rewraps paragraphs according to the console
479    width.
480
481    Line feeds between paragraphs preserved.
482    Formatting of paragraphs that starts with additional indentation
483    preserved.
484    """
485
486    if width is None:
487        width = 80
488
489    # Remove common indentation.
490    text = textwrap.dedent(text)
491
492    def needs_wrapping(line):
493        # Line always non-empty.
494        return not line[0].isspace()
495
496    # Split text by lines and group lines that comprise paragraphs.
497    wrapped_text = ""
498    for do_wrap, lines in itertools.groupby(text.splitlines(True),
499                                            key=needs_wrapping):
500        paragraph = ''.join(lines)
501
502        if do_wrap:
503            paragraph = textwrap.fill(paragraph, width)
504
505        wrapped_text += paragraph
506
507    return wrapped_text
508
509
510def dictionary_merge(a, b):
511    """merges dictionary b into a
512       Like dict.update, but recursive
513    """
514    for key, value in b.items():
515        if key in a and isinstance(a[key], dict) and isinstance(value, dict):
516            dictionary_merge(a[key], b[key])
517            continue
518        a[key] = b[key]
519    return a
520
521
522__all__ = [
523    'naturalSort', 'now', 'formatInterval', 'ComparableMixin',
524    'safeTranslate', 'none_or_str',
525    'NotABranch', 'deferredLocked', 'UTC',
526    'diffSets', 'makeList', 'in_reactor', 'string2boolean',
527    'check_functional_environment', 'human_readable_delta',
528    'rewrap',
529    'Notifier',
530    "giturlparse",
531]
532