1
2import codecs
3import re
4import socket
5import time
6import os
7import os.path
8import itertools
9import subprocess
10import sys
11import hashlib
12
13# various utilities that are handy
14
15def getAllSourceFiles( arr=None , prefix="." ):
16    if arr is None:
17        arr = []
18
19    if not os.path.isdir( prefix ):
20        # assume a file
21        arr.append( prefix )
22        return arr
23
24    for x in os.listdir( prefix ):
25        if x.startswith( "." ) or x.startswith( "pcre-" ) or x.startswith( "32bit" ) or x.startswith( "mongodb-" ) or x.startswith("debian") or x.startswith( "mongo-cxx-driver" ):
26            continue
27        # XXX: Avoid conflict between v8 and v8-3.25 source files in
28        #      src/mongo/scripting
29        #      Remove after v8-3.25 migration.
30        if x.find("v8-3.25") != -1:
31            continue
32        full = prefix + "/" + x
33        if os.path.isdir( full ) and not os.path.islink( full ):
34            getAllSourceFiles( arr , full )
35        else:
36            if full.endswith( ".cpp" ) or full.endswith( ".h" ) or full.endswith( ".c" ):
37                full = full.replace( "//" , "/" )
38                arr.append( full )
39
40    return arr
41
42
43def getGitBranch():
44    if not os.path.exists( ".git" ) or not os.path.isdir(".git"):
45        return None
46
47    version = open( ".git/HEAD" ,'r' ).read().strip()
48    if not version.startswith( "ref: " ):
49        return version
50    version = version.split( "/" )
51    version = version[len(version)-1]
52    return version
53
54def getGitBranchString( prefix="" , postfix="" ):
55    t = re.compile( '[/\\\]' ).split( os.getcwd() )
56    if len(t) > 2 and t[len(t)-1] == "mongo":
57        par = t[len(t)-2]
58        m = re.compile( ".*_([vV]\d+\.\d+)$" ).match( par )
59        if m is not None:
60            return prefix + m.group(1).lower() + postfix
61        if par.find("Nightly") > 0:
62            return ""
63
64
65    b = getGitBranch()
66    if b == None or b == "master":
67        return ""
68    return prefix + b + postfix
69
70def getGitVersion():
71    if not os.path.exists( ".git" ) or not os.path.isdir(".git"):
72        return "nogitversion"
73
74    version = open( ".git/HEAD" ,'r' ).read().strip()
75    if not version.startswith( "ref: " ):
76        return version
77    version = version[5:]
78    f = ".git/" + version
79    if not os.path.exists( f ):
80        return version
81    return open( f , 'r' ).read().strip()
82
83def execsys( args ):
84    import subprocess
85    if isinstance( args , str ):
86        r = re.compile( "\s+" )
87        args = r.split( args )
88    p = subprocess.Popen( args , stdout=subprocess.PIPE , stderr=subprocess.PIPE )
89    r = p.communicate()
90    return r;
91
92def getprocesslist():
93    raw = ""
94    try:
95        raw = execsys( "/bin/ps axww" )[0]
96    except Exception,e:
97        print( "can't get processlist: " + str( e ) )
98
99    r = re.compile( "[\r\n]+" )
100    return r.split( raw )
101
102def removeIfInList( lst , thing ):
103    if thing in lst:
104        lst.remove( thing )
105
106def findVersion( root , choices ):
107    for c in choices:
108        if ( os.path.exists( root + c ) ):
109            return root + c
110    raise "can't find a version of [" + root + "] choices: " + choices
111
112def choosePathExist( choices , default=None):
113    for c in choices:
114        if c != None and os.path.exists( c ):
115            return c
116    return default
117
118def filterExists(paths):
119    return filter(os.path.exists, paths)
120
121def ensureDir( name ):
122    d = os.path.dirname( name )
123    if not os.path.exists( d ):
124        print( "Creating dir: " + name );
125        os.makedirs( d )
126        if not os.path.exists( d ):
127            raise "Failed to create dir: " + name
128
129
130def distinctAsString( arr ):
131    s = set()
132    for x in arr:
133        s.add( str(x) )
134    return list(s)
135
136def checkMongoPort( port=27017 ):
137    sock = socket.socket()
138    sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
139    sock.settimeout(1)
140    sock.connect(("localhost", port))
141    sock.close()
142
143
144def didMongodStart( port=27017 , timeout=20 ):
145    while timeout > 0:
146        time.sleep( 1 )
147        try:
148            checkMongoPort( port )
149            return True
150        except Exception,e:
151            print( e )
152            timeout = timeout - 1
153    return False
154
155def which(executable):
156    if sys.platform == 'win32':
157        paths = os.environ.get('Path', '').split(';')
158    else:
159        paths = os.environ.get('PATH', '').split(':')
160
161    for path in paths:
162        path = os.path.expandvars(path)
163        path = os.path.expanduser(path)
164        path = os.path.abspath(path)
165        executable_path = os.path.join(path, executable)
166        if os.path.exists(executable_path):
167            return executable_path
168
169    return executable
170
171def md5sum( file ):
172    #TODO error handling, etc..
173    return execsys( "md5sum " + file )[0].partition(" ")[0]
174
175def md5string( a_string ):
176    return hashlib.md5(a_string).hexdigest()
177
178def find_python(min_version=(2, 5)):
179    try:
180        if sys.version_info >= min_version:
181            return sys.executable
182    except AttributeError:
183        # In case the version of Python is somehow missing sys.version_info or sys.executable.
184        pass
185
186    version = re.compile(r'[Pp]ython ([\d\.]+)', re.MULTILINE)
187    binaries = ('python27', 'python2.7', 'python26', 'python2.6', 'python25', 'python2.5', 'python')
188    for binary in binaries:
189        try:
190            out, err = subprocess.Popen([binary, '-V'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
191            for stream in (out, err):
192                match = version.search(stream)
193                if match:
194                    versiontuple = tuple(map(int, match.group(1).split('.')))
195                    if versiontuple >= min_version:
196                        return which(binary)
197        except:
198            pass
199
200    raise Exception('could not find suitable Python (version >= %s)' % '.'.join(str(v) for v in min_version))
201
202def smoke_command(*args):
203    # return a list of arguments that comprises a complete
204    # invocation of smoke.py
205    here = os.path.dirname(__file__)
206    smoke_py = os.path.abspath(os.path.join(here, 'smoke.py'))
207    # the --with-cleanbb argument causes smoke.py to run
208    # buildscripts/cleanbb.py before each test phase; this
209    # prevents us from running out of disk space on slaves
210    return [find_python(), smoke_py, '--with-cleanbb'] + list(args)
211
212def run_smoke_command(*args):
213    # to run a command line script from a scons Alias (or any
214    # Action), the command sequence must be enclosed in a list,
215    # otherwise SCons treats it as a list of dependencies.
216    return [smoke_command(*args)]
217
218# unicode is a pain. some strings cannot be unicode()'d
219# but we want to just preserve the bytes in a human-readable
220# fashion. this codec error handler will substitute the
221# repr() of the offending bytes into the decoded string
222# at the position they occurred
223def replace_with_repr(unicode_error):
224    offender = unicode_error.object[unicode_error.start:unicode_error.end]
225    return (unicode(repr(offender).strip("'").strip('"')), unicode_error.end)
226
227codecs.register_error('repr', replace_with_repr)
228
229def unicode_dammit(string, encoding='utf8'):
230    # convert a string to a unicode, using the Python
231    # representation of non-ascii bytes when necessary
232    #
233    # name inpsired by BeautifulSoup's "UnicodeDammit"
234    return string.decode(encoding, 'repr')
235
236