1#!/usr/bin/env python
2# coding: UTF-8
3
4'''This scirpt builds the seafile command line client (With no gui).
5
6Some notes:
7
8'''
9
10import sys
11
12####################
13### Requires Python 2.6+
14####################
15if sys.version_info[0] == 3:
16    print 'Python 3 not supported yet. Quit now.'
17    sys.exit(1)
18if sys.version_info[1] < 6:
19    print 'Python 2.6 or above is required. Quit now.'
20    sys.exit(1)
21
22import os
23import commands
24import tempfile
25import shutil
26import re
27import subprocess
28import optparse
29import atexit
30
31####################
32### Global variables
33####################
34
35# command line configuartion
36conf = {}
37
38# key names in the conf dictionary.
39CONF_VERSION            = 'version'
40CONF_SEAFILE_VERSION    = 'seafile_version'
41CONF_LIBSEARPC_VERSION  = 'libsearpc_version'
42CONF_CCNET_VERSION      = 'ccnet_version'
43CONF_SRCDIR             = 'srcdir'
44CONF_KEEP               = 'keep'
45CONF_BUILDDIR           = 'builddir'
46CONF_OUTPUTDIR          = 'outputdir'
47CONF_THIRDPARTDIR       = 'thirdpartdir'
48CONF_NO_STRIP           = 'nostrip'
49
50####################
51### Common helper functions
52####################
53def highlight(content, is_error=False):
54    '''Add ANSI color to content to get it highlighted on terminal'''
55    if is_error:
56        return '\x1b[1;31m%s\x1b[m' % content
57    else:
58        return '\x1b[1;32m%s\x1b[m' % content
59
60def info(msg):
61    print highlight('[INFO] ') + msg
62
63def exist_in_path(prog):
64    '''Test whether prog exists in system path'''
65    dirs = os.environ['PATH'].split(':')
66    for d in dirs:
67        if d == '':
68            continue
69        path = os.path.join(d, prog)
70        if os.path.exists(path):
71            return True
72
73    return False
74
75def prepend_env_value(name, value, seperator=':'):
76    '''append a new value to a list'''
77    try:
78        current_value = os.environ[name]
79    except KeyError:
80        current_value = ''
81
82    new_value = value
83    if current_value:
84        new_value += seperator + current_value
85
86    os.environ[name] = new_value
87
88def error(msg=None, usage=None):
89    if msg:
90        print highlight('[ERROR] ') + msg
91    if usage:
92        print usage
93    sys.exit(1)
94
95def run_argv(argv, cwd=None, env=None, suppress_stdout=False, suppress_stderr=False):
96    '''Run a program and wait it to finish, and return its exit code. The
97    standard output of this program is supressed.
98
99    '''
100    with open(os.devnull, 'w') as devnull:
101        if suppress_stdout:
102            stdout = devnull
103        else:
104            stdout = sys.stdout
105
106        if suppress_stderr:
107            stderr = devnull
108        else:
109            stderr = sys.stderr
110
111        proc = subprocess.Popen(argv,
112                                cwd=cwd,
113                                stdout=stdout,
114                                stderr=stderr,
115                                env=env)
116        return proc.wait()
117
118def run(cmdline, cwd=None, env=None, suppress_stdout=False, suppress_stderr=False):
119    '''Like run_argv but specify a command line string instead of argv'''
120    with open(os.devnull, 'w') as devnull:
121        if suppress_stdout:
122            stdout = devnull
123        else:
124            stdout = sys.stdout
125
126        if suppress_stderr:
127            stderr = devnull
128        else:
129            stderr = sys.stderr
130
131        proc = subprocess.Popen(cmdline,
132                                cwd=cwd,
133                                stdout=stdout,
134                                stderr=stderr,
135                                env=env,
136                                shell=True)
137        return proc.wait()
138
139def must_mkdir(path):
140    '''Create a directory, exit on failure'''
141    try:
142        os.mkdir(path)
143    except OSError, e:
144        error('failed to create directory %s:%s' % (path, e))
145
146def must_copy(src, dst):
147    '''Copy src to dst, exit on failure'''
148    try:
149        shutil.copy(src, dst)
150    except Exception, e:
151        error('failed to copy %s to %s: %s' % (src, dst, e))
152
153class Project(object):
154    '''Base class for a project'''
155    # Probject name, i.e. libseaprc/ccnet/seafile/
156    name = ''
157
158    # A list of shell commands to configure/build the project
159    build_commands = []
160
161    def __init__(self):
162        # the path to pass to --prefix=/<prefix>
163        self.prefix = os.path.join(conf[CONF_BUILDDIR], 'seafile-cli')
164        self.version = self.get_version()
165        self.src_tarball = os.path.join(conf[CONF_SRCDIR],
166                            '%s-%s.tar.gz' % (self.name, self.version))
167        # project dir, like <builddir>/seafile-1.2.2/
168        self.projdir = os.path.join(conf[CONF_BUILDDIR], '%s-%s' % (self.name, self.version))
169
170    def get_version(self):
171        # libsearpc and ccnet can have different versions from seafile.
172        raise NotImplementedError
173
174    def get_source_commit_id(self):
175        '''By convetion, we record the commit id of the source code in the
176        file "<projdir>/latest_commit"
177
178        '''
179        latest_commit_file = os.path.join(self.projdir, 'latest_commit')
180        with open(latest_commit_file, 'r') as fp:
181            commit_id = fp.read().strip('\n\r\t ')
182
183        return commit_id
184
185    def append_cflags(self, macros):
186        cflags = ' '.join([ '-D%s=%s' % (k, macros[k]) for k in macros ])
187        prepend_env_value('CPPFLAGS',
188                          cflags,
189                          seperator=' ')
190
191    def uncompress(self):
192        '''Uncompress the source from the tarball'''
193        info('Uncompressing %s' % self.name)
194
195        if run('tar xf %s' % self.src_tarball) < 0:
196            error('failed to uncompress source of %s' % self.name)
197
198    def before_build(self):
199        '''Hook method to do project-specific stuff before running build commands'''
200        pass
201
202    def build(self):
203        '''Build the source'''
204        self.before_build()
205        info('Building %s' % self.name)
206        for cmd in self.build_commands:
207            if run(cmd, cwd=self.projdir) != 0:
208                error('error when running command:\n\t%s\n' % cmd)
209
210class Libsearpc(Project):
211    name = 'libsearpc'
212
213    def __init__(self):
214        Project.__init__(self)
215        self.build_commands = [
216            './configure --prefix=%s --disable-compile-demo' % self.prefix,
217            'make',
218            'make install'
219        ]
220
221    def get_version(self):
222        return conf[CONF_LIBSEARPC_VERSION]
223
224class Ccnet(Project):
225    name = 'ccnet'
226    def __init__(self):
227        Project.__init__(self)
228        self.build_commands = [
229            './configure --prefix=%s --disable-compile-demo' % self.prefix,
230            'make',
231            'make install'
232        ]
233
234    def get_version(self):
235        return conf[CONF_CCNET_VERSION]
236
237    def before_build(self):
238        macros = {}
239        # SET CCNET_SOURCE_COMMIT_ID, so it can be printed in the log
240        macros['CCNET_SOURCE_COMMIT_ID'] = '\\"%s\\"' % self.get_source_commit_id()
241
242        self.append_cflags(macros)
243
244class Seafile(Project):
245    name = 'seafile'
246    def __init__(self):
247        Project.__init__(self)
248        self.build_commands = [
249            './configure --prefix=%s --disable-gui' % self.prefix,
250            'make',
251            'make install'
252        ]
253
254    def get_version(self):
255        return conf[CONF_SEAFILE_VERSION]
256
257    def update_cli_version(self):
258        '''Substitute the version number in seaf-cli'''
259        cli_py = os.path.join(self.projdir, 'app', 'seaf-cli')
260        with open(cli_py, 'r') as fp:
261            lines = fp.readlines()
262
263        ret = []
264        for line in lines:
265            old = '''SEAF_CLI_VERSION = ""'''
266            new = '''SEAF_CLI_VERSION = "%s"''' % conf[CONF_VERSION]
267            line = line.replace(old, new)
268            ret.append(line)
269
270        with open(cli_py, 'w') as fp:
271            fp.writelines(ret)
272
273    def before_build(self):
274        self.update_cli_version()
275        macros = {}
276        # SET SEAFILE_SOURCE_COMMIT_ID, so it can be printed in the log
277        macros['SEAFILE_SOURCE_COMMIT_ID'] = '\\"%s\\"' % self.get_source_commit_id()
278        self.append_cflags(macros)
279
280def check_targz_src(proj, version, srcdir):
281    src_tarball = os.path.join(srcdir, '%s-%s.tar.gz' % (proj, version))
282    if not os.path.exists(src_tarball):
283        error('%s not exists' % src_tarball)
284
285def validate_args(usage, options):
286    required_args = [
287        CONF_VERSION,
288        CONF_LIBSEARPC_VERSION,
289        CONF_CCNET_VERSION,
290        CONF_SEAFILE_VERSION,
291        CONF_SRCDIR,
292    ]
293
294    # fist check required args
295    for optname in required_args:
296        if getattr(options, optname, None) == None:
297            error('%s must be specified' % optname, usage=usage)
298
299    def get_option(optname):
300        return getattr(options, optname)
301
302    # [ version ]
303    def check_project_version(version):
304        '''A valid version must be like 1.2.2, 1.3'''
305        if not re.match('^[0-9]+(\.([0-9])+)+$', version):
306            error('%s is not a valid version' % version, usage=usage)
307
308    version = get_option(CONF_VERSION)
309    seafile_version = get_option(CONF_SEAFILE_VERSION)
310    libsearpc_version = get_option(CONF_LIBSEARPC_VERSION)
311    ccnet_version = get_option(CONF_CCNET_VERSION)
312
313    check_project_version(version)
314    check_project_version(libsearpc_version)
315    check_project_version(ccnet_version)
316    check_project_version(seafile_version)
317
318    # [ srcdir ]
319    srcdir = get_option(CONF_SRCDIR)
320    check_targz_src('libsearpc', libsearpc_version, srcdir)
321    check_targz_src('ccnet', ccnet_version, srcdir)
322    check_targz_src('seafile', seafile_version, srcdir)
323
324    # [ builddir ]
325    builddir = get_option(CONF_BUILDDIR)
326    if not os.path.exists(builddir):
327        error('%s does not exist' % builddir, usage=usage)
328
329    builddir = os.path.join(builddir, 'seafile-cli-build')
330
331    # [ outputdir ]
332    outputdir = get_option(CONF_OUTPUTDIR)
333    if outputdir:
334        if not os.path.exists(outputdir):
335            error('outputdir %s does not exist' % outputdir, usage=usage)
336    else:
337        outputdir = os.getcwd()
338
339    # [ keep ]
340    keep = get_option(CONF_KEEP)
341
342    # [ no strip]
343    nostrip = get_option(CONF_NO_STRIP)
344
345    conf[CONF_VERSION] = version
346    conf[CONF_LIBSEARPC_VERSION] = libsearpc_version
347    conf[CONF_SEAFILE_VERSION] = seafile_version
348    conf[CONF_CCNET_VERSION] = ccnet_version
349
350    conf[CONF_BUILDDIR] = builddir
351    conf[CONF_SRCDIR] = srcdir
352    conf[CONF_OUTPUTDIR] = outputdir
353    conf[CONF_KEEP] = keep
354    conf[CONF_NO_STRIP] = nostrip
355
356    prepare_builddir(builddir)
357    show_build_info()
358
359def show_build_info():
360    '''Print all conf information. Confirm before continue.'''
361    info('------------------------------------------')
362    info('Seafile command line client %s: BUILD INFO' % conf[CONF_VERSION])
363    info('------------------------------------------')
364    info('seafile:          %s' % conf[CONF_SEAFILE_VERSION])
365    info('ccnet:            %s' % conf[CONF_CCNET_VERSION])
366    info('libsearpc:        %s' % conf[CONF_LIBSEARPC_VERSION])
367    info('builddir:         %s' % conf[CONF_BUILDDIR])
368    info('outputdir:        %s' % conf[CONF_OUTPUTDIR])
369    info('source dir:       %s' % conf[CONF_SRCDIR])
370    info('strip symbols:    %s' % (not conf[CONF_NO_STRIP]))
371    info('clean on exit:    %s' % (not conf[CONF_KEEP]))
372    info('------------------------------------------')
373    info('press any key to continue ')
374    info('------------------------------------------')
375    dummy = raw_input()
376
377def prepare_builddir(builddir):
378    must_mkdir(builddir)
379
380    if not conf[CONF_KEEP]:
381        def remove_builddir():
382            '''Remove the builddir when exit'''
383            info('remove builddir before exit')
384            shutil.rmtree(builddir, ignore_errors=True)
385        atexit.register(remove_builddir)
386
387    os.chdir(builddir)
388
389    must_mkdir(os.path.join(builddir, 'seafile-cli'))
390
391def parse_args():
392    parser = optparse.OptionParser()
393    def long_opt(opt):
394        return '--' + opt
395
396    parser.add_option(long_opt(CONF_VERSION),
397                      dest=CONF_VERSION,
398                      nargs=1,
399                      help='the version to build. Must be digits delimited by dots, like 1.3.0')
400
401    parser.add_option(long_opt(CONF_SEAFILE_VERSION),
402                      dest=CONF_SEAFILE_VERSION,
403                      nargs=1,
404                      help='the version of seafile as specified in its "configure.ac". Must be digits delimited by dots, like 1.3.0')
405
406    parser.add_option(long_opt(CONF_LIBSEARPC_VERSION),
407                      dest=CONF_LIBSEARPC_VERSION,
408                      nargs=1,
409                      help='the version of libsearpc as specified in its "configure.ac". Must be digits delimited by dots, like 1.3.0')
410
411    parser.add_option(long_opt(CONF_CCNET_VERSION),
412                      dest=CONF_CCNET_VERSION,
413                      nargs=1,
414                      help='the version of ccnet as specified in its "configure.ac". Must be digits delimited by dots, like 1.3.0')
415
416    parser.add_option(long_opt(CONF_BUILDDIR),
417                      dest=CONF_BUILDDIR,
418                      nargs=1,
419                      help='the directory to build the source. Defaults to /tmp',
420                      default=tempfile.gettempdir())
421
422    parser.add_option(long_opt(CONF_OUTPUTDIR),
423                      dest=CONF_OUTPUTDIR,
424                      nargs=1,
425                      help='the output directory to put the generated tarball. Defaults to the current directory.',
426                      default=os.getcwd())
427
428    parser.add_option(long_opt(CONF_SRCDIR),
429                      dest=CONF_SRCDIR,
430                      nargs=1,
431                      help='''Source tarballs must be placed in this directory.''')
432
433    parser.add_option(long_opt(CONF_KEEP),
434                      dest=CONF_KEEP,
435                      action='store_true',
436                      help='''keep the build directory after the script exits. By default, the script would delete the build directory at exit.''')
437
438    parser.add_option(long_opt(CONF_NO_STRIP),
439                      dest=CONF_NO_STRIP,
440                      action='store_true',
441                      help='''do not strip debug symbols''')
442    usage = parser.format_help()
443    options, remain = parser.parse_args()
444    if remain:
445        error(usage=usage)
446
447    validate_args(usage, options)
448
449def setup_build_env():
450    '''Setup environment variables, such as export PATH=$BUILDDDIR/bin:$PATH'''
451    prefix = os.path.join(conf[CONF_BUILDDIR], 'seafile-cli')
452
453    prepend_env_value('CPPFLAGS',
454                     '-I%s' % os.path.join(prefix, 'include'),
455                     seperator=' ')
456
457    prepend_env_value('CPPFLAGS',
458                     '-DSEAFILE_CLIENT_VERSION=\\"%s\\"' % conf[CONF_VERSION],
459                     seperator=' ')
460
461    if conf[CONF_NO_STRIP]:
462        prepend_env_value('CPPFLAGS',
463                         '-g -O0',
464                         seperator=' ')
465
466    prepend_env_value('LDFLAGS',
467                     '-L%s' % os.path.join(prefix, 'lib'),
468                     seperator=' ')
469
470    prepend_env_value('LDFLAGS',
471                     '-L%s' % os.path.join(prefix, 'lib64'),
472                     seperator=' ')
473
474    prepend_env_value('PATH', os.path.join(prefix, 'bin'))
475    prepend_env_value('PKG_CONFIG_PATH', os.path.join(prefix, 'lib', 'pkgconfig'))
476    prepend_env_value('PKG_CONFIG_PATH', os.path.join(prefix, 'lib64', 'pkgconfig'))
477
478def copy_scripts_and_libs():
479    '''Copy scripts and shared libs'''
480    builddir = conf[CONF_BUILDDIR]
481    seafile_dir = os.path.join(builddir, Seafile().projdir)
482    scripts_srcdir = os.path.join(seafile_dir, 'scripts')
483    doc_dir = os.path.join(seafile_dir, 'doc')
484    cli_dir = os.path.join(builddir, 'seafile-cli')
485
486    # copy the wrapper shell script for seaf-cli.py
487    src = os.path.join(scripts_srcdir, 'seaf-cli-wrapper.sh')
488    dst = os.path.join(cli_dir, 'seaf-cli')
489
490    must_copy(src, dst)
491
492    # copy Readme for cli client
493    src = os.path.join(doc_dir, 'cli-readme.txt')
494    dst = os.path.join(cli_dir, 'Readme.txt')
495
496    must_copy(src, dst)
497
498    # rename seaf-cli to seaf-cli.py to avoid confusing users
499    src = os.path.join(cli_dir, 'bin', 'seaf-cli')
500    dst = os.path.join(cli_dir, 'bin', 'seaf-cli.py')
501
502    try:
503        shutil.move(src, dst)
504    except Exception, e:
505        error('failed to move %s to %s: %s' % (src, dst, e))
506
507    # copy shared c libs
508    copy_shared_libs()
509
510def get_dependent_libs(executable):
511    syslibs = ['libsearpc', 'libccnet', 'libseafile', 'libpthread.so', 'libc.so', 'libm.so', 'librt.so', 'libdl.so', 'libselinux.so']
512    def is_syslib(lib):
513        for syslib in syslibs:
514            if syslib in lib:
515                return True
516        return False
517
518    ldd_output = commands.getoutput('ldd %s' % executable)
519    ret = []
520    for line in ldd_output.splitlines():
521        tokens = line.split()
522        if len(tokens) != 4:
523            continue
524        if is_syslib(tokens[0]):
525            continue
526
527        ret.append(tokens[2])
528
529    return ret
530
531def copy_shared_libs():
532    '''copy shared c libs, such as libevent, glib, libmysqlclient'''
533    builddir = conf[CONF_BUILDDIR]
534
535    dst_dir = os.path.join(builddir,
536                           'seafile-cli',
537                           'lib')
538
539    ccnet_daemon_path = os.path.join(builddir,
540                                     'seafile-cli',
541                                     'bin',
542                                     'ccnet')
543
544    seaf_daemon_path = os.path.join(builddir,
545                                    'seafile-cli',
546                                    'bin',
547                                    'seaf-daemon')
548
549    ccnet_daemon_libs = get_dependent_libs(ccnet_daemon_path)
550    seaf_daemon_libs = get_dependent_libs(seaf_daemon_path)
551
552    libs = ccnet_daemon_libs
553    for lib in seaf_daemon_libs:
554        if lib not in libs:
555            libs.append(lib)
556
557    for lib in libs:
558        info('Copying %s' % lib)
559        shutil.copy(lib, dst_dir)
560
561def strip_symbols():
562    def do_strip(fn):
563        run('chmod u+w %s' % fn)
564        info('stripping:    %s' % fn)
565        run('strip "%s"' % fn)
566
567    def remove_static_lib(fn):
568        info('removing:     %s' % fn)
569        os.remove(fn)
570
571    builddir = conf[CONF_BUILDDIR]
572    topdir = os.path.join(builddir, 'seafile-cli')
573    for parent, dnames, fnames in os.walk(topdir):
574        dummy = dnames          # avoid pylint 'unused' warning
575        for fname in fnames:
576            fn = os.path.join(parent, fname)
577            if os.path.isdir(fn):
578                continue
579
580            if fn.endswith(".a") or fn.endswith(".la"):
581                remove_static_lib(fn)
582                continue
583
584            if os.path.islink(fn):
585                continue
586
587            finfo = commands.getoutput('file "%s"' % fn)
588
589            if 'not stripped' in finfo:
590                do_strip(fn)
591
592def create_tarball(tarball_name):
593    '''call tar command to generate a tarball'''
594    version  = conf[CONF_VERSION]
595
596    cli_dir = 'seafile-cli'
597    versioned_cli_dir = 'seafile-cli-' + version
598
599    # move seafile-cli to seafile-cli-${version}
600    try:
601        shutil.move(cli_dir, versioned_cli_dir)
602    except Exception, e:
603        error('failed to move %s to %s: %s' % (cli_dir, versioned_cli_dir, e))
604
605    ignored_patterns = [
606        # common ignored files
607        '*.pyc',
608        '*~',
609        '*#',
610
611        # seafile
612        os.path.join(versioned_cli_dir, 'share*'),
613        os.path.join(versioned_cli_dir, 'include*'),
614        os.path.join(versioned_cli_dir, 'lib', 'pkgconfig*'),
615        os.path.join(versioned_cli_dir, 'lib64', 'pkgconfig*'),
616        os.path.join(versioned_cli_dir, 'bin', 'ccnet-demo*'),
617        os.path.join(versioned_cli_dir, 'bin', 'ccnet-tool'),
618        os.path.join(versioned_cli_dir, 'bin', 'ccnet-servtool'),
619        os.path.join(versioned_cli_dir, 'bin', 'searpc-codegen.py'),
620        os.path.join(versioned_cli_dir, 'bin', 'seafile-admin'),
621        os.path.join(versioned_cli_dir, 'bin', 'seafile'),
622    ]
623
624    excludes_list = [ '--exclude=%s' % pattern for pattern in ignored_patterns ]
625    excludes = ' '.join(excludes_list)
626
627    tar_cmd = 'tar czvf %(tarball_name)s %(versioned_cli_dir)s %(excludes)s' \
628              % dict(tarball_name=tarball_name,
629                     versioned_cli_dir=versioned_cli_dir,
630                     excludes=excludes)
631
632    if run(tar_cmd) != 0:
633        error('failed to generate the tarball')
634
635def gen_tarball():
636    # strip symbols of libraries to reduce size
637    if not conf[CONF_NO_STRIP]:
638        try:
639            strip_symbols()
640        except Exception, e:
641            error('failed to strip symbols: %s' % e)
642
643    # determine the output name
644    # 64-bit: seafile-cli_1.2.2_x86-64.tar.gz
645    # 32-bit: seafile-cli_1.2.2_i386.tar.gz
646    version = conf[CONF_VERSION]
647    arch = os.uname()[-1].replace('_', '-')
648    if arch != 'x86-64':
649        arch = 'i386'
650
651    dbg = ''
652    if conf[CONF_NO_STRIP]:
653        dbg = '.dbg'
654
655    tarball_name = 'seafile-cli_%(version)s_%(arch)s%(dbg)s.tar.gz' \
656                   % dict(version=version, arch=arch, dbg=dbg)
657    dst_tarball = os.path.join(conf[CONF_OUTPUTDIR], tarball_name)
658
659    # generate the tarball
660    try:
661        create_tarball(tarball_name)
662    except Exception, e:
663        error('failed to generate tarball: %s' % e)
664
665    # move tarball to outputdir
666    try:
667        shutil.copy(tarball_name, dst_tarball)
668    except Exception, e:
669        error('failed to copy %s to %s: %s' % (tarball_name, dst_tarball, e))
670
671    print '---------------------------------------------'
672    print 'The build is successfully. Output is:\t%s' % dst_tarball
673    print '---------------------------------------------'
674
675def main():
676    parse_args()
677    setup_build_env()
678
679    libsearpc = Libsearpc()
680    ccnet = Ccnet()
681    seafile = Seafile()
682
683    libsearpc.uncompress()
684    libsearpc.build()
685
686    ccnet.uncompress()
687    ccnet.build()
688
689    seafile.uncompress()
690    seafile.build()
691
692    copy_scripts_and_libs()
693    gen_tarball()
694
695if __name__ == '__main__':
696    main()
697