1#!/usr/bin/env python
2# ***** BEGIN LICENSE BLOCK *****
3# This Source Code Form is subject to the terms of the Mozilla Public
4# License, v. 2.0. If a copy of the MPL was not distributed with this file,
5# You can obtain one at http://mozilla.org/MPL/2.0/.
6# ***** END LICENSE BLOCK *****
7"""desktop_l10n.py
8
9This script manages Desktop repacks for nightly builds.
10"""
11import os
12import glob
13import re
14import sys
15import shlex
16
17# load modules from parent dir
18sys.path.insert(1, os.path.dirname(sys.path[0]))  # noqa
19
20from mozharness.base.errors import MakefileErrorList
21from mozharness.base.script import BaseScript
22from mozharness.base.vcs.vcsbase import VCSMixin
23from mozharness.mozilla.automation import AutomationMixin
24from mozharness.mozilla.building.buildbase import (
25    MakeUploadOutputParser,
26    get_mozconfig_path,
27)
28from mozharness.mozilla.l10n.locales import LocalesMixin
29
30try:
31    import simplejson as json
32    assert json
33except ImportError:
34    import json
35
36
37# needed by _map
38SUCCESS = 0
39FAILURE = 1
40
41SUCCESS_STR = "Success"
42FAILURE_STR = "Failed"
43
44
45# some other values such as "%(version)s", ...
46# are defined at run time and they cannot be enforced in the _pre_config_lock
47# phase
48runtime_config_tokens = ('version', 'locale', 'abs_objdir',
49                         'en_us_installer_binary_url')
50
51
52# DesktopSingleLocale {{{1
53class DesktopSingleLocale(LocalesMixin, AutomationMixin,
54                          VCSMixin, BaseScript):
55    """Manages desktop repacks"""
56    config_options = [[
57        ['--locale', ],
58        {"action": "extend",
59         "dest": "locales",
60         "type": "string",
61         "help": "Specify the locale(s) to sign and update. Optionally pass"
62                 " revision separated by colon, en-GB:default."}
63    ], [
64        ['--tag-override', ],
65        {"action": "store",
66         "dest": "tag_override",
67         "type": "string",
68         "help": "Override the tags set for all repos"}
69    ], [
70        ['--en-us-installer-url', ],
71        {"action": "store",
72         "dest": "en_us_installer_url",
73         "type": "string",
74         "help": "Specify the url of the en-us binary"}
75    ], [
76        ['--scm-level'], {  # Ignored on desktop for now: see Bug 1414678.
77         "action": "store",
78         "type": "int",
79         "dest": "scm_level",
80         "default": 1,
81         "help": "This sets the SCM level for the branch being built."
82                 " See https://www.mozilla.org/en-US/about/"
83                 "governance/policies/commit/access-policy/"}
84    ]]
85
86    def __init__(self, require_config_file=True):
87        # fxbuild style:
88        buildscript_kwargs = {
89            'all_actions': [
90                "clone-locales",
91                "list-locales",
92                "setup",
93                "repack",
94                "summary",
95            ],
96            'config': {
97                "ignore_locales": ["en-US"],
98                "locales_dir": "browser/locales",
99                "log_name": "single_locale",
100                "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
101            },
102        }
103
104        LocalesMixin.__init__(self)
105        BaseScript.__init__(
106            self,
107            config_options=self.config_options,
108            require_config_file=require_config_file,
109            **buildscript_kwargs
110        )
111
112        self.bootstrap_env = None
113        self.upload_env = None
114        self.upload_urls = {}
115        self.pushdate = None
116        # upload_files is a dictionary of files to upload, keyed by locale.
117        self.upload_files = {}
118
119    def _pre_config_lock(self, rw_config):
120        """replaces 'configuration_tokens' with their values, before the
121           configuration gets locked. If some of the configuration_tokens
122           are not present, stops the execution of the script"""
123        # now, only runtime_config_tokens should be present in config
124        # we should parse self.config and fail if any other  we spot any
125        # other token
126        tokens_left = set(self._get_configuration_tokens(self.config))
127        unknown_tokens = set(tokens_left) - set(runtime_config_tokens)
128        if unknown_tokens:
129            msg = ['unknown tokens in configuration:']
130            for t in unknown_tokens:
131                msg.append(t)
132            self.fatal(' '.join(msg))
133        self.info('configuration looks ok')
134        return
135
136    def _get_configuration_tokens(self, iterable):
137        """gets a list of tokens in iterable"""
138        regex = re.compile('%\(\w+\)s')
139        results = []
140        try:
141            for element in iterable:
142                if isinstance(iterable, str):
143                    # this is a string, look for tokens
144                    # self.debug("{0}".format(re.findall(regex, element)))
145                    tokens = re.findall(regex, iterable)
146                    for token in tokens:
147                        # clean %(branch)s => branch
148                        # remove %(
149                        token_name = token.partition('%(')[2]
150                        # remove )s
151                        token_name = token_name.partition(')s')[0]
152                        results.append(token_name)
153                    break
154
155                elif isinstance(iterable, (list, tuple)):
156                    results.extend(self._get_configuration_tokens(element))
157
158                elif isinstance(iterable, dict):
159                    results.extend(self._get_configuration_tokens(iterable[element]))
160
161        except TypeError:
162            # element is a int/float/..., nothing to do here
163            pass
164
165        # remove duplicates, and return results
166
167        return list(set(results))
168
169    def __detokenise_element(self, config_option, token, value):
170        """reads config_options and returns a version of the same config_option
171           replacing token with value recursively"""
172        # config_option is a string, let's replace token with value
173        if isinstance(config_option, str):
174            # if token does not appear in this string,
175            # nothing happens and the original value is returned
176            return config_option.replace(token, value)
177        # it's a dictionary
178        elif isinstance(config_option, dict):
179            # replace token for each element of this dictionary
180            for element in config_option:
181                config_option[element] = self.__detokenise_element(
182                    config_option[element], token, value)
183            return config_option
184        # it's a list
185        elif isinstance(config_option, list):
186            # create a new list and append the replaced elements
187            new_list = []
188            for element in config_option:
189                new_list.append(self.__detokenise_element(element, token, value))
190            return new_list
191        elif isinstance(config_option, tuple):
192            # create a new list and append the replaced elements
193            new_list = []
194            for element in config_option:
195                new_list.append(self.__detokenise_element(element, token, value))
196            return tuple(new_list)
197        else:
198            # everything else, bool, number, ...
199            return config_option
200
201    # Helper methods {{{2
202    def query_bootstrap_env(self):
203        """returns the env for repacks"""
204        if self.bootstrap_env:
205            return self.bootstrap_env
206        config = self.config
207        replace_dict = self.query_abs_dirs()
208
209        bootstrap_env = self.query_env(partial_env=config.get("bootstrap_env"),
210                                       replace_dict=replace_dict)
211        if self.query_is_nightly():
212            bootstrap_env["IS_NIGHTLY"] = "yes"
213            # we might set update_channel explicitly
214            if config.get('update_channel'):
215                update_channel = config['update_channel']
216            else:  # Let's just give the generic channel based on branch.
217                update_channel = "nightly-%s" % (config['branch'],)
218            if isinstance(update_channel, unicode):
219                update_channel = update_channel.encode("utf-8")
220            bootstrap_env["MOZ_UPDATE_CHANNEL"] = update_channel
221            self.info("Update channel set to: {}".format(bootstrap_env["MOZ_UPDATE_CHANNEL"]))
222        self.bootstrap_env = bootstrap_env
223        return self.bootstrap_env
224
225    def _query_upload_env(self):
226        """returns the environment used for the upload step"""
227        if self.upload_env:
228            return self.upload_env
229        config = self.config
230
231        upload_env = self.query_env(partial_env=config.get("upload_env"))
232        # check if there are any extra option from the platform configuration
233        # and append them to the env
234
235        if 'upload_env_extra' in config:
236            for extra in config['upload_env_extra']:
237                upload_env[extra] = config['upload_env_extra'][extra]
238
239        self.upload_env = upload_env
240        return self.upload_env
241
242    def query_l10n_env(self):
243        l10n_env = self._query_upload_env().copy()
244        l10n_env.update(self.query_bootstrap_env())
245        return l10n_env
246
247    def _query_make_variable(self, variable, make_args=None):
248        """returns the value of make echo-variable-<variable>
249           it accepts extra make arguements (make_args)
250        """
251        dirs = self.query_abs_dirs()
252        make_args = make_args or []
253        target = ["echo-variable-%s" % variable] + make_args
254        cwd = dirs['abs_locales_dir']
255        raw_output = self._get_output_from_make(target, cwd=cwd,
256                                                env=self.query_bootstrap_env())
257        # we want to log all the messages from make
258        output = []
259        for line in raw_output.split("\n"):
260            output.append(line.strip())
261        output = " ".join(output).strip()
262        self.info('echo-variable-%s: %s' % (variable, output))
263        return output
264
265    def _map(self, func, items):
266        """runs func for any item in items, calls the add_failure() for each
267           error. It assumes that function returns 0 when successful.
268           returns a two element tuple with (success_count, total_count)"""
269        success_count = 0
270        total_count = len(items)
271        name = func.__name__
272        for item in items:
273            result = func(item)
274            if result == SUCCESS:
275                #  success!
276                success_count += 1
277            else:
278                #  func failed...
279                message = 'failure: %s(%s)' % (name, item)
280                self.add_failure(item, message)
281        return (success_count, total_count)
282
283    # Actions {{{2
284    def clone_locales(self):
285        self.pull_locale_source()
286
287    def setup(self):
288        """setup step"""
289        self._run_tooltool()
290        self._copy_mozconfig()
291        self._mach_configure()
292        self._run_make_in_config_dir()
293        self.make_wget_en_US()
294        self.make_unpack_en_US()
295
296    def _run_make_in_config_dir(self):
297        """this step creates nsinstall, needed my make_wget_en_US()
298        """
299        dirs = self.query_abs_dirs()
300        config_dir = os.path.join(dirs['abs_objdir'], 'config')
301        env = self.query_bootstrap_env()
302        return self._make(target=['export'], cwd=config_dir, env=env)
303
304    def _copy_mozconfig(self):
305        """copies the mozconfig file into abs_mozilla_dir/.mozconfig
306           and logs the content
307        """
308        config = self.config
309        dirs = self.query_abs_dirs()
310        src = get_mozconfig_path(self, config, dirs)
311        dst = os.path.join(dirs['abs_mozilla_dir'], '.mozconfig')
312        self.copyfile(src, dst)
313        self.read_from_file(dst, verbose=True)
314
315    def _mach(self, target, env, halt_on_failure=True, output_parser=None):
316        dirs = self.query_abs_dirs()
317        mach = self._get_mach_executable()
318        return self.run_command(mach + target,
319                                halt_on_failure=True,
320                                env=env,
321                                cwd=dirs['abs_mozilla_dir'],
322                                output_parser=None)
323
324    def _mach_configure(self):
325        """calls mach configure"""
326        env = self.query_bootstrap_env()
327        target = ["configure"]
328        return self._mach(target=target, env=env)
329
330    def _get_mach_executable(self):
331        return [sys.executable, 'mach']
332
333    def _get_make_executable(self):
334        config = self.config
335        dirs = self.query_abs_dirs()
336        if config.get('enable_mozmake'):  # e.g. windows
337            make = r"/".join([dirs['abs_mozilla_dir'], 'mozmake.exe'])
338            # mysterious subprocess errors, let's try to fix this path...
339            make = make.replace('\\', '/')
340            make = [make]
341        else:
342            make = ['make']
343        return make
344
345    def _make(self, target, cwd, env, error_list=MakefileErrorList,
346              halt_on_failure=True, output_parser=None):
347        """Runs make. Returns the exit code"""
348        make = self._get_make_executable()
349        if target:
350            make = make + target
351        return self.run_command(make,
352                                cwd=cwd,
353                                env=env,
354                                error_list=error_list,
355                                halt_on_failure=halt_on_failure,
356                                output_parser=output_parser)
357
358    def _get_output_from_make(self, target, cwd, env, halt_on_failure=True, ignore_errors=False):
359        """runs make and returns the output of the command"""
360        make = self._get_make_executable()
361        return self.get_output_from_command(make + target,
362                                            cwd=cwd,
363                                            env=env,
364                                            silent=True,
365                                            halt_on_failure=halt_on_failure,
366                                            ignore_errors=ignore_errors)
367
368    def make_unpack_en_US(self):
369        """wrapper for make unpack"""
370        config = self.config
371        dirs = self.query_abs_dirs()
372        env = self.query_bootstrap_env()
373        cwd = os.path.join(dirs['abs_objdir'], config['locales_dir'])
374        return self._make(target=["unpack"], cwd=cwd, env=env)
375
376    def make_wget_en_US(self):
377        """wrapper for make wget-en-US"""
378        env = self.query_bootstrap_env()
379        dirs = self.query_abs_dirs()
380        cwd = dirs['abs_locales_dir']
381        return self._make(target=["wget-en-US"], cwd=cwd, env=env)
382
383    def make_upload(self, locale):
384        """wrapper for make upload command"""
385        env = self.query_l10n_env()
386        dirs = self.query_abs_dirs()
387        target = ['upload', 'AB_CD=%s' % (locale)]
388        cwd = dirs['abs_locales_dir']
389        parser = MakeUploadOutputParser(config=self.config,
390                                        log_obj=self.log_obj)
391        retval = self._make(target=target, cwd=cwd, env=env,
392                            halt_on_failure=False, output_parser=parser)
393        if retval == SUCCESS:
394            self.info('Upload successful (%s)' % locale)
395            ret = SUCCESS
396        else:
397            self.error('failed to upload %s' % locale)
398            ret = FAILURE
399
400        if ret == FAILURE:
401            # If we failed above, we shouldn't even attempt a SIMPLE_NAME move
402            # even if we are configured to do so
403            return ret
404
405        # XXX Move the files to a SIMPLE_NAME format until we can enable
406        #     Simple names in the build system
407        if self.config.get("simple_name_move"):
408            # Assume an UPLOAD PATH
409            upload_target = self.config["upload_env"]["UPLOAD_PATH"]
410            target_path = os.path.join(upload_target, locale)
411            self.mkdir_p(target_path)
412            glob_name = "*.%s.*" % locale
413            matches = (glob.glob(os.path.join(upload_target, glob_name)) +
414                       glob.glob(os.path.join(upload_target, 'update', glob_name)) +
415                       glob.glob(os.path.join(upload_target, '*', 'xpi', glob_name)) +
416                       glob.glob(os.path.join(upload_target, 'install', 'sea', glob_name)) +
417                       glob.glob(os.path.join(upload_target, 'setup.exe')) +
418                       glob.glob(os.path.join(upload_target, 'setup-stub.exe')))
419            targets_exts = ["tar.bz2", "dmg", "langpack.xpi",
420                            "checksums", "zip",
421                            "installer.exe", "installer-stub.exe"]
422            targets = [(".%s" % (ext,), "target.%s" % (ext,)) for ext in targets_exts]
423            targets.extend([(f, f) for f in 'setup.exe', 'setup-stub.exe'])
424            for f in matches:
425                possible_targets = [
426                    (tail, target_file)
427                    for (tail, target_file) in targets
428                    if f.endswith(tail)
429                ]
430                if len(possible_targets) == 1:
431                    _, target_file = possible_targets[0]
432                    # Remove from list of available options for this locale
433                    targets.remove(possible_targets[0])
434                else:
435                    # wasn't valid (or already matched)
436                    raise RuntimeError("Unexpected matching file name encountered: %s"
437                                       % f)
438                self.move(os.path.join(f),
439                          os.path.join(target_path, target_file))
440            self.log("Converted uploads for %s to simple names" % locale)
441        return ret
442
443    def set_upload_files(self, locale):
444        # The tree doesn't have a good way of exporting the list of files
445        # created during locale generation, but we can grab them by echoing the
446        # UPLOAD_FILES variable for each locale.
447        env = self.query_l10n_env()
448        target = ['echo-variable-UPLOAD_FILES', 'echo-variable-CHECKSUM_FILES',
449                  'AB_CD=%s' % locale]
450        dirs = self.query_abs_dirs()
451        cwd = dirs['abs_locales_dir']
452        # Bug 1242771 - echo-variable-UPLOAD_FILES via mozharness fails when stderr is found
453        #    we should ignore stderr as unfortunately it's expected when parsing for values
454        output = self._get_output_from_make(target=target, cwd=cwd, env=env,
455                                            ignore_errors=True)
456        self.info('UPLOAD_FILES is "%s"' % output)
457        files = shlex.split(output)
458        if not files:
459            self.error('failed to get upload file list for locale %s' % locale)
460            return FAILURE
461
462        self.upload_files[locale] = [
463            os.path.abspath(os.path.join(cwd, f)) for f in files
464        ]
465        return SUCCESS
466
467    def make_installers(self, locale):
468        """wrapper for make installers-(locale)"""
469        env = self.query_l10n_env()
470        self._copy_mozconfig()
471        dirs = self.query_abs_dirs()
472        cwd = os.path.join(dirs['abs_locales_dir'])
473        target = ["installers-%s" % locale, ]
474        return self._make(target=target, cwd=cwd,
475                          env=env, halt_on_failure=False)
476
477    def repack_locale(self, locale):
478        """wraps the logic for make installers and generating
479           complete updates."""
480
481        # run make installers
482        if self.make_installers(locale) != SUCCESS:
483            self.error("make installers-%s failed" % (locale))
484            return FAILURE
485
486        # now try to upload the artifacts
487        if self.make_upload(locale):
488            self.error("make upload for locale %s failed!" % (locale))
489            return FAILURE
490
491        # set_upload_files() should be called after make upload, to make sure
492        # we have all files in place (checksums, etc)
493        if self.set_upload_files(locale):
494            self.error("failed to get list of files to upload for locale %s" % locale)
495            return FAILURE
496
497        return SUCCESS
498
499    def repack(self):
500        """creates the repacks and udpates"""
501        self._map(self.repack_locale, self.query_locales())
502
503    def _query_objdir(self):
504        """returns objdir name from configuration"""
505        return self.config['objdir']
506
507    def query_abs_dirs(self):
508        if self.abs_dirs:
509            return self.abs_dirs
510        abs_dirs = super(DesktopSingleLocale, self).query_abs_dirs()
511        for directory in abs_dirs:
512            value = abs_dirs[directory]
513            abs_dirs[directory] = value
514        dirs = {}
515        dirs['abs_tools_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tools')
516        dirs['abs_src_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'src')
517        for key in dirs.keys():
518            if key not in abs_dirs:
519                abs_dirs[key] = dirs[key]
520        self.abs_dirs = abs_dirs
521        return self.abs_dirs
522
523    # TODO: replace with ToolToolMixin
524    def _get_tooltool_auth_file(self):
525        # set the default authentication file based on platform; this
526        # corresponds to where puppet puts the token
527        if 'tooltool_authentication_file' in self.config:
528            fn = self.config['tooltool_authentication_file']
529        elif self._is_windows():
530            fn = r'c:\builds\relengapi.tok'
531        else:
532            fn = '/builds/relengapi.tok'
533
534        # if the file doesn't exist, don't pass it to tooltool (it will just
535        # fail).  In taskcluster, this will work OK as the relengapi-proxy will
536        # take care of auth.  Everywhere else, we'll get auth failures if
537        # necessary.
538        if os.path.exists(fn):
539            return fn
540
541    def _run_tooltool(self):
542        env = self.query_bootstrap_env()
543        config = self.config
544        dirs = self.query_abs_dirs()
545        toolchains = os.environ.get('MOZ_TOOLCHAINS')
546        manifest_src = os.environ.get('TOOLTOOL_MANIFEST')
547        if not manifest_src:
548            manifest_src = config.get('tooltool_manifest_src')
549        if not manifest_src and not toolchains:
550            return
551        python = sys.executable
552
553        cmd = [
554            python, '-u',
555            os.path.join(dirs['abs_mozilla_dir'], 'mach'),
556            'artifact',
557            'toolchain',
558            '-v',
559            '--retry', '4',
560            '--artifact-manifest',
561            os.path.join(dirs['abs_mozilla_dir'], 'toolchains.json'),
562        ]
563        if manifest_src:
564            cmd.extend([
565                '--tooltool-manifest',
566                os.path.join(dirs['abs_mozilla_dir'], manifest_src),
567                '--tooltool-url',
568                config['tooltool_url'],
569            ])
570            auth_file = self._get_tooltool_auth_file()
571            if auth_file and os.path.exists(auth_file):
572                cmd.extend(['--authentication-file', auth_file])
573        cache = config['bootstrap_env'].get('TOOLTOOL_CACHE')
574        if cache:
575            cmd.extend(['--cache-dir', cache])
576        if toolchains:
577            cmd.extend(toolchains.split())
578        self.info(str(cmd))
579        self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True,
580                         env=env)
581
582
583# main {{{
584if __name__ == '__main__':
585    single_locale = DesktopSingleLocale()
586    single_locale.run_and_exit()
587