1#!/usr/bin/env python
2# ***** BEGIN LICENSE BLOCK *****
3# This Source Code Form is subject to the terms of the Mozilla Public
4# License, v. 2.0. If a copy of the MPL was not distributed with this file,
5# You can obtain one at http://mozilla.org/MPL/2.0/.
6# ***** END LICENSE BLOCK *****
7"""desktop_l10n.py
8
9This script manages Desktop repacks for nightly builds.
10"""
11from __future__ import absolute_import
12import os
13import glob
14import sys
15import shlex
16
17# load modules from parent dir
18sys.path.insert(1, os.path.dirname(sys.path[0]))  # noqa
19
20from mozharness.base.errors import MakefileErrorList
21from mozharness.base.script import BaseScript
22from mozharness.base.vcs.vcsbase import VCSMixin
23from mozharness.mozilla.automation import AutomationMixin
24from mozharness.mozilla.building.buildbase import (
25    MakeUploadOutputParser,
26    get_mozconfig_path,
27)
28from mozharness.mozilla.l10n.locales import LocalesMixin
29
30try:
31    import simplejson as json
32
33    assert json
34except ImportError:
35    import json
36
37
38# needed by _map
39SUCCESS = 0
40FAILURE = 1
41
42SUCCESS_STR = "Success"
43FAILURE_STR = "Failed"
44
45
46# DesktopSingleLocale {{{1
47class DesktopSingleLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript):
48    """Manages desktop repacks"""
49
50    config_options = [
51        [
52            [
53                "--locale",
54            ],
55            {
56                "action": "extend",
57                "dest": "locales",
58                "type": "string",
59                "help": "Specify the locale(s) to sign and update. Optionally pass"
60                " revision separated by colon, en-GB:default.",
61            },
62        ],
63        [
64            [
65                "--tag-override",
66            ],
67            {
68                "action": "store",
69                "dest": "tag_override",
70                "type": "string",
71                "help": "Override the tags set for all repos",
72            },
73        ],
74        [
75            [
76                "--en-us-installer-url",
77            ],
78            {
79                "action": "store",
80                "dest": "en_us_installer_url",
81                "type": "string",
82                "help": "Specify the url of the en-us binary",
83            },
84        ],
85    ]
86
87    def __init__(self, require_config_file=True):
88        # fxbuild style:
89        buildscript_kwargs = {
90            "all_actions": [
91                "clone-locales",
92                "list-locales",
93                "setup",
94                "repack",
95                "summary",
96            ],
97            "config": {
98                "ignore_locales": ["en-US"],
99                "locales_dir": "browser/locales",
100                "log_name": "single_locale",
101                "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
102            },
103        }
104
105        LocalesMixin.__init__(self)
106        BaseScript.__init__(
107            self,
108            config_options=self.config_options,
109            require_config_file=require_config_file,
110            **buildscript_kwargs
111        )
112
113        self.bootstrap_env = None
114        self.upload_env = None
115        self.upload_urls = {}
116        self.pushdate = None
117        # upload_files is a dictionary of files to upload, keyed by locale.
118        self.upload_files = {}
119
120    # Helper methods {{{2
121    def query_bootstrap_env(self):
122        """returns the env for repacks"""
123        if self.bootstrap_env:
124            return self.bootstrap_env
125        config = self.config
126        abs_dirs = self.query_abs_dirs()
127
128        bootstrap_env = self.query_env(
129            partial_env=config.get("bootstrap_env"), replace_dict=abs_dirs
130        )
131
132        bootstrap_env["L10NBASEDIR"] = abs_dirs["abs_l10n_dir"]
133        if self.query_is_nightly():
134            # we might set update_channel explicitly
135            if config.get("update_channel"):
136                update_channel = config["update_channel"]
137            else:  # Let's just give the generic channel based on branch.
138                update_channel = "nightly-%s" % (config["branch"],)
139            if not isinstance(update_channel, bytes):
140                update_channel = update_channel.encode("utf-8")
141            bootstrap_env["MOZ_UPDATE_CHANNEL"] = update_channel
142            self.info(
143                "Update channel set to: {}".format(bootstrap_env["MOZ_UPDATE_CHANNEL"])
144            )
145        self.bootstrap_env = bootstrap_env
146        return self.bootstrap_env
147
148    def _query_upload_env(self):
149        """returns the environment used for the upload step"""
150        if self.upload_env:
151            return self.upload_env
152        config = self.config
153
154        upload_env = self.query_env(partial_env=config.get("upload_env"))
155        # check if there are any extra option from the platform configuration
156        # and append them to the env
157
158        if "upload_env_extra" in config:
159            for extra in config["upload_env_extra"]:
160                upload_env[extra] = config["upload_env_extra"][extra]
161
162        self.upload_env = upload_env
163        return self.upload_env
164
165    def query_l10n_env(self):
166        l10n_env = self._query_upload_env().copy()
167        l10n_env.update(self.query_bootstrap_env())
168        return l10n_env
169
170    def _query_make_variable(self, variable, make_args=None):
171        """returns the value of make echo-variable-<variable>
172        it accepts extra make arguements (make_args)
173        """
174        dirs = self.query_abs_dirs()
175        make_args = make_args or []
176        target = ["echo-variable-%s" % variable] + make_args
177        cwd = dirs["abs_locales_dir"]
178        raw_output = self._get_output_from_make(
179            target, cwd=cwd, env=self.query_bootstrap_env()
180        )
181        # we want to log all the messages from make
182        output = []
183        for line in raw_output.split("\n"):
184            output.append(line.strip())
185        output = " ".join(output).strip()
186        self.info("echo-variable-%s: %s" % (variable, output))
187        return output
188
189    def _map(self, func, items):
190        """runs func for any item in items, calls the add_failure() for each
191        error. It assumes that function returns 0 when successful.
192        returns a two element tuple with (success_count, total_count)"""
193        success_count = 0
194        total_count = len(items)
195        name = func.__name__
196        for item in items:
197            result = func(item)
198            if result == SUCCESS:
199                #  success!
200                success_count += 1
201            else:
202                #  func failed...
203                message = "failure: %s(%s)" % (name, item)
204                self.add_failure(item, message)
205        return (success_count, total_count)
206
207    # Actions {{{2
208    def clone_locales(self):
209        self.pull_locale_source()
210
211    def setup(self):
212        """setup step"""
213        self._run_tooltool()
214        self._copy_mozconfig()
215        self._mach_configure()
216        self._run_make_in_config_dir()
217        self.make_wget_en_US()
218        self.make_unpack_en_US()
219
220    def _run_make_in_config_dir(self):
221        """this step creates nsinstall, needed my make_wget_en_US()"""
222        dirs = self.query_abs_dirs()
223        config_dir = os.path.join(dirs["abs_obj_dir"], "config")
224        env = self.query_bootstrap_env()
225        return self._make(target=["export"], cwd=config_dir, env=env)
226
227    def _copy_mozconfig(self):
228        """copies the mozconfig file into abs_src_dir/.mozconfig
229        and logs the content
230        """
231        config = self.config
232        dirs = self.query_abs_dirs()
233        src = get_mozconfig_path(self, config, dirs)
234        dst = os.path.join(dirs["abs_src_dir"], ".mozconfig")
235        self.copyfile(src, dst)
236        self.read_from_file(dst, verbose=True)
237
238    def _mach(self, target, env, halt_on_failure=True, output_parser=None):
239        dirs = self.query_abs_dirs()
240        mach = self._get_mach_executable()
241        return self.run_command(
242            mach + target,
243            halt_on_failure=True,
244            env=env,
245            cwd=dirs["abs_src_dir"],
246            output_parser=None,
247        )
248
249    def _mach_configure(self):
250        """calls mach configure"""
251        env = self.query_bootstrap_env()
252        target = ["configure"]
253        return self._mach(target=target, env=env)
254
255    def _get_mach_executable(self):
256        return [sys.executable, "mach"]
257
258    def _make(
259        self,
260        target,
261        cwd,
262        env,
263        error_list=MakefileErrorList,
264        halt_on_failure=True,
265        output_parser=None,
266    ):
267        """Runs make. Returns the exit code"""
268        make = ["make"]
269        if target:
270            make = make + target
271        return self.run_command(
272            make,
273            cwd=cwd,
274            env=env,
275            error_list=error_list,
276            halt_on_failure=halt_on_failure,
277            output_parser=output_parser,
278        )
279
280    def _get_output_from_make(
281        self, target, cwd, env, halt_on_failure=True, ignore_errors=False
282    ):
283        """runs make and returns the output of the command"""
284        return self.get_output_from_command(
285            ["make"] + target,
286            cwd=cwd,
287            env=env,
288            silent=True,
289            halt_on_failure=halt_on_failure,
290            ignore_errors=ignore_errors,
291        )
292
293    def make_unpack_en_US(self):
294        """wrapper for make unpack"""
295        config = self.config
296        dirs = self.query_abs_dirs()
297        env = self.query_bootstrap_env()
298        cwd = os.path.join(dirs["abs_obj_dir"], config["locales_dir"])
299        return self._make(target=["unpack"], cwd=cwd, env=env)
300
301    def make_wget_en_US(self):
302        """wrapper for make wget-en-US"""
303        env = self.query_bootstrap_env()
304        dirs = self.query_abs_dirs()
305        cwd = dirs["abs_locales_dir"]
306        return self._make(target=["wget-en-US"], cwd=cwd, env=env)
307
308    def make_upload(self, locale):
309        """wrapper for make upload command"""
310        env = self.query_l10n_env()
311        dirs = self.query_abs_dirs()
312        target = ["upload", "AB_CD=%s" % (locale)]
313        cwd = dirs["abs_locales_dir"]
314        parser = MakeUploadOutputParser(config=self.config, log_obj=self.log_obj)
315        retval = self._make(
316            target=target, cwd=cwd, env=env, halt_on_failure=False, output_parser=parser
317        )
318        if retval == SUCCESS:
319            self.info("Upload successful (%s)" % locale)
320            ret = SUCCESS
321        else:
322            self.error("failed to upload %s" % locale)
323            ret = FAILURE
324
325        if ret == FAILURE:
326            # If we failed above, we shouldn't even attempt a SIMPLE_NAME move
327            # even if we are configured to do so
328            return ret
329
330        # XXX Move the files to a SIMPLE_NAME format until we can enable
331        #     Simple names in the build system
332        if self.config.get("simple_name_move"):
333            # Assume an UPLOAD PATH
334            upload_target = self.config["upload_env"]["UPLOAD_PATH"]
335            target_path = os.path.join(upload_target, locale)
336            self.mkdir_p(target_path)
337            glob_name = "*.%s.*" % locale
338            matches = (
339                glob.glob(os.path.join(upload_target, glob_name))
340                + glob.glob(os.path.join(upload_target, "update", glob_name))
341                + glob.glob(os.path.join(upload_target, "*", "xpi", glob_name))
342                + glob.glob(os.path.join(upload_target, "install", "sea", glob_name))
343                + glob.glob(os.path.join(upload_target, "setup.exe"))
344                + glob.glob(os.path.join(upload_target, "setup-stub.exe"))
345            )
346            targets_exts = [
347                "tar.bz2",
348                "dmg",
349                "langpack.xpi",
350                "checksums",
351                "zip",
352                "installer.exe",
353                "installer-stub.exe",
354            ]
355            targets = [(".%s" % (ext,), "target.%s" % (ext,)) for ext in targets_exts]
356            targets.extend([(f, f) for f in ("setup.exe", "setup-stub.exe")])
357            for f in matches:
358                possible_targets = [
359                    (tail, target_file)
360                    for (tail, target_file) in targets
361                    if f.endswith(tail)
362                ]
363                if len(possible_targets) == 1:
364                    _, target_file = possible_targets[0]
365                    # Remove from list of available options for this locale
366                    targets.remove(possible_targets[0])
367                else:
368                    # wasn't valid (or already matched)
369                    raise RuntimeError(
370                        "Unexpected matching file name encountered: %s" % f
371                    )
372                self.move(os.path.join(f), os.path.join(target_path, target_file))
373            self.log("Converted uploads for %s to simple names" % locale)
374        return ret
375
376    def set_upload_files(self, locale):
377        # The tree doesn't have a good way of exporting the list of files
378        # created during locale generation, but we can grab them by echoing the
379        # UPLOAD_FILES variable for each locale.
380        env = self.query_l10n_env()
381        target = [
382            "echo-variable-UPLOAD_FILES",
383            "echo-variable-CHECKSUM_FILES",
384            "AB_CD=%s" % locale,
385        ]
386        dirs = self.query_abs_dirs()
387        cwd = dirs["abs_locales_dir"]
388        # Bug 1242771 - echo-variable-UPLOAD_FILES via mozharness fails when stderr is found
389        #    we should ignore stderr as unfortunately it's expected when parsing for values
390        output = self._get_output_from_make(
391            target=target, cwd=cwd, env=env, ignore_errors=True
392        )
393        self.info('UPLOAD_FILES is "%s"' % output)
394        files = shlex.split(output)
395        if not files:
396            self.error("failed to get upload file list for locale %s" % locale)
397            return FAILURE
398
399        self.upload_files[locale] = [
400            os.path.abspath(os.path.join(cwd, f)) for f in files
401        ]
402        return SUCCESS
403
404    def make_installers(self, locale):
405        """wrapper for make installers-(locale)"""
406        env = self.query_l10n_env()
407        env["PYTHONIOENCODING"] = "utf-8"
408        self._copy_mozconfig()
409        dirs = self.query_abs_dirs()
410        cwd = os.path.join(dirs["abs_locales_dir"])
411        target = [
412            "installers-%s" % locale,
413        ]
414        return self._make(target=target, cwd=cwd, env=env, halt_on_failure=False)
415
416    def repack_locale(self, locale):
417        """wraps the logic for make installers and generating
418        complete updates."""
419
420        # run make installers
421        if self.make_installers(locale) != SUCCESS:
422            self.error("make installers-%s failed" % (locale))
423            return FAILURE
424
425        # now try to upload the artifacts
426        if self.make_upload(locale):
427            self.error("make upload for locale %s failed!" % (locale))
428            return FAILURE
429
430        # set_upload_files() should be called after make upload, to make sure
431        # we have all files in place (checksums, etc)
432        if self.set_upload_files(locale):
433            self.error("failed to get list of files to upload for locale %s" % locale)
434            return FAILURE
435
436        return SUCCESS
437
438    def repack(self):
439        """creates the repacks and udpates"""
440        self._map(self.repack_locale, self.query_locales())
441
442    def _run_tooltool(self):
443        env = self.query_bootstrap_env()
444        config = self.config
445        dirs = self.query_abs_dirs()
446        manifest_src = os.environ.get("TOOLTOOL_MANIFEST")
447        if not manifest_src:
448            manifest_src = config.get("tooltool_manifest_src")
449        if not manifest_src:
450            return
451        python = sys.executable
452
453        cmd = [
454            python,
455            "-u",
456            os.path.join(dirs["abs_src_dir"], "mach"),
457            "artifact",
458            "toolchain",
459            "-v",
460            "--retry",
461            "4",
462            "--artifact-manifest",
463            os.path.join(dirs["abs_src_dir"], "toolchains.json"),
464        ]
465        if manifest_src:
466            cmd.extend(
467                [
468                    "--tooltool-manifest",
469                    os.path.join(dirs["abs_src_dir"], manifest_src),
470                ]
471            )
472        cache = config["bootstrap_env"].get("TOOLTOOL_CACHE")
473        if cache:
474            cmd.extend(["--cache-dir", cache])
475        self.info(str(cmd))
476        self.run_command(cmd, cwd=dirs["abs_src_dir"], halt_on_failure=True, env=env)
477
478
479# main {{{
480if __name__ == "__main__":
481    single_locale = DesktopSingleLocale()
482    single_locale.run_and_exit()
483