1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this,
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import, print_function, unicode_literals
6
7from appdirs import user_config_dir
8from hglib.error import CommandError
9from mach.decorators import (
10    CommandArgument,
11    CommandProvider,
12    Command,
13)
14from mach.base import FailedCommandError
15from mozbuild.base import MachCommandBase
16from mozrelease.scriptworker_canary import get_secret
17from pathlib import Path
18from redo import retry
19import argparse
20import logging
21import os
22import tempfile
23
24
25@CommandProvider
26class CompareLocales(MachCommandBase):
27    """Run compare-locales."""
28
29    @Command(
30        "compare-locales",
31        category="build",
32        description="Run source checks on a localization.",
33    )
34    @CommandArgument(
35        "config_paths",
36        metavar="l10n.toml",
37        nargs="+",
38        help="TOML or INI file for the project",
39    )
40    @CommandArgument(
41        "l10n_base_dir",
42        metavar="l10n-base-dir",
43        help="Parent directory of localizations",
44    )
45    @CommandArgument(
46        "locales",
47        nargs="*",
48        metavar="locale-code",
49        help="Locale code and top-level directory of " "each localization",
50    )
51    @CommandArgument(
52        "-q",
53        "--quiet",
54        action="count",
55        default=0,
56        help="""Show less data.
57Specified once, don't show obsolete entities. Specified twice, also hide
58missing entities. Specify thrice to exclude warnings and four times to
59just show stats""",
60    )
61    @CommandArgument(
62        "-m", "--merge", help="""Use this directory to stage merged files"""
63    )
64    @CommandArgument(
65        "--validate", action="store_true", help="Run compare-locales against reference"
66    )
67    @CommandArgument(
68        "--json",
69        help="""Serialize to JSON. Value is the name of
70the output file, pass "-" to serialize to stdout and hide the default output.
71""",
72    )
73    @CommandArgument(
74        "-D",
75        action="append",
76        metavar="var=value",
77        default=[],
78        dest="defines",
79        help="Overwrite variables in TOML files",
80    )
81    @CommandArgument(
82        "--full", action="store_true", help="Compare projects that are disabled"
83    )
84    @CommandArgument(
85        "--return-zero", action="store_true", help="Return 0 regardless of l10n status"
86    )
87    def compare(self, command_context, **kwargs):
88        from compare_locales.commands import CompareLocales
89
90        class ErrorHelper(object):
91            """Dummy ArgumentParser to marshall compare-locales
92            commandline errors to mach exceptions.
93            """
94
95            def error(self, msg):
96                raise FailedCommandError(msg)
97
98            def exit(self, message=None, status=0):
99                raise FailedCommandError(message, exit_code=status)
100
101        cmd = CompareLocales()
102        cmd.parser = ErrorHelper()
103        return cmd.handle(**kwargs)
104
105
106# https://stackoverflow.com/a/14117511
107def _positive_int(value):
108    value = int(value)
109    if value <= 0:
110        raise argparse.ArgumentTypeError(f"{value} must be a positive integer.")
111    return value
112
113
114class RetryError(Exception):
115    ...
116
117
118VCT_PATH = Path(".").resolve() / "vct"
119VCT_URL = "https://hg.mozilla.org/hgcustom/version-control-tools/"
120FXTREE_PATH = VCT_PATH / "hgext" / "firefoxtree"
121HGRC_PATH = Path(user_config_dir("hg")).joinpath("hgrc")
122
123
124@CommandProvider
125class CrossChannel(MachCommandBase):
126    """Run l10n cross-channel content generation."""
127
128    @Command(
129        "l10n-cross-channel",
130        category="misc",
131        description="Create cross-channel content.",
132    )
133    @CommandArgument(
134        "--strings-path",
135        "-s",
136        metavar="en-US",
137        type=Path,
138        default=Path("en-US"),
139        help="Path to mercurial repository for gecko-strings-quarantine",
140    )
141    @CommandArgument(
142        "--outgoing-path",
143        "-o",
144        type=Path,
145        help="create an outgoing() patch if there are changes",
146    )
147    @CommandArgument(
148        "--attempts",
149        type=_positive_int,
150        default=1,
151        help="Number of times to try (for automation)",
152    )
153    @CommandArgument(
154        "--ssh-secret",
155        action="store",
156        help="Taskcluster secret to use to push (for automation)",
157    )
158    @CommandArgument(
159        "actions",
160        choices=("prep", "create", "push"),
161        nargs="+",
162        # This help block will be poorly formatted until we fix bug 1714239
163        help="""
164        "prep": clone repos and pull heads.
165        "create": create the en-US strings commit an optionally create an
166                  outgoing() patch.
167        "push": push the en-US strings to the quarantine repo.
168        """,
169    )
170    def cross_channel(
171        self,
172        command_context,
173        strings_path,
174        outgoing_path,
175        actions,
176        attempts,
177        ssh_secret,
178        **kwargs,
179    ):
180        # This can be any path, as long as the name of the directory is en-US.
181        # Not entirely sure where this is a requirement; perhaps in l10n
182        # string manipulation logic?
183        if strings_path.name != "en-US":
184            raise FailedCommandError("strings_path needs to be named `en-US`")
185        self.activate_virtualenv()
186        # XXX pin python requirements
187        self.virtualenv_manager.install_pip_requirements(
188            Path(os.path.dirname(__file__)) / "requirements.in"
189        )
190        strings_path = strings_path.resolve()  # abspath
191        if outgoing_path:
192            outgoing_path = outgoing_path.resolve()  # abspath
193        try:
194            with tempfile.TemporaryDirectory() as ssh_key_dir:
195                retry(
196                    self._do_create_content,
197                    attempts=attempts,
198                    retry_exceptions=(RetryError,),
199                    args=(
200                        command_context,
201                        strings_path,
202                        outgoing_path,
203                        ssh_secret,
204                        Path(ssh_key_dir),
205                        actions,
206                    ),
207                )
208        except RetryError as exc:
209            raise FailedCommandError(exc) from exc
210
211    def _do_create_content(
212        self,
213        command_context,
214        strings_path,
215        outgoing_path,
216        ssh_secret,
217        ssh_key_dir,
218        actions,
219    ):
220
221        from mozxchannel import CrossChannelCreator, get_default_config
222
223        config = get_default_config(Path(self.topsrcdir), strings_path)
224        ccc = CrossChannelCreator(config)
225        status = 0
226        changes = False
227        ssh_key_secret = None
228        ssh_key_file = None
229
230        if "prep" in actions:
231            if ssh_secret:
232                if not os.environ.get("MOZ_AUTOMATION"):
233                    raise CommandError(
234                        "I don't know how to fetch the ssh secret outside of automation!"
235                    )
236                ssh_key_secret = get_secret(ssh_secret)
237                ssh_key_file = ssh_key_dir.joinpath("id_rsa")
238                ssh_key_file.write_text(ssh_key_secret["ssh_privkey"])
239                ssh_key_file.chmod(0o600)
240            # Set up firefoxtree for comm per bug 1659691 comment 22
241            if os.environ.get("MOZ_AUTOMATION") and not HGRC_PATH.exists():
242                self._clone_hg_repo(VCT_URL, VCT_PATH)
243                hgrc_content = [
244                    "[extensions]",
245                    f"firefoxtree = {FXTREE_PATH}",
246                    "",
247                    "[ui]",
248                    "username = trybld",
249                ]
250                if ssh_key_file:
251                    hgrc_content.extend(
252                        [
253                            f"ssh = ssh -i {ssh_key_file} -l {ssh_key_secret['user']}",
254                        ]
255                    )
256                HGRC_PATH.write_text("\n".join(hgrc_content))
257            if strings_path.exists() and self._check_outgoing(strings_path):
258                self._strip_outgoing(strings_path)
259            # Clone strings + source repos, pull heads
260            for repo_config in (config["strings"], *config["source"].values()):
261                if not repo_config["path"].exists():
262                    self._clone_hg_repo(repo_config["url"], str(repo_config["path"]))
263                for head in repo_config["heads"].keys():
264                    command = ["hg", "--cwd", str(repo_config["path"]), "pull"]
265                    command.append(head)
266                    status = self._retry_run_process(command, ensure_exit_code=False)
267                    if status not in (0, 255):  # 255 on pull with no changes
268                        raise RetryError(f"Failure on pull: status {status}!")
269                    if repo_config.get("update_on_pull"):
270                        command = [
271                            "hg",
272                            "--cwd",
273                            str(repo_config["path"]),
274                            "up",
275                            "-C",
276                            "-r",
277                            head,
278                        ]
279                        status = self._retry_run_process(
280                            command, ensure_exit_code=False
281                        )
282                        if status not in (0, 255):  # 255 on pull with no changes
283                            raise RetryError(f"Failure on update: status {status}!")
284                self._check_hg_repo(
285                    repo_config["path"], heads=repo_config.get("heads", {}).keys()
286                )
287        else:
288            self._check_hg_repo(strings_path)
289            for repo_config in config.get("source", {}).values():
290                self._check_hg_repo(
291                    repo_config["path"], heads=repo_config.get("heads", {}).keys()
292                )
293            if self._check_outgoing(strings_path):
294                raise RetryError(f"check: Outgoing changes in {strings_path}!")
295
296        if "create" in actions:
297            try:
298                status = ccc.create_content()
299                changes = True
300                self._create_outgoing_patch(outgoing_path, strings_path)
301            except CommandError as exc:
302                if exc.ret != 1:
303                    raise RetryError(exc) from exc
304                self.log(logging.INFO, "create", {}, "No new strings.")
305
306        if "push" in actions:
307            if changes:
308                self._retry_run_process(
309                    [
310                        "hg",
311                        "--cwd",
312                        str(strings_path),
313                        "push",
314                        "-r",
315                        ".",
316                        config["strings"]["push_url"],
317                    ],
318                    line_handler=print,
319                )
320            else:
321                self.log(logging.INFO, "push", {}, "Skipping empty push.")
322
323        return status
324
325    def _check_outgoing(self, strings_path):
326        status = self._retry_run_process(
327            ["hg", "--cwd", str(strings_path), "out", "-r", "."],
328            ensure_exit_code=False,
329        )
330        if status == 0:
331            return True
332        if status == 1:
333            return False
334        raise RetryError(
335            f"Outgoing check in {strings_path} returned unexpected {status}!"
336        )
337
338    def _strip_outgoing(self, strings_path):
339        self._retry_run_process(
340            [
341                "hg",
342                "--config",
343                "extensions.strip=",
344                "--cwd",
345                str(strings_path),
346                "strip",
347                "--no-backup",
348                "outgoing()",
349            ],
350        )
351
352    def _create_outgoing_patch(self, path, strings_path):
353        if not path:
354            return
355        if not path.parent.exists():
356            os.makedirs(path.parent)
357        with open(path, "w") as fh:
358
359            def writeln(line):
360                fh.write(f"{line}\n")
361
362            self._retry_run_process(
363                [
364                    "hg",
365                    "--cwd",
366                    str(strings_path),
367                    "log",
368                    "--patch",
369                    "--verbose",
370                    "-r",
371                    "outgoing()",
372                ],
373                line_handler=writeln,
374            )
375
376    def _retry_run_process(self, *args, error_msg=None, **kwargs):
377        try:
378            return self.run_process(*args, **kwargs)
379        except Exception as exc:
380            raise RetryError(error_msg or str(exc)) from exc
381
382    def _check_hg_repo(self, path, heads=None):
383        if not (path.is_dir() and (path / ".hg").is_dir()):
384            raise RetryError(f"{path} is not a Mercurial repository")
385        if heads:
386            for head in heads:
387                self._retry_run_process(
388                    ["hg", "--cwd", str(path), "log", "-r", head],
389                    error_msg=f"check: {path} has no head {head}!",
390                )
391
392    def _clone_hg_repo(self, url, path):
393        self._retry_run_process(["hg", "clone", url, str(path)])
394