1#!/usr/bin/env python
2# ***** BEGIN LICENSE BLOCK *****
3# This Source Code Form is subject to the terms of the Mozilla Public
4# License, v. 2.0. If a copy of the MPL was not distributed with this file,
5# You can obtain one at http://mozilla.org/MPL/2.0/.
6# ***** END LICENSE BLOCK *****
7""" l10n_bumper.py
8
9    Updates a gecko repo with up to date changesets from l10n.mozilla.org.
10
11    Specifically, it updates l10n-changesets.json which is used by mobile releases.
12
13    This is to allow for `mach taskgraph` to reference specific l10n revisions
14    without having to resort to task.extra or commandline base64 json hacks.
15"""
16from __future__ import absolute_import
17import codecs
18import os
19import pprint
20import sys
21import time
22
23try:
24    import simplejson as json
25
26    assert json
27except ImportError:
28    import json
29
30sys.path.insert(1, os.path.dirname(sys.path[0]))
31
32from mozharness.base.errors import HgErrorList
33from mozharness.base.vcs.vcsbase import VCSScript
34from mozharness.base.log import FATAL
35
36
37class L10nBumper(VCSScript):
38    config_options = [
39        [
40            [
41                "--ignore-closed-tree",
42            ],
43            {
44                "action": "store_true",
45                "dest": "ignore_closed_tree",
46                "default": False,
47                "help": "Bump l10n changesets on a closed tree.",
48            },
49        ],
50        [
51            [
52                "--build",
53            ],
54            {
55                "action": "store_false",
56                "dest": "dontbuild",
57                "default": True,
58                "help": "Trigger new builds on push.",
59            },
60        ],
61    ]
62
63    def __init__(self, require_config_file=True):
64        super(L10nBumper, self).__init__(
65            all_actions=[
66                "clobber",
67                "check-treestatus",
68                "checkout-gecko",
69                "bump-changesets",
70                "push",
71                "push-loop",
72            ],
73            default_actions=[
74                "push-loop",
75            ],
76            require_config_file=require_config_file,
77            config_options=self.config_options,
78            # Default config options
79            config={
80                "treestatus_base_url": "https://treestatus.mozilla-releng.net",
81                "log_max_rotate": 99,
82            },
83        )
84
85    # Helper methods {{{1
86    def query_abs_dirs(self):
87        if self.abs_dirs:
88            return self.abs_dirs
89
90        abs_dirs = super(L10nBumper, self).query_abs_dirs()
91
92        abs_dirs.update(
93            {
94                "gecko_local_dir": os.path.join(
95                    abs_dirs["abs_work_dir"],
96                    self.config.get(
97                        "gecko_local_dir",
98                        os.path.basename(self.config["gecko_pull_url"]),
99                    ),
100                ),
101            }
102        )
103        self.abs_dirs = abs_dirs
104        return self.abs_dirs
105
106    def hg_commit(self, path, repo_path, message):
107        """
108        Commits changes in repo_path, with specified user and commit message
109        """
110        user = self.config["hg_user"]
111        hg = self.query_exe("hg", return_type="list")
112        env = self.query_env(partial_env={"LANG": "en_US.UTF-8"})
113        cmd = hg + ["add", path]
114        self.run_command(cmd, cwd=repo_path, env=env)
115        cmd = hg + ["commit", "-u", user, "-m", message]
116        self.run_command(cmd, cwd=repo_path, env=env)
117
118    def hg_push(self, repo_path):
119        hg = self.query_exe("hg", return_type="list")
120        command = hg + [
121            "push",
122            "-e",
123            "ssh -oIdentityFile=%s -l %s"
124            % (
125                self.config["ssh_key"],
126                self.config["ssh_user"],
127            ),
128            "-r",
129            ".",
130            self.config["gecko_push_url"],
131        ]
132        status = self.run_command(command, cwd=repo_path, error_list=HgErrorList)
133        if status != 0:
134            # We failed; get back to a known state so we can either retry
135            # or fail out and continue later.
136            self.run_command(
137                hg
138                + ["--config", "extensions.mq=", "strip", "--no-backup", "outgoing()"],
139                cwd=repo_path,
140            )
141            self.run_command(hg + ["up", "-C"], cwd=repo_path)
142            self.run_command(
143                hg + ["--config", "extensions.purge=", "purge", "--all"], cwd=repo_path
144            )
145            return False
146        return True
147
148    def _read_json(self, path):
149        contents = self.read_from_file(path)
150        try:
151            json_contents = json.loads(contents)
152            return json_contents
153        except ValueError:
154            self.error("%s is invalid json!" % path)
155
156    def _read_version(self, path):
157        contents = self.read_from_file(path).split("\n")[0]
158        return contents.split(".")
159
160    def _build_locale_map(self, old_contents, new_contents):
161        locale_map = {}
162        for key in old_contents:
163            if key not in new_contents:
164                locale_map[key] = "removed"
165        for k, v in new_contents.items():
166            if old_contents.get(k, {}).get("revision") != v["revision"]:
167                locale_map[k] = v["revision"]
168            elif old_contents.get(k, {}).get("platforms") != v["platforms"]:
169                locale_map[k] = v["platforms"]
170        return locale_map
171
172    def _build_platform_dict(self, bump_config):
173        dirs = self.query_abs_dirs()
174        repo_path = dirs["gecko_local_dir"]
175        platform_dict = {}
176        ignore_config = bump_config.get("ignore_config", {})
177        for platform_config in bump_config["platform_configs"]:
178            path = os.path.join(repo_path, platform_config["path"])
179            self.info(
180                "Reading %s for %s locales..." % (path, platform_config["platforms"])
181            )
182            contents = self.read_from_file(path)
183            for locale in contents.splitlines():
184                # locale is 1st word in line in shipped-locales
185                if platform_config.get("format") == "shipped-locales":
186                    locale = locale.split(" ")[0]
187                existing_platforms = set(
188                    platform_dict.get(locale, {}).get("platforms", [])
189                )
190                platforms = set(platform_config["platforms"])
191                ignore_platforms = set(ignore_config.get(locale, []))
192                platforms = (platforms | existing_platforms) - ignore_platforms
193                platform_dict[locale] = {"platforms": sorted(list(platforms))}
194        self.info("Built platform_dict:\n%s" % pprint.pformat(platform_dict))
195        return platform_dict
196
197    def _build_revision_dict(self, bump_config, version_list):
198        self.info("Building revision dict...")
199        platform_dict = self._build_platform_dict(bump_config)
200        revision_dict = {}
201        if bump_config.get("revision_url"):
202            repl_dict = {
203                "MAJOR_VERSION": version_list[0],
204                "COMBINED_MAJOR_VERSION": str(
205                    int(version_list[0]) + int(version_list[1])
206                ),
207            }
208
209            url = bump_config["revision_url"] % repl_dict
210            path = self.download_file(url, error_level=FATAL)
211            revision_info = self.read_from_file(path)
212            self.info("Got %s" % revision_info)
213            for line in revision_info.splitlines():
214                locale, revision = line.split(" ")
215                if locale in platform_dict:
216                    revision_dict[locale] = platform_dict[locale]
217                    revision_dict[locale]["revision"] = revision
218        else:
219            for k, v in platform_dict.items():
220                v["revision"] = "default"
221                revision_dict[k] = v
222        self.info("revision_dict:\n%s" % pprint.pformat(revision_dict))
223        return revision_dict
224
225    def build_commit_message(self, name, locale_map):
226        comments = ""
227        approval_str = "r=release a=l10n-bump"
228        for locale, revision in sorted(locale_map.items()):
229            comments += "%s -> %s\n" % (locale, revision)
230        if self.config["dontbuild"]:
231            approval_str += " DONTBUILD"
232        if self.config["ignore_closed_tree"]:
233            approval_str += " CLOSED TREE"
234        message = "no bug - Bumping %s %s\n\n" % (name, approval_str)
235        message += comments
236        message = message.encode("utf-8")
237        return message
238
239    def query_treestatus(self):
240        "Return True if we can land based on treestatus"
241        c = self.config
242        dirs = self.query_abs_dirs()
243        tree = c.get(
244            "treestatus_tree", os.path.basename(c["gecko_pull_url"].rstrip("/"))
245        )
246        treestatus_url = "%s/trees/%s" % (c["treestatus_base_url"], tree)
247        treestatus_json = os.path.join(dirs["abs_work_dir"], "treestatus.json")
248        if not os.path.exists(dirs["abs_work_dir"]):
249            self.mkdir_p(dirs["abs_work_dir"])
250        self.rmtree(treestatus_json)
251
252        self.run_command(
253            ["curl", "--retry", "4", "-o", treestatus_json, treestatus_url],
254            throw_exception=True,
255        )
256
257        treestatus = self._read_json(treestatus_json)
258        if treestatus["result"]["status"] != "closed":
259            self.info(
260                "treestatus is %s - assuming we can land"
261                % repr(treestatus["result"]["status"])
262            )
263            return True
264
265        return False
266
267    # Actions {{{1
268    def check_treestatus(self):
269        if not self.config["ignore_closed_tree"] and not self.query_treestatus():
270            self.info("breaking early since treestatus is closed")
271            sys.exit(0)
272
273    def checkout_gecko(self):
274        c = self.config
275        dirs = self.query_abs_dirs()
276        dest = dirs["gecko_local_dir"]
277        repos = [
278            {
279                "repo": c["gecko_pull_url"],
280                "tag": c.get("gecko_tag", "default"),
281                "dest": dest,
282                "vcs": "hg",
283            }
284        ]
285        self.vcs_checkout_repos(repos)
286
287    def bump_changesets(self):
288        dirs = self.query_abs_dirs()
289        repo_path = dirs["gecko_local_dir"]
290        version_path = os.path.join(repo_path, self.config["version_path"])
291        changes = False
292        version_list = self._read_version(version_path)
293        for bump_config in self.config["bump_configs"]:
294            path = os.path.join(repo_path, bump_config["path"])
295            # For now, assume format == 'json'.  When we add desktop support,
296            # we may need to add flatfile support
297            if os.path.exists(path):
298                old_contents = self._read_json(path)
299            else:
300                old_contents = {}
301
302            new_contents = self._build_revision_dict(bump_config, version_list)
303
304            if new_contents == old_contents:
305                continue
306            # super basic sanity check
307            if not isinstance(new_contents, dict) or len(new_contents) < 5:
308                self.error(
309                    "Cowardly refusing to land a broken-seeming changesets file!"
310                )
311                continue
312
313            # Write to disk
314            content_string = json.dumps(
315                new_contents,
316                sort_keys=True,
317                indent=4,
318                separators=(",", ": "),
319            )
320            fh = codecs.open(path, encoding="utf-8", mode="w+")
321            fh.write(content_string + "\n")
322            fh.close()
323
324            locale_map = self._build_locale_map(old_contents, new_contents)
325
326            # Commit
327            message = self.build_commit_message(bump_config["name"], locale_map)
328            self.hg_commit(path, repo_path, message)
329            changes = True
330        return changes
331
332    def push(self):
333        dirs = self.query_abs_dirs()
334        repo_path = dirs["gecko_local_dir"]
335        return self.hg_push(repo_path)
336
337    def push_loop(self):
338        max_retries = 5
339        for _ in range(max_retries):
340            changed = False
341            if not self.config["ignore_closed_tree"] and not self.query_treestatus():
342                # Tree is closed; exit early to avoid a bunch of wasted time
343                self.info("breaking early since treestatus is closed")
344                break
345
346            self.checkout_gecko()
347            if self.bump_changesets():
348                changed = True
349
350            if not changed:
351                # Nothing changed, we're all done
352                self.info("No changes - all done")
353                break
354
355            if self.push():
356                # We did it! Hurray!
357                self.info("Great success!")
358                break
359            # If we're here, then the push failed. It also stripped any
360            # outgoing commits, so we should be in a pristine state again
361            # Empty our local cache of manifests so they get loaded again next
362            # time through this loop. This makes sure we get fresh upstream
363            # manifests, and avoids problems like bug 979080
364            self.device_manifests = {}
365
366            # Sleep before trying again
367            self.info("Sleeping 60 before trying again")
368            time.sleep(60)
369        else:
370            self.fatal("Didn't complete successfully (hit max_retries)")
371
372        # touch status file for nagios
373        dirs = self.query_abs_dirs()
374        status_path = os.path.join(dirs["base_work_dir"], self.config["status_path"])
375        self._touch_file(status_path)
376
377
378# __main__ {{{1
379if __name__ == "__main__":
380    bumper = L10nBumper()
381    bumper.run_and_exit()
382