1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this, 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from collections import defaultdict 6from dataclasses import dataclass, field 7from datetime import datetime 8import os 9import shutil 10from compare_locales import merge 11import hglib 12from pathlib import Path 13 14from mozpack import path as mozpath 15from . import projectconfig 16from . import source 17 18 19def get_default_config(topsrcdir, strings_path): 20 assert isinstance(topsrcdir, Path) 21 assert isinstance(strings_path, Path) 22 return { 23 "strings": { 24 "path": strings_path, 25 "url": "https://hg.mozilla.org/l10n/gecko-strings-quarantine/", 26 "heads": {"default": "default"}, 27 "update_on_pull": True, 28 "push_url": "ssh://hg.mozilla.org/l10n/gecko-strings-quarantine/", 29 }, 30 "source": { 31 "mozilla-unified": { 32 "path": topsrcdir, 33 "url": "https://hg.mozilla.org/mozilla-unified/", 34 "heads": { 35 # This list of repositories is ordered, starting with the 36 # one with the most recent content (central) to the oldest 37 # (ESR). In case two ESR versions are supported, the oldest 38 # ESR goes last (e.g. esr78 goes after esr91). 39 "central": "mozilla-central", 40 "beta": "releases/mozilla-beta", 41 "release": "releases/mozilla-release", 42 "esr91": "releases/mozilla-esr91", 43 }, 44 "config_files": [ 45 "browser/locales/l10n.toml", 46 "mobile/android/locales/l10n.toml", 47 ], 48 }, 49 "comm-central": { 50 "path": topsrcdir / "comm", 51 "post-clobber": True, 52 "url": "https://hg.mozilla.org/comm-central/", 53 "heads": { 54 # This list of repositories is ordered, starting with the 55 # one with the most recent content (central) to the oldest 56 # (ESR). In case two ESR versions are supported, the oldest 57 # ESR goes last (e.g. esr78 goes after esr91). 58 "comm": "comm-central", 59 "comm-beta": "releases/comm-beta", 60 "comm-esr91": "releases/comm-esr91", 61 }, 62 "config_files": [ 63 "comm/calendar/locales/l10n.toml", 64 "comm/mail/locales/l10n.toml", 65 "comm/suite/locales/l10n.toml", 66 ], 67 }, 68 }, 69 } 70 71 72@dataclass 73class TargetRevs: 74 target: bytes = None 75 revs: list = field(default_factory=list) 76 77 78@dataclass 79class CommitRev: 80 repo: str 81 rev: bytes 82 83 @property 84 def message(self): 85 return ( 86 f"X-Channel-Repo: {self.repo}\n" 87 f'X-Channel-Revision: {self.rev.decode("ascii")}' 88 ) 89 90 91class CrossChannelCreator: 92 def __init__(self, config): 93 self.config = config 94 self.strings_path = config["strings"]["path"] 95 self.message = ( 96 f"cross-channel content for {datetime.utcnow().strftime('%Y-%m-%d %H:%M')}" 97 ) 98 99 def create_content(self): 100 self.prune_target() 101 revs = [] 102 for repo_name, repo_config in self.config["source"].items(): 103 with hglib.open(repo_config["path"]) as repo: 104 revs.extend(self.create_for_repo(repo, repo_name, repo_config)) 105 self.commit(revs) 106 return 0 107 108 def prune_target(self): 109 for leaf in self.config["strings"]["path"].iterdir(): 110 if leaf.name == ".hg": 111 continue 112 shutil.rmtree(leaf) 113 114 def create_for_repo(self, repo, repo_name, repo_config): 115 print(f"Processing {repo_name} in {repo_config['path']}") 116 source_target_revs = defaultdict(TargetRevs) 117 revs_for_commit = [] 118 parse_kwargs = { 119 "env": {"l10n_base": str(self.strings_path.parent)}, 120 "ignore_missing_includes": True, 121 } 122 for head, head_name in repo_config["heads"].items(): 123 print(f"Gathering files for {head}") 124 rev = repo.log(revrange=head)[0].node 125 revs_for_commit.append(CommitRev(head_name, rev)) 126 p = source.HgTOMLParser(repo, rev) 127 project_configs = [] 128 for config_file in repo_config["config_files"]: 129 project_configs.append(p.parse(config_file, **parse_kwargs)) 130 project_configs[-1].set_locales(["en-US"], deep=True) 131 hgfiles = source.HGFiles(repo, rev, project_configs) 132 for targetpath, refpath, _, _ in hgfiles: 133 source_target_revs[refpath].revs.append(rev) 134 source_target_revs[refpath].target = targetpath 135 root = repo.root() 136 print(f"Writing {repo_name} content to target") 137 for refpath, targetrevs in source_target_revs.items(): 138 local_ref = mozpath.relpath(refpath, root) 139 content = self.get_content(local_ref, repo, targetrevs.revs) 140 target_dir = mozpath.dirname(targetrevs.target) 141 if not os.path.isdir(target_dir): 142 os.makedirs(target_dir) 143 with open(targetrevs.target, "wb") as fh: 144 fh.write(content) 145 return revs_for_commit 146 147 def commit(self, revs): 148 message = self.message + "\n\n" 149 if "TASK_ID" in os.environ: 150 message += f"X-Task-ID: {os.environ['TASK_ID']}\n\n" 151 message += "\n".join(rev.message for rev in revs) 152 with hglib.open(self.strings_path) as repo: 153 repo.commit(message=message, addremove=True) 154 155 def get_content(self, local_ref, repo, revs): 156 if local_ref.endswith(b".toml"): 157 return self.get_config_content(local_ref, repo, revs) 158 if len(revs) < 2: 159 return repo.cat([b"path:" + local_ref], rev=revs[0]) 160 contents = [repo.cat([b"path:" + local_ref], rev=rev) for rev in revs] 161 try: 162 return merge.merge_channels(local_ref.decode("utf-8"), contents) 163 except merge.MergeNotSupportedError: 164 return contents[0] 165 166 def get_config_content(self, local_ref, repo, revs): 167 # We don't support merging toml files 168 content = repo.cat([b"path:" + local_ref], rev=revs[0]) 169 return projectconfig.process_config(content) 170