1#!/usr/bin/env python 2# This Source Code Form is subject to the terms of the Mozilla Public 3# License, v. 2.0. If a copy of the MPL was not distributed with this file, 4# You can obtain one at http://mozilla.org/MPL/2.0/. 5 6""" 7Imports a test suite from a remote repository. Takes one argument, a file in 8the format described in README. 9Note: removes both source and destination directory before starting. Do not 10 use with outstanding changes in either directory. 11""" 12 13from __future__ import print_function, unicode_literals 14 15import os 16import shutil 17import subprocess 18import sys 19 20import parseManifest 21import writeBuildFiles 22 23def readManifests(iden, dirs): 24 def parseManifestFile(iden, path): 25 pathstr = "hg-%s/%s/MANIFEST" % (iden, path) 26 subdirs, mochitests, reftests, _, supportfiles = parseManifest.parseManifestFile(pathstr) 27 return subdirs, mochitests, reftests, supportfiles 28 29 data = [] 30 for path in dirs: 31 subdirs, mochitests, reftests, supportfiles = parseManifestFile(iden, path) 32 data.append({ 33 "path": path, 34 "mochitests": mochitests, 35 "reftests": reftests, 36 "supportfiles": supportfiles, 37 }) 38 data.extend(readManifests(iden, ["%s/%s" % (path, d) for d in subdirs])) 39 return data 40 41 42def getData(confFile): 43 """This function parses a file of the form 44 (hg or git)|URL of remote repository|identifier for the local directory 45 First directory of tests 46 ... 47 Last directory of tests""" 48 vcs = "" 49 url = "" 50 iden = "" 51 directories = [] 52 try: 53 with open(confFile, 'r') as fp: 54 first = True 55 for line in fp: 56 if first: 57 vcs, url, iden = line.strip().split("|") 58 first = False 59 else: 60 directories.append(line.strip()) 61 finally: 62 return vcs, url, iden, directories 63 64 65def makePathInternal(a, b): 66 if not b: 67 # Empty directory, i.e., the repository root. 68 return a 69 return "%s/%s" % (a, b) 70 71 72def makeSourcePath(a, b): 73 """Make a path in the source (upstream) directory.""" 74 return makePathInternal("hg-%s" % a, b) 75 76 77def makeDestPath(a, b): 78 """Make a path in the destination (mozilla-central) directory, shortening as 79 appropriate.""" 80 def shorten(path): 81 path = path.replace('dom-tree-accessors', 'dta') 82 path = path.replace('document.getElementsByName', 'doc.gEBN') 83 path = path.replace('requirements-for-implementations', 'implreq') 84 path = path.replace('other-elements-attributes-and-apis', 'oeaaa') 85 return path 86 87 return shorten(makePathInternal(a, b)) 88 89 90def extractReftestFiles(reftests): 91 """Returns the set of files referenced in the reftests argument""" 92 files = set() 93 for line in reftests: 94 files.update([line[1], line[2]]) 95 return files 96 97 98def copy(dest, directories): 99 """Copy mochitests and support files from the external HG directory to their 100 place in mozilla-central. 101 """ 102 print("Copying tests...") 103 for d in directories: 104 sourcedir = makeSourcePath(dest, d["path"]) 105 destdir = makeDestPath(dest, d["path"]) 106 os.makedirs(destdir) 107 108 reftestfiles = extractReftestFiles(d["reftests"]) 109 110 for mochitest in d["mochitests"]: 111 shutil.copy("%s/%s" % (sourcedir, mochitest), "%s/test_%s" % (destdir, mochitest)) 112 for reftest in sorted(reftestfiles): 113 shutil.copy("%s/%s" % (sourcedir, reftest), "%s/%s" % (destdir, reftest)) 114 for support in d["supportfiles"]: 115 shutil.copy("%s/%s" % (sourcedir, support), "%s/%s" % (destdir, support)) 116 117def printBuildFiles(dest, directories): 118 """Create a mochitest.ini that all the contains tests we import. 119 """ 120 print("Creating manifest...") 121 all_mochitests = set() 122 all_support = set() 123 124 for d in directories: 125 path = makeDestPath(dest, d["path"]) 126 127 all_mochitests |= set('%s/test_%s' % (d['path'], mochitest) 128 for mochitest in d['mochitests']) 129 all_support |= set('%s/%s' % (d['path'], p) for p in d['supportfiles']) 130 131 if d["reftests"]: 132 with open(path + "/reftest.list", "w") as fh: 133 result = writeBuildFiles.substReftestList("importTestsuite.py", 134 d["reftests"]) 135 fh.write(result) 136 137 manifest_path = dest + '/mochitest.ini' 138 with open(manifest_path, 'w') as fh: 139 result = writeBuildFiles.substManifest('importTestsuite.py', 140 all_mochitests, all_support) 141 fh.write(result) 142 subprocess.check_call(["hg", "add", manifest_path]) 143 144def hgadd(dest, directories): 145 """Inform hg of the files in |directories|.""" 146 print("hg addremoving...") 147 for d in directories: 148 subprocess.check_call(["hg", "addremove", makeDestPath(dest, d)]) 149 150def removeAndCloneRepo(vcs, url, dest): 151 """Replaces the repo at dest by a fresh clone from url using vcs""" 152 assert vcs in ('hg', 'git') 153 154 print("Removing %s..." % dest) 155 subprocess.check_call(["rm", "-rf", dest]) 156 157 print("Cloning %s to %s with %s..." % (url, dest, vcs)) 158 subprocess.check_call([vcs, "clone", url, dest]) 159 160def importRepo(confFile): 161 try: 162 vcs, url, iden, directories = getData(confFile) 163 dest = iden 164 hgdest = "hg-%s" % iden 165 166 print("Removing %s..." % dest) 167 subprocess.check_call(["rm", "-rf", dest]) 168 169 removeAndCloneRepo(vcs, url, hgdest) 170 171 data = readManifests(iden, directories) 172 print("Going to import %s..." % [d["path"] for d in data]) 173 174 copy(dest, data) 175 printBuildFiles(dest, data) 176 hgadd(dest, directories) 177 print("Removing %s again..." % hgdest) 178 subprocess.check_call(["rm", "-rf", hgdest]) 179 except subprocess.CalledProcessError as e: 180 print(e.returncode) 181 finally: 182 print("Done") 183 184if __name__ == "__main__": 185 if len(sys.argv) != 2: 186 print("Need one argument.") 187 else: 188 importRepo(sys.argv[1]) 189 190