1#!/usr/bin/env python
2# This Source Code Form is subject to the terms of the Mozilla Public
3# License, v. 2.0. If a copy of the MPL was not distributed with this file,
4# You can obtain one at http://mozilla.org/MPL/2.0/.
5
6"""
7Imports a test suite from a remote repository. Takes one argument, a file in
8the format described in README.
9Note: removes both source and destination directory before starting. Do not
10      use with outstanding changes in either directory.
11"""
12
13from __future__ import print_function, unicode_literals
14
15import os
16import shutil
17import subprocess
18import sys
19
20import parseManifest
21import writeBuildFiles
22
23def readManifests(iden, dirs):
24    def parseManifestFile(iden, path):
25        pathstr = "hg-%s/%s/MANIFEST" % (iden, path)
26        subdirs, mochitests, reftests, _, supportfiles = parseManifest.parseManifestFile(pathstr)
27        return subdirs, mochitests, reftests, supportfiles
28
29    data = []
30    for path in dirs:
31        subdirs, mochitests, reftests, supportfiles = parseManifestFile(iden, path)
32        data.append({
33          "path": path,
34          "mochitests": mochitests,
35          "reftests": reftests,
36          "supportfiles": supportfiles,
37        })
38        data.extend(readManifests(iden, ["%s/%s" % (path, d) for d in subdirs]))
39    return data
40
41
42def getData(confFile):
43    """This function parses a file of the form
44    (hg or git)|URL of remote repository|identifier for the local directory
45    First directory of tests
46    ...
47    Last directory of tests"""
48    vcs = ""
49    url = ""
50    iden = ""
51    directories = []
52    try:
53        with open(confFile, 'r') as fp:
54            first = True
55            for line in fp:
56                if first:
57                    vcs, url, iden = line.strip().split("|")
58                    first = False
59                else:
60                    directories.append(line.strip())
61    finally:
62        return vcs, url, iden, directories
63
64
65def makePathInternal(a, b):
66    if not b:
67        # Empty directory, i.e., the repository root.
68        return a
69    return "%s/%s" % (a, b)
70
71
72def makeSourcePath(a, b):
73    """Make a path in the source (upstream) directory."""
74    return makePathInternal("hg-%s" % a, b)
75
76
77def makeDestPath(a, b):
78    """Make a path in the destination (mozilla-central) directory, shortening as
79    appropriate."""
80    def shorten(path):
81        path = path.replace('dom-tree-accessors', 'dta')
82        path = path.replace('document.getElementsByName', 'doc.gEBN')
83        path = path.replace('requirements-for-implementations', 'implreq')
84        path = path.replace('other-elements-attributes-and-apis', 'oeaaa')
85        return path
86
87    return shorten(makePathInternal(a, b))
88
89
90def extractReftestFiles(reftests):
91    """Returns the set of files referenced in the reftests argument"""
92    files = set()
93    for line in reftests:
94        files.update([line[1], line[2]])
95    return files
96
97
98def copy(dest, directories):
99    """Copy mochitests and support files from the external HG directory to their
100    place in mozilla-central.
101    """
102    print("Copying tests...")
103    for d in directories:
104        sourcedir = makeSourcePath(dest, d["path"])
105        destdir = makeDestPath(dest, d["path"])
106        os.makedirs(destdir)
107
108        reftestfiles = extractReftestFiles(d["reftests"])
109
110        for mochitest in d["mochitests"]:
111            shutil.copy("%s/%s" % (sourcedir, mochitest), "%s/test_%s" % (destdir, mochitest))
112        for reftest in sorted(reftestfiles):
113            shutil.copy("%s/%s" % (sourcedir, reftest), "%s/%s" % (destdir, reftest))
114        for support in d["supportfiles"]:
115            shutil.copy("%s/%s" % (sourcedir, support), "%s/%s" % (destdir, support))
116
117def printBuildFiles(dest, directories):
118    """Create a mochitest.ini that all the contains tests we import.
119    """
120    print("Creating manifest...")
121    all_mochitests = set()
122    all_support = set()
123
124    for d in directories:
125        path = makeDestPath(dest, d["path"])
126
127        all_mochitests |= set('%s/test_%s' % (d['path'], mochitest)
128            for mochitest in d['mochitests'])
129        all_support |= set('%s/%s' % (d['path'], p) for p in d['supportfiles'])
130
131        if d["reftests"]:
132            with open(path + "/reftest.list", "w") as fh:
133                result = writeBuildFiles.substReftestList("importTestsuite.py",
134                    d["reftests"])
135                fh.write(result)
136
137    manifest_path = dest + '/mochitest.ini'
138    with open(manifest_path, 'w') as fh:
139        result = writeBuildFiles.substManifest('importTestsuite.py',
140            all_mochitests, all_support)
141        fh.write(result)
142    subprocess.check_call(["hg", "add", manifest_path])
143
144def hgadd(dest, directories):
145    """Inform hg of the files in |directories|."""
146    print("hg addremoving...")
147    for d in directories:
148        subprocess.check_call(["hg", "addremove", makeDestPath(dest, d)])
149
150def removeAndCloneRepo(vcs, url, dest):
151    """Replaces the repo at dest by a fresh clone from url using vcs"""
152    assert vcs in ('hg', 'git')
153
154    print("Removing %s..." % dest)
155    subprocess.check_call(["rm", "-rf", dest])
156
157    print("Cloning %s to %s with %s..." % (url, dest, vcs))
158    subprocess.check_call([vcs, "clone", url, dest])
159
160def importRepo(confFile):
161    try:
162        vcs, url, iden, directories = getData(confFile)
163        dest = iden
164        hgdest = "hg-%s" % iden
165
166        print("Removing %s..." % dest)
167        subprocess.check_call(["rm", "-rf", dest])
168
169        removeAndCloneRepo(vcs, url, hgdest)
170
171        data = readManifests(iden, directories)
172        print("Going to import %s..." % [d["path"] for d in data])
173
174        copy(dest, data)
175        printBuildFiles(dest, data)
176        hgadd(dest, directories)
177        print("Removing %s again..." % hgdest)
178        subprocess.check_call(["rm", "-rf", hgdest])
179    except subprocess.CalledProcessError as e:
180        print(e.returncode)
181    finally:
182        print("Done")
183
184if __name__ == "__main__":
185    if len(sys.argv) != 2:
186        print("Need one argument.")
187    else:
188        importRepo(sys.argv[1])
189
190