1#!/usr/bin/python
2# This Source Code Form is subject to the terms of the Mozilla Public
3# License, v. 2.0. If a copy of the MPL was not distributed with this
4# file, You can obtain one at http://mozilla.org/MPL/2.0/.
5import os
6from optparse import OptionParser
7from subprocess import Popen, PIPE
8import xml.dom.minidom
9import html5lib
10import fnmatch
11import shutil
12import sys
13import re
14
15# FIXME:
16#   * Import more tests rather than just the very limited set currently
17#     chosen.
18#   * Read in a (checked-in) input file with a list of test assertions
19#     expected to fail.
20#   * Read in a (checked-in) input file with a list of reference choices
21#     for tests with multiple rel="match" references.  (But still go
22#     though all those references in case they, in turn, have references.)
23
24# Eventually we should import all the tests that have references.  (At
25# least for a subset of secs.  And we probably want to organize the
26# directory structure by spec to avoid constant file moves when files
27# move in the W3C repository.  And we probably also want to import each
28# test only once, even if it covers more than one spec.)
29
30# But for now, let's just import a few sets of tests.
31
32gSubtrees = [
33    os.path.join("css-namespaces"),
34    os.path.join("css-conditional"),
35    os.path.join("css-values"),
36    os.path.join("css-multicol"),
37    os.path.join("css-writing-modes"),
38    os.path.join("selectors"),
39]
40
41gPrefixedProperties = [
42    "column-count",
43    "column-fill",
44    "column-gap",
45    "column-rule",
46    "column-rule-color",
47    "column-rule-style",
48    "column-rule-width",
49    "columns",
50    "column-span",
51    "column-width"
52]
53
54gPrefixRegexp = re.compile(
55    r"([^-#]|^)(" + r"|".join(gPrefixedProperties) + r")\b")
56
57# Map of about:config prefs that need toggling, for a given test subdirectory.
58# Entries should look like:
59#  "$SUBDIR_NAME": "pref($PREF_NAME, $PREF_VALUE)"
60#
61# For example, when "@supports" was behind a pref, gDefaultPreferences had:
62#  "css3-conditional": "pref(layout.css.supports-rule.enabled,true)"
63gDefaultPreferences = {
64}
65
66gLog = None
67gFailList = []
68gDestPath = None
69gSrcPath = None
70support_dirs_mapped = set()
71filemap = {}
72speclinkmap = {}
73propsaddedfor = []
74tests = []
75gOptions = None
76gArgs = None
77gTestfiles = []
78gTestFlags = {}
79
80def to_unix_path_sep(path):
81    return path.replace('\\', '/')
82
83def log_output_of(subprocess):
84    global gLog
85    subprocess.wait()
86    if (subprocess.returncode != 0):
87        raise StandardError("error while running subprocess")
88    gLog.write(subprocess.stdout.readline().rstrip())
89
90def write_log_header():
91    global gLog, gSrcPath
92    gLog.write("Importing revision: ")
93    log_output_of(Popen(["git", "rev-parse", "HEAD"],
94                  stdout=PIPE, cwd=gSrcPath))
95    gLog.write("\nfrom repository: ")
96    branches = Popen(["git", "branch", "--format",
97                      "%(HEAD)%(upstream:lstrip=2)"],
98                     stdout=PIPE, cwd=gSrcPath)
99    for branch in branches.stdout:
100        if branch[0] == "*":
101            upstream = branch[1:].split("/")[0]
102            break
103    if len(upstream.strip()) == 0:
104        raise StandardError("No upstream repository found")
105    log_output_of(Popen(["git", "remote", "get-url", upstream],
106                        stdout=PIPE, cwd=gSrcPath))
107    gLog.write("\n")
108
109def remove_existing_dirs():
110    global gDestPath
111    # Remove existing directories that we're going to regenerate.  This
112    # is necessary so that we can give errors in cases where our import
113    # might copy two files to the same location, which we do by giving
114    # errors if a copy would overwrite a file.
115    for dirname in os.listdir(gDestPath):
116        fulldir = os.path.join(gDestPath, dirname)
117        if not os.path.isdir(fulldir):
118            continue
119        shutil.rmtree(fulldir)
120
121def populate_test_files():
122    global gSubtrees, gTestfiles
123    excludeDirs = ["support", "reftest", "reference", "reports", "tools"]
124    for subtree in gSubtrees:
125        for dirpath, dirnames, filenames in os.walk(subtree, topdown=True):
126            for exclDir in excludeDirs:
127                if exclDir in dirnames:
128                    dirnames.remove(exclDir)
129            for f in filenames:
130                if f == "README" or \
131                   f.find("-ref.") != -1:
132                    continue
133                gTestfiles.append(os.path.join(dirpath, f))
134
135    gTestfiles.sort()
136
137def copy_file(test, srcfile, destname, isSupportFile=False):
138    global gDestPath, gLog, gSrcPath
139    if not srcfile.startswith(gSrcPath):
140        raise StandardError("Filename " + srcfile + " does not start with " + gSrcPath)
141    logname = srcfile[len(gSrcPath):]
142    gLog.write("Importing " + to_unix_path_sep(logname) +
143               " to " + to_unix_path_sep(destname) + "\n")
144    destfile = os.path.join(gDestPath, destname)
145    destdir = os.path.dirname(destfile)
146    if not os.path.exists(destdir):
147        os.makedirs(destdir)
148    if os.path.exists(destfile):
149        raise StandardError("file " + destfile + " already exists")
150    copy_and_prefix(test, srcfile, destfile, isSupportFile)
151
152def copy_support_files(test, dirname):
153    global gSrcPath
154    if dirname in support_dirs_mapped:
155        return
156    support_dirs_mapped.add(dirname)
157    support_dir = os.path.join(dirname, "support")
158    if not os.path.exists(support_dir):
159        return
160    for dirpath, dirnames, filenames in os.walk(support_dir):
161        for srcname in filenames:
162            if srcname == "LOCK":
163                continue
164            full_srcname = os.path.join(dirpath, srcname)
165            destname = to_unix_path_sep(os.path.relpath(full_srcname, gSrcPath))
166            copy_file(test, full_srcname, destname, True)
167
168def map_file(srcname):
169    global gSrcPath
170    srcname = to_unix_path_sep(os.path.normpath(srcname))
171    if srcname in filemap:
172        return filemap[srcname]
173    destname = to_unix_path_sep(os.path.relpath(srcname, gSrcPath))
174    destdir = os.path.dirname(destname)
175    filemap[srcname] = destname
176    load_flags_for(srcname, destname)
177    copy_file(destname, srcname, destname, False)
178    copy_support_files(destname, os.path.dirname(srcname))
179    return destname
180
181def load_flags_for(srcname, destname):
182    global gTestFlags
183    gTestFlags[destname] = []
184
185    if not (is_html(srcname) or is_xml(srcname)):
186        return
187    document = get_document_for(srcname)
188    for meta in document.getElementsByTagName("meta"):
189        name = meta.getAttribute("name")
190        if name == "flags":
191            gTestFlags[destname] = meta.getAttribute("content").split()
192
193def is_html(fn):
194    return fn.endswith(".htm") or fn.endswith(".html")
195
196def is_xml(fn):
197    return fn.endswith(".xht") or fn.endswith(".xml") or fn.endswith(".xhtml") or fn.endswith(".svg")
198
199def get_document_for(srcname):
200    document = None # an xml.dom.minidom document
201    if is_html(srcname):
202        # An HTML file
203        f = open(srcname, "rb")
204        parser = html5lib.HTMLParser(tree=html5lib.treebuilders.getTreeBuilder("dom"))
205        document = parser.parse(f)
206        f.close()
207    else:
208        # An XML file
209        document = xml.dom.minidom.parse(srcname)
210    return document
211
212def add_test_items(srcname):
213    if not (is_html(srcname) or is_xml(srcname)):
214        map_file(srcname)
215        return None
216    document = get_document_for(srcname)
217    refs = []
218    notrefs = []
219    for link in document.getElementsByTagName("link"):
220        rel = link.getAttribute("rel")
221        if rel == "match":
222            arr = refs
223        elif rel == "mismatch":
224            arr = notrefs
225        else:
226            continue
227        if str(link.getAttribute("href")) != "":
228            arr.append(os.path.join(os.path.dirname(srcname), str(link.getAttribute("href"))))
229        else:
230            gLog.write("Warning: href attribute found empty in " + srcname + "\n")
231    if len(refs) > 1:
232        raise StandardError("Need to add code to specify which reference we want to match.")
233    for ref in refs:
234        tests.append(["==", map_file(srcname), map_file(ref)])
235    for notref in notrefs:
236        tests.append(["!=", map_file(srcname), map_file(notref)])
237    # Add chained references too
238    for ref in refs:
239        add_test_items(ref)
240    for notref in notrefs:
241        add_test_items(notref)
242
243AHEM_FONT_PATH = os.path.normpath(
244    os.path.join(os.path.dirname(__file__), "../fonts/Ahem.ttf"))
245AHEM_DECL_CONTENT = """@font-face {{
246  font-family: Ahem;
247  src: url("{}");
248}}"""
249AHEM_DECL_HTML = """<style type="text/css">
250""" + AHEM_DECL_CONTENT + """
251</style>
252"""
253AHEM_DECL_XML = """<style type="text/css"><![CDATA[
254""" + AHEM_DECL_CONTENT + """
255]]></style>
256"""
257
258def copy_and_prefix(test, aSourceFileName, aDestFileName, isSupportFile=False):
259    global gTestFlags, gPrefixRegexp
260    newFile = open(aDestFileName, 'wb')
261    unPrefixedFile = open(aSourceFileName, 'rb')
262    testName = aDestFileName[len(gDestPath)+1:]
263    ahemFontAdded = False
264    for line in unPrefixedFile:
265        replacementLine = line
266        searchRegex = "\s*<style\s*"
267
268        if not isSupportFile and not ahemFontAdded and 'ahem' in gTestFlags[test] and re.search(searchRegex, line):
269            # First put our ahem font declation before the first <style>
270            # element
271            template = AHEM_DECL_HTML if is_html(aDestFileName) else AHEM_DECL_XML
272            ahemPath = os.path.relpath(AHEM_FONT_PATH, os.path.dirname(aDestFileName))
273            newFile.write(template.format(to_unix_path_sep(ahemPath)))
274            ahemFontAdded = True
275
276        replacementLine = gPrefixRegexp.sub(r"\1-moz-\2", replacementLine)
277        newFile.write(replacementLine)
278
279    newFile.close()
280    unPrefixedFile.close()
281
282def read_options():
283    global gArgs, gOptions
284    op = OptionParser()
285    op.usage = \
286    '''%prog <clone of git repository>
287            Import CSS reftests from a web-platform-tests git repository clone.
288            The location of the git repository must be given on the command
289            line.'''
290    (gOptions, gArgs) = op.parse_args()
291    if len(gArgs) != 1:
292        op.error("Too few arguments specified.")
293
294def setup_paths():
295    global gSubtrees, gDestPath, gSrcPath
296    # FIXME: generate gSrcPath with consistent trailing / regardless of input.
297    # (We currently expect the argument to have a trailing slash.)
298    gSrcPath = gArgs[0]
299    if not os.path.isdir(gSrcPath) or \
300       not os.path.isdir(os.path.join(gSrcPath, ".git")):
301        raise StandardError("source path does not appear to be a git clone")
302    gSrcPath = os.path.join(gSrcPath, "css") + "/"
303    if not os.path.isdir(gSrcPath):
304        raise StandardError("source path does not appear to be " +
305                            "a wpt clone which contains css tests")
306
307    gDestPath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "received")
308    newSubtrees = []
309    for relPath in gSubtrees:
310        newSubtrees[len(gSubtrees):] = [os.path.join(gSrcPath, relPath)]
311    gSubtrees = newSubtrees
312
313def setup_log():
314    global gLog
315    # Since we're going to commit the tests, we should also commit
316    # information about where they came from.
317    gLog = open(os.path.join(gDestPath, "import.log"), "wb")
318
319def read_fail_list():
320    global gFailList
321    dirname = os.path.realpath(__file__).split(os.path.sep)
322    dirname = os.path.sep.join(dirname[:len(dirname)-1])
323    with open(os.path.join(dirname, "failures.list"), "rb") as f:
324        for line in f:
325            line = line.strip()
326            if not line or line.startswith("#"):
327                continue
328            items = line.split()
329            refpat = None
330            if items[-1].startswith("ref:"):
331                refpat = re.compile(fnmatch.translate(items.pop()[4:]))
332            pat = re.compile(fnmatch.translate(items.pop()))
333            gFailList.append((pat, refpat, items))
334
335def main():
336    global gDestPath, gLog, gTestfiles, gTestFlags, gFailList
337    read_options()
338    setup_paths()
339    read_fail_list()
340    setup_log()
341    write_log_header()
342    remove_existing_dirs()
343    populate_test_files()
344
345    for t in gTestfiles:
346        add_test_items(t)
347
348    listfile = open(os.path.join(gDestPath, "reftest.list"), "wb")
349    listfile.write("# THIS FILE IS AUTOGENERATED BY {0}\n# DO NOT EDIT!\n".format(os.path.basename(__file__)))
350    lastDefaultPreferences = None
351    for test in tests:
352        defaultPreferences = gDefaultPreferences.get(test[1].split("/")[0], None)
353        if defaultPreferences != lastDefaultPreferences:
354            if defaultPreferences is None:
355                listfile.write("\ndefault-preferences\n\n")
356            else:
357                listfile.write("\ndefault-preferences {0}\n\n".format(defaultPreferences))
358            lastDefaultPreferences = defaultPreferences
359        key = 1
360        while not test[key] in gTestFlags.keys() and key < len(test):
361            key = key + 1
362        testType = test[key - 1]
363        testFlags = gTestFlags[test[key]]
364        # Replace the Windows separators if any. Our internal strings
365        # all use the system separator, however the failure/skip lists
366        # and reftest.list always use '/' so we fix the paths here.
367        test[key] = to_unix_path_sep(test[key])
368        test[key + 1] = to_unix_path_sep(test[key + 1])
369        testKey = test[key]
370        refKey = test[key + 1]
371        fail = []
372        for pattern, refpattern, failureType in gFailList:
373            if (refpattern is None or refpattern.match(refKey)) and \
374               pattern.match(testKey):
375                fail = failureType
376        test = fail + test
377        listfile.write(" ".join(test) + "\n")
378    listfile.close()
379
380    gLog.close()
381
382if __name__ == '__main__':
383    main()
384