1#!/usr/bin/env python
2# LexGen.py - implemented 2002 by Neil Hodgson neilh@scintilla.org
3# Released to the public domain.
4
5# Regenerate the Scintilla and SciTE source files that list
6# all the lexers and all the properties files.
7# Should be run whenever a new lexer is added or removed.
8# Requires Python 2.4 or later
9# Most files are regenerated in place with templates stored in comments.
10# The VS .NET project file is generated into a different file as the
11# VS .NET environment will not retain comments when modifying the file.
12# The files are copied to a string apart from sections between a
13# ++Autogenerated comment and a --Autogenerated comment which is
14# generated by the CopyWithInsertion function. After the whole
15# string is instantiated, it is compared with the target file and
16# if different the file is rewritten.
17# Does not regenerate the Visual C++ 6 project files but does the VS .NET
18# project file.
19
20import string
21import sys
22import os
23import glob
24
25# EOL constants
26CR = "\r"
27LF = "\n"
28CRLF = "\r\n"
29if sys.platform == "win32":
30    NATIVE = CRLF
31else:
32    # Yes, LF is the native EOL even on Mac OS X. CR is just for
33    # Mac OS <=9 (a.k.a. "Mac Classic")
34    NATIVE = LF
35
36# Automatically generated sections contain start and end comments,
37# a definition line and the results.
38# The results are replaced by regenerating based on the definition line.
39# The definition line is a comment prefix followed by "**".
40# If there is a digit after the ** then this indicates which list to use
41# and the digit and next character are not part of the definition
42# Backslash is used as an escape within the definition line.
43# The part between \( and \) is repeated for each item in the list.
44# \* is replaced by each list item. \t, and \n are tab and newline.
45def CopyWithInsertion(input, commentPrefix, retainDefs, eolType, *lists):
46    copying = 1
47    listid = 0
48    output = []
49    for line in input.splitlines(0):
50        isStartGenerated = line.startswith(commentPrefix + "++Autogenerated")
51        if copying and not isStartGenerated:
52            output.append(line)
53        if isStartGenerated:
54            if retainDefs:
55                output.append(line)
56            copying = 0
57            definition = ""
58        elif not copying and line.startswith(commentPrefix + "**"):
59            if retainDefs:
60                output.append(line)
61            definition = line[len(commentPrefix + "**"):]
62            if (commentPrefix == "<!--") and (" -->" in definition):
63                definition = definition.replace(" -->", "")
64            listid = 0
65            if definition[0] in string.digits:
66                listid = int(definition[:1])
67                definition = definition[2:]
68            # Hide double slashes as a control character
69            definition = definition.replace("\\\\", "\001")
70            # Do some normal C style transforms
71            definition = definition.replace("\\n", "\n")
72            definition = definition.replace("\\t", "\t")
73            # Get the doubled backslashes back as single backslashes
74            definition = definition.replace("\001", "\\")
75            startRepeat = definition.find("\\(")
76            endRepeat = definition.find("\\)")
77            intro = definition[:startRepeat]
78            out = ""
79            if intro.endswith("\n"):
80                pos = 0
81            else:
82                pos = len(intro)
83            out += intro
84            middle = definition[startRepeat+2:endRepeat]
85            for i in lists[listid]:
86                item = middle.replace("\\*", i)
87                if pos and (pos + len(item) >= 80):
88                    out += "\\\n"
89                    pos = 0
90                out += item
91                pos += len(item)
92                if item.endswith("\n"):
93                    pos = 0
94            outro = definition[endRepeat+2:]
95            out += outro
96            out = out.replace("\n", eolType) # correct EOLs in generated content
97            output.append(out)
98        elif line.startswith(commentPrefix + "--Autogenerated"):
99            copying = 1
100            if retainDefs:
101                output.append(line)
102    output = [line.rstrip(" \t") for line in output] # trim trailing whitespace
103    return eolType.join(output) + eolType
104
105def UpdateFile(filename, updated):
106    """ If the file is different to updated then copy updated
107    into the file else leave alone so CVS and make don't treat
108    it as modified. """
109    try:
110        infile = open(filename, "rb")
111    except IOError:	# File is not there yet
112        out = open(filename, "wb")
113        out.write(updated.encode('utf-8'))
114        out.close()
115        print("New %s" % filename)
116        return
117    original = infile.read()
118    infile.close()
119    original = original.decode('utf-8')
120    if updated != original:
121        os.unlink(filename)
122        out = open(filename, "wb")
123        out.write(updated.encode('utf-8'))
124        out.close()
125        print("Changed %s " % filename)
126    #~ else:
127        #~ print "Unchanged", filename
128
129def Generate(inpath, outpath, commentPrefix, eolType, *lists):
130    """Generate 'outpath' from 'inpath'.
131
132        "eolType" indicates the type of EOLs to use in the generated
133            file. It should be one of following constants: LF, CRLF,
134            CR, or NATIVE.
135    """
136    #print "generate '%s' -> '%s' (comment prefix: %r, eols: %r)"\
137    #      % (inpath, outpath, commentPrefix, eolType)
138    try:
139        infile = open(inpath, "rb")
140    except IOError:
141        print("Can not open %s" % inpath)
142        return
143    original = infile.read()
144    infile.close()
145    original = original.decode('utf-8')
146    updated = CopyWithInsertion(original, commentPrefix,
147        inpath == outpath, eolType, *lists)
148    UpdateFile(outpath, updated)
149
150def Regenerate(filename, commentPrefix, eolType, *lists):
151    """Regenerate the given file.
152
153        "eolType" indicates the type of EOLs to use in the generated
154            file. It should be one of following constants: LF, CRLF,
155            CR, or NATIVE.
156    """
157    Generate(filename, filename, commentPrefix, eolType, *lists)
158
159def FindModules(lexFile):
160    modules = []
161    f = open(lexFile)
162    for l in f.readlines():
163        if l.startswith("LexerModule"):
164            l = l.replace("(", " ")
165            modules.append(l.split()[1])
166    return modules
167
168# Properties that start with lexer. or fold. are automatically found but there are some
169# older properties that don't follow this pattern so must be explicitly listed.
170knownIrregularProperties = [
171    "fold",
172    "styling.within.preprocessor",
173    "tab.timmy.whinge.level",
174    "asp.default.language",
175    "html.tags.case.sensitive",
176    "ps.level",
177    "ps.tokenize",
178    "sql.backslash.escapes",
179    "nsis.uservars",
180    "nsis.ignorecase"
181]
182
183def FindProperties(lexFile):
184    properties = {}
185    f = open(lexFile)
186    for l in f.readlines():
187        if ("GetProperty" in l or "DefineProperty" in l) and "\"" in l:
188            l = l.strip()
189            if not l.startswith("//"):	# Drop comments
190                propertyName = l.split("\"")[1]
191                if propertyName.lower() == propertyName:
192                    # Only allow lower case property names
193                    if propertyName in knownIrregularProperties or \
194                        propertyName.startswith("fold.") or \
195                        propertyName.startswith("lexer."):
196                        properties[propertyName] = 1
197    return properties
198
199def FindPropertyDocumentation(lexFile):
200    documents = {}
201    f = open(lexFile)
202    name = ""
203    for l in f.readlines():
204        l = l.strip()
205        if "// property " in l:
206            propertyName = l.split()[2]
207            if propertyName.lower() == propertyName:
208                # Only allow lower case property names
209                name = propertyName
210                documents[name] = ""
211        elif "DefineProperty" in l and "\"" in l:
212            propertyName = l.split("\"")[1]
213            if propertyName.lower() == propertyName:
214                # Only allow lower case property names
215                name = propertyName
216                documents[name] = ""
217        elif name:
218            if l.startswith("//"):
219                if documents[name]:
220                    documents[name] += " "
221                documents[name] += l[2:].strip()
222            elif l.startswith("\""):
223                l = l[1:].strip()
224                if l.endswith(";"):
225                    l = l[:-1].strip()
226                if l.endswith(")"):
227                    l = l[:-1].strip()
228                if l.endswith("\""):
229                    l = l[:-1]
230                # Fix escaped double quotes
231                l = l.replace("\\\"", "\"")
232                documents[name] += l
233            else:
234                name = ""
235    for name in list(documents.keys()):
236        if documents[name] == "":
237            del documents[name]
238    return documents
239
240def ciCompare(a,b):
241    return cmp(a.lower(), b.lower())
242
243def ciKey(a):
244    return a.lower()
245
246def sortListInsensitive(l):
247    try:    # Try key function
248        l.sort(key=ciKey)
249    except TypeError:    # Earlier version of Python, so use comparison function
250        l.sort(ciCompare)
251
252def UpdateLineInFile(path, linePrefix, lineReplace):
253    lines = []
254    with open(path, "r") as f:
255        for l in f.readlines():
256            l = l.rstrip()
257            if l.startswith(linePrefix):
258                lines.append(lineReplace)
259            else:
260                lines.append(l)
261    contents = NATIVE.join(lines) + NATIVE
262    UpdateFile(path, contents)
263
264def UpdateVersionNumbers(root):
265    with open(root + "scintilla/version.txt") as f:
266        version = f.read()
267    versionDotted = version[0] + '.' + version[1] + '.' + version[2]
268    versionCommad = version[0] + ', ' + version[1] + ', ' + version[2] + ', 0'
269
270    UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_SCINTILLA",
271        "#define VERSION_SCINTILLA \"" + versionDotted + "\"")
272    UpdateLineInFile(root + "scintilla/win32/ScintRes.rc", "#define VERSION_WORDS",
273        "#define VERSION_WORDS " + versionCommad)
274    UpdateLineInFile(root + "scintilla/qt/ScintillaEditBase/ScintillaEditBase.pro",
275        "VERSION =",
276        "VERSION = " + versionDotted)
277    UpdateLineInFile(root + "scintilla/qt/ScintillaEdit/ScintillaEdit.pro",
278        "VERSION =",
279        "VERSION = " + versionDotted)
280    UpdateLineInFile(root + "scintilla/doc/ScintillaDownload.html", "       Release",
281        "       Release " + versionDotted)
282    UpdateLineInFile(root + "scintilla/doc/index.html",
283        '          <font color="#FFCC99" size="3"> Release version',
284        '          <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />')
285
286    if os.path.exists(root + "scite"):
287        UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_SCITE",
288            "#define VERSION_SCITE \"" + versionDotted + "\"")
289        UpdateLineInFile(root + "scite/src/SciTE.h", "#define VERSION_WORDS",
290            "#define VERSION_WORDS " + versionCommad)
291        UpdateLineInFile(root + "scite/doc/SciTEDownload.html", "       Release",
292            "       Release " + versionDotted)
293        UpdateLineInFile(root + "scite/doc/SciTE.html",
294            '          <font color="#FFCC99" size="3"> Release version',
295            '          <font color="#FFCC99" size="3"> Release version ' + versionDotted + '<br />')
296
297def RegenerateAll():
298    root="../../"
299
300    # Find all the lexer source code files
301    lexFilePaths = glob.glob(root + "scintilla/lexers/Lex*.cxx")
302    sortListInsensitive(lexFilePaths)
303    lexFiles = [os.path.basename(f)[:-4] for f in lexFilePaths]
304    print(lexFiles)
305    lexerModules = []
306    lexerProperties = {}
307    propertyDocuments = {}
308    for lexFile in lexFilePaths:
309        lexerModules.extend(FindModules(lexFile))
310        for k in FindProperties(lexFile).keys():
311            lexerProperties[k] = 1
312        documents = FindPropertyDocumentation(lexFile)
313        for k in documents.keys():
314            propertyDocuments[k] = documents[k]
315    sortListInsensitive(lexerModules)
316    lexerProperties = list(lexerProperties.keys())
317    sortListInsensitive(lexerProperties)
318
319    # Generate HTML to document each property
320    # This is done because tags can not be safely put inside comments in HTML
321    documentProperties = list(propertyDocuments.keys())
322    sortListInsensitive(documentProperties)
323    propertiesHTML = []
324    for k in documentProperties:
325        propertiesHTML.append("\t<tr id='property-%s'>\n\t<td>%s</td>\n\t<td>%s</td>\n\t</tr>" %
326            (k, k, propertyDocuments[k]))
327
328    # Find all the SciTE properties files
329    otherProps = ["abbrev.properties", "Embedded.properties", "SciTEGlobal.properties", "SciTE.properties"]
330    if os.path.exists(root + "scite"):
331        propFilePaths = glob.glob(root + "scite/src/*.properties")
332        sortListInsensitive(propFilePaths)
333        propFiles = [os.path.basename(f) for f in propFilePaths if os.path.basename(f) not in otherProps]
334        sortListInsensitive(propFiles)
335        print(propFiles)
336
337    Regenerate(root + "scintilla/src/Catalogue.cxx", "//", NATIVE, lexerModules)
338    Regenerate(root + "scintilla/win32/scintilla.mak", "#", NATIVE, lexFiles)
339    Regenerate(root + "scintilla/win32/scintilla_vc6.mak", "#", NATIVE, lexFiles)
340    if os.path.exists(root + "scite"):
341        Regenerate(root + "scite/win32/makefile", "#", NATIVE, propFiles)
342        Regenerate(root + "scite/win32/scite.mak", "#", NATIVE, propFiles)
343        Regenerate(root + "scite/src/SciTEProps.cxx", "//", NATIVE, lexerProperties)
344        Regenerate(root + "scite/doc/SciTEDoc.html", "<!--", NATIVE, propertiesHTML)
345        Generate(root + "scite/boundscheck/vcproj.gen",
346         root + "scite/boundscheck/SciTE.vcproj", "#", NATIVE, lexFiles)
347
348    UpdateVersionNumbers(root)
349
350RegenerateAll()
351