1# LexGen.py - implemented 2002 by Neil Hodgson neilh@scintilla.org 2# Released to the public domain. 3 4# Regenerate the Scintilla and SciTE source files that list 5# all the lexers and all the properties files. 6# Should be run whenever a new lexer is added or removed. 7# Requires Python 2.1 or later 8# Most files are regenerated in place with templates stored in comments. 9# The VS .NET project file is generated into a different file as the 10# VS .NET environment will not retain comments when modifying the file. 11# The files are copied to a string apart from sections between a 12# ++Autogenerated comment and a --Autogenerated comment which is 13# generated by the CopyWithInsertion function. After the whole 14# string is instantiated, it is compared with the target file and 15# if different the file is rewritten. 16# Does not regenerate the Visual C++ 6 project files but does the VS .NET 17# project file. 18 19import string 20import sys 21import os 22import glob 23 24# EOL constants 25CR = "\r" 26LF = "\n" 27CRLF = "\r\n" 28if sys.platform == "win32": 29 NATIVE = CRLF 30else: 31 # Yes, LF is the native EOL even on Mac OS X. CR is just for 32 # Mac OS <=9 (a.k.a. "Mac Classic") 33 NATIVE = LF 34 35# Automatically generated sections contain start and end comments, 36# a definition line and the results. 37# The results are replaced by regenerating based on the definition line. 38# The definition line is a comment prefix followed by "**". 39# If there is a digit after the ** then this indicates which list to use 40# and the digit and next character are not part of the definition 41# Backslash is used as an escape within the definition line. 42# The part between \( and \) is repeated for each item in the list. 43# \* is replaced by each list item. \t, and \n are tab and newline. 44def CopyWithInsertion(input, commentPrefix, retainDefs, eolType, *lists): 45 copying = 1 46 listid = 0 47 output = [] 48 for line in input.splitlines(0): 49 isStartGenerated = line.startswith(commentPrefix + "++Autogenerated") 50 if copying and not isStartGenerated: 51 output.append(line) 52 if isStartGenerated: 53 if retainDefs: 54 output.append(line) 55 copying = 0 56 definition = "" 57 elif not copying and line.startswith(commentPrefix + "**"): 58 if retainDefs: 59 output.append(line) 60 definition = line[len(commentPrefix + "**"):] 61 listid = 0 62 if definition[0] in string.digits: 63 listid = int(definition[:1]) 64 definition = definition[2:] 65 # Hide double slashes as a control character 66 definition = definition.replace("\\\\", "\001") 67 # Do some normal C style transforms 68 definition = definition.replace("\\n", "\n") 69 definition = definition.replace("\\t", "\t") 70 # Get the doubled backslashes back as single backslashes 71 definition = definition.replace("\001", "\\") 72 startRepeat = definition.find("\\(") 73 endRepeat = definition.find("\\)") 74 intro = definition[:startRepeat] 75 out = "" 76 if intro.endswith("\n"): 77 pos = 0 78 else: 79 pos = len(intro) 80 out += intro 81 middle = definition[startRepeat+2:endRepeat] 82 for i in lists[listid]: 83 item = middle.replace("\\*", i) 84 if pos and (pos + len(item) >= 80): 85 out += "\\\n" 86 pos = 0 87 out += item 88 pos += len(item) 89 if item.endswith("\n"): 90 pos = 0 91 outro = definition[endRepeat+2:] 92 out += outro 93 out = out.replace("\n", eolType) # correct EOLs in generated content 94 output.append(out) 95 elif line.startswith(commentPrefix + "--Autogenerated"): 96 copying = 1 97 if retainDefs: 98 output.append(line) 99 output = [line.rstrip(" \t") for line in output] # trim trailing whitespace 100 return eolType.join(output) + eolType 101 102def UpdateFile(filename, updated): 103 """ If the file is different to updated then copy updated 104 into the file else leave alone so CVS and make don't treat 105 it as modified. """ 106 try: 107 infile = open(filename, "rb") 108 except IOError: # File is not there yet 109 out = open(filename, "wb") 110 out.write(updated) 111 out.close() 112 print "New", filename 113 return 114 original = infile.read() 115 infile.close() 116 if updated != original: 117 os.unlink(filename) 118 out = open(filename, "wb") 119 out.write(updated) 120 out.close() 121 print "Changed", filename 122 #~ else: 123 #~ print "Unchanged", filename 124 125def Generate(inpath, outpath, commentPrefix, eolType, *lists): 126 """Generate 'outpath' from 'inpath'. 127 128 "eolType" indicates the type of EOLs to use in the generated 129 file. It should be one of following constants: LF, CRLF, 130 CR, or NATIVE. 131 """ 132 #print "generate '%s' -> '%s' (comment prefix: %r, eols: %r)"\ 133 # % (inpath, outpath, commentPrefix, eolType) 134 try: 135 infile = open(inpath, "r") 136 except IOError: 137 print "Can not open", inpath 138 return 139 original = infile.read() 140 infile.close() 141 updated = CopyWithInsertion(original, commentPrefix, 142 inpath == outpath, eolType, *lists) 143 UpdateFile(outpath, updated) 144 145def Regenerate(filename, commentPrefix, eolType, *lists): 146 """Regenerate the given file. 147 148 "eolType" indicates the type of EOLs to use in the generated 149 file. It should be one of following constants: LF, CRLF, 150 CR, or NATIVE. 151 """ 152 Generate(filename, filename, commentPrefix, eolType, *lists) 153 154def FindModules(lexFile): 155 modules = [] 156 f = open(lexFile) 157 for l in f.readlines(): 158 if l.startswith("LexerModule"): 159 l = l.replace("(", " ") 160 modules.append(l.split()[1]) 161 return modules 162 163knownIrregularProperties = [ 164 "fold", 165 "styling.within.preprocessor", 166 "tab.timmy.whinge.level", 167 "asp.default.language", 168 "html.tags.case.sensitive", 169 "ps.level", 170 "ps.tokenize", 171 "sql.backslash.escapes", 172 "nsis.uservars", 173 "nsis.ignorecase" 174] 175 176def FindProperties(lexFile): 177 properties = set() 178 f = open(lexFile) 179 for l in f.readlines(): 180 if "GetProperty" in l: 181 l = l.strip() 182 if not l.startswith("//"): # Drop comments 183 propertyName = l.split("\"")[1] 184 if propertyName.lower() == propertyName: 185 # Only allow lower case property names 186 if propertyName in knownIrregularProperties or \ 187 propertyName.startswith("fold.") or \ 188 propertyName.startswith("lexer."): 189 properties.add(propertyName) 190 return properties 191 192def ciCompare(a,b): 193 return cmp(a.lower(), b.lower()) 194 195def RegenerateAll(): 196 root="../../" 197 198 # Find all the lexer source code files 199 lexFilePaths = glob.glob(root + "scintilla/src/Lex*.cxx") 200 lexFiles = [os.path.basename(f)[:-4] for f in lexFilePaths] 201 print lexFiles 202 lexerModules = [] 203 lexerProperties = set() 204 for lexFile in lexFilePaths: 205 lexerModules.extend(FindModules(lexFile)) 206 lexerProperties.update(FindProperties(lexFile)) 207 lexerModules.sort(ciCompare) 208 lexerProperties.remove("fold.comment.python") 209 lexerProperties = list(lexerProperties) 210 lexerProperties.sort(ciCompare) 211 212 # Find all the SciTE properties files 213 otherProps = ["abbrev.properties", "Embedded.properties", "SciTEGlobal.properties", "SciTE.properties"] 214 propFilePaths = glob.glob(root + "scite/src/*.properties") 215 propFiles = [os.path.basename(f) for f in propFilePaths if os.path.basename(f) not in otherProps] 216 propFiles.sort(ciCompare) 217 print propFiles 218 219 # Find all the menu command IDs in the SciTE header 220 SciTEHeader = file(root + "scite/src/SciTE.h") 221 lines = SciTEHeader.read().split("\n") 222 SciTEHeader.close() 223 ids = [id for id in [l.split()[1] for l in lines if l.startswith("#define")] if id.startswith("IDM_")] 224 #print ids 225 226 Regenerate(root + "scintilla/src/KeyWords.cxx", "//", NATIVE, lexerModules) 227 Regenerate(root + "scintilla/win32/makefile", "#", NATIVE, lexFiles) 228 Regenerate(root + "scintilla/win32/scintilla.mak", "#", NATIVE, lexFiles) 229 Regenerate(root + "scintilla/win32/scintilla_vc6.mak", "#", NATIVE, lexFiles) 230 # Use Unix EOLs for gtk Makefiles so they work for Linux users when 231 # extracted from the Scintilla source ZIP (typically created on 232 # Windows). 233 Regenerate(root + "scintilla/gtk/makefile", "#", LF, lexFiles) 234 Regenerate(root + "scintilla/gtk/scintilla.mak", "#", NATIVE, lexFiles) 235 Regenerate(root + "scite/win32/makefile", "#", NATIVE, lexFiles, propFiles) 236 Regenerate(root + "scite/win32/scite.mak", "#", NATIVE, lexFiles, propFiles) 237 Regenerate(root + "scite/src/SciTEProps.cxx", "//", NATIVE, lexerProperties, ids) 238 Generate(root + "scite/boundscheck/vcproj.gen", 239 root + "scite/boundscheck/SciTE.vcproj", "#", NATIVE, lexFiles) 240 241RegenerateAll() 242