1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Requires Python 2.7 or later 4 5import io, os, sys, unittest 6 7if sys.platform == "win32": 8 import XiteWin as Xite 9else: 10 import XiteQt as Xite 11 12keywordsHTML = [ 13b"b body content head href html link meta " 14 b"name rel script strong title type xmlns", 15b"function", 16b"sub" 17] 18 19keywordsPerl = [ 20b"NULL __FILE__ __LINE__ __PACKAGE__ __DATA__ __END__ AUTOLOAD " 21b"BEGIN CORE DESTROY END EQ GE GT INIT LE LT NE CHECK abs accept " 22b"alarm and atan2 bind binmode bless caller chdir chmod chomp chop " 23b"chown chr chroot close closedir cmp connect continue cos crypt " 24b"dbmclose dbmopen defined delete die do dump each else elsif endgrent " 25b"endhostent endnetent endprotoent endpwent endservent eof eq eval " 26b"exec exists exit exp fcntl fileno flock for foreach fork format " 27b"formline ge getc getgrent getgrgid getgrnam gethostbyaddr gethostbyname " 28b"gethostent getlogin getnetbyaddr getnetbyname getnetent getpeername " 29b"getpgrp getppid getpriority getprotobyname getprotobynumber getprotoent " 30b"getpwent getpwnam getpwuid getservbyname getservbyport getservent " 31b"getsockname getsockopt glob gmtime goto grep gt hex if index " 32b"int ioctl join keys kill last lc lcfirst le length link listen " 33b"local localtime lock log lstat lt map mkdir msgctl msgget msgrcv " 34b"msgsnd my ne next no not oct open opendir or ord our pack package " 35b"pipe pop pos print printf prototype push quotemeta qu " 36b"rand read readdir readline readlink readpipe recv redo " 37b"ref rename require reset return reverse rewinddir rindex rmdir " 38b"scalar seek seekdir select semctl semget semop send setgrent " 39b"sethostent setnetent setpgrp setpriority setprotoent setpwent " 40b"setservent setsockopt shift shmctl shmget shmread shmwrite shutdown " 41b"sin sleep socket socketpair sort splice split sprintf sqrt srand " 42b"stat study sub substr symlink syscall sysopen sysread sysseek " 43b"system syswrite tell telldir tie tied time times truncate " 44b"uc ucfirst umask undef unless unlink unpack unshift untie until " 45b"use utime values vec wait waitpid wantarray warn while write " 46b"xor " 47b"given when default break say state UNITCHECK __SUB__ fc" 48] 49 50class TestLexers(unittest.TestCase): 51 52 def setUp(self): 53 self.xite = Xite.xiteFrame 54 self.ed = self.xite.ed 55 self.ed.ClearAll() 56 self.ed.EmptyUndoBuffer() 57 58 def AsStyled(self, withWindowsLineEnds): 59 text = self.ed.Contents() 60 data = io.BytesIO() 61 prevStyle = -1 62 for o in range(self.ed.Length): 63 styleNow = self.ed.GetStyleAt(o) 64 if styleNow != prevStyle: 65 styleBuf = "{%0d}" % styleNow 66 data.write(styleBuf.encode('utf-8')) 67 prevStyle = styleNow 68 data.write(text[o:o+1]) 69 if withWindowsLineEnds: 70 return data.getvalue().replace(b"\n", b"\r\n") 71 else: 72 return data.getvalue() 73 74 def LexExample(self, name, lexerName, keywords, fileMode="b"): 75 self.ed.ClearAll() 76 self.ed.EmptyUndoBuffer() 77 self.ed.SetCodePage(65001) 78 self.ed.LexerLanguage = lexerName 79 mask = 0xff 80 for i in range(len(keywords)): 81 self.ed.SetKeyWords(i, keywords[i]) 82 83 nameExample = os.path.join("examples", name) 84 namePrevious = nameExample +".styled" 85 nameNew = nameExample +".new" 86 with open(nameExample, "rb") as f: 87 prog = f.read() 88 if fileMode == "t" and sys.platform == "win32": 89 prog = prog.replace(b"\r\n", b"\n") 90 BOM = b"\xEF\xBB\xBF" 91 if prog.startswith(BOM): 92 prog = prog[len(BOM):] 93 lenDocument = len(prog) 94 self.ed.AddText(lenDocument, prog) 95 self.ed.Colourise(0, lenDocument) 96 self.assertEquals(self.ed.EndStyled, lenDocument) 97 try: 98 with open(namePrevious, "rb") as f: 99 prevStyled = f.read() 100 if fileMode == "t" and sys.platform == "win32": 101 prog = prog.replace(b"\r\n", b"\n") 102 except EnvironmentError: 103 prevStyled = "" 104 progStyled = self.AsStyled(fileMode == "t" and sys.platform == "win32") 105 if progStyled != prevStyled: 106 with open(nameNew, "wb") as f: 107 f.write(progStyled) 108 print("Incorrect lex for " + name) 109 print(progStyled) 110 print(prevStyled) 111 self.assertEquals(progStyled, prevStyled) 112 # The whole file doesn't parse like it did before so don't try line by line 113 # as that is likely to fail many times. 114 return 115 116 if fileMode == "b": # "t" files are large and this is a quadratic check 117 # Try partial lexes from the start of every line which should all be identical. 118 for line in range(self.ed.LineCount): 119 lineStart = self.ed.PositionFromLine(line) 120 self.ed.StartStyling(lineStart, mask) 121 self.assertEquals(self.ed.EndStyled, lineStart) 122 self.ed.Colourise(lineStart, lenDocument) 123 progStyled = self.AsStyled(fileMode == "t" and sys.platform == "win32") 124 if progStyled != prevStyled: 125 print("Incorrect partial lex for " + name + " at line " + line) 126 with open(nameNew, "wb") as f: 127 f.write(progStyled) 128 self.assertEquals(progStyled, prevStyled) 129 # Give up after one failure 130 return 131 132 # Test lexing just once from beginning to end in text form. 133 # This is used for test cases that are too long to be exhaustively tested by lines and 134 # may be sensitive to line ends so are tested as if using Unix LF line ends. 135 def LexLongCase(self, name, lexerName, keywords, fileMode="b"): 136 self.LexExample(name, lexerName, keywords, "t") 137 138 def testCXX(self): 139 self.LexExample("x.cxx", b"cpp", [b"int"]) 140 141 def testPython(self): 142 self.LexExample("x.py", b"python", 143 [b"class def else for if import in print return while"]) 144 145 def testHTML(self): 146 self.LexExample("x.html", b"hypertext", keywordsHTML) 147 148 def testASP(self): 149 self.LexExample("x.asp", b"hypertext", keywordsHTML) 150 151 def testPHP(self): 152 self.LexExample("x.php", b"hypertext", keywordsHTML) 153 154 def testVB(self): 155 self.LexExample("x.vb", b"vb", [b"as dim or string"]) 156 157 def testLua(self): 158 self.LexExample("x.lua", b"lua", [b"function end"]) 159 160 def testNim(self): 161 self.LexExample("x.nim", b"nim", [b"else end if let proc"]) 162 163 def testRuby(self): 164 self.LexExample("x.rb", b"ruby", [b"class def end"]) 165 166 def testPerl(self): 167 self.LexExample("x.pl", b"perl", keywordsPerl) 168 169 def testPerl52(self): 170 self.LexLongCase("perl-test-5220delta.pl", b"perl", keywordsPerl) 171 172 def testPerlPrototypes(self): 173 self.LexLongCase("perl-test-sub-prototypes.pl", b"perl", keywordsPerl) 174 175 def testD(self): 176 self.LexExample("x.d", b"d", 177 [b"keyword1", b"keyword2", b"", b"keyword4", b"keyword5", 178 b"keyword6", b"keyword7"]) 179 180 def testTCL(self): 181 self.LexExample("x.tcl", b"tcl", [b"proc set socket vwait"]) 182 183if __name__ == '__main__': 184 Xite.main("lexTests") 185