1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# Requires Python 2.7 or later
4
5import io, os, sys, unittest
6
7if sys.platform == "win32":
8	import XiteWin as Xite
9else:
10	import XiteQt as Xite
11
12keywordsHTML = [
13b"b body content head href html link meta "
14	b"name rel script strong title type xmlns",
15b"function",
16b"sub"
17]
18
19keywordsPerl = [
20b"NULL __FILE__ __LINE__ __PACKAGE__ __DATA__ __END__ AUTOLOAD "
21b"BEGIN CORE DESTROY END EQ GE GT INIT LE LT NE CHECK abs accept "
22b"alarm and atan2 bind binmode bless caller chdir chmod chomp chop "
23b"chown chr chroot close closedir cmp connect continue cos crypt "
24b"dbmclose dbmopen defined delete die do dump each else elsif endgrent "
25b"endhostent endnetent endprotoent endpwent endservent eof eq eval "
26b"exec exists exit exp fcntl fileno flock for foreach fork format "
27b"formline ge getc getgrent getgrgid getgrnam gethostbyaddr gethostbyname "
28b"gethostent getlogin getnetbyaddr getnetbyname getnetent getpeername "
29b"getpgrp getppid getpriority getprotobyname getprotobynumber getprotoent "
30b"getpwent getpwnam getpwuid getservbyname getservbyport getservent "
31b"getsockname getsockopt glob gmtime goto grep gt hex if index "
32b"int ioctl join keys kill last lc lcfirst le length link listen "
33b"local localtime lock log lstat lt map mkdir msgctl msgget msgrcv "
34b"msgsnd my ne next no not oct open opendir or ord our pack package "
35b"pipe pop pos print printf prototype push quotemeta qu "
36b"rand read readdir readline readlink readpipe recv redo "
37b"ref rename require reset return reverse rewinddir rindex rmdir "
38b"scalar seek seekdir select semctl semget semop send setgrent "
39b"sethostent setnetent setpgrp setpriority setprotoent setpwent "
40b"setservent setsockopt shift shmctl shmget shmread shmwrite shutdown "
41b"sin sleep socket socketpair sort splice split sprintf sqrt srand "
42b"stat study sub substr symlink syscall sysopen sysread sysseek "
43b"system syswrite tell telldir tie tied time times truncate "
44b"uc ucfirst umask undef unless unlink unpack unshift untie until "
45b"use utime values vec wait waitpid wantarray warn while write "
46b"xor "
47b"given when default break say state UNITCHECK __SUB__ fc"
48]
49
50class TestLexers(unittest.TestCase):
51
52	def setUp(self):
53		self.xite = Xite.xiteFrame
54		self.ed = self.xite.ed
55		self.ed.ClearAll()
56		self.ed.EmptyUndoBuffer()
57
58	def AsStyled(self, withWindowsLineEnds):
59		text = self.ed.Contents()
60		data = io.BytesIO()
61		prevStyle = -1
62		for o in range(self.ed.Length):
63			styleNow = self.ed.GetStyleAt(o)
64			if styleNow != prevStyle:
65				styleBuf = "{%0d}" % styleNow
66				data.write(styleBuf.encode('utf-8'))
67				prevStyle = styleNow
68			data.write(text[o:o+1])
69		if withWindowsLineEnds:
70			return data.getvalue().replace(b"\n", b"\r\n")
71		else:
72			return data.getvalue()
73
74	def LexExample(self, name, lexerName, keywords, fileMode="b"):
75		self.ed.ClearAll()
76		self.ed.EmptyUndoBuffer()
77		self.ed.SetCodePage(65001)
78		self.ed.LexerLanguage = lexerName
79		mask = 0xff
80		for i in range(len(keywords)):
81			self.ed.SetKeyWords(i, keywords[i])
82
83		nameExample = os.path.join("examples", name)
84		namePrevious = nameExample +".styled"
85		nameNew = nameExample +".new"
86		with open(nameExample, "rb") as f:
87			prog = f.read()
88		if fileMode == "t" and sys.platform == "win32":
89			prog = prog.replace(b"\r\n", b"\n")
90		BOM = b"\xEF\xBB\xBF"
91		if prog.startswith(BOM):
92			prog = prog[len(BOM):]
93		lenDocument = len(prog)
94		self.ed.AddText(lenDocument, prog)
95		self.ed.Colourise(0, lenDocument)
96		self.assertEquals(self.ed.EndStyled, lenDocument)
97		try:
98			with open(namePrevious, "rb") as f:
99				prevStyled = f.read()
100			if fileMode == "t" and sys.platform == "win32":
101				prog = prog.replace(b"\r\n", b"\n")
102		except EnvironmentError:
103			prevStyled = ""
104		progStyled = self.AsStyled(fileMode == "t" and sys.platform == "win32")
105		if progStyled != prevStyled:
106			with open(nameNew, "wb") as f:
107				f.write(progStyled)
108			print("Incorrect lex for " + name)
109			print(progStyled)
110			print(prevStyled)
111			self.assertEquals(progStyled, prevStyled)
112			# The whole file doesn't parse like it did before so don't try line by line
113			# as that is likely to fail many times.
114			return
115
116		if fileMode == "b":	# "t" files are large and this is a quadratic check
117			# Try partial lexes from the start of every line which should all be identical.
118			for line in range(self.ed.LineCount):
119				lineStart = self.ed.PositionFromLine(line)
120				self.ed.StartStyling(lineStart, mask)
121				self.assertEquals(self.ed.EndStyled, lineStart)
122				self.ed.Colourise(lineStart, lenDocument)
123				progStyled = self.AsStyled(fileMode == "t" and sys.platform == "win32")
124				if progStyled != prevStyled:
125					print("Incorrect partial lex for " + name + " at line " + line)
126					with open(nameNew, "wb") as f:
127						f.write(progStyled)
128					self.assertEquals(progStyled, prevStyled)
129					# Give up after one failure
130					return
131
132	# Test lexing just once from beginning to end in text form.
133	# This is used for test cases that are too long to be exhaustively tested by lines and
134	# may be sensitive to line ends so are tested as if using Unix LF line ends.
135	def LexLongCase(self, name, lexerName, keywords, fileMode="b"):
136		self.LexExample(name, lexerName, keywords, "t")
137
138	def testCXX(self):
139		self.LexExample("x.cxx", b"cpp", [b"int"])
140
141	def testPython(self):
142		self.LexExample("x.py", b"python",
143			[b"class def else for if import in print return while"])
144
145	def testHTML(self):
146		self.LexExample("x.html", b"hypertext", keywordsHTML)
147
148	def testASP(self):
149		self.LexExample("x.asp", b"hypertext", keywordsHTML)
150
151	def testPHP(self):
152		self.LexExample("x.php", b"hypertext", keywordsHTML)
153
154	def testVB(self):
155		self.LexExample("x.vb", b"vb", [b"as dim or string"])
156
157	def testLua(self):
158		self.LexExample("x.lua", b"lua", [b"function end"])
159
160	def testNim(self):
161		self.LexExample("x.nim", b"nim", [b"else end if let proc"])
162
163	def testRuby(self):
164		self.LexExample("x.rb", b"ruby", [b"class def end"])
165
166	def testPerl(self):
167		self.LexExample("x.pl", b"perl", keywordsPerl)
168
169	def testPerl52(self):
170		self.LexLongCase("perl-test-5220delta.pl", b"perl", keywordsPerl)
171
172	def testPerlPrototypes(self):
173		self.LexLongCase("perl-test-sub-prototypes.pl", b"perl", keywordsPerl)
174
175	def testD(self):
176		self.LexExample("x.d", b"d",
177			[b"keyword1", b"keyword2", b"", b"keyword4", b"keyword5",
178			b"keyword6", b"keyword7"])
179
180	def testTCL(self):
181		self.LexExample("x.tcl", b"tcl", [b"proc set socket vwait"])
182
183if __name__ == '__main__':
184	Xite.main("lexTests")
185