1#! /usr/bin/env python
2# encoding: utf-8
3# Thomas Nagy, 2007-2010 (ita)
4
5"""
6Debugging helper for parallel compilation.
7
8Copy it to your project and load it with::
9
10	def options(opt):
11		opt.load('parallel_debug', tooldir='.')
12	def build(bld):
13		...
14
15The build will then output a file named pdebug.svg in the source directory.
16"""
17
18import re, sys, threading, time, traceback
19try:
20	from Queue import Queue
21except:
22	from queue import Queue
23from waflib import Runner, Options, Task, Logs, Errors
24
25SVG_TEMPLATE = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
26<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
27<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.0"
28   x="${project.x}" y="${project.y}" width="${project.width}" height="${project.height}" id="svg602" xml:space="preserve">
29
30<style type='text/css' media='screen'>
31	g.over rect { stroke:#FF0000; fill-opacity:0.4 }
32</style>
33
34<script type='text/javascript'><![CDATA[
35var svg  = document.getElementsByTagName('svg')[0];
36
37svg.addEventListener('mouseover', function(e) {
38	var g = e.target.parentNode;
39	var x = document.getElementById('r_' + g.id);
40	if (x) {
41		g.setAttribute('class', g.getAttribute('class') + ' over');
42		x.setAttribute('class', x.getAttribute('class') + ' over');
43		showInfo(e, g.id, e.target.attributes.tooltip.value);
44	}
45}, false);
46
47svg.addEventListener('mouseout', function(e) {
48		var g = e.target.parentNode;
49		var x = document.getElementById('r_' + g.id);
50		if (x) {
51			g.setAttribute('class', g.getAttribute('class').replace(' over', ''));
52			x.setAttribute('class', x.getAttribute('class').replace(' over', ''));
53			hideInfo(e);
54		}
55}, false);
56
57function showInfo(evt, txt, details) {
58${if project.tooltip}
59	tooltip = document.getElementById('tooltip');
60
61	var t = document.getElementById('tooltiptext');
62	t.firstChild.data = txt + " " + details;
63
64	var x = evt.clientX + 9;
65	if (x > 250) { x -= t.getComputedTextLength() + 16; }
66	var y = evt.clientY + 20;
67	tooltip.setAttribute("transform", "translate(" + x + "," + y + ")");
68	tooltip.setAttributeNS(null, "visibility", "visible");
69
70	var r = document.getElementById('tooltiprect');
71	r.setAttribute('width', t.getComputedTextLength() + 6);
72${endif}
73}
74
75function hideInfo(evt) {
76	var tooltip = document.getElementById('tooltip');
77	tooltip.setAttributeNS(null,"visibility","hidden");
78}
79]]></script>
80
81<!-- inkscape requires a big rectangle or it will not export the pictures properly -->
82<rect
83   x='${project.x}' y='${project.y}' width='${project.width}' height='${project.height}'
84   style="font-size:10;fill:#ffffff;fill-opacity:0.01;fill-rule:evenodd;stroke:#ffffff;"></rect>
85
86${if project.title}
87  <text x="${project.title_x}" y="${project.title_y}"
88    style="font-size:15px; text-anchor:middle; font-style:normal;font-weight:normal;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;font-family:Bitstream Vera Sans">${project.title}</text>
89${endif}
90
91
92${for cls in project.groups}
93  <g id='${cls.classname}'>
94    ${for rect in cls.rects}
95    <rect x='${rect.x}' y='${rect.y}' width='${rect.width}' height='${rect.height}' tooltip='${rect.name}' style="font-size:10;fill:${rect.color};fill-rule:evenodd;stroke:#000000;stroke-width:0.4;" />
96    ${endfor}
97  </g>
98${endfor}
99
100${for info in project.infos}
101  <g id='r_${info.classname}'>
102   <rect x='${info.x}' y='${info.y}' width='${info.width}' height='${info.height}' style="font-size:10;fill:${info.color};fill-rule:evenodd;stroke:#000000;stroke-width:0.4;" />
103   <text x="${info.text_x}" y="${info.text_y}"
104       style="font-size:12px;font-style:normal;font-weight:normal;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;font-family:Bitstream Vera Sans"
105   >${info.text}</text>
106  </g>
107${endfor}
108
109${if project.tooltip}
110  <g transform="translate(0,0)" visibility="hidden" id="tooltip">
111       <rect id="tooltiprect" y="-15" x="-3" width="1" height="20" style="stroke:black;fill:#edefc2;stroke-width:1"/>
112       <text id="tooltiptext" style="font-family:Arial; font-size:12;fill:black;"> </text>
113  </g>
114${endif}
115
116</svg>
117"""
118
119COMPILE_TEMPLATE = '''def f(project):
120	lst = []
121	def xml_escape(value):
122		return value.replace("&", "&amp;").replace('"', "&quot;").replace("'", "&apos;").replace("<", "&lt;").replace(">", "&gt;")
123
124	%s
125	return ''.join(lst)
126'''
127reg_act = re.compile(r"(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<code>[^}]*?)\})", re.M)
128def compile_template(line):
129
130	extr = []
131	def repl(match):
132		g = match.group
133		if g('dollar'):
134			return "$"
135		elif g('backslash'):
136			return "\\"
137		elif g('subst'):
138			extr.append(g('code'))
139			return "<<|@|>>"
140		return None
141
142	line2 = reg_act.sub(repl, line)
143	params = line2.split('<<|@|>>')
144	assert(extr)
145
146
147	indent = 0
148	buf = []
149	app = buf.append
150
151	def app(txt):
152		buf.append(indent * '\t' + txt)
153
154	for x in range(len(extr)):
155		if params[x]:
156			app("lst.append(%r)" % params[x])
157
158		f = extr[x]
159		if f.startswith(('if', 'for')):
160			app(f + ':')
161			indent += 1
162		elif f.startswith('py:'):
163			app(f[3:])
164		elif f.startswith(('endif', 'endfor')):
165			indent -= 1
166		elif f.startswith(('else', 'elif')):
167			indent -= 1
168			app(f + ':')
169			indent += 1
170		elif f.startswith('xml:'):
171			app('lst.append(xml_escape(%s))' % f[4:])
172		else:
173			#app('lst.append((%s) or "cannot find %s")' % (f, f))
174			app('lst.append(str(%s))' % f)
175
176	if extr:
177		if params[-1]:
178			app("lst.append(%r)" % params[-1])
179
180	fun = COMPILE_TEMPLATE % "\n\t".join(buf)
181	# uncomment the following to debug the template
182	#for i, x in enumerate(fun.splitlines()):
183	#	print i, x
184	return Task.funex(fun)
185
186# red   #ff4d4d
187# green #4da74d
188# lila  #a751ff
189
190color2code = {
191	'GREEN'  : '#4da74d',
192	'YELLOW' : '#fefe44',
193	'PINK'   : '#a751ff',
194	'RED'    : '#cc1d1d',
195	'BLUE'   : '#6687bb',
196	'CYAN'   : '#34e2e2',
197}
198
199mp = {}
200info = [] # list of (text,color)
201
202def map_to_color(name):
203	if name in mp:
204		return mp[name]
205	try:
206		cls = Task.classes[name]
207	except KeyError:
208		return color2code['RED']
209	if cls.color in mp:
210		return mp[cls.color]
211	if cls.color in color2code:
212		return color2code[cls.color]
213	return color2code['RED']
214
215def process(self):
216	m = self.generator.bld.producer
217	try:
218		# TODO another place for this?
219		del self.generator.bld.task_sigs[self.uid()]
220	except KeyError:
221		pass
222
223	self.generator.bld.producer.set_running(1, self)
224
225	try:
226		ret = self.run()
227	except Exception:
228		self.err_msg = traceback.format_exc()
229		self.hasrun = Task.EXCEPTION
230
231		# TODO cleanup
232		m.error_handler(self)
233		return
234
235	if ret:
236		self.err_code = ret
237		self.hasrun = Task.CRASHED
238	else:
239		try:
240			self.post_run()
241		except Errors.WafError:
242			pass
243		except Exception:
244			self.err_msg = traceback.format_exc()
245			self.hasrun = Task.EXCEPTION
246		else:
247			self.hasrun = Task.SUCCESS
248	if self.hasrun != Task.SUCCESS:
249		m.error_handler(self)
250
251	self.generator.bld.producer.set_running(-1, self)
252
253Task.Task.process_back = Task.Task.process
254Task.Task.process = process
255
256old_start = Runner.Parallel.start
257def do_start(self):
258	try:
259		Options.options.dband
260	except AttributeError:
261		self.bld.fatal('use def options(opt): opt.load("parallel_debug")!')
262
263	self.taskinfo = Queue()
264	old_start(self)
265	if self.dirty:
266		make_picture(self)
267Runner.Parallel.start = do_start
268
269lock_running = threading.Lock()
270def set_running(self, by, tsk):
271	with lock_running:
272		try:
273			cache = self.lock_cache
274		except AttributeError:
275			cache = self.lock_cache = {}
276
277		i = 0
278		if by > 0:
279			vals = cache.values()
280			for i in range(self.numjobs):
281				if i not in vals:
282					cache[tsk] = i
283					break
284		else:
285			i = cache[tsk]
286			del cache[tsk]
287
288		self.taskinfo.put( (i, id(tsk), time.time(), tsk.__class__.__name__, self.processed, self.count, by, ",".join(map(str, tsk.outputs)))  )
289Runner.Parallel.set_running = set_running
290
291def name2class(name):
292	return name.replace(' ', '_').replace('.', '_')
293
294def make_picture(producer):
295	# first, cast the parameters
296	if not hasattr(producer.bld, 'path'):
297		return
298
299	tmp = []
300	try:
301		while True:
302			tup = producer.taskinfo.get(False)
303			tmp.append(list(tup))
304	except:
305		pass
306
307	try:
308		ini = float(tmp[0][2])
309	except:
310		return
311
312	if not info:
313		seen = []
314		for x in tmp:
315			name = x[3]
316			if not name in seen:
317				seen.append(name)
318			else:
319				continue
320
321			info.append((name, map_to_color(name)))
322		info.sort(key=lambda x: x[0])
323
324	thread_count = 0
325	acc = []
326	for x in tmp:
327		thread_count += x[6]
328		acc.append("%d %d %f %r %d %d %d %s" % (x[0], x[1], x[2] - ini, x[3], x[4], x[5], thread_count, x[7]))
329
330	data_node = producer.bld.path.make_node('pdebug.dat')
331	data_node.write('\n'.join(acc))
332
333	tmp = [lst[:2] + [float(lst[2]) - ini] + lst[3:] for lst in tmp]
334
335	st = {}
336	for l in tmp:
337		if not l[0] in st:
338			st[l[0]] = len(st.keys())
339	tmp = [  [st[lst[0]]] + lst[1:] for lst in tmp ]
340	THREAD_AMOUNT = len(st.keys())
341
342	st = {}
343	for l in tmp:
344		if not l[1] in st:
345			st[l[1]] = len(st.keys())
346	tmp = [  [lst[0]] + [st[lst[1]]] + lst[2:] for lst in tmp ]
347
348
349	BAND = Options.options.dband
350
351	seen = {}
352	acc = []
353	for x in range(len(tmp)):
354		line = tmp[x]
355		id = line[1]
356
357		if id in seen:
358			continue
359		seen[id] = True
360
361		begin = line[2]
362		thread_id = line[0]
363		for y in range(x + 1, len(tmp)):
364			line = tmp[y]
365			if line[1] == id:
366				end = line[2]
367				#print id, thread_id, begin, end
368				#acc.append(  ( 10*thread_id, 10*(thread_id+1), 10*begin, 10*end ) )
369				acc.append( (BAND * begin, BAND*thread_id, BAND*end - BAND*begin, BAND, line[3], line[7]) )
370				break
371
372	if Options.options.dmaxtime < 0.1:
373		gwidth = 1
374		for x in tmp:
375			m = BAND * x[2]
376			if m > gwidth:
377				gwidth = m
378	else:
379		gwidth = BAND * Options.options.dmaxtime
380
381	ratio = float(Options.options.dwidth) / gwidth
382	gwidth = Options.options.dwidth
383	gheight = BAND * (THREAD_AMOUNT + len(info) + 1.5)
384
385
386	# simple data model for our template
387	class tobject(object):
388		pass
389
390	model = tobject()
391	model.x = 0
392	model.y = 0
393	model.width = gwidth + 4
394	model.height = gheight + 4
395
396	model.tooltip = not Options.options.dnotooltip
397
398	model.title = Options.options.dtitle
399	model.title_x = gwidth / 2
400	model.title_y = gheight + - 5
401
402	groups = {}
403	for (x, y, w, h, clsname, name) in acc:
404		try:
405			groups[clsname].append((x, y, w, h, name))
406		except:
407			groups[clsname] = [(x, y, w, h, name)]
408
409	# groups of rectangles (else js highlighting is slow)
410	model.groups = []
411	for cls in groups:
412		g = tobject()
413		model.groups.append(g)
414		g.classname = name2class(cls)
415		g.rects = []
416		for (x, y, w, h, name) in groups[cls]:
417			r = tobject()
418			g.rects.append(r)
419			r.x = 2 + x * ratio
420			r.y = 2 + y
421			r.width = w * ratio
422			r.height = h
423			r.name = name
424			r.color = map_to_color(cls)
425
426	cnt = THREAD_AMOUNT
427
428	# caption
429	model.infos = []
430	for (text, color) in info:
431		inf = tobject()
432		model.infos.append(inf)
433		inf.classname = name2class(text)
434		inf.x = 2 + BAND
435		inf.y = 5 + (cnt + 0.5) * BAND
436		inf.width = BAND/2
437		inf.height = BAND/2
438		inf.color = color
439
440		inf.text = text
441		inf.text_x = 2 + 2 * BAND
442		inf.text_y = 5 + (cnt + 0.5) * BAND + 10
443
444		cnt += 1
445
446	# write the file...
447	template1 = compile_template(SVG_TEMPLATE)
448	txt = template1(model)
449
450	node = producer.bld.path.make_node('pdebug.svg')
451	node.write(txt)
452	Logs.warn('Created the diagram %r', node)
453
454def options(opt):
455	opt.add_option('--dtitle', action='store', default='Parallel build representation for %r' % ' '.join(sys.argv),
456		help='title for the svg diagram', dest='dtitle')
457	opt.add_option('--dwidth', action='store', type='int', help='diagram width', default=800, dest='dwidth')
458	opt.add_option('--dtime', action='store', type='float', help='recording interval in seconds', default=0.009, dest='dtime')
459	opt.add_option('--dband', action='store', type='int', help='band width', default=22, dest='dband')
460	opt.add_option('--dmaxtime', action='store', type='float', help='maximum time, for drawing fair comparisons', default=0, dest='dmaxtime')
461	opt.add_option('--dnotooltip', action='store_true', help='disable tooltips', default=False, dest='dnotooltip')
462
463