1#! /usr/bin/python
2#
3# CDDL HEADER START
4#
5# The contents of this file are subject to the terms of the
6# Common Development and Distribution License (the "License").
7# You may not use this file except in compliance with the License.
8#
9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10# or http://www.opensolaris.org/os/licensing.
11# See the License for the specific language governing permissions
12# and limitations under the License.
13#
14# When distributing Covered Code, include this CDDL HEADER in each
15# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16# If applicable, add the following below this CDDL HEADER, with the
17# fields enclosed by brackets "[]" replaced with your own identifying
18# information: Portions Copyright [yyyy] [name of copyright owner]
19#
20# CDDL HEADER END
21#
22
23#
24# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
25# Use is subject to license terms.
26#
27
28#
29# Various database lookup classes/methods, i.e.:
30#     * monaco
31#     * bugs.opensolaris.org (b.o.o.)
32#     * opensolaris.org/cgi/arc.py (for ARC)
33#
34
35import re
36import urllib
37import urllib2
38import htmllib
39import os
40from socket import socket, AF_INET, SOCK_STREAM
41
42from onbld.Checks import onSWAN
43
44class BugException(Exception):
45	def __init__(self, data=''):
46		self.data = data
47		Exception.__init__(self, data)
48
49	def __str__(self):
50		return "Unknown error: %s" % self.data
51
52class NonExistentBug(BugException):
53	def __str__(self):
54		return "Bug %s does not exist" % self.data
55
56class BugDBException(Exception):
57	def __init__(self, data=''):
58		self.data = data
59		Exception.__init__(self, data)
60
61	def __str__(self):
62		return "Unknown bug database: %s" % self.data
63
64class BugDB(object):
65	"""Lookup change requests.
66
67	Object can be used on or off of SWAN, using either monaco or
68	bugs.opensolaris.org as a database.
69
70	Usage:
71	bdb = BugDB()
72	r = bdb.lookup("6455550")
73	print r["6455550"]["synopsis"]
74	r = bdb.lookup(["6455550", "6505625"])
75	print r["6505625"]["synopsis"]
76	"""
77
78	def __init__(self, priority = ("bugster",), forceBoo = False):
79		"""Create a BugDB object.
80
81		Keyword argument:
82		forceBoo: use b.o.o even from SWAN (default=False)
83		priority: use bug databases in this order
84		"""
85		self.__validBugDB = ["bugster"]
86		self.__onSWAN = not forceBoo and onSWAN()
87		for database in priority:
88			if database not in self.__validBugDB:
89				raise BugDBException, database
90		self.__priority = priority
91
92
93	def __boobug(self, cr):
94		cr = str(cr)
95		url = "http://bugs.opensolaris.org/view_bug.do"
96   		req = urllib2.Request(url, urllib.urlencode({"bug_id": cr}))
97		results = {}
98		try:
99			data = urllib2.urlopen(req).readlines()
100		except urllib2.HTTPError, e:
101			if e.code != 404:
102				print "ERROR: HTTP error at " + \
103					req.get_full_url() + \
104					" got error: " + str(e.code)
105				raise e
106			else:
107				raise NonExistentBug
108		except urllib2.URLError, e:
109			print "ERROR: could not connect to " + \
110				req.get_full_url() + \
111				' got error: "' + e.reason[1] + '"'
112			raise e
113		htmlParser = htmllib.HTMLParser(None)
114		metaHtmlRe = re.compile(r'^<meta name="([^"]+)" content="([^"]*)">$')
115		for line in data:
116			m = metaHtmlRe.search(line)
117			if not m:
118				continue
119			val = urllib.unquote(m.group(2))
120			htmlParser.save_bgn()
121			htmlParser.feed(val)
122			results[m.group(1)] = htmlParser.save_end()
123		htmlParser.close()
124
125		if "synopsis" not in results:
126			raise NonExistentBug(cr)
127
128		results["cr_number"] = cr
129		results["sub_category"] = results.pop("subcategory")
130		results["status"] = results.pop("state")
131		results["date_submitted"] = results.pop("submit_date")
132
133		return results
134
135
136	def __monaco(self, crs):
137		"""Return all info for requested change reports.
138
139		Argument:
140		crs: list of change request ids
141
142		Returns:
143		Dictionary, mapping CR=>dictionary, where the nested dictionary
144		is a mapping of field=>value
145		"""
146
147		#
148		# We request synopsis last, and split on only
149		# the number of separators that we expect to
150		# see such that a | in the synopsis doesn't
151		# throw us out of whack.
152		#
153		monacoFields = [ "cr_number", "category", "sub_category",
154			"area", "release", "build", "responsible_manager",
155			"responsible_engineer", "priority", "status", "sub_status",
156			"submitted_by", "date_submitted", "synopsis" ]
157		cmd = []
158		cmd.append("set What = cr." + ', cr.'.join(monacoFields))
159		cmd.append("")
160		cmd.append("set Which = cr.cr_number in (" + ','.join(crs) +")")
161		cmd.append("")
162		cmd.append("set FinalClauses = order by cr.cr_number")
163		cmd.append("")
164		cmd.append("doMeta genQuery cr")
165		url = "http://hestia.sfbay.sun.com/cgi-bin/expert?format="
166		url += "Pipe-delimited+text;Go=2;no_header=on;cmds="
167		url += urllib.quote_plus("\n".join(cmd))
168		results = {}
169		try:
170			data = urllib2.urlopen(url).readlines()
171		except urllib2.HTTPError, e:
172			print "ERROR: HTTP error at " + url + \
173				" got error: " + str(e.code)
174			raise e
175
176		except urllib2.URLError, e:
177			print "ERROR: could not connect to " + url + \
178				' got error: "' + e.reason[1] + '"'
179			raise e
180		for line in data:
181			line = line.rstrip('\n')
182			values = line.split('|', len(monacoFields) - 1)
183			v = 0
184			cr = values[0]
185			results[cr] = {}
186			for field in monacoFields:
187				results[cr][field] = values[v]
188				v += 1
189		return results
190
191	def lookup(self, crs):
192		"""Return all info for requested change reports.
193
194		Argument:
195		crs: one change request id (may be integer, string, or list),
196	             or multiple change request ids (must be a list)
197
198		Returns:
199		Dictionary, mapping CR=>dictionary, where the nested dictionary
200		is a mapping of field=>value
201		"""
202		results = {}
203		if not isinstance(crs, list):
204			crs = [str(crs)]
205		for database in self.__priority:
206			if database == "bugster":
207				if self.__onSWAN:
208					results.update(self.__monaco(crs))
209				# else we're off-swan and querying via boo, which we can
210				# only do one bug at a time
211				else:
212					for cr in crs:
213						cr = str(cr)
214						try:
215							results[cr] = self.__boobug(cr)
216						except NonExistentBug:
217							continue
218
219			# the CR has already been found by one bug database
220			# so don't bother looking it up in the others
221			for cr in crs:
222				if cr in results:
223					crs.remove(cr)
224
225		return results
226####################################################################
227def ARC(arclist):
228	opts = {}
229	url = "http://opensolaris.org/cgi/arc.py"
230	opts["n"] = str(len(arclist))
231	for i, arc in enumerate(arclist):
232		arc, case = arc
233		opts["arc" + str(i)] = arc
234		opts["case" + str(i)] = case
235	req = urllib2.Request(url, urllib.urlencode(opts))
236	try:
237		data = urllib2.urlopen(req).readlines()
238	except urllib2.HTTPError, e:
239		print "ERROR: HTTP error at " + req.get_full_url() + \
240			" got error: " + str(e.code)
241		raise e
242
243	except urllib2.URLError, e:
244		print "ERROR: could not connect to " + req.get_full_url() + \
245			' got error: "' + e.reason[1] + '"'
246		raise e
247	ret = {}
248	for line in data:
249		oneline = line.rstrip('\n')
250		fields = oneline.split('|')
251		# check if each is valid ( fields[0]::validity )
252		if fields[0] != "0":
253			continue
254		arc, case = fields[1].split(" ")
255		ret[(arc, case)] = fields[2]
256	return ret
257
258####################################################################
259
260# Pointers to the webrti server hostname & port to use
261# Using it directly is probably not *officially* supported, so we'll
262# have a pointer to the official `webrticli` command line interface
263# if using a direct socket connection fails for some reason, so we
264# have a fallback
265WEBRTI_HOST = 'webrti.sfbay.sun.com'
266WEBRTI_PORT = 9188
267WEBRTICLI = '/net/onnv.sfbay.sun.com/export/onnv-gate/public/bin/webrticli'
268
269
270class RtiException(Exception):
271	def __init__(self, data=''):
272		self.data = data
273		Exception.__init__(self, data)
274
275	def __str__(self):
276		return "Unknown error: %s" % self.data
277
278class RtiCallFailed(RtiException):
279	def __str__(self):
280		return "Unable to call webrti: %s" % self.data
281
282class RtiSystemProblem(RtiException):
283	def __str__(self):
284		return "RTI status cannot be determined: %s" % self.data
285
286class RtiIncorrectCR(RtiException):
287	def __str__(self):
288		return "Incorrect CR number specified: %s" % self.data
289
290class RtiNotFound(RtiException):
291	def __str__(self):
292		return "RTI not found: %s" % self.data
293
294class RtiNeedConsolidation(RtiException):
295	def __str__(self):
296		return "More than one consolidation has this CR: %s" % self.data
297
298class RtiBadGate(RtiException):
299	def __str__(self):
300		return "Incorrect gate name specified: %s" % self.data
301
302class RtiOffSwan(RtiException):
303	def __str__(self):
304		return "RTI status checks need SWAN access: %s" % self.data
305
306WEBRTI_ERRORS = {
307	'1': RtiSystemProblem,
308	'2': RtiIncorrectCR,
309	'3': RtiNotFound,
310	'4': RtiNeedConsolidation,
311	'5': RtiBadGate,
312}
313
314# Our Rti object which we'll use to represent an Rti query
315# It's really just a wrapper around the Rti connection, and attempts
316# to establish a direct socket connection and query the webrti server
317# directly (thus avoiding a system/fork/exec call).  If it fails, it
318# falls back to the webrticli command line client.
319
320returnCodeRe = re.compile(r'.*RETURN_CODE=(\d+)')
321class Rti:
322	"""Lookup an RTI.
323
324	Usage:
325	r = Rti("6640538")
326	print r.rtiNumber();
327	"""
328
329	def __init__(self, cr, gate=None, consolidation=None):
330		"""Create an Rti object for the specified change request.
331
332		Argument:
333		cr: change request id
334
335		Keyword arguments, to limit scope of RTI search:
336		gate: path to gate workspace (default=None)
337		consolidation: consolidation name (default=None)
338		"""
339
340		bufSz = 1024
341		addr = (WEBRTI_HOST, WEBRTI_PORT)
342		# If the passed 'cr' was given as an int, then wrap it
343		# into a string to make our life easier
344		if isinstance(cr, int):
345			cr = str(cr)
346		self.__queryCr = cr
347		self.__queryGate = gate
348		self.__queryConsolidation = consolidation
349
350		self.__webRtiOutput = []
351		self.__mainCR = []
352		self.__rtiNumber = []
353		self.__consolidation = []
354		self.__project = []
355		self.__status = []
356		self.__rtiType = []
357		try:
358			# try to use a direct connection to the
359			# webrti server first
360			sock = socket(AF_INET, SOCK_STREAM)
361			sock.connect(addr)
362			command = "WEBRTICLI/1.0\nRTIstatus\n%s\n" % cr
363			if consolidation:
364				command += "-c\n%s\n" % consolidation
365			if gate:
366				command += "-g\n%s\n" % gate
367			command += "\n"
368			sock.send(command)
369			dataList = []
370			# keep receiving data from the socket until the
371			# server closes the connection
372			stillReceiving = True
373			while stillReceiving:
374				dataPiece = sock.recv(bufSz)
375				if dataPiece:
376					dataList.append(dataPiece)
377				else:
378					stillReceiving = False
379			# create the lines, skipping the first
380			# ("WEBRTCLI/1.0\n")
381			data = '\n'.join(''.join(dataList).split('\n')[1:])
382		except:
383			if not onSWAN():
384				raise RtiOffSwan(cr)
385
386			if not os.path.exists(WEBRTICLI):
387				raise RtiCallFailed('not found')
388
389			# fallback to the "supported" webrticli interface
390			command = WEBRTICLI
391			if consolidation:
392				command += " -c " + consolidation
393			if gate:
394				command += " -g " + gate
395			command += " RTIstatus " + cr
396
397			try:
398				cliPipe = os.popen(command)
399			except:
400				# we couldn't call the webrticli for some
401				# reason, so return a failure
402				raise RtiCallFailed('unknown')
403
404			data = cliPipe.readline()
405
406		# parse the data to see if we got a return code
407		# if we did, then that's bad.  if we didn't,
408		# then our call was successfully
409		m = returnCodeRe.search(data)
410		if m:
411			# we got a return code, set it in our
412			# object, set the webRtiOutput for debugging
413			# or logging, and return a failure
414			if m.group(1) in WEBRTI_ERRORS:
415				exc = WEBRTI_ERRORS[m.group(1)]
416			else:
417				exc = RtiException
418			raise exc(data)
419
420		data = data.splitlines()
421		# At this point, we should have valid data
422		for line in data:
423			line = line.rstrip('\r\n')
424			self.__webRtiOutput.append(line)
425			fields = line.split(':')
426			self.__mainCR.append(fields[0])
427			self.__rtiNumber.append(fields[1])
428			self.__consolidation.append(fields[2])
429			self.__project.append(fields[3])
430			self.__status.append(fields[4])
431			self.__rtiType.append(fields[5])
432
433	# accessors in case callers need the raw data
434	def mainCR(self):
435		return self.__mainCR
436	def rtiNumber(self):
437		return self.__rtiNumber
438	def consolidation(self):
439		return self.__consolidation
440	def project(self):
441		return self.__project
442	def status(self):
443		return self.__status
444	def rtiType(self):
445		return self.__rtiType
446	def queryCr(self):
447		return self.__queryCr
448	def queryGate(self):
449		return self.__queryGate
450	def queryConsolidation(self):
451		return self.__queryConsolidation
452
453	# in practice, most callers only care about the following
454	def accepted(self):
455		for status in self.__status:
456			if status != "S_ACCEPTED":
457				return False
458		return True
459
460	# for logging/debugging in case the caller wants the raw webrti output
461	def webRtiOutput(self):
462		return self.__webRtiOutput
463
464