1# @HEADER 2# ************************************************************************ 3# 4# TriBITS: Tribal Build, Integrate, and Test System 5# Copyright 2013 Sandia Corporation 6# 7# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, 8# the U.S. Government retains certain rights in this software. 9# 10# Redistribution and use in source and binary forms, with or without 11# modification, are permitted provided that the following conditions are 12# met: 13# 14# 1. Redistributions of source code must retain the above copyright 15# notice, this list of conditions and the following disclaimer. 16# 17# 2. Redistributions in binary form must reproduce the above copyright 18# notice, this list of conditions and the following disclaimer in the 19# documentation and/or other materials provided with the distribution. 20# 21# 3. Neither the name of the Corporation nor the names of the 22# contributors may be used to endorse or promote products derived from 23# this software without specific prior written permission. 24# 25# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY 26# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE 29# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 30# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 31# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 32# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 33# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 34# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 35# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 36# 37# ************************************************************************ 38# @HEADER 39 40""" 41Python module containing general support functions for creating scripts 42""" 43 44# 45# Check the python version 46# 47 48import sys 49if sys.version < '2.6': 50 print("Error, Python version is " + sys.version + " < 2.6!") 51 sys.exit(1) 52 53# 54# Import commands 55# 56import os 57import re 58import math 59import subprocess 60import time 61import datetime 62import optparse 63import traceback 64 65# 66# Byte array / string / unicode support across Python 2 & 3 67# 68# Note that the str class in Python 2 is an ASCII string (byte) array and in 69# Python 3 it is a Unicode object. For Python 3 code that is backward compatible 70# with Python 2, we sometimes need version-specific conversion functions to give 71# us the data type we desire. These functions are: 72# 73# b(x) return a byte array of str x, much like b'<string const>' in 74# Python 3 75# s(x) return a version-specific str object equivalent to x 76# u(x) return a unicode object equivalent to x, much like 77# u'<string const>' in Python 2 78# 79if sys.version_info < (3,): 80 # Python 2 81 def b(x): return x 82 def s(x): return x 83 def u(x): return unicode(x) 84else: 85 # Python 3 86 import codecs 87 def b(x): return codecs.latin_1_encode(x)[0] 88 def s(x): 89 try: 90 return x.decode("utf-8") 91 except AttributeError: 92 return x 93 def u(x): return x 94 95verboseDebug = False 96 97 98# 99# Determine what system we are on: 100# 101 102rePlatformName = re.compile(r"^[a-zA-Z]+") 103platformStr = rePlatformName.findall(sys.platform)[0] 104#print("\nplatformStr = " + platformStr) 105 106 107###################################### 108# Script location functions 109###################################### 110 111 112def getScriptBaseDir(): 113 return os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) 114 115 116def getScriptName(): 117 return os.path.basename(os.path.dirname(sys.argv[0])) 118 119def getCompleteFileDirname(filename): 120 return os.path.dirname(os.path.realpath(os.path.abspath(filename))) 121 122###################################### 123# List helper functions 124###################################### 125 126 127def findInSequence(seq, item): 128 for i in range(0, len(seq)): 129 if seq[i] == item: 130 return i 131 return -1 132 133 134def removeItemsFromList(list, items): 135 numItemsRemoved = 0 136 for item in items: 137 if item in list: 138 idx = list.index(item) 139 del list[idx] 140 numItemsRemoved = numItemsRemoved + 1 141 return numItemsRemoved 142 143 144###################################### 145# String helper functions 146###################################### 147 148 149def stripWhitespaceFromStringList(stringList): 150 return [x.strip() for x in stringList] 151 152 153def isSubstrInMultiLineString(inputStr, findStr): 154 return inputStr.find(findStr) >= 0 155 156 157def getStrUnderlineStr(width): 158 underlineStr = "" 159 for i in range(width): 160 underlineStr += "-" 161 return underlineStr 162 163 164def arrayToFormattedString(array_in, offsetStr = ""): 165 sout = "" 166 sout += offsetStr + "[\n" 167 for i in range(0, len(array_in)): 168 if i != len(array_in)-1: 169 commaChar = "," 170 else: 171 commaChar = "" 172 sout += (offsetStr + " \'" + str(array_in[i]) + "\'"+commaChar+"\n") 173 sout += offsetStr + "]\n" 174 return sout 175 176 177def extractLinesAfterRegex(string_in, regex_in): 178 #print("regex_in = " + regex_in) 179 reMatch = re.compile(regex_in) 180 linesExtracted = "" 181 foundRegex = False 182 for line in string_in.strip().splitlines(): 183 #print("line = '" + line + "'") 184 if not foundRegex: 185 matchObj = reMatch.match(line) 186 #print("matchObj = " + matchObj) 187 if matchObj: 188 foundRegex = True 189 if foundRegex: 190 linesExtracted += line + "\n" 191 return linesExtracted 192 193 194def extractLinesMatchingRegex(string_in, regex_in): 195 #print("regex_in = " + regex_in) 196 string_in = s(string_in) 197 reMatch = re.compile(regex_in) 198 linesExtracted = "" 199 for line in string_in.strip().splitlines(): 200 #print("line = '" + line + "'") 201 matchObj = reMatch.match(line) 202 #print("matchObj = " + matchObj) 203 if matchObj: 204 linesExtracted += line + "\n" 205 return linesExtracted 206# NOTE: Above is *NOT* using tested! 207 208 209def extractLinesMatchingSubstr(string_in, substr_in): 210 #print("substr_in = '" + substr_in + "'") 211 string_in = s(string_in) 212 linesExtracted = "" 213 for line in string_in.strip().splitlines(): 214 #print("line = '" + line + "'") 215 if substr_in in line: 216 #print("matched '" + substr_in + "'") 217 linesExtracted += line + "\n" 218 return linesExtracted 219# NOTE: Above is *NOT* unit tested! 220 221 222# Convert a dictionary to a string, using a sorted set of keys. 223# 224# This is needed to provide a portable string representation across various 225# versions of Python and platforms (see TriBITS GitHub Issue #119). 226def sorted_dict_str(d): 227 items = [] 228 keys = list(d.keys()) 229 keys.sort() 230 for key in keys: 231 if isinstance(d[key], dict): 232 value = sorted_dict_str(d[key]) 233 else: 234 value = repr(d[key]) 235 items.append(repr(key) + ": " + value) 236 return "{" + ", ".join(items) + "}" 237 238 239 240############################################## 241# System command unit testing utiltities 242############################################## 243 244 245class InterceptedCmndStruct: 246 247 def __init__(self, cmndRegex, cmndReturn, cmndOutput): 248 self.cmndRegex = cmndRegex 249 self.cmndReturn = cmndReturn 250 self.cmndOutput = cmndOutput 251 252 def __str__(self): 253 return "{cmndRegex='"+self.cmndRegex+"'," \ 254 " cmndReturn="+str(self.cmndReturn)+"," \ 255 " cmndOutput='"+str(self.cmndOutput)+"'}" 256 257# 258# Class that is used to record a set of commands that will be used to 259# intercept commands 260# 261 262class SysCmndInterceptor: 263 264 def __init__(self): 265 self.__fallThroughCmndRegexList = [] 266 self.__interceptedCmndStructList = [] 267 self.__allowExtraCmnds = True 268 269 def setFallThroughCmndRegex(self, cmndRegex): 270 self.__fallThroughCmndRegexList.append(cmndRegex) 271 272 def setInterceptedCmnd(self, cmndRegex, cmndReturn, cmndOutput=None): 273 self.__interceptedCmndStructList.append( 274 InterceptedCmndStruct(cmndRegex, cmndReturn, cmndOutput) ) 275 276 def setAllowExtraCmnds(self, allowExtraCmnds): 277 self.__allowExtraCmnds = allowExtraCmnds 278 279 def hasInterceptedCmnds(self): 280 return len(self.__interceptedCmndStructList) > 0 281 282 def getFallThroughCmndRegexList(self): 283 return self.__fallThroughCmndRegexList[:] 284 285 def getInterceptedCmndStructList(self): 286 return self.__interceptedCmndStructList[:] 287 288 def doProcessInterceptedCmnd(self, cmnd): 289 #print("doProcessInterceptedCmnd(): cmnd='" + cmnd + "'") 290 if self.isFallThroughCmnd(cmnd): 291 return False 292 if len(self.__interceptedCmndStructList) > 0: 293 return True 294 if not self.__allowExtraCmnds: 295 return True 296 return False 297 298 def isFallThroughCmnd(self, cmnd): 299 for interceptCmndStruct in self.__interceptedCmndStructList: 300 if re.match(interceptCmndStruct.cmndRegex, cmnd): 301 return False 302 for cmndRegex in self.__fallThroughCmndRegexList: 303 if re.match(cmndRegex, cmnd): 304 return True 305 return False 306 307 def nextInterceptedCmndStruct(self, cmnd): 308 assert(not self.isFallThroughCmnd(cmnd)) 309 if len(self.__interceptedCmndStructList) == 0: 310 raise Exception("Error, cmnd='"+cmnd+"' is past the last expected command!") 311 ics = self.__interceptedCmndStructList[0] 312 if not re.match(ics.cmndRegex, cmnd): 313 raise Exception("Error, cmnd='" + cmnd + "' did not match the" \ 314 " expected regex='" + ics.cmndRegex + "'!") 315 self.__interceptedCmndStructList.pop(0) 316 return (ics.cmndReturn, ics.cmndOutput) 317 318 def clear(self): 319 self.__fallThroughCmndRegexList = [] 320 self.__interceptedCmndStructList = [] 321 self.__allowExtraCmnds = True 322 323 def readCommandsFromStr(self, cmndsStr): 324 lines = cmndsStr.splitlines() 325 for line in lines: 326 #print("line: '" + line + "'") 327 if line == "": 328 continue 329 splitArray = line.split(':') 330 (tag, entry) = (splitArray[0], ':'.join(splitArray[1:])) 331 #(tag, entry) = line.split(':') 332 #print("(tag, entry) = " + str((tag, entry))) 333 if tag == "FT": 334 self.__fallThroughCmndRegexList.append(entry.strip()) 335 elif tag == "IT": 336 entryArray = entry.split(';') 337 if len(entryArray) < 3: 338 raise Exception("Error, invalid line {"+line+"}") 339 cmndRegex = entryArray[0] 340 cmndReturn = entryArray[1] 341 cmndOutput = "" 342 for cmndOutputEntry in entryArray[2:]: 343 #print("cmndOutputEntry = {" + cmndOutputEntry + "}") 344 cmndOutput += cmndOutputEntry.strip()[1:-1]+"\n" 345 #print("(cmndRegex, cmndReturn, cmndOutput) = " + 346 # str((cmndRegex, cmndReturn, cmndOutput))) 347 self.__interceptedCmndStructList.append( 348 InterceptedCmndStruct(cmndRegex.strip(), int(cmndReturn), cmndOutput) 349 ) 350 else: 351 raise Exception("Error, invalid tag = '"+tag+"'!") 352 353 def assertAllCommandsRun(self): 354 if len(self.__interceptedCmndStructList) > 0: 355 raise Exception("Error, all of the commands have not been run starting with" \ 356 " the command " + str(self.__interceptedCmndStructList[0]) 357 + "!") 358 359 360g_sysCmndInterceptor = SysCmndInterceptor() 361 362 363# Read the command interepts from a file? 364cmndInterceptsFile = os.environ.get( 365 "GENERAL_SCRIPT_SUPPORT_CMND_INTERCEPTS_FILE","") 366if cmndInterceptsFile: 367 cmndInterceptsFileStr = open(cmndInterceptsFile, 'r').read() 368 print("\nReading system command intercepts from file '" + 369 cmndInterceptsFile + 370 "' with contents:\n" + 371 "-----------------------------------\n" + 372 cmndInterceptsFileStr + 373 "-----------------------------------\n") 374 g_sysCmndInterceptor.readCommandsFromStr(cmndInterceptsFileStr) 375 g_sysCmndInterceptor.setAllowExtraCmnds(False) 376 377 378# Dump all commands being performed? 379g_dumpAllSysCmnds = "GENERAL_SCRIPT_SUPPORT_DUMD_COMMANDS" in os.environ 380 381 382def runSysCmndInterface(cmnd, outFile=None, rtnOutput=False, extraEnv=None, \ 383 workingDir="", getStdErr=False \ 384 ): 385 if g_dumpAllSysCmnds: 386 print("\nDUMP SYS CMND: " + cmnd + "\n") 387 if outFile!=None and rtnOutput==True: 388 raise Exception("Error, both outFile and rtnOutput can not be true!") 389 if g_sysCmndInterceptor.doProcessInterceptedCmnd(cmnd): 390 (cmndReturn, cmndOutput) = g_sysCmndInterceptor.nextInterceptedCmndStruct(cmnd) 391 if rtnOutput: 392 if cmndOutput==None: 393 raise Exception("Error, the command '"+cmnd+"' gave None output when" \ 394 " non-null output was expected!") 395 return (cmndOutput, cmndReturn) 396 if outFile: 397 writeStrToFile(outFile, cmndOutput) 398 return cmndReturn 399 # Else, fall through 400 if extraEnv: 401 fullEnv = os.environ.copy() 402 fullEnv.update(extraEnv) 403 else: 404 fullEnv = None 405 pwd = None 406 if workingDir: 407 pwd = os.getcwd() 408 os.chdir(workingDir) 409 rtnObject = None 410 try: 411 if rtnOutput: 412 if getStdErr: 413 child = subprocess.Popen(cmnd, shell=True, stdout=subprocess.PIPE, 414 stderr = subprocess.STDOUT, env=fullEnv) 415 else: 416 child = subprocess.Popen(cmnd, shell=True, stdout=subprocess.PIPE, 417 env=fullEnv) 418 data = child.stdout.read() 419 #print("data = '" + str(data) + "'") 420 child.wait() 421 rtnCode = child.returncode 422 #print("rtnCode = '" + str(rtnCode) + "'") 423 rtnObject = (data, rtnCode) 424 else: 425 outFileHandle = None 426 if outFile: 427 outFileHandle = open(outFile, 'w') 428 rtnCode = subprocess.call(cmnd, shell=True, stderr=subprocess.STDOUT, 429 stdout=outFileHandle, env=fullEnv) 430 rtnObject = rtnCode 431 finally: 432 if pwd: os.chdir(pwd) 433 return rtnObject 434 435 436###################################### 437# System interaction utilties 438###################################### 439 440 441def runSysCmnd(cmnd, throwExcept=True, outFile=None, workingDir="", 442 extraEnv=None, echoCmndForDebugging=False \ 443 ): 444 """Run system command and optionally throw on failure""" 445 sys.stdout.flush() 446 sys.stderr.flush() 447 try: 448 outFileHandle = None 449 if echoCmndForDebugging: print("Running: "+cmnd) 450 rtnCode = runSysCmndInterface(cmnd, outFile=outFile, extraEnv=extraEnv, 451 workingDir=workingDir) 452 except OSError as e: 453 rtnCode = 1 # Just some error code != 0 please! 454 if rtnCode != 0 and throwExcept: 455 raise RuntimeError("Error, the command '%s' failed with error code %d" 456 % (cmnd, rtnCode)) 457 return rtnCode 458 459 460def echoRunSysCmnd(cmnd, throwExcept=True, outFile=None, msg=None, 461 timeCmnd=False, verbose=True, workingDir="", returnTimeCmnd=False, 462 extraEnv=None 463 ): 464 """Echo command to be run and run command with runSysCmnd()""" 465 if verbose: 466 print("\nRunning: " + cmnd + "\n") 467 if workingDir: 468 print(" Running in working directory: " + workingDir + " ...\n") 469 if extraEnv: 470 print(" Appending environment:" + sorted_dict_str(extraEnv) + "\n") 471 if outFile: 472 print(" Writing console output to file " + outFile + " ...") 473 if msg and verbose: 474 print(" " + msg + "\n") 475 t1 = time.time() 476 totalTimeMin = -1.0 477 try: 478 rtn = runSysCmnd(cmnd, throwExcept, outFile, workingDir, extraEnv) 479 finally: 480 if timeCmnd: 481 t2 = time.time() 482 totalTimeMin = (t2-t1)/60.0 483 if verbose: 484 print("\n Runtime for command = %f minutes" % totalTimeMin) 485 if returnTimeCmnd: 486 return (rtn, totalTimeMin) 487 return rtn 488 489 490def getCmndOutput(cmnd, stripTrailingSpaces=False, throwOnError=True, workingDir="", \ 491 getStdErr=False, rtnCode=False \ 492 ): 493 """Run a shell command and return its output""" 494 (data, errCode) = runSysCmndInterface(cmnd, rtnOutput=True, workingDir=workingDir, 495 getStdErr=getStdErr) 496 if errCode != 0: 497 if throwOnError: 498 raise RuntimeError('%s failed w/ exit code %d:\n\n%s' % (cmnd, errCode, data)) 499 dataToReturn = data 500 if stripTrailingSpaces: 501 dataToReturn = data.rstrip() 502 if rtnCode: 503 return (dataToReturn, errCode) 504 return dataToReturn 505 506 507def pidStillRunning(pid): 508 #print("\npid = '" + pid + "'") 509 cmnd = "kill -s 0 "+pid 510 cmndReturn = runSysCmnd(cmnd, False) 511 #print("\ncmndReturn = " + cmndReturn) 512 return cmndReturn == 0 513 514 515###################################### 516# File/path utilities 517###################################### 518 519 520def getFilePathArray(filePathStr): 521 return filePathStr.split('/') 522 523 524def joinDirs(dirArray): 525 """ 526 Join directories. 527 528 2009/06/09: rabartl: We should be able to just use os.path.join(...) but I 529 found when used in at least on context that it resulted in not joining the 530 elements but instead just returning the array. 531 """ 532 dirPath = "" 533 for dir in dirArray: 534 if not dirPath: 535 dirPath = dir 536 else: 537 dirPath = dirPath + "/" + dir 538 return dirPath 539 540 541def downDirsArray(numDownDirs): 542 downDirsPathArray = [] 543 for i in range(0, numDownDirs): 544 downDirsPathArray.append("..") 545 #print("\ndownDirsPathArray = " + downDirsPathArray) 546 return downDirsPathArray 547 548 549def normalizePath(path): 550 return os.path.normpath(path) 551 552 553def echoChDir(dirName, verbose=True): 554 if verbose: 555 print("\nChanging current directory to \'" + dirName + "\'") 556 if not os.path.isdir(dirName): 557 raise OSError("Error, the directory \'"+dirName+"\' does not exist in the" \ 558 + " base directory \'"+os.getcwd()+"\"!" ) 559 os.chdir(dirName) 560 if verbose: 561 print("\nCurrent directory is \'" + os.getcwd() + "\'\n") 562 563 564def createDir(dirName, cdIntoDir=False, verbose=False): 565 """Create a directory if it does not exist""" 566 if os.path.exists(dirName): 567 if not os.path.isdir(dirName): 568 errMsg = "\nError the path '" + dirName + \ 569 "'already exists but it is not a directory!" 570 if verbose: print(errMsg) 571 raise RuntimeError(errMsg) 572 if verbose: print("\nThe directory " + dirName + "already exists!") 573 else: 574 if verbose: print("\nCreating directory " + dirName + " ...") 575 os.mkdir(dirName) 576 if cdIntoDir: 577 echoChDir(dirName, verbose=verbose) 578 579 580def createDirsFromPath(path): 581 #print("\npath = " + path) 582 pathList = path.split("/") 583 #print("\npathList = " + pathList) 584 if not pathList[0]: 585 currDir = "/" 586 for dir in pathList: 587 currDir = os.path.join(currDir, dir) 588 if currDir and not os.path.exists(currDir): 589 #print("\ncurrDir = " + currDir) 590 createDir(currDir) 591 592 593def expandDirsDict(trilinosDirsDict_inout): 594 595 for dir in list(trilinosDirsDict_inout): 596 subdirsList = dir.split("/") 597 #print("\nsubdirsList = " + subdirsList) 598 for i in range(len(subdirsList)): 599 trilinosDirsDict_inout.update({joinDirs(subdirsList[:i+1]) : 0}) 600 601 602def removeIfExists(fileName): 603 if os.path.exists(fileName): 604 echoRunSysCmnd("rm "+fileName) 605 606 607def removeDirIfExists(dirName, verbose=False): 608 if os.path.exists(dirName): 609 if verbose: 610 print("Removing existing directory '" + dirName + "' ...") 611 echoRunSysCmnd("rm -rf "+dirName) 612 613 614def writeStrToFile(fileName, fileBodyStr): 615 open(fileName, 'w').write(fileBodyStr) 616 617 618def readStrFromFile(fileName): 619 return open(fileName, 'r').read() 620 621 622def getFileNamesWithFileTag( baseDir, fileTag ): 623 """Get a list of file names with a given date stamp""" 624 return getCmndOutput( 625 'cd %s && ls *%s*' % (baseDir, fileTag), 626 throwOnError=False 627 ).split() 628 629 630def getFileNameFromGlob( baseDir, fileNameGlob ): 631 return getCmndOutput("cd "+baseDir+" && ls "+fileNameGlob, True, False) 632 633 634def isEmptyDir( absDir ): 635 return (len(os.listdir(absDir)) == 0) 636 637 638def getDirSizeInGb(dir): 639 sizeIn1024Kb = int(getCmndOutput("du -s "+dir).split('\t')[0]) 640 #print("\nsizeIn1024Kb = " + str(sizeIn1024Kb)) 641 return float(sizeIn1024Kb)/1e+6 # Size in Gb! 642 643 644def isPathChar(char): 645 return (char.isalnum() or char == '/') and (not char == ' ') 646 647 648# 2008/07/08: rabartl: This silly function is needed because on the sun 649# machine (i.e. sass8000), the 'which' command returns some non-printable 650# chars from the beginning of the output string that don't form a valid path. 651# This was *very* hard to debug but this function is able to select the valid 652# path string. This has been tested at least on linux and the sun and should 653# work anywhere. 654def cleanBadPath(inputPath): 655 cleanPath = "" 656 for i in range(len(inputPath)-1, -1, -1): 657 char = inputPath[i] 658 if not isPathChar(char): 659 break 660 cleanPath = char + cleanPath 661 return cleanPath 662 663 664def getRelativePathFrom1to2(absPath1, absPath2): 665 #print("\nabsPath1 =" + absPath1) 666 #print("\nabsPath2 =" + absPath2) 667 absPath1_array = absPath1.split('/') 668 absPath2_array = absPath2.split('/') 669 #print("\nabsPath1_array =" + absPath1_array) 670 #print("\nabsPath2_array =" + absPath2_array) 671 absPath1_array_len = len(absPath1_array) 672 absPath2_array_len = len(absPath2_array) 673 maxBaseDirDepth = min(absPath1_array_len, absPath2_array_len) 674 baseDirDepth = 0 675 for dirIdx in range(0, maxBaseDirDepth): 676 dir1 = absPath1_array[dirIdx] 677 dir2 = absPath2_array[dirIdx] 678 if dir1 != dir2: 679 break 680 baseDirDepth = baseDirDepth + 1 681 #print("\nbaseDirDepth = %d" % baseDirDepth) 682 numDownDirs = absPath1_array_len - baseDirDepth 683 #print("\nnumDownDirs = %d" % numDownDirs) 684 if numDownDirs > 0: 685 downDirPath = joinDirs(downDirsArray(numDownDirs)) 686 else: 687 downDirPath = "." 688 #print("\ndownDirPath = '" + downDirPath + "'") 689 if baseDirDepth == absPath2_array_len: 690 upDirPath = "." 691 else: 692 upDirPath = joinDirs(absPath2_array[baseDirDepth:]) 693 #print("\nupDirPath = " + upDirPath ) 694 #print("\nupDirPath = '" + upDirPath + "'") 695 relPath = os.path.join(downDirPath, upDirPath) 696 if relPath == "./.": 697 return "." 698 return relPath 699 700 701def getExecBaseDir(execName): 702 whichOutput = getCmndOutput("type -p "+execName, True, False) 703 # Handle the outpue 'execName is execFullPath' output 704 execFullPath = whichOutput.split(' ')[-1] 705 #print("\nexecFullPath = " + execFullPath) 706 execNameMatchRe = r"^(.+)/"+execName 707 execNameGroups = re.findall(execNameMatchRe, execFullPath) 708 #print("\nexecNameGroups = " + execNameGroups) 709 if not execNameGroups: 710 return None 711 execBaseDir = execNameGroups[0] 712 #print("\nexecBaseDir = \"" + execBaseDir + "\"") 713 #execBaseDir = cleanBadPath(execBaseDir) 714 #print("\nexecBaseDir = \"" + execBaseDir + "\"") 715 return execBaseDir 716 717 718def extractAppendUniqueDirsDictFromFileNames(filenamesArray, dirsDict): 719 for filename in filenamesArray: 720 dirsDict.update( { normalizePath(os.path.dirname(filename)) : 0 } ) 721 722 723def copyFileAndReplaceTokens( scriptsDir, inputFile, tokenReplacementList, 724 outputFile ): 725 726 """Copies an input stub file and then does a set of token replacements""" 727 728 echoRunSysCmnd("cp "+inputFile+" "+outputFile, verbose=verboseDebug) 729 730 for tokenReplacementPair in tokenReplacementList: 731 oldToken = tokenReplacementPair[0] 732 newToken = tokenReplacementPair[1] 733 echoRunSysCmnd( scriptsDir+"/token-replace.pl "+oldToken+" "+newToken\ 734 +" "+outputFile+" "+outputFile, verbose=verboseDebug ); 735 # ToDo: Replace above with native re commands 736 737 738class TeeOutput(object): 739 """ 740 Object that directs all calls to its write method to stdout as well 741 as a file. This is to be used as a simple replacement for the Unix 742 tee command. 743 """ 744 def __init__(self, outputfile): 745 """ Constructor takes a file-like object to write output to.""" 746 self._realstdout = sys.stdout 747 self._outputfile = outputfile 748 749 def _safe_outputfile_method(self, methodname, *args): 750 """ 751 Calls the method specified by methodname with the given args on 752 the internal file object if it is non-null. 753 """ 754 if self._outputfile is not None: 755 if hasattr(self._outputfile, methodname): 756 method = getattr(self._outputfile, methodname) 757 if method and callable(method): 758 method(*args) 759 760 def write(self, data): 761 """ 762 Write the given data to stdout and to the log file. 763 """ 764 self._realstdout.write(data) 765 self._safe_outputfile_method('write', data) 766 767 def flush(self): 768 """ 769 Flush the internal file buffers. 770 """ 771 self._realstdout.flush() 772 self._safe_outputfile_method('flush') 773 774 775###################################### 776# Shell argument helpers 777###################################### 778 779 780reCmndLineArg = re.compile(r"(--.+=)(.+)") 781 782 783def requoteCmndLineArgs(inArgs): 784 argsStr = "" 785 for arg in inArgs: 786 splitArg = arg.split("=") 787 newArg = None 788 if len(splitArg) == 1: 789 newArg = arg 790 else: 791 newArg = splitArg[0]+"=\""+'='.join(splitArg[1:])+"\"" 792 #print("\nnewArg = " + newArg) 793 argsStr = argsStr+" "+newArg 794 return argsStr 795 796 797def commandLineOptionsToList(stringOptions): 798 """ 799 Convert a string of space separated command line options to a python 800 list of the individual optionstrings. 801 TODO: Handle shell quoting. 802 """ 803 return stringOptions.split() 804 805 806class ConfigurableOptionParser(optparse.OptionParser): 807 """ 808 OptionParser that accepts a python dictionary as a configuration 809 file to provide default value overrides for the options. 810 """ 811 def __init__(self, configuration, **kwargs): 812 """ 813 Constructor accepts a configuration dictionary with default values 814 for arguments and all of the OptionParser arguments as well. 815 """ 816 optparse.OptionParser.__init__(self, **kwargs) 817 self._configuration = configuration 818 819 def add_option(self, *args, **kwargs): 820 """ 821 Checks for a default override in the configuration dictionary and 822 modifies the default and help arguments before dispatching them to 823 the base class implementation. 824 """ 825 if 'default' in kwargs: 826 for arg in args: 827 kwargs['default'] = self._configuration.get(arg, kwargs['default']) 828 optparse.OptionParser.add_option(self, *args, **kwargs) 829 830 831###################################### 832# Debugging support 833###################################### 834 835 836def printStackTrace(): 837 sys.stdout.flush() 838 traceback.print_exc() 839 840 841class ErrorCaptureOptionParser(optparse.OptionParser): 842 __sawError = None 843 def __init__(self, usage="%prog [options]", version=None): 844 optparse.OptionParser.__init__(self, usage, version) 845 __sawError = False 846 def error(self, msg): 847 raise Exception("Received error message: " + msg) 848 849 850###################################### 851# HTML directory browsing 852###################################### 853 854 855def createIndexHtmlBrowserList(baseDir, fileDirList = None): 856 htmlList = "" 857 # Get the fileDirList if empty 858 if not fileDirList: 859 fileDirList = os.listdir(baseDir) 860 fileDirList.sort() 861 # Create the HTML header 862 htmlList += "" \ 863 + "<ul>\n" \ 864 + "<li><a href=\"..\">..</a></li>\n" 865 # Fill in links to directories first 866 for fd in fileDirList: 867 absFd = baseDir+"/"+fd 868 #print("isfile(" + fd + ") = " + str(os.path.isfile(absFd))) 869 #print("isdir(" + fd + ") = " + str(os.path.isdir(absFd) )) 870 if not os.path.isfile(absFd): 871 htmlList = htmlList \ 872 +"<li>dir: <a href=\""+fd+"\">"+fd+"</a></li>\n" 873 # Fill in links to regular files second 874 for fd in fileDirList: 875 absFd = baseDir+"/"+fd 876 if os.path.isfile(absFd): 877 if fd != 'index.html': 878 htmlList = htmlList \ 879 +"<li>file: <a href=\""+fd+"\">"+fd+"</a></li>\n" 880 # Write the footer 881 htmlList = htmlList \ 882 + "</ul>\n" 883 return htmlList 884 885 886 887def createIndexHtmlBrowserFile(baseDir, fileDirList): 888 """Creates an HTML browser file as a returned string.""" 889 htmlFile = "" \ 890 + "<html>\n" \ 891 + "<head>\n" \ 892 + "<title>"+baseDir+"</title>\n" \ 893 + "</head>\n" \ 894 + "<body>\n" \ 895 + "<b>"+baseDir+"</b>\n" \ 896 + createIndexHtmlBrowserList(baseDir, fileDirList) \ 897 + "</body>\n" \ 898 + "</html>\n" 899 return htmlFile 900 901 902def createHtmlBrowserFiles(absBaseDir, depth, verbose=False): 903 904 """Create a hierarchy of index.html files that will build a directory/file 905 browser for a web server that will not allow directory/file browsing.""" 906 907 #print("\nEntering createHtmlBrowserFiles(" + absBaseDir + ",%d" % (depth) + ")") 908 909 # Get the list of all of the files/directories in absBaseDir 910 fileDirList = os.listdir(absBaseDir) 911 fileDirList.sort() 912 #print("\nfileDirList = " + str(fileDirList) 913 #sys.stdout.flush() 914 915 # Get the index.html file HTML 916 indexHtml = createIndexHtmlBrowserFile(absBaseDir, fileDirList) 917 #print("\nindexHtml:\n" + indexHtml) 918 919 # Write the index.html file 920 indexFileName = absBaseDir+"/index.html" 921 if verbose: 922 print("\nWriting " + indexFileName) 923 open(indexFileName,'w').write(indexHtml) 924 925 # Loop through all of the directories and recursively call this function 926 if depth > 0: 927 for fd in fileDirList: 928 absFd = absBaseDir+"/"+fd 929 if os.path.isdir(absFd): 930 subDir = absFd 931 #print("\nCalling createHtmlBrowserFiles(" + subDir + ",%d" % (depth-1) 932 # + ")") 933 createHtmlBrowserFiles(absBaseDir+"/"+fd,depth-1) 934 935 #print("\nLeaving createHtmlBrowserFiles(" + absBaseDir + ",%d" % (depth) + 936 # ")") 937