1# @HEADER
2# ************************************************************************
3#
4#            TriBITS: Tribal Build, Integrate, and Test System
5#                    Copyright 2013 Sandia Corporation
6#
7# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8# the U.S. Government retains certain rights in this software.
9#
10# Redistribution and use in source and binary forms, with or without
11# modification, are permitted provided that the following conditions are
12# met:
13#
14# 1. Redistributions of source code must retain the above copyright
15# notice, this list of conditions and the following disclaimer.
16#
17# 2. Redistributions in binary form must reproduce the above copyright
18# notice, this list of conditions and the following disclaimer in the
19# documentation and/or other materials provided with the distribution.
20#
21# 3. Neither the name of the Corporation nor the names of the
22# contributors may be used to endorse or promote products derived from
23# this software without specific prior written permission.
24#
25# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36#
37# ************************************************************************
38# @HEADER
39
40"""
41Python module containing general support functions for creating scripts
42"""
43
44#
45# Check the python version
46#
47
48import sys
49if sys.version < '2.6':
50   print("Error, Python version is " + sys.version + " < 2.6!")
51   sys.exit(1)
52
53#
54# Import commands
55#
56import os
57import re
58import math
59import subprocess
60import time
61import datetime
62import optparse
63import traceback
64
65#
66# Byte array / string / unicode support across Python 2 & 3
67#
68# Note that the str class in Python 2 is an ASCII string (byte) array and in
69# Python 3 it is a Unicode object. For Python 3 code that is backward compatible
70# with Python 2, we sometimes need version-specific conversion functions to give
71# us the data type we desire. These functions are:
72#
73#     b(x)    return a byte array of str x, much like b'<string const>' in
74#             Python 3
75#     s(x)    return a version-specific str object equivalent to x
76#     u(x)    return a unicode object equivalent to x, much like
77#             u'<string const>' in Python 2
78#
79if sys.version_info < (3,):
80   # Python 2
81   def b(x): return x
82   def s(x): return x
83   def u(x): return unicode(x)
84else:
85   # Python 3
86   import codecs
87   def b(x): return codecs.latin_1_encode(x)[0]
88   def s(x):
89      try:
90         return x.decode("utf-8")
91      except AttributeError:
92         return x
93   def u(x): return x
94
95verboseDebug = False
96
97
98#
99# Determine what system we are on:
100#
101
102rePlatformName = re.compile(r"^[a-zA-Z]+")
103platformStr = rePlatformName.findall(sys.platform)[0]
104#print("\nplatformStr = " + platformStr)
105
106
107######################################
108# Script location functions
109######################################
110
111
112def getScriptBaseDir():
113  return os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
114
115
116def getScriptName():
117  return os.path.basename(os.path.dirname(sys.argv[0]))
118
119def getCompleteFileDirname(filename):
120  return os.path.dirname(os.path.realpath(os.path.abspath(filename)))
121
122######################################
123# List helper functions
124######################################
125
126
127def findInSequence(seq, item):
128  for i in range(0, len(seq)):
129    if seq[i] == item:
130      return i
131  return -1
132
133
134def removeItemsFromList(list, items):
135  numItemsRemoved = 0
136  for item in items:
137    if item in list:
138      idx = list.index(item)
139      del list[idx]
140      numItemsRemoved = numItemsRemoved + 1
141  return numItemsRemoved
142
143
144######################################
145# String helper functions
146######################################
147
148
149def stripWhitespaceFromStringList(stringList):
150  return [x.strip() for x in stringList]
151
152
153def isSubstrInMultiLineString(inputStr, findStr):
154  return inputStr.find(findStr) >= 0
155
156
157def getStrUnderlineStr(width):
158  underlineStr = ""
159  for i in range(width):
160    underlineStr += "-"
161  return underlineStr
162
163
164def arrayToFormattedString(array_in, offsetStr = ""):
165  sout = ""
166  sout += offsetStr + "[\n"
167  for i in range(0, len(array_in)):
168    if i != len(array_in)-1:
169      commaChar = ","
170    else:
171      commaChar = ""
172    sout += (offsetStr + "  \'" + str(array_in[i]) + "\'"+commaChar+"\n")
173  sout += offsetStr + "]\n"
174  return sout
175
176
177def extractLinesAfterRegex(string_in, regex_in):
178  #print("regex_in = " + regex_in)
179  reMatch = re.compile(regex_in)
180  linesExtracted = ""
181  foundRegex = False
182  for line in string_in.strip().splitlines():
183    #print("line = '" + line + "'")
184    if not foundRegex:
185      matchObj = reMatch.match(line)
186      #print("matchObj = " + matchObj)
187      if matchObj:
188        foundRegex = True
189    if foundRegex:
190      linesExtracted += line + "\n"
191  return linesExtracted
192
193
194def extractLinesMatchingRegex(string_in, regex_in):
195  #print("regex_in = " + regex_in)
196  string_in = s(string_in)
197  reMatch = re.compile(regex_in)
198  linesExtracted = ""
199  for line in string_in.strip().splitlines():
200    #print("line = '" + line + "'")
201    matchObj = reMatch.match(line)
202    #print("matchObj = " + matchObj)
203    if matchObj:
204      linesExtracted += line + "\n"
205  return linesExtracted
206# NOTE: Above is *NOT* using tested!
207
208
209def extractLinesMatchingSubstr(string_in, substr_in):
210  #print("substr_in = '" + substr_in + "'")
211  string_in = s(string_in)
212  linesExtracted = ""
213  for line in string_in.strip().splitlines():
214    #print("line = '" + line + "'")
215    if substr_in in line:
216      #print("matched '" + substr_in + "'")
217      linesExtracted += line + "\n"
218  return linesExtracted
219# NOTE: Above is *NOT* unit tested!
220
221
222# Convert a dictionary to a string, using a sorted set of keys.
223#
224# This is needed to provide a portable string representation across various
225# versions of Python and platforms (see TriBITS GitHub Issue #119).
226def sorted_dict_str(d):
227  items = []
228  keys = list(d.keys())
229  keys.sort()
230  for key in keys:
231    if isinstance(d[key], dict):
232      value = sorted_dict_str(d[key])
233    else:
234      value = repr(d[key])
235    items.append(repr(key) + ": " + value)
236  return "{" + ", ".join(items) + "}"
237
238
239
240##############################################
241# System command unit testing utiltities
242##############################################
243
244
245class InterceptedCmndStruct:
246
247  def __init__(self, cmndRegex, cmndReturn, cmndOutput):
248    self.cmndRegex = cmndRegex
249    self.cmndReturn = cmndReturn
250    self.cmndOutput = cmndOutput
251
252  def __str__(self):
253    return "{cmndRegex='"+self.cmndRegex+"'," \
254     " cmndReturn="+str(self.cmndReturn)+"," \
255     " cmndOutput='"+str(self.cmndOutput)+"'}"
256
257#
258# Class that is used to record a set of commands that will be used to
259# intercept commands
260#
261
262class SysCmndInterceptor:
263
264  def __init__(self):
265    self.__fallThroughCmndRegexList = []
266    self.__interceptedCmndStructList = []
267    self.__allowExtraCmnds = True
268
269  def setFallThroughCmndRegex(self, cmndRegex):
270    self.__fallThroughCmndRegexList.append(cmndRegex)
271
272  def setInterceptedCmnd(self, cmndRegex, cmndReturn, cmndOutput=None):
273    self.__interceptedCmndStructList.append(
274       InterceptedCmndStruct(cmndRegex, cmndReturn, cmndOutput) )
275
276  def setAllowExtraCmnds(self, allowExtraCmnds):
277    self.__allowExtraCmnds = allowExtraCmnds
278
279  def hasInterceptedCmnds(self):
280     return len(self.__interceptedCmndStructList) > 0
281
282  def getFallThroughCmndRegexList(self):
283    return self.__fallThroughCmndRegexList[:]
284
285  def getInterceptedCmndStructList(self):
286    return self.__interceptedCmndStructList[:]
287
288  def doProcessInterceptedCmnd(self, cmnd):
289    #print("doProcessInterceptedCmnd(): cmnd='" + cmnd + "'")
290    if self.isFallThroughCmnd(cmnd):
291      return False
292    if len(self.__interceptedCmndStructList) > 0:
293      return True
294    if not self.__allowExtraCmnds:
295      return True
296    return False
297
298  def isFallThroughCmnd(self, cmnd):
299     for interceptCmndStruct in self.__interceptedCmndStructList:
300       if re.match(interceptCmndStruct.cmndRegex, cmnd):
301         return False
302     for cmndRegex in self.__fallThroughCmndRegexList:
303       if re.match(cmndRegex, cmnd):
304         return True
305     return False
306
307  def nextInterceptedCmndStruct(self, cmnd):
308    assert(not self.isFallThroughCmnd(cmnd))
309    if len(self.__interceptedCmndStructList) == 0:
310      raise Exception("Error, cmnd='"+cmnd+"' is past the last expected command!")
311    ics = self.__interceptedCmndStructList[0]
312    if not re.match(ics.cmndRegex, cmnd):
313      raise Exception("Error, cmnd='" + cmnd + "' did not match the" \
314                      " expected regex='" + ics.cmndRegex + "'!")
315    self.__interceptedCmndStructList.pop(0)
316    return (ics.cmndReturn, ics.cmndOutput)
317
318  def clear(self):
319    self.__fallThroughCmndRegexList = []
320    self.__interceptedCmndStructList = []
321    self.__allowExtraCmnds = True
322
323  def readCommandsFromStr(self, cmndsStr):
324    lines = cmndsStr.splitlines()
325    for line in lines:
326      #print("line: '" + line + "'")
327      if line == "":
328        continue
329      splitArray = line.split(':')
330      (tag, entry) = (splitArray[0], ':'.join(splitArray[1:]))
331      #(tag, entry) = line.split(':')
332      #print("(tag, entry) = " + str((tag, entry)))
333      if tag == "FT":
334        self.__fallThroughCmndRegexList.append(entry.strip())
335      elif tag == "IT":
336        entryArray = entry.split(';')
337        if len(entryArray) < 3:
338          raise Exception("Error, invalid line {"+line+"}")
339        cmndRegex = entryArray[0]
340        cmndReturn = entryArray[1]
341        cmndOutput = ""
342        for cmndOutputEntry in entryArray[2:]:
343          #print("cmndOutputEntry = {" + cmndOutputEntry + "}")
344          cmndOutput += cmndOutputEntry.strip()[1:-1]+"\n"
345        #print("(cmndRegex, cmndReturn, cmndOutput) = " +
346        #      str((cmndRegex, cmndReturn, cmndOutput)))
347        self.__interceptedCmndStructList.append(
348          InterceptedCmndStruct(cmndRegex.strip(), int(cmndReturn), cmndOutput)
349          )
350      else:
351        raise Exception("Error, invalid tag = '"+tag+"'!")
352
353  def assertAllCommandsRun(self):
354    if len(self.__interceptedCmndStructList) > 0:
355      raise Exception("Error, all of the commands have not been run starting with" \
356                      " the command " + str(self.__interceptedCmndStructList[0])
357                      + "!")
358
359
360g_sysCmndInterceptor = SysCmndInterceptor()
361
362
363# Read the command interepts from a file?
364cmndInterceptsFile = os.environ.get(
365  "GENERAL_SCRIPT_SUPPORT_CMND_INTERCEPTS_FILE","")
366if cmndInterceptsFile:
367  cmndInterceptsFileStr = open(cmndInterceptsFile, 'r').read()
368  print("\nReading system command intercepts from file '" +
369        cmndInterceptsFile                                +
370        "' with contents:\n"                              +
371        "-----------------------------------\n"           +
372        cmndInterceptsFileStr                             +
373        "-----------------------------------\n")
374  g_sysCmndInterceptor.readCommandsFromStr(cmndInterceptsFileStr)
375  g_sysCmndInterceptor.setAllowExtraCmnds(False)
376
377
378# Dump all commands being performed?
379g_dumpAllSysCmnds = "GENERAL_SCRIPT_SUPPORT_DUMD_COMMANDS" in os.environ
380
381
382def runSysCmndInterface(cmnd, outFile=None, rtnOutput=False, extraEnv=None, \
383  workingDir="", getStdErr=False \
384  ):
385  if g_dumpAllSysCmnds:
386    print("\nDUMP SYS CMND: " + cmnd + "\n")
387  if outFile!=None and rtnOutput==True:
388    raise Exception("Error, both outFile and rtnOutput can not be true!")
389  if g_sysCmndInterceptor.doProcessInterceptedCmnd(cmnd):
390    (cmndReturn, cmndOutput) = g_sysCmndInterceptor.nextInterceptedCmndStruct(cmnd)
391    if rtnOutput:
392      if cmndOutput==None:
393        raise Exception("Error, the command '"+cmnd+"' gave None output when" \
394                        " non-null output was expected!")
395      return (cmndOutput, cmndReturn)
396    if outFile:
397      writeStrToFile(outFile, cmndOutput)
398    return cmndReturn
399  # Else, fall through
400  if extraEnv:
401    fullEnv = os.environ.copy()
402    fullEnv.update(extraEnv)
403  else:
404    fullEnv = None
405  pwd = None
406  if workingDir:
407    pwd = os.getcwd()
408    os.chdir(workingDir)
409  rtnObject = None
410  try:
411    if rtnOutput:
412      if getStdErr:
413        child = subprocess.Popen(cmnd, shell=True, stdout=subprocess.PIPE,
414          stderr = subprocess.STDOUT, env=fullEnv)
415      else:
416        child = subprocess.Popen(cmnd, shell=True, stdout=subprocess.PIPE,
417          env=fullEnv)
418      data = child.stdout.read()
419      #print("data = '" + str(data) + "'")
420      child.wait()
421      rtnCode = child.returncode
422      #print("rtnCode = '" + str(rtnCode) + "'")
423      rtnObject = (data, rtnCode)
424    else:
425      outFileHandle = None
426      if outFile:
427        outFileHandle = open(outFile, 'w')
428      rtnCode = subprocess.call(cmnd, shell=True, stderr=subprocess.STDOUT,
429        stdout=outFileHandle, env=fullEnv)
430      rtnObject = rtnCode
431  finally:
432    if pwd: os.chdir(pwd)
433  return rtnObject
434
435
436######################################
437# System interaction utilties
438######################################
439
440
441def runSysCmnd(cmnd, throwExcept=True, outFile=None, workingDir="",
442  extraEnv=None, echoCmndForDebugging=False \
443  ):
444  """Run system command and optionally throw on failure"""
445  sys.stdout.flush()
446  sys.stderr.flush()
447  try:
448    outFileHandle = None
449    if echoCmndForDebugging: print("Running: "+cmnd)
450    rtnCode = runSysCmndInterface(cmnd, outFile=outFile, extraEnv=extraEnv,
451      workingDir=workingDir)
452  except OSError as e:
453    rtnCode = 1 # Just some error code != 0 please!
454  if rtnCode != 0 and throwExcept:
455    raise RuntimeError("Error, the command '%s' failed with error code %d"
456                       % (cmnd, rtnCode))
457  return rtnCode
458
459
460def echoRunSysCmnd(cmnd, throwExcept=True, outFile=None, msg=None,
461  timeCmnd=False, verbose=True, workingDir="", returnTimeCmnd=False,
462  extraEnv=None
463  ):
464  """Echo command to be run and run command with runSysCmnd()"""
465  if verbose:
466    print("\nRunning: " + cmnd + "\n")
467    if workingDir:
468      print("  Running in working directory: " + workingDir + " ...\n")
469    if extraEnv:
470      print("  Appending environment:" + sorted_dict_str(extraEnv) + "\n")
471    if outFile:
472      print("  Writing console output to file " + outFile + " ...")
473  if msg and verbose:
474    print("  " + msg + "\n")
475  t1 = time.time()
476  totalTimeMin = -1.0
477  try:
478    rtn = runSysCmnd(cmnd, throwExcept, outFile, workingDir, extraEnv)
479  finally:
480    if timeCmnd:
481      t2 = time.time()
482      totalTimeMin = (t2-t1)/60.0
483      if verbose:
484        print("\n  Runtime for command = %f minutes" % totalTimeMin)
485  if returnTimeCmnd:
486    return (rtn, totalTimeMin)
487  return rtn
488
489
490def getCmndOutput(cmnd, stripTrailingSpaces=False, throwOnError=True, workingDir="", \
491  getStdErr=False, rtnCode=False \
492  ):
493  """Run a shell command and return its output"""
494  (data, errCode) = runSysCmndInterface(cmnd, rtnOutput=True, workingDir=workingDir,
495    getStdErr=getStdErr)
496  if errCode != 0:
497    if throwOnError:
498      raise RuntimeError('%s failed w/ exit code %d:\n\n%s' % (cmnd, errCode, data))
499  dataToReturn = data
500  if stripTrailingSpaces:
501    dataToReturn = data.rstrip()
502  if rtnCode:
503    return (dataToReturn, errCode)
504  return dataToReturn
505
506
507def pidStillRunning(pid):
508  #print("\npid = '" + pid + "'")
509  cmnd = "kill -s 0 "+pid
510  cmndReturn = runSysCmnd(cmnd, False)
511  #print("\ncmndReturn = " + cmndReturn)
512  return cmndReturn == 0
513
514
515######################################
516# File/path utilities
517######################################
518
519
520def getFilePathArray(filePathStr):
521  return filePathStr.split('/')
522
523
524def joinDirs(dirArray):
525  """
526  Join directories.
527
528  2009/06/09: rabartl: We should be able to just use os.path.join(...) but I
529  found when used in at least on context that it resulted in not joining the
530  elements but instead just returning the array.
531  """
532  dirPath = ""
533  for dir in dirArray:
534    if not dirPath:
535      dirPath = dir
536    else:
537      dirPath = dirPath + "/" + dir
538  return dirPath
539
540
541def downDirsArray(numDownDirs):
542  downDirsPathArray = []
543  for i in range(0, numDownDirs):
544    downDirsPathArray.append("..")
545  #print("\ndownDirsPathArray = " + downDirsPathArray)
546  return downDirsPathArray
547
548
549def normalizePath(path):
550  return os.path.normpath(path)
551
552
553def echoChDir(dirName, verbose=True):
554  if verbose:
555    print("\nChanging current directory to \'" + dirName + "\'")
556  if not os.path.isdir(dirName):
557    raise OSError("Error, the directory \'"+dirName+"\' does not exist in the" \
558      + " base directory \'"+os.getcwd()+"\"!" )
559  os.chdir(dirName)
560  if verbose:
561    print("\nCurrent directory is \'" + os.getcwd() + "\'\n")
562
563
564def createDir(dirName, cdIntoDir=False, verbose=False):
565  """Create a directory if it does not exist"""
566  if os.path.exists(dirName):
567    if not os.path.isdir(dirName):
568      errMsg = "\nError the path '" + dirName + \
569               "'already exists but it is not a directory!"
570      if verbose: print(errMsg)
571      raise RuntimeError(errMsg)
572    if verbose: print("\nThe directory " + dirName + "already exists!")
573  else:
574    if verbose: print("\nCreating directory " + dirName + " ...")
575    os.mkdir(dirName)
576  if cdIntoDir:
577    echoChDir(dirName, verbose=verbose)
578
579
580def createDirsFromPath(path):
581  #print("\npath = " + path)
582  pathList = path.split("/")
583  #print("\npathList = " + pathList)
584  if not pathList[0]:
585    currDir = "/"
586  for dir in pathList:
587    currDir = os.path.join(currDir, dir)
588    if currDir and not os.path.exists(currDir):
589      #print("\ncurrDir = " + currDir)
590      createDir(currDir)
591
592
593def expandDirsDict(trilinosDirsDict_inout):
594
595  for dir in list(trilinosDirsDict_inout):
596    subdirsList = dir.split("/")
597    #print("\nsubdirsList = " + subdirsList)
598    for i in range(len(subdirsList)):
599      trilinosDirsDict_inout.update({joinDirs(subdirsList[:i+1]) : 0})
600
601
602def removeIfExists(fileName):
603  if os.path.exists(fileName):
604    echoRunSysCmnd("rm "+fileName)
605
606
607def removeDirIfExists(dirName, verbose=False):
608  if os.path.exists(dirName):
609    if verbose:
610      print("Removing existing directory '" + dirName + "' ...")
611    echoRunSysCmnd("rm -rf "+dirName)
612
613
614def writeStrToFile(fileName, fileBodyStr):
615  open(fileName, 'w').write(fileBodyStr)
616
617
618def readStrFromFile(fileName):
619  return open(fileName, 'r').read()
620
621
622def getFileNamesWithFileTag( baseDir, fileTag ):
623  """Get a list of file names with a given date stamp"""
624  return getCmndOutput(
625    'cd %s && ls *%s*' % (baseDir, fileTag),
626    throwOnError=False
627    ).split()
628
629
630def getFileNameFromGlob( baseDir, fileNameGlob ):
631  return getCmndOutput("cd "+baseDir+" && ls "+fileNameGlob, True, False)
632
633
634def isEmptyDir( absDir ):
635  return (len(os.listdir(absDir)) == 0)
636
637
638def getDirSizeInGb(dir):
639  sizeIn1024Kb = int(getCmndOutput("du -s "+dir).split('\t')[0])
640  #print("\nsizeIn1024Kb = " + str(sizeIn1024Kb))
641  return float(sizeIn1024Kb)/1e+6 # Size in Gb!
642
643
644def isPathChar(char):
645  return (char.isalnum() or char == '/') and (not char == ' ')
646
647
648# 2008/07/08: rabartl: This silly function is needed because on the sun
649# machine (i.e. sass8000), the 'which' command returns some non-printable
650# chars from the beginning of the output string that don't form a valid path.
651# This was *very* hard to debug but this function is able to select the valid
652# path string.  This has been tested at least on linux and the sun and should
653# work anywhere.
654def cleanBadPath(inputPath):
655  cleanPath = ""
656  for i in range(len(inputPath)-1, -1, -1):
657    char = inputPath[i]
658    if not isPathChar(char):
659      break
660    cleanPath = char + cleanPath
661  return cleanPath
662
663
664def getRelativePathFrom1to2(absPath1, absPath2):
665  #print("\nabsPath1 =" + absPath1)
666  #print("\nabsPath2 =" + absPath2)
667  absPath1_array = absPath1.split('/')
668  absPath2_array = absPath2.split('/')
669  #print("\nabsPath1_array =" + absPath1_array)
670  #print("\nabsPath2_array =" + absPath2_array)
671  absPath1_array_len = len(absPath1_array)
672  absPath2_array_len = len(absPath2_array)
673  maxBaseDirDepth = min(absPath1_array_len, absPath2_array_len)
674  baseDirDepth = 0
675  for dirIdx in range(0, maxBaseDirDepth):
676    dir1 = absPath1_array[dirIdx]
677    dir2 = absPath2_array[dirIdx]
678    if dir1 != dir2:
679      break
680    baseDirDepth = baseDirDepth + 1
681  #print("\nbaseDirDepth = %d" % baseDirDepth)
682  numDownDirs = absPath1_array_len - baseDirDepth
683  #print("\nnumDownDirs = %d" % numDownDirs)
684  if numDownDirs > 0:
685    downDirPath = joinDirs(downDirsArray(numDownDirs))
686  else:
687    downDirPath = "."
688  #print("\ndownDirPath = '" + downDirPath + "'")
689  if baseDirDepth == absPath2_array_len:
690    upDirPath = "."
691  else:
692    upDirPath = joinDirs(absPath2_array[baseDirDepth:])
693  #print("\nupDirPath = "  + upDirPath      )
694  #print("\nupDirPath = '" + upDirPath + "'")
695  relPath = os.path.join(downDirPath, upDirPath)
696  if relPath == "./.":
697    return "."
698  return relPath
699
700
701def getExecBaseDir(execName):
702  whichOutput = getCmndOutput("type -p "+execName, True, False)
703  # Handle the outpue 'execName is execFullPath' output
704  execFullPath = whichOutput.split(' ')[-1]
705  #print("\nexecFullPath = " + execFullPath)
706  execNameMatchRe = r"^(.+)/"+execName
707  execNameGroups = re.findall(execNameMatchRe, execFullPath)
708  #print("\nexecNameGroups = " + execNameGroups)
709  if not execNameGroups:
710    return None
711  execBaseDir = execNameGroups[0]
712  #print("\nexecBaseDir = \"" + execBaseDir + "\"")
713  #execBaseDir = cleanBadPath(execBaseDir)
714  #print("\nexecBaseDir = \"" + execBaseDir + "\"")
715  return execBaseDir
716
717
718def extractAppendUniqueDirsDictFromFileNames(filenamesArray, dirsDict):
719  for filename in filenamesArray:
720    dirsDict.update( { normalizePath(os.path.dirname(filename)) : 0 } )
721
722
723def copyFileAndReplaceTokens( scriptsDir, inputFile, tokenReplacementList,
724  outputFile ):
725
726  """Copies an input stub file and then does a set of token replacements"""
727
728  echoRunSysCmnd("cp "+inputFile+" "+outputFile, verbose=verboseDebug)
729
730  for tokenReplacementPair in tokenReplacementList:
731    oldToken = tokenReplacementPair[0]
732    newToken = tokenReplacementPair[1]
733    echoRunSysCmnd( scriptsDir+"/token-replace.pl "+oldToken+" "+newToken\
734      +" "+outputFile+" "+outputFile, verbose=verboseDebug );
735    # ToDo: Replace above with native re commands
736
737
738class TeeOutput(object):
739  """
740  Object that directs all calls to its write method to stdout as well
741  as a file. This is to be used as a simple replacement for the Unix
742  tee command.
743  """
744  def __init__(self, outputfile):
745    """ Constructor takes a file-like object to write output to."""
746    self._realstdout = sys.stdout
747    self._outputfile = outputfile
748
749  def _safe_outputfile_method(self, methodname, *args):
750    """
751    Calls the method specified by methodname with the given args on
752    the internal file object if it is non-null.
753    """
754    if self._outputfile is not None:
755      if hasattr(self._outputfile, methodname):
756        method = getattr(self._outputfile, methodname)
757        if method and callable(method):
758          method(*args)
759
760  def write(self, data):
761    """
762    Write the given data to stdout and to the log file.
763    """
764    self._realstdout.write(data)
765    self._safe_outputfile_method('write', data)
766
767  def flush(self):
768    """
769    Flush the internal file buffers.
770    """
771    self._realstdout.flush()
772    self._safe_outputfile_method('flush')
773
774
775######################################
776# Shell argument helpers
777######################################
778
779
780reCmndLineArg = re.compile(r"(--.+=)(.+)")
781
782
783def requoteCmndLineArgs(inArgs):
784  argsStr = ""
785  for arg in inArgs:
786    splitArg = arg.split("=")
787    newArg = None
788    if len(splitArg) == 1:
789      newArg = arg
790    else:
791      newArg = splitArg[0]+"=\""+'='.join(splitArg[1:])+"\""
792    #print("\nnewArg = " + newArg)
793    argsStr = argsStr+" "+newArg
794  return argsStr
795
796
797def commandLineOptionsToList(stringOptions):
798  """
799  Convert a string of space separated command line options to a python
800  list of the individual optionstrings.
801  TODO: Handle shell quoting.
802  """
803  return stringOptions.split()
804
805
806class ConfigurableOptionParser(optparse.OptionParser):
807  """
808  OptionParser that accepts a python dictionary as a configuration
809  file to provide default value overrides for the options.
810  """
811  def __init__(self, configuration, **kwargs):
812    """
813    Constructor accepts a configuration dictionary with default values
814    for arguments and all of the OptionParser arguments as well.
815    """
816    optparse.OptionParser.__init__(self, **kwargs)
817    self._configuration = configuration
818
819  def add_option(self, *args, **kwargs):
820    """
821    Checks for a default override in the configuration dictionary and
822    modifies the default and help arguments before dispatching them to
823    the base class implementation.
824    """
825    if 'default' in kwargs:
826      for arg in args:
827        kwargs['default'] = self._configuration.get(arg, kwargs['default'])
828    optparse.OptionParser.add_option(self, *args, **kwargs)
829
830
831######################################
832# Debugging support
833######################################
834
835
836def printStackTrace():
837  sys.stdout.flush()
838  traceback.print_exc()
839
840
841class ErrorCaptureOptionParser(optparse.OptionParser):
842  __sawError = None
843  def __init__(self, usage="%prog [options]", version=None):
844    optparse.OptionParser.__init__(self, usage, version)
845    __sawError = False
846  def error(self, msg):
847    raise Exception("Received error message: " + msg)
848
849
850######################################
851# HTML directory browsing
852######################################
853
854
855def createIndexHtmlBrowserList(baseDir, fileDirList = None):
856  htmlList = ""
857  # Get the fileDirList if empty
858  if not fileDirList:
859    fileDirList = os.listdir(baseDir)
860    fileDirList.sort()
861  # Create the HTML header
862  htmlList += "" \
863    + "<ul>\n" \
864    + "<li><a href=\"..\">..</a></li>\n"
865  # Fill in links to directories first
866  for fd in fileDirList:
867    absFd = baseDir+"/"+fd
868    #print("isfile(" + fd + ") = " + str(os.path.isfile(absFd)))
869    #print("isdir("  + fd + ") = " + str(os.path.isdir(absFd) ))
870    if not os.path.isfile(absFd):
871      htmlList = htmlList \
872                 +"<li>dir: <a href=\""+fd+"\">"+fd+"</a></li>\n"
873  # Fill in links to regular files second
874  for fd in fileDirList:
875    absFd = baseDir+"/"+fd
876    if os.path.isfile(absFd):
877      if fd != 'index.html':
878        htmlList = htmlList \
879                 +"<li>file: <a href=\""+fd+"\">"+fd+"</a></li>\n"
880  # Write the footer
881  htmlList = htmlList \
882    + "</ul>\n"
883  return htmlList
884
885
886
887def createIndexHtmlBrowserFile(baseDir, fileDirList):
888  """Creates an HTML browser file as a returned string."""
889  htmlFile = "" \
890    + "<html>\n" \
891    + "<head>\n" \
892    + "<title>"+baseDir+"</title>\n" \
893    + "</head>\n" \
894    + "<body>\n" \
895    + "<b>"+baseDir+"</b>\n" \
896    + createIndexHtmlBrowserList(baseDir, fileDirList) \
897    + "</body>\n" \
898    + "</html>\n"
899  return htmlFile
900
901
902def createHtmlBrowserFiles(absBaseDir, depth, verbose=False):
903
904  """Create a hierarchy of index.html files that will build a directory/file
905  browser for a web server that will not allow directory/file browsing."""
906
907  #print("\nEntering createHtmlBrowserFiles(" + absBaseDir + ",%d" % (depth) + ")")
908
909  # Get the list of all of the files/directories in absBaseDir
910  fileDirList = os.listdir(absBaseDir)
911  fileDirList.sort()
912  #print("\nfileDirList = " + str(fileDirList)
913  #sys.stdout.flush()
914
915  # Get the index.html file HTML
916  indexHtml = createIndexHtmlBrowserFile(absBaseDir, fileDirList)
917  #print("\nindexHtml:\n" + indexHtml)
918
919  # Write the index.html file
920  indexFileName = absBaseDir+"/index.html"
921  if verbose:
922    print("\nWriting " + indexFileName)
923  open(indexFileName,'w').write(indexHtml)
924
925  # Loop through all of the directories and recursively call this function
926  if depth > 0:
927    for fd in fileDirList:
928      absFd = absBaseDir+"/"+fd
929      if os.path.isdir(absFd):
930        subDir = absFd
931        #print("\nCalling createHtmlBrowserFiles(" + subDir + ",%d" % (depth-1)
932        #      + ")")
933        createHtmlBrowserFiles(absBaseDir+"/"+fd,depth-1)
934
935  #print("\nLeaving createHtmlBrowserFiles(" + absBaseDir + ",%d" % (depth) +
936  #      ")")
937