1#!/usr/bin/python 2# encoding: UTF-8 3 4#=============================================================================== 5# Define global imports 6#=============================================================================== 7import os 8import re 9import sys 10import codecs 11import shutil 12import filecmp 13import subprocess as sp 14from . import constants 15from .GLError import GLError 16from .GLConfig import GLConfig 17 18 19#=============================================================================== 20# Define module information 21#=============================================================================== 22__author__ = constants.__author__ 23__license__ = constants.__license__ 24__copyright__ = constants.__copyright__ 25 26 27#=============================================================================== 28# Define global constants 29#=============================================================================== 30PYTHON3 = constants.PYTHON3 31NoneType = type(None) 32APP = constants.APP 33DIRS = constants.DIRS 34ENCS = constants.ENCS 35UTILS = constants.UTILS 36MODES = constants.MODES 37TESTS = constants.TESTS 38compiler = constants.compiler 39joinpath = constants.joinpath 40cleaner = constants.cleaner 41string = constants.string 42isabs = os.path.isabs 43isdir = os.path.isdir 44isfile = os.path.isfile 45normpath = os.path.normpath 46relpath = os.path.relpath 47 48 49#=============================================================================== 50# Define GLFileSystem class 51#=============================================================================== 52class GLFileSystem(object): 53 '''GLFileSystem class is used to create virtual filesystem, which is based on 54 the gnulib directory and directory specified by localdir argument. Its main 55 method lookup(file) is used to find file in these directories or combine it 56 using Linux 'patch' utility.''' 57 58 def __init__(self, config): 59 '''Create new GLFileSystem instance. The only argument is localdir, 60 which can be an empty string too.''' 61 if type(config) is not GLConfig: 62 raise(TypeError('config must be a GLConfig, not %s' % 63 type(config).__name__)) 64 self.config = config 65 66 def __repr__(self): 67 '''x.__repr__ <==> repr(x)''' 68 result = '<pygnulib.GLFileSystem %s>' % hex(id(self)) 69 return(result) 70 71 def lookup(self, name): 72 '''GLFileSystem.lookup(name) -> tuple 73 74 Lookup a file in gnulib and localdir directories or combine it using Linux 75 'patch' utility. If file was found, method returns string, else it raises 76 GLError telling that file was not found. Function also returns flag which 77 indicates whether file is a temporary file. 78 GLConfig: localdir.''' 79 if type(name) is bytes or type(name) is string: 80 if type(name) is bytes: 81 name = name.decode(ENCS['default']) 82 else: # if name has not bytes or string type 83 raise(TypeError( 84 'name must be a string, not %s' % type(module).__name__)) 85 # If name exists in localdir, then we use it 86 path_gnulib = joinpath(DIRS['root'], name) 87 path_local = joinpath(self.config['localdir'], name) 88 path_diff = joinpath(self.config['localdir'], '%s.diff' % name) 89 path_temp = joinpath(self.config['tempdir'], name) 90 try: # Try to create directories 91 os.makedirs(os.path.dirname(path_temp)) 92 except OSError as error: 93 pass # Skip errors if directory exists 94 if isfile(path_temp): 95 os.remove(path_temp) 96 if self.config['localdir'] and isfile(path_local): 97 result = (path_local, False) 98 else: # if path_local does not exist 99 if isfile(path_gnulib): 100 if self.config['localdir'] and isfile(path_diff): 101 shutil.copy(path_gnulib, path_temp) 102 command = 'patch -s "%s" < "%s" >&2' % (path_temp, path_diff) 103 try: # Try to apply patch 104 sp.check_call(command, shell=True) 105 except sp.CalledProcessError as error: 106 raise(GLError(2, name)) 107 result = (path_temp, True) 108 else: # if path_diff does not exist 109 result = (path_gnulib, False) 110 else: # if path_gnulib does not exist 111 raise(GLError(1, name)) 112 return(result) 113 114 115#=============================================================================== 116# Define GLFileAssistant class 117#=============================================================================== 118class GLFileAssistant(object): 119 '''GLFileAssistant is used to help with file processing.''' 120 121 def __init__(self, config, transformers=dict()): 122 '''Create GLFileAssistant instance.''' 123 if type(config) is not GLConfig: 124 raise(TypeError('config must be a GLConfig, not %s' % 125 type(config).__name__)) 126 if type(transformers) is not dict: 127 raise(TypeError('transformers must be a dict, not %s' % 128 type(transformers).__name__)) 129 for key in ['lib', 'aux', 'main', 'tests']: 130 if key not in transformers: 131 transformers[key] = 's,x,x,' 132 else: # if key in transformers 133 value = transformers[key] 134 if type(value) is bytes or type(value) is string: 135 if type(value) is bytes: 136 transformers[key] = value.decode(ENCS['default']) 137 else: # if value has not bytes or string type 138 raise(TypeError('transformers[%s] must be a string, not %s' % 139 (key, type(value).__name__))) 140 self.original = None 141 self.rewritten = None 142 self.added = list() 143 self.makefile = list() 144 self.config = config 145 self.transformers = transformers 146 self.filesystem = GLFileSystem(self.config) 147 148 def __repr__(self): 149 '''x.__repr__() <==> repr(x)''' 150 result = '<pygnulib.GLFileAssistant %s>' % hex(id(self)) 151 return(result) 152 153 def tmpfilename(self, path): 154 '''GLFileAssistant.tmpfilename() -> string 155 156 Return the name of a temporary file (file is relative to destdir).''' 157 if type(path) is bytes or type(path) is string: 158 if type(path) is bytes: 159 path = path.decode(ENCS['default']) 160 else: # if path has not bytes or string type 161 raise(TypeError( 162 'path must be a string, not %s' % (type(path).__name__))) 163 if not self.config['dryrun']: 164 # Put the new contents of $file in a file in the same directory (needed 165 # to guarantee that an 'mv' to "$destdir/$file" works). 166 result = joinpath(self.config['destdir'], '%s.tmp' % path) 167 dirname = os.path.dirname(result) 168 if dirname and not isdir(dirname): 169 os.makedirs(dirname) 170 else: # if self.config['dryrun'] 171 # Put the new contents of $file in a file in a temporary directory 172 # (because the directory of "$file" might not exist). 173 tempdir = self.config['tempdir'] 174 result = joinpath(tempdir, '%s.tmp' % os.path.basename(path)) 175 dirname = os.path.dirname(result) 176 if not isdir(dirname): 177 os.makedirs(dirname) 178 if type(result) is bytes: 179 result = bytes.decode(ENCS['default']) 180 return(result) 181 182 def setOriginal(self, original): 183 '''GLFileAssistant.setOriginal(original) 184 185 Set the name of the original file which will be used.''' 186 if type(original) is bytes or type(original) is string: 187 if type(original) is bytes: 188 original = original.decode(ENCS['default']) 189 else: # if original has not bytes or string type 190 raise(TypeError( 191 'original must be a string, not %s' % (type(original).__name__))) 192 self.original = original 193 194 def setRewritten(self, rewritten): 195 '''GLFileAssistant.setRewritten(rewritten) 196 197 Set the name of the rewritten file which will be used.''' 198 if type(rewritten) is bytes or type(rewritten) is string: 199 if type(rewritten) is bytes: 200 rewritten = rewritten.decode(ENCS['default']) 201 else: # if rewritten has not bytes or string type 202 raise(TypeError( 203 'rewritten must be a string, not %s' % type(rewritten).__name__)) 204 self.rewritten = rewritten 205 206 def addFile(self, file): 207 '''GLFileAssistant.addFile(file) 208 209 Add file to the list of added files.''' 210 if file not in self.added: 211 self.added += [file] 212 213 def removeFile(self, file): 214 '''GLFileAssistant.removeFile(file) 215 216 Remove file from the list of added files.''' 217 if file in self.added: 218 self.added.pop(file) 219 220 def getFiles(self): 221 '''Return list of the added files.''' 222 return(list(self.added)) 223 224 def add(self, lookedup, tmpflag, tmpfile): 225 '''GLFileAssistant.add(lookedup, tmpflag, tmpfile) 226 227 This method copies a file from gnulib into the destination directory. 228 The destination is known to exist. If tmpflag is True, then lookedup file 229 is a temporary one.''' 230 original = self.original 231 rewritten = self.rewritten 232 destdir = self.config['destdir'] 233 symbolic = self.config['symbolic'] 234 lsymbolic = self.config['lsymbolic'] 235 if original == None: 236 raise(TypeError('original must be set before applying the method')) 237 elif rewritten == None: 238 raise(TypeError('rewritten must be set before applying the method')) 239 if not self.config['dryrun']: 240 print('Copying file %s' % rewritten) 241 loriginal = joinpath(self.config['localdir'], original) 242 if (symbolic or (lsymbolic and lookedup == loriginal)) \ 243 and not tmpflag and filecmp.cmp(lookedup, tmpfile): 244 constants.link_if_changed( 245 lookedup, joinpath(destdir, rewritten)) 246 else: # if any of these conditions is not met 247 try: # Try to move file 248 shutil.move(tmpfile, joinpath(destdir, rewritten)) 249 except Exception as error: 250 raise(GLError(17, original)) 251 else: # if self.config['dryrun'] 252 print('Copy file %s' % rewritten) 253 254 def update(self, lookedup, tmpflag, tmpfile, already_present): 255 '''GLFileAssistant.update(lookedup, tmpflag, tmpfile, already_present) 256 257 This method copies a file from gnulib into the destination directory. 258 The destination is known to exist. If tmpflag is True, then lookedup file 259 is a temporary one.''' 260 original = self.original 261 rewritten = self.rewritten 262 destdir = self.config['destdir'] 263 symbolic = self.config['symbolic'] 264 lsymbolic = self.config['lsymbolic'] 265 if original == None: 266 raise(TypeError('original must be set before applying the method')) 267 elif rewritten == None: 268 raise(TypeError('rewritten must be set before applying the method')) 269 if type(lookedup) is bytes or type(lookedup) is string: 270 if type(lookedup) is bytes: 271 lookedup = lookedup.decode(ENCS['default']) 272 else: # if lookedup has not bytes or string type 273 raise(TypeError('lookedup must be a string, not %s' % 274 type(lookedup).__name__)) 275 if type(already_present) is not bool: 276 raise(TypeError('already_present must be a bool, not %s' % 277 type(already_present).__name__)) 278 basename = rewritten 279 backupname = string('%s~' % basename) 280 basepath = joinpath(destdir, basename) 281 backuppath = joinpath(destdir, backupname) 282 if not filecmp.cmp(basepath, tmpfile): 283 if not self.config['dryrun']: 284 if already_present: 285 print('Updating file %s (backup in %s)' % 286 (basename, backupname)) 287 else: # if not already_present 288 message = 'Replacing file ' 289 message += '%s (non-gnulib code backed up in ' % basename 290 message += '%s) !!' % backupname 291 print(message) 292 if isfile(backuppath): 293 os.remove(backuppath) 294 try: # Try to replace the given file 295 shutil.move(basepath, backuppath) 296 except Exception as error: 297 raise(GLError(17, original)) 298 loriginal = joinpath(self.config['localdir'], original) 299 if (symbolic or (lsymbolic and lookedup == loriginal)) \ 300 and not tmpflag and filecmp.cmp(lookedup, tmpfile): 301 constants.link_if_changed(lookedup, basepath) 302 else: # if any of these conditions is not met 303 try: # Try to move file 304 if os.path.exists(basepath): 305 os.remove(basepath) 306 shutil.copy(tmpfile, rewritten) 307 except Exception as error: 308 raise(GLError(17, original)) 309 else: # if self.config['dryrun'] 310 if already_present: 311 print('Update file %s (backup in %s)' % 312 (rewritten, backup)) 313 else: # if not already_present 314 print('Replace file %s (backup in %s)' % 315 (rewritten, backup)) 316 317 def add_or_update(self, already_present): 318 '''GLFileAssistant.add_or_update(already_present) 319 320 This method handles a file that ought to be present afterwards.''' 321 original = self.original 322 rewritten = self.rewritten 323 if original == None: 324 raise(TypeError('original must be set before applying the method')) 325 elif rewritten == None: 326 raise(TypeError('rewritten must be set before applying the method')) 327 if type(already_present) is not bool: 328 raise(TypeError('already_present must be a bool, not %s' % 329 type(already_present).__name__)) 330 xoriginal = original 331 if original.startswith('tests=lib/'): 332 xoriginal = constants.substart('tests=lib/', 'lib/', original) 333 lookedup, tmpflag = self.filesystem.lookup(xoriginal) 334 tmpfile = self.tmpfilename(rewritten) 335 sed_transform_lib_file = self.transformers.get('lib', '') 336 sed_transform_build_aux_file = self.transformers.get('aux', '') 337 sed_transform_main_lib_file = self.transformers.get('main', '') 338 sed_transform_testsrelated_lib_file = self.transformers.get( 339 'tests', '') 340 try: # Try to copy lookedup file to tmpfile 341 shutil.copy(lookedup, tmpfile) 342 except Exception as error: 343 raise(GLError(15, lookedup)) 344 # Don't process binary files with sed. 345 if not (original.endswith(".class") or original.endswith(".mo")): 346 transformer = string() 347 if original.startswith('lib/'): 348 if sed_transform_main_lib_file: 349 transformer = sed_transform_main_lib_file 350 elif original.startswith('build-aux/'): 351 if sed_transform_build_aux_file: 352 transformer = sed_transform_build_aux_file 353 elif original.startswith('tests=lib/'): 354 if sed_transform_testsrelated_lib_file: 355 transformer = sed_transform_testsrelated_lib_file 356 if transformer: 357 args = ['sed', '-e', transformer] 358 stdin = codecs.open(lookedup, 'rb', 'UTF-8') 359 try: # Try to transform file 360 data = sp.check_output(args, stdin=stdin, shell=False) 361 data = data.decode("UTF-8") 362 except Exception as error: 363 raise(GLError(16, lookedup)) 364 with codecs.open(tmpfile, 'wb', 'UTF-8') as file: 365 file.write(data) 366 path = joinpath(self.config['destdir'], rewritten) 367 if isfile(path): 368 self.update(lookedup, tmpflag, tmpfile, already_present) 369 os.remove(tmpfile) 370 else: # if not isfile(path) 371 self.add(lookedup, tmpflag, tmpfile) 372 self.addFile(rewritten) 373 374 def super_update(self, basename, tmpfile): 375 '''GLFileAssistant.super_update(basename, tmpfile) -> tuple 376 377 Move tmpfile to destdir/basename path, making a backup of it. 378 Returns tuple, which contains basename, backupname and status. 379 0: tmpfile is the same as destfile; 380 1: tmpfile was used to update destfile; 381 2: destfile was created, because it didn't exist.''' 382 backupname = '%s~' % basename 383 basepath = joinpath(self.config['destdir'], basename) 384 backuppath = joinpath(self.config['destdir'], backupname) 385 if isfile(basepath): 386 if filecmp.cmp(basepath, tmpfile): 387 result_flag = 0 388 else: # if not filecmp.cmp(basepath, tmpfile) 389 result_flag = 1 390 if not self.config['dryrun']: 391 if isfile(backuppath): 392 os.remove(backuppath) 393 shutil.move(basepath, backuppath) 394 shutil.move(tmpfile, basepath) 395 else: # if self.config['dryrun'] 396 os.remove(tmpfile) 397 else: # if not isfile(basepath) 398 result_flag = 2 399 if not self.config['dryrun']: 400 if isfile(basepath): 401 os.remove(basepath) 402 shutil.move(tmpfile, basepath) 403 else: # if self.config['dryrun'] 404 os.remove(tmpfile) 405 result = tuple([basename, backupname, result_flag]) 406 return(result) 407