1# SPDX-License-Identifier: GPL-2.0+ 2# 3# Copyright (c) 2016 Google, Inc 4# 5 6import glob 7import os 8import shutil 9import struct 10import sys 11import tempfile 12 13from patman import command 14from patman import tout 15 16# Output directly (generally this is temporary) 17outdir = None 18 19# True to keep the output directory around after exiting 20preserve_outdir = False 21 22# Path to the Chrome OS chroot, if we know it 23chroot_path = None 24 25# Search paths to use for Filename(), used to find files 26search_paths = [] 27 28tool_search_paths = [] 29 30# Tools and the packages that contain them, on debian 31packages = { 32 'lz4': 'liblz4-tool', 33 } 34 35# List of paths to use when looking for an input file 36indir = [] 37 38def PrepareOutputDir(dirname, preserve=False): 39 """Select an output directory, ensuring it exists. 40 41 This either creates a temporary directory or checks that the one supplied 42 by the user is valid. For a temporary directory, it makes a note to 43 remove it later if required. 44 45 Args: 46 dirname: a string, name of the output directory to use to store 47 intermediate and output files. If is None - create a temporary 48 directory. 49 preserve: a Boolean. If outdir above is None and preserve is False, the 50 created temporary directory will be destroyed on exit. 51 52 Raises: 53 OSError: If it cannot create the output directory. 54 """ 55 global outdir, preserve_outdir 56 57 preserve_outdir = dirname or preserve 58 if dirname: 59 outdir = dirname 60 if not os.path.isdir(outdir): 61 try: 62 os.makedirs(outdir) 63 except OSError as err: 64 raise CmdError("Cannot make output directory '%s': '%s'" % 65 (outdir, err.strerror)) 66 tout.Debug("Using output directory '%s'" % outdir) 67 else: 68 outdir = tempfile.mkdtemp(prefix='binman.') 69 tout.Debug("Using temporary directory '%s'" % outdir) 70 71def _RemoveOutputDir(): 72 global outdir 73 74 shutil.rmtree(outdir) 75 tout.Debug("Deleted temporary directory '%s'" % outdir) 76 outdir = None 77 78def FinaliseOutputDir(): 79 global outdir, preserve_outdir 80 81 """Tidy up: delete output directory if temporary and not preserved.""" 82 if outdir and not preserve_outdir: 83 _RemoveOutputDir() 84 outdir = None 85 86def GetOutputFilename(fname): 87 """Return a filename within the output directory. 88 89 Args: 90 fname: Filename to use for new file 91 92 Returns: 93 The full path of the filename, within the output directory 94 """ 95 return os.path.join(outdir, fname) 96 97def GetOutputDir(): 98 """Return the current output directory 99 100 Returns: 101 str: The output directory 102 """ 103 return outdir 104 105def _FinaliseForTest(): 106 """Remove the output directory (for use by tests)""" 107 global outdir 108 109 if outdir: 110 _RemoveOutputDir() 111 outdir = None 112 113def SetInputDirs(dirname): 114 """Add a list of input directories, where input files are kept. 115 116 Args: 117 dirname: a list of paths to input directories to use for obtaining 118 files needed by binman to place in the image. 119 """ 120 global indir 121 122 indir = dirname 123 tout.Debug("Using input directories %s" % indir) 124 125def GetInputFilename(fname, allow_missing=False): 126 """Return a filename for use as input. 127 128 Args: 129 fname: Filename to use for new file 130 allow_missing: True if the filename can be missing 131 132 Returns: 133 fname, if indir is None; 134 full path of the filename, within the input directory; 135 None, if file is missing and allow_missing is True 136 137 Raises: 138 ValueError if file is missing and allow_missing is False 139 """ 140 if not indir or fname[:1] == '/': 141 return fname 142 for dirname in indir: 143 pathname = os.path.join(dirname, fname) 144 if os.path.exists(pathname): 145 return pathname 146 147 if allow_missing: 148 return None 149 raise ValueError("Filename '%s' not found in input path (%s) (cwd='%s')" % 150 (fname, ','.join(indir), os.getcwd())) 151 152def GetInputFilenameGlob(pattern): 153 """Return a list of filenames for use as input. 154 155 Args: 156 pattern: Filename pattern to search for 157 158 Returns: 159 A list of matching files in all input directories 160 """ 161 if not indir: 162 return glob.glob(fname) 163 files = [] 164 for dirname in indir: 165 pathname = os.path.join(dirname, pattern) 166 files += glob.glob(pathname) 167 return sorted(files) 168 169def Align(pos, align): 170 if align: 171 mask = align - 1 172 pos = (pos + mask) & ~mask 173 return pos 174 175def NotPowerOfTwo(num): 176 return num and (num & (num - 1)) 177 178def SetToolPaths(toolpaths): 179 """Set the path to search for tools 180 181 Args: 182 toolpaths: List of paths to search for tools executed by Run() 183 """ 184 global tool_search_paths 185 186 tool_search_paths = toolpaths 187 188def PathHasFile(path_spec, fname): 189 """Check if a given filename is in the PATH 190 191 Args: 192 path_spec: Value of PATH variable to check 193 fname: Filename to check 194 195 Returns: 196 True if found, False if not 197 """ 198 for dir in path_spec.split(':'): 199 if os.path.exists(os.path.join(dir, fname)): 200 return True 201 return False 202 203def GetHostCompileTool(name): 204 """Get the host-specific version for a compile tool 205 206 This checks the environment variables that specify which version of 207 the tool should be used (e.g. ${HOSTCC}). 208 209 The following table lists the host-specific versions of the tools 210 this function resolves to: 211 212 Compile Tool | Host version 213 --------------+---------------- 214 as | ${HOSTAS} 215 ld | ${HOSTLD} 216 cc | ${HOSTCC} 217 cpp | ${HOSTCPP} 218 c++ | ${HOSTCXX} 219 ar | ${HOSTAR} 220 nm | ${HOSTNM} 221 ldr | ${HOSTLDR} 222 strip | ${HOSTSTRIP} 223 objcopy | ${HOSTOBJCOPY} 224 objdump | ${HOSTOBJDUMP} 225 dtc | ${HOSTDTC} 226 227 Args: 228 name: Command name to run 229 230 Returns: 231 host_name: Exact command name to run instead 232 extra_args: List of extra arguments to pass 233 """ 234 host_name = None 235 extra_args = [] 236 if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip', 237 'objcopy', 'objdump', 'dtc'): 238 host_name, *host_args = env.get('HOST' + name.upper(), '').split(' ') 239 elif name == 'c++': 240 host_name, *host_args = env.get('HOSTCXX', '').split(' ') 241 242 if host_name: 243 return host_name, extra_args 244 return name, [] 245 246def GetTargetCompileTool(name, cross_compile=None): 247 """Get the target-specific version for a compile tool 248 249 This first checks the environment variables that specify which 250 version of the tool should be used (e.g. ${CC}). If those aren't 251 specified, it checks the CROSS_COMPILE variable as a prefix for the 252 tool with some substitutions (e.g. "${CROSS_COMPILE}gcc" for cc). 253 254 The following table lists the target-specific versions of the tools 255 this function resolves to: 256 257 Compile Tool | First choice | Second choice 258 --------------+----------------+---------------------------- 259 as | ${AS} | ${CROSS_COMPILE}as 260 ld | ${LD} | ${CROSS_COMPILE}ld.bfd 261 | | or ${CROSS_COMPILE}ld 262 cc | ${CC} | ${CROSS_COMPILE}gcc 263 cpp | ${CPP} | ${CROSS_COMPILE}gcc -E 264 c++ | ${CXX} | ${CROSS_COMPILE}g++ 265 ar | ${AR} | ${CROSS_COMPILE}ar 266 nm | ${NM} | ${CROSS_COMPILE}nm 267 ldr | ${LDR} | ${CROSS_COMPILE}ldr 268 strip | ${STRIP} | ${CROSS_COMPILE}strip 269 objcopy | ${OBJCOPY} | ${CROSS_COMPILE}objcopy 270 objdump | ${OBJDUMP} | ${CROSS_COMPILE}objdump 271 dtc | ${DTC} | (no CROSS_COMPILE version) 272 273 Args: 274 name: Command name to run 275 276 Returns: 277 target_name: Exact command name to run instead 278 extra_args: List of extra arguments to pass 279 """ 280 env = dict(os.environ) 281 282 target_name = None 283 extra_args = [] 284 if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip', 285 'objcopy', 'objdump', 'dtc'): 286 target_name, *extra_args = env.get(name.upper(), '').split(' ') 287 elif name == 'c++': 288 target_name, *extra_args = env.get('CXX', '').split(' ') 289 290 if target_name: 291 return target_name, extra_args 292 293 if cross_compile is None: 294 cross_compile = env.get('CROSS_COMPILE', '') 295 if not cross_compile: 296 return name, [] 297 298 if name in ('as', 'ar', 'nm', 'ldr', 'strip', 'objcopy', 'objdump'): 299 target_name = cross_compile + name 300 elif name == 'ld': 301 try: 302 if Run(cross_compile + 'ld.bfd', '-v'): 303 target_name = cross_compile + 'ld.bfd' 304 except: 305 target_name = cross_compile + 'ld' 306 elif name == 'cc': 307 target_name = cross_compile + 'gcc' 308 elif name == 'cpp': 309 target_name = cross_compile + 'gcc' 310 extra_args = ['-E'] 311 elif name == 'c++': 312 target_name = cross_compile + 'g++' 313 else: 314 target_name = name 315 return target_name, extra_args 316 317def Run(name, *args, **kwargs): 318 """Run a tool with some arguments 319 320 This runs a 'tool', which is a program used by binman to process files and 321 perhaps produce some output. Tools can be located on the PATH or in a 322 search path. 323 324 Args: 325 name: Command name to run 326 args: Arguments to the tool 327 for_host: True to resolve the command to the version for the host 328 for_target: False to run the command as-is, without resolving it 329 to the version for the compile target 330 331 Returns: 332 CommandResult object 333 """ 334 try: 335 binary = kwargs.get('binary') 336 for_host = kwargs.get('for_host', False) 337 for_target = kwargs.get('for_target', not for_host) 338 env = None 339 if tool_search_paths: 340 env = dict(os.environ) 341 env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH'] 342 if for_target: 343 name, extra_args = GetTargetCompileTool(name) 344 args = tuple(extra_args) + args 345 elif for_host: 346 name, extra_args = GetHostCompileTool(name) 347 args = tuple(extra_args) + args 348 name = os.path.expanduser(name) # Expand paths containing ~ 349 all_args = (name,) + args 350 result = command.RunPipe([all_args], capture=True, capture_stderr=True, 351 env=env, raise_on_error=False, binary=binary) 352 if result.return_code: 353 raise Exception("Error %d running '%s': %s" % 354 (result.return_code,' '.join(all_args), 355 result.stderr)) 356 return result.stdout 357 except: 358 if env and not PathHasFile(env['PATH'], name): 359 msg = "Please install tool '%s'" % name 360 package = packages.get(name) 361 if package: 362 msg += " (e.g. from package '%s')" % package 363 raise ValueError(msg) 364 raise 365 366def Filename(fname): 367 """Resolve a file path to an absolute path. 368 369 If fname starts with ##/ and chroot is available, ##/ gets replaced with 370 the chroot path. If chroot is not available, this file name can not be 371 resolved, `None' is returned. 372 373 If fname is not prepended with the above prefix, and is not an existing 374 file, the actual file name is retrieved from the passed in string and the 375 search_paths directories (if any) are searched to for the file. If found - 376 the path to the found file is returned, `None' is returned otherwise. 377 378 Args: 379 fname: a string, the path to resolve. 380 381 Returns: 382 Absolute path to the file or None if not found. 383 """ 384 if fname.startswith('##/'): 385 if chroot_path: 386 fname = os.path.join(chroot_path, fname[3:]) 387 else: 388 return None 389 390 # Search for a pathname that exists, and return it if found 391 if fname and not os.path.exists(fname): 392 for path in search_paths: 393 pathname = os.path.join(path, os.path.basename(fname)) 394 if os.path.exists(pathname): 395 return pathname 396 397 # If not found, just return the standard, unchanged path 398 return fname 399 400def ReadFile(fname, binary=True): 401 """Read and return the contents of a file. 402 403 Args: 404 fname: path to filename to read, where ## signifiies the chroot. 405 406 Returns: 407 data read from file, as a string. 408 """ 409 with open(Filename(fname), binary and 'rb' or 'r') as fd: 410 data = fd.read() 411 #self._out.Info("Read file '%s' size %d (%#0x)" % 412 #(fname, len(data), len(data))) 413 return data 414 415def WriteFile(fname, data, binary=True): 416 """Write data into a file. 417 418 Args: 419 fname: path to filename to write 420 data: data to write to file, as a string 421 """ 422 #self._out.Info("Write file '%s' size %d (%#0x)" % 423 #(fname, len(data), len(data))) 424 with open(Filename(fname), binary and 'wb' or 'w') as fd: 425 fd.write(data) 426 427def GetBytes(byte, size): 428 """Get a string of bytes of a given size 429 430 Args: 431 byte: Numeric byte value to use 432 size: Size of bytes/string to return 433 434 Returns: 435 A bytes type with 'byte' repeated 'size' times 436 """ 437 return bytes([byte]) * size 438 439def ToBytes(string): 440 """Convert a str type into a bytes type 441 442 Args: 443 string: string to convert 444 445 Returns: 446 A bytes type 447 """ 448 return string.encode('utf-8') 449 450def ToString(bval): 451 """Convert a bytes type into a str type 452 453 Args: 454 bval: bytes value to convert 455 456 Returns: 457 Python 3: A bytes type 458 Python 2: A string type 459 """ 460 return bval.decode('utf-8') 461 462def Compress(indata, algo, with_header=True): 463 """Compress some data using a given algorithm 464 465 Note that for lzma this uses an old version of the algorithm, not that 466 provided by xz. 467 468 This requires 'lz4' and 'lzma_alone' tools. It also requires an output 469 directory to be previously set up, by calling PrepareOutputDir(). 470 471 Args: 472 indata: Input data to compress 473 algo: Algorithm to use ('none', 'gzip', 'lz4' or 'lzma') 474 475 Returns: 476 Compressed data 477 """ 478 if algo == 'none': 479 return indata 480 fname = GetOutputFilename('%s.comp.tmp' % algo) 481 WriteFile(fname, indata) 482 if algo == 'lz4': 483 data = Run('lz4', '--no-frame-crc', '-B4', '-5', '-c', fname, 484 binary=True) 485 # cbfstool uses a very old version of lzma 486 elif algo == 'lzma': 487 outfname = GetOutputFilename('%s.comp.otmp' % algo) 488 Run('lzma_alone', 'e', fname, outfname, '-lc1', '-lp0', '-pb0', '-d8') 489 data = ReadFile(outfname) 490 elif algo == 'gzip': 491 data = Run('gzip', '-c', fname, binary=True) 492 else: 493 raise ValueError("Unknown algorithm '%s'" % algo) 494 if with_header: 495 hdr = struct.pack('<I', len(data)) 496 data = hdr + data 497 return data 498 499def Decompress(indata, algo, with_header=True): 500 """Decompress some data using a given algorithm 501 502 Note that for lzma this uses an old version of the algorithm, not that 503 provided by xz. 504 505 This requires 'lz4' and 'lzma_alone' tools. It also requires an output 506 directory to be previously set up, by calling PrepareOutputDir(). 507 508 Args: 509 indata: Input data to decompress 510 algo: Algorithm to use ('none', 'gzip', 'lz4' or 'lzma') 511 512 Returns: 513 Compressed data 514 """ 515 if algo == 'none': 516 return indata 517 if with_header: 518 data_len = struct.unpack('<I', indata[:4])[0] 519 indata = indata[4:4 + data_len] 520 fname = GetOutputFilename('%s.decomp.tmp' % algo) 521 with open(fname, 'wb') as fd: 522 fd.write(indata) 523 if algo == 'lz4': 524 data = Run('lz4', '-dc', fname, binary=True) 525 elif algo == 'lzma': 526 outfname = GetOutputFilename('%s.decomp.otmp' % algo) 527 Run('lzma_alone', 'd', fname, outfname) 528 data = ReadFile(outfname, binary=True) 529 elif algo == 'gzip': 530 data = Run('gzip', '-cd', fname, binary=True) 531 else: 532 raise ValueError("Unknown algorithm '%s'" % algo) 533 return data 534 535CMD_CREATE, CMD_DELETE, CMD_ADD, CMD_REPLACE, CMD_EXTRACT = range(5) 536 537IFWITOOL_CMDS = { 538 CMD_CREATE: 'create', 539 CMD_DELETE: 'delete', 540 CMD_ADD: 'add', 541 CMD_REPLACE: 'replace', 542 CMD_EXTRACT: 'extract', 543 } 544 545def RunIfwiTool(ifwi_file, cmd, fname=None, subpart=None, entry_name=None): 546 """Run ifwitool with the given arguments: 547 548 Args: 549 ifwi_file: IFWI file to operation on 550 cmd: Command to execute (CMD_...) 551 fname: Filename of file to add/replace/extract/create (None for 552 CMD_DELETE) 553 subpart: Name of sub-partition to operation on (None for CMD_CREATE) 554 entry_name: Name of directory entry to operate on, or None if none 555 """ 556 args = ['ifwitool', ifwi_file] 557 args.append(IFWITOOL_CMDS[cmd]) 558 if fname: 559 args += ['-f', fname] 560 if subpart: 561 args += ['-n', subpart] 562 if entry_name: 563 args += ['-d', '-e', entry_name] 564 Run(*args) 565 566def ToHex(val): 567 """Convert an integer value (or None) to a string 568 569 Returns: 570 hex value, or 'None' if the value is None 571 """ 572 return 'None' if val is None else '%#x' % val 573 574def ToHexSize(val): 575 """Return the size of an object in hex 576 577 Returns: 578 hex value of size, or 'None' if the value is None 579 """ 580 return 'None' if val is None else '%#x' % len(val) 581