1#!/usr/bin/env python3 2# vim: set list et ts=8 sts=4 sw=4 ft=python: 3 4# acefile - read/test/extract ACE 1.0 and 2.0 archives in pure python 5# Copyright (C) 2017-2019, Daniel Roethlisberger <daniel@roe.ch> 6# All rights reserved. 7# 8# Redistribution and use in source and binary forms, with or without 9# modification, are permitted provided that the following conditions 10# are met: 11# 1. Redistributions of source code must retain the above copyright 12# notice, this list of conditions, and the following disclaimer. 13# 2. Redistributions in binary form must reproduce the above copyright 14# notice, this list of conditions and the following disclaimer in the 15# documentation and/or other materials provided with the distribution. 16# 17# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 28# NOTE: The ACE archive format and ACE compression and decompression 29# algorithms have been designed by Marcel Lemke. The above copyright 30# notice and license does not constitute a claim of intellectual property 31# over ACE technology beyond the copyright of this python implementation. 32 33""" 34Read/test/extract ACE 1.0 and 2.0 archives in pure python. 35 36This single-file, pure python 3, no-dependencies implementation is intended 37to be used as a library, but also provides a stand-alone unace utility. 38As mostly pure-python implementation, it is significantly slower than 39native implementations, but more robust against vulnerabilities. 40 41This implementation supports up to version 2.0 of the ACE archive format, 42including the EXE, DELTA, PIC and SOUND modes of ACE 2.0, password protected 43archives and multi-volume archives. It does not support writing to archives. 44It is an implementation from scratch, based on the 1998 document titled 45"Technical information of the archiver ACE v1.2" by Marcel Lemke, using 46unace 2.5 and WinAce 2.69 by Marcel Lemke as reference implementations. 47 48For more information, API documentation, source code, packages and release 49notifications, refer to: 50 51- https://www.roe.ch/acefile 52- https://apidoc.roe.ch/acefile 53- https://github.com/droe/acefile 54- https://pypi.python.org/pypi/acefile 55- https://twitter.com/droethlisberger 56""" 57 58__version__ = '0.6.12' 59__author__ = 'Daniel Roethlisberger' 60__email__ = 'daniel@roe.ch' 61__copyright__ = 'Copyright 2017-2019, Daniel Roethlisberger' 62__credits__ = ['Marcel Lemke'] 63__license__ = 'BSD' 64__url__ = 'https://www.roe.ch/acefile' 65 66 67 68import array 69import builtins 70import ctypes 71import datetime 72import io 73import math 74import os 75import platform 76import re 77import stat 78import struct 79import sys 80import zlib 81 82try: 83 import acebitstream 84except: 85 acebitstream = None 86 87 88 89# Very basic debugging facility; if set to True, exceptions raised during 90# testing of archives will be raised and a minimal set of state information 91# will be printed to stderr. 92DEBUG = False 93 94 95 96# Arbitrarily chosen buffer size to use for buffered file operations that 97# have no obvious natural block size. 98FILE_BLOCKSIZE = 131072 99assert FILE_BLOCKSIZE % 4 == 0 100 101 102 103if platform.system() == 'Windows': 104 # BOOL WINAPI SetFileAttributes( 105 # _In_ LPCTSTR lpFileName, 106 # _In_ DWORD dwFileAttributes 107 # ); 108 try: 109 SetFileAttributes = ctypes.windll.kernel32.SetFileAttributesW 110 except: 111 SetFileAttributes = None 112 # BOOL WINAPI SetFileSecurity( 113 # _In_ LPCTSTR lpFileName, 114 # _In_ SECURITY_INFORMATION SecurityInformation, 115 # _In_ PSECURITY_DESCRIPTOR pSecurityDescriptor 116 # ); 117 try: 118 SetFileSecurity = ctypes.windll.advapi32.SetFileSecurityW 119 except: 120 SetFileSecurity = None 121else: 122 SetFileAttributes = None 123 SetFileSecurity = None 124 125 126 127def eprint(*args, **kwargs): 128 """ 129 Print to stderr. 130 """ 131 print(*args, file=sys.stderr, **kwargs) 132 133 134 135# haklib.dt 136def _dt_fromdos(dosdt): 137 """ 138 Convert DOS format 32bit timestamp to datetime object. 139 Timestamps with illegal values out of the allowed range are ignored and a 140 datetime object representing 1980-01-01 00:00:00 is returned instead. 141 https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx 142 143 >>> _dt_fromdos(0x4a5c48fd) 144 datetime.datetime(2017, 2, 28, 9, 7, 58) 145 >>> _dt_fromdos(0) 146 datetime.datetime(1980, 1, 1, 0, 0) 147 >>> _dt_fromdos(-1) 148 datetime.datetime(1980, 1, 1, 0, 0) 149 """ 150 try: 151 return datetime.datetime( 152 ((dosdt >> 25) & 0x7F) + 1980, 153 (dosdt >> 21) & 0x0F, 154 (dosdt >> 16) & 0x1F, 155 (dosdt >> 11) & 0x1F, 156 (dosdt >> 5) & 0x3F, 157 ((dosdt ) & 0x1F) * 2) 158 except ValueError: 159 return datetime.datetime(1980, 1, 1, 0, 0, 0) 160 161 162 163# haklib.c 164def c_div(q, d): 165 """ 166 Arbitrary signed integer division with c behaviour. 167 168 >>> (c_div(10, 3), c_div(-10, -3), c_div(-10, 3), c_div(10, -3)) 169 (3, 3, -3, -3) 170 >>> c_div(-11, 0) 171 Traceback (most recent call last): 172 ... 173 ZeroDivisionError 174 """ 175 s = int(math.copysign(1, q) * math.copysign(1, d)) 176 return s * int(abs(q) / abs(d)) 177 178def c_schar(i): 179 """ 180 Convert arbitrary integer to c signed char type range as if casted in c. 181 182 >>> c_schar(0x12345678) 183 120 184 >>> (c_schar(-128), c_schar(-129), c_schar(127), c_schar(128)) 185 (-128, 127, 127, -128) 186 """ 187 return ((i + 128) % 256) - 128 188 189def c_uchar(i): 190 """ 191 Convert arbitrary integer to c unsigned char type range as if casted in c. 192 193 >>> c_uchar(0x12345678) 194 120 195 >>> (c_uchar(-123), c_uchar(-1), c_uchar(255), c_uchar(256)) 196 (133, 255, 255, 0) 197 """ 198 return i & 0xFF 199 200def c_rot32(i, n): 201 """ 202 Rotate *i* left by *n* bits within the uint32 value range. 203 204 >>> c_rot32(0xF0000000, 4) 205 15 206 >>> c_rot32(0xF0, -4) 207 15 208 """ 209 if n < 0: 210 n = 32 + n 211 return (((i << n) & 0xFFFFFFFF) | (i >> (32 - n))) 212 213def c_add32(a, b): 214 """ 215 Add *a* and *b* within the uint32 value range. 216 217 >>> c_add32(0xFFFFFFFF, 1) 218 0 219 >>> c_add32(0xFFFFFFFF, 0xFFFFFFFF) 220 4294967294 221 """ 222 return (a + b) & 0xFFFFFFFF 223 224def c_sum32(*args): 225 """ 226 Add all elements of *args* within the uint32 value range. 227 228 >>> c_sum32(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF) 229 4294967293 230 """ 231 return sum(args) & 0xFFFFFFFF 232 233 234 235def asciibox(msg, title=None, minwidth=None): 236 """ 237 Returns message string *msg* wrapped in a plain ASCII box. 238 If *title* is given, add *title* in the top horizontal bar. 239 Lines will be padded to the longest out of *minwidth* characters, the 240 length of the longest line, or the length of the title plus six. 241 Caller is responsible for ensuring a sensible line length in *msg*. 242 """ 243 out = [] 244 lines = msg.splitlines() 245 width = 0 246 for line in lines: 247 width = max(width, len(line)) 248 if minwidth != None: 249 width = max(width, minwidth) 250 if title != None: 251 width = max(width, len(title) + 6) 252 ftr = "+" + ("-" * (width + 2)) + "+" 253 if title != None: 254 hdr = ("+--[ %s ]--" % title) + ("-" * (width - 6 - len(title))) + "+" 255 else: 256 hdr = ftr 257 fmt = "| %%-%is |" % width 258 out.append(hdr) 259 for line in msg.splitlines(): 260 out.append(fmt % line) 261 out.append(ftr) 262 return '\n'.join(out) 263 264 265 266class FileSegmentIO: 267 """ 268 Seekable file-like object that wraps and reads from seekable file-like 269 object and fakes EOF when a read would extend beyond a defined boundary. 270 271 >>> FileSegmentIO(io.BytesIO(b'0123456789'), 3, 4).read() 272 b'3456' 273 """ 274 def __init__(self, f, base, size): 275 assert f.seekable() 276 self.__file = f 277 self.__base = base 278 self.__eof = base + size 279 self.__file.seek(self.__base, 0) 280 281 def seekable(self): 282 return True 283 284 def _tell(self): 285 """ 286 Returns the current absolute position in the file and asserts that it 287 lies within the defined file segment. 288 """ 289 pos = self.__file.tell() 290 assert pos >= self.__base and pos <= self.__eof 291 return pos 292 293 def tell(self): 294 return self._tell() - self.__base 295 296 def seek(self, offset, whence=0): 297 if whence == 0: 298 newpos = self.__base + offset 299 elif whence == 1: 300 newpos = self._tell() + offset 301 elif whence == 2: 302 newpos = self.__eof + offset 303 assert newpos >= self.__base and newpos <= self.__eof 304 self.__file.seek(newpos, 0) 305 306 def read(self, n=None): 307 pos = self._tell() 308 if n == None: 309 amount = self.__eof - pos 310 else: 311 amount = min(n, self.__eof - pos) 312 if amount == 0: 313 return b'' 314 return self.__file.read(amount) 315 316 317 318class MultipleFilesIO: 319 """ 320 Seekable file-like object that wraps and reads from multiple 321 seekable lower-level file-like objects. 322 323 >>> MultipleFilesIO((io.BytesIO(b'01234'), io.BytesIO(b'56789'))).read() 324 b'0123456789' 325 """ 326 def __init__(self, files): 327 assert len(files) > 0 328 self.__files = files 329 self.__sizes = [] 330 for f in files: 331 f.seek(0, 2) 332 self.__sizes.append(f.tell()) 333 self.__files[0].seek(0) 334 self.__idx = 0 335 self.__pos = 0 336 self.__eof = sum(self.__sizes) 337 338 def seekable(): 339 return True 340 341 def tell(self): 342 return self.__pos 343 344 def seek(self, offset, whence=0): 345 if whence == 0: 346 newpos = offset 347 elif whence == 1: 348 newpos = self.__pos + offset 349 elif whence == 2: 350 newpos = self.__eof + offset 351 assert newpos >= 0 and newpos <= self.__eof 352 idx = 0 353 relpos = newpos 354 while relpos > self.__sizes[idx]: 355 relpos -= self.__sizes[idx] 356 idx += 1 357 self.__files[idx].seek(relpos) 358 self.__idx = idx 359 360 def read(self, n=None): 361 if n == None: 362 n = self.__eof - self.__pos 363 out = [] 364 have_size = 0 365 while have_size < n: 366 if self.__idx >= len(self.__files): 367 break 368 chunk = self.__files[self.__idx].read(n - have_size) 369 if len(chunk) == 0: 370 self.__idx += 1 371 if self.__idx < len(self.__files): 372 self.__files[self.__idx].seek(0) 373 continue 374 out.append(chunk) 375 self.__pos += len(chunk) 376 have_size += len(chunk) 377 return b''.join(out) 378 379 380 381class EncryptedFileIO: 382 """ 383 Non-seekable file-like object that reads from a lower-level seekable 384 file-like object, and transparently decrypts the data stream using a 385 decryption engine. The decryption engine is assumed to support a 386 decrypt() method and a blocksize property. The underlying file-like 387 object is expected to contain a multiple of blocksize bytes, if not, 388 CorruptedArchiveError is raised. 389 390 >>> EncryptedFileIO(io.BytesIO(b'7'*16), AceBlowfish(b'123456789')).read() 391 b'\\t_\\xd0a}\\x1dh\\xdd>h\\xe7VJ*_\\xea' 392 >>> EncryptedFileIO(io.BytesIO(b'7'*17), AceBlowfish(b'123456789')).read() 393 Traceback (most recent call last): 394 ... 395 CorruptedArchiveError 396 """ 397 def __init__(self, f, engine): 398 self.__file = f 399 self.__file.seek(0, 2) 400 self.__eof = self.__file.tell() 401 self.__file.seek(0) 402 self.__engine = engine 403 self.__buffer = b'' 404 405 def seekable(): 406 return False 407 408 def read(self, n=None): 409 if n == None: 410 n = self.__eof - (self.__file.tell() - len(self.__buffer)) 411 if n < len(self.__buffer): 412 rbuf = self.__buffer[:n] 413 self.__buffer = self.__buffer[n:] 414 return rbuf 415 want_bytes = n - len(self.__buffer) 416 read_bytes = want_bytes 417 blocksize = self.__engine.blocksize 418 if want_bytes % blocksize: 419 read_bytes += blocksize - (want_bytes % blocksize) 420 buf = self.__file.read(read_bytes) 421 if len(buf) % blocksize: 422 raise CorruptedArchiveError("Truncated ciphertext block") 423 buf = self.__engine.decrypt(buf) 424 rbuf = self.__buffer + buf[:n] 425 self.__buffer = buf[n:] 426 return rbuf 427 428 429 430class AceBlowfish: 431 """ 432 Decryption engine for ACE Blowfish. 433 434 >>> bf = AceBlowfish(b'123456789') 435 >>> bf.blocksize 436 8 437 >>> bf.decrypt(b'\\xFF'*8) 438 b'\\xb7wF@5.er' 439 >>> bf.decrypt(b'\\xC7'*8) 440 b'eE\\x05\\xc4\\xa5\\x85)\\xbc' 441 >>> bf.decrypt(b'123') 442 Traceback (most recent call last): 443 ... 444 AssertionError 445 """ 446 447 SHA1_A = 0x67452301 448 SHA1_B = 0xefcdab89 449 SHA1_C = 0x98badcfe 450 SHA1_D = 0x10325476 451 SHA1_E = 0xc3d2e1f0 452 453 BF_P = ( 454 0x243F6A88, 0x85A308D3, 0x13198A2E, 0x03707344, 455 0xA4093822, 0x299F31D0, 0x082EFA98, 0xEC4E6C89, 456 0x452821E6, 0x38D01377, 0xBE5466CF, 0x34E90C6C, 457 0xC0AC29B7, 0xC97C50DD, 0x3F84D5B5, 0xB5470917, 458 0x9216D5D9, 0x8979FB1B) 459 460 BF_S0 = ( 461 0xD1310BA6, 0x98DFB5AC, 0x2FFD72DB, 0xD01ADFB7, 462 0xB8E1AFED, 0x6A267E96, 0xBA7C9045, 0xF12C7F99, 463 0x24A19947, 0xB3916CF7, 0x0801F2E2, 0x858EFC16, 464 0x636920D8, 0x71574E69, 0xA458FEA3, 0xF4933D7E, 465 0x0D95748F, 0x728EB658, 0x718BCD58, 0x82154AEE, 466 0x7B54A41D, 0xC25A59B5, 0x9C30D539, 0x2AF26013, 467 0xC5D1B023, 0x286085F0, 0xCA417918, 0xB8DB38EF, 468 0x8E79DCB0, 0x603A180E, 0x6C9E0E8B, 0xB01E8A3E, 469 0xD71577C1, 0xBD314B27, 0x78AF2FDA, 0x55605C60, 470 0xE65525F3, 0xAA55AB94, 0x57489862, 0x63E81440, 471 0x55CA396A, 0x2AAB10B6, 0xB4CC5C34, 0x1141E8CE, 472 0xA15486AF, 0x7C72E993, 0xB3EE1411, 0x636FBC2A, 473 0x2DA9C55D, 0x741831F6, 0xCE5C3E16, 0x9B87901E, 474 0xAFD6BA33, 0x6C24CF5C, 0x7A325381, 0x28958677, 475 0x3B8F4898, 0x6B4BB9AF, 0xC4BFE81B, 0x66282193, 476 0x61D809CC, 0xFB21A991, 0x487CAC60, 0x5DEC8032, 477 0xEF845D5D, 0xE98575B1, 0xDC262302, 0xEB651B88, 478 0x23893E81, 0xD396ACC5, 0x0F6D6FF3, 0x83F44239, 479 0x2E0B4482, 0xA4842004, 0x69C8F04A, 0x9E1F9B5E, 480 0x21C66842, 0xF6E96C9A, 0x670C9C61, 0xABD388F0, 481 0x6A51A0D2, 0xD8542F68, 0x960FA728, 0xAB5133A3, 482 0x6EEF0B6C, 0x137A3BE4, 0xBA3BF050, 0x7EFB2A98, 483 0xA1F1651D, 0x39AF0176, 0x66CA593E, 0x82430E88, 484 0x8CEE8619, 0x456F9FB4, 0x7D84A5C3, 0x3B8B5EBE, 485 0xE06F75D8, 0x85C12073, 0x401A449F, 0x56C16AA6, 486 0x4ED3AA62, 0x363F7706, 0x1BFEDF72, 0x429B023D, 487 0x37D0D724, 0xD00A1248, 0xDB0FEAD3, 0x49F1C09B, 488 0x075372C9, 0x80991B7B, 0x25D479D8, 0xF6E8DEF7, 489 0xE3FE501A, 0xB6794C3B, 0x976CE0BD, 0x04C006BA, 490 0xC1A94FB6, 0x409F60C4, 0x5E5C9EC2, 0x196A2463, 491 0x68FB6FAF, 0x3E6C53B5, 0x1339B2EB, 0x3B52EC6F, 492 0x6DFC511F, 0x9B30952C, 0xCC814544, 0xAF5EBD09, 493 0xBEE3D004, 0xDE334AFD, 0x660F2807, 0x192E4BB3, 494 0xC0CBA857, 0x45C8740F, 0xD20B5F39, 0xB9D3FBDB, 495 0x5579C0BD, 0x1A60320A, 0xD6A100C6, 0x412C7279, 496 0x679F25FE, 0xFB1FA3CC, 0x8EA5E9F8, 0xDB3222F8, 497 0x3C7516DF, 0xFD616B15, 0x2F501EC8, 0xAD0552AB, 498 0x323DB5FA, 0xFD238760, 0x53317B48, 0x3E00DF82, 499 0x9E5C57BB, 0xCA6F8CA0, 0x1A87562E, 0xDF1769DB, 500 0xD542A8F6, 0x287EFFC3, 0xAC6732C6, 0x8C4F5573, 501 0x695B27B0, 0xBBCA58C8, 0xE1FFA35D, 0xB8F011A0, 502 0x10FA3D98, 0xFD2183B8, 0x4AFCB56C, 0x2DD1D35B, 503 0x9A53E479, 0xB6F84565, 0xD28E49BC, 0x4BFB9790, 504 0xE1DDF2DA, 0xA4CB7E33, 0x62FB1341, 0xCEE4C6E8, 505 0xEF20CADA, 0x36774C01, 0xD07E9EFE, 0x2BF11FB4, 506 0x95DBDA4D, 0xAE909198, 0xEAAD8E71, 0x6B93D5A0, 507 0xD08ED1D0, 0xAFC725E0, 0x8E3C5B2F, 0x8E7594B7, 508 0x8FF6E2FB, 0xF2122B64, 0x8888B812, 0x900DF01C, 509 0x4FAD5EA0, 0x688FC31C, 0xD1CFF191, 0xB3A8C1AD, 510 0x2F2F2218, 0xBE0E1777, 0xEA752DFE, 0x8B021FA1, 511 0xE5A0CC0F, 0xB56F74E8, 0x18ACF3D6, 0xCE89E299, 512 0xB4A84FE0, 0xFD13E0B7, 0x7CC43B81, 0xD2ADA8D9, 513 0x165FA266, 0x80957705, 0x93CC7314, 0x211A1477, 514 0xE6AD2065, 0x77B5FA86, 0xC75442F5, 0xFB9D35CF, 515 0xEBCDAF0C, 0x7B3E89A0, 0xD6411BD3, 0xAE1E7E49, 516 0x00250E2D, 0x2071B35E, 0x226800BB, 0x57B8E0AF, 517 0x2464369B, 0xF009B91E, 0x5563911D, 0x59DFA6AA, 518 0x78C14389, 0xD95A537F, 0x207D5BA2, 0x02E5B9C5, 519 0x83260376, 0x6295CFA9, 0x11C81968, 0x4E734A41, 520 0xB3472DCA, 0x7B14A94A, 0x1B510052, 0x9A532915, 521 0xD60F573F, 0xBC9BC6E4, 0x2B60A476, 0x81E67400, 522 0x08BA6FB5, 0x571BE91F, 0xF296EC6B, 0x2A0DD915, 523 0xB6636521, 0xE7B9F9B6, 0xFF34052E, 0xC5855664, 524 0x53B02D5D, 0xA99F8FA1, 0x08BA4799, 0x6E85076A) 525 526 BF_S1 = ( 527 0x4B7A70E9, 0xB5B32944, 0xDB75092E, 0xC4192623, 528 0xAD6EA6B0, 0x49A7DF7D, 0x9CEE60B8, 0x8FEDB266, 529 0xECAA8C71, 0x699A17FF, 0x5664526C, 0xC2B19EE1, 530 0x193602A5, 0x75094C29, 0xA0591340, 0xE4183A3E, 531 0x3F54989A, 0x5B429D65, 0x6B8FE4D6, 0x99F73FD6, 532 0xA1D29C07, 0xEFE830F5, 0x4D2D38E6, 0xF0255DC1, 533 0x4CDD2086, 0x8470EB26, 0x6382E9C6, 0x021ECC5E, 534 0x09686B3F, 0x3EBAEFC9, 0x3C971814, 0x6B6A70A1, 535 0x687F3584, 0x52A0E286, 0xB79C5305, 0xAA500737, 536 0x3E07841C, 0x7FDEAE5C, 0x8E7D44EC, 0x5716F2B8, 537 0xB03ADA37, 0xF0500C0D, 0xF01C1F04, 0x0200B3FF, 538 0xAE0CF51A, 0x3CB574B2, 0x25837A58, 0xDC0921BD, 539 0xD19113F9, 0x7CA92FF6, 0x94324773, 0x22F54701, 540 0x3AE5E581, 0x37C2DADC, 0xC8B57634, 0x9AF3DDA7, 541 0xA9446146, 0x0FD0030E, 0xECC8C73E, 0xA4751E41, 542 0xE238CD99, 0x3BEA0E2F, 0x3280BBA1, 0x183EB331, 543 0x4E548B38, 0x4F6DB908, 0x6F420D03, 0xF60A04BF, 544 0x2CB81290, 0x24977C79, 0x5679B072, 0xBCAF89AF, 545 0xDE9A771F, 0xD9930810, 0xB38BAE12, 0xDCCF3F2E, 546 0x5512721F, 0x2E6B7124, 0x501ADDE6, 0x9F84CD87, 547 0x7A584718, 0x7408DA17, 0xBC9F9ABC, 0xE94B7D8C, 548 0xEC7AEC3A, 0xDB851DFA, 0x63094366, 0xC464C3D2, 549 0xEF1C1847, 0x3215D908, 0xDD433B37, 0x24C2BA16, 550 0x12A14D43, 0x2A65C451, 0x50940002, 0x133AE4DD, 551 0x71DFF89E, 0x10314E55, 0x81AC77D6, 0x5F11199B, 552 0x043556F1, 0xD7A3C76B, 0x3C11183B, 0x5924A509, 553 0xF28FE6ED, 0x97F1FBFA, 0x9EBABF2C, 0x1E153C6E, 554 0x86E34570, 0xEAE96FB1, 0x860E5E0A, 0x5A3E2AB3, 555 0x771FE71C, 0x4E3D06FA, 0x2965DCB9, 0x99E71D0F, 556 0x803E89D6, 0x5266C825, 0x2E4CC978, 0x9C10B36A, 557 0xC6150EBA, 0x94E2EA78, 0xA5FC3C53, 0x1E0A2DF4, 558 0xF2F74EA7, 0x361D2B3D, 0x1939260F, 0x19C27960, 559 0x5223A708, 0xF71312B6, 0xEBADFE6E, 0xEAC31F66, 560 0xE3BC4595, 0xA67BC883, 0xB17F37D1, 0x018CFF28, 561 0xC332DDEF, 0xBE6C5AA5, 0x65582185, 0x68AB9802, 562 0xEECEA50F, 0xDB2F953B, 0x2AEF7DAD, 0x5B6E2F84, 563 0x1521B628, 0x29076170, 0xECDD4775, 0x619F1510, 564 0x13CCA830, 0xEB61BD96, 0x0334FE1E, 0xAA0363CF, 565 0xB5735C90, 0x4C70A239, 0xD59E9E0B, 0xCBAADE14, 566 0xEECC86BC, 0x60622CA7, 0x9CAB5CAB, 0xB2F3846E, 567 0x648B1EAF, 0x19BDF0CA, 0xA02369B9, 0x655ABB50, 568 0x40685A32, 0x3C2AB4B3, 0x319EE9D5, 0xC021B8F7, 569 0x9B540B19, 0x875FA099, 0x95F7997E, 0x623D7DA8, 570 0xF837889A, 0x97E32D77, 0x11ED935F, 0x16681281, 571 0x0E358829, 0xC7E61FD6, 0x96DEDFA1, 0x7858BA99, 572 0x57F584A5, 0x1B227263, 0x9B83C3FF, 0x1AC24696, 573 0xCDB30AEB, 0x532E3054, 0x8FD948E4, 0x6DBC3128, 574 0x58EBF2EF, 0x34C6FFEA, 0xFE28ED61, 0xEE7C3C73, 575 0x5D4A14D9, 0xE864B7E3, 0x42105D14, 0x203E13E0, 576 0x45EEE2B6, 0xA3AAABEA, 0xDB6C4F15, 0xFACB4FD0, 577 0xC742F442, 0xEF6ABBB5, 0x654F3B1D, 0x41CD2105, 578 0xD81E799E, 0x86854DC7, 0xE44B476A, 0x3D816250, 579 0xCF62A1F2, 0x5B8D2646, 0xFC8883A0, 0xC1C7B6A3, 580 0x7F1524C3, 0x69CB7492, 0x47848A0B, 0x5692B285, 581 0x095BBF00, 0xAD19489D, 0x1462B174, 0x23820E00, 582 0x58428D2A, 0x0C55F5EA, 0x1DADF43E, 0x233F7061, 583 0x3372F092, 0x8D937E41, 0xD65FECF1, 0x6C223BDB, 584 0x7CDE3759, 0xCBEE7460, 0x4085F2A7, 0xCE77326E, 585 0xA6078084, 0x19F8509E, 0xE8EFD855, 0x61D99735, 586 0xA969A7AA, 0xC50C06C2, 0x5A04ABFC, 0x800BCADC, 587 0x9E447A2E, 0xC3453484, 0xFDD56705, 0x0E1E9EC9, 588 0xDB73DBD3, 0x105588CD, 0x675FDA79, 0xE3674340, 589 0xC5C43465, 0x713E38D8, 0x3D28F89E, 0xF16DFF20, 590 0x153E21E7, 0x8FB03D4A, 0xE6E39F2B, 0xDB83ADF7) 591 592 BF_S2 = ( 593 0xE93D5A68, 0x948140F7, 0xF64C261C, 0x94692934, 594 0x411520F7, 0x7602D4F7, 0xBCF46B2E, 0xD4A20068, 595 0xD4082471, 0x3320F46A, 0x43B7D4B7, 0x500061AF, 596 0x1E39F62E, 0x97244546, 0x14214F74, 0xBF8B8840, 597 0x4D95FC1D, 0x96B591AF, 0x70F4DDD3, 0x66A02F45, 598 0xBFBC09EC, 0x03BD9785, 0x7FAC6DD0, 0x31CB8504, 599 0x96EB27B3, 0x55FD3941, 0xDA2547E6, 0xABCA0A9A, 600 0x28507825, 0x530429F4, 0x0A2C86DA, 0xE9B66DFB, 601 0x68DC1462, 0xD7486900, 0x680EC0A4, 0x27A18DEE, 602 0x4F3FFEA2, 0xE887AD8C, 0xB58CE006, 0x7AF4D6B6, 603 0xAACE1E7C, 0xD3375FEC, 0xCE78A399, 0x406B2A42, 604 0x20FE9E35, 0xD9F385B9, 0xEE39D7AB, 0x3B124E8B, 605 0x1DC9FAF7, 0x4B6D1856, 0x26A36631, 0xEAE397B2, 606 0x3A6EFA74, 0xDD5B4332, 0x6841E7F7, 0xCA7820FB, 607 0xFB0AF54E, 0xD8FEB397, 0x454056AC, 0xBA489527, 608 0x55533A3A, 0x20838D87, 0xFE6BA9B7, 0xD096954B, 609 0x55A867BC, 0xA1159A58, 0xCCA92963, 0x99E1DB33, 610 0xA62A4A56, 0x3F3125F9, 0x5EF47E1C, 0x9029317C, 611 0xFDF8E802, 0x04272F70, 0x80BB155C, 0x05282CE3, 612 0x95C11548, 0xE4C66D22, 0x48C1133F, 0xC70F86DC, 613 0x07F9C9EE, 0x41041F0F, 0x404779A4, 0x5D886E17, 614 0x325F51EB, 0xD59BC0D1, 0xF2BCC18F, 0x41113564, 615 0x257B7834, 0x602A9C60, 0xDFF8E8A3, 0x1F636C1B, 616 0x0E12B4C2, 0x02E1329E, 0xAF664FD1, 0xCAD18115, 617 0x6B2395E0, 0x333E92E1, 0x3B240B62, 0xEEBEB922, 618 0x85B2A20E, 0xE6BA0D99, 0xDE720C8C, 0x2DA2F728, 619 0xD0127845, 0x95B794FD, 0x647D0862, 0xE7CCF5F0, 620 0x5449A36F, 0x877D48FA, 0xC39DFD27, 0xF33E8D1E, 621 0x0A476341, 0x992EFF74, 0x3A6F6EAB, 0xF4F8FD37, 622 0xA812DC60, 0xA1EBDDF8, 0x991BE14C, 0xDB6E6B0D, 623 0xC67B5510, 0x6D672C37, 0x2765D43B, 0xDCD0E804, 624 0xF1290DC7, 0xCC00FFA3, 0xB5390F92, 0x690FED0B, 625 0x667B9FFB, 0xCEDB7D9C, 0xA091CF0B, 0xD9155EA3, 626 0xBB132F88, 0x515BAD24, 0x7B9479BF, 0x763BD6EB, 627 0x37392EB3, 0xCC115979, 0x8026E297, 0xF42E312D, 628 0x6842ADA7, 0xC66A2B3B, 0x12754CCC, 0x782EF11C, 629 0x6A124237, 0xB79251E7, 0x06A1BBE6, 0x4BFB6350, 630 0x1A6B1018, 0x11CAEDFA, 0x3D25BDD8, 0xE2E1C3C9, 631 0x44421659, 0x0A121386, 0xD90CEC6E, 0xD5ABEA2A, 632 0x64AF674E, 0xDA86A85F, 0xBEBFE988, 0x64E4C3FE, 633 0x9DBC8057, 0xF0F7C086, 0x60787BF8, 0x6003604D, 634 0xD1FD8346, 0xF6381FB0, 0x7745AE04, 0xD736FCCC, 635 0x83426B33, 0xF01EAB71, 0xB0804187, 0x3C005E5F, 636 0x77A057BE, 0xBDE8AE24, 0x55464299, 0xBF582E61, 637 0x4E58F48F, 0xF2DDFDA2, 0xF474EF38, 0x8789BDC2, 638 0x5366F9C3, 0xC8B38E74, 0xB475F255, 0x46FCD9B9, 639 0x7AEB2661, 0x8B1DDF84, 0x846A0E79, 0x915F95E2, 640 0x466E598E, 0x20B45770, 0x8CD55591, 0xC902DE4C, 641 0xB90BACE1, 0xBB8205D0, 0x11A86248, 0x7574A99E, 642 0xB77F19B6, 0xE0A9DC09, 0x662D09A1, 0xC4324633, 643 0xE85A1F02, 0x09F0BE8C, 0x4A99A025, 0x1D6EFE10, 644 0x1AB93D1D, 0x0BA5A4DF, 0xA186F20F, 0x2868F169, 645 0xDCB7DA83, 0x573906FE, 0xA1E2CE9B, 0x4FCD7F52, 646 0x50115E01, 0xA70683FA, 0xA002B5C4, 0x0DE6D027, 647 0x9AF88C27, 0x773F8641, 0xC3604C06, 0x61A806B5, 648 0xF0177A28, 0xC0F586E0, 0x006058AA, 0x30DC7D62, 649 0x11E69ED7, 0x2338EA63, 0x53C2DD94, 0xC2C21634, 650 0xBBCBEE56, 0x90BCB6DE, 0xEBFC7DA1, 0xCE591D76, 651 0x6F05E409, 0x4B7C0188, 0x39720A3D, 0x7C927C24, 652 0x86E3725F, 0x724D9DB9, 0x1AC15BB4, 0xD39EB8FC, 653 0xED545578, 0x08FCA5B5, 0xD83D7CD3, 0x4DAD0FC4, 654 0x1E50EF5E, 0xB161E6F8, 0xA28514D9, 0x6C51133C, 655 0x6FD5C7E7, 0x56E14EC4, 0x362ABFCE, 0xDDC6C837, 656 0xD79A3234, 0x92638212, 0x670EFA8E, 0x406000E0) 657 658 BF_S3 = ( 659 0x3A39CE37, 0xD3FAF5CF, 0xABC27737, 0x5AC52D1B, 660 0x5CB0679E, 0x4FA33742, 0xD3822740, 0x99BC9BBE, 661 0xD5118E9D, 0xBF0F7315, 0xD62D1C7E, 0xC700C47B, 662 0xB78C1B6B, 0x21A19045, 0xB26EB1BE, 0x6A366EB4, 663 0x5748AB2F, 0xBC946E79, 0xC6A376D2, 0x6549C2C8, 664 0x530FF8EE, 0x468DDE7D, 0xD5730A1D, 0x4CD04DC6, 665 0x2939BBDB, 0xA9BA4650, 0xAC9526E8, 0xBE5EE304, 666 0xA1FAD5F0, 0x6A2D519A, 0x63EF8CE2, 0x9A86EE22, 667 0xC089C2B8, 0x43242EF6, 0xA51E03AA, 0x9CF2D0A4, 668 0x83C061BA, 0x9BE96A4D, 0x8FE51550, 0xBA645BD6, 669 0x2826A2F9, 0xA73A3AE1, 0x4BA99586, 0xEF5562E9, 670 0xC72FEFD3, 0xF752F7DA, 0x3F046F69, 0x77FA0A59, 671 0x80E4A915, 0x87B08601, 0x9B09E6AD, 0x3B3EE593, 672 0xE990FD5A, 0x9E34D797, 0x2CF0B7D9, 0x022B8B51, 673 0x96D5AC3A, 0x017DA67D, 0xD1CF3ED6, 0x7C7D2D28, 674 0x1F9F25CF, 0xADF2B89B, 0x5AD6B472, 0x5A88F54C, 675 0xE029AC71, 0xE019A5E6, 0x47B0ACFD, 0xED93FA9B, 676 0xE8D3C48D, 0x283B57CC, 0xF8D56629, 0x79132E28, 677 0x785F0191, 0xED756055, 0xF7960E44, 0xE3D35E8C, 678 0x15056DD4, 0x88F46DBA, 0x03A16125, 0x0564F0BD, 679 0xC3EB9E15, 0x3C9057A2, 0x97271AEC, 0xA93A072A, 680 0x1B3F6D9B, 0x1E6321F5, 0xF59C66FB, 0x26DCF319, 681 0x7533D928, 0xB155FDF5, 0x03563482, 0x8ABA3CBB, 682 0x28517711, 0xC20AD9F8, 0xABCC5167, 0xCCAD925F, 683 0x4DE81751, 0x3830DC8E, 0x379D5862, 0x9320F991, 684 0xEA7A90C2, 0xFB3E7BCE, 0x5121CE64, 0x774FBE32, 685 0xA8B6E37E, 0xC3293D46, 0x48DE5369, 0x6413E680, 686 0xA2AE0810, 0xDD6DB224, 0x69852DFD, 0x09072166, 687 0xB39A460A, 0x6445C0DD, 0x586CDECF, 0x1C20C8AE, 688 0x5BBEF7DD, 0x1B588D40, 0xCCD2017F, 0x6BB4E3BB, 689 0xDDA26A7E, 0x3A59FF45, 0x3E350A44, 0xBCB4CDD5, 690 0x72EACEA8, 0xFA6484BB, 0x8D6612AE, 0xBF3C6F47, 691 0xD29BE463, 0x542F5D9E, 0xAEC2771B, 0xF64E6370, 692 0x740E0D8D, 0xE75B1357, 0xF8721671, 0xAF537D5D, 693 0x4040CB08, 0x4EB4E2CC, 0x34D2466A, 0x0115AF84, 694 0xE1B00428, 0x95983A1D, 0x06B89FB4, 0xCE6EA048, 695 0x6F3F3B82, 0x3520AB82, 0x011A1D4B, 0x277227F8, 696 0x611560B1, 0xE7933FDC, 0xBB3A792B, 0x344525BD, 697 0xA08839E1, 0x51CE794B, 0x2F32C9B7, 0xA01FBAC9, 698 0xE01CC87E, 0xBCC7D1F6, 0xCF0111C3, 0xA1E8AAC7, 699 0x1A908749, 0xD44FBD9A, 0xD0DADECB, 0xD50ADA38, 700 0x0339C32A, 0xC6913667, 0x8DF9317C, 0xE0B12B4F, 701 0xF79E59B7, 0x43F5BB3A, 0xF2D519FF, 0x27D9459C, 702 0xBF97222C, 0x15E6FC2A, 0x0F91FC71, 0x9B941525, 703 0xFAE59361, 0xCEB69CEB, 0xC2A86459, 0x12BAA8D1, 704 0xB6C1075E, 0xE3056A0C, 0x10D25065, 0xCB03A442, 705 0xE0EC6E0E, 0x1698DB3B, 0x4C98A0BE, 0x3278E964, 706 0x9F1F9532, 0xE0D392DF, 0xD3A0342B, 0x8971F21E, 707 0x1B0A7441, 0x4BA3348C, 0xC5BE7120, 0xC37632D8, 708 0xDF359F8D, 0x9B992F2E, 0xE60B6F47, 0x0FE3F11D, 709 0xE54CDA54, 0x1EDAD891, 0xCE6279CF, 0xCD3E7E6F, 710 0x1618B166, 0xFD2C1D05, 0x848FD2C5, 0xF6FB2299, 711 0xF523F357, 0xA6327623, 0x93A83531, 0x56CCCD02, 712 0xACF08162, 0x5A75EBB5, 0x6E163697, 0x88D273CC, 713 0xDE966292, 0x81B949D0, 0x4C50901B, 0x71C65614, 714 0xE6C6C7BD, 0x327A140A, 0x45E1D006, 0xC3F27B9A, 715 0xC9AA53FD, 0x62A80F00, 0xBB25BFE2, 0x35BDD2F6, 716 0x71126905, 0xB2040222, 0xB6CBCF7C, 0xCD769C2B, 717 0x53113EC0, 0x1640E3D3, 0x38ABBD60, 0x2547ADF0, 718 0xBA38209C, 0xF746CE76, 0x77AFA1C5, 0x20756060, 719 0x85CBFE4E, 0x8AE88DD8, 0x7AAAF9B0, 0x4CF9AA7E, 720 0x1948C25C, 0x02FB8A8C, 0x01C36AE4, 0xD6EBE1F9, 721 0x90D4F869, 0xA65CDEA0, 0x3F09252D, 0xC208E69F, 722 0xB74E6132, 0xCE77E25B, 0x578FDFE3, 0x3AC372E6) 723 724 def __init__(self, pwd): 725 """ 726 Initialize decryption engine with a key derived from password *pwd*, 727 which can be str or bytes. 728 """ 729 if isinstance(pwd, str): 730 pwd = pwd.encode('utf-8') 731 self._bf_init(self._derive_key(pwd)) 732 733 def _derive_key(self, pwd): 734 """ 735 Derive the decryption key from password bytes *pwd* using a single 736 application of SHA-1 using non-standard padding. The password is 737 truncated to a maximum of 50 bytes before key derivation. 738 739 >>> AceBlowfish._derive_key(None, b'123456789') 740 (3071200156, 3325860325, 4058316933, 1308772094, 896611998) 741 """ 742 if len(pwd) > 50: 743 pwd = pwd[:50] 744 buf = pwd + bytes([0x80] + [0] * (64 - len(pwd) - 5)) 745 state = [] 746 state.extend(struct.unpack('<15L', buf)) 747 state.append(len(pwd) << 3) 748 for i in range(len(state), 80): 749 state.append(state[i-16] ^ state[i-14] ^ state[i-8] ^ state[i-3]) 750 a = AceBlowfish.SHA1_A 751 b = AceBlowfish.SHA1_B 752 c = AceBlowfish.SHA1_C 753 d = AceBlowfish.SHA1_D 754 e = AceBlowfish.SHA1_E 755 for i in range(20): 756 a, b, c, d, e = \ 757 c_sum32(c_rot32(a, 5), ((b&c)|(~b&d)), e, state[i], 758 0x5a827999), a, c_rot32(b, 30), c, d 759 for i in range(20, 40): 760 a, b, c, d, e = \ 761 c_sum32(c_rot32(a, 5), (b^c^d), e, state[i], 762 0x6ed9eba1), a, c_rot32(b, 30), c, d 763 for i in range(40, 60): 764 a, b, c, d, e = \ 765 c_sum32(c_rot32(a, 5), ((b&c)|(b&d)|(c&d)), e, state[i], 766 0x8f1bbcdc), a, c_rot32(b, 30), c, d 767 for i in range(60, 80): 768 a, b, c, d, e = \ 769 c_sum32(c_rot32(a, 5), (b^c^d), e, state[i], 770 0xca62c1d6), a, c_rot32(b, 30), c, d 771 a = c_add32(a, AceBlowfish.SHA1_A) 772 b = c_add32(b, AceBlowfish.SHA1_B) 773 c = c_add32(c, AceBlowfish.SHA1_C) 774 d = c_add32(d, AceBlowfish.SHA1_D) 775 e = c_add32(e, AceBlowfish.SHA1_E) 776 return (a, b, c, d, e) 777 778 def _bf_init(self, key): 779 """ 780 Initialize blowfish state using 160-bit key *key* as list or tuple of 781 integers. 782 """ 783 self.__p = [self.BF_P[i] ^ key[i % len(key)] \ 784 for i in list(range(len(self.BF_P)))] 785 self.__s = (list(self.BF_S0), list(self.BF_S1), 786 list(self.BF_S2), list(self.BF_S3)) 787 self.__lastcl = 0 788 self.__lastcr = 0 789 l = r = 0 790 for i in range(0, 18, 2): 791 l, r = self._bf_encrypt_block(l, r) 792 self.__p[i] = l 793 self.__p[i + 1] = r 794 for i in range(0, 4): 795 for j in range(0, 256, 2): 796 l, r = self._bf_encrypt_block(l, r) 797 self.__s[i][j] = l 798 self.__s[i][j + 1] = r 799 800 def _bf_func(self, x): 801 """ 802 The blowfish round function operating on an integer. 803 """ 804 h = c_add32(self.__s[0][x >> 24], self.__s[1][x >> 16 & 0xff]) 805 return c_add32((h ^ self.__s[2][x >> 8 & 0xff]), self.__s[3][x & 0xff]) 806 807 def _bf_encrypt_block(self, l, r): 808 """ 809 Encrypt a single block consisting of integers *l* and *r*. 810 """ 811 for i in range(0, 16, 2): 812 l ^= self.__p[i] 813 r ^= self._bf_func(l) 814 r ^= self.__p[i+1] 815 l ^= self._bf_func(r) 816 l ^= self.__p[16] 817 r ^= self.__p[17] 818 return (r, l) 819 820 def _bf_decrypt_block(self, l, r): 821 """ 822 Decrypt a single block consisting of integers *l* and *r*. 823 """ 824 for i in range(16, 0, -2): 825 l ^= self.__p[i+1] 826 r ^= self._bf_func(l) 827 r ^= self.__p[i] 828 l ^= self._bf_func(r) 829 l ^= self.__p[1] 830 r ^= self.__p[0] 831 return (r, l) 832 833 def decrypt(self, buf): 834 """ 835 Decrypt a buffer of complete blocks, i.e. of length that is a multiple 836 of the block size returned by the blocksize property. 837 AceBlowfish uses Blowfish in CBC mode with an IV of all zeroes on the 838 first call, and an IV of the last ciphertext block on subsequent calls. 839 Does not remove any padding. 840 """ 841 assert len(buf) % self.blocksize == 0 842 out = [] 843 for i in range(0, len(buf), 8): 844 cl, cr = struct.unpack('<LL', buf[i:i+8]) 845 pl, pr = self._bf_decrypt_block(cl, cr) 846 pl ^= self.__lastcl 847 pr ^= self.__lastcr 848 self.__lastcl = cl 849 self.__lastcr = cr 850 out.append(struct.pack('<LL', pl, pr)) 851 return b''.join(out) 852 853 @property 854 def blocksize(self): 855 """ 856 Return the block size of the decryption engine in bytes. 857 The decrypt() method will only accept buffers containing a multiple of 858 the block size of bytes. 859 """ 860 return 8 861 862 863 864class AceCRC32: 865 """ 866 Calculate an ACE CRC-32 checksum. 867 868 ACE CRC-32 uses the standard CRC-32 polynomial, bit ordering and 869 initialization vector, but does not invert the resulting checksum. 870 This implementation uses :meth:`zlib.crc32` with inverted state, 871 inverted initialization vector and inverted output in order to 872 construct ACE CRC-32 from standard CRC-32. 873 874 >>> crc = AceCRC32() 875 >>> crc += b"12345" 876 >>> crc += b"6789" 877 >>> crc.sum 878 873187033 879 >>> crc == 873187033 880 True 881 """ 882 883 def __init__(self, buf=b''): 884 """ 885 Initialize and add bytes in *buf* into checksum. 886 """ 887 self.__state = 0 888 if len(buf) > 0: 889 self += buf 890 891 def __iadd__(self, buf): 892 """ 893 Adding a buffer of bytes into the checksum, updating the rolling 894 checksum from all previously added buffers. 895 """ 896 self.__state = zlib.crc32(buf, self.__state) 897 return self 898 899 def __eq__(self, other): 900 """ 901 Compare the checksum to a fixed value or another ACE CRC32 object. 902 """ 903 return self.sum == other 904 905 def __format__(self, format_spec): 906 """ 907 Format the checksum for printing. 908 """ 909 return self.sum.__format__(format_spec) 910 911 def __str__(self): 912 """ 913 String representation of object is hex value of checksum. 914 """ 915 return "0x%08x" % self.sum 916 917 @property 918 def sum(self): 919 """ 920 The final checksum. 921 """ 922 return self.__state ^ 0xFFFFFFFF 923 924class AceCRC16(AceCRC32): 925 """ 926 Calculate an ACE CRC-16 checksum, which is actually just the lower 16 bits 927 of an ACE CRC-32. 928 929 >>> crc = AceCRC16() 930 >>> crc += b"12345" 931 >>> crc += b"6789" 932 >>> crc.sum 933 50905 934 >>> crc == 50905 935 True 936 """ 937 def __str__(self): 938 """ 939 String representation of object is hex value of checksum. 940 """ 941 return "0x%04x" % self.sum 942 943 @property 944 def sum(self): 945 """ 946 The checksum. 947 """ 948 return super().sum & 0xFFFF 949 950def ace_crc32(buf): 951 """ 952 Return the ACE CRC-32 checksum of the bytes in *buf*. 953 954 >>> ace_crc32(b"123456789") 955 873187033 956 """ 957 return AceCRC32(buf).sum 958 959def ace_crc16(buf): 960 """ 961 Return the ACE CRC-16 checksum of the bytes in *buf*. 962 963 >>> ace_crc16(b"123456789") 964 50905 965 """ 966 return AceCRC16(buf).sum 967 968 969 970class BitStream: 971 """ 972 Intel-endian 32bit-byte-swapped, MSB first bitstream, reading from an 973 underlying file-like object that does not need to be seekable, but is 974 expected to be a multiple of 4 in length. 975 976 >>> bs = BitStream(io.BytesIO(b'01234567')) 977 >>> bs.peek_bits(31) 978 429463704 979 >>> bs.read_bits(31) 980 429463704 981 >>> bs.skip_bits(3) 982 >>> bs.read_bits(5) 983 27 984 >>> bs.read_golomb_rice(3) 985 20 986 >>> bs.read_golomb_rice(2, True) 987 -2 988 >>> bs.read_knownwidth_uint(10) 989 618 990 >>> bs.read_bits(7) 991 52 992 >>> bs.peek_bits(31) 993 0 994 >>> bs.read_bits(1) 995 Traceback (most recent call last): 996 ... 997 EOFError 998 >>> BitStream(io.BytesIO(b'012')).read_bits(31) 999 Traceback (most recent call last): 1000 ... 1001 ValueError 1002 """ 1003 1004 @staticmethod 1005 def _getbits(value, start, length): 1006 """ 1007 Return *length* bits from byte *value*, starting at position *start*. 1008 Behaviour is undefined for start < 0, length < 0 or start + length > 32. 1009 """ 1010 #assert start >= 0 and length >= 0 and start + length <= 32 1011 mask = ((0xFFFFFFFF << (32 - length)) & 0xFFFFFFFF) >> start 1012 return (value & mask) >> (32 - length - start) 1013 1014 def __init__(self, f): 1015 """ 1016 Initialize BitStream reading from file-like object *f* until EOF. 1017 """ 1018 self.__file = f 1019 self.__buf = array.array('I') 1020 self.__len = 0 # in bits 1021 self.__pos = 0 # in bits 1022 self._refill() 1023 1024 def _refill(self): 1025 """ 1026 Refill the internal buffer with data read from file. 1027 """ 1028 tmpbuf = self.__file.read(FILE_BLOCKSIZE) 1029 if len(tmpbuf) == 0: 1030 raise EOFError("Cannot refill beyond EOF") 1031 if len(tmpbuf) % 4 != 0: 1032 raise ValueError("Truncated 32-bit word from file-like object") 1033 1034 newbuf = self.__buf[-1:] 1035 for i in range(0, len(tmpbuf), 4): 1036 newbuf.append(struct.unpack('<L', tmpbuf[i:i+4])[0]) 1037 if self.__pos > 0: 1038 self.__pos -= (self.__len - 32) 1039 self.__buf = newbuf 1040 self.__len = 32 * len(newbuf) 1041 1042 def skip_bits(self, bits): 1043 """ 1044 Skip *bits* bits in the stream. 1045 Raise EOFError when skipping beyond the end of the input file data. 1046 The pure-python implementation supports skipping arbitrarily many 1047 *bits* while the c implementation is limited to a maximum of 31. 1048 """ 1049 if self.__pos + bits > self.__len: 1050 self._refill() 1051 self.__pos += bits 1052 1053 def peek_bits(self, bits): 1054 """ 1055 Peek at next *bits* bits in the stream without incrementing position. 1056 A maximum of 31 bits beyond the end of the input file data are 1057 guaranteed to be peekable; these bits are always unset. 1058 The pure-python implementation supports peeking arbitrarily many 1059 *bits* while the c implementation is limited to a maximum of 31. 1060 """ 1061 if self.__pos + bits > self.__len: 1062 try: 1063 self._refill() 1064 except EOFError: 1065 if len(self.__buf) * 32 == self.__len: 1066 self.__buf.append(0) 1067 if self.__pos + bits > self.__len + 31: 1068 raise 1069 1070 peeked = min(bits, 32 - (self.__pos % 32)) 1071 res = self._getbits(self.__buf[self.__pos // 32], 1072 self.__pos % 32, peeked) 1073 while bits - peeked >= 32: 1074 res <<= 32 1075 res += self.__buf[(self.__pos + peeked) // 32] 1076 peeked += 32 1077 if bits - peeked > 0: 1078 res <<= bits - peeked 1079 res += self._getbits(self.__buf[(self.__pos + peeked) // 32], 1080 0, bits - peeked) 1081 return res 1082 1083 def read_bits(self, bits): 1084 """ 1085 Read *bits* bits from bitstream and increment position accordingly. 1086 The pure-python implementation supports reading arbitrarily many 1087 *bits* while the c implementation is limited to a maximum of 31. 1088 """ 1089 value = self.peek_bits(bits) 1090 self.skip_bits(bits) 1091 return value 1092 1093 def read_golomb_rice(self, r_bits, signed=False): 1094 """ 1095 Read a Golomb-Rice code with *r_bits* remainder bits and an arbitrary 1096 number of quotient bits from bitstream and return the represented 1097 value. Iff *signed* is True, interpret the lowest order bit as sign 1098 bit and return a signed integer. 1099 """ 1100 if r_bits == 0: 1101 value = 0 1102 else: 1103 assert r_bits > 0 1104 value = self.read_bits(r_bits) 1105 while self.read_bits(1) == 1: 1106 value += 1 << r_bits 1107 if signed == False: 1108 return value 1109 if value & 1: 1110 return - (value >> 1) - 1 1111 else: 1112 return value >> 1 1113 1114 def read_knownwidth_uint(self, bits): 1115 """ 1116 Read an unsigned integer with previously known bit width *bits* from 1117 stream. The most significant bit is not encoded in the bit stream, 1118 because it is always 1. 1119 """ 1120 if bits < 2: 1121 return bits 1122 bits -= 1 1123 return self.read_bits(bits) + (1 << bits) 1124 1125 1126 1127if acebitstream != None: 1128 class BitStream_c(acebitstream.BitStream): 1129 read_golomb_rice = BitStream.read_golomb_rice 1130 read_knownwidth_uint = BitStream.read_knownwidth_uint 1131 1132 BitStream_c.__doc__ = BitStream.__doc__ 1133 BitStream = BitStream_c 1134 1135 1136 1137class AceMode: 1138 """ 1139 Represent and parse compression submode information from a bitstream. 1140 """ 1141 @classmethod 1142 def read_from(cls, bs): 1143 mode = cls(bs.read_bits(8)) 1144 if mode.mode == ACE.MODE_LZ77_DELTA: 1145 mode.delta_dist = bs.read_bits(8) 1146 mode.delta_len = bs.read_bits(17) 1147 elif mode.mode == ACE.MODE_LZ77_EXE: 1148 mode.exe_mode = bs.read_bits(8) 1149 if DEBUG: 1150 eprint(mode) 1151 return mode 1152 1153 def __init__(self, mode): 1154 self.mode = mode 1155 1156 def __str__(self): 1157 args = '' 1158 if self.mode == ACE.MODE_LZ77_DELTA: 1159 args = " delta_dist=%i delta_len=%i" % (self.delta_dist, 1160 self.delta_len) 1161 elif self.mode == ACE.MODE_LZ77_EXE: 1162 args = " exe_mode=%i" % self.exe_mode 1163 return "mode %s(%i)%s" % (ACE.mode_str(self.mode), self.mode, args) 1164 1165 1166 1167class Huffman: 1168 """ 1169 Huffman decoder engine. 1170 """ 1171 1172 class Tree: 1173 """ 1174 Huffman tree reconstructed from bitstream, internally represented by 1175 a table mapping (length-extended) codes to symbols and a table mapping 1176 symbols to bit widths. 1177 """ 1178 def __init__(self, codes, widths, max_width): 1179 self.codes = codes 1180 self.widths = widths 1181 self.max_width = max_width 1182 1183 def read_symbol(self, bs): 1184 """ 1185 Read a single Huffman symbol from BitStream *bs* by peeking the 1186 maximum code length in bits from the bit stream, looking up the 1187 symbol and its width, and finally skipping the actual width of 1188 the code for the symbol in the bit stream. 1189 """ 1190 maxwidth_code = bs.peek_bits(self.max_width) 1191 if maxwidth_code >= len(self.codes): 1192 # Not sure if we could prevent this from happening on malformed 1193 # input by improving how we choose max_width and codes; so far 1194 # only happens on malformed input. 1195 raise CorruptedArchiveError("maxwidth_code >= len(codes)") 1196 symbol = self.codes[maxwidth_code] 1197 bs.skip_bits(self.widths[symbol]) 1198 return symbol 1199 1200 1201 WIDTHWIDTHBITS = 3 1202 MAXWIDTHWIDTH = (1 << WIDTHWIDTHBITS) - 1 1203 1204 @staticmethod 1205 def _quicksort(keys, values): 1206 """ 1207 In-place quicksort of lists *keys* and *values* in descending order of 1208 *keys*. Python uses a stable sorting algorithm, while the 1209 reconstruction of the correct Huffman trees depends on the sorting 1210 being unstable in exactly the way of this quicksort implementation. 1211 1212 >>> k, v = [1, 0, 0, 1, 2, 0, 0], list(range(7)) 1213 >>> Huffman._quicksort(k, v) 1214 >>> (k, v) 1215 ([2, 1, 1, 0, 0, 0, 0], [4, 0, 3, 5, 6, 2, 1]) 1216 """ 1217 def _quicksort_subrange(left, right): 1218 def _list_swap(_list, a, b): 1219 """ 1220 >>> a = list(range(9)) 1221 >>> _list_swap(a, 3, 6) 1222 [0, 1, 2, 6, 4, 5, 3, 7, 8, 9] 1223 """ 1224 _list[a], _list[b] = _list[b], _list[a] 1225 1226 new_left = left 1227 new_right = right 1228 m = keys[right] 1229 while True: 1230 while keys[new_left] > m: 1231 new_left += 1 1232 while keys[new_right] < m: 1233 new_right -= 1 1234 if new_left <= new_right: 1235 _list_swap(keys, new_left, new_right) 1236 _list_swap(values, new_left, new_right) 1237 new_left += 1 1238 new_right -= 1 1239 if new_left >= new_right: 1240 break 1241 if left < new_right: 1242 if left < new_right - 1: 1243 _quicksort_subrange(left, new_right) 1244 else: 1245 if keys[left] < keys[new_right]: 1246 _list_swap(keys, left, new_right) 1247 _list_swap(values, left, new_right) 1248 if right > new_left: 1249 if new_left < right - 1: 1250 _quicksort_subrange(new_left, right) 1251 else: 1252 if keys[new_left] < keys[right]: 1253 _list_swap(keys, new_left, right) 1254 _list_swap(values, new_left, right) 1255 1256 assert len(keys) == len(values) 1257 _quicksort_subrange(0, len(keys) - 1) 1258 1259 @staticmethod 1260 def _make_tree(widths, max_width): 1261 """ 1262 Calculate the list of Huffman codes corresponding to the symbols 1263 implicitly described by the list of *widths* and maximal width 1264 *max_width*, and return a Huffman.Tree object representing the 1265 resulting Huffman tree. 1266 """ 1267 sorted_symbols = list(range(len(widths))) 1268 sorted_widths = list(widths) 1269 Huffman._quicksort(sorted_widths, sorted_symbols) 1270 1271 used = 0 1272 while used < len(sorted_widths) and sorted_widths[used] != 0: 1273 used += 1 1274 1275 if used < 2: 1276 widths[sorted_symbols[0]] = 1 1277 if used == 0: 1278 used += 1 1279 del sorted_symbols[used:] 1280 del sorted_widths[used:] 1281 1282 codes = [] 1283 max_codes = 1 << max_width 1284 for sym, wdt in zip(reversed(sorted_symbols), reversed(sorted_widths)): 1285 if wdt > max_width: 1286 raise CorruptedArchiveError("wdt > max_width") 1287 repeat = 1 << (max_width - wdt) 1288 codes.extend([sym] * repeat) 1289 if len(codes) > max_codes: 1290 raise CorruptedArchiveError("len(codes) > max_codes") 1291 1292 return Huffman.Tree(codes, widths, max_width) 1293 1294 @staticmethod 1295 def read_tree(bs, max_width, num_codes): 1296 """ 1297 Read a Huffman tree consisting of codes and their widths from 1298 BitStream *bs*. The caller specifies the maximum width of a single 1299 code *max_width* and the number of codes *num_codes*; these are 1300 required to reconstruct the Huffman tree from the widths stored in 1301 the bit stream. 1302 """ 1303 num_widths = bs.read_bits(9) + 1 1304 if num_widths > num_codes + 1: 1305 num_widths = num_codes + 1 1306 lower_width = bs.read_bits(4) 1307 upper_width = bs.read_bits(4) 1308 1309 width_widths = [] 1310 width_num_widths = upper_width + 1 1311 for i in range(width_num_widths): 1312 width_widths.append(bs.read_bits(Huffman.WIDTHWIDTHBITS)) 1313 width_tree = Huffman._make_tree(width_widths, Huffman.MAXWIDTHWIDTH) 1314 1315 widths = [] 1316 while len(widths) < num_widths: 1317 symbol = width_tree.read_symbol(bs) 1318 if symbol < upper_width: 1319 widths.append(symbol) 1320 else: 1321 length = bs.read_bits(4) + 4 1322 length = min(length, num_widths - len(widths)) 1323 widths.extend([0] * length) 1324 1325 if upper_width > 0: 1326 for i in range(1, len(widths)): 1327 widths[i] = (widths[i] + widths[i - 1]) % upper_width 1328 1329 for i in range(len(widths)): 1330 if widths[i] > 0: 1331 widths[i] += lower_width 1332 1333 return Huffman._make_tree(widths, max_width) 1334 1335 1336 1337class LZ77: 1338 """ 1339 ACE 1.0 and ACE 2.0 LZ77 mode decompression engine. 1340 1341 Plain LZ77 compression over a Huffman-encoded symbol stream. 1342 """ 1343 1344 class SymbolReader: 1345 """ 1346 Read blocks of Huffman-encoded LZ77 symbols. 1347 Two Huffman trees are used, one for the LZ77 symbols (main codes) and 1348 one for the length parameters (len codes). 1349 """ 1350 1351 def __init__(self): 1352 self.__syms_to_read = 0 1353 1354 def _read_trees(self, bs): 1355 """ 1356 Read the Huffman trees as well as the blocksize from BitStream 1357 *bs*; essentially this starts reading into a next block of symbols. 1358 """ 1359 self.__main_tree = Huffman.read_tree(bs, LZ77.MAXCODEWIDTH, 1360 LZ77.NUMMAINCODES) 1361 self.__len_tree = Huffman.read_tree(bs, LZ77.MAXCODEWIDTH, 1362 LZ77.NUMLENCODES) 1363 self.__syms_to_read = bs.read_bits(15) 1364 1365 def read_main_symbol(self, bs): 1366 """ 1367 Read a main symbol from BitStream *bs*. 1368 """ 1369 if self.__syms_to_read == 0: 1370 self._read_trees(bs) 1371 self.__syms_to_read -= 1 1372 return self.__main_tree.read_symbol(bs) 1373 1374 def read_len_symbol(self, bs): 1375 """ 1376 Read a length symbol from BitStream *bs*. 1377 """ 1378 return self.__len_tree.read_symbol(bs) 1379 1380 1381 class DistHist: 1382 """ 1383 Distance value cache for storing the last SIZE used LZ77 distances. 1384 1385 >>> dh = LZ77.DistHist() 1386 >>> dh.append(1);dh.append(2);dh.append(3);dh.append(4);dh.append(5) 1387 >>> dh.retrieve(2) 1388 3 1389 >>> dh.retrieve(0) 1390 3 1391 >>> dh.retrieve(1) 1392 5 1393 >>> dh.retrieve(1) 1394 3 1395 """ 1396 SIZE = 4 1397 1398 def __init__(self): 1399 self.__hist = [0] * self.SIZE 1400 1401 def append(self, dist): 1402 self.__hist.pop(0) 1403 self.__hist.append(dist) 1404 1405 def retrieve(self, offset): 1406 assert offset >= 0 and offset < self.SIZE 1407 dist = self.__hist.pop(self.SIZE - offset - 1) 1408 self.__hist.append(dist) 1409 return dist 1410 1411 1412 class Dictionary: 1413 """ 1414 LZ77 dictionary. Stores at least the last dictionary-size number of 1415 decompressed bytes and supports the LZ77 copy operation. Also doubles 1416 as decompressed bytes buffer. Consequently, the dictionary will grow 1417 as bytes are appended to it until copyout or copyin are called. 1418 1419 >>> dic = LZ77.Dictionary(4, 8) 1420 >>> dic.append(1); dic.append(2); dic.extend((3,4)) 1421 >>> dic.copy(4, 4) 1422 >>> dic.copyout(8) 1423 [1, 2, 3, 4, 1, 2, 3, 4] 1424 >>> dic.copy(9, 1) 1425 Traceback (most recent call last): 1426 ... 1427 CorruptedArchiveError 1428 """ 1429 def __init__(self, minsize, maxsize): 1430 self.__dicdata = [] 1431 self.__dicsize = minsize 1432 self.__maxsize = maxsize 1433 1434 def set_size(self, dicsize): 1435 """ 1436 Set expected dictionary size for next decompression run. 1437 """ 1438 self.__dicsize = min(max(dicsize, self.__dicsize), self.__maxsize) 1439 1440 def append(self, char): 1441 """ 1442 Append output byte *char* to dictionary. 1443 """ 1444 self.__dicdata.append(char) 1445 1446 def extend(self, buf): 1447 """ 1448 Append output bytes *buf* to dictionary. 1449 """ 1450 self.__dicdata.extend(buf) 1451 1452 def copy(self, dist, n): 1453 """ 1454 Copy *n* previously produced output bytes to end of dictionary, 1455 starting from position *dist* away from the end. 1456 """ 1457 source_pos = len(self.__dicdata) - dist 1458 if source_pos < 0: 1459 raise CorruptedArchiveError("LZ77 copy src out of bounds") 1460 # copy needs to be byte-wise for overlapping src and dst 1461 for i in range(source_pos, source_pos + n): 1462 self.__dicdata.append(self.__dicdata[i]) 1463 1464 def copyin(self, buf): 1465 """ 1466 Copy output bytes produced by other decompression methods 1467 from *buf* into dictionary. This operation may cause the 1468 dictionary to shrink to the indended dictionary size; make sure 1469 to copyout all needed output bytes before calling copyin. 1470 """ 1471 self.extend(buf) 1472 self._truncate() 1473 1474 def copyout(self, n): 1475 """ 1476 Return copy of last *n* appended output bytes for handing them 1477 to the caller. This operation may cause the dictionary to shrink 1478 to the intended dictionary size. 1479 """ 1480 if n > 0: 1481 assert n <= len(self.__dicdata) 1482 chunk = self.__dicdata[-n:] 1483 else: 1484 chunk = [] 1485 self._truncate() 1486 return chunk 1487 1488 def _truncate(self): 1489 # only perform actual truncation when dictionary exceeds 4 times 1490 # it's supposed size in order to prevent excessive data copying 1491 if len(self.__dicdata) > 4 * self.__dicsize: 1492 self.__dicdata = self.__dicdata[-self.__dicsize:] 1493 1494 1495 # 0..255 character literals 1496 # 256..259 copy from dictionary, dist from dist history -1..-4 1497 # 260..282 copy from dictionary, dist 0..22 bits from bitstream 1498 # 283 type code 1499 MAXCODEWIDTH = 11 1500 MAXLEN = 259 1501 MAXDISTATLEN2 = 255 1502 MAXDISTATLEN3 = 8191 1503 MINDICBITS = 10 1504 MAXDICBITS = 22 1505 MINDICSIZE = 1 << MINDICBITS 1506 MAXDICSIZE = 1 << MAXDICBITS 1507 TYPECODE = 260 + MAXDICBITS + 1 1508 NUMMAINCODES = 260 + MAXDICBITS + 2 1509 NUMLENCODES = 256 - 1 1510 1511 def __init__(self): 1512 self.__dictionary = LZ77.Dictionary(LZ77.MINDICSIZE, LZ77.MAXDICSIZE) 1513 1514 def reinit(self): 1515 """ 1516 Reinitialize the LZ77 decompression engine. 1517 Reset all data dependent state to initial values. 1518 """ 1519 self.__symreader = LZ77.SymbolReader() 1520 self.__disthist = LZ77.DistHist() 1521 self.__leftover = [] 1522 1523 def dic_setsize(self, dicsize): 1524 """ 1525 Set the required dictionary size for the next LZ77 decompression run. 1526 """ 1527 self.__dictionary.set_size(dicsize) 1528 1529 def dic_register(self, buf): 1530 """ 1531 Register bytes in *buf* produced by other decompression modes into 1532 the LZ77 dictionary. 1533 """ 1534 self.__dictionary.copyin(buf) 1535 1536 def read(self, bs, want_size): 1537 """ 1538 Read a block of LZ77 compressed data from BitStream *bs*. 1539 Reading will stop when *want_size* output bytes can be provided, 1540 or when a block ends, i.e. when a mode instruction is found. 1541 Returns a tuple of the output byte-like and the mode instruction. 1542 """ 1543 assert want_size > 0 1544 have_size = 0 1545 1546 if len(self.__leftover) > 0: 1547 self.__dictionary.extend(self.__leftover) 1548 have_size += len(self.__leftover) 1549 self.__leftover = [] 1550 1551 next_mode = None 1552 while have_size < want_size: 1553 symbol = self.__symreader.read_main_symbol(bs) 1554 if symbol <= 255: 1555 self.__dictionary.append(symbol) 1556 have_size += 1 1557 elif symbol < LZ77.TYPECODE: 1558 if symbol <= 259: 1559 copy_len = self.__symreader.read_len_symbol(bs) 1560 offset = symbol & 0x03 1561 copy_dist = self.__disthist.retrieve(offset) 1562 if offset > 1: 1563 copy_len += 3 1564 else: 1565 copy_len += 2 1566 else: 1567 copy_dist = bs.read_knownwidth_uint(symbol - 260) 1568 copy_len = self.__symreader.read_len_symbol(bs) 1569 self.__disthist.append(copy_dist) 1570 if copy_dist <= LZ77.MAXDISTATLEN2: 1571 copy_len += 2 1572 elif copy_dist <= LZ77.MAXDISTATLEN3: 1573 copy_len += 3 1574 else: 1575 copy_len += 4 1576 copy_dist += 1 1577 if have_size + copy_len > want_size: 1578 raise CorruptedArchiveError("LZ77 copy exceeds want_size") 1579 self.__dictionary.copy(copy_dist, copy_len) 1580 have_size += copy_len 1581 elif symbol == LZ77.TYPECODE: 1582 next_mode = AceMode.read_from(bs) 1583 break 1584 else: 1585 raise CorruptedArchiveError("LZ77 symbol > LZ77.TYPECODE") 1586 1587 chunk = self.__dictionary.copyout(have_size) 1588 return (chunk, next_mode) 1589 1590 1591 1592class Sound: 1593 """ 1594 ACE 2.0 SOUND mode decompression engine. 1595 1596 Multi-channel audio predictor over Huffman-encoding, resulting in a higher 1597 compression ratio for uncompressed mono/stereo 8/16 bit sound data. 1598 """ 1599 1600 class SymbolReader: 1601 """ 1602 Read blocks of Huffman-encoded SOUND symbols. 1603 For each channel, three Huffman trees are used. 1604 """ 1605 1606 def __init__(self, num_models): 1607 self.__trees = [None] * num_models 1608 self.__syms_to_read = 0 1609 1610 def _read_trees(self, bs): 1611 """ 1612 Read the Huffman trees as well as the blocksize from BitStream 1613 *bs*; essentially this starts reading into a next block of symbols. 1614 """ 1615 for i in range(len(self.__trees)): 1616 self.__trees[i] = Huffman.read_tree(bs, Sound.MAXCODEWIDTH, 1617 Sound.NUMCODES) 1618 self.__syms_to_read = bs.read_bits(15) 1619 1620 def read_symbol(self, bs, model): 1621 """ 1622 Read a symbol from BitStream *bs* using the Huffman tree for model 1623 *model*. 1624 """ 1625 if self.__syms_to_read == 0: 1626 self._read_trees(bs) 1627 self.__syms_to_read -= 1 1628 return self.__trees[model].read_symbol(bs) 1629 1630 1631 def classinit_sound_quantizer(cls): 1632 """ 1633 Decorator that adds the static quantizer table to class *cls*. 1634 """ 1635 cls._quantizer = [0] * 256 1636 for i in range(1, 129): 1637 # [-i] is equivalent to [256 - i] 1638 cls._quantizer[-i] = cls._quantizer[i] = i.bit_length() 1639 return cls 1640 1641 @classinit_sound_quantizer 1642 class Channel: 1643 """ 1644 Decompression parameters and methods for a single audio channel. 1645 """ 1646 def __init__(self, symreader, channel_idx): 1647 """ 1648 Initialize a channel with index *channel_idx*, using symbol 1649 reader *symreader* to fetch new symbols. 1650 """ 1651 self.__symreader = symreader 1652 self.__model_base_idx = 3 * channel_idx 1653 self.__pred_dif_cnt = [0] * 2 1654 self.__last_pred_dif_cnt = [0] * 2 1655 self.__rar_dif_cnt = [0] * 4 1656 self.__rar_coeff = [0] * 4 1657 self.__rar_dif = [0] * 9 1658 self.__byte_count = 0 1659 self.__last_sample = 0 1660 self.__last_delta = 0 1661 self.__adapt_model_cnt = 0 1662 self.__adapt_model_use = 0 1663 self.__get_state = 0 1664 self.__get_code = 0 1665 1666 def _get_symbol(self, bs): 1667 """ 1668 Get next symbol from BitStream *bs*. 1669 """ 1670 model = self.__get_state << 1 1671 if model == 0: 1672 model += self.__adapt_model_use 1673 model += self.__model_base_idx 1674 return self.__symreader.read_symbol(bs, model) 1675 1676 def get(self, bs): 1677 """ 1678 Get next sample, reading from BitStream *bs* if necessary. 1679 """ 1680 if self.__get_state != 2: 1681 self.__get_code = self._get_symbol(bs) 1682 if self.__get_code == Sound.TYPECODE: 1683 return AceMode.read_from(bs) 1684 1685 if self.__get_state == 0: 1686 if self.__get_code >= Sound.RUNLENCODES: 1687 value = self.__get_code - Sound.RUNLENCODES 1688 self.__adapt_model_cnt = \ 1689 (self.__adapt_model_cnt * 7 >> 3) + value 1690 if self.__adapt_model_cnt > 40: 1691 self.__adapt_model_use = 1 1692 else: 1693 self.__adapt_model_use = 0 1694 else: 1695 self.__get_state = 2 1696 elif self.__get_state == 1: 1697 value = self.__get_code 1698 self.__get_state = 0 1699 1700 if self.__get_state == 2: 1701 if self.__get_code == 0: 1702 self.__get_state = 1 1703 else: 1704 self.__get_code -= 1 1705 value = 0 1706 1707 if value & 1: 1708 return 255 - (value >> 1) 1709 else: 1710 return value >> 1 1711 1712 def rar_predict(self): 1713 if self.__pred_dif_cnt[0] > self.__pred_dif_cnt[1]: 1714 return self.__last_sample 1715 else: 1716 return self._get_predicted_sample() 1717 1718 def rar_adjust(self, sample): 1719 self.__byte_count += 1 1720 pred_sample = self._get_predicted_sample() 1721 pred_dif = c_schar(pred_sample - sample) << 3 1722 self.__rar_dif[0] += abs(pred_dif - self.__rar_dif_cnt[0]) 1723 self.__rar_dif[1] += abs(pred_dif + self.__rar_dif_cnt[0]) 1724 self.__rar_dif[2] += abs(pred_dif - self.__rar_dif_cnt[1]) 1725 self.__rar_dif[3] += abs(pred_dif + self.__rar_dif_cnt[1]) 1726 self.__rar_dif[4] += abs(pred_dif - self.__rar_dif_cnt[2]) 1727 self.__rar_dif[5] += abs(pred_dif + self.__rar_dif_cnt[2]) 1728 self.__rar_dif[6] += abs(pred_dif - self.__rar_dif_cnt[3]) 1729 self.__rar_dif[7] += abs(pred_dif + self.__rar_dif_cnt[3]) 1730 self.__rar_dif[8] += abs(pred_dif) 1731 1732 self.__last_delta = c_schar(sample - self.__last_sample) 1733 self.__pred_dif_cnt[0] += self._quantizer[pred_dif >> 3] 1734 self.__pred_dif_cnt[1] += self._quantizer[self.__last_sample-sample] 1735 self.__last_sample = sample 1736 1737 if self.__byte_count & 0x1F == 0: 1738 min_dif = 0xFFFF 1739 for i in reversed(range(9)): 1740 if self.__rar_dif[i] <= min_dif: 1741 min_dif = self.__rar_dif[i] 1742 min_dif_pos = i 1743 self.__rar_dif[i] = 0 1744 if min_dif_pos != 8: 1745 i = min_dif_pos >> 1 1746 if min_dif_pos & 1 == 0: 1747 if self.__rar_coeff[i] >= -16: 1748 self.__rar_coeff[i] -= 1 1749 else: 1750 if self.__rar_coeff[i] <= 16: 1751 self.__rar_coeff[i] += 1 1752 if self.__byte_count & 0xFF == 0: 1753 for i in range(2): 1754 self.__pred_dif_cnt[i] -= self.__last_pred_dif_cnt[i] 1755 self.__last_pred_dif_cnt[i] = self.__pred_dif_cnt[i] 1756 1757 self.__rar_dif_cnt[3] = self.__rar_dif_cnt[2] 1758 self.__rar_dif_cnt[2] = self.__rar_dif_cnt[1] 1759 self.__rar_dif_cnt[1] = self.__last_delta - self.__rar_dif_cnt[0] 1760 self.__rar_dif_cnt[0] = self.__last_delta 1761 1762 def _get_predicted_sample(self): 1763 return c_uchar((8 * self.__last_sample + \ 1764 self.__rar_coeff[0] * self.__rar_dif_cnt[0] + \ 1765 self.__rar_coeff[1] * self.__rar_dif_cnt[1] + \ 1766 self.__rar_coeff[2] * self.__rar_dif_cnt[2] + \ 1767 self.__rar_coeff[3] * self.__rar_dif_cnt[3]) >> 3) 1768 1769 1770 RUNLENCODES = 32 1771 TYPECODE = 256 + RUNLENCODES 1772 NUMCODES = 256 + RUNLENCODES + 1 1773 MAXCODEWIDTH = 10 1774 NUMCHANNELS = (1, 2, 3, 3) 1775 USECHANNELS = ((0, 0, 0, 0), 1776 (0, 1, 0, 1), 1777 (0, 1, 0, 2), 1778 (1, 0, 2, 0)) 1779 1780 def __init__(self): 1781 self.__mode = None 1782 self.__channels = None 1783 1784 def reinit(self, mode): 1785 """ 1786 Reinitialize the SOUND decompression engine. 1787 Reset all data dependent state to initial values. 1788 """ 1789 self.__mode = mode - ACE.MODE_SOUND_8 1790 num_channels = Sound.NUMCHANNELS[self.__mode] 1791 num_models = num_channels * 3 1792 sr = Sound.SymbolReader(num_models) 1793 self.__channels = [self.Channel(sr, i) for i in range(num_channels)] 1794 1795 def read(self, bs, want_size): 1796 """ 1797 Read a block of SOUND compressed data from BitStream *bs*. 1798 Reading will stop when *want_size* output bytes can be provided, 1799 or when a block ends, i.e. when a mode instruction is found. 1800 Returns a tuple of the output byte-like and the mode instruction. 1801 """ 1802 assert want_size > 0 1803 chunk = [] 1804 next_mode = None 1805 for i in range(want_size & 0xFFFFFFFC): 1806 channel = Sound.USECHANNELS[self.__mode][i % 4] 1807 value = self.__channels[channel].get(bs) 1808 if isinstance(value, AceMode): 1809 next_mode = value 1810 break 1811 sample = c_uchar(value + self.__channels[channel].rar_predict()) 1812 chunk.append(sample) 1813 self.__channels[channel].rar_adjust(c_schar(sample)) 1814 return (chunk, next_mode) 1815 1816 1817 1818class Pic: 1819 """ 1820 ACE 2.0 PIC mode decompression engine. 1821 1822 Two-dimensional pixel value predictor over Huffman encoding, resulting in a 1823 higher compression ratio for uncompressed picture data. 1824 """ 1825 1826 class ErrContext: 1827 """ 1828 A prediction error context. 1829 """ 1830 def __init__(self): 1831 self.used_counter = 0 1832 self.predictor_number = 0 1833 self.average_counter = 4 1834 self.error_counters = [0] * 4 1835 1836 1837 class ErrModel: 1838 """ 1839 A prediction error model comprising of N error contexts. 1840 """ 1841 N = 365 1842 1843 def __init__(self): 1844 self.contexts = [Pic.ErrContext() for _ in range(Pic.ErrModel.N)] 1845 1846 1847 def classinit_pic_dif_bit_width(cls): 1848 """ 1849 Decorator that adds the PIC dif_bit_width static table to *cls*. 1850 """ 1851 cls._dif_bit_width = [] 1852 for i in range(0, 128): 1853 cls._dif_bit_width.append((2 * i).bit_length()) 1854 for i in range(-128, 0): 1855 cls._dif_bit_width.append((- 2 * i - 1).bit_length()) 1856 return cls 1857 1858 def classinit_pic_quantizers(cls): 1859 """ 1860 Decorator that adds the PIC quantizer static tables to *cls*. 1861 """ 1862 cls._quantizer = [] 1863 cls._quantizer9 = [] 1864 cls._quantizer81 = [] 1865 for i in range(-255, -20): 1866 cls._quantizer.append(-4) 1867 for i in range(-20, -6): 1868 cls._quantizer.append(-3) 1869 for i in range(-6, -2): 1870 cls._quantizer.append(-2) 1871 for i in range(-2, 0): 1872 cls._quantizer.append(-1) 1873 cls._quantizer.append(0) 1874 for i in range(1, 3): 1875 cls._quantizer.append(1) 1876 for i in range(3, 7): 1877 cls._quantizer.append(2) 1878 for i in range(7, 21): 1879 cls._quantizer.append(3) 1880 for i in range(21, 256): 1881 cls._quantizer.append(4) 1882 for q in cls._quantizer: 1883 cls._quantizer9.append(9 * q) 1884 cls._quantizer81.append(81 * q) 1885 return cls 1886 1887 @classinit_pic_dif_bit_width 1888 @classinit_pic_quantizers 1889 class PixelDecoder: 1890 _producers = [] 1891 1892 @classmethod 1893 def register(cls, othercls): 1894 cls._producers.append(othercls) 1895 return othercls 1896 1897 @classmethod 1898 def read_from(cls, bs): 1899 """ 1900 Read a pixel decoder identifier from BitStream *bs* and return 1901 the appropriate PixelDecoder instance. 1902 """ 1903 try: 1904 return cls._producers[bs.read_bits(2)]() 1905 except IndexError: 1906 raise CorruptedArchiveError("Unknown producer requested") 1907 1908 def shift_pixels(self): 1909 """ 1910 Shift pixels to the left to prepare for the next column. 1911 1912 C A D 1913 B X 1914 """ 1915 self._pixel_c = self._pixel_a 1916 self._pixel_a = self._pixel_d 1917 self._pixel_b = self._pixel_x 1918 1919 def get_context(self): 1920 """ 1921 Calculate the error context to use based on the differences 1922 between the neighbouring pixels D-A, A-C and C-B: 1923 1924 C A D 1925 B X 1926 """ 1927 ctx = self._quantizer81[255 + self._pixel_d - self._pixel_a] + \ 1928 self._quantizer9 [255 + self._pixel_a - self._pixel_c] + \ 1929 self._quantizer [255 + self._pixel_c - self._pixel_b] 1930 return abs(ctx) 1931 1932 def _predict(self, use_predictor): 1933 """ 1934 With X being the current position, the predictors use the pixels 1935 above (A), to the left (B) and in the corner (C): 1936 1937 C A D 1938 B X 1939 """ 1940 if use_predictor == 0: 1941 return self._pixel_a 1942 elif use_predictor == 1: 1943 return self._pixel_b 1944 elif use_predictor == 2: 1945 return (self._pixel_a + self._pixel_b) >> 1 1946 elif use_predictor == 3: 1947 return c_uchar(self._pixel_a + self._pixel_b - self._pixel_c) 1948 1949 def update_pixel_x(self, bs, context): 1950 """ 1951 Read the next data point from BitStream *bs* and store the 1952 resulting pixel X based on ErrContext *context*. 1953 """ 1954 context.used_counter += 1 1955 r = c_div(context.average_counter, context.used_counter) 1956 epsilon = bs.read_golomb_rice(r.bit_length(), signed=True) 1957 predicted = self._predict(context.predictor_number) 1958 self._pixel_x = c_uchar(predicted + epsilon) 1959 1960 context.average_counter += abs(epsilon) 1961 if context.used_counter == 128: 1962 context.used_counter >>= 1 1963 context.average_counter >>= 1 1964 1965 for i in range(len(context.error_counters)): 1966 context.error_counters[i] += \ 1967 self._dif_bit_width[self._pixel_x - self._predict(i)] 1968 if i == 0 or context.error_counters[i] < \ 1969 context.error_counters[best_predictor]: 1970 best_predictor = i 1971 context.predictor_number = best_predictor 1972 1973 if any([ec > 0x7F for ec in context.error_counters]): 1974 for i in range(len(context.error_counters)): 1975 context.error_counters[i] >>= 1 1976 1977 @PixelDecoder.register 1978 class PixelDecoder0(PixelDecoder): 1979 def __init__(self): 1980 self._pixel_a = 0 1981 self._pixel_b = 0 1982 self._pixel_c = 0 1983 self._pixel_x = 0 1984 1985 def update_pixel_d(self, thisplane_d, refplane_d): 1986 self._pixel_d = thisplane_d 1987 1988 def produce(self, refplane_x): 1989 return self._pixel_x 1990 1991 class DifferentialPixelDecoder(PixelDecoder): 1992 def __init__(self): 1993 self._pixel_a = 128 1994 self._pixel_b = 128 1995 self._pixel_c = 128 1996 self._pixel_x = 128 1997 1998 @PixelDecoder.register 1999 class PixelDecoder1(DifferentialPixelDecoder): 2000 def update_pixel_d(self, thisplane_d, refplane_d): 2001 self._pixel_d = c_uchar(128 + thisplane_d - refplane_d) 2002 2003 def produce(self, refplane_x): 2004 return c_uchar(self._pixel_x + refplane_x - 128) 2005 2006 @PixelDecoder.register 2007 class PixelDecoder2(DifferentialPixelDecoder): 2008 def update_pixel_d(self, thisplane_d, refplane_d): 2009 self._pixel_d = c_uchar(128 + thisplane_d - (refplane_d * 11 >> 4)) 2010 2011 def produce(self, refplane_x): 2012 return c_uchar(self._pixel_x + (refplane_x * 11 >> 4) - 128) 2013 2014 2015 def __init__(self): 2016 pass 2017 2018 def reinit(self, bs): 2019 """ 2020 Reinitialize the PIC decompression engine. 2021 Read width and planes from BitStream *bs* and reset all data dependent 2022 state to initial values. 2023 Note that width does not need to be a multiple of planes. 2024 """ 2025 self.__width = bs.read_golomb_rice(12) 2026 self.__planes = bs.read_golomb_rice(2) 2027 self.__errmodel_plane0 = self.ErrModel() 2028 self.__errmodel_plane1toN = self.ErrModel() 2029 self.__prevrow = [0] * (self.__width + self.__planes) 2030 self.__leftover = [] 2031 2032 def _row(self, bs): 2033 """ 2034 Decompress a row of pixels. 2035 """ 2036 # NOTE 2037 # Some indices into row and self.__prevrow are outside of the 2038 # expected range 0..width, as indicated below. Additionally, when 2039 # processing the first row, self.__prevrow is all zeroes. 2040 # Furthermore, self.__width is not necessarily a multiple of 2041 # self.__planes. 2042 row = [0] * (self.__width + self.__planes) 2043 for plane in range(self.__planes): 2044 if plane == 0: 2045 errmodel = self.__errmodel_plane0 2046 decoder = Pic.PixelDecoder0() 2047 else: 2048 errmodel = self.__errmodel_plane1toN 2049 decoder = Pic.PixelDecoder.read_from(bs) 2050 # plane-1 is -1 for first plane 2051 decoder.update_pixel_d(self.__prevrow[plane], 2052 self.__prevrow[plane - 1]) 2053 2054 for col in range(plane, self.__width, self.__planes): 2055 decoder.shift_pixels() 2056 # col+self.__planes is > width for last col in plane 2057 decoder.update_pixel_d(self.__prevrow[col + self.__planes], 2058 self.__prevrow[col + self.__planes - 1]) 2059 context = errmodel.contexts[decoder.get_context()] 2060 decoder.update_pixel_x(bs, context) 2061 # col-1 is -1 for first col in first plane 2062 row[col] = decoder.produce(row[col - 1]) 2063 2064 self.__prevrow = row 2065 return row[:self.__width] 2066 2067 def read(self, bs, want_size): 2068 """ 2069 Read a block of PIC compressed data from BitStream *bs*. 2070 Reading will stop when *want_size* output bytes can be provided, 2071 or when a block ends, i.e. when a mode instruction is found. 2072 Returns a tuple of the output byte-like and the mode instruction. 2073 """ 2074 assert want_size > 0 2075 chunk = [] 2076 next_mode = None 2077 if len(self.__leftover) > 0: 2078 chunk.extend(self.__leftover) 2079 self.__leftover = [] 2080 while len(chunk) < want_size: 2081 if bs.read_bits(1) == 0: 2082 next_mode = AceMode.read_from(bs) 2083 break 2084 data = self._row(bs) 2085 n = min(want_size - len(chunk), len(data)) 2086 if n == len(data): 2087 chunk.extend(data) 2088 else: 2089 chunk.extend(data[0:n]) 2090 self.__leftover = data[n:] 2091 return (chunk, next_mode) 2092 2093 2094 2095class ACE: 2096 """ 2097 Core decompression engine for ACE compression up to version 2.0. 2098 """ 2099 MODE_LZ77 = 0 # LZ77 2100 MODE_LZ77_DELTA = 1 # LZ77 after byte reordering 2101 MODE_LZ77_EXE = 2 # LZ77 after patching JMP/CALL targets 2102 MODE_SOUND_8 = 3 # 8 bit sound compression 2103 MODE_SOUND_16 = 4 # 16 bit sound compression 2104 MODE_SOUND_32A = 5 # 32 bit sound compression, variant 1 2105 MODE_SOUND_32B = 6 # 32 bit sound compression, variant 2 2106 MODE_PIC = 7 # picture compression 2107 MODE_STRINGS = ('LZ77', 'LZ77_DELTA', 'LZ77_EXE', 2108 'SOUND_8', 'SOUND_16', 'SOUND_32A', 'SOUND_32B', 2109 'PIC') 2110 2111 @staticmethod 2112 def mode_str(mode): 2113 try: 2114 return ACE.MODE_STRINGS[mode] 2115 except IndexError: 2116 return '?' 2117 2118 @staticmethod 2119 def decompress_comment(buf): 2120 """ 2121 Decompress an ACE MAIN or FILE comment from bytes *buf* and return the 2122 decompressed bytes. 2123 """ 2124 bs = BitStream(io.BytesIO(buf)) 2125 want_size = bs.read_bits(15) 2126 huff_tree = Huffman.read_tree(bs, LZ77.MAXCODEWIDTH, LZ77.NUMMAINCODES) 2127 comment = [] 2128 htab = [0] * 511 2129 while len(comment) < want_size: 2130 if len(comment) > 1: 2131 hval = comment[-1] + comment[-2] 2132 source_pos = htab[hval] 2133 htab[hval] = len(comment) 2134 else: 2135 source_pos = 0 2136 code = huff_tree.read_symbol(bs) 2137 if code < 256: 2138 comment.append(code) 2139 else: 2140 for i in range(code - 256 + 2): 2141 comment.append(comment[source_pos + i]) 2142 return bytes(comment) 2143 2144 def __init__(self): 2145 self.__lz77 = LZ77() 2146 self.__sound = Sound() 2147 self.__pic = Pic() 2148 2149 def decompress_stored(self, f, filesize, dicsize): 2150 """ 2151 Decompress data compressed using the store method from file-like-object 2152 *f* containing compressed bytes that will be decompressed to *filesize* 2153 bytes. Decompressed data will be yielded in blocks of undefined size 2154 upon availability. Empty files will return without yielding anything. 2155 """ 2156 self.__lz77.dic_setsize(dicsize) 2157 producedsize = 0 2158 while producedsize < filesize: 2159 wantsize = min(filesize - producedsize, FILE_BLOCKSIZE) 2160 outchunk = f.read(wantsize) 2161 if len(outchunk) == 0: 2162 raise CorruptedArchiveError("Truncated stored file") 2163 self.__lz77.dic_register(outchunk) 2164 yield outchunk 2165 producedsize += len(outchunk) 2166 2167 def decompress_lz77(self, f, filesize, dicsize): 2168 """ 2169 Decompress data compressed using the ACE 1.0 legacy LZ77 method from 2170 file-like-object *f* containing compressed bytes that will be 2171 decompressed to *filesize* bytes. Decompressed data will be yielded 2172 in blocks of undefined size upon availability. 2173 """ 2174 self.__lz77.dic_setsize(dicsize) 2175 self.__lz77.reinit() 2176 bs = BitStream(f) 2177 producedsize = 0 2178 while producedsize < filesize: 2179 outchunk, next_mode = self.__lz77.read(bs, filesize) 2180 if next_mode: 2181 raise CorruptedArchiveError("LZ77.TYPECODE in ACE 1.0 LZ77") 2182 yield bytes(outchunk) 2183 producedsize += len(outchunk) 2184 2185 def decompress_blocked(self, f, filesize, dicsize): 2186 """ 2187 Decompress data compressed using the ACE 2.0 blocked method from 2188 file-like-object *f* containing compressed bytes that will be 2189 decompressed to *filesize* bytes. Decompressed data will be yielded 2190 in blocks of undefined size upon availability. 2191 """ 2192 bs = BitStream(f) 2193 self.__lz77.dic_setsize(dicsize) 2194 self.__lz77.reinit() 2195 2196 # LZ77_EXE 2197 exe_leftover = [] 2198 2199 # LZ77_DELTA 2200 last_delta = 0 2201 2202 next_mode = None 2203 mode = AceMode(ACE.MODE_LZ77) 2204 2205 producedsize = 0 2206 while producedsize < filesize: 2207 if next_mode != None: 2208 if mode.mode != next_mode.mode: 2209 if next_mode.mode in (ACE.MODE_SOUND_8, 2210 ACE.MODE_SOUND_16, 2211 ACE.MODE_SOUND_32A, 2212 ACE.MODE_SOUND_32B): 2213 self.__sound.reinit(next_mode.mode) 2214 elif next_mode.mode == ACE.MODE_PIC: 2215 self.__pic.reinit(bs) 2216 2217 mode = next_mode 2218 next_mode = None 2219 2220 outchunk = [] 2221 if mode.mode == ACE.MODE_LZ77_DELTA: 2222 # Preprocessor that rearranges chunks of data and calculates 2223 # differences between byte values, resulting in a higher 2224 # LZ77 compression ratio for some inputs. 2225 delta = [] 2226 while len(delta) < mode.delta_len: 2227 chunk, nm = self.__lz77.read(bs, 2228 mode.delta_len - len(delta)) 2229 if len(delta) == 0: 2230 # avoid costly copy 2231 delta = chunk 2232 else: 2233 delta.extend(chunk) 2234 if nm != None: 2235 if next_mode: 2236 raise CorruptedArchiveError("DELTA clobbers mode") 2237 next_mode = nm 2238 if len(delta) == 0: 2239 break 2240 if len(delta) == 0 and next_mode != None: 2241 continue 2242 2243 for i in range(len(delta)): 2244 delta[i] = c_uchar(delta[i] + last_delta) 2245 last_delta = delta[i] 2246 2247 delta_plane = 0 2248 delta_plane_pos = 0 2249 delta_plane_size = mode.delta_len // mode.delta_dist 2250 while delta_plane_pos < delta_plane_size: 2251 while delta_plane < mode.delta_len: 2252 outchunk.append(delta[delta_plane + delta_plane_pos]) 2253 delta_plane += delta_plane_size 2254 delta_plane = 0 2255 delta_plane_pos += 1 2256 # end of ACE.MODE_LZ77_DELTA 2257 2258 elif mode.mode in (ACE.MODE_LZ77, ACE.MODE_LZ77_EXE): 2259 if len(exe_leftover) > 0: 2260 outchunk.extend(exe_leftover) 2261 exe_leftover = [] 2262 chunk, next_mode = self.__lz77.read(bs, 2263 filesize - producedsize - len(outchunk)) 2264 outchunk.extend(chunk) 2265 2266 if mode.mode == ACE.MODE_LZ77_EXE: 2267 # Preprocessor that adjusts target addresses of 2268 # x86 JMP and CALL instructions in order to achieve a 2269 # higher LZ77 compression ratio for executables. 2270 it = iter(range(len(outchunk))) 2271 for i in it: 2272 if i + 4 >= len(outchunk): 2273 break 2274 if outchunk[i] == 0xE8: # CALL rel16/rel32 2275 pos = producedsize + i 2276 if mode.exe_mode == 0: 2277 # rel16 2278 #assert i + 2 < len(outchunk) 2279 rel16 = outchunk[i+1] + (outchunk[i+2] << 8) 2280 rel16 = (rel16 - pos) & 0xFFFF 2281 outchunk[i+1] = rel16 & 0xFF 2282 outchunk[i+2] = (rel16 >> 8) & 0xFF 2283 next(it); next(it) 2284 else: 2285 # rel32 2286 #assert i + 4 < len(outchunk) 2287 rel32 = outchunk[i+1] + \ 2288 (outchunk[i+2] << 8) + \ 2289 (outchunk[i+3] << 16) + \ 2290 (outchunk[i+4] << 24) 2291 rel32 = (rel32 - pos) & 0xFFFFFFFF 2292 outchunk[i+1] = rel32 & 0xFF 2293 outchunk[i+2] = (rel32 >> 8) & 0xFF 2294 outchunk[i+3] = (rel32 >> 16) & 0xFF 2295 outchunk[i+4] = (rel32 >> 24) & 0xFF 2296 next(it); next(it); next(it); next(it) 2297 elif outchunk[i] == 0xE9: # JMP rel16/rel32 2298 pos = producedsize + i 2299 # rel16 2300 #assert i + 2 < len(outchunk) 2301 rel16 = outchunk[i+1] + (outchunk[i+2] << 8) 2302 rel16 = (rel16 - pos) & 0xFFFF 2303 outchunk[i+1] = rel16 & 0xFF 2304 outchunk[i+2] = (rel16 >> 8) & 0xFF 2305 next(it); next(it) 2306 # store max 4 bytes for next loop; this can happen when 2307 # changing between different exe modes after the opcode 2308 # but before completing the machine instruction 2309 for i in it: 2310 #assert i + 4 >= len(outchunk) 2311 if outchunk[i] == 0xE8 or outchunk[i] == 0xE9: 2312 exe_leftover = outchunk[i:] 2313 outchunk = outchunk[:i] 2314 # end of ACE.MODE_LZ77_EXE 2315 # end of ACE.MODE_LZ77 or ACE.MODE_LZ77_EXE 2316 2317 elif mode.mode in (ACE.MODE_SOUND_8, ACE.MODE_SOUND_16, 2318 ACE.MODE_SOUND_32A, ACE.MODE_SOUND_32B): 2319 outchunk, next_mode = self.__sound.read(bs, 2320 filesize - producedsize) 2321 self.__lz77.dic_register(outchunk) 2322 # end of ACE.MODE_SOUND_* 2323 2324 elif mode.mode == ACE.MODE_PIC: 2325 outchunk, next_mode = self.__pic.read(bs, 2326 filesize - producedsize) 2327 self.__lz77.dic_register(outchunk) 2328 # end of ACE.MODE_PIC 2329 2330 else: 2331 raise CorruptedArchiveError("unknown mode: %s" % mode) 2332 2333 yield bytes(outchunk) 2334 producedsize += len(outchunk) 2335 # end of block loop 2336 return producedsize 2337 2338 2339 2340class Header: 2341 """ 2342 Base class for all ACE file format headers. 2343 Header classes are dumb by design and only serve as fancy structs. 2344 """ 2345 MAGIC = b'**ACE**' 2346 2347 TYPE_MAIN = 0 2348 TYPE_FILE32 = 1 2349 TYPE_RECOVERY32 = 2 2350 TYPE_FILE64 = 3 2351 TYPE_RECOVERY64A = 4 2352 TYPE_RECOVERY64B = 5 2353 TYPE_STRINGS = ('MAIN', 'FILE32', 'RECOVERY32', 2354 'FILE64', 'RECOVERY64A', 'RECOVERY64B') 2355 2356 FLAG_ADDSIZE = 1 << 0 # 1 iff addsize field present MFR 2357 FLAG_COMMENT = 1 << 1 # 1 iff comment present MF- 2358 FLAG_64BIT = 1 << 2 # 1 iff 64bit addsize field -FR 2359 FLAG_V20FORMAT = 1 << 8 # 1 iff ACE 2.0 format M-- 2360 FLAG_SFX = 1 << 9 # 1 iff self extracting archive M-- 2361 FLAG_LIMITSFXJR = 1 << 10 # 1 iff dict size limited to 256K M-- 2362 FLAG_NTSECURITY = 1 << 10 # 1 iff NTFS security data present -F- 2363 FLAG_MULTIVOLUME = 1 << 11 # 1 iff archive has multiple volumes M-- 2364 FLAG_ADVERT = 1 << 12 # 1 iff advert string present M-- 2365 FLAG_CONTPREV = 1 << 12 # 1 iff continued from previous volume -F- 2366 FLAG_RECOVERY = 1 << 13 # 1 iff recovery record present M-- 2367 FLAG_CONTNEXT = 1 << 13 # 1 iff continued in next volume -F- 2368 FLAG_LOCKED = 1 << 14 # 1 iff archive is locked M-- 2369 FLAG_PASSWORD = 1 << 14 # 1 iff password encrypted -F- 2370 FLAG_SOLID = 1 << 15 # 1 iff archive is solid MF- 2371 FLAG_STRINGS_M = ('ADDSIZE', 'COMMENT', '4', '8', 2372 '16', '32', '64', '128', 2373 'V20FORMAT', 'SFX', 'LIMITSFXJR', 'MULTIVOLUME', 2374 'ADVERT', 'RECOVERY', 'LOCKED', 'SOLID') 2375 FLAG_STRINGS_F = ('ADDSIZE', 'COMMENT', '64BIT', '8', 2376 '16', '32', '64', '128', 2377 '256', '512', 'NTSECURITY', '2048', 2378 'CONTPREV', 'CONTNEXT', 'PASSWORD', 'SOLID') 2379 FLAG_STRINGS_R = ('ADDSIZE', '2', '64BIT', '8', 2380 '16', '32', '64', '128', 2381 '256', '512', '1024', '2048', 2382 '4096', '8192', '16384', '32768') 2383 FLAG_STRINGS_BYTYPE = (FLAG_STRINGS_M, FLAG_STRINGS_F, FLAG_STRINGS_R, 2384 FLAG_STRINGS_F, FLAG_STRINGS_R, FLAG_STRINGS_R) 2385 2386 HOST_MSDOS = 0 2387 HOST_OS2 = 1 2388 HOST_WIN32 = 2 2389 HOST_UNIX = 3 2390 HOST_MAC_OS = 4 2391 HOST_WIN_NT = 5 2392 HOST_PRIMOS = 6 2393 HOST_APPLE_GS = 7 2394 HOST_ATARI = 8 2395 HOST_VAX_VMS = 9 2396 HOST_AMIGA = 10 2397 HOST_NEXT = 11 2398 HOST_LINUX = 12 2399 HOST_STRINGS = ('MS-DOS', 'OS/2', 'Win32', 'Unix', 'Mac OS', 2400 'Win NT', 'Primos', 'Apple GS', 'ATARI', 'VAX VMS', 2401 'AMIGA', 'NeXT', 'Linux') 2402 2403 COMP_STORED = 0 2404 COMP_LZ77 = 1 2405 COMP_BLOCKED = 2 2406 COMP_STRINGS = ('stored', 'lz77', 'blocked') 2407 2408 QUAL_NONE = 0 2409 QUAL_FASTEST = 1 2410 QUAL_FAST = 2 2411 QUAL_NORMAL = 3 2412 QUAL_GOOD = 4 2413 QUAL_BEST = 5 2414 QUAL_STRINGS = ('store', 'fastest', 'fast', 'normal', 'good', 'best') 2415 2416 # winnt.h 2417 ATTR_READONLY = 0x00000001 2418 ATTR_HIDDEN = 0x00000002 2419 ATTR_SYSTEM = 0x00000004 2420 ATTR_VOLUME_ID = 0x00000008 2421 ATTR_DIRECTORY = 0x00000010 2422 ATTR_ARCHIVE = 0x00000020 2423 ATTR_DEVICE = 0x00000040 2424 ATTR_NORMAL = 0x00000080 2425 ATTR_TEMPORARY = 0x00000100 2426 ATTR_SPARSE_FILE = 0x00000200 2427 ATTR_REPARSE_POINT = 0x00000400 2428 ATTR_COMPRESSED = 0x00000800 2429 ATTR_OFFLINE = 0x00001000 2430 ATTR_NOT_CONTENT_INDEXED = 0x00002000 2431 ATTR_ENCRYPTED = 0x00004000 2432 ATTR_INTEGRITY_STREAM = 0x00008000 2433 ATTR_VIRTUAL = 0x00010000 2434 ATTR_NO_SCRUB_DATA = 0x00020000 2435 ATTR_EA = 0x00040000 2436 ATTR_STRINGS = ('READONLY', 'HIDDEN', 'SYSTEM', 'VOLUME_ID', 2437 'DIRECTORY', 'ARCHIVE', 'DEVICE', 'NORMAL', 2438 'TEMPORARY', 'SPARSE_FILE', 2439 'REPARSE_POINT', 'COMPRESSED', 2440 'OFFLINE', 'NOT_CONTENT_INDEXED', 2441 'ENCRYPTED', 'INTEGRITY_STREAM', 2442 'VIRTUAL', 'NO_SCRUB_DATA', 'EA') 2443 2444 @staticmethod 2445 def _format_bitfield(strings, field): 2446 labels = [] 2447 for i in range(field.bit_length()): 2448 bit = 1 << i 2449 if field & bit == bit: 2450 try: 2451 labels.append(strings[i]) 2452 except IndexError: 2453 labels.append(str(bit)) 2454 return '|'.join(labels) 2455 2456 def __init__(self, crc, size, type, flags): 2457 self.hdr_crc = crc # uint16 header crc without crc,sz 2458 self.hdr_size = size # uint16 header size without crc,sz 2459 self.hdr_type = type # uint8 2460 self.hdr_flags = flags # uint16 2461 2462 def __str__(self): 2463 return """header 2464 hdr_crc 0x%04x 2465 hdr_size %i 2466 hdr_type 0x%02x %s 2467 hdr_flags 0x%04x %s""" % ( 2468 self.hdr_crc, 2469 self.hdr_size, 2470 self.hdr_type, self.hdr_type_str, 2471 self.hdr_flags, self.hdr_flags_str) 2472 2473 def flag(self, flag): 2474 return self.hdr_flags & flag == flag 2475 2476 @property 2477 def hdr_type_str(self): 2478 try: 2479 return Header.TYPE_STRINGS[self.hdr_type] 2480 except IndexError: 2481 return '?' 2482 2483 @property 2484 def hdr_flags_str(self): 2485 try: 2486 strings = self.FLAG_STRINGS_BYTYPE[self.hdr_type] 2487 return self._format_bitfield(strings, self.hdr_flags) 2488 except IndexError: 2489 return '?' 2490 2491 2492 2493class UnknownHeader(Header): 2494 pass 2495 2496 2497 2498class MainHeader(Header): 2499 def __init__(self, *args): 2500 super().__init__(*args) 2501 self.magic = None # uint8[7] **ACE** 2502 self.eversion = None # uint8 extract version 2503 self.cversion = None # uint8 creator version 2504 self.host = None # uint8 platform 2505 self.volume = None # uint8 volume number 2506 self.datetime = None # uint32 date/time in MS-DOS format 2507 self.reserved1 = None # uint8[8] 2508 self.advert = b'' # [uint8] optional 2509 self.comment = b'' # [uint16] optional, compressed 2510 self.reserved2 = None # [?] optional 2511 2512 def __str__(self): 2513 return super().__str__() + """ 2514 magic %r 2515 eversion %i %s 2516 cversion %i %s 2517 host 0x%02x %s 2518 volume %i 2519 datetime 0x%08x %s 2520 reserved1 %02x %02x %02x %02x %02x %02x %02x %02x 2521 advert %r 2522 comment %r 2523 reserved2 %r""" % ( 2524 self.magic, 2525 self.eversion, self.eversion/10, 2526 self.cversion, self.cversion/10, 2527 self.host, self.host_str, 2528 self.volume, 2529 self.datetime, 2530 _dt_fromdos(self.datetime).strftime('%Y-%m-%d %H:%M:%S'), 2531 self.reserved1[0], self.reserved1[1], 2532 self.reserved1[2], self.reserved1[3], 2533 self.reserved1[4], self.reserved1[5], 2534 self.reserved1[6], self.reserved1[7], 2535 self.advert, 2536 self.comment, 2537 self.reserved2) 2538 2539 @property 2540 def host_str(self): 2541 try: 2542 return Header.HOST_STRINGS[self.host] 2543 except IndexError: 2544 return '?' 2545 2546 2547 2548class FileHeader(Header): 2549 def __init__(self, *args): 2550 super().__init__(*args) 2551 self.packsize = None # uint32|64 packed size 2552 self.origsize = None # uint32|64 original size 2553 self.datetime = None # uint32 ctime 2554 self.attribs = None # uint32 file attributes 2555 self.crc32 = None # uint32 checksum over compressed file 2556 self.comptype = None # uint8 compression type 2557 self.compqual = None # uint8 compression quality 2558 self.params = None # uint16 decompression parameters 2559 self.reserved1 = None # uint16 2560 self.filename = None # [uint16] 2561 self.comment = b'' # [uint16] optional, compressed 2562 self.ntsecurity = b'' # [uint16] optional 2563 self.reserved2 = None # ? 2564 self.dataoffset = None # position of data after hdr 2565 2566 def __str__(self): 2567 return super().__str__() + """ 2568 packsize %i 2569 origsize %i 2570 datetime 0x%08x %s 2571 attribs 0x%08x %s 2572 crc32 0x%08x 2573 comptype 0x%02x %s 2574 compqual 0x%02x %s 2575 params 0x%04x 2576 reserved1 0x%04x 2577 filename %r 2578 comment %r 2579 ntsecurity %r 2580 reserved2 %r""" % ( 2581 self.packsize, 2582 self.origsize, 2583 self.datetime, 2584 _dt_fromdos(self.datetime).strftime('%Y-%m-%d %H:%M:%S'), 2585 self.attribs, self.attribs_str, 2586 self.crc32, 2587 self.comptype, self.comptype_str, 2588 self.compqual, self.compqual_str, 2589 self.params, 2590 self.reserved1, 2591 self.filename, 2592 self.comment, 2593 self.ntsecurity, 2594 self.reserved2) 2595 2596 def attrib(self, attrib): 2597 return self.attribs & attrib == attrib 2598 2599 @property 2600 def attribs_str(self): 2601 return self._format_bitfield(Header.ATTR_STRINGS, self.attribs) 2602 2603 @property 2604 def comptype_str(self): 2605 try: 2606 return Header.COMP_STRINGS[self.comptype] 2607 except IndexError: 2608 return '?' 2609 2610 @property 2611 def compqual_str(self): 2612 try: 2613 return Header.QUAL_STRINGS[self.compqual] 2614 except IndexError: 2615 return '?' 2616 2617 2618 2619class RecoveryHeader(Header): 2620 def __init__(self, *args): 2621 super().__init__(*args) 2622 # R32 R64A R64B 2623 self.rcvrsize = None # uint32 uint64 uint64 recovery data size 2624 self.magic = None # uint8[7] **ACE** 2625 self.relstart = None # uint32 uint64 uint64 ? 2626 self.cluster = None # uint32 uint32 - ? 2627 self.sectors = None # - - uint16 ? 2628 self.spc = None # - - uint16 ? 2629 self.clustersize = None # uint32 uint32 uint32 ? 2630 self.rcvrcrc = None # uint16 uint16 - recovery data crc 2631 2632 def __str__(self): 2633 segments = [super().__str__()] 2634 segments.append(""" 2635 rcvrsize %i 2636 magic %r 2637 relstart %i""" % ( 2638 self.rcvrsize, 2639 self.magic, 2640 self.relstart)) 2641 if self.hdr_type in (Header.TYPE_RECOVERY32, 2642 Header.TYPE_RECOVERY64A): 2643 segments.append(""" 2644 cluster %i 2645 clustersize %i 2646 rcvrcrc 0x%02x""" % ( 2647 self.cluster, 2648 self.clustersize, 2649 self.rcvrcrc)) 2650 elif self.hdr_type == Header.TYPE_RECOVERY64B: 2651 segments.append(""" 2652 sectors %i 2653 spc %i 2654 clustersize %i""" % ( 2655 self.sectors, 2656 self.spc, 2657 self.clustersize)) 2658 return ''.join(segments) 2659 2660 2661 2662class AceError(Exception): 2663 """ 2664 Base class for all :mod:`acefile` exceptions. 2665 """ 2666 pass 2667 2668class MainHeaderNotFoundError(AceError): 2669 """ 2670 The main ACE header marked by the magic bytes ``**ACE**`` could not be 2671 found. 2672 Either the *search* argument was to small or the archive is not an ACE 2673 format archive. 2674 """ 2675 pass 2676 2677class MultiVolumeArchiveError(AceError): 2678 """ 2679 A multi-volume archive was expected but a normal archive was found, or 2680 mismatching volumes were provided, or while reading a member from a 2681 multi-volume archive, the member headers indicate that the member 2682 continues in the next volume, but no next volume was found or provided. 2683 """ 2684 pass 2685 2686class CorruptedArchiveError(AceError): 2687 """ 2688 Archive is corrupted. Either a header or data CRC check failed, an invalid 2689 value was read from the archive or the archive is truncated. 2690 """ 2691 pass 2692 2693class EncryptedArchiveError(AceError): 2694 """ 2695 Archive member is encrypted but either no password was provided, or 2696 decompression failed with the given password. 2697 Also raised when processing an encrypted solid archive member out of order, 2698 when any previous archive member uses a different password than the archive 2699 member currently being accessed. 2700 2701 .. note:: 2702 2703 Due to the lack of a password verifier in the ACE file format, there is 2704 no straightforward way to distinguish a wrong password from a corrupted 2705 archive. If the CRC check of an encrypted archive member fails or an 2706 :class:`CorruptedArchiveError` is encountered during decompression, it 2707 is assumed that the password was wrong and as a consequence, 2708 :class:`EncryptedArchiveError` is raised. 2709 """ 2710 pass 2711 2712class UnknownCompressionMethodError(AceError): 2713 """ 2714 Data was compressed using an unknown compression method and therefore 2715 cannot be decompressed using this implementation. This should not happen 2716 for ACE 1.0 or ACE 2.0 archives since this implementation implements all 2717 existing compression methods. 2718 """ 2719 pass 2720 2721 2722 2723class AceMember: 2724 """ 2725 Represents a single archive member, potentially spanning multiple 2726 archive volumes. 2727 :class:`AceMember` is not directly instantiated; instead, instances are 2728 returned by :meth:`AceArchive.getmember` and :meth:`AceArchive.getmembers`. 2729 """ 2730 2731 # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247 2732 if platform.system() == 'Windows': 2733 RESERVED_NAMES = ('CON', 'PRN', 'AUX', 'NUL', 'COM1', 'COM2', 'COM3', 2734 'COM4', 'COM5', 'COM6', 'COM7', 'COM8', 'COM9', 2735 'LPT1', 'LPT2', 'LPT3', 'LPT4', 'LPT5', 'LPT6', 2736 'LPT7', 'LPT8', 'LPT9') 2737 else: 2738 RESERVED_NAMES = () 2739 2740 RESERVED_CHARS = ':<>"?*|' + ''.join([chr(x) for x in range(1,32)]) 2741 TRANSLATION_TAB = str.maketrans(dict.fromkeys(RESERVED_CHARS)) 2742 2743 @staticmethod 2744 def _sanitize_filename(filename): 2745 """ 2746 Decode and sanitize filename for security and platform independence. 2747 Returns either a sanitized relative path, or an empty string. 2748 This python implementation was not vulnerable to CVE-2018-20250, but we 2749 include a test vector to make sure anyway. 2750 While the author believes this sanitization function to be safe, it is 2751 entirely possible that it is not. For maximum safety against path 2752 traversal attacks, for instance when unpacking malicious code, you may 2753 want to ignore the filenames in the archive headers and instead 2754 generate your own filename for each archive member. 2755 2756 >>> AceMember._sanitize_filename(b'a.exe\\0b.txt') 2757 'a.exe' 2758 >>> AceMember._sanitize_filename(b'\\\\etc\\\\foo/bar\\\\baz.txt') 2759 'etc/foo/bar/baz.txt' 2760 >>> AceMember._sanitize_filename(b'a/b/../b/.//.././c/.//../d/file.txt') 2761 'a/d/file.txt' 2762 >>> AceMember._sanitize_filename(b'/etc/passwd') 2763 'etc/passwd' 2764 >>> AceMember._sanitize_filename(b'.././.././.././.././../etc/passwd') 2765 'etc/passwd' 2766 >>> AceMember._sanitize_filename(b'C:\\\\Windows\\\\foo.exe') 2767 'C/Windows/foo.exe' 2768 >>> AceMember._sanitize_filename(b'\\\\\\\\server\\\\share\\\\file') 2769 'server/share/file' 2770 >>> AceMember._sanitize_filename(b'\\\\\\\\.\\\\CdRom0') 2771 'CdRom0' 2772 >>> AceMember._sanitize_filename(b'\\\\\\\\?\\\\raw\\\\path') 2773 'raw/path' 2774 >>> AceMember._sanitize_filename(b'hello\x05world') 2775 'helloworld' 2776 >>> AceMember._sanitize_filename(b'.././.././.././.././../etc/../') 2777 '' 2778 >>> AceMember._sanitize_filename(b'c:\\\\c:\\\\CVE-2018-20250\\\\p.lnk') 2779 'c/c/CVE-2018-20250/p.lnk' 2780 """ 2781 filename = filename.decode('utf-8', errors='replace') 2782 # treat null byte as filename terminator 2783 nullbyte = filename.find(chr(0)) 2784 if nullbyte >= 0: 2785 filename = filename[0:nullbyte] 2786 # ensure path separators are consistent with current platform 2787 if os.sep != '/': 2788 filename = filename.replace('/', os.sep) 2789 if os.sep != '\\': 2790 filename = filename.replace('\\', os.sep) 2791 # eliminate characters illegal on some platforms, including some 2792 # characters that are relevant for path traversal attacks (i.e. colon) 2793 filename = filename.translate(AceMember.TRANSLATION_TAB) 2794 # first eliminate all /./, foo/../ and similar sequences, then remove 2795 # all remaining .. labels in order to avoid path traversal attacks but 2796 # still allow a safe subset of dot syntax in the filename 2797 filename = os.path.normpath(filename) 2798 escsep = re.escape(os.sep) 2799 pattern = r'(^|%s)(?:\.\.(?:%s|$))+' % (escsep, escsep) 2800 filename = re.sub(pattern, r'\1', filename) 2801 # remove leading path separators to ensure a relative path 2802 filename = filename.lstrip(os.sep) 2803 # avoid reserved file names 2804 if filename in AceMember.RESERVED_NAMES: 2805 return '_' + filename 2806 return filename 2807 2808 def __init__(self, idx, filehdrs, f): 2809 """ 2810 Initialize an :class:`AceMember` object with index within archive *idx*, 2811 initial file header *filehdr* and underlying file-like object *f*. 2812 """ 2813 self._idx = idx 2814 self._file = f 2815 self._headers = filehdrs 2816 self.__attribs = filehdrs[0].attribs 2817 self.__comment = filehdrs[0].comment.decode('utf-8', 2818 errors='replace') 2819 self.__crc32 = filehdrs[-1].crc32 2820 self.__comptype = filehdrs[0].comptype 2821 self.__compqual = filehdrs[0].compqual 2822 self.__datetime = _dt_fromdos(filehdrs[0].datetime) 2823 self.__dicsizebits = (filehdrs[0].params & 15) + 10 2824 self.__dicsize = 1 << self.__dicsizebits 2825 self.__raw_filename = filehdrs[0].filename 2826 self.__filename = self._sanitize_filename(filehdrs[0].filename) 2827 if self.__filename == '': 2828 self.__filename = 'file%04i' % self._idx 2829 self.__ntsecurity = filehdrs[0].ntsecurity 2830 self.__size = filehdrs[0].origsize 2831 self.__packsize = 0 2832 for hdr in filehdrs: 2833 self.__packsize += hdr.packsize 2834 2835 def is_dir(self): 2836 """ 2837 True iff :class:`AceMember` instance describes a directory. 2838 """ 2839 return self.attribs & Header.ATTR_DIRECTORY != 0 2840 2841 def is_enc(self): 2842 """ 2843 True iff :class:`AceMember` instance describes an encrypted archive 2844 member. 2845 """ 2846 return self._headers[0].flag(Header.FLAG_PASSWORD) 2847 2848 def is_reg(self): 2849 """ 2850 True iff :class:`AceMember` instance describes a regular file. 2851 """ 2852 return not self.is_dir() 2853 2854 @property 2855 def attribs(self): 2856 """ 2857 DOS/Windows file attribute bit field, as :class:`int`, 2858 as produced by the Windows :func:`GetFileAttributes` API. 2859 """ 2860 return self.__attribs 2861 2862 @property 2863 def comment(self): 2864 """ 2865 File-level comment, as :class:`str`. 2866 If absent, empty :class:`str`. 2867 """ 2868 return self.__comment 2869 2870 @property 2871 def comptype(self): 2872 """ 2873 Compression type used; one of 2874 :data:`COMP_STORED`, 2875 :data:`COMP_LZ77` or 2876 :data:`COMP_BLOCKED`. 2877 """ 2878 return self.__comptype 2879 2880 @property 2881 def compqual(self): 2882 """ 2883 Compression quality used; one of 2884 :data:`QUAL_NONE`, 2885 :data:`QUAL_FASTEST`, 2886 :data:`QUAL_FAST`, 2887 :data:`QUAL_NORMAL`, 2888 :data:`QUAL_GOOD` or 2889 :data:`QUAL_BEST`. 2890 """ 2891 return self.__compqual 2892 2893 @property 2894 def crc32(self): 2895 """ 2896 ACE CRC-32 checksum of decompressed data as recorded in the archive, 2897 as :class:`int`. 2898 ACE CRC-32 is the bitwise inverse of standard CRC-32. 2899 """ 2900 return self.__crc32 2901 2902 @property 2903 def datetime(self): 2904 """ 2905 Timestamp as recorded in the archive, as :class:`datetime.datetime` 2906 instance. 2907 """ 2908 return self.__datetime 2909 2910 @property 2911 def dicsize(self): 2912 """ 2913 LZ77 dictionary size required for extraction of this archive member 2914 in literal symbols, ranging from 1K to 4M. 2915 """ 2916 return 1 << self.__dicsizebits 2917 2918 @property 2919 def dicsizebits(self): 2920 """ 2921 LZ77 dictionary size bit length, i.e. the base-two logarithm of the 2922 dictionary size required for extraction of this archive member. 2923 """ 2924 return self.__dicsizebits 2925 2926 @property 2927 def filename(self): 2928 """ 2929 Sanitized filename, as :class:`str`, safe for use with file 2930 operations on the current platform. 2931 """ 2932 return self.__filename 2933 2934 @property 2935 def ntsecurity(self): 2936 """ 2937 NT security descriptor as :class:`bytes`, describing the owner, primary 2938 group and discretionary access control list (DACL) of the archive 2939 member, as produced by the Windows :func:`GetFileSecurity` API with the 2940 :data:`OWNER_SECURITY_INFORMATION`, 2941 :data:`GROUP_SECURITY_INFORMATION` and 2942 :data:`DACL_SECURITY_INFORMATION` flags set. 2943 If absent, empty :class:`bytes`. 2944 """ 2945 return self.__ntsecurity 2946 2947 @property 2948 def packsize(self): 2949 """ 2950 Size before decompression (packed size). 2951 """ 2952 return self.__packsize 2953 2954 @property 2955 def raw_filename(self): 2956 """ 2957 Raw, unsanitized filename, as :class:`bytes`, not safe for use with 2958 file operations and possibly using path syntax from other platforms. 2959 """ 2960 return self.__raw_filename 2961 2962 @property 2963 def size(self): 2964 """ 2965 Size after decompression (original size). 2966 """ 2967 return self.__size 2968 2969 def __str__(self): 2970 return "member idx %i sz %i/%i type %i qual %i: %s" % ( 2971 self._idx, self.packsize, self.size, 2972 self.comptype, self.compqual, 2973 self.filename) 2974 2975 2976 2977class AceVolume: 2978 """ 2979 Parse and represent a single archive volume. 2980 """ 2981 2982 def __init__(self, file, mode='r', *, search=524288, _idx=0, _am=None): 2983 if mode != 'r': 2984 raise NotImplementedError("mode != 'r' not implemented") 2985 if isinstance(file, str): 2986 self.__file = builtins.open(file, 'rb') 2987 self.__filename = file 2988 else: 2989 if not file.seekable(): 2990 raise TypeError("file must be filename or " 2991 "seekable file-like object") 2992 self.__file = file 2993 self.__filename = '-' 2994 self.__file.seek(0, 2) 2995 self.__filesize = self.__file.tell() 2996 self.__main_header = None 2997 self.__file_headers = [] 2998 self.__recovery_headers = [] 2999 self.__all_headers = [] 3000 try: 3001 self._parse_headers(search) 3002 if self.__main_header == None: 3003 raise CorruptedArchiveError("no main header") 3004 except: 3005 self.close() 3006 raise 3007 3008 def close(self): 3009 """ 3010 Close the underlying file object for this volume. 3011 Can safely be called multiple times. 3012 """ 3013 if self.__file != None: 3014 self.__file.close() 3015 self.__file = None 3016 3017 def dumpheaders(self, file=sys.stdout): 3018 """ 3019 Dump ACE headers in this archive volume to *file*. 3020 """ 3021 print("""volume 3022 filename %s 3023 filesize %i 3024 headers MAIN:1 FILE:%i RECOVERY:%i others:%i""" % ( 3025 self.__filename, 3026 self.__filesize, 3027 len(self.__file_headers), 3028 len(self.__recovery_headers), 3029 len(self.__all_headers) - len(self.__file_headers) - \ 3030 len(self.__recovery_headers) - 1), 3031 file=file) 3032 for h in self.__all_headers: 3033 print(h, file=file) 3034 3035 def file_segment_for(self, fhdr): 3036 """ 3037 Returns a :class:`FileSegmentIO` object for the file header *fhdr* 3038 belonging to this volume. 3039 """ 3040 assert fhdr in self.__file_headers 3041 return FileSegmentIO(self.__file, fhdr.dataoffset, fhdr.packsize) 3042 3043 def _next_filename(self): 3044 """ 3045 Derive the filename of the next volume after this one. 3046 If the filename ends in ``.[cC]XX``, XX is incremented by 1. 3047 Otherwise self is assumed to be the first in the series and 3048 ``.[cC]00`` is used as extension. 3049 Returns the derived filename in two variants, upper and lower case, 3050 to allow for finding the file on fully case-sensitive filesystems. 3051 """ 3052 base, ext = os.path.splitext(self.__filename) 3053 ext = ext.lower() 3054 if ext[:2] == '.c': 3055 try: 3056 n = int(ext[2:]) 3057 return (base + ('.c%02i' % (n + 1)), 3058 base + ('.C%02i' % (n + 1))) 3059 except ValueError: 3060 pass 3061 return (base + '.c00', 3062 base + '.C00') 3063 3064 def try_load_next_volume(self, mode): 3065 """ 3066 Open the next volume following this one in a multi-volume archive 3067 and return the instantiated :class:`AceVolume` object. 3068 """ 3069 for nextname in self._next_filename(): 3070 try: 3071 return AceVolume(nextname, mode=mode, search=0) 3072 except FileNotFoundError: 3073 continue 3074 return None 3075 3076 def _parse_headers(self, search): 3077 """ 3078 Parse ACE headers from self.__file. If *search* is > 0, search for 3079 the magic bytes in the first *search* bytes of the file. 3080 Raises MainHeaderNotFoundError if the main header could not be located. 3081 Raises other exceptions if parsing fails for other reasons. 3082 On success, loads all the parsed headers into self.__main_header, 3083 self.__file_headers, self.__recovery_headers and self.__all_headers. 3084 """ 3085 self.__file.seek(0, 0) 3086 buf = self.__file.read(512) 3087 found_at_start = False 3088 if buf[7:14] == MainHeader.MAGIC: 3089 self.__file.seek(0, 0) 3090 try: 3091 self._parse_header() 3092 found_at_start = True 3093 except CorruptedArchiveError: 3094 pass 3095 if not found_at_start: 3096 if search == 0: 3097 raise MainHeaderNotFoundError("no ACE header at offset 0") 3098 self.__file.seek(0, 0) 3099 buf = self.__file.read(search) 3100 magicpos = 7 3101 while magicpos < search: 3102 magicpos = buf.find(MainHeader.MAGIC, magicpos + 1, search) 3103 if magicpos == -1: 3104 raise MainHeaderNotFoundError( 3105 "no ACE header within first %i bytes" % search) 3106 self.__file.seek(magicpos - 7, 0) 3107 try: 3108 self._parse_header() 3109 break 3110 except CorruptedArchiveError: 3111 continue 3112 while self.__file.tell() < self.__filesize: 3113 self._parse_header() 3114 3115 def _parse_header(self): 3116 """ 3117 Parse a single header from self.__file at the current file position. 3118 Raises CorruptedArchiveError if the header cannot be parsed. 3119 Guarantees that no data is written to object state 3120 if an exception is thrown, otherwise the header is added to 3121 self.__main_header, self.__file_headers, self.__recovery_headers 3122 and self.__all_headers. 3123 """ 3124 buf = self.__file.read(4) 3125 if len(buf) < 4: 3126 raise CorruptedArchiveError("truncated header") 3127 hcrc, hsize = struct.unpack('<HH', buf) 3128 buf = self.__file.read(hsize) 3129 if len(buf) < hsize: 3130 raise CorruptedArchiveError("truncated header") 3131 if ace_crc16(buf) != hcrc: 3132 raise CorruptedArchiveError("header CRC failed") 3133 htype, hflags = struct.unpack('<BH', buf[0:3]) 3134 i = 3 3135 3136 if htype == Header.TYPE_MAIN: 3137 header = MainHeader(hcrc, hsize, htype, hflags) 3138 if header.flag(Header.FLAG_ADDSIZE): 3139 raise CorruptedArchiveError("main header has addsize > 0") 3140 header.magic = buf[3:10] 3141 if header.magic != MainHeader.MAGIC: 3142 raise CorruptedArchiveError("main header without magic") 3143 header.eversion, \ 3144 header.cversion, \ 3145 header.host, \ 3146 header.volume, \ 3147 header.datetime = struct.unpack('<BBBBL', buf[10:18]) 3148 header.reserved1 = buf[18:26] 3149 i = 26 3150 if header.flag(Header.FLAG_ADVERT): 3151 if i + 1 > len(buf): 3152 raise CorruptedArchiveError("truncated header") 3153 avsz, = struct.unpack('<B', buf[i:i+1]) 3154 i += 1 3155 if i + avsz > len(buf): 3156 raise CorruptedArchiveError("truncated header") 3157 header.advert = buf[i:i+avsz] 3158 i += avsz 3159 if header.flag(Header.FLAG_COMMENT): 3160 if i + 2 > len(buf): 3161 raise CorruptedArchiveError("truncated header") 3162 cmsz, = struct.unpack('<H', buf[i:i+2]) 3163 i += 2 3164 if i + cmsz > len(buf): 3165 raise CorruptedArchiveError("truncated header") 3166 header.comment = ACE.decompress_comment(buf[i:i+cmsz]) 3167 i += cmsz 3168 header.reserved2 = buf[i:] 3169 if self.__main_header != None: 3170 raise CorruptedArchiveError("multiple main headers") 3171 self.__main_header = header 3172 3173 elif htype in (Header.TYPE_FILE32, Header.TYPE_FILE64): 3174 header = FileHeader(hcrc, hsize, htype, hflags) 3175 if not header.flag(Header.FLAG_ADDSIZE): 3176 raise CorruptedArchiveError("file header with addsize == 0") 3177 if header.flag(Header.FLAG_64BIT): 3178 if htype != Header.TYPE_FILE64: 3179 raise CorruptedArchiveError("64 bit flag in 32 bit header") 3180 if i + 16 > len(buf): 3181 raise CorruptedArchiveError("truncated header") 3182 header.packsize, \ 3183 header.origsize, = struct.unpack('<QQ', buf[i:i+16]) 3184 i += 16 3185 else: 3186 if htype != Header.TYPE_FILE32: 3187 raise CorruptedArchiveError("32 bit flag in 64 bit header") 3188 if i + 8 > len(buf): 3189 raise CorruptedArchiveError("truncated header") 3190 header.packsize, \ 3191 header.origsize, = struct.unpack('<LL', buf[i:i+8]) 3192 i += 8 3193 if i + 20 > len(buf): 3194 raise CorruptedArchiveError("truncated header") 3195 header.datetime, \ 3196 header.attribs, \ 3197 header.crc32, \ 3198 header.comptype, \ 3199 header.compqual, \ 3200 header.params, \ 3201 header.reserved1, \ 3202 fnsz = struct.unpack('<LLLBBHHH', buf[i:i+20]) 3203 i += 20 3204 if i + fnsz > len(buf): 3205 raise CorruptedArchiveError("truncated header") 3206 header.filename = buf[i:i+fnsz] 3207 i += fnsz 3208 if header.flag(Header.FLAG_COMMENT): 3209 if i + 2 > len(buf): 3210 raise CorruptedArchiveError("truncated header") 3211 cmsz, = struct.unpack('<H', buf[i:i+2]) 3212 i += 2 3213 if i + cmsz > len(buf): 3214 raise CorruptedArchiveError("truncated header") 3215 header.comment = ACE.decompress_comment(buf[i:i+cmsz]) 3216 i += cmsz 3217 if header.flag(Header.FLAG_NTSECURITY): 3218 if i + 2 > len(buf): 3219 raise CorruptedArchiveError("truncated header") 3220 nssz, = struct.unpack('<H', buf[i:i+2]) 3221 i += 2 3222 if i + nssz > len(buf): 3223 raise CorruptedArchiveError("truncated header") 3224 header.ntsecurity = buf[i:i+nssz] 3225 i += nssz 3226 header.reserved2 = buf[i:] 3227 header.dataoffset = self.__file.tell() 3228 self.__file_headers.append(header) 3229 self.__file.seek(header.packsize, 1) 3230 3231 elif htype in (Header.TYPE_RECOVERY32, 3232 Header.TYPE_RECOVERY64A, 3233 Header.TYPE_RECOVERY64B): 3234 header = RecoveryHeader(hcrc, hsize, htype, hflags) 3235 if not header.flag(Header.FLAG_ADDSIZE): 3236 raise CorruptedArchiveError("recovery header with addsize == 0") 3237 if header.flag(Header.FLAG_64BIT) and \ 3238 htype == Header.TYPE_RECOVERY32: 3239 raise CorruptedArchiveError("64 bit flag in 32 bit header") 3240 if not header.flag(Header.FLAG_64BIT) and \ 3241 htype != Header.TYPE_RECOVERY32: 3242 raise CorruptedArchiveError("32 bit flag in 64 bit header") 3243 if header.flag(Header.FLAG_64BIT): 3244 if i + 23 > len(buf): 3245 raise CorruptedArchiveError("truncated header") 3246 header.rcvrsize, \ 3247 header.magic, \ 3248 header.relstart = struct.unpack('<Q7sQ', buf[i:i+23]) 3249 i += 23 3250 else: 3251 if i + 15 > len(buf): 3252 raise CorruptedArchiveError("truncated header") 3253 header.rcvrsize, \ 3254 header.magic, \ 3255 header.relstart = struct.unpack('<L7sL', buf[i:i+15]) 3256 i += 15 3257 if htype == Header.TYPE_RECOVERY64B: 3258 if i + 8 > len(buf): 3259 raise CorruptedArchiveError("truncated header") 3260 header.sectors, \ 3261 header.spc, \ 3262 header.clustersize = struct.unpack('<HHL', buf[i:i+8]) 3263 i += 8 3264 else: 3265 if i + 10 > len(buf): 3266 raise CorruptedArchiveError("truncated header") 3267 header.cluster, \ 3268 header.clustersize, \ 3269 header.rcvrcrc = struct.unpack('<LLH', buf[i:i+10]) 3270 i += 10 3271 header.dataoffset = self.__file.tell() 3272 self.__recovery_headers.append(header) 3273 self.__file.seek(header.rcvrsize, 1) 3274 3275 else: 3276 header = UnknownHeader(hcrc, hsize, htype, hflags) 3277 addsz = 0 3278 if header.flag(Header.FLAG_ADDSIZE): 3279 if header.flag(Header.FLAG_64BIT): 3280 if i + 8 > len(buf): 3281 raise CorruptedArchiveError("truncated header") 3282 addsz, = struct.unpack('<Q', buf[i:i+8]) 3283 else: 3284 if i + 4 > len(buf): 3285 raise CorruptedArchiveError("truncated header") 3286 addsz, = struct.unpack('<L', buf[i:i+4]) 3287 self.__file.seek(addsz, 1) 3288 3289 self.__all_headers.append(header) 3290 3291 def get_file_headers(self): 3292 return self.__file_headers 3293 3294 def get_recovery_headers(self): 3295 return self.__recovery_headers 3296 3297 def is_locked(self): 3298 return self.__main_header.flag(Header.FLAG_LOCKED) 3299 3300 def is_multivolume(self): 3301 return self.__main_header.flag(Header.FLAG_MULTIVOLUME) 3302 3303 def is_solid(self): 3304 return self.__main_header.flag(Header.FLAG_SOLID) 3305 3306 @property 3307 def advert(self): 3308 return self.__main_header.advert.decode('utf-8', errors='replace') 3309 3310 @property 3311 def comment(self): 3312 return self.__main_header.comment.decode('utf-8', errors='replace') 3313 3314 @property 3315 def cversion(self): 3316 return self.__main_header.cversion 3317 3318 @property 3319 def eversion(self): 3320 return self.__main_header.eversion 3321 3322 @property 3323 def filename(self): 3324 return self.__filename 3325 3326 @property 3327 def datetime(self): 3328 return _dt_fromdos(self.__main_header.datetime) 3329 3330 @property 3331 def platform(self): 3332 return self.__main_header.host_str 3333 3334 @property 3335 def volume(self): 3336 return self.__main_header.volume 3337 3338 3339 3340class AceArchive: 3341 """ 3342 Represents an ACE archive, possibly consisting of multiple volumes. 3343 :class:`AceArchive` is not directly instantiated; instead, instances are 3344 returned by :meth:`acefile.open`. 3345 3346 When used as a context manager, :class:`AceArchive` ensures that 3347 :meth:`AceArchive.close` is called after the block. 3348 When used as an iterator, :class:`AceArchive` yields instances of 3349 :class:`AceMember` representing all archive members in order of 3350 appearance in the archive. 3351 """ 3352 3353 @classmethod 3354 def _open(cls, file, mode='r', *, search=524288): 3355 """ 3356 Open archive from *file*, which is either a filename or seekable 3357 file-like object, and return an instance of :class:`AceArchive` 3358 representing the opened archive that can function as a context 3359 manager. 3360 Only *mode* 'r' is implemented. 3361 If *search* is 0, the archive must start at position 0 in *file*, 3362 otherwise the first *search* bytes are searched for the magic bytes 3363 ``**ACE**`` that mark the ACE main header. 3364 For 1:1 compatibility with the official unace, 1024 sectors are 3365 searched by default, even though none of the SFX stubs that come with 3366 ACE compressors are that large. 3367 3368 Multi-volume archives are represented by a single :class:`AceArchive` 3369 object to the caller, all operations transparently read into subsequent 3370 volumes as required. 3371 To load a multi-volume archive, either open the first volume of the 3372 series by filename, or provide a list or tuple of all file-like 3373 objects or filenames in the correct order in *file*. 3374 """ 3375 return cls(file, mode, search=search) 3376 3377 def __init__(self, file, mode='r', *, search=524288): 3378 """ 3379 See :meth:`AceArchive._open`. 3380 """ 3381 if mode != 'r': 3382 raise NotImplementedError("mode != 'r' not implemented") 3383 if isinstance(file, (tuple, list)): 3384 if len(file) == 0: 3385 raise ValueError("file is empty tuple/list") 3386 else: 3387 file = (file,) 3388 3389 self.__volumes = [] 3390 try: 3391 # load volumes 3392 self.__tmp_file = file[0] 3393 self.__volumes.append(AceVolume(file[0], mode, search=search)) 3394 self.__tmp_file = None 3395 if self.__volumes[0].is_multivolume(): 3396 for f in file[1:]: 3397 self.__tmp_file = f 3398 self.__volumes.append(AceVolume(f, mode, search=0)) 3399 self.__tmp_file = None 3400 if len(self.__volumes) == 1 and isinstance(file[0], str): 3401 vol = self.__volumes[0] 3402 while True: 3403 vol = vol.try_load_next_volume(mode) 3404 if not vol: 3405 break 3406 self.__volumes.append(vol) 3407 3408 # check volume linkage 3409 if len(self.__volumes) > 1: 3410 last_volume = None 3411 for vol in self.__volumes: 3412 if not vol.is_multivolume(): 3413 raise MultiVolumeArchiveError("single-volume archive") 3414 if last_volume != None and vol.volume != last_volume + 1: 3415 raise MultiVolumeArchiveError("volumes do not match") 3416 last_volume = vol.volume 3417 3418 # build list of members and their file segments across volumes 3419 self.__members = [] 3420 headers = [] 3421 segments = [] 3422 for volume in self.__volumes: 3423 for hdr in volume.get_file_headers(): 3424 if len(headers) == 0: 3425 if hdr.flag(Header.FLAG_CONTPREV): 3426 if len(self.__members) > 0: 3427 raise MultiVolumeArchiveError("incomplete file") 3428 # don't raise an error if this is the first file 3429 # in the first volume, to allow opening subsequent 3430 # volumes of multi-volume archives separately 3431 continue 3432 else: 3433 if not hdr.flag(Header.FLAG_CONTPREV): 3434 raise MultiVolumeArchiveError("unexpected new file") 3435 if hdr.filename != headers[-1].filename: 3436 raise MultiVolumeArchiveError("filename mismatch") 3437 headers.append(hdr) 3438 segments.append(volume.file_segment_for(hdr)) 3439 if not hdr.flag(Header.FLAG_CONTNEXT): 3440 if len(segments) > 1: 3441 f = MultipleFilesIO(segments) 3442 else: 3443 f = segments[0] 3444 self.__members.append(AceMember(len(self.__members), 3445 headers, f)) 3446 headers = [] 3447 segments = [] 3448 3449 self.__next_read_idx = 0 3450 self.__ace = ACE() 3451 except: 3452 self.close() 3453 raise 3454 3455 def __enter__(self): 3456 """ 3457 Using :class:`AceArchive` as a context manager ensures that 3458 :meth:`AceArchive.close` is called after leaving the block. 3459 """ 3460 return self 3461 3462 def __exit__(self, type, value, traceback): 3463 self.close() 3464 3465 def __iter__(self): 3466 """ 3467 Using :class:`AceArchive` as an iterater will iterate over 3468 :class:`AceMember` objects for all archive members. 3469 """ 3470 return self.__members.__iter__() 3471 3472 def __repr__(self): 3473 return "<%s %r at %#x>" % (self.__class__.__name__, 3474 self.filename, 3475 id(self)) 3476 3477 def close(self): 3478 """ 3479 Close the archive and all open files. 3480 No other methods may be called after having called 3481 :meth:`AceArchive.close`, but calling :meth:`AceArchive.close` 3482 multiple times is permitted. 3483 """ 3484 if self.__tmp_file != None: 3485 if not isinstance(self.__tmp_file, str): 3486 self.__tmp_file.close() 3487 self.__tmp_file = None 3488 for volume in self.__volumes: 3489 volume.close() 3490 3491 def _getmember_byname(self, name): 3492 """ 3493 Return an :class:`AceMember` object corresponding to archive member 3494 name *name*. 3495 Raise :class:`KeyError` if *name* is not present in the archive. 3496 If *name* occurs multiple times in the archive, then the last occurence 3497 is returned. 3498 """ 3499 match = None 3500 for am in self.__members: 3501 if am.filename == name: 3502 match = am 3503 if match == None: 3504 raise KeyError("no member '%s' in archive" % name) 3505 return match 3506 3507 def _getmember_byidx(self, idx): 3508 """ 3509 Return an :class:`AceMember` object corresponding to archive member 3510 index *idx*. 3511 Raise :class:`IndexError` if *idx* is not present in the archive. 3512 """ 3513 return self.__members[idx] 3514 3515 def getmember(self, member): 3516 """ 3517 Return an :class:`AceMember` object corresponding to archive 3518 member *member*. 3519 Raise :class:`KeyError` or :class:`IndexError` if *member* is not 3520 found in archive. 3521 *Member* can refer to an :class:`AceMember` object, a member name or 3522 an index into the archive member list. 3523 If *member* is a name and it occurs multiple times in the archive, 3524 then the last member with matching filename is returned. 3525 """ 3526 if isinstance(member, int): 3527 return self._getmember_byidx(member) 3528 elif isinstance(member, AceMember): 3529 return member 3530 elif isinstance(member, str): 3531 return self._getmember_byname(member) 3532 else: 3533 raise TypeError("member argument has unsupported type") 3534 3535 def getmembers(self): 3536 """ 3537 Return a list of :class:`AceMember` objects for the members of the 3538 archive. 3539 The objects are in the same order as they are in the archive. 3540 For simply iterating over the members of an archive, it is more concise 3541 and functionally equivalent to directly iterate over the 3542 :class:`AceArchive` instance instead of over the list returned by 3543 :meth:`AceArchive.getmembers`. 3544 """ 3545 return self.__members 3546 3547 def getnames(self): 3548 """ 3549 Return a list of the (file)names of all the members in the archive 3550 in the order they are in the archive. 3551 """ 3552 return [am.filename for am in self.getmembers()] 3553 3554 def extract(self, member, *, path=None, pwd=None, restore=False): 3555 """ 3556 Extract an archive member to *path* or the current working directory. 3557 *Member* can refer to an :class:`AceMember` object, a member name or 3558 an index into the archive member list. 3559 Password *pwd* is used to decrypt the archive member if it is 3560 encrypted. 3561 Raises :class:`EncryptedArchiveError` if an archive member is 3562 encrypted but no password was provided. 3563 Iff *restore* is True, restore mtime and atime for non-dir members, 3564 file attributes and NT security information as far as supported by 3565 the platform. 3566 3567 .. note:: 3568 3569 For **solid** archives, extracting members in a different order 3570 than they appear in the archive works, but is potentially very 3571 slow, because the decompressor needs to restart decompression at 3572 the beginning of the solid archive to restore internal decompressor 3573 state. 3574 For **encrypted solid** archives, out of order access may fail when 3575 archive members use different passwords. 3576 """ 3577 am = self.getmember(member) 3578 3579 if path != None: 3580 fn = os.path.join(path, am.filename) 3581 else: 3582 fn = am.filename 3583 if am.is_dir(): 3584 try: 3585 os.mkdir(fn) 3586 except FileExistsError: 3587 pass 3588 else: 3589 basedir = os.path.dirname(fn) 3590 if basedir != '': 3591 os.makedirs(basedir, exist_ok=True) 3592 with builtins.open(fn, 'wb') as f: 3593 for buf in self.readblocks(am, pwd=pwd): 3594 f.write(buf) 3595 if restore: 3596 if SetFileAttributes: 3597 SetFileAttributes(fn, am.attribs) 3598 elif am.attribs & Header.ATTR_READONLY != 0: 3599 mode = stat.S_IMODE(os.lstat(fn).st_mode) 3600 all_w = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH 3601 os.chmod(fn, mode & ~all_w) 3602 if SetFileSecurity and am.ntsecurity: 3603 SetFileSecurity(fn, 0x7, am.ntsecurity) 3604 if not am.is_dir(): 3605 ts = am.datetime.timestamp() 3606 os.utime(fn, (ts, ts)) 3607 3608 def extractall(self, *, path=None, members=None, pwd=None, restore=False): 3609 """ 3610 Extract *members* or all members from archive to *path* or the current 3611 working directory. 3612 *Members* can contain :class:`AceMember` objects, member names or 3613 indexes into the archive member list. 3614 Password *pwd* is used to decrypt encrypted archive members. 3615 To extract archives that use multiple different passwords for different 3616 archive members, you must use :meth:`AceArchive.extract` instead. 3617 Raises :class:`EncryptedArchiveError` if an archive member is 3618 encrypted but no password was provided. 3619 Iff *restore* is True, restore mtime and atime for non-dir members, 3620 file attributes and NT security information as far as supported by 3621 the platform. 3622 """ 3623 if members == None or members == []: 3624 members = self.getmembers() 3625 else: 3626 if self.is_solid(): 3627 # ensure members subset is in order of appearance 3628 sorted_members = [] 3629 for member in self.getmembers(): 3630 if member in members or \ 3631 member.filename in members or \ 3632 member._idx in members: 3633 sorted_members.append(member) 3634 members = sorted_members 3635 for am in members: 3636 self.extract(am, path=path, pwd=pwd, restore=restore) 3637 3638 def read(self, member, *, pwd=None): 3639 """ 3640 Read the decompressed bytes of an archive member. 3641 *Member* can refer to an :class:`AceMember` object, a member name or 3642 an index into the archive member list. 3643 Password *pwd* is used to decrypt the archive member if it is 3644 encrypted. 3645 Raises :class:`EncryptedArchiveError` if the archive member is 3646 encrypted but no password was provided. 3647 3648 .. note:: 3649 3650 For **solid** archives, reading members in a different order than 3651 they appear in the archive works, but is potentially very slow, 3652 because the decompressor needs to restart decompression at the 3653 beginning of the solid archive to restore internal decompressor 3654 state. 3655 For **encrypted solid** archives, out of order access may fail when 3656 archive members use different passwords. 3657 3658 .. note:: 3659 3660 Using :meth:`AceArchive.read` for large files is inefficient and 3661 may fail for very large files. 3662 Using :meth:`AceArchive.readblocks` to write the data to disk in 3663 blocks ensures that large files can be handled efficiently. 3664 """ 3665 return b''.join(self.readblocks(member, pwd=pwd)) 3666 3667 def readblocks(self, member, *, pwd=None): 3668 """ 3669 Read the archive member by yielding blocks of decompressed bytes. 3670 *Member* can refer to an :class:`AceMember` object, a member name or 3671 an index into the archive member list. 3672 Password *pwd* is used to decrypt the archive member if it is 3673 encrypted. 3674 Raises :class:`EncryptedArchiveError` if the archive member is 3675 encrypted but no password was provided. 3676 3677 .. note:: 3678 3679 For **solid** archives, reading members in a different order than 3680 they appear in the archive works, but is potentially very slow, 3681 because the decompressor needs to restart decompression at the 3682 beginning of the solid archive to restore internal decompressor 3683 state. 3684 For **encrypted solid** archives, out of order access may fail when 3685 archive members use different passwords. 3686 """ 3687 am = self.getmember(member) 3688 3689 if DEBUG: 3690 eprint(am) 3691 3692 # Need first volume available to read from solid multi-volume archives. 3693 if self.is_solid() and self.is_multivolume() and self.volume > 0: 3694 raise MultiVolumeArchiveError("need first volume") 3695 3696 # For solid archives, ensure the LZ77 state corresponds to the state 3697 # after extracting the previous file by re-starting extraction from 3698 # the beginning or the last extracted file. This is what makes out 3699 # of order access to solid archive members prohibitively slow. 3700 if self.is_solid() and self.__next_read_idx != am._idx: 3701 if self.__next_read_idx < am._idx: 3702 restart_idx = self.__next_read_idx 3703 else: 3704 restart_idx = self.__next_read_idx = 0 3705 for i in range(restart_idx, am._idx): 3706 if not self.test(i): 3707 raise CorruptedArchiveError("failed to restore solid state") 3708 3709 if (not am.is_dir()) and am.size > 0: 3710 f = am._file 3711 f.seek(0, 0) 3712 3713 # For password protected members, wrap the file-like object in 3714 # a decrypting wrapper object. 3715 if am.is_enc(): 3716 if not pwd: 3717 raise EncryptedArchiveError("need password") 3718 f = EncryptedFileIO(f, AceBlowfish(pwd)) 3719 3720 # Choose the matching decompressor based on the first header. 3721 if am.comptype == Header.COMP_STORED: 3722 decompressor = self.__ace.decompress_stored 3723 elif am.comptype == Header.COMP_LZ77: 3724 decompressor = self.__ace.decompress_lz77 3725 elif am.comptype == Header.COMP_BLOCKED: 3726 decompressor = self.__ace.decompress_blocked 3727 else: 3728 raise UnknownCompressionMethodError( 3729 "method %i unknown" % am.comptype) 3730 3731 # Decompress and calculate CRC over full decompressed data, 3732 # i.e. after decryption and across all segments that may have 3733 # been read from different volumes. 3734 crc = AceCRC32() 3735 try: 3736 for block in decompressor(f, am.size, am.dicsize): 3737 crc += block 3738 yield block 3739 except ValueError: 3740 if am.is_enc(): 3741 raise EncryptedArchiveError("wrong password or corrupted") 3742 else: 3743 raise CorruptedArchiveError("ValueError during decomp") 3744 except CorruptedArchiveError: 3745 if am.is_enc(): 3746 raise EncryptedArchiveError("wrong password or corrupted") 3747 raise 3748 if crc != am.crc32: 3749 if am.is_enc(): 3750 raise EncryptedArchiveError("wrong password or corrupted") 3751 raise CorruptedArchiveError("CRC mismatch") 3752 3753 self.__next_read_idx += 1 3754 3755 def test(self, member, *, pwd=None): 3756 """ 3757 Test an archive member. Returns False if any corruption was 3758 found, True if the header and decompression was okay. 3759 *Member* can refer to an :class:`AceMember` object, a member name or 3760 an index into the archive member list. 3761 Password *pwd* is used to decrypt the archive member if it is 3762 encrypted. 3763 Raises :class:`EncryptedArchiveError` if the archive member is 3764 encrypted but no password was provided. 3765 3766 .. note:: 3767 3768 For **solid** archives, testing members in a different order than 3769 they appear in the archive works, but is potentially very slow, 3770 because the decompressor needs to restart decompression at the 3771 beginning of the solid archive to restore internal decompressor 3772 state. 3773 For **encrypted solid** archives, out of order access may fail when 3774 archive members use different passwords. 3775 """ 3776 try: 3777 for buf in self.readblocks(member, pwd=pwd): 3778 pass 3779 return True 3780 except EncryptedArchiveError: 3781 raise 3782 except AceError: 3783 if DEBUG: 3784 raise 3785 return False 3786 3787 def testall(self, *, pwd=None): 3788 """ 3789 Test all the members in the archive. Returns the name of the first 3790 archive member with a failing header or content CRC, or None if all 3791 members were okay. 3792 Password *pwd* is used to decrypt encrypted archive members. 3793 To test archives that use multiple different passwords for different 3794 archive members, use :meth:`AceArchive.test` instead. 3795 Raises :class:`EncryptedArchiveError` if an archive member is 3796 encrypted but no password was provided. 3797 """ 3798 for am in self.getmembers(): 3799 if not self.test(am, pwd=pwd): 3800 return am.filename 3801 return None 3802 3803 def dumpheaders(self, file=sys.stdout): 3804 """ 3805 Dump all ACE file format headers in this archive and all its volumes 3806 to *file*. 3807 """ 3808 for volume in self.__volumes: 3809 volume.dumpheaders() 3810 3811 def is_locked(self): 3812 """ 3813 Return True iff archive is locked for further modifications. 3814 Since this implementation does not support writing to archives, 3815 presence or absence of the flag in an archive does not change any 3816 behaviour of :mod:`acefile`. 3817 """ 3818 return self.__volumes[0].is_locked() 3819 3820 def is_multivolume(self): 3821 """ 3822 Return True iff archive is a multi-volume archive as determined 3823 by the archive headers. When opening the last volume of a 3824 multi-volume archive, this returns True even though only a single 3825 volume was loaded. 3826 """ 3827 return self.__volumes[0].is_multivolume() 3828 3829 def is_solid(self): 3830 """ 3831 Return True iff archive is a solid archive, i.e. iff the archive 3832 members are linked to each other by sharing the same LZ77 dictionary. 3833 Members of solid archives should always be read/tested/extracted in 3834 the order they appear in the archive in order to avoid costly 3835 decompression restarts from the beginning of the archive. 3836 """ 3837 return self.__volumes[0].is_solid() 3838 3839 @property 3840 def advert(self): 3841 """ 3842 ACE archive advert string as :class:`str`. 3843 Unregistered versions of ACE compressors communicate that they are 3844 unregistered by including an advert string of 3845 ``*UNREGISTERED VERSION*`` in archives they create. 3846 If absent, empty :class:`str`. 3847 """ 3848 return self.__volumes[0].advert 3849 3850 @property 3851 def comment(self): 3852 """ 3853 ACE archive level comment as :class:`str`. 3854 If absent, empty :class:`str`. 3855 """ 3856 return self.__volumes[0].comment 3857 3858 @property 3859 def cversion(self): 3860 """ 3861 ACE creator version. This is equal to the major version of the ACE 3862 compressor used to create the archive, which equals the highest 3863 version of the ACE format supported by the ACE compressor which 3864 produced the archive. 3865 """ 3866 return self.__volumes[0].cversion 3867 3868 @property 3869 def eversion(self): 3870 """ 3871 ACE extractor version. This is the version of the ACE decompressor 3872 required to extract, which equals the version of the ACE format this 3873 archive is compliant with. 3874 """ 3875 return self.__volumes[0].eversion 3876 3877 @property 3878 def filename(self): 3879 """ 3880 ACE archive filename. This is not a property of the archive but rather 3881 just the filename passed to :func:`acefile.open`. 3882 """ 3883 return self.__volumes[0].filename 3884 3885 @property 3886 def datetime(self): 3887 """ 3888 Archive timestamp as :class:`datetime.datetime` object. 3889 """ 3890 return self.__volumes[0].datetime 3891 3892 @property 3893 def platform(self): 3894 """ 3895 String describing the platform on which the ACE archive was created. 3896 This is derived from the *host* field in the archive header. 3897 """ 3898 return self.__volumes[0].platform 3899 3900 @property 3901 def volume(self): 3902 """ 3903 ACE archive volume number of the first volume of this ACE archive. 3904 """ 3905 return self.__volumes[0].volume 3906 3907 @property 3908 def volumes_loaded(self): 3909 """ 3910 Number of loaded volumes in this archives. When opening a subsequent 3911 volume of a multi-volume archive, this may be lower than the 3912 theoretical volume count. 3913 """ 3914 return len(self.__volumes) 3915 3916 3917 3918def is_acefile(file, *, search=524288): 3919 """ 3920 Return True iff *file* refers to an ACE archive by filename or seekable 3921 file-like object. 3922 If *search* is 0, the archive must start at position 0 in *file*, 3923 otherwise the first *search* bytes are searched for the magic bytes 3924 ``**ACE**`` that mark the ACE main header. 3925 For 1:1 compatibility with the official unace, 1024 sectors are 3926 searched by default, even though none of the SFX stubs that come with 3927 ACE compressors are that large. 3928 """ 3929 try: 3930 with open(file, search=search) as f: 3931 pass 3932 return True 3933 except AceError: 3934 return False 3935 3936 3937 3938#: The compression type constant for no compression. 3939COMP_STORED = Header.COMP_STORED 3940#: The compression type constant for ACE 1.0 LZ77 mode. 3941COMP_LZ77 = Header.COMP_LZ77 3942#: The compression type constant for ACE 2.0 blocked mode. 3943COMP_BLOCKED = Header.COMP_BLOCKED 3944 3945#: The compression quality constant for no compression. 3946QUAL_NONE = Header.QUAL_NONE 3947#: The compression quality constant for fastest compression. 3948QUAL_FASTEST = Header.QUAL_FASTEST 3949#: The compression quality constant for fast compression. 3950QUAL_FAST = Header.QUAL_FAST 3951#: The compression quality constant for normal compression. 3952QUAL_NORMAL = Header.QUAL_NORMAL 3953#: The compression quality constant for good compression. 3954QUAL_GOOD = Header.QUAL_GOOD 3955#: The compression quality constant for best compression. 3956QUAL_BEST = Header.QUAL_BEST 3957 3958open = AceArchive._open 3959 3960__all__ = ['is_acefile', 'open'] 3961__all__.extend(filter(lambda name: name.startswith('COMP_'), 3962 sorted(list(globals())))) 3963__all__.extend(filter(lambda name: name.startswith('QUAL_'), 3964 sorted(list(globals())))) 3965__all__.extend(filter(lambda name: name.endswith('Error'), 3966 sorted(list(globals())))) 3967 3968 3969 3970def unace(): 3971 import argparse 3972 import getpass 3973 import signal 3974 3975 def title(docstr): 3976 return docstr.strip().split('\n', 1)[0] 3977 3978 class Status: 3979 def __init__(self, argv0, action, archive): 3980 self.argv0 = os.path.basename(argv0) 3981 self.action = action + 'ing' 3982 self.archive = os.path.basename(archive) 3983 self.member = '' 3984 3985 def __str__(self): 3986 return "%s: %s %s %s" % (self.argv0, self.action, 3987 self.archive, self.member) 3988 3989 status = None 3990 3991 def siginfo_handler(signum, frame): 3992 eprint(status) 3993 3994 parser = argparse.ArgumentParser(description=title(__doc__)) 3995 3996 parser.add_argument('archive', type=str, 3997 help='archive to read from') 3998 parser.add_argument('file', nargs='*', type=str, 3999 help='file(s) in archive to operate on, default all') 4000 4001 parser.add_argument('-V', '--version', action='version', 4002 version='acefile %s' % __version__, 4003 help='show version and exit') 4004 4005 group = parser.add_mutually_exclusive_group() 4006 group.add_argument('--extract', '-x', default='extract', 4007 action='store_const', dest='mode', const='extract', 4008 help='extract files in archive (default)') 4009 group.add_argument('--test', '-t', 4010 action='store_const', dest='mode', const='test', 4011 help='test archive integrity') 4012 group.add_argument('--list', '-l', 4013 action='store_const', dest='mode', const='list', 4014 help='list files in archive') 4015 group.add_argument('--headers', 4016 action='store_const', dest='mode', const='headers', 4017 help='dump archive headers') 4018 group.add_argument('--selftest', 4019 action='store_const', dest='mode', const='selftest', 4020 help=argparse.SUPPRESS) 4021 4022 parser.add_argument('-d', '--basedir', type=str, default='.', metavar='X', 4023 help='base directory for extraction') 4024 parser.add_argument('-p', '--password', type=str, metavar='X', 4025 help='password for decryption') 4026 parser.add_argument('-r', '--restore', action='store_true', 4027 help='restore mtime/atime, attribs and ntsecurity on extraction') 4028 parser.add_argument('-b', '--batch', action='store_true', 4029 help='suppress all interactive input') 4030 parser.add_argument('-v', '--verbose', action='store_true', 4031 help='be more verbose') 4032 parser.add_argument('--debug', action='store_true', 4033 help='show mode transitions and expose internal exceptions') 4034 4035 # not implemented arguments that other unace implementations have: 4036 # --(no-)full-path always full path extraction 4037 # --(no-)show-comments show comments iff verbose 4038 # --(no-)overwrite-files always overwrite files 4039 # --(no-)full-path-matching always full path matching 4040 # --exclude(-list) feature not implemented 4041 4042 args = parser.parse_args() 4043 4044 if args.mode != 'extract' and len(args.file) > 0: 4045 eprint("%s: error: not extracting, but files were specified" % 4046 os.path.basename(sys.argv[0])) 4047 sys.exit(1) 4048 4049 if args.debug: 4050 global DEBUG 4051 DEBUG = True 4052 4053 if hasattr(signal, 'SIGINFO'): 4054 signal.signal(signal.SIGINFO, siginfo_handler) 4055 status = Status(sys.argv[0], args.mode, args.archive) 4056 4057 if args.archive == '-': 4058 if sys.stdin.buffer.seekable() and platform.system() != 'Windows': 4059 archive = sys.stdin.buffer 4060 else: 4061 archive = io.BytesIO(sys.stdin.buffer.read()) 4062 else: 4063 archive = args.archive 4064 4065 try: 4066 with open(archive) as f: 4067 if args.verbose: 4068 if acebitstream == None: 4069 eprint(("warning: acebitstream c extension unavailable, " 4070 "using pure-python bit stream")) 4071 eprint("processing archive %s" % f.filename) 4072 eprint("loaded %i volume(s) starting at volume %i" % ( 4073 f.volumes_loaded, f.volume)) 4074 archinfo = [] 4075 if not f.is_locked(): 4076 archinfo.append('not ') 4077 archinfo.append('locked, ') 4078 if not f.is_multivolume(): 4079 archinfo.append('not ') 4080 archinfo.append('multi-volume, ') 4081 if not f.is_solid(): 4082 archinfo.append('not ') 4083 archinfo.append('solid') 4084 eprint("archive is", ''.join(archinfo)) 4085 eprint("last modified %s" % ( 4086 f.datetime.strftime('%Y-%m-%d %H:%M:%S'))) 4087 eprint("created on %s with ACE %s for extraction with %s+" % ( 4088 f.platform, f.cversion/10, f.eversion/10)) 4089 if f.advert: 4090 eprint("advert [%s]" % f.advert) 4091 4092 if f.is_multivolume() and f.volume > 0: 4093 eprint("warning: this is not the initial volume of this " 4094 "multi-volume archive") 4095 if f.comment: 4096 eprint(asciibox(f.comment, title='archive comment')) 4097 4098 if args.mode == 'extract': 4099 if f.is_multivolume() and f.volume > 0 and f.is_solid(): 4100 eprint(("error: need complete set of volumes to extract " 4101 "from solid multivolume archive")) 4102 sys.exit(1) 4103 failed = 0 4104 password = args.password 4105 if args.file: 4106 members = [f.getmember(m) for m in args.file] 4107 else: 4108 members = f.getmembers() 4109 for am in members: 4110 if status: 4111 status.member = am.filename 4112 if am.is_enc() and password == None and not args.batch: 4113 try: 4114 password = getpass.getpass("%s password: " % \ 4115 am.filename) 4116 except EOFError: 4117 password = None 4118 while True: 4119 try: 4120 f.extract(am, path=args.basedir, 4121 pwd=password, 4122 restore=args.restore) 4123 if args.verbose: 4124 eprint("%s" % am.filename) 4125 break 4126 except EncryptedArchiveError: 4127 if args.verbose or args.batch or not password: 4128 eprint("%s failed to decrypt" % am.filename) 4129 if args.batch or not password: 4130 failed += 1 4131 break 4132 try: 4133 password = getpass.getpass("%s password: " % \ 4134 am.filename) 4135 except EOFError: 4136 password = '' 4137 if password == '': 4138 password = args.password 4139 eprint("%s skipped" % am.filename) 4140 failed += 1 4141 break 4142 except AceError: 4143 eprint("%s failed to extract" % am.filename) 4144 failed += 1 4145 break 4146 if f.is_solid() and failed > 0: 4147 eprint("error extracting from solid archive, aborting") 4148 sys.exit(1) 4149 if args.verbose and am.comment: 4150 eprint(asciibox(am.comment, title='file comment')) 4151 if failed > 0: 4152 sys.exit(1) 4153 4154 elif args.mode == 'test': 4155 if f.is_multivolume() and f.volume > 0 and f.is_solid(): 4156 eprint(("error: need complete set of volumes to test " 4157 "solid multivolume archive")) 4158 sys.exit(1) 4159 failed = 0 4160 ok = 0 4161 password = args.password 4162 for am in f: 4163 if status: 4164 status.member = am.filename 4165 if f.is_solid() and failed > 0: 4166 print("failure %s" % am.filename) 4167 failed += 1 4168 continue 4169 if am.is_enc() and password == None and not args.batch: 4170 try: 4171 password = getpass.getpass("%s password: " % \ 4172 am.filename) 4173 except EOFError: 4174 password = None 4175 while True: 4176 try: 4177 if f.test(am, pwd=password): 4178 print("success %s" % am.filename) 4179 ok += 1 4180 else: 4181 print("failure %s" % am.filename) 4182 failed += 1 4183 break 4184 except EncryptedArchiveError: 4185 if args.batch or not password: 4186 print("needpwd %s" % am.filename) 4187 failed += 1 4188 break 4189 eprint("last used password failed") 4190 try: 4191 password = getpass.getpass("%s password: " % \ 4192 am.filename) 4193 except EOFError: 4194 password = '' 4195 if password == '': 4196 password = args.password 4197 print("needpwd %s" % am.filename) 4198 failed += 1 4199 break 4200 if args.verbose and am.comment: 4201 eprint(asciibox(am.comment, title='file comment')) 4202 eprint("total %i tested, %i ok, %i failed" % ( 4203 ok + failed, ok, failed)) 4204 if failed > 0: 4205 sys.exit(1) 4206 4207 elif args.mode == 'list': 4208 if args.verbose: 4209 eprint(("CQD FES size packed rel " 4210 "timestamp filename")) 4211 count = count_size = count_packsize = 0 4212 for am in f: 4213 if am.is_dir(): 4214 ft = 'd' 4215 else: 4216 ft = 'f' 4217 if am.is_enc(): 4218 en = '+' 4219 else: 4220 en = ' ' 4221 if am.ntsecurity: 4222 ns = 's' 4223 else: 4224 ns = ' ' 4225 if am.size > 0: 4226 ratio = (100 * am.packsize) // am.size 4227 else: 4228 ratio = 100 4229 print("%i%i%s %s%s%s %9i %9i %3i%% %s %s" % ( 4230 am.comptype, am.compqual, 4231 hex(am.dicsizebits - 10)[2:], 4232 ft, en, ns, 4233 am.size, 4234 am.packsize, 4235 ratio, 4236 am.datetime.strftime('%Y-%m-%d %H:%M:%S'), 4237 am.filename)) 4238 if am.comment: 4239 eprint(asciibox(am.comment, title='file comment')) 4240 count_size += am.size 4241 count_packsize += am.packsize 4242 count += 1 4243 eprint("total %i members, %i bytes, %i bytes compressed" % ( 4244 count, count_size, count_packsize)) 4245 else: 4246 for fn in f.getnames(): 4247 print("%s" % fn) 4248 4249 elif args.mode == 'headers': 4250 f.dumpheaders() 4251 4252 elif args.mode == 'selftest': 4253 eprint('dumpheaders():') 4254 f.dumpheaders() 4255 eprint('-' * 78) 4256 eprint('getnames():') 4257 for fn in f.getnames(): 4258 eprint("%s" % fn) 4259 eprint('-' * 78) 4260 eprint('testall():') 4261 rv = f.testall() 4262 if rv != None: 4263 eprint("Test failed: member %s is corrupted" % rv) 4264 sys.exit(1) 4265 eprint('-' * 78) 4266 eprint('test() in order:') 4267 for member in f: 4268 if f.test(member): 4269 eprint("%s: CRC OK" % member.filename) 4270 else: 4271 eprint("%s: CRC FAILED" % member.filename) 4272 sys.exit(1) 4273 eprint('-' * 78) 4274 eprint('test() in reverse order:') 4275 for member in reversed(f.getmembers()): 4276 if f.test(member): 4277 eprint("%s: CRC OK" % member.filename) 4278 else: 4279 eprint("%s: CRC FAILED" % member.filename) 4280 sys.exit(1) 4281 # end of with open 4282 4283 except AceError as e: 4284 if DEBUG: 4285 raise 4286 eprint("%s: %s: %s" % (args.archive, type(e).__name__, e)) 4287 sys.exit(1) 4288 4289 sys.exit(0) 4290 4291 4292 4293def testsuite(): 4294 import doctest 4295 return doctest.DocTestSuite(optionflags=doctest.IGNORE_EXCEPTION_DETAIL) 4296 4297def test(): 4298 import doctest 4299 fails, tests = doctest.testmod(optionflags=doctest.IGNORE_EXCEPTION_DETAIL) 4300 sys.exit(min(1, fails)) 4301 4302 4303 4304if __name__ == '__main__': 4305 if '--doctest' in sys.argv: 4306 test() 4307 unace() 4308 4309