1#!~/.wine/drive_c/Python25/python.exe 2# -*- coding: utf-8 -*- 3 4# Copyright (c) 2009-2014, Mario Vilas 5# All rights reserved. 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions are met: 9# 10# * Redistributions of source code must retain the above copyright notice, 11# this list of conditions and the following disclaimer. 12# * Redistributions in binary form must reproduce the above copyright 13# notice,this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# * Neither the name of the copyright holder nor the names of its 16# contributors may be used to endorse or promote products derived from 17# this software without specific prior written permission. 18# 19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29# POSSIBILITY OF SUCH DAMAGE. 30 31""" 32SQL database storage support. 33 34@group Crash reporting: 35 CrashDAO 36""" 37 38__revision__ = "$Id$" 39 40__all__ = ['CrashDAO'] 41 42import sqlite3 43import datetime 44import warnings 45 46from sqlalchemy import create_engine, Column, ForeignKey, Sequence 47from sqlalchemy.engine.url import URL 48from sqlalchemy.ext.compiler import compiles 49from sqlalchemy.ext.declarative import declarative_base 50from sqlalchemy.interfaces import PoolListener 51from sqlalchemy.orm import sessionmaker, deferred 52from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound 53from sqlalchemy.types import Integer, BigInteger, Boolean, DateTime, String, \ 54 LargeBinary, Enum, VARCHAR 55from sqlalchemy.sql.expression import asc, desc 56 57from crash import Crash, Marshaller, pickle, HIGHEST_PROTOCOL 58from textio import CrashDump 59import win32 60 61#------------------------------------------------------------------------------ 62 63try: 64 from decorator import decorator 65except ImportError: 66 import functools 67 def decorator(w): 68 """ 69 The C{decorator} module was not found. You can install it from: 70 U{http://pypi.python.org/pypi/decorator/} 71 """ 72 def d(fn): 73 @functools.wraps(fn) 74 def x(*argv, **argd): 75 return w(fn, *argv, **argd) 76 return x 77 return d 78 79#------------------------------------------------------------------------------ 80 81@compiles(String, 'mysql') 82@compiles(VARCHAR, 'mysql') 83def _compile_varchar_mysql(element, compiler, **kw): 84 """MySQL hack to avoid the "VARCHAR requires a length" error.""" 85 if not element.length or element.length == 'max': 86 return "TEXT" 87 else: 88 return compiler.visit_VARCHAR(element, **kw) 89 90#------------------------------------------------------------------------------ 91 92class _SQLitePatch (PoolListener): 93 """ 94 Used internally by L{BaseDAO}. 95 96 After connecting to an SQLite database, ensure that the foreign keys 97 support is enabled. If not, abort the connection. 98 99 @see: U{http://sqlite.org/foreignkeys.html} 100 """ 101 def connect(dbapi_connection, connection_record): 102 """ 103 Called once by SQLAlchemy for each new SQLite DB-API connection. 104 105 Here is where we issue some PRAGMA statements to configure how we're 106 going to access the SQLite database. 107 108 @param dbapi_connection: 109 A newly connected raw SQLite DB-API connection. 110 111 @param connection_record: 112 Unused by this method. 113 """ 114 try: 115 cursor = dbapi_connection.cursor() 116 try: 117 cursor.execute("PRAGMA foreign_keys = ON;") 118 cursor.execute("PRAGMA foreign_keys;") 119 if cursor.fetchone()[0] != 1: 120 raise Exception() 121 finally: 122 cursor.close() 123 except Exception: 124 dbapi_connection.close() 125 raise sqlite3.Error() 126 127#------------------------------------------------------------------------------ 128 129class BaseDTO (object): 130 """ 131 Customized declarative base for SQLAlchemy. 132 """ 133 134 __table_args__ = { 135 136 # Don't use MyISAM in MySQL. It doesn't support ON DELETE CASCADE. 137 'mysql_engine': 'InnoDB', 138 139 # Don't use BlitzDB in Drizzle. It doesn't support foreign keys. 140 'drizzle_engine': 'InnoDB', 141 142 # Collate to UTF-8. 143 'mysql_charset': 'utf8', 144 145 } 146 147BaseDTO = declarative_base(cls = BaseDTO) 148 149#------------------------------------------------------------------------------ 150 151# TODO: if using mssql, check it's at least SQL Server 2005 152# (LIMIT and OFFSET support is required). 153# TODO: if using mysql, check it's at least MySQL 5.0.3 154# (nested transactions are required). 155# TODO: maybe in mysql check the tables are not myisam? 156# TODO: maybe create the database if it doesn't exist? 157# TODO: maybe add a method to compact the database? 158# http://stackoverflow.com/questions/1875885 159# http://www.sqlite.org/lang_vacuum.html 160# http://dev.mysql.com/doc/refman/5.1/en/optimize-table.html 161# http://msdn.microsoft.com/en-us/library/ms174459(v=sql.90).aspx 162 163class BaseDAO (object): 164 """ 165 Data Access Object base class. 166 167 @type _url: sqlalchemy.url.URL 168 @ivar _url: Database connection URL. 169 170 @type _dialect: str 171 @ivar _dialect: SQL dialect currently being used. 172 173 @type _driver: str 174 @ivar _driver: Name of the database driver currently being used. 175 To get the actual Python module use L{_url}.get_driver() instead. 176 177 @type _session: sqlalchemy.orm.Session 178 @ivar _session: Database session object. 179 180 @type _new_session: class 181 @cvar _new_session: Custom configured Session class used to create the 182 L{_session} instance variable. 183 184 @type _echo: bool 185 @cvar _echo: Set to C{True} to print all SQL queries to standard output. 186 """ 187 188 _echo = False 189 190 _new_session = sessionmaker(autoflush = True, 191 autocommit = True, 192 expire_on_commit = True, 193 weak_identity_map = True) 194 195 def __init__(self, url, creator = None): 196 """ 197 Connect to the database using the given connection URL. 198 199 The current implementation uses SQLAlchemy and so it will support 200 whatever database said module supports. 201 202 @type url: str 203 @param url: 204 URL that specifies the database to connect to. 205 206 Some examples: 207 - Opening an SQLite file: 208 C{dao = CrashDAO("sqlite:///C:\\some\\path\\database.sqlite")} 209 - Connecting to a locally installed SQL Express database: 210 C{dao = CrashDAO("mssql://.\\SQLEXPRESS/Crashes?trusted_connection=yes")} 211 - Connecting to a MySQL database running locally, using the 212 C{oursql} library, authenticating as the "winappdbg" user with 213 no password: 214 C{dao = CrashDAO("mysql+oursql://winappdbg@localhost/Crashes")} 215 - Connecting to a Postgres database running locally, 216 authenticating with user and password: 217 C{dao = CrashDAO("postgresql://winappdbg:winappdbg@localhost/Crashes")} 218 219 For more information see the C{SQLAlchemy} documentation online: 220 U{http://docs.sqlalchemy.org/en/latest/core/engines.html} 221 222 Note that in all dialects except for SQLite the database 223 must already exist. The tables schema, however, is created 224 automatically when connecting for the first time. 225 226 To create the database in MSSQL, you can use the 227 U{SQLCMD<http://msdn.microsoft.com/en-us/library/ms180944.aspx>} 228 command:: 229 sqlcmd -Q "CREATE DATABASE Crashes" 230 231 In MySQL you can use something like the following:: 232 mysql -u root -e "CREATE DATABASE Crashes;" 233 234 And in Postgres:: 235 createdb Crashes -h localhost -U winappdbg -p winappdbg -O winappdbg 236 237 Some small changes to the schema may be tolerated (for example, 238 increasing the maximum length of string columns, or adding new 239 columns with default values). Of course, it's best to test it 240 first before making changes in a live database. This all depends 241 very much on the SQLAlchemy version you're using, but it's best 242 to use the latest version always. 243 244 @type creator: callable 245 @param creator: (Optional) Callback function that creates the SQL 246 database connection. 247 248 Normally it's not necessary to use this argument. However in some 249 odd cases you may need to customize the database connection. 250 """ 251 252 # Parse the connection URL. 253 parsed_url = URL(url) 254 schema = parsed_url.drivername 255 if '+' in schema: 256 dialect, driver = schema.split('+') 257 else: 258 dialect, driver = schema, 'base' 259 dialect = dialect.strip().lower() 260 driver = driver.strip() 261 262 # Prepare the database engine arguments. 263 arguments = {'echo' : self._echo} 264 if dialect == 'sqlite': 265 arguments['module'] = sqlite3.dbapi2 266 arguments['listeners'] = [_SQLitePatch()] 267 if creator is not None: 268 arguments['creator'] = creator 269 270 # Load the database engine. 271 engine = create_engine(url, **arguments) 272 273 # Create a new session. 274 session = self._new_session(bind = engine) 275 276 # Create the required tables if they don't exist. 277 BaseDTO.metadata.create_all(engine) 278 # TODO: create a dialect specific index on the "signature" column. 279 280 # Set the instance properties. 281 self._url = parsed_url 282 self._driver = driver 283 self._dialect = dialect 284 self._session = session 285 286 def _transactional(self, method, *argv, **argd): 287 """ 288 Begins a transaction and calls the given DAO method. 289 290 If the method executes successfully the transaction is commited. 291 292 If the method fails, the transaction is rolled back. 293 294 @type method: callable 295 @param method: Bound method of this class or one of its subclasses. 296 The first argument will always be C{self}. 297 298 @return: The return value of the method call. 299 300 @raise Exception: Any exception raised by the method. 301 """ 302 self._session.begin(subtransactions = True) 303 try: 304 result = method(self, *argv, **argd) 305 self._session.commit() 306 return result 307 except: 308 self._session.rollback() 309 raise 310 311#------------------------------------------------------------------------------ 312 313@decorator 314def Transactional(fn, self, *argv, **argd): 315 """ 316 Decorator that wraps DAO methods to handle transactions automatically. 317 318 It may only work with subclasses of L{BaseDAO}. 319 """ 320 return self._transactional(fn, *argv, **argd) 321 322#============================================================================== 323 324# Generates all possible memory access flags. 325def _gen_valid_access_flags(): 326 f = [] 327 for a1 in ("---", "R--", "RW-", "RC-", "--X", "R-X", "RWX", "RCX", "???"): 328 for a2 in ("G", "-"): 329 for a3 in ("N", "-"): 330 for a4 in ("W", "-"): 331 f.append("%s %s%s%s" % (a1, a2, a3, a4)) 332 return tuple(f) 333_valid_access_flags = _gen_valid_access_flags() 334 335# Enumerated types for the memory table. 336n_MEM_ACCESS_ENUM = {"name" : "MEM_ACCESS_ENUM"} 337n_MEM_ALLOC_ACCESS_ENUM = {"name" : "MEM_ALLOC_ACCESS_ENUM"} 338MEM_ACCESS_ENUM = Enum(*_valid_access_flags, 339 **n_MEM_ACCESS_ENUM) 340MEM_ALLOC_ACCESS_ENUM = Enum(*_valid_access_flags, 341 **n_MEM_ALLOC_ACCESS_ENUM) 342MEM_STATE_ENUM = Enum("Reserved", "Commited", "Free", "Unknown", 343 name = "MEM_STATE_ENUM") 344MEM_TYPE_ENUM = Enum("Image", "Mapped", "Private", "Unknown", 345 name = "MEM_TYPE_ENUM") 346 347# Cleanup the namespace. 348del _gen_valid_access_flags 349del _valid_access_flags 350del n_MEM_ACCESS_ENUM 351del n_MEM_ALLOC_ACCESS_ENUM 352 353#------------------------------------------------------------------------------ 354 355class MemoryDTO (BaseDTO): 356 """ 357 Database mapping for memory dumps. 358 """ 359 360 # Declare the table mapping. 361 __tablename__ = 'memory' 362 id = Column(Integer, Sequence(__tablename__ + '_seq'), 363 primary_key = True, autoincrement = True) 364 crash_id = Column(Integer, ForeignKey('crashes.id', 365 ondelete = 'CASCADE', 366 onupdate = 'CASCADE'), 367 nullable = False) 368 address = Column(BigInteger, nullable = False, index = True) 369 size = Column(BigInteger, nullable = False) 370 state = Column(MEM_STATE_ENUM, nullable = False) 371 access = Column(MEM_ACCESS_ENUM) 372 type = Column(MEM_TYPE_ENUM) 373 alloc_base = Column(BigInteger) 374 alloc_access = Column(MEM_ALLOC_ACCESS_ENUM) 375 filename = Column(String) 376 content = deferred(Column(LargeBinary)) 377 378 def __init__(self, crash_id, mbi): 379 """ 380 Process a L{win32.MemoryBasicInformation} object for database storage. 381 """ 382 383 # Crash ID. 384 self.crash_id = crash_id 385 386 # Address. 387 self.address = mbi.BaseAddress 388 389 # Size. 390 self.size = mbi.RegionSize 391 392 # State (free or allocated). 393 if mbi.State == win32.MEM_RESERVE: 394 self.state = "Reserved" 395 elif mbi.State == win32.MEM_COMMIT: 396 self.state = "Commited" 397 elif mbi.State == win32.MEM_FREE: 398 self.state = "Free" 399 else: 400 self.state = "Unknown" 401 402 # Page protection bits (R/W/X/G). 403 if mbi.State != win32.MEM_COMMIT: 404 self.access = None 405 else: 406 self.access = self._to_access(mbi.Protect) 407 408 # Type (file mapping, executable image, or private memory). 409 if mbi.Type == win32.MEM_IMAGE: 410 self.type = "Image" 411 elif mbi.Type == win32.MEM_MAPPED: 412 self.type = "Mapped" 413 elif mbi.Type == win32.MEM_PRIVATE: 414 self.type = "Private" 415 elif mbi.Type == 0: 416 self.type = None 417 else: 418 self.type = "Unknown" 419 420 # Allocation info. 421 self.alloc_base = mbi.AllocationBase 422 if not mbi.AllocationProtect: 423 self.alloc_access = None 424 else: 425 self.alloc_access = self._to_access(mbi.AllocationProtect) 426 427 # Filename (for memory mappings). 428 try: 429 self.filename = mbi.filename 430 except AttributeError: 431 self.filename = None 432 433 # Memory contents. 434 try: 435 self.content = mbi.content 436 except AttributeError: 437 self.content = None 438 439 def _to_access(self, protect): 440 if protect & win32.PAGE_NOACCESS: 441 access = "--- " 442 elif protect & win32.PAGE_READONLY: 443 access = "R-- " 444 elif protect & win32.PAGE_READWRITE: 445 access = "RW- " 446 elif protect & win32.PAGE_WRITECOPY: 447 access = "RC- " 448 elif protect & win32.PAGE_EXECUTE: 449 access = "--X " 450 elif protect & win32.PAGE_EXECUTE_READ: 451 access = "R-X " 452 elif protect & win32.PAGE_EXECUTE_READWRITE: 453 access = "RWX " 454 elif protect & win32.PAGE_EXECUTE_WRITECOPY: 455 access = "RCX " 456 else: 457 access = "??? " 458 if protect & win32.PAGE_GUARD: 459 access += "G" 460 else: 461 access += "-" 462 if protect & win32.PAGE_NOCACHE: 463 access += "N" 464 else: 465 access += "-" 466 if protect & win32.PAGE_WRITECOMBINE: 467 access += "W" 468 else: 469 access += "-" 470 return access 471 472 def toMBI(self, getMemoryDump = False): 473 """ 474 Returns a L{win32.MemoryBasicInformation} object using the data 475 retrieved from the database. 476 477 @type getMemoryDump: bool 478 @param getMemoryDump: (Optional) If C{True} retrieve the memory dump. 479 Defaults to C{False} since this may be a costly operation. 480 481 @rtype: L{win32.MemoryBasicInformation} 482 @return: Memory block information. 483 """ 484 mbi = win32.MemoryBasicInformation() 485 mbi.BaseAddress = self.address 486 mbi.RegionSize = self.size 487 mbi.State = self._parse_state(self.state) 488 mbi.Protect = self._parse_access(self.access) 489 mbi.Type = self._parse_type(self.type) 490 if self.alloc_base is not None: 491 mbi.AllocationBase = self.alloc_base 492 else: 493 mbi.AllocationBase = mbi.BaseAddress 494 if self.alloc_access is not None: 495 mbi.AllocationProtect = self._parse_access(self.alloc_access) 496 else: 497 mbi.AllocationProtect = mbi.Protect 498 if self.filename is not None: 499 mbi.filename = self.filename 500 if getMemoryDump and self.content is not None: 501 mbi.content = self.content 502 return mbi 503 504 @staticmethod 505 def _parse_state(state): 506 if state: 507 if state == "Reserved": 508 return win32.MEM_RESERVE 509 if state == "Commited": 510 return win32.MEM_COMMIT 511 if state == "Free": 512 return win32.MEM_FREE 513 return 0 514 515 @staticmethod 516 def _parse_type(type): 517 if type: 518 if type == "Image": 519 return win32.MEM_IMAGE 520 if type == "Mapped": 521 return win32.MEM_MAPPED 522 if type == "Private": 523 return win32.MEM_PRIVATE 524 return -1 525 return 0 526 527 @staticmethod 528 def _parse_access(access): 529 if not access: 530 return 0 531 perm = access[:3] 532 if perm == "R--": 533 protect = win32.PAGE_READONLY 534 elif perm == "RW-": 535 protect = win32.PAGE_READWRITE 536 elif perm == "RC-": 537 protect = win32.PAGE_WRITECOPY 538 elif perm == "--X": 539 protect = win32.PAGE_EXECUTE 540 elif perm == "R-X": 541 protect = win32.PAGE_EXECUTE_READ 542 elif perm == "RWX": 543 protect = win32.PAGE_EXECUTE_READWRITE 544 elif perm == "RCX": 545 protect = win32.PAGE_EXECUTE_WRITECOPY 546 else: 547 protect = win32.PAGE_NOACCESS 548 if access[5] == "G": 549 protect = protect | win32.PAGE_GUARD 550 if access[6] == "N": 551 protect = protect | win32.PAGE_NOCACHE 552 if access[7] == "W": 553 protect = protect | win32.PAGE_WRITECOMBINE 554 return protect 555 556#------------------------------------------------------------------------------ 557 558class CrashDTO (BaseDTO): 559 """ 560 Database mapping for crash dumps. 561 """ 562 563 # Table name. 564 __tablename__ = "crashes" 565 566 # Primary key. 567 id = Column(Integer, Sequence(__tablename__ + '_seq'), 568 primary_key = True, autoincrement = True) 569 570 # Timestamp. 571 timestamp = Column(DateTime, nullable = False, index = True) 572 573 # Exploitability test. 574 exploitable = Column(Integer, nullable = False) 575 exploitability_rule = Column(String(32), nullable = False) 576 exploitability_rating = Column(String(32), nullable = False) 577 exploitability_desc = Column(String, nullable = False) 578 579 # Platform description. 580 os = Column(String(32), nullable = False) 581 arch = Column(String(16), nullable = False) 582 bits = Column(Integer, nullable = False) # Integer(4) is deprecated :( 583 584 # Event description. 585 event = Column(String, nullable = False) 586 pid = Column(Integer, nullable = False) 587 tid = Column(Integer, nullable = False) 588 pc = Column(BigInteger, nullable = False) 589 sp = Column(BigInteger, nullable = False) 590 fp = Column(BigInteger, nullable = False) 591 pc_label = Column(String, nullable = False) 592 593 # Exception description. 594 exception = Column(String(64)) 595 exception_text = Column(String(64)) 596 exception_address = Column(BigInteger) 597 exception_label = Column(String) 598 first_chance = Column(Boolean) 599 fault_type = Column(Integer) 600 fault_address = Column(BigInteger) 601 fault_label = Column(String) 602 fault_disasm = Column(String) 603 stack_trace = Column(String) 604 605 # Environment description. 606 command_line = Column(String) 607 environment = Column(String) 608 609 # Debug strings. 610 debug_string = Column(String) 611 612 # Notes. 613 notes = Column(String) 614 615 # Heuristic signature. 616 signature = Column(String, nullable = False) 617 618 # Pickled Crash object, minus the memory dump. 619 data = deferred(Column(LargeBinary, nullable = False)) 620 621 def __init__(self, crash): 622 """ 623 @type crash: Crash 624 @param crash: L{Crash} object to store into the database. 625 """ 626 627 # Timestamp and signature. 628 self.timestamp = datetime.datetime.fromtimestamp( crash.timeStamp ) 629 self.signature = pickle.dumps(crash.signature, protocol = 0) 630 631 # Marshalled Crash object, minus the memory dump. 632 # This code is *not* thread safe! 633 memoryMap = crash.memoryMap 634 try: 635 crash.memoryMap = None 636 self.data = buffer( Marshaller.dumps(crash) ) 637 finally: 638 crash.memoryMap = memoryMap 639 640 # Exploitability test. 641 self.exploitability_rating, \ 642 self.exploitability_rule, \ 643 self.exploitability_desc = crash.isExploitable() 644 645 # Exploitability test as an integer result (for sorting). 646 self.exploitable = [ 647 "Not an exception", 648 "Not exploitable", 649 "Not likely exploitable", 650 "Unknown", 651 "Probably exploitable", 652 "Exploitable", 653 ].index(self.exploitability_rating) 654 655 # Platform description. 656 self.os = crash.os 657 self.arch = crash.arch 658 self.bits = crash.bits 659 660 # Event description. 661 self.event = crash.eventName 662 self.pid = crash.pid 663 self.tid = crash.tid 664 self.pc = crash.pc 665 self.sp = crash.sp 666 self.fp = crash.fp 667 self.pc_label = crash.labelPC 668 669 # Exception description. 670 self.exception = crash.exceptionName 671 self.exception_text = crash.exceptionDescription 672 self.exception_address = crash.exceptionAddress 673 self.exception_label = crash.exceptionLabel 674 self.first_chance = crash.firstChance 675 self.fault_type = crash.faultType 676 self.fault_address = crash.faultAddress 677 self.fault_label = crash.faultLabel 678 self.fault_disasm = CrashDump.dump_code( crash.faultDisasm, 679 crash.pc ) 680 self.stack_trace = CrashDump.dump_stack_trace_with_labels( 681 crash.stackTracePretty ) 682 683 # Command line. 684 self.command_line = crash.commandLine 685 686 # Environment. 687 if crash.environment: 688 envList = crash.environment.items() 689 envList.sort() 690 environment = '' 691 for envKey, envVal in envList: 692 # Must concatenate here instead of using a substitution, 693 # so strings can be automatically promoted to Unicode. 694 environment += envKey + '=' + envVal + '\n' 695 if environment: 696 self.environment = environment 697 698 # Debug string. 699 self.debug_string = crash.debugString 700 701 # Notes. 702 self.notes = crash.notesReport() 703 704 def toCrash(self, getMemoryDump = False): 705 """ 706 Returns a L{Crash} object using the data retrieved from the database. 707 708 @type getMemoryDump: bool 709 @param getMemoryDump: If C{True} retrieve the memory dump. 710 Defaults to C{False} since this may be a costly operation. 711 712 @rtype: L{Crash} 713 @return: Crash object. 714 """ 715 crash = Marshaller.loads(str(self.data)) 716 if not isinstance(crash, Crash): 717 raise TypeError( 718 "Expected Crash instance, got %s instead" % type(crash)) 719 crash._rowid = self.id 720 if not crash.memoryMap: 721 memory = getattr(self, "memory", []) 722 if memory: 723 crash.memoryMap = [dto.toMBI(getMemoryDump) for dto in memory] 724 return crash 725 726#============================================================================== 727 728# TODO: add a method to modify already stored crash dumps. 729 730class CrashDAO (BaseDAO): 731 """ 732 Data Access Object to read, write and search for L{Crash} objects in a 733 database. 734 """ 735 736 @Transactional 737 def add(self, crash, allow_duplicates = True): 738 """ 739 Add a new crash dump to the database, optionally filtering them by 740 signature to avoid duplicates. 741 742 @type crash: L{Crash} 743 @param crash: Crash object. 744 745 @type allow_duplicates: bool 746 @param allow_duplicates: (Optional) 747 C{True} to always add the new crash dump. 748 C{False} to only add the crash dump if no other crash with the 749 same signature is found in the database. 750 751 Sometimes, your fuzzer turns out to be I{too} good. Then you find 752 youself browsing through gigabytes of crash dumps, only to find 753 a handful of actual bugs in them. This simple heuristic filter 754 saves you the trouble by discarding crashes that seem to be similar 755 to another one you've already found. 756 """ 757 758 # Filter out duplicated crashes, if requested. 759 if not allow_duplicates: 760 signature = pickle.dumps(crash.signature, protocol = 0) 761 if self._session.query(CrashDTO.id) \ 762 .filter_by(signature = signature) \ 763 .count() > 0: 764 return 765 766 # Fill out a new row for the crashes table. 767 crash_id = self.__add_crash(crash) 768 769 # Fill out new rows for the memory dump. 770 self.__add_memory(crash_id, crash.memoryMap) 771 772 # On success set the row ID for the Crash object. 773 # WARNING: In nested calls, make sure to delete 774 # this property before a session rollback! 775 crash._rowid = crash_id 776 777 # Store the Crash object into the crashes table. 778 def __add_crash(self, crash): 779 session = self._session 780 r_crash = None 781 try: 782 783 # Fill out a new row for the crashes table. 784 r_crash = CrashDTO(crash) 785 session.add(r_crash) 786 787 # Flush and get the new row ID. 788 session.flush() 789 crash_id = r_crash.id 790 791 finally: 792 try: 793 794 # Make the ORM forget the CrashDTO object. 795 if r_crash is not None: 796 session.expire(r_crash) 797 798 finally: 799 800 # Delete the last reference to the CrashDTO 801 # object, so the Python garbage collector claims it. 802 del r_crash 803 804 # Return the row ID. 805 return crash_id 806 807 # Store the memory dump into the memory table. 808 def __add_memory(self, crash_id, memoryMap): 809 session = self._session 810 if memoryMap: 811 for mbi in memoryMap: 812 r_mem = MemoryDTO(crash_id, mbi) 813 session.add(r_mem) 814 session.flush() 815 816 @Transactional 817 def find(self, 818 signature = None, order = 0, 819 since = None, until = None, 820 offset = None, limit = None): 821 """ 822 Retrieve all crash dumps in the database, optionally filtering them by 823 signature and timestamp, and/or sorting them by timestamp. 824 825 Results can be paged to avoid consuming too much memory if the database 826 is large. 827 828 @see: L{find_by_example} 829 830 @type signature: object 831 @param signature: (Optional) Return only through crashes matching 832 this signature. See L{Crash.signature} for more details. 833 834 @type order: int 835 @param order: (Optional) Sort by timestamp. 836 If C{== 0}, results are not sorted. 837 If C{> 0}, results are sorted from older to newer. 838 If C{< 0}, results are sorted from newer to older. 839 840 @type since: datetime 841 @param since: (Optional) Return only the crashes after and 842 including this date and time. 843 844 @type until: datetime 845 @param until: (Optional) Return only the crashes before this date 846 and time, not including it. 847 848 @type offset: int 849 @param offset: (Optional) Skip the first I{offset} results. 850 851 @type limit: int 852 @param limit: (Optional) Return at most I{limit} results. 853 854 @rtype: list(L{Crash}) 855 @return: List of Crash objects. 856 """ 857 858 # Validate the parameters. 859 if since and until and since > until: 860 warnings.warn("CrashDAO.find() got the 'since' and 'until'" 861 " arguments reversed, corrected automatically.") 862 since, until = until, since 863 if limit is not None and not limit: 864 warnings.warn("CrashDAO.find() was set a limit of 0 results," 865 " returning without executing a query.") 866 return [] 867 868 # Build the SQL query. 869 query = self._session.query(CrashDTO) 870 if signature is not None: 871 sig_pickled = pickle.dumps(signature, protocol = 0) 872 query = query.filter(CrashDTO.signature == sig_pickled) 873 if since: 874 query = query.filter(CrashDTO.timestamp >= since) 875 if until: 876 query = query.filter(CrashDTO.timestamp < until) 877 if order: 878 if order > 0: 879 query = query.order_by(asc(CrashDTO.timestamp)) 880 else: 881 query = query.order_by(desc(CrashDTO.timestamp)) 882 else: 883 # Default ordering is by row ID, to get consistent results. 884 # Also some database engines require ordering when using offsets. 885 query = query.order_by(asc(CrashDTO.id)) 886 if offset: 887 query = query.offset(offset) 888 if limit: 889 query = query.limit(limit) 890 891 # Execute the SQL query and convert the results. 892 try: 893 return [dto.toCrash() for dto in query.all()] 894 except NoResultFound: 895 return [] 896 897 @Transactional 898 def find_by_example(self, crash, offset = None, limit = None): 899 """ 900 Find all crash dumps that have common properties with the crash dump 901 provided. 902 903 Results can be paged to avoid consuming too much memory if the database 904 is large. 905 906 @see: L{find} 907 908 @type crash: L{Crash} 909 @param crash: Crash object to compare with. Fields set to C{None} are 910 ignored, all other fields but the signature are used in the 911 comparison. 912 913 To search for signature instead use the L{find} method. 914 915 @type offset: int 916 @param offset: (Optional) Skip the first I{offset} results. 917 918 @type limit: int 919 @param limit: (Optional) Return at most I{limit} results. 920 921 @rtype: list(L{Crash}) 922 @return: List of similar crash dumps found. 923 """ 924 925 # Validate the parameters. 926 if limit is not None and not limit: 927 warnings.warn("CrashDAO.find_by_example() was set a limit of 0" 928 " results, returning without executing a query.") 929 return [] 930 931 # Build the query. 932 query = self._session.query(CrashDTO) 933 934 # Order by row ID to get consistent results. 935 # Also some database engines require ordering when using offsets. 936 query = query.asc(CrashDTO.id) 937 938 # Build a CrashDTO from the Crash object. 939 dto = CrashDTO(crash) 940 941 # Filter all the fields in the crashes table that are present in the 942 # CrashDTO object and not set to None, except for the row ID. 943 for name, column in compat.iteritems(CrashDTO.__dict__): 944 if not name.startswith('__') and name not in ('id', 945 'signature', 946 'data'): 947 if isinstance(column, Column): 948 value = getattr(dto, name, None) 949 if value is not None: 950 query = query.filter(column == value) 951 952 # Page the query. 953 if offset: 954 query = query.offset(offset) 955 if limit: 956 query = query.limit(limit) 957 958 # Execute the SQL query and convert the results. 959 try: 960 return [dto.toCrash() for dto in query.all()] 961 except NoResultFound: 962 return [] 963 964 @Transactional 965 def count(self, signature = None): 966 """ 967 Counts how many crash dumps have been stored in this database. 968 Optionally filters the count by heuristic signature. 969 970 @type signature: object 971 @param signature: (Optional) Count only the crashes that match 972 this signature. See L{Crash.signature} for more details. 973 974 @rtype: int 975 @return: Count of crash dumps stored in this database. 976 """ 977 query = self._session.query(CrashDTO.id) 978 if signature: 979 sig_pickled = pickle.dumps(signature, protocol = 0) 980 query = query.filter_by(signature = sig_pickled) 981 return query.count() 982 983 @Transactional 984 def delete(self, crash): 985 """ 986 Remove the given crash dump from the database. 987 988 @type crash: L{Crash} 989 @param crash: Crash dump to remove. 990 """ 991 query = self._session.query(CrashDTO).filter_by(id = crash._rowid) 992 query.delete(synchronize_session = False) 993 del crash._rowid 994