1#!/usr/local/bin/python3.8 2# 3# Electrum - lightweight Bitcoin client 4# Copyright (C) 2015 Thomas Voegtlin 5# 6# Permission is hereby granted, free of charge, to any person 7# obtaining a copy of this software and associated documentation files 8# (the "Software"), to deal in the Software without restriction, 9# including without limitation the rights to use, copy, modify, merge, 10# publish, distribute, sublicense, and/or sell copies of the Software, 11# and to permit persons to whom the Software is furnished to do so, 12# subject to the following conditions: 13# 14# The above copyright notice and this permission notice shall be 15# included in all copies or substantial portions of the Software. 16# 17# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 21# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 23# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24# SOFTWARE. 25import os 26import ast 27import json 28import copy 29import threading 30from collections import defaultdict 31from typing import Dict, Optional, List, Tuple, Set, Iterable, NamedTuple, Sequence, TYPE_CHECKING, Union 32import binascii 33 34from . import util, bitcoin 35from .util import profiler, WalletFileException, multisig_type, TxMinedInfo, bfh 36from .invoices import PR_TYPE_ONCHAIN, Invoice 37from .keystore import bip44_derivation 38from .transaction import Transaction, TxOutpoint, tx_from_any, PartialTransaction, PartialTxOutput 39from .logging import Logger 40from .lnutil import LOCAL, REMOTE, FeeUpdate, UpdateAddHtlc, LocalConfig, RemoteConfig, Keypair, OnlyPubkeyKeypair, RevocationStore 41from .lnutil import ImportedChannelBackupStorage, OnchainChannelBackupStorage 42from .lnutil import ChannelConstraints, Outpoint, ShachainElement 43from .json_db import StoredDict, JsonDB, locked, modifier 44from .plugin import run_hook, plugin_loaders 45from .paymentrequest import PaymentRequest 46from .submarine_swaps import SwapData 47 48if TYPE_CHECKING: 49 from .storage import WalletStorage 50 51 52# seed_version is now used for the version of the wallet file 53 54OLD_SEED_VERSION = 4 # electrum versions < 2.0 55NEW_SEED_VERSION = 11 # electrum versions >= 2.0 56FINAL_SEED_VERSION = 41 # electrum >= 2.7 will set this to prevent 57 # old versions from overwriting new format 58 59 60class TxFeesValue(NamedTuple): 61 fee: Optional[int] = None 62 is_calculated_by_us: bool = False 63 num_inputs: Optional[int] = None 64 65 66class WalletDB(JsonDB): 67 68 def __init__(self, raw, *, manual_upgrades: bool): 69 JsonDB.__init__(self, {}) 70 self._manual_upgrades = manual_upgrades 71 self._called_after_upgrade_tasks = False 72 if raw: # loading existing db 73 self.load_data(raw) 74 self.load_plugins() 75 else: # creating new db 76 self.put('seed_version', FINAL_SEED_VERSION) 77 self._after_upgrade_tasks() 78 79 def load_data(self, s): 80 try: 81 self.data = json.loads(s) 82 except: 83 try: 84 d = ast.literal_eval(s) 85 labels = d.get('labels', {}) 86 except Exception as e: 87 raise WalletFileException("Cannot read wallet file. (parsing failed)") 88 self.data = {} 89 for key, value in d.items(): 90 try: 91 json.dumps(key) 92 json.dumps(value) 93 except: 94 self.logger.info(f'Failed to convert label to json format: {key}') 95 continue 96 self.data[key] = value 97 if not isinstance(self.data, dict): 98 raise WalletFileException("Malformed wallet file (not dict)") 99 100 if not self._manual_upgrades and self.requires_split(): 101 raise WalletFileException("This wallet has multiple accounts and must be split") 102 103 if not self.requires_upgrade(): 104 self._after_upgrade_tasks() 105 elif not self._manual_upgrades: 106 self.upgrade() 107 108 def requires_split(self): 109 d = self.get('accounts', {}) 110 return len(d) > 1 111 112 def get_split_accounts(self): 113 result = [] 114 # backward compatibility with old wallets 115 d = self.get('accounts', {}) 116 if len(d) < 2: 117 return 118 wallet_type = self.get('wallet_type') 119 if wallet_type == 'old': 120 assert len(d) == 2 121 data1 = copy.deepcopy(self.data) 122 data1['accounts'] = {'0': d['0']} 123 data1['suffix'] = 'deterministic' 124 data2 = copy.deepcopy(self.data) 125 data2['accounts'] = {'/x': d['/x']} 126 data2['seed'] = None 127 data2['seed_version'] = None 128 data2['master_public_key'] = None 129 data2['wallet_type'] = 'imported' 130 data2['suffix'] = 'imported' 131 result = [data1, data2] 132 133 elif wallet_type in ['bip44', 'trezor', 'keepkey', 'ledger', 'btchip', 'digitalbitbox', 'safe_t']: 134 mpk = self.get('master_public_keys') 135 for k in d.keys(): 136 i = int(k) 137 x = d[k] 138 if x.get("pending"): 139 continue 140 xpub = mpk["x/%d'"%i] 141 new_data = copy.deepcopy(self.data) 142 # save account, derivation and xpub at index 0 143 new_data['accounts'] = {'0': x} 144 new_data['master_public_keys'] = {"x/0'": xpub} 145 new_data['derivation'] = bip44_derivation(k) 146 new_data['suffix'] = k 147 result.append(new_data) 148 else: 149 raise WalletFileException("This wallet has multiple accounts and must be split") 150 return result 151 152 def requires_upgrade(self): 153 return self.get_seed_version() < FINAL_SEED_VERSION 154 155 @profiler 156 def upgrade(self): 157 self.logger.info('upgrading wallet format') 158 if self._called_after_upgrade_tasks: 159 # we need strict ordering between upgrade() and after_upgrade_tasks() 160 raise Exception("'after_upgrade_tasks' must NOT be called before 'upgrade'") 161 self._convert_imported() 162 self._convert_wallet_type() 163 self._convert_account() 164 self._convert_version_13_b() 165 self._convert_version_14() 166 self._convert_version_15() 167 self._convert_version_16() 168 self._convert_version_17() 169 self._convert_version_18() 170 self._convert_version_19() 171 self._convert_version_20() 172 self._convert_version_21() 173 self._convert_version_22() 174 self._convert_version_23() 175 self._convert_version_24() 176 self._convert_version_25() 177 self._convert_version_26() 178 self._convert_version_27() 179 self._convert_version_28() 180 self._convert_version_29() 181 self._convert_version_30() 182 self._convert_version_31() 183 self._convert_version_32() 184 self._convert_version_33() 185 self._convert_version_34() 186 self._convert_version_35() 187 self._convert_version_36() 188 self._convert_version_37() 189 self._convert_version_38() 190 self._convert_version_39() 191 self._convert_version_40() 192 self._convert_version_41() 193 self.put('seed_version', FINAL_SEED_VERSION) # just to be sure 194 195 self._after_upgrade_tasks() 196 197 def _after_upgrade_tasks(self): 198 self._called_after_upgrade_tasks = True 199 self._load_transactions() 200 201 def _convert_wallet_type(self): 202 if not self._is_upgrade_method_needed(0, 13): 203 return 204 205 wallet_type = self.get('wallet_type') 206 if wallet_type == 'btchip': wallet_type = 'ledger' 207 if self.get('keystore') or self.get('x1/') or wallet_type=='imported': 208 return False 209 assert not self.requires_split() 210 seed_version = self.get_seed_version() 211 seed = self.get('seed') 212 xpubs = self.get('master_public_keys') 213 xprvs = self.get('master_private_keys', {}) 214 mpk = self.get('master_public_key') 215 keypairs = self.get('keypairs') 216 key_type = self.get('key_type') 217 if seed_version == OLD_SEED_VERSION or wallet_type == 'old': 218 d = { 219 'type': 'old', 220 'seed': seed, 221 'mpk': mpk, 222 } 223 self.put('wallet_type', 'standard') 224 self.put('keystore', d) 225 226 elif key_type == 'imported': 227 d = { 228 'type': 'imported', 229 'keypairs': keypairs, 230 } 231 self.put('wallet_type', 'standard') 232 self.put('keystore', d) 233 234 elif wallet_type in ['xpub', 'standard']: 235 xpub = xpubs["x/"] 236 xprv = xprvs.get("x/") 237 d = { 238 'type': 'bip32', 239 'xpub': xpub, 240 'xprv': xprv, 241 'seed': seed, 242 } 243 self.put('wallet_type', 'standard') 244 self.put('keystore', d) 245 246 elif wallet_type in ['bip44']: 247 xpub = xpubs["x/0'"] 248 xprv = xprvs.get("x/0'") 249 d = { 250 'type': 'bip32', 251 'xpub': xpub, 252 'xprv': xprv, 253 } 254 self.put('wallet_type', 'standard') 255 self.put('keystore', d) 256 257 elif wallet_type in ['trezor', 'keepkey', 'ledger', 'digitalbitbox', 'safe_t']: 258 xpub = xpubs["x/0'"] 259 derivation = self.get('derivation', bip44_derivation(0)) 260 d = { 261 'type': 'hardware', 262 'hw_type': wallet_type, 263 'xpub': xpub, 264 'derivation': derivation, 265 } 266 self.put('wallet_type', 'standard') 267 self.put('keystore', d) 268 269 elif (wallet_type == '2fa') or multisig_type(wallet_type): 270 for key in xpubs.keys(): 271 d = { 272 'type': 'bip32', 273 'xpub': xpubs[key], 274 'xprv': xprvs.get(key), 275 } 276 if key == 'x1/' and seed: 277 d['seed'] = seed 278 self.put(key, d) 279 else: 280 raise WalletFileException('Unable to tell wallet type. Is this even a wallet file?') 281 # remove junk 282 self.put('master_public_key', None) 283 self.put('master_public_keys', None) 284 self.put('master_private_keys', None) 285 self.put('derivation', None) 286 self.put('seed', None) 287 self.put('keypairs', None) 288 self.put('key_type', None) 289 290 def _convert_version_13_b(self): 291 # version 13 is ambiguous, and has an earlier and a later structure 292 if not self._is_upgrade_method_needed(0, 13): 293 return 294 295 if self.get('wallet_type') == 'standard': 296 if self.get('keystore').get('type') == 'imported': 297 pubkeys = self.get('keystore').get('keypairs').keys() 298 d = {'change': []} 299 receiving_addresses = [] 300 for pubkey in pubkeys: 301 addr = bitcoin.pubkey_to_address('p2pkh', pubkey) 302 receiving_addresses.append(addr) 303 d['receiving'] = receiving_addresses 304 self.put('addresses', d) 305 self.put('pubkeys', None) 306 307 self.put('seed_version', 13) 308 309 def _convert_version_14(self): 310 # convert imported wallets for 3.0 311 if not self._is_upgrade_method_needed(13, 13): 312 return 313 314 if self.get('wallet_type') =='imported': 315 addresses = self.get('addresses') 316 if type(addresses) is list: 317 addresses = dict([(x, None) for x in addresses]) 318 self.put('addresses', addresses) 319 elif self.get('wallet_type') == 'standard': 320 if self.get('keystore').get('type')=='imported': 321 addresses = set(self.get('addresses').get('receiving')) 322 pubkeys = self.get('keystore').get('keypairs').keys() 323 assert len(addresses) == len(pubkeys) 324 d = {} 325 for pubkey in pubkeys: 326 addr = bitcoin.pubkey_to_address('p2pkh', pubkey) 327 assert addr in addresses 328 d[addr] = { 329 'pubkey': pubkey, 330 'redeem_script': None, 331 'type': 'p2pkh' 332 } 333 self.put('addresses', d) 334 self.put('pubkeys', None) 335 self.put('wallet_type', 'imported') 336 self.put('seed_version', 14) 337 338 def _convert_version_15(self): 339 if not self._is_upgrade_method_needed(14, 14): 340 return 341 if self.get('seed_type') == 'segwit': 342 # should not get here; get_seed_version should have caught this 343 raise Exception('unsupported derivation (development segwit, v14)') 344 self.put('seed_version', 15) 345 346 def _convert_version_16(self): 347 # fixes issue #3193 for Imported_Wallets with addresses 348 # also, previous versions allowed importing any garbage as an address 349 # which we now try to remove, see pr #3191 350 if not self._is_upgrade_method_needed(15, 15): 351 return 352 353 def remove_address(addr): 354 def remove_from_dict(dict_name): 355 d = self.get(dict_name, None) 356 if d is not None: 357 d.pop(addr, None) 358 self.put(dict_name, d) 359 360 def remove_from_list(list_name): 361 lst = self.get(list_name, None) 362 if lst is not None: 363 s = set(lst) 364 s -= {addr} 365 self.put(list_name, list(s)) 366 367 # note: we don't remove 'addr' from self.get('addresses') 368 remove_from_dict('addr_history') 369 remove_from_dict('labels') 370 remove_from_dict('payment_requests') 371 remove_from_list('frozen_addresses') 372 373 if self.get('wallet_type') == 'imported': 374 addresses = self.get('addresses') 375 assert isinstance(addresses, dict) 376 addresses_new = dict() 377 for address, details in addresses.items(): 378 if not bitcoin.is_address(address): 379 remove_address(address) 380 continue 381 if details is None: 382 addresses_new[address] = {} 383 else: 384 addresses_new[address] = details 385 self.put('addresses', addresses_new) 386 387 self.put('seed_version', 16) 388 389 def _convert_version_17(self): 390 # delete pruned_txo; construct spent_outpoints 391 if not self._is_upgrade_method_needed(16, 16): 392 return 393 394 self.put('pruned_txo', None) 395 396 transactions = self.get('transactions', {}) # txid -> raw_tx 397 spent_outpoints = defaultdict(dict) 398 for txid, raw_tx in transactions.items(): 399 tx = Transaction(raw_tx) 400 for txin in tx.inputs(): 401 if txin.is_coinbase_input(): 402 continue 403 prevout_hash = txin.prevout.txid.hex() 404 prevout_n = txin.prevout.out_idx 405 spent_outpoints[prevout_hash][str(prevout_n)] = txid 406 self.put('spent_outpoints', spent_outpoints) 407 408 self.put('seed_version', 17) 409 410 def _convert_version_18(self): 411 # delete verified_tx3 as its structure changed 412 if not self._is_upgrade_method_needed(17, 17): 413 return 414 self.put('verified_tx3', None) 415 self.put('seed_version', 18) 416 417 def _convert_version_19(self): 418 # delete tx_fees as its structure changed 419 if not self._is_upgrade_method_needed(18, 18): 420 return 421 self.put('tx_fees', None) 422 self.put('seed_version', 19) 423 424 def _convert_version_20(self): 425 # store 'derivation' (prefix) and 'root_fingerprint' in all xpub-based keystores. 426 # store explicit None values if we cannot retroactively determine them 427 if not self._is_upgrade_method_needed(19, 19): 428 return 429 430 from .bip32 import BIP32Node, convert_bip32_intpath_to_strpath 431 # note: This upgrade method reimplements bip32.root_fp_and_der_prefix_from_xkey. 432 # This is done deliberately, to avoid introducing that method as a dependency to this upgrade. 433 for ks_name in ('keystore', *['x{}/'.format(i) for i in range(1, 16)]): 434 ks = self.get(ks_name, None) 435 if ks is None: continue 436 xpub = ks.get('xpub', None) 437 if xpub is None: continue 438 bip32node = BIP32Node.from_xkey(xpub) 439 # derivation prefix 440 derivation_prefix = ks.get('derivation', None) 441 if derivation_prefix is None: 442 assert bip32node.depth >= 0, bip32node.depth 443 if bip32node.depth == 0: 444 derivation_prefix = 'm' 445 elif bip32node.depth == 1: 446 child_number_int = int.from_bytes(bip32node.child_number, 'big') 447 derivation_prefix = convert_bip32_intpath_to_strpath([child_number_int]) 448 ks['derivation'] = derivation_prefix 449 # root fingerprint 450 root_fingerprint = ks.get('ckcc_xfp', None) 451 if root_fingerprint is not None: 452 root_fingerprint = root_fingerprint.to_bytes(4, byteorder="little", signed=False).hex().lower() 453 if root_fingerprint is None: 454 if bip32node.depth == 0: 455 root_fingerprint = bip32node.calc_fingerprint_of_this_node().hex().lower() 456 elif bip32node.depth == 1: 457 root_fingerprint = bip32node.fingerprint.hex() 458 ks['root_fingerprint'] = root_fingerprint 459 ks.pop('ckcc_xfp', None) 460 self.put(ks_name, ks) 461 462 self.put('seed_version', 20) 463 464 def _convert_version_21(self): 465 if not self._is_upgrade_method_needed(20, 20): 466 return 467 channels = self.get('channels') 468 if channels: 469 for channel in channels: 470 channel['state'] = 'OPENING' 471 self.put('channels', channels) 472 self.put('seed_version', 21) 473 474 def _convert_version_22(self): 475 # construct prevouts_by_scripthash 476 if not self._is_upgrade_method_needed(21, 21): 477 return 478 479 from .bitcoin import script_to_scripthash 480 transactions = self.get('transactions', {}) # txid -> raw_tx 481 prevouts_by_scripthash = defaultdict(list) 482 for txid, raw_tx in transactions.items(): 483 tx = Transaction(raw_tx) 484 for idx, txout in enumerate(tx.outputs()): 485 outpoint = f"{txid}:{idx}" 486 scripthash = script_to_scripthash(txout.scriptpubkey.hex()) 487 prevouts_by_scripthash[scripthash].append((outpoint, txout.value)) 488 self.put('prevouts_by_scripthash', prevouts_by_scripthash) 489 490 self.put('seed_version', 22) 491 492 def _convert_version_23(self): 493 if not self._is_upgrade_method_needed(22, 22): 494 return 495 channels = self.get('channels', []) 496 LOCAL = 1 497 REMOTE = -1 498 for c in channels: 499 # move revocation store from remote_config 500 r = c['remote_config'].pop('revocation_store') 501 c['revocation_store'] = r 502 # convert fee updates 503 log = c.get('log', {}) 504 for sub in LOCAL, REMOTE: 505 l = log[str(sub)]['fee_updates'] 506 d = {} 507 for i, fu in enumerate(l): 508 d[str(i)] = { 509 'rate':fu['rate'], 510 'ctn_local':fu['ctns'][str(LOCAL)], 511 'ctn_remote':fu['ctns'][str(REMOTE)] 512 } 513 log[str(int(sub))]['fee_updates'] = d 514 self.data['channels'] = channels 515 516 self.data['seed_version'] = 23 517 518 def _convert_version_24(self): 519 if not self._is_upgrade_method_needed(23, 23): 520 return 521 channels = self.get('channels', []) 522 for c in channels: 523 # convert revocation store to dict 524 r = c['revocation_store'] 525 d = {} 526 for i in range(49): 527 v = r['buckets'][i] 528 if v is not None: 529 d[str(i)] = v 530 r['buckets'] = d 531 c['revocation_store'] = r 532 # convert channels to dict 533 self.data['channels'] = {x['channel_id']: x for x in channels} 534 # convert txi & txo 535 txi = self.get('txi', {}) 536 for tx_hash, d in list(txi.items()): 537 d2 = {} 538 for addr, l in d.items(): 539 d2[addr] = {} 540 for ser, v in l: 541 d2[addr][ser] = v 542 txi[tx_hash] = d2 543 self.data['txi'] = txi 544 txo = self.get('txo', {}) 545 for tx_hash, d in list(txo.items()): 546 d2 = {} 547 for addr, l in d.items(): 548 d2[addr] = {} 549 for n, v, cb in l: 550 d2[addr][str(n)] = (v, cb) 551 txo[tx_hash] = d2 552 self.data['txo'] = txo 553 554 self.data['seed_version'] = 24 555 556 def _convert_version_25(self): 557 if not self._is_upgrade_method_needed(24, 24): 558 return 559 # add 'type' field to onchain requests 560 requests = self.data.get('payment_requests', {}) 561 for k, r in list(requests.items()): 562 if r.get('address') == k: 563 requests[k] = { 564 'address': r['address'], 565 'amount': r.get('amount'), 566 'exp': r.get('exp'), 567 'id': r.get('id'), 568 'memo': r.get('memo'), 569 'time': r.get('time'), 570 'type': PR_TYPE_ONCHAIN, 571 } 572 # convert bip70 invoices 573 invoices = self.data.get('invoices', {}) 574 for k, r in list(invoices.items()): 575 data = r.get("hex") 576 if data: 577 pr = PaymentRequest(bytes.fromhex(data)) 578 if pr.id != k: 579 continue 580 invoices[k] = { 581 'type': PR_TYPE_ONCHAIN, 582 'amount': pr.get_amount(), 583 'bip70': data, 584 'exp': pr.get_expiration_date() - pr.get_time(), 585 'id': pr.id, 586 'message': pr.get_memo(), 587 'outputs': [x.to_legacy_tuple() for x in pr.get_outputs()], 588 'time': pr.get_time(), 589 'requestor': pr.get_requestor(), 590 } 591 self.data['seed_version'] = 25 592 593 def _convert_version_26(self): 594 if not self._is_upgrade_method_needed(25, 25): 595 return 596 channels = self.data.get('channels', {}) 597 channel_timestamps = self.data.pop('lightning_channel_timestamps', {}) 598 for channel_id, c in channels.items(): 599 item = channel_timestamps.get(channel_id) 600 if item: 601 funding_txid, funding_height, funding_timestamp, closing_txid, closing_height, closing_timestamp = item 602 if funding_txid: 603 c['funding_height'] = funding_txid, funding_height, funding_timestamp 604 if closing_txid: 605 c['closing_height'] = closing_txid, closing_height, closing_timestamp 606 self.data['seed_version'] = 26 607 608 def _convert_version_27(self): 609 if not self._is_upgrade_method_needed(26, 26): 610 return 611 channels = self.data.get('channels', {}) 612 for channel_id, c in channels.items(): 613 c['local_config']['htlc_minimum_msat'] = 1 614 self.data['seed_version'] = 27 615 616 def _convert_version_28(self): 617 if not self._is_upgrade_method_needed(27, 27): 618 return 619 channels = self.data.get('channels', {}) 620 for channel_id, c in channels.items(): 621 c['local_config']['channel_seed'] = None 622 self.data['seed_version'] = 28 623 624 def _convert_version_29(self): 625 if not self._is_upgrade_method_needed(28, 28): 626 return 627 requests = self.data.get('payment_requests', {}) 628 invoices = self.data.get('invoices', {}) 629 for d in [invoices, requests]: 630 for key, r in list(d.items()): 631 _type = r.get('type', 0) 632 item = { 633 'type': _type, 634 'message': r.get('message') or r.get('memo', ''), 635 'amount': r.get('amount'), 636 'exp': r.get('exp') or 0, 637 'time': r.get('time', 0), 638 } 639 if _type == PR_TYPE_ONCHAIN: 640 address = r.pop('address', None) 641 if address: 642 outputs = [(0, address, r.get('amount'))] 643 else: 644 outputs = r.get('outputs') 645 item.update({ 646 'outputs': outputs, 647 'id': r.get('id'), 648 'bip70': r.get('bip70'), 649 'requestor': r.get('requestor'), 650 }) 651 else: 652 item.update({ 653 'rhash': r['rhash'], 654 'invoice': r['invoice'], 655 }) 656 d[key] = item 657 self.data['seed_version'] = 29 658 659 def _convert_version_30(self): 660 if not self._is_upgrade_method_needed(29, 29): 661 return 662 663 from .invoices import PR_TYPE_ONCHAIN, PR_TYPE_LN 664 requests = self.data.get('payment_requests', {}) 665 invoices = self.data.get('invoices', {}) 666 for d in [invoices, requests]: 667 for key, item in list(d.items()): 668 _type = item['type'] 669 if _type == PR_TYPE_ONCHAIN: 670 item['amount_sat'] = item.pop('amount') 671 elif _type == PR_TYPE_LN: 672 amount_sat = item.pop('amount') 673 item['amount_msat'] = 1000 * amount_sat if amount_sat is not None else None 674 item.pop('exp') 675 item.pop('message') 676 item.pop('rhash') 677 item.pop('time') 678 else: 679 raise Exception(f"unknown invoice type: {_type}") 680 self.data['seed_version'] = 30 681 682 def _convert_version_31(self): 683 if not self._is_upgrade_method_needed(30, 30): 684 return 685 686 from .invoices import PR_TYPE_ONCHAIN 687 requests = self.data.get('payment_requests', {}) 688 invoices = self.data.get('invoices', {}) 689 for d in [invoices, requests]: 690 for key, item in list(d.items()): 691 if item['type'] == PR_TYPE_ONCHAIN: 692 item['amount_sat'] = item['amount_sat'] or 0 693 item['exp'] = item['exp'] or 0 694 item['time'] = item['time'] or 0 695 self.data['seed_version'] = 31 696 697 def _convert_version_32(self): 698 if not self._is_upgrade_method_needed(31, 31): 699 return 700 PR_TYPE_ONCHAIN = 0 701 invoices_old = self.data.get('invoices', {}) 702 invoices_new = {k: item for k, item in invoices_old.items() 703 if not (item['type'] == PR_TYPE_ONCHAIN and item['outputs'] is None)} 704 self.data['invoices'] = invoices_new 705 self.data['seed_version'] = 32 706 707 def _convert_version_33(self): 708 if not self._is_upgrade_method_needed(32, 32): 709 return 710 PR_TYPE_ONCHAIN = 0 711 requests = self.data.get('payment_requests', {}) 712 invoices = self.data.get('invoices', {}) 713 for d in [invoices, requests]: 714 for key, item in list(d.items()): 715 if item['type'] == PR_TYPE_ONCHAIN: 716 item['height'] = item.get('height') or 0 717 self.data['seed_version'] = 33 718 719 def _convert_version_34(self): 720 if not self._is_upgrade_method_needed(33, 33): 721 return 722 channels = self.data.get('channels', {}) 723 for key, item in channels.items(): 724 item['local_config']['upfront_shutdown_script'] = \ 725 item['local_config'].get('upfront_shutdown_script') or "" 726 item['remote_config']['upfront_shutdown_script'] = \ 727 item['remote_config'].get('upfront_shutdown_script') or "" 728 self.data['seed_version'] = 34 729 730 def _convert_version_35(self): 731 # same as 32, but for payment_requests 732 if not self._is_upgrade_method_needed(34, 34): 733 return 734 PR_TYPE_ONCHAIN = 0 735 requests_old = self.data.get('payment_requests', {}) 736 requests_new = {k: item for k, item in requests_old.items() 737 if not (item['type'] == PR_TYPE_ONCHAIN and item['outputs'] is None)} 738 self.data['payment_requests'] = requests_new 739 self.data['seed_version'] = 35 740 741 def _convert_version_36(self): 742 if not self._is_upgrade_method_needed(35, 35): 743 return 744 old_frozen_coins = self.data.get('frozen_coins', []) 745 new_frozen_coins = {coin: True for coin in old_frozen_coins} 746 self.data['frozen_coins'] = new_frozen_coins 747 self.data['seed_version'] = 36 748 749 def _convert_version_37(self): 750 if not self._is_upgrade_method_needed(36, 36): 751 return 752 payments = self.data.get('lightning_payments', {}) 753 for k, v in list(payments.items()): 754 amount_sat, direction, status = v 755 amount_msat = amount_sat * 1000 if amount_sat is not None else None 756 payments[k] = amount_msat, direction, status 757 self.data['lightning_payments'] = payments 758 self.data['seed_version'] = 37 759 760 def _convert_version_38(self): 761 if not self._is_upgrade_method_needed(37, 37): 762 return 763 PR_TYPE_ONCHAIN = 0 764 PR_TYPE_LN = 2 765 from .bitcoin import TOTAL_COIN_SUPPLY_LIMIT_IN_BTC, COIN 766 max_sats = TOTAL_COIN_SUPPLY_LIMIT_IN_BTC * COIN 767 requests = self.data.get('payment_requests', {}) 768 invoices = self.data.get('invoices', {}) 769 for d in [invoices, requests]: 770 for key, item in list(d.items()): 771 if item['type'] == PR_TYPE_ONCHAIN: 772 amount_sat = item['amount_sat'] 773 if amount_sat == '!': 774 continue 775 if not (isinstance(amount_sat, int) and 0 <= amount_sat <= max_sats): 776 del d[key] 777 elif item['type'] == PR_TYPE_LN: 778 amount_msat = item['amount_msat'] 779 if not amount_msat: 780 continue 781 if not (isinstance(amount_msat, int) and 0 <= amount_msat <= max_sats * 1000): 782 del d[key] 783 self.data['seed_version'] = 38 784 785 def _convert_version_39(self): 786 # this upgrade prevents initialization of lightning_privkey2 after lightning_xprv has been set 787 if not self._is_upgrade_method_needed(38, 38): 788 return 789 self.data['imported_channel_backups'] = self.data.pop('channel_backups', {}) 790 self.data['seed_version'] = 39 791 792 def _convert_version_40(self): 793 # put 'seed_type' into keystores 794 if not self._is_upgrade_method_needed(39, 39): 795 return 796 for ks_name in ('keystore', *['x{}/'.format(i) for i in range(1, 16)]): 797 ks = self.data.get(ks_name, None) 798 if ks is None: continue 799 seed = ks.get('seed') 800 if not seed: continue 801 seed_type = None 802 xpub = ks.get('xpub') or None 803 if xpub: 804 assert isinstance(xpub, str) 805 if xpub[0:4] in ('xpub', 'tpub'): 806 seed_type = 'standard' 807 elif xpub[0:4] in ('zpub', 'Zpub', 'vpub', 'Vpub'): 808 seed_type = 'segwit' 809 elif ks.get('type') == 'old': 810 seed_type = 'old' 811 if seed_type is not None: 812 ks['seed_type'] = seed_type 813 self.data['seed_version'] = 40 814 815 def _convert_version_41(self): 816 # this is a repeat of upgrade 39, to fix wallet backup files (see #7339) 817 if not self._is_upgrade_method_needed(40, 40): 818 return 819 imported_channel_backups = self.data.pop('channel_backups', {}) 820 imported_channel_backups.update(self.data.get('imported_channel_backups', {})) 821 self.data['imported_channel_backups'] = imported_channel_backups 822 self.data['seed_version'] = 41 823 824 def _convert_imported(self): 825 if not self._is_upgrade_method_needed(0, 13): 826 return 827 828 # '/x' is the internal ID for imported accounts 829 d = self.get('accounts', {}).get('/x', {}).get('imported',{}) 830 if not d: 831 return False 832 addresses = [] 833 keypairs = {} 834 for addr, v in d.items(): 835 pubkey, privkey = v 836 if privkey: 837 keypairs[pubkey] = privkey 838 else: 839 addresses.append(addr) 840 if addresses and keypairs: 841 raise WalletFileException('mixed addresses and privkeys') 842 elif addresses: 843 self.put('addresses', addresses) 844 self.put('accounts', None) 845 elif keypairs: 846 self.put('wallet_type', 'standard') 847 self.put('key_type', 'imported') 848 self.put('keypairs', keypairs) 849 self.put('accounts', None) 850 else: 851 raise WalletFileException('no addresses or privkeys') 852 853 def _convert_account(self): 854 if not self._is_upgrade_method_needed(0, 13): 855 return 856 self.put('accounts', None) 857 858 def _is_upgrade_method_needed(self, min_version, max_version): 859 assert min_version <= max_version 860 cur_version = self.get_seed_version() 861 if cur_version > max_version: 862 return False 863 elif cur_version < min_version: 864 raise WalletFileException( 865 'storage upgrade: unexpected version {} (should be {}-{})' 866 .format(cur_version, min_version, max_version)) 867 else: 868 return True 869 870 @locked 871 def get_seed_version(self): 872 seed_version = self.get('seed_version') 873 if not seed_version: 874 seed_version = OLD_SEED_VERSION if len(self.get('master_public_key','')) == 128 else NEW_SEED_VERSION 875 if seed_version > FINAL_SEED_VERSION: 876 raise WalletFileException('This version of Electrum is too old to open this wallet.\n' 877 '(highest supported storage version: {}, version of this file: {})' 878 .format(FINAL_SEED_VERSION, seed_version)) 879 if seed_version==14 and self.get('seed_type') == 'segwit': 880 self._raise_unsupported_version(seed_version) 881 if seed_version >=12: 882 return seed_version 883 if seed_version not in [OLD_SEED_VERSION, NEW_SEED_VERSION]: 884 self._raise_unsupported_version(seed_version) 885 return seed_version 886 887 def _raise_unsupported_version(self, seed_version): 888 msg = f"Your wallet has an unsupported seed version: {seed_version}." 889 if seed_version in [5, 7, 8, 9, 10, 14]: 890 msg += "\n\nTo open this wallet, try 'git checkout seed_v%d'"%seed_version 891 if seed_version == 6: 892 # version 1.9.8 created v6 wallets when an incorrect seed was entered in the restore dialog 893 msg += '\n\nThis file was created because of a bug in version 1.9.8.' 894 if self.get('master_public_keys') is None and self.get('master_private_keys') is None and self.get('imported_keys') is None: 895 # pbkdf2 (at that time an additional dependency) was not included with the binaries, and wallet creation aborted. 896 msg += "\nIt does not contain any keys, and can safely be removed." 897 else: 898 # creation was complete if electrum was run from source 899 msg += "\nPlease open this file with Electrum 1.9.8, and move your coins to a new wallet." 900 raise WalletFileException(msg) 901 902 @locked 903 def get_txi_addresses(self, tx_hash: str) -> List[str]: 904 """Returns list of is_mine addresses that appear as inputs in tx.""" 905 assert isinstance(tx_hash, str) 906 return list(self.txi.get(tx_hash, {}).keys()) 907 908 @locked 909 def get_txo_addresses(self, tx_hash: str) -> List[str]: 910 """Returns list of is_mine addresses that appear as outputs in tx.""" 911 assert isinstance(tx_hash, str) 912 return list(self.txo.get(tx_hash, {}).keys()) 913 914 @locked 915 def get_txi_addr(self, tx_hash: str, address: str) -> Iterable[Tuple[str, int]]: 916 """Returns an iterable of (prev_outpoint, value).""" 917 assert isinstance(tx_hash, str) 918 assert isinstance(address, str) 919 d = self.txi.get(tx_hash, {}).get(address, {}) 920 return list(d.items()) 921 922 @locked 923 def get_txo_addr(self, tx_hash: str, address: str) -> Dict[int, Tuple[int, bool]]: 924 """Returns a dict: output_index -> (value, is_coinbase).""" 925 assert isinstance(tx_hash, str) 926 assert isinstance(address, str) 927 d = self.txo.get(tx_hash, {}).get(address, {}) 928 return {int(n): (v, cb) for (n, (v, cb)) in d.items()} 929 930 @modifier 931 def add_txi_addr(self, tx_hash: str, addr: str, ser: str, v: int) -> None: 932 assert isinstance(tx_hash, str) 933 assert isinstance(addr, str) 934 assert isinstance(ser, str) 935 assert isinstance(v, int) 936 if tx_hash not in self.txi: 937 self.txi[tx_hash] = {} 938 d = self.txi[tx_hash] 939 if addr not in d: 940 d[addr] = {} 941 d[addr][ser] = v 942 943 @modifier 944 def add_txo_addr(self, tx_hash: str, addr: str, n: Union[int, str], v: int, is_coinbase: bool) -> None: 945 n = str(n) 946 assert isinstance(tx_hash, str) 947 assert isinstance(addr, str) 948 assert isinstance(n, str) 949 assert isinstance(v, int) 950 assert isinstance(is_coinbase, bool) 951 if tx_hash not in self.txo: 952 self.txo[tx_hash] = {} 953 d = self.txo[tx_hash] 954 if addr not in d: 955 d[addr] = {} 956 d[addr][n] = (v, is_coinbase) 957 958 @locked 959 def list_txi(self) -> Sequence[str]: 960 return list(self.txi.keys()) 961 962 @locked 963 def list_txo(self) -> Sequence[str]: 964 return list(self.txo.keys()) 965 966 @modifier 967 def remove_txi(self, tx_hash: str) -> None: 968 assert isinstance(tx_hash, str) 969 self.txi.pop(tx_hash, None) 970 971 @modifier 972 def remove_txo(self, tx_hash: str) -> None: 973 assert isinstance(tx_hash, str) 974 self.txo.pop(tx_hash, None) 975 976 @locked 977 def list_spent_outpoints(self) -> Sequence[Tuple[str, str]]: 978 return [(h, n) 979 for h in self.spent_outpoints.keys() 980 for n in self.get_spent_outpoints(h) 981 ] 982 983 @locked 984 def get_spent_outpoints(self, prevout_hash: str) -> Sequence[str]: 985 assert isinstance(prevout_hash, str) 986 return list(self.spent_outpoints.get(prevout_hash, {}).keys()) 987 988 @locked 989 def get_spent_outpoint(self, prevout_hash: str, prevout_n: Union[int, str]) -> Optional[str]: 990 assert isinstance(prevout_hash, str) 991 prevout_n = str(prevout_n) 992 return self.spent_outpoints.get(prevout_hash, {}).get(prevout_n) 993 994 @modifier 995 def remove_spent_outpoint(self, prevout_hash: str, prevout_n: Union[int, str]) -> None: 996 assert isinstance(prevout_hash, str) 997 prevout_n = str(prevout_n) 998 self.spent_outpoints[prevout_hash].pop(prevout_n, None) 999 if not self.spent_outpoints[prevout_hash]: 1000 self.spent_outpoints.pop(prevout_hash) 1001 1002 @modifier 1003 def set_spent_outpoint(self, prevout_hash: str, prevout_n: Union[int, str], tx_hash: str) -> None: 1004 assert isinstance(prevout_hash, str) 1005 assert isinstance(tx_hash, str) 1006 prevout_n = str(prevout_n) 1007 if prevout_hash not in self.spent_outpoints: 1008 self.spent_outpoints[prevout_hash] = {} 1009 self.spent_outpoints[prevout_hash][prevout_n] = tx_hash 1010 1011 @modifier 1012 def add_prevout_by_scripthash(self, scripthash: str, *, prevout: TxOutpoint, value: int) -> None: 1013 assert isinstance(scripthash, str) 1014 assert isinstance(prevout, TxOutpoint) 1015 assert isinstance(value, int) 1016 if scripthash not in self._prevouts_by_scripthash: 1017 self._prevouts_by_scripthash[scripthash] = set() 1018 self._prevouts_by_scripthash[scripthash].add((prevout.to_str(), value)) 1019 1020 @modifier 1021 def remove_prevout_by_scripthash(self, scripthash: str, *, prevout: TxOutpoint, value: int) -> None: 1022 assert isinstance(scripthash, str) 1023 assert isinstance(prevout, TxOutpoint) 1024 assert isinstance(value, int) 1025 self._prevouts_by_scripthash[scripthash].discard((prevout.to_str(), value)) 1026 if not self._prevouts_by_scripthash[scripthash]: 1027 self._prevouts_by_scripthash.pop(scripthash) 1028 1029 @locked 1030 def get_prevouts_by_scripthash(self, scripthash: str) -> Set[Tuple[TxOutpoint, int]]: 1031 assert isinstance(scripthash, str) 1032 prevouts_and_values = self._prevouts_by_scripthash.get(scripthash, set()) 1033 return {(TxOutpoint.from_str(prevout), value) for prevout, value in prevouts_and_values} 1034 1035 @modifier 1036 def add_transaction(self, tx_hash: str, tx: Transaction) -> None: 1037 assert isinstance(tx_hash, str) 1038 assert isinstance(tx, Transaction), tx 1039 # note that tx might be a PartialTransaction 1040 # serialize and de-serialize tx now. this might e.g. convert a complete PartialTx to a Tx 1041 tx = tx_from_any(str(tx)) 1042 if not tx_hash: 1043 raise Exception("trying to add tx to db without txid") 1044 if tx_hash != tx.txid(): 1045 raise Exception(f"trying to add tx to db with inconsistent txid: {tx_hash} != {tx.txid()}") 1046 # don't allow overwriting complete tx with partial tx 1047 tx_we_already_have = self.transactions.get(tx_hash, None) 1048 if tx_we_already_have is None or isinstance(tx_we_already_have, PartialTransaction): 1049 self.transactions[tx_hash] = tx 1050 1051 @modifier 1052 def remove_transaction(self, tx_hash: str) -> Optional[Transaction]: 1053 assert isinstance(tx_hash, str) 1054 return self.transactions.pop(tx_hash, None) 1055 1056 @locked 1057 def get_transaction(self, tx_hash: Optional[str]) -> Optional[Transaction]: 1058 if tx_hash is None: 1059 return None 1060 assert isinstance(tx_hash, str) 1061 return self.transactions.get(tx_hash) 1062 1063 @locked 1064 def list_transactions(self) -> Sequence[str]: 1065 return list(self.transactions.keys()) 1066 1067 @locked 1068 def get_history(self) -> Sequence[str]: 1069 return list(self.history.keys()) 1070 1071 def is_addr_in_history(self, addr: str) -> bool: 1072 # does not mean history is non-empty! 1073 assert isinstance(addr, str) 1074 return addr in self.history 1075 1076 @locked 1077 def get_addr_history(self, addr: str) -> Sequence[Tuple[str, int]]: 1078 assert isinstance(addr, str) 1079 return self.history.get(addr, []) 1080 1081 @modifier 1082 def set_addr_history(self, addr: str, hist) -> None: 1083 assert isinstance(addr, str) 1084 self.history[addr] = hist 1085 1086 @modifier 1087 def remove_addr_history(self, addr: str) -> None: 1088 assert isinstance(addr, str) 1089 self.history.pop(addr, None) 1090 1091 @locked 1092 def list_verified_tx(self) -> Sequence[str]: 1093 return list(self.verified_tx.keys()) 1094 1095 @locked 1096 def get_verified_tx(self, txid: str) -> Optional[TxMinedInfo]: 1097 assert isinstance(txid, str) 1098 if txid not in self.verified_tx: 1099 return None 1100 height, timestamp, txpos, header_hash = self.verified_tx[txid] 1101 return TxMinedInfo(height=height, 1102 conf=None, 1103 timestamp=timestamp, 1104 txpos=txpos, 1105 header_hash=header_hash) 1106 1107 @modifier 1108 def add_verified_tx(self, txid: str, info: TxMinedInfo): 1109 assert isinstance(txid, str) 1110 assert isinstance(info, TxMinedInfo) 1111 self.verified_tx[txid] = (info.height, info.timestamp, info.txpos, info.header_hash) 1112 1113 @modifier 1114 def remove_verified_tx(self, txid: str): 1115 assert isinstance(txid, str) 1116 self.verified_tx.pop(txid, None) 1117 1118 def is_in_verified_tx(self, txid: str) -> bool: 1119 assert isinstance(txid, str) 1120 return txid in self.verified_tx 1121 1122 @modifier 1123 def add_tx_fee_from_server(self, txid: str, fee_sat: Optional[int]) -> None: 1124 assert isinstance(txid, str) 1125 # note: when called with (fee_sat is None), rm currently saved value 1126 if txid not in self.tx_fees: 1127 self.tx_fees[txid] = TxFeesValue() 1128 tx_fees_value = self.tx_fees[txid] 1129 if tx_fees_value.is_calculated_by_us: 1130 return 1131 self.tx_fees[txid] = tx_fees_value._replace(fee=fee_sat, is_calculated_by_us=False) 1132 1133 @modifier 1134 def add_tx_fee_we_calculated(self, txid: str, fee_sat: Optional[int]) -> None: 1135 assert isinstance(txid, str) 1136 if fee_sat is None: 1137 return 1138 assert isinstance(fee_sat, int) 1139 if txid not in self.tx_fees: 1140 self.tx_fees[txid] = TxFeesValue() 1141 self.tx_fees[txid] = self.tx_fees[txid]._replace(fee=fee_sat, is_calculated_by_us=True) 1142 1143 @locked 1144 def get_tx_fee(self, txid: str, *, trust_server: bool = False) -> Optional[int]: 1145 assert isinstance(txid, str) 1146 """Returns tx_fee.""" 1147 tx_fees_value = self.tx_fees.get(txid) 1148 if tx_fees_value is None: 1149 return None 1150 if not trust_server and not tx_fees_value.is_calculated_by_us: 1151 return None 1152 return tx_fees_value.fee 1153 1154 @modifier 1155 def add_num_inputs_to_tx(self, txid: str, num_inputs: int) -> None: 1156 assert isinstance(txid, str) 1157 assert isinstance(num_inputs, int) 1158 if txid not in self.tx_fees: 1159 self.tx_fees[txid] = TxFeesValue() 1160 self.tx_fees[txid] = self.tx_fees[txid]._replace(num_inputs=num_inputs) 1161 1162 @locked 1163 def get_num_all_inputs_of_tx(self, txid: str) -> Optional[int]: 1164 assert isinstance(txid, str) 1165 tx_fees_value = self.tx_fees.get(txid) 1166 if tx_fees_value is None: 1167 return None 1168 return tx_fees_value.num_inputs 1169 1170 @locked 1171 def get_num_ismine_inputs_of_tx(self, txid: str) -> int: 1172 assert isinstance(txid, str) 1173 txins = self.txi.get(txid, {}) 1174 return sum([len(tupls) for addr, tupls in txins.items()]) 1175 1176 @modifier 1177 def remove_tx_fee(self, txid: str) -> None: 1178 assert isinstance(txid, str) 1179 self.tx_fees.pop(txid, None) 1180 1181 @locked 1182 def get_dict(self, name) -> dict: 1183 # Warning: interacts un-intuitively with 'put': certain parts 1184 # of 'data' will have pointers saved as separate variables. 1185 if name not in self.data: 1186 self.data[name] = {} 1187 return self.data[name] 1188 1189 @locked 1190 def num_change_addresses(self) -> int: 1191 return len(self.change_addresses) 1192 1193 @locked 1194 def num_receiving_addresses(self) -> int: 1195 return len(self.receiving_addresses) 1196 1197 @locked 1198 def get_change_addresses(self, *, slice_start=None, slice_stop=None) -> List[str]: 1199 # note: slicing makes a shallow copy 1200 return self.change_addresses[slice_start:slice_stop] 1201 1202 @locked 1203 def get_receiving_addresses(self, *, slice_start=None, slice_stop=None) -> List[str]: 1204 # note: slicing makes a shallow copy 1205 return self.receiving_addresses[slice_start:slice_stop] 1206 1207 @modifier 1208 def add_change_address(self, addr: str) -> None: 1209 assert isinstance(addr, str) 1210 self._addr_to_addr_index[addr] = (1, len(self.change_addresses)) 1211 self.change_addresses.append(addr) 1212 1213 @modifier 1214 def add_receiving_address(self, addr: str) -> None: 1215 assert isinstance(addr, str) 1216 self._addr_to_addr_index[addr] = (0, len(self.receiving_addresses)) 1217 self.receiving_addresses.append(addr) 1218 1219 @locked 1220 def get_address_index(self, address: str) -> Optional[Sequence[int]]: 1221 assert isinstance(address, str) 1222 return self._addr_to_addr_index.get(address) 1223 1224 @modifier 1225 def add_imported_address(self, addr: str, d: dict) -> None: 1226 assert isinstance(addr, str) 1227 self.imported_addresses[addr] = d 1228 1229 @modifier 1230 def remove_imported_address(self, addr: str) -> None: 1231 assert isinstance(addr, str) 1232 self.imported_addresses.pop(addr) 1233 1234 @locked 1235 def has_imported_address(self, addr: str) -> bool: 1236 assert isinstance(addr, str) 1237 return addr in self.imported_addresses 1238 1239 @locked 1240 def get_imported_addresses(self) -> Sequence[str]: 1241 return list(sorted(self.imported_addresses.keys())) 1242 1243 @locked 1244 def get_imported_address(self, addr: str) -> Optional[dict]: 1245 assert isinstance(addr, str) 1246 return self.imported_addresses.get(addr) 1247 1248 def load_addresses(self, wallet_type): 1249 """ called from Abstract_Wallet.__init__ """ 1250 if wallet_type == 'imported': 1251 self.imported_addresses = self.get_dict('addresses') # type: Dict[str, dict] 1252 else: 1253 self.get_dict('addresses') 1254 for name in ['receiving', 'change']: 1255 if name not in self.data['addresses']: 1256 self.data['addresses'][name] = [] 1257 self.change_addresses = self.data['addresses']['change'] 1258 self.receiving_addresses = self.data['addresses']['receiving'] 1259 self._addr_to_addr_index = {} # type: Dict[str, Sequence[int]] # key: address, value: (is_change, index) 1260 for i, addr in enumerate(self.receiving_addresses): 1261 self._addr_to_addr_index[addr] = (0, i) 1262 for i, addr in enumerate(self.change_addresses): 1263 self._addr_to_addr_index[addr] = (1, i) 1264 1265 @profiler 1266 def _load_transactions(self): 1267 self.data = StoredDict(self.data, self, []) 1268 # references in self.data 1269 # TODO make all these private 1270 # txid -> address -> prev_outpoint -> value 1271 self.txi = self.get_dict('txi') # type: Dict[str, Dict[str, Dict[str, int]]] 1272 # txid -> address -> output_index -> (value, is_coinbase) 1273 self.txo = self.get_dict('txo') # type: Dict[str, Dict[str, Dict[str, Tuple[int, bool]]]] 1274 self.transactions = self.get_dict('transactions') # type: Dict[str, Transaction] 1275 self.spent_outpoints = self.get_dict('spent_outpoints') # txid -> output_index -> next_txid 1276 self.history = self.get_dict('addr_history') # address -> list of (txid, height) 1277 self.verified_tx = self.get_dict('verified_tx3') # txid -> (height, timestamp, txpos, header_hash) 1278 self.tx_fees = self.get_dict('tx_fees') # type: Dict[str, TxFeesValue] 1279 # scripthash -> set of (outpoint, value) 1280 self._prevouts_by_scripthash = self.get_dict('prevouts_by_scripthash') # type: Dict[str, Set[Tuple[str, int]]] 1281 # remove unreferenced tx 1282 for tx_hash in list(self.transactions.keys()): 1283 if not self.get_txi_addresses(tx_hash) and not self.get_txo_addresses(tx_hash): 1284 self.logger.info(f"removing unreferenced tx: {tx_hash}") 1285 self.transactions.pop(tx_hash) 1286 # remove unreferenced outpoints 1287 for prevout_hash in self.spent_outpoints.keys(): 1288 d = self.spent_outpoints[prevout_hash] 1289 for prevout_n, spending_txid in list(d.items()): 1290 if spending_txid not in self.transactions: 1291 self.logger.info("removing unreferenced spent outpoint") 1292 d.pop(prevout_n) 1293 1294 @modifier 1295 def clear_history(self): 1296 self.txi.clear() 1297 self.txo.clear() 1298 self.spent_outpoints.clear() 1299 self.transactions.clear() 1300 self.history.clear() 1301 self.verified_tx.clear() 1302 self.tx_fees.clear() 1303 self._prevouts_by_scripthash.clear() 1304 1305 def _convert_dict(self, path, key, v): 1306 if key == 'transactions': 1307 # note: for performance, "deserialize=False" so that we will deserialize these on-demand 1308 v = dict((k, tx_from_any(x, deserialize=False)) for k, x in v.items()) 1309 if key == 'invoices': 1310 v = dict((k, Invoice.from_json(x)) for k, x in v.items()) 1311 if key == 'payment_requests': 1312 v = dict((k, Invoice.from_json(x)) for k, x in v.items()) 1313 elif key == 'adds': 1314 v = dict((k, UpdateAddHtlc.from_tuple(*x)) for k, x in v.items()) 1315 elif key == 'fee_updates': 1316 v = dict((k, FeeUpdate(**x)) for k, x in v.items()) 1317 elif key == 'submarine_swaps': 1318 v = dict((k, SwapData(**x)) for k, x in v.items()) 1319 elif key == 'imported_channel_backups': 1320 v = dict((k, ImportedChannelBackupStorage(**x)) for k, x in v.items()) 1321 elif key == 'onchain_channel_backups': 1322 v = dict((k, OnchainChannelBackupStorage(**x)) for k, x in v.items()) 1323 elif key == 'tx_fees': 1324 v = dict((k, TxFeesValue(*x)) for k, x in v.items()) 1325 elif key == 'prevouts_by_scripthash': 1326 v = dict((k, {(prevout, value) for (prevout, value) in x}) for k, x in v.items()) 1327 elif key == 'buckets': 1328 v = dict((k, ShachainElement(bfh(x[0]), int(x[1]))) for k, x in v.items()) 1329 elif key == 'data_loss_protect_remote_pcp': 1330 v = dict((k, bfh(x)) for k, x in v.items()) 1331 return v 1332 1333 def _convert_value(self, path, key, v): 1334 if key == 'local_config': 1335 v = LocalConfig(**v) 1336 elif key == 'remote_config': 1337 v = RemoteConfig(**v) 1338 elif key == 'constraints': 1339 v = ChannelConstraints(**v) 1340 elif key == 'funding_outpoint': 1341 v = Outpoint(**v) 1342 return v 1343 1344 def _should_convert_to_stored_dict(self, key) -> bool: 1345 if key == 'keystore': 1346 return False 1347 multisig_keystore_names = [('x%d/' % i) for i in range(1, 16)] 1348 if key in multisig_keystore_names: 1349 return False 1350 return True 1351 1352 def write(self, storage: 'WalletStorage'): 1353 with self.lock: 1354 self._write(storage) 1355 1356 @profiler 1357 def _write(self, storage: 'WalletStorage'): 1358 if threading.currentThread().isDaemon(): 1359 self.logger.warning('daemon thread cannot write db') 1360 return 1361 if not self.modified(): 1362 return 1363 json_str = self.dump(human_readable=not storage.is_encrypted()) 1364 storage.write(json_str) 1365 self.set_modified(False) 1366 1367 def is_ready_to_be_used_by_wallet(self): 1368 return not self.requires_upgrade() and self._called_after_upgrade_tasks 1369 1370 def split_accounts(self, root_path): 1371 from .storage import WalletStorage 1372 out = [] 1373 result = self.get_split_accounts() 1374 for data in result: 1375 path = root_path + '.' + data['suffix'] 1376 storage = WalletStorage(path) 1377 db = WalletDB(json.dumps(data), manual_upgrades=False) 1378 db._called_after_upgrade_tasks = False 1379 db.upgrade() 1380 db.write(storage) 1381 out.append(path) 1382 return out 1383 1384 def get_action(self): 1385 action = run_hook('get_action', self) 1386 return action 1387 1388 def load_plugins(self): 1389 wallet_type = self.get('wallet_type') 1390 if wallet_type in plugin_loaders: 1391 plugin_loaders[wallet_type]() 1392 1393 def set_keystore_encryption(self, enable): 1394 self.put('use_encryption', enable) 1395