1# Copyright (c) 2017-2021 Cedric Bellegarde <cedric.bellegarde@adishatz.org> 2# Fork of https://github.com/firefox-services/syncclient 3# This program is free software: you can redistribute it and/or modify 4# it under the terms of the GNU General Public License as published by 5# the Free Software Foundation, either version 3 of the License, or 6# (at your option) any later version. 7# This program is distributed in the hope that it will be useful, 8# but WITHOUT ANY WARRANTY; without even the implied warranty of 9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10# GNU General Public License for more details. 11# You should have received a copy of the GNU General Public License 12# along with this program. If not, see <http://www.gnu.org/licenses/>. 13 14from gi.repository import Gio, GLib, GObject 15 16from pickle import dump, load 17from hashlib import sha256 18import json 19from fcntl import flock, LOCK_EX, LOCK_NB, LOCK_UN 20from time import time, sleep 21 22from eolie.helper_task import TaskHelper 23from eolie.define import App, EOLIE_DATA_PATH 24from eolie.sqlcursor import SqlCursor 25from eolie.helper_passwords import PasswordsHelper 26from eolie.logger import Logger 27from eolie.utils import emit_signal 28 29 30TOKENSERVER_URL = "https://%s/" %\ 31 App().settings.get_value("firefox-sync-server").get_string() 32 33 34class SyncWorker(GObject.Object): 35 """ 36 Manage sync with firefox server, will start syncing on init 37 """ 38 39 __gsignals__ = { 40 "syncing": (GObject.SignalFlags.RUN_FIRST, None, (str,)) 41 } 42 43 def check_modules(): 44 """ 45 True if deps are installed 46 @return bool 47 """ 48 from importlib import util 49 fxa = util.find_spec("fxa") 50 crypto = util.find_spec("Crypto") 51 hawk = util.find_spec("requests_hawk") 52 if fxa is None: 53 Logger.info("PyFxA missing: pip3 install --user pyfxa") 54 if crypto is None: 55 Logger.info("Cyrpto missing: pip3 install --user pycrypto") 56 if hawk is None: 57 Logger.info("Hawk missing: pip3 install --user requests_hawk") 58 return fxa is not None and crypto is not None and hawk is not None 59 60 def __init__(self): 61 """ 62 Init worker 63 """ 64 GObject.Object.__init__(self) 65 self.__sync_cancellable = Gio.Cancellable() 66 self.__timeout_id = None 67 self.__username = "" 68 self.__password = "" 69 self.__token = "" 70 self.__uid = "" 71 self.__keyB = b"" 72 self.__mtimes = {"bookmarks": 0.1, "history": 0.1, "passwords": 0.1} 73 # We do not create this here because it's slow down Eolie startup 74 # See __firefox_sync property 75 self.__mz = None 76 self.__state_lock = True 77 self.__session = None 78 self.__syncing = False 79 self.__syncing_pendings = False 80 try: 81 self.__pending_records = load(open(EOLIE_DATA_PATH + 82 "/firefox_sync_pendings.bin", 83 "rb")) 84 self.__sync_pendings() 85 except: 86 self.__pending_records = {"history": [], 87 "passwords": [], 88 "bookmarks": []} 89 self.__helper = PasswordsHelper() 90 if App().settings.get_value("enable-firefox-sync"): 91 self.set_credentials() 92 93 def login(self, attributes, password, code): 94 """ 95 Login to service 96 @param attributes as {} 97 @param password as str 98 @param code as str 99 """ 100 self.__username = "" 101 self.__password = "" 102 self.__uid = "" 103 self.__token = "" 104 self.__keyB = b"" 105 if attributes is None or not attributes["login"] or not password: 106 Logger.warning("SyncWorker::login(): %s", attributes) 107 return 108 from base64 import b64encode 109 import json 110 session = None 111 self.__username = attributes["login"] 112 self.__password = password 113 # Connect to firefox sync 114 try: 115 session = self.__firefox_sync.login( 116 self.__username, password, code) 117 bid_assertion, key = self.__firefox_sync.\ 118 get_browserid_assertion(session) 119 self.__token = session.token 120 self.__uid = session.uid 121 self.__keyB = session.keys[1] 122 keyB_encoded = b64encode(self.__keyB).decode("utf-8") 123 record = {"uid": self.__uid, 124 "token": self.__token, 125 "keyB": keyB_encoded} 126 self.__helper.clear_sync(self.__helper.store_sync, 127 self.__username, 128 json.dumps(record)) 129 except Exception as e: 130 self.__helper.clear_sync(self.__helper.store_sync, 131 self.__username, 132 "") 133 raise(e) 134 135 def new_session(self): 136 """ 137 Start a new session 138 """ 139 # Just reset session, will be set by get_session_bulk_keys() 140 self.__session = None 141 142 def set_credentials(self): 143 """ 144 Set credentials using Secret 145 """ 146 self.__helper.get_sync(self.__set_credentials) 147 148 def pull_loop(self): 149 """ 150 Start pulling every hours 151 """ 152 def loop(): 153 self.pull() 154 return True 155 if App().settings.get_value("enable-firefox-sync"): 156 self.pull() 157 if self.__timeout_id is None: 158 self.__timeout_id = GLib.timeout_add_seconds(600, loop) 159 160 def pull(self, force=False): 161 """ 162 Start pulling from Firefox sync 163 @param force as bool 164 """ 165 if Gio.NetworkMonitor.get_default().get_network_available() and\ 166 self.__username: 167 task_helper = TaskHelper() 168 task_helper.run(self.__pull, force) 169 170 def push(self): 171 """ 172 Start pushing to Firefox sync 173 Will push all data 174 """ 175 if Gio.NetworkMonitor.get_default().get_network_available() and\ 176 self.__username: 177 task_helper = TaskHelper() 178 task_helper.run(self.__push,) 179 180 def push_history(self, history_id): 181 """ 182 Push history id 183 @param history_id as int 184 """ 185 if self.__username: 186 task_helper = TaskHelper() 187 task_helper.run(self.__push_history, history_id) 188 189 def push_bookmark(self, bookmark_id): 190 """ 191 Push bookmark id 192 @param bookmark_id as int 193 """ 194 if self.__username: 195 task_helper = TaskHelper() 196 task_helper.run(self.__push_bookmark, bookmark_id) 197 198 def push_password(self, user_form_name, user_form_value, pass_form_name, 199 pass_form_value, uri, form_uri, uuid): 200 """ 201 Push password 202 @param user_form_name as str 203 @param user_form_value as str 204 @param pass_form_name as str 205 @param pass_form_value as str 206 @param uri as str 207 @param form_uri as str 208 @param uuid as str 209 """ 210 if self.__username: 211 task_helper = TaskHelper() 212 task_helper.run(self.__push_password, 213 user_form_name, user_form_value, pass_form_name, 214 pass_form_value, uri, form_uri, uuid) 215 216 def remove_from_history(self, guid): 217 """ 218 Remove history guid from remote history 219 @param guid as str 220 """ 221 if self.__username: 222 task_helper = TaskHelper() 223 task_helper.run(self.__remove_from_history, guid) 224 225 def remove_from_bookmarks(self, guid): 226 """ 227 Remove bookmark guid from remote bookmarks 228 @param guid as str 229 """ 230 if self.__username: 231 task_helper = TaskHelper() 232 task_helper.run(self.__remove_from_bookmarks, guid) 233 234 def remove_from_passwords(self, uuid): 235 """ 236 Remove password from passwords collection 237 @param uuid as str 238 """ 239 if self.__username: 240 task_helper = TaskHelper() 241 task_helper.run(self.__remove_from_passwords, uuid) 242 243 def delete_secret(self): 244 """ 245 Delete sync secret 246 """ 247 self.__username = "" 248 self.__password = "" 249 self.__session = None 250 self.__helper.clear_sync(None) 251 252 def stop(self, force=False): 253 """ 254 Stop update, if force, kill session too 255 @param force as bool 256 """ 257 if self.__timeout_id is not None: 258 GLib.source_remove(self.__timeout_id) 259 self.__timeout_id = None 260 self.__sync_cancellable.cancel() 261 self.__sync_cancellable = Gio.Cancellable() 262 if force: 263 self.__session = None 264 265 def save_pendings(self): 266 """ 267 Save pending records 268 """ 269 try: 270 dump(self.__pending_records, 271 open(EOLIE_DATA_PATH + "/firefox_sync_pendings.bin", "wb")) 272 except Exception as e: 273 Logger.Error("SyncWorker::save_pendings(): %s", e) 274 275 @property 276 def syncing(self): 277 """ 278 True if syncing 279 @return bool 280 """ 281 return self.__syncing 282 283 @property 284 def status(self): 285 """ 286 True if sync is working 287 @return bool 288 """ 289 try: 290 if self.__username: 291 self.__get_session_bulk_keys() 292 self.__firefox_sync.client.info_collections() 293 return True 294 except Exception as e: 295 Logger.error("SyncWorker::status(): %s", e) 296 return False 297 298 @property 299 def username(self): 300 """ 301 Get username 302 @return str 303 """ 304 return self.__username 305 306####################### 307# PRIVATE # 308####################### 309 @property 310 def __firefox_sync(self): 311 """ 312 Get firefox sync, create if None 313 """ 314 if self.__mz is None: 315 self.__mz = FirefoxSync() 316 return self.__mz 317 318 def __get_session_bulk_keys(self): 319 """ 320 Get session decrypt keys 321 @return keys as (b"", b"") 322 """ 323 if self.__session is None: 324 from fxa.core import Session as FxASession 325 from fxa.crypto import quick_stretch_password 326 self.__session = FxASession(self.__firefox_sync.fxa_client, 327 self.__username, 328 quick_stretch_password( 329 self.__username, 330 self.__password), 331 self.__uid, 332 self.__token) 333 self.__session.keys = [b"", self.__keyB] 334 self.__session.check_session_status() 335 bid_assertion, key = self.__firefox_sync.get_browserid_assertion( 336 self.__session) 337 bulk_keys = self.__firefox_sync.connect(bid_assertion, key) 338 return bulk_keys 339 340 def __update_state(self): 341 """ 342 Update state file 343 """ 344 try: 345 f = open(EOLIE_DATA_PATH + "/firefox_sync.bin", "wb") 346 # Lock file 347 flock(f, LOCK_EX | LOCK_NB) 348 self.__mtimes = self.__firefox_sync.client.info_collections() 349 dump(self.__mtimes, f) 350 # Unlock file 351 flock(f, LOCK_UN) 352 except Exception as e: 353 # Not an error, just the lock exception 354 Logger.info("SyncWorker::__update_state(): %s", e) 355 356 def __sync_pendings(self): 357 """ 358 Sync pendings record 359 """ 360 try: 361 if Gio.NetworkMonitor.get_default().get_network_available() and\ 362 self.__username and not self.__syncing_pendings: 363 self.__syncing_pendings = True 364 Logger.sync_debug("Elements to push to Firefox sync: %s", 365 self.__pending_records) 366 self.__check_worker() 367 bulk_keys = self.__get_session_bulk_keys() 368 for key in self.__pending_records.keys(): 369 while self.__pending_records[key]: 370 try: 371 record = self.__pending_records[key].pop(0) 372 Logger.sync_debug("syncing %s", record) 373 if "bmkUri" in record.keys(): 374 emit_signal(self, "syncing", 375 record["bmkUri"]) 376 elif "histUri" in record.keys(): 377 emit_signal(self, "syncing", 378 record["histUri"]) 379 elif "hostname" in record.keys(): 380 emit_signal(self, "syncing", 381 record["hostname"]) 382 self.__firefox_sync.add(record, key, bulk_keys) 383 except: 384 self.__pending_records[key].append(record) 385 self.__update_state() 386 self.__syncing_pendings = False 387 except Exception as e: 388 Logger.error("SyncWorker::__sync_pendings(): %s", e) 389 self.__syncing_pendings = False 390 391 def __push_history(self, history_id, sync=True): 392 """ 393 Push history item 394 @param history is as int 395 @param sync as bool 396 """ 397 try: 398 record = {} 399 atimes = App().history.get_atimes(history_id) 400 guid = App().history.get_guid(history_id) 401 record["histUri"] = App().history.get_uri(history_id) 402 record["id"] = guid 403 record["title"] = App().history.get_title(history_id) 404 record["visits"] = [] 405 for atime in atimes: 406 record["visits"].append({"date": atime * 1000000, 407 "type": 1}) 408 self.__pending_records["history"].append(record) 409 if sync: 410 self.__sync_pendings() 411 except Exception as e: 412 Logger.error("SyncWorker::__push_history(): %s", e) 413 414 def __push_bookmark(self, bookmark_id, sync=True): 415 """ 416 Push bookmark 417 @param bookmark_id as int 418 @param sync as bool 419 """ 420 try: 421 parent_guid = App().bookmarks.get_parent_guid(bookmark_id) 422 # No parent, move it to unfiled 423 if parent_guid is None: 424 parent_guid = "unfiled" 425 parent_id = App().bookmarks.get_id_by_guid(parent_guid) 426 record = {} 427 record["bmkUri"] = App().bookmarks.get_uri(bookmark_id) 428 record["id"] = App().bookmarks.get_guid(bookmark_id) 429 record["title"] = App().bookmarks.get_title(bookmark_id) 430 record["tags"] = App().bookmarks.get_tags(bookmark_id) 431 record["parentid"] = parent_guid 432 record["parentName"] = App().bookmarks.get_parent_name(bookmark_id) 433 record["type"] = "bookmark" 434 self.__pending_records["bookmarks"].append(record) 435 parent_guid = App().bookmarks.get_guid(parent_id) 436 parent_name = App().bookmarks.get_title(parent_id) 437 children = App().bookmarks.get_children(parent_guid) 438 record = {} 439 record["id"] = parent_guid 440 record["type"] = "folder" 441 # A parent with parent as unfiled needs to be moved to places 442 # Firefox internal 443 grand_parent_guid = App().bookmarks.get_parent_guid(parent_id) 444 if grand_parent_guid == "unfiled": 445 grand_parent_guid = "places" 446 record["parentid"] = grand_parent_guid 447 record["parentName"] = App().bookmarks.get_parent_name(parent_id) 448 record["title"] = parent_name 449 record["children"] = children 450 self.__pending_records["bookmarks"].append(record) 451 if sync: 452 self.__sync_pendings() 453 except Exception as e: 454 Logger.error("SyncWorker::__push_bookmark(): %s", e) 455 456 def __push_password(self, user_form_name, user_form_value, pass_form_name, 457 pass_form_value, uri, form_uri, uuid, sync=True): 458 """ 459 Push password 460 @param user_form_name as str 461 @param user_form_value as str 462 @param pass_form_name as str 463 @param pass_form_value as str 464 @param uri as str 465 @param uuid as str 466 @param sync as bool 467 """ 468 try: 469 record = {} 470 record["id"] = "{%s}" % uuid 471 record["hostname"] = uri 472 record["formSubmitURL"] = form_uri 473 record["httpRealm"] = None 474 record["username"] = user_form_value 475 record["password"] = pass_form_value 476 record["usernameField"] = user_form_name 477 record["passwordField"] = pass_form_name 478 mtime = int(time() * 1000) 479 record["timeCreated"] = mtime 480 record["timePasswordChanged"] = mtime 481 self.__pending_records["passwords"].append(record) 482 if sync: 483 self.__sync_pendings() 484 except Exception as e: 485 Logger.error("SyncWorker::__push_password(): %s", e) 486 487 def __remove_from_history(self, guid, sync=True): 488 """ 489 Remove from history 490 @param guid as str 491 @param sync as bool 492 """ 493 try: 494 record = {} 495 record["id"] = guid 496 record["type"] = "item" 497 record["deleted"] = True 498 self.__pending_records["history"].append(record) 499 if sync: 500 self.__sync_pendings() 501 except Exception as e: 502 Logger.sync_debug("SyncWorker::__remove_from_history(): %s", e) 503 504 def __remove_from_bookmarks(self, guid, sync=True): 505 """ 506 Remove from history 507 @param guid as str 508 @param sync as bool 509 """ 510 try: 511 record = {} 512 record["id"] = guid 513 record["type"] = "bookmark" 514 record["deleted"] = True 515 self.__pending_records["bookmarks"].append(record) 516 if sync: 517 self.__sync_pendings() 518 except Exception as e: 519 Logger.sync_debug("SyncWorker::__remove_from_bookmarks(): %s", e) 520 521 def __remove_from_passwords(self, uuid, sync=True): 522 """ 523 Remove password from passwords collection 524 @param uuid as str 525 @param sync as bool 526 """ 527 try: 528 record = {} 529 record["id"] = uuid 530 record["deleted"] = True 531 self.__pending_records["passwords"].append(record) 532 if sync: 533 self.__sync_pendings() 534 except Exception as e: 535 Logger.sync_debug("SyncWorker::__remove_from_passwords(): %s", e) 536 537 def __pull(self, force): 538 """ 539 Pull bookmarks, history, ... from Firefox Sync 540 @param force as bool 541 """ 542 if self.__syncing: 543 return 544 Logger.sync_debug("Start pulling") 545 self.__syncing = True 546 self.__sync_cancellable.cancel() 547 self.__sync_cancellable = Gio.Cancellable() 548 try: 549 if force: 550 raise 551 self.__mtimes = load(open(EOLIE_DATA_PATH + "/firefox_sync.bin", 552 "rb")) 553 except: 554 self.__mtimes = {"bookmarks": 0.1, 555 "history": 0.1, 556 "passwords": 0.1} 557 try: 558 self.__check_worker() 559 560 bulk_keys = self.__get_session_bulk_keys() 561 new_mtimes = self.__firefox_sync.client.info_collections() 562 563 self.__check_worker() 564 ######################## 565 # Passwords Management # 566 ######################## 567 try: 568 Logger.sync_debug("local passwords: %s, remote passwords: %s", 569 self.__mtimes["passwords"], 570 new_mtimes["passwords"]) 571 # Only pull if something new available 572 if self.__mtimes["passwords"] != new_mtimes["passwords"]: 573 self.__pull_passwords(bulk_keys) 574 except: 575 pass # No passwords in sync 576 577 self.__check_worker() 578 ###################### 579 # History Management # 580 ###################### 581 try: 582 Logger.sync_debug("local history: %s, remote history: %s", 583 self.__mtimes["history"], 584 new_mtimes["history"]) 585 # Only pull if something new available 586 if self.__mtimes["history"] != new_mtimes["history"]: 587 self.__pull_history(bulk_keys) 588 except: 589 pass 590 591 self.__check_worker() 592 ######################## 593 # Bookmarks Management # 594 ######################## 595 try: 596 Logger.sync_debug("local bookmarks: %s, remote bookmarks: %s", 597 self.__mtimes["bookmarks"], 598 new_mtimes["bookmarks"]) 599 # Only pull if something new available 600 if self.__mtimes["bookmarks"] != new_mtimes["bookmarks"]: 601 self.__pull_bookmarks(bulk_keys) 602 except: 603 pass 604 self.__update_state() 605 Logger.sync_debug("Stop pulling") 606 except Exception as e: 607 Logger.error("SyncWorker::__pull(): %s", e) 608 self.__syncing = False 609 610 def __push(self): 611 """ 612 Push bookmarks, history, ... to Firefox Sync 613 @param force as bool 614 """ 615 if self.__syncing: 616 return 617 Logger.sync_debug("Start pushing") 618 self.__syncing = True 619 self.__sync_cancellable.cancel() 620 self.__sync_cancellable = Gio.Cancellable() 621 try: 622 self.__check_worker() 623 624 ######################## 625 # Passwords Management # 626 ######################## 627 self.__helper.get_all(self.__on_helper_get_all) 628 629 self.__check_worker() 630 ###################### 631 # History Management # 632 ###################### 633 for history_id in App().history.get_from_atime(0): 634 self.__push_history(history_id, False) 635 636 self.__check_worker() 637 ######################## 638 # Bookmarks Management # 639 ######################## 640 for (bookmark_id, title, uri) in App().bookmarks.get_bookmarks(): 641 self.__push_bookmark(bookmark_id, False) 642 self.__check_worker() 643 self.__sync_pendings() 644 Logger.sync_debug("Stop pushing") 645 except Exception as e: 646 Logger.error("SyncWorker::__push(): %s", e) 647 self.__syncing = False 648 649 def __pull_bookmarks(self, bulk_keys): 650 """ 651 Pull from bookmarks 652 @param bulk_keys as KeyBundle 653 @raise StopIteration 654 """ 655 Logger.sync_debug("pull bookmarks") 656 SqlCursor.add(App().bookmarks) 657 records = self.__firefox_sync.get_records("bookmarks", bulk_keys) 658 children_array = [] 659 for record in records: 660 self.__check_worker() 661 if record["modified"] < self.__mtimes["bookmarks"]: 662 continue 663 sleep(0.01) 664 bookmark = record["payload"] 665 bookmark_id = App().bookmarks.get_id_by_guid(bookmark["id"]) 666 # Nothing to apply, continue 667 if App().bookmarks.get_mtime(bookmark_id) >= record["modified"]: 668 continue 669 Logger.sync_debug("pulling %s", record) 670 # Deleted bookmark 671 if "deleted" in bookmark.keys(): 672 App().bookmarks.remove(bookmark_id) 673 # Keep folder only for firefox compatiblity 674 elif "type" in bookmark.keys() and bookmark["type"] == "folder"\ 675 and bookmark["id"] is not None\ 676 and bookmark["title"]: 677 if bookmark_id is None: 678 bookmark_id = App().bookmarks.add(bookmark["title"], 679 bookmark["id"], 680 bookmark["id"], 681 [], 682 0) 683 # Will calculate position later 684 if "children" in bookmark.keys(): 685 children_array.append(bookmark["children"]) 686 # We have a bookmark, add it 687 elif "type" in bookmark.keys() and bookmark["type"] == "bookmark"\ 688 and bookmark["id"] is not None\ 689 and bookmark["title"]: 690 # Update bookmark 691 if bookmark_id is not None: 692 App().bookmarks.set_title(bookmark_id, 693 bookmark["title"]) 694 App().bookmarks.set_uri(bookmark_id, 695 bookmark["bmkUri"]) 696 # Update tags 697 current_tags = App().bookmarks.get_tags(bookmark_id) 698 for tag in App().bookmarks.get_tags(bookmark_id): 699 if "tags" in bookmark.keys() and\ 700 tag not in bookmark["tags"]: 701 tag_id = App().bookmarks.get_tag_id(tag) 702 current_tags.remove(tag) 703 App().bookmarks.del_tag_from(tag_id, 704 bookmark_id) 705 if "tags" in bookmark.keys(): 706 for tag in bookmark["tags"]: 707 # Tag already associated 708 if tag in current_tags: 709 continue 710 tag_id = App().bookmarks.get_tag_id(tag) 711 if tag_id is None: 712 tag_id = App().bookmarks.add_tag(tag) 713 App().bookmarks.add_tag_to(tag_id, 714 bookmark_id) 715 # Add a new bookmark 716 else: 717 bookmark_id = App().bookmarks.get_id(bookmark["bmkUri"]) 718 # Add a new bookmark 719 if bookmark_id is None: 720 # Use parent name if no bookmarks tags 721 if "tags" not in bookmark.keys() or\ 722 not bookmark["tags"]: 723 if "parentName" in bookmark.keys() and\ 724 bookmark["parentName"]: 725 bookmark["tags"] = [bookmark["parentName"]] 726 else: 727 bookmark["tags"] = [] 728 bookmark_id = App().bookmarks.add(bookmark["title"], 729 bookmark["bmkUri"], 730 bookmark["id"], 731 bookmark["tags"], 732 0) 733 else: 734 # Update guid 735 App().bookmarks.set_guid(bookmark_id, bookmark["id"]) 736 # Update parent name if available 737 if bookmark_id is not None and "parentName" in bookmark.keys(): 738 App().bookmarks.set_parent(bookmark_id, 739 bookmark["parentid"], 740 bookmark["parentName"]) 741 App().bookmarks.set_mtime(bookmark_id, 742 record["modified"]) 743 # Update bookmark position 744 for children in children_array: 745 position = 0 746 for child in children: 747 bid = App().bookmarks.get_id_by_guid(child) 748 App().bookmarks.set_position(bid, 749 position) 750 position += 1 751 App().bookmarks.clean_tags() # Will commit 752 SqlCursor.remove(App().bookmarks) 753 754 def __pull_passwords(self, bulk_keys): 755 """ 756 Pull from passwords 757 @param bulk_keys as KeyBundle 758 @raise StopIteration 759 """ 760 Logger.sync_debug("pull passwords") 761 records = self.__firefox_sync.get_records("passwords", bulk_keys) 762 for record in records: 763 self.__check_worker() 764 if record["modified"] < self.__mtimes["passwords"]: 765 continue 766 sleep(0.01) 767 Logger.sync_debug("pulling %s", record) 768 password = record["payload"] 769 password_id = password["id"].strip("{}") 770 if "formSubmitURL" in password.keys(): 771 self.__helper.clear(password_id, 772 self.__helper.store, 773 password["usernameField"], 774 password["username"], 775 password["passwordField"], 776 password["password"], 777 password["hostname"], 778 password["formSubmitURL"], 779 password_id, 780 None) 781 elif "deleted" in password.keys(): # We assume True 782 self.__helper.clear(password_id) 783 784 def __pull_history(self, bulk_keys): 785 """ 786 Pull from history 787 @param bulk_keys as KeyBundle 788 @raise StopIteration 789 """ 790 Logger.sync_debug("pull history") 791 records = self.__firefox_sync.get_records("history", bulk_keys) 792 for record in records: 793 self.__check_worker() 794 if record["modified"] < self.__mtimes["history"]: 795 continue 796 sleep(0.01) 797 history = record["payload"] 798 keys = history.keys() 799 history_id = App().history.get_id_by_guid(history["id"]) 800 # Check we have a valid history item 801 if "histUri" in keys and\ 802 "title" in keys and\ 803 history["title"] and\ 804 App().history.get_mtime(history_id) < record["modified"]: 805 # Try to get visit date 806 atimes = [] 807 try: 808 for visit in history["visits"]: 809 atimes.append(round(int(visit["date"]) / 1000000, 2)) 810 except: 811 continue 812 Logger.sync_debug("pulling %s", record) 813 title = history["title"].rstrip().lstrip() 814 history_id = App().history.add(title, 815 history["histUri"], 816 record["modified"], 817 history["id"], 818 atimes, 819 True) 820 elif "deleted" in keys: 821 history_id = App().history.get_id_by_guid(history_id) 822 App().history.remove(history_id) 823 824 def __set_credentials(self, attributes, password, uri, index, count): 825 """ 826 Set credentials 827 @param attributes as {} 828 @param password as str 829 @param uri as None 830 @param index as int 831 @param count as int 832 """ 833 if attributes is None: 834 return 835 from base64 import b64decode 836 import json 837 try: 838 self.__username = attributes["login"] 839 record = json.loads(password) 840 self.__token = record["token"] 841 self.__uid = record["uid"] 842 self.__keyB = b64decode(record["keyB"]) 843 except Exception as e: 844 Logger.error("SyncWorker::__set_credentials(): %s" % e) 845 846 def __check_worker(self): 847 """ 848 Raise an exception if worker should not be syncing: error&cancel 849 """ 850 if self.__sync_cancellable.is_cancelled(): 851 raise StopIteration("SyncWorker: cancelled") 852 elif not self.__username: 853 raise StopIteration("SyncWorker: missing username") 854 elif not self.__token: 855 raise StopIteration("SyncWorker: missing token") 856 857 def __on_helper_get_all(self, attributes, password, uri, index, count): 858 """ 859 Push password 860 @param attributes as {} 861 @param password as str 862 @param uri as None 863 @param index as int 864 @param count as int 865 """ 866 if attributes is None: 867 return 868 try: 869 self.__check_worker() 870 user_form_name = attributes["userform"] 871 user_form_value = attributes["login"] 872 pass_form_name = attributes["passform"] 873 pass_form_value = password 874 uri = attributes["hostname"] 875 form_uri = attributes["formSubmitURL"] 876 uuid = attributes["uuid"] 877 task_helper = TaskHelper() 878 task_helper.run(self.__push_password, 879 user_form_name, user_form_value, pass_form_name, 880 pass_form_value, uri, form_uri, uuid, False) 881 except Exception as e: 882 Logger.error("SyncWorker::__on_helper_get_all(): %s" % e) 883 884 885class FirefoxSync(object): 886 """ 887 Sync client 888 """ 889 890 def __init__(self): 891 """ 892 Init client 893 """ 894 from fxa.core import Client as FxAClient 895 self.__fxa_client = FxAClient() 896 897 def login(self, login, password, code): 898 """ 899 Login to FxA and get the keys. 900 @param login as str 901 @param password as str 902 @param code as str 903 @return fxaSession 904 """ 905 fxaSession = self.__fxa_client.login(login, password, keys=True) 906 if code: 907 fxaSession.totp_verify(code) 908 fxaSession.fetch_keys() 909 return fxaSession 910 911 def connect(self, bid_assertion, key): 912 """ 913 Connect to sync using FxA browserid assertion 914 @param session as fxaSession 915 @return bundle keys as KeyBundle 916 """ 917 state = None 918 if key is not None: 919 from binascii import hexlify 920 state = hexlify(sha256(key).digest()[0:16]) 921 self.__client = SyncClient(bid_assertion, state) 922 sync_keys = KeyBundle.fromMasterKey( 923 key, 924 "identity.mozilla.com/picl/v1/oldsync") 925 926 # Fetch the sync bundle keys out of storage. 927 # They're encrypted with the account-level key. 928 keys = self.__decrypt_payload(self.__client.get_record("crypto", 929 "keys"), 930 sync_keys) 931 932 # There's some provision for using separate 933 # key bundles for separate collections 934 # but I haven't bothered digging through 935 # to see what that's about because 936 # it doesn't seem to be in use, at least on my account. 937 if keys["collections"]: 938 Logger.error("""no support for per-collection 939 key bundles yet sorry :-(""") 940 return None 941 942 # Now use those keys to decrypt the records of interest. 943 from base64 import b64decode 944 bulk_keys = KeyBundle(b64decode(keys["default"][0]), 945 b64decode(keys["default"][1])) 946 return bulk_keys 947 948 def get_records(self, collection, bulk_keys): 949 """ 950 Return records payload 951 @param collection as str 952 @param bulk keys as KeyBundle 953 @return [{}] 954 """ 955 records = [] 956 for record in self.__client.get_records(collection): 957 record["payload"] = self.__decrypt_payload(record, bulk_keys) 958 records.append(record) 959 return records 960 961 def add(self, item, collection, bulk_keys): 962 """ 963 Add bookmark 964 @param bookmark as {} 965 @param collection as str 966 @param bulk_keys as KeyBundle 967 """ 968 payload = self.__encrypt_payload(item, bulk_keys) 969 record = {} 970 record["modified"] = round(time(), 2) 971 record["payload"] = payload 972 record["id"] = item["id"] 973 self.__client.put_record(collection, record) 974 975 def get_browserid_assertion(self, session, 976 tokenserver_url=TOKENSERVER_URL): 977 """ 978 Get browser id assertion and state 979 @param session as fxaSession 980 @return (bid_assertion, state) as (str, str) 981 """ 982 bid_assertion = session.get_identity_assertion(tokenserver_url) 983 return bid_assertion, session.keys[1] 984 985 @property 986 def client(self): 987 """ 988 Get client 989 """ 990 return self.__client 991 992 @property 993 def fxa_client(self): 994 """ 995 Get fxa client 996 """ 997 return self.__fxa_client 998 999####################### 1000# PRIVATE # 1001####################### 1002 def __encrypt_payload(self, record, key_bundle): 1003 """ 1004 Encrypt payload 1005 @param record as {} 1006 @param key bundle as KeyBundle 1007 @return encrypted record payload 1008 """ 1009 from Crypto.Cipher import AES 1010 from Crypto import Random 1011 from hmac import new 1012 from base64 import b64encode 1013 plaintext = json.dumps(record).encode("utf-8") 1014 # Input strings must be a multiple of 16 in length 1015 length = 16 - (len(plaintext) % 16) 1016 plaintext += bytes([length]) * length 1017 iv = Random.new().read(16) 1018 aes = AES.new(key_bundle.encryption_key, AES.MODE_CBC, iv) 1019 ciphertext = b64encode(aes.encrypt(plaintext)) 1020 _hmac = new(key_bundle.hmac_key, ciphertext, sha256).hexdigest() 1021 payload = {"ciphertext": ciphertext.decode("utf-8"), 1022 "IV": b64encode(iv).decode("utf-8"), "hmac": _hmac} 1023 return json.dumps(payload) 1024 1025 def __decrypt_payload(self, record, key_bundle): 1026 """ 1027 Descrypt payload 1028 @param record as str (json) 1029 @param key bundle as KeyBundle 1030 @return uncrypted record payload 1031 """ 1032 from Crypto.Cipher import AES 1033 from hmac import new 1034 from base64 import b64decode 1035 j = json.loads(record["payload"]) 1036 # Always check the hmac before decrypting anything. 1037 expected_hmac = new(key_bundle.hmac_key, 1038 j['ciphertext'].encode("utf-8"), 1039 sha256).hexdigest() 1040 if j['hmac'] != expected_hmac: 1041 raise ValueError("HMAC mismatch: %s != %s" % (j['hmac'], 1042 expected_hmac)) 1043 ciphertext = b64decode(j['ciphertext']) 1044 iv = b64decode(j['IV']) 1045 aes = AES.new(key_bundle.encryption_key, AES.MODE_CBC, iv) 1046 plaintext = aes.decrypt(ciphertext).strip().decode("utf-8") 1047 # Remove any CBC block padding, 1048 # assuming it's a well-formed JSON payload. 1049 plaintext = plaintext[:plaintext.rfind("}") + 1] 1050 return json.loads(plaintext) 1051 1052 1053class KeyBundle: 1054 """ 1055 RFC-5869 1056 """ 1057 1058 def __init__(self, encryption_key, hmac_key): 1059 self.encryption_key = encryption_key 1060 self.hmac_key = hmac_key 1061 1062 @classmethod 1063 def fromMasterKey(cls, master_key, info): 1064 key_material = KeyBundle.HKDF(master_key, None, info, 2 * 32) 1065 return cls(key_material[:32], key_material[32:]) 1066 1067 def HKDF_extract(salt, IKM, hashmod=sha256): 1068 """ 1069 Extract a pseudorandom key suitable for use with HKDF_expand 1070 @param salt as str 1071 @param IKM as str 1072 """ 1073 from hmac import new 1074 if salt is None: 1075 salt = b"\x00" * hashmod().digest_size 1076 return new(salt, IKM, hashmod).digest() 1077 1078 def HKDF_expand(PRK, info, length, hashmod=sha256): 1079 """ 1080 Expand pseudo random key and info 1081 @param PRK as str 1082 @param info as str 1083 @param length as int 1084 """ 1085 from hmac import new 1086 from math import ceil 1087 digest_size = hashmod().digest_size 1088 N = int(ceil(length * 1.0 / digest_size)) 1089 assert N <= 255 1090 T = b"" 1091 output = [] 1092 for i in range(1, N + 1): 1093 data = T + (info + chr(i)).encode() 1094 T = new(PRK, data, hashmod).digest() 1095 output.append(T) 1096 return b"".join(output)[:length] 1097 1098 def HKDF(secret, salt, info, length, hashmod=sha256): 1099 """ 1100 HKDF-extract-and-expand as a single function. 1101 @param secret as str 1102 @param salt as str 1103 @param info as str 1104 @param length as int 1105 """ 1106 PRK = KeyBundle.HKDF_extract(salt, secret, hashmod) 1107 return KeyBundle.HKDF_expand(PRK, info, length, hashmod) 1108 1109 1110class TokenserverClient(object): 1111 """ 1112 Client for the Firefox Sync Token Server. 1113 """ 1114 1115 def __init__(self, bid_assertion, client_state, 1116 server_url=TOKENSERVER_URL): 1117 """ 1118 Init client 1119 @param bid assertion as str 1120 @param client_state as ??? 1121 @param server_url as str 1122 """ 1123 self.__bid_assertion = bid_assertion 1124 self.__client_state = client_state 1125 self.__server_url = server_url 1126 1127 def get_hawk_credentials(self, duration=None): 1128 """ 1129 Asks for new temporary token given a BrowserID assertion 1130 @param duration as str 1131 """ 1132 from requests import get 1133 authorization = 'BrowserID %s' % self.__bid_assertion 1134 headers = { 1135 'Authorization': authorization, 1136 'X-Client-State': self.__client_state 1137 } 1138 params = {} 1139 1140 if duration is not None: 1141 params['duration'] = int(duration) 1142 1143 url = self.__server_url.rstrip('/') + '/1.0/sync/1.5' 1144 raw_resp = get(url, headers=headers, params=params, verify=True) 1145 raw_resp.raise_for_status() 1146 return raw_resp.json() 1147 1148 1149class SyncClient(object): 1150 """ 1151 Client for the Firefox Sync server. 1152 """ 1153 1154 def __init__(self, bid_assertion=None, client_state=None, 1155 credentials={}, tokenserver_url=TOKENSERVER_URL): 1156 """ 1157 Init client 1158 @param bid assertion as str 1159 @param client_state as ??? 1160 @param credentials as {} 1161 @param server_url as str 1162 """ 1163 from requests_hawk import HawkAuth 1164 if bid_assertion is not None and client_state is not None: 1165 ts_client = TokenserverClient(bid_assertion, client_state, 1166 tokenserver_url) 1167 credentials = ts_client.get_hawk_credentials() 1168 self.__user_id = credentials['uid'] 1169 self.__api_endpoint = credentials['api_endpoint'] 1170 self.__auth = HawkAuth(algorithm=credentials['hashalg'], 1171 id=credentials['id'], 1172 key=credentials['key'], 1173 always_hash_content=False) 1174 1175 def _request(self, method, url, **kwargs): 1176 """ 1177 Utility to request an endpoint with the correct authentication 1178 setup, raises on errors and returns the JSON. 1179 @param method as str 1180 @param url as str 1181 @param kwargs as requests.request named args 1182 """ 1183 from requests import request, exceptions 1184 url = self.__api_endpoint.rstrip('/') + '/' + url.lstrip('/') 1185 raw_resp = request(method, url, auth=self.__auth, **kwargs) 1186 raw_resp.raise_for_status() 1187 1188 if raw_resp.status_code == 304: 1189 http_error_msg = '%s Client Error: %s for url: %s' % ( 1190 raw_resp.status_code, 1191 raw_resp.reason, 1192 raw_resp.url) 1193 raise exceptions.HTTPError(http_error_msg, response=raw_resp) 1194 return raw_resp.json() 1195 1196 def info_collections(self, **kwargs): 1197 """ 1198 Returns an object mapping collection names associated with the 1199 account to the last-modified time for each collection. 1200 1201 The server may allow requests to this endpoint to be authenticated 1202 with an expired token, so that clients can check for server-side 1203 changes before fetching an updated token from the Token Server. 1204 """ 1205 return self._request('get', '/info/collections', **kwargs) 1206 1207 def info_quota(self, **kwargs): 1208 """ 1209 Returns a two-item list giving the user's current usage and quota 1210 (in KB). The second item will be null if the server 1211 does not enforce quotas. 1212 1213 Note that usage numbers may be approximate. 1214 """ 1215 return self._request('get', '/info/quota', **kwargs) 1216 1217 def get_collection_usage(self, **kwargs): 1218 """ 1219 Returns an object mapping collection names associated with the 1220 account to the data volume used for each collection (in KB). 1221 1222 Note that these results may be very expensive as it calculates more 1223 detailed and accurate usage information than the info_quota method. 1224 """ 1225 return self._request('get', '/info/collection_usage', **kwargs) 1226 1227 def get_collection_counts(self, **kwargs): 1228 """ 1229 Returns an object mapping collection names associated with the 1230 account to the total number of items in each collection. 1231 """ 1232 return self._request('get', '/info/collection_counts', **kwargs) 1233 1234 def delete_all_records(self, **kwargs): 1235 """ 1236 Deletes all records for the user 1237 """ 1238 return self._request('delete', '/', **kwargs) 1239 1240 def get_records(self, collection, full=True, ids=None, newer=None, 1241 limit=None, offset=None, sort=None, **kwargs): 1242 """ 1243 Returns a list of the BSOs contained in a collection. For example: 1244 1245 >>> ["GXS58IDC_12", "GXS58IDC_13", "GXS58IDC_15"] 1246 1247 By default only the BSO ids are returned, but full objects can be 1248 requested using the full parameter. If the collection does not 1249 exist, an empty list is returned. 1250 1251 :param ids: 1252 a comma-separated list of ids. Only objects whose id is in 1253 this list will be returned. A maximum of 100 ids may be 1254 provided. 1255 1256 :param newer: 1257 a timestamp. Only objects whose last-modified time is strictly 1258 greater than this value will be returned. 1259 1260 :param full: 1261 any value. If provided then the response will be a list of full 1262 BSO objects rather than a list of ids. 1263 1264 :param limit: 1265 a positive integer. At most that many objects will be returned. 1266 If more than that many objects matched the query, 1267 an X-Weave-Next-Offset header will be returned. 1268 1269 :param offset: 1270 a string, as returned in the X-Weave-Next-Offset header of a 1271 previous request using the limit parameter. 1272 1273 :param sort: 1274 sorts the output: 1275 "newest" - orders by last-modified time, largest first 1276 "index" - orders by the sortindex, highest weight first 1277 "oldest" - orders by last-modified time, oldest first 1278 """ 1279 params = kwargs.pop('params', {}) 1280 if full: 1281 params['full'] = True 1282 if ids is not None: 1283 params['ids'] = ','.join(map(str, ids)) 1284 if newer is not None: 1285 params['newer'] = newer 1286 if limit is not None: 1287 params['limit'] = limit 1288 if offset is not None: 1289 params['offset'] = offset 1290 if sort is not None and sort in ('newest', 'index', 'oldest'): 1291 params['sort'] = sort 1292 1293 return self._request('get', '/storage/%s' % collection.lower(), 1294 params=params, **kwargs) 1295 1296 def get_record(self, collection, record_id, **kwargs): 1297 """Returns the BSO in the collection corresponding to the requested id. 1298 """ 1299 return self._request('get', '/storage/%s/%s' % (collection.lower(), 1300 record_id), **kwargs) 1301 1302 def delete_record(self, collection, record_id, **kwargs): 1303 """Deletes the BSO at the given location. 1304 """ 1305 try: 1306 return self._request('delete', '/storage/%s/%s' % ( 1307 collection.lower(), record_id), **kwargs) 1308 except Exception as e: 1309 Logger.error("SyncClient::delete_record(): %s", e) 1310 1311 def put_record(self, collection, record, **kwargs): 1312 """ 1313 Creates or updates a specific BSO within a collection. 1314 The passed record must be a python object containing new data for 1315 the BSO. 1316 1317 If the target BSO already exists then it will be updated with the 1318 data from the request body. Fields that are not provided will not 1319 be overwritten, so it is possible to e.g. update the ttl field of a 1320 BSO without re-submitting its payload. Fields that are explicitly 1321 set to null in the request body will be set to their default value 1322 by the server. 1323 1324 If the target BSO does not exist, then fields that are not provided 1325 in the python object will be set to their default value 1326 by the server. 1327 1328 Successful responses will return the new last-modified time for the 1329 collection. 1330 1331 Note that the server may impose a limit on the amount of data 1332 submitted for storage in a single BSO. 1333 """ 1334 import six 1335 # XXX: Workaround until request-hawk supports the json parameter. (#17) 1336 if isinstance(record, six.string_types): 1337 record = json.loads(record) 1338 record = record.copy() 1339 record_id = record.pop('id') 1340 headers = {} 1341 if 'headers' in kwargs: 1342 headers = kwargs.pop('headers') 1343 headers['Content-Type'] = 'application/json; charset=utf-8' 1344 1345 return self._request('put', '/storage/%s/%s' % ( 1346 collection.lower(), record_id), data=json.dumps(record), 1347 headers=headers, **kwargs) 1348