1# -*- coding: utf-8 -*- 2# Copyright: Ankitects Pty Ltd and contributors 3# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html 4 5import pprint 6import re 7import time 8import os 9import random 10import stat 11import datetime 12import copy 13import traceback 14import json 15 16from anki.lang import _, ngettext 17from anki.utils import ids2str, fieldChecksum, \ 18 intTime, splitFields, joinFields, maxID, devMode, stripHTMLMedia 19from anki.hooks import runFilter, runHook 20from anki.models import ModelManager 21from anki.media import MediaManager 22from anki.decks import DeckManager 23from anki.tags import TagManager 24from anki.consts import * 25from anki.errors import AnkiError 26from anki.sound import stripSounds 27import anki.latex # sets up hook 28import anki.cards 29import anki.notes 30import anki.template 31import anki.find 32 33defaultConf = { 34 # review options 35 'activeDecks': [1], 36 'curDeck': 1, 37 'newSpread': NEW_CARDS_DISTRIBUTE, 38 'collapseTime': 1200, 39 'timeLim': 0, 40 'estTimes': True, 41 'dueCounts': True, 42 # other config 43 'curModel': None, 44 'nextPos': 1, 45 'sortType': "noteFld", 46 'sortBackwards': False, 47 'addToCur': True, # add new to currently selected deck? 48 'dayLearnFirst': False, 49 'schedVer': 2, 50} 51 52def timezoneOffset(): 53 if time.localtime().tm_isdst: 54 return time.altzone//60 55 else: 56 return time.timezone//60 57 58# this is initialized by storage.Collection 59class _Collection: 60 61 def __init__(self, db, server=False, log=False): 62 self._debugLog = log 63 self.db = db 64 self.path = db._path 65 self._openLog() 66 self.log(self.path, anki.version) 67 self.server = server 68 self._lastSave = time.time() 69 self.clearUndo() 70 self.media = MediaManager(self, server) 71 self.models = ModelManager(self) 72 self.decks = DeckManager(self) 73 self.tags = TagManager(self) 74 self.load() 75 if not self.crt: 76 d = datetime.datetime.today() 77 d -= datetime.timedelta(hours=4) 78 d = datetime.datetime(d.year, d.month, d.day) 79 d += datetime.timedelta(hours=4) 80 self.crt = int(time.mktime(d.timetuple())) 81 self.conf['localOffset'] = timezoneOffset() 82 self._loadScheduler() 83 if not self.conf.get("newBury", False): 84 self.conf['newBury'] = True 85 self.setMod() 86 87 def name(self): 88 n = os.path.splitext(os.path.basename(self.path))[0] 89 return n 90 91 # Scheduler 92 ########################################################################## 93 94 supportedSchedulerVersions = (1, 2) 95 96 def schedVer(self): 97 ver = self.conf.get("schedVer", 1) 98 if ver in self.supportedSchedulerVersions: 99 return ver 100 else: 101 raise Exception("Unsupported scheduler version") 102 103 def _loadScheduler(self): 104 ver = self.schedVer() 105 if ver == 1: 106 from anki.sched import Scheduler 107 elif ver == 2: 108 from anki.schedv2 import Scheduler 109 110 self.sched = Scheduler(self) 111 112 def changeSchedulerVer(self, ver): 113 if ver == self.schedVer(): 114 return 115 if ver not in self.supportedSchedulerVersions: 116 raise Exception("Unsupported scheduler version") 117 118 self.modSchema(check=True) 119 self.clearUndo() 120 121 from anki.schedv2 import Scheduler 122 v2Sched = Scheduler(self) 123 124 if ver == 1: 125 v2Sched.moveToV1() 126 else: 127 v2Sched.moveToV2() 128 129 self.conf['schedVer'] = ver 130 self.setMod() 131 132 self._loadScheduler() 133 134 # DB-related 135 ########################################################################## 136 137 def load(self): 138 (self.crt, 139 self.mod, 140 self.scm, 141 self.dty, # no longer used 142 self._usn, 143 self.ls, 144 self.conf, 145 models, 146 decks, 147 dconf, 148 tags) = self.db.first(""" 149select crt, mod, scm, dty, usn, ls, 150conf, models, decks, dconf, tags from col""") 151 self.conf = json.loads(self.conf) 152 self.models.load(models) 153 self.decks.load(decks, dconf) 154 self.tags.load(tags) 155 156 def setMod(self): 157 """Mark DB modified. 158 159DB operations and the deck/tag/model managers do this automatically, so this 160is only necessary if you modify properties of this object or the conf dict.""" 161 self.db.mod = True 162 163 def flush(self, mod=None): 164 "Flush state to DB, updating mod time." 165 self.mod = intTime(1000) if mod is None else mod 166 self.db.execute( 167 """update col set 168crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", 169 self.crt, self.mod, self.scm, self.dty, 170 self._usn, self.ls, json.dumps(self.conf)) 171 172 def save(self, name=None, mod=None): 173 "Flush, commit DB, and take out another write lock." 174 # let the managers conditionally flush 175 self.models.flush() 176 self.decks.flush() 177 self.tags.flush() 178 # and flush deck + bump mod if db has been changed 179 if self.db.mod: 180 self.flush(mod=mod) 181 self.db.commit() 182 self.lock() 183 self.db.mod = False 184 self._markOp(name) 185 self._lastSave = time.time() 186 187 def autosave(self): 188 "Save if 5 minutes has passed since last save. True if saved." 189 if time.time() - self._lastSave > 300: 190 self.save() 191 return True 192 193 def lock(self): 194 # make sure we don't accidentally bump mod time 195 mod = self.db.mod 196 self.db.execute("update col set mod=mod") 197 self.db.mod = mod 198 199 def close(self, save=True): 200 "Disconnect from DB." 201 if self.db: 202 if save: 203 self.save() 204 else: 205 self.db.rollback() 206 if not self.server: 207 self.db.setAutocommit(True) 208 self.db.execute("pragma journal_mode = delete") 209 self.db.setAutocommit(False) 210 self.db.close() 211 self.db = None 212 self.media.close() 213 self._closeLog() 214 215 def reopen(self): 216 "Reconnect to DB (after changing threads, etc)." 217 import anki.db 218 if not self.db: 219 self.db = anki.db.DB(self.path) 220 self.media.connect() 221 self._openLog() 222 223 def rollback(self): 224 self.db.rollback() 225 self.load() 226 self.lock() 227 228 def modSchema(self, check): 229 "Mark schema modified. Call this first so user can abort if necessary." 230 if not self.schemaChanged(): 231 if check and not runFilter("modSchema", True): 232 raise AnkiError("abortSchemaMod") 233 self.scm = intTime(1000) 234 self.setMod() 235 236 def schemaChanged(self): 237 "True if schema changed since last sync." 238 return self.scm > self.ls 239 240 def usn(self): 241 return self._usn if self.server else -1 242 243 def beforeUpload(self): 244 "Called before a full upload." 245 tbls = "notes", "cards", "revlog" 246 for t in tbls: 247 self.db.execute("update %s set usn=0 where usn=-1" % t) 248 # we can save space by removing the log of deletions 249 self.db.execute("delete from graves") 250 self._usn += 1 251 self.models.beforeUpload() 252 self.tags.beforeUpload() 253 self.decks.beforeUpload() 254 self.modSchema(check=False) 255 self.ls = self.scm 256 # ensure db is compacted before upload 257 self.db.setAutocommit(True) 258 self.db.execute("vacuum") 259 self.db.execute("analyze") 260 self.close() 261 262 # Object creation helpers 263 ########################################################################## 264 265 def getCard(self, id): 266 return anki.cards.Card(self, id) 267 268 def getNote(self, id): 269 return anki.notes.Note(self, id=id) 270 271 # Utils 272 ########################################################################## 273 274 def nextID(self, type, inc=True): 275 type = "next"+type.capitalize() 276 id = self.conf.get(type, 1) 277 if inc: 278 self.conf[type] = id+1 279 return id 280 281 def reset(self): 282 "Rebuild the queue and reload data after DB modified." 283 self.sched.reset() 284 285 # Deletion logging 286 ########################################################################## 287 288 def _logRem(self, ids, type): 289 self.db.executemany("insert into graves values (%d, ?, %d)" % ( 290 self.usn(), type), ([x] for x in ids)) 291 292 # Notes 293 ########################################################################## 294 295 def noteCount(self): 296 return self.db.scalar("select count() from notes") 297 298 def newNote(self, forDeck=True): 299 "Return a new note with the current model." 300 return anki.notes.Note(self, self.models.current(forDeck)) 301 302 def addNote(self, note): 303 "Add a note to the collection. Return number of new cards." 304 # check we have card models available, then save 305 cms = self.findTemplates(note) 306 if not cms: 307 return 0 308 note.flush() 309 # deck conf governs which of these are used 310 due = self.nextID("pos") 311 # add cards 312 ncards = 0 313 for template in cms: 314 self._newCard(note, template, due) 315 ncards += 1 316 return ncards 317 318 def remNotes(self, ids): 319 self.remCards(self.db.list("select id from cards where nid in "+ 320 ids2str(ids))) 321 322 def _remNotes(self, ids): 323 "Bulk delete notes by ID. Don't call this directly." 324 if not ids: 325 return 326 strids = ids2str(ids) 327 # we need to log these independently of cards, as one side may have 328 # more card templates 329 runHook("remNotes", self, ids) 330 self._logRem(ids, REM_NOTE) 331 self.db.execute("delete from notes where id in %s" % strids) 332 333 # Card creation 334 ########################################################################## 335 336 def findTemplates(self, note): 337 "Return (active), non-empty templates." 338 model = note.model() 339 avail = self.models.availOrds(model, joinFields(note.fields)) 340 return self._tmplsFromOrds(model, avail) 341 342 def _tmplsFromOrds(self, model, avail): 343 ok = [] 344 if model['type'] == MODEL_STD: 345 for t in model['tmpls']: 346 if t['ord'] in avail: 347 ok.append(t) 348 else: 349 # cloze - generate temporary templates from first 350 for ord in avail: 351 t = copy.copy(model['tmpls'][0]) 352 t['ord'] = ord 353 ok.append(t) 354 return ok 355 356 def genCards(self, nids): 357 "Generate cards for non-empty templates, return ids to remove." 358 # build map of (nid,ord) so we don't create dupes 359 snids = ids2str(nids) 360 have = {} 361 dids = {} 362 dues = {} 363 for id, nid, ord, did, due, odue, odid, type in self.db.execute( 364 "select id, nid, ord, did, due, odue, odid, type from cards where nid in "+snids): 365 # existing cards 366 if nid not in have: 367 have[nid] = {} 368 have[nid][ord] = id 369 # if in a filtered deck, add new cards to original deck 370 if odid != 0: 371 did = odid 372 # and their dids 373 if nid in dids: 374 if dids[nid] and dids[nid] != did: 375 # cards are in two or more different decks; revert to 376 # model default 377 dids[nid] = None 378 else: 379 # first card or multiple cards in same deck 380 dids[nid] = did 381 # save due 382 if odid != 0: 383 due = odue 384 if nid not in dues and type == 0: 385 # Add due to new card only if it's the due of a new sibling 386 dues[nid] = due 387 # build cards for each note 388 data = [] 389 ts = maxID(self.db) 390 now = intTime() 391 rem = [] 392 usn = self.usn() 393 for nid, mid, flds in self.db.execute( 394 "select id, mid, flds from notes where id in "+snids): 395 model = self.models.get(mid) 396 avail = self.models.availOrds(model, flds) 397 did = dids.get(nid) or model['did'] 398 due = dues.get(nid) 399 # add any missing cards 400 for t in self._tmplsFromOrds(model, avail): 401 doHave = nid in have and t['ord'] in have[nid] 402 if not doHave: 403 # check deck is not a cram deck 404 did = t['did'] or did 405 if self.decks.isDyn(did): 406 did = 1 407 # if the deck doesn't exist, use default instead 408 did = self.decks.get(did)['id'] 409 # use sibling due# if there is one, else use a new id 410 if due is None: 411 due = self.nextID("pos") 412 data.append((ts, nid, did, t['ord'], 413 now, usn, due)) 414 ts += 1 415 # note any cards that need removing 416 if nid in have: 417 for ord, id in list(have[nid].items()): 418 if ord not in avail: 419 rem.append(id) 420 # bulk update 421 self.db.executemany(""" 422insert into cards values (?,?,?,?,?,?,0,0,?,0,0,0,0,0,0,0,0,"")""", 423 data) 424 return rem 425 426 # type 0 - when previewing in add dialog, only non-empty 427 # type 1 - when previewing edit, only existing 428 # type 2 - when previewing in models dialog, all templates 429 def previewCards(self, note, type=0, did=None): 430 if type == 0: 431 cms = self.findTemplates(note) 432 elif type == 1: 433 cms = [c.template() for c in note.cards()] 434 else: 435 cms = note.model()['tmpls'] 436 if not cms: 437 return [] 438 cards = [] 439 for template in cms: 440 cards.append(self._newCard(note, template, 1, flush=False, did=did)) 441 return cards 442 443 def _newCard(self, note, template, due, flush=True, did=None): 444 "Create a new card." 445 card = anki.cards.Card(self) 446 card.nid = note.id 447 card.ord = template['ord'] 448 card.did = self.db.scalar("select did from cards where nid = ? and ord = ?", card.nid, card.ord) 449 # Use template did (deck override) if valid, otherwise did in argument, otherwise model did 450 if not card.did: 451 if template['did'] and str(template['did']) in self.decks.decks: 452 card.did = template['did'] 453 elif did: 454 card.did = did 455 else: 456 card.did = note.model()['did'] 457 # if invalid did, use default instead 458 deck = self.decks.get(card.did) 459 if deck['dyn']: 460 # must not be a filtered deck 461 card.did = 1 462 else: 463 card.did = deck['id'] 464 card.due = self._dueForDid(card.did, due) 465 if flush: 466 card.flush() 467 return card 468 469 def _dueForDid(self, did, due): 470 conf = self.decks.confForDid(did) 471 # in order due? 472 if conf['new']['order'] == NEW_CARDS_DUE: 473 return due 474 else: 475 # random mode; seed with note ts so all cards of this note get the 476 # same random number 477 r = random.Random() 478 r.seed(due) 479 return r.randrange(1, max(due, 1000)) 480 481 # Cards 482 ########################################################################## 483 484 def isEmpty(self): 485 return not self.db.scalar("select 1 from cards limit 1") 486 487 def cardCount(self): 488 return self.db.scalar("select count() from cards") 489 490 def remCards(self, ids, notes=True): 491 "Bulk delete cards by ID." 492 if not ids: 493 return 494 sids = ids2str(ids) 495 nids = self.db.list("select nid from cards where id in "+sids) 496 # remove cards 497 self._logRem(ids, REM_CARD) 498 self.db.execute("delete from cards where id in "+sids) 499 # then notes 500 if not notes: 501 return 502 nids = self.db.list(""" 503select id from notes where id in %s and id not in (select nid from cards)""" % 504 ids2str(nids)) 505 self._remNotes(nids) 506 507 def emptyCids(self): 508 rem = [] 509 for m in self.models.all(): 510 rem += self.genCards(self.models.nids(m)) 511 return rem 512 513 def emptyCardReport(self, cids): 514 rep = "" 515 for ords, cnt, flds in self.db.all(""" 516select group_concat(ord+1), count(), flds from cards c, notes n 517where c.nid = n.id and c.id in %s group by nid""" % ids2str(cids)): 518 rep += _("Empty card numbers: %(c)s\nFields: %(f)s\n\n") % dict( 519 c=ords, f=flds.replace("\x1f", " / ")) 520 return rep 521 522 # Field checksums and sorting fields 523 ########################################################################## 524 525 def _fieldData(self, snids): 526 return self.db.execute( 527 "select id, mid, flds from notes where id in "+snids) 528 529 def updateFieldCache(self, nids): 530 "Update field checksums and sort cache, after find&replace, etc." 531 snids = ids2str(nids) 532 r = [] 533 for (nid, mid, flds) in self._fieldData(snids): 534 fields = splitFields(flds) 535 model = self.models.get(mid) 536 if not model: 537 # note points to invalid model 538 continue 539 r.append((stripHTMLMedia(fields[self.models.sortIdx(model)]), 540 fieldChecksum(fields[0]), 541 nid)) 542 # apply, relying on calling code to bump usn+mod 543 self.db.executemany("update notes set sfld=?, csum=? where id=?", r) 544 545 # Q/A generation 546 ########################################################################## 547 548 def renderQA(self, ids=None, type="card"): 549 # gather metadata 550 if type == "card": 551 where = "and c.id in " + ids2str(ids) 552 elif type == "note": 553 where = "and f.id in " + ids2str(ids) 554 elif type == "model": 555 where = "and m.id in " + ids2str(ids) 556 elif type == "all": 557 where = "" 558 else: 559 raise Exception() 560 return [self._renderQA(row) 561 for row in self._qaData(where)] 562 563 def _renderQA(self, data, qfmt=None, afmt=None): 564 "Returns hash of id, question, answer." 565 # data is [cid, nid, mid, did, ord, tags, flds, cardFlags] 566 # unpack fields and create dict 567 flist = splitFields(data[6]) 568 fields = {} 569 model = self.models.get(data[2]) 570 for (name, (idx, conf)) in list(self.models.fieldMap(model).items()): 571 fields[name] = flist[idx] 572 fields['Tags'] = data[5].strip() 573 fields['Type'] = model['name'] 574 fields['Deck'] = self.decks.name(data[3]) 575 fields['Subdeck'] = fields['Deck'].split('::')[-1] 576 fields['CardFlag'] = self._flagNameFromCardFlags(data[7]) 577 if model['type'] == MODEL_STD: 578 template = model['tmpls'][data[4]] 579 else: 580 template = model['tmpls'][0] 581 fields['Card'] = template['name'] 582 fields['c%d' % (data[4]+1)] = "1" 583 # render q & a 584 d = dict(id=data[0]) 585 qfmt = qfmt or template['qfmt'] 586 afmt = afmt or template['afmt'] 587 for (type, format) in (("q", qfmt), ("a", afmt)): 588 if type == "q": 589 format = re.sub("{{(?!type:)(.*?)cloze:", r"{{\1cq-%d:" % (data[4]+1), format) 590 format = format.replace("<%cloze:", "<%%cq:%d:" % ( 591 data[4]+1)) 592 else: 593 format = re.sub("{{(.*?)cloze:", r"{{\1ca-%d:" % (data[4]+1), format) 594 format = format.replace("<%cloze:", "<%%ca:%d:" % ( 595 data[4]+1)) 596 fields['FrontSide'] = stripSounds(d['q']) 597 fields = runFilter("mungeFields", fields, model, data, self) 598 html = anki.template.render(format, fields) 599 d[type] = runFilter( 600 "mungeQA", html, type, fields, model, data, self) 601 # empty cloze? 602 if type == 'q' and model['type'] == MODEL_CLOZE: 603 if not self.models._availClozeOrds(model, data[6], False): 604 d['q'] += ("<p>" + _( 605 "Please edit this note and add some cloze deletions. (%s)") % ( 606 "<a href=%s#cloze>%s</a>" % (HELP_SITE, _("help")))) 607 return d 608 609 def _qaData(self, where=""): 610 "Return [cid, nid, mid, did, ord, tags, flds, cardFlags] db query" 611 return self.db.execute(""" 612select c.id, f.id, f.mid, c.did, c.ord, f.tags, f.flds, c.flags 613from cards c, notes f 614where c.nid == f.id 615%s""" % where) 616 617 def _flagNameFromCardFlags(self, flags): 618 flag = flags & 0b111 619 if not flag: 620 return "" 621 return "flag%d" % flag 622 623 # Finding cards 624 ########################################################################## 625 626 def findCards(self, query, order=False): 627 return anki.find.Finder(self).findCards(query, order) 628 629 def findNotes(self, query): 630 return anki.find.Finder(self).findNotes(query) 631 632 def findReplace(self, nids, src, dst, regex=None, field=None, fold=True): 633 return anki.find.findReplace(self, nids, src, dst, regex, field, fold) 634 635 def findDupes(self, fieldName, search=""): 636 return anki.find.findDupes(self, fieldName, search) 637 638 # Stats 639 ########################################################################## 640 641 def cardStats(self, card): 642 from anki.stats import CardStats 643 return CardStats(self, card).report() 644 645 def stats(self): 646 from anki.stats import CollectionStats 647 return CollectionStats(self) 648 649 # Timeboxing 650 ########################################################################## 651 652 def startTimebox(self): 653 self._startTime = time.time() 654 self._startReps = self.sched.reps 655 656 def timeboxReached(self): 657 "Return (elapsedTime, reps) if timebox reached, or False." 658 if not self.conf['timeLim']: 659 # timeboxing disabled 660 return False 661 elapsed = time.time() - self._startTime 662 if elapsed > self.conf['timeLim']: 663 return (self.conf['timeLim'], self.sched.reps - self._startReps) 664 665 # Undo 666 ########################################################################## 667 668 def clearUndo(self): 669 # [type, undoName, data] 670 # type 1 = review; type 2 = checkpoint 671 self._undo = None 672 673 def undoName(self): 674 "Undo menu item name, or None if undo unavailable." 675 if not self._undo: 676 return None 677 return self._undo[1] 678 679 def undo(self): 680 if self._undo[0] == 1: 681 return self._undoReview() 682 else: 683 self._undoOp() 684 685 def markReview(self, card): 686 old = [] 687 if self._undo: 688 if self._undo[0] == 1: 689 old = self._undo[2] 690 self.clearUndo() 691 wasLeech = card.note().hasTag("leech") or False 692 self._undo = [1, _("Review"), old + [copy.copy(card)], wasLeech] 693 694 def _undoReview(self): 695 data = self._undo[2] 696 wasLeech = self._undo[3] 697 c = data.pop() 698 if not data: 699 self.clearUndo() 700 # remove leech tag if it didn't have it before 701 if not wasLeech and c.note().hasTag("leech"): 702 c.note().delTag("leech") 703 c.note().flush() 704 # write old data 705 c.flush() 706 # and delete revlog entry 707 last = self.db.scalar( 708 "select id from revlog where cid = ? " 709 "order by id desc limit 1", c.id) 710 self.db.execute("delete from revlog where id = ?", last) 711 # restore any siblings 712 self.db.execute( 713 "update cards set queue=type,mod=?,usn=? where queue=-2 and nid=?", 714 intTime(), self.usn(), c.nid) 715 # and finally, update daily counts 716 n = 1 if c.queue == 3 else c.queue 717 type = ("new", "lrn", "rev")[n] 718 self.sched._updateStats(c, type, -1) 719 self.sched.reps -= 1 720 return c.id 721 722 def _markOp(self, name): 723 "Call via .save()" 724 if name: 725 self._undo = [2, name] 726 else: 727 # saving disables old checkpoint, but not review undo 728 if self._undo and self._undo[0] == 2: 729 self.clearUndo() 730 731 def _undoOp(self): 732 self.rollback() 733 self.clearUndo() 734 735 # DB maintenance 736 ########################################################################## 737 738 def basicCheck(self): 739 "Basic integrity check for syncing. True if ok." 740 # cards without notes 741 if self.db.scalar(""" 742select 1 from cards where nid not in (select id from notes) limit 1"""): 743 return 744 # notes without cards or models 745 if self.db.scalar(""" 746select 1 from notes where id not in (select distinct nid from cards) 747or mid not in %s limit 1""" % ids2str(self.models.ids())): 748 return 749 # invalid ords 750 for m in self.models.all(): 751 # ignore clozes 752 if m['type'] != MODEL_STD: 753 continue 754 if self.db.scalar(""" 755select 1 from cards where ord not in %s and nid in ( 756select id from notes where mid = ?) limit 1""" % 757 ids2str([t['ord'] for t in m['tmpls']]), 758 m['id']): 759 return 760 return True 761 762 def fixIntegrity(self): 763 "Fix possible problems and rebuild caches." 764 problems = [] 765 curs = self.db.cursor() 766 self.save() 767 oldSize = os.stat(self.path)[stat.ST_SIZE] 768 if self.db.scalar("pragma integrity_check") != "ok": 769 return (_("Collection is corrupt. Please see the manual."), False) 770 # note types with a missing model 771 ids = self.db.list(""" 772select id from notes where mid not in """ + ids2str(self.models.ids())) 773 if ids: 774 problems.append( 775 ngettext("Deleted %d note with missing note type.", 776 "Deleted %d notes with missing note type.", len(ids)) 777 % len(ids)) 778 self.remNotes(ids) 779 # for each model 780 for m in self.models.all(): 781 for t in m['tmpls']: 782 if t['did'] == "None": 783 t['did'] = None 784 problems.append(_("Fixed AnkiDroid deck override bug.")) 785 self.models.save(m) 786 if m['type'] == MODEL_STD: 787 # model with missing req specification 788 if 'req' not in m: 789 self.models._updateRequired(m) 790 problems.append(_("Fixed note type: %s") % m['name']) 791 # cards with invalid ordinal 792 ids = self.db.list(""" 793select id from cards where ord not in %s and nid in ( 794select id from notes where mid = ?)""" % 795 ids2str([t['ord'] for t in m['tmpls']]), 796 m['id']) 797 if ids: 798 problems.append( 799 ngettext("Deleted %d card with missing template.", 800 "Deleted %d cards with missing template.", 801 len(ids)) % len(ids)) 802 self.remCards(ids) 803 # notes with invalid field count 804 ids = [] 805 for id, flds in self.db.execute( 806 "select id, flds from notes where mid = ?", m['id']): 807 if (flds.count("\x1f") + 1) != len(m['flds']): 808 ids.append(id) 809 if ids: 810 problems.append( 811 ngettext("Deleted %d note with wrong field count.", 812 "Deleted %d notes with wrong field count.", 813 len(ids)) % len(ids)) 814 self.remNotes(ids) 815 # delete any notes with missing cards 816 ids = self.db.list(""" 817select id from notes where id not in (select distinct nid from cards)""") 818 if ids: 819 cnt = len(ids) 820 problems.append( 821 ngettext("Deleted %d note with no cards.", 822 "Deleted %d notes with no cards.", cnt) % cnt) 823 self._remNotes(ids) 824 # cards with missing notes 825 ids = self.db.list(""" 826select id from cards where nid not in (select id from notes)""") 827 if ids: 828 cnt = len(ids) 829 problems.append( 830 ngettext("Deleted %d card with missing note.", 831 "Deleted %d cards with missing note.", cnt) % cnt) 832 self.remCards(ids) 833 # cards with odue set when it shouldn't be 834 ids = self.db.list(""" 835select id from cards where odue > 0 and (type=1 or queue=2) and not odid""") 836 if ids: 837 cnt = len(ids) 838 problems.append( 839 ngettext("Fixed %d card with invalid properties.", 840 "Fixed %d cards with invalid properties.", cnt) % cnt) 841 self.db.execute("update cards set odue=0 where id in "+ 842 ids2str(ids)) 843 # cards with odid set when not in a dyn deck 844 dids = [id for id in self.decks.allIds() if not self.decks.isDyn(id)] 845 ids = self.db.list(""" 846select id from cards where odid > 0 and did in %s""" % ids2str(dids)) 847 if ids: 848 cnt = len(ids) 849 problems.append( 850 ngettext("Fixed %d card with invalid properties.", 851 "Fixed %d cards with invalid properties.", cnt) % cnt) 852 self.db.execute("update cards set odid=0, odue=0 where id in "+ 853 ids2str(ids)) 854 # tags 855 self.tags.registerNotes() 856 # field cache 857 for m in self.models.all(): 858 self.updateFieldCache(self.models.nids(m)) 859 # new cards can't have a due position > 32 bits, so wrap items over 860 # 2 million back to 1 million 861 curs.execute(""" 862update cards set due=1000000+due%1000000,mod=?,usn=? where due>=1000000 863and type=0""", [intTime(), self.usn()]) 864 if curs.rowcount: 865 problems.append("Found %d new cards with a due number >= 1,000,000 - consider repositioning them in the Browse screen." % curs.rowcount) 866 # new card position 867 self.conf['nextPos'] = self.db.scalar( 868 "select max(due)+1 from cards where type = 0") or 0 869 # reviews should have a reasonable due # 870 ids = self.db.list( 871 "select id from cards where queue = 2 and due > 100000") 872 if ids: 873 problems.append("Reviews had incorrect due date.") 874 self.db.execute( 875 "update cards set due = ?, ivl = 1, mod = ?, usn = ? where id in %s" 876 % ids2str(ids), self.sched.today, intTime(), self.usn()) 877 # v2 sched had a bug that could create decimal intervals 878 curs.execute("update cards set ivl=round(ivl),due=round(due) where ivl!=round(ivl) or due!=round(due)") 879 if curs.rowcount: 880 problems.append("Fixed %d cards with v2 scheduler bug." % curs.rowcount) 881 882 curs.execute("update revlog set ivl=round(ivl),lastIvl=round(lastIvl) where ivl!=round(ivl) or lastIvl!=round(lastIvl)") 883 if curs.rowcount: 884 problems.append("Fixed %d review history entries with v2 scheduler bug." % curs.rowcount) 885 # models 886 if self.models.ensureNotEmpty(): 887 problems.append("Added missing note type.") 888 # and finally, optimize 889 self.optimize() 890 newSize = os.stat(self.path)[stat.ST_SIZE] 891 txt = _("Database rebuilt and optimized.") 892 ok = not problems 893 problems.append(txt) 894 # if any problems were found, force a full sync 895 if not ok: 896 self.modSchema(check=False) 897 self.save() 898 return ("\n".join(problems), ok) 899 900 def optimize(self): 901 self.db.setAutocommit(True) 902 self.db.execute("vacuum") 903 self.db.execute("analyze") 904 self.db.setAutocommit(False) 905 self.lock() 906 907 # Logging 908 ########################################################################## 909 910 def log(self, *args, **kwargs): 911 if not self._debugLog: 912 return 913 def customRepr(x): 914 if isinstance(x, str): 915 return x 916 return pprint.pformat(x) 917 path, num, fn, y = traceback.extract_stack( 918 limit=2+kwargs.get("stack", 0))[0] 919 buf = "[%s] %s:%s(): %s" % (intTime(), os.path.basename(path), fn, 920 ", ".join([customRepr(x) for x in args])) 921 self._logHnd.write(buf + "\n") 922 if devMode: 923 print(buf) 924 925 def _openLog(self): 926 if not self._debugLog: 927 return 928 lpath = re.sub(r"\.anki2$", ".log", self.path) 929 if os.path.exists(lpath) and os.path.getsize(lpath) > 10*1024*1024: 930 lpath2 = lpath + ".old" 931 if os.path.exists(lpath2): 932 os.unlink(lpath2) 933 os.rename(lpath, lpath2) 934 self._logHnd = open(lpath, "a", encoding="utf8") 935 936 def _closeLog(self): 937 if not self._debugLog: 938 return 939 self._logHnd.close() 940 self._logHnd = None 941 942 # Card Flags 943 ########################################################################## 944 945 def setUserFlag(self, flag, cids): 946 assert 0 <= flag <= 7 947 self.db.execute("update cards set flags = (flags & ~?) | ?, usn=?, mod=? where id in %s" % 948 ids2str(cids), 0b111, flag, self.usn(), intTime()) 949