1# -*- coding: utf-8 -*- 2# Copyright: Damien Elmes <anki@ichi2.net> 3# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html 4 5import re 6import os 7import random 8import time 9import math 10from html.entities import name2codepoint 11import subprocess 12import tempfile 13import string 14import sys 15import locale 16from hashlib import sha1 17import platform 18import traceback 19import json 20 21from .lang import _, ngettext 22 23# Time handling 24############################################################################## 25 26def intTime(scale=1): 27 "The time in integer seconds. Pass scale=1000 to get milliseconds." 28 return int(time.time()*scale) 29 30timeTable = { 31 "years": lambda n: ngettext("%s year", "%s years", n), 32 "months": lambda n: ngettext("%s month", "%s months", n), 33 "days": lambda n: ngettext("%s day", "%s days", n), 34 "hours": lambda n: ngettext("%s hour", "%s hours", n), 35 "minutes": lambda n: ngettext("%s minute", "%s minutes", n), 36 "seconds": lambda n: ngettext("%s second", "%s seconds", n), 37 } 38 39afterTimeTable = { 40 "years": lambda n: ngettext("%s year<!--after-->", "%s years<!--after-->", n), 41 "months": lambda n: ngettext("%s month<!--after-->", "%s months<!--after-->", n), 42 "days": lambda n: ngettext("%s day<!--after-->", "%s days<!--after-->", n), 43 "hours": lambda n: ngettext("%s hour<!--after-->", "%s hours<!--after-->", n), 44 "minutes": lambda n: ngettext("%s minute<!--after-->", "%s minutes<!--after-->", n), 45 "seconds": lambda n: ngettext("%s second<!--after-->", "%s seconds<!--after-->", n), 46 } 47 48def shortTimeFmt(type): 49 return { 50 "years": _("%sy"), 51 "months": _("%smo"), 52 "days": _("%sd"), 53 "hours": _("%sh"), 54 "minutes": _("%sm"), 55 "seconds": _("%ss"), 56 }[type] 57 58def fmtTimeSpan(time, pad=0, point=0, short=False, after=False, unit=99): 59 "Return a string representing a time span (eg '2 days')." 60 (type, point) = optimalPeriod(time, point, unit) 61 time = convertSecondsTo(time, type) 62 if not point: 63 time = int(round(time)) 64 if short: 65 fmt = shortTimeFmt(type) 66 else: 67 if after: 68 fmt = afterTimeTable[type](_pluralCount(time, point)) 69 else: 70 fmt = timeTable[type](_pluralCount(time, point)) 71 timestr = "%(a)d.%(b)df" % {'a': pad, 'b': point} 72 return locale.format_string("%" + (fmt % timestr), time) 73 74def optimalPeriod(time, point, unit): 75 if abs(time) < 60 or unit < 1: 76 type = "seconds" 77 point -= 1 78 elif abs(time) < 3600 or unit < 2: 79 type = "minutes" 80 elif abs(time) < 60 * 60 * 24 or unit < 3: 81 type = "hours" 82 elif abs(time) < 60 * 60 * 24 * 30 or unit < 4: 83 type = "days" 84 elif abs(time) < 60 * 60 * 24 * 365 or unit < 5: 85 type = "months" 86 point += 1 87 else: 88 type = "years" 89 point += 1 90 return (type, max(point, 0)) 91 92def convertSecondsTo(seconds, type): 93 if type == "seconds": 94 return seconds 95 elif type == "minutes": 96 return seconds / 60 97 elif type == "hours": 98 return seconds / 3600 99 elif type == "days": 100 return seconds / 86400 101 elif type == "months": 102 return seconds / 2592000 103 elif type == "years": 104 return seconds / 31536000 105 assert False 106 107def _pluralCount(time, point): 108 if point: 109 return 2 110 return math.floor(time) 111 112# Locale 113############################################################################## 114 115def fmtPercentage(float_value, point=1): 116 "Return float with percentage sign" 117 fmt = '%' + "0.%(b)df" % {'b': point} 118 return locale.format_string(fmt, float_value) + "%" 119 120def fmtFloat(float_value, point=1): 121 "Return a string with decimal separator according to current locale" 122 fmt = '%' + "0.%(b)df" % {'b': point} 123 return locale.format_string(fmt, float_value) 124 125# HTML 126############################################################################## 127reStyle = re.compile("(?si)<style.*?>.*?</style>") 128reScript = re.compile("(?si)<script.*?>.*?</script>") 129reTag = re.compile("<.*?>") 130reEnts = re.compile("&#?\w+;") 131reMedia = re.compile("(?i)<img[^>]+src=[\"']?([^\"'>]+)[\"']?[^>]*>") 132 133def stripHTML(s): 134 s = reStyle.sub("", s) 135 s = reScript.sub("", s) 136 s = reTag.sub("", s) 137 s = entsToTxt(s) 138 return s 139 140def stripHTMLMedia(s): 141 "Strip HTML but keep media filenames" 142 s = reMedia.sub(" \\1 ", s) 143 return stripHTML(s) 144 145def minimizeHTML(s): 146 "Correct Qt's verbose bold/underline/etc." 147 s = re.sub('<span style="font-weight:600;">(.*?)</span>', '<b>\\1</b>', 148 s) 149 s = re.sub('<span style="font-style:italic;">(.*?)</span>', '<i>\\1</i>', 150 s) 151 s = re.sub('<span style="text-decoration: underline;">(.*?)</span>', 152 '<u>\\1</u>', s) 153 return s 154 155def htmlToTextLine(s): 156 s = s.replace("<br>", " ") 157 s = s.replace("<br />", " ") 158 s = s.replace("<div>", " ") 159 s = s.replace("\n", " ") 160 s = re.sub("\[sound:[^]]+\]", "", s) 161 s = re.sub("\[\[type:[^]]+\]\]", "", s) 162 s = stripHTMLMedia(s) 163 s = s.strip() 164 return s 165 166def entsToTxt(html): 167 # entitydefs defines nbsp as \xa0 instead of a standard space, so we 168 # replace it first 169 html = html.replace(" ", " ") 170 def fixup(m): 171 text = m.group(0) 172 if text[:2] == "&#": 173 # character reference 174 try: 175 if text[:3] == "&#x": 176 return chr(int(text[3:-1], 16)) 177 else: 178 return chr(int(text[2:-1])) 179 except ValueError: 180 pass 181 else: 182 # named entity 183 try: 184 text = chr(name2codepoint[text[1:-1]]) 185 except KeyError: 186 pass 187 return text # leave as is 188 return reEnts.sub(fixup, html) 189 190# IDs 191############################################################################## 192 193def hexifyID(id): 194 return "%x" % int(id) 195 196def dehexifyID(id): 197 return int(id, 16) 198 199def ids2str(ids): 200 """Given a list of integers, return a string '(int1,int2,...)'.""" 201 return "(%s)" % ",".join(str(i) for i in ids) 202 203def timestampID(db, table): 204 "Return a non-conflicting timestamp for table." 205 # be careful not to create multiple objects without flushing them, or they 206 # may share an ID. 207 t = intTime(1000) 208 while db.scalar("select id from %s where id = ?" % table, t): 209 t += 1 210 return t 211 212def maxID(db): 213 "Return the first safe ID to use." 214 now = intTime(1000) 215 for tbl in "cards", "notes": 216 now = max(now, db.scalar("select max(id) from %s" % tbl) or 0) 217 return now + 1 218 219# used in ankiweb 220def base62(num, extra=""): 221 s = string; table = s.ascii_letters + s.digits + extra 222 buf = "" 223 while num: 224 num, i = divmod(num, len(table)) 225 buf = table[i] + buf 226 return buf 227 228_base91_extra_chars = "!#$%&()*+,-./:;<=>?@[]^_`{|}~" 229def base91(num): 230 # all printable characters minus quotes, backslash and separators 231 return base62(num, _base91_extra_chars) 232 233def guid64(): 234 "Return a base91-encoded 64bit random number." 235 return base91(random.randint(0, 2**64-1)) 236 237# increment a guid by one, for note type conflicts 238def incGuid(guid): 239 return _incGuid(guid[::-1])[::-1] 240 241def _incGuid(guid): 242 s = string; table = s.ascii_letters + s.digits + _base91_extra_chars 243 idx = table.index(guid[0]) 244 if idx + 1 == len(table): 245 # overflow 246 guid = table[0] + _incGuid(guid[1:]) 247 else: 248 guid = table[idx+1] + guid[1:] 249 return guid 250 251# Fields 252############################################################################## 253 254def joinFields(list): 255 return "\x1f".join(list) 256 257def splitFields(string): 258 return string.split("\x1f") 259 260# Checksums 261############################################################################## 262 263def checksum(data): 264 if isinstance(data, str): 265 data = data.encode("utf-8") 266 return sha1(data).hexdigest() 267 268def fieldChecksum(data): 269 # 32 bit unsigned number from first 8 digits of sha1 hash 270 return int(checksum(stripHTMLMedia(data).encode("utf-8"))[:8], 16) 271 272# Temp files 273############################################################################## 274 275_tmpdir = None 276 277def tmpdir(): 278 "A reusable temp folder which we clean out on each program invocation." 279 global _tmpdir 280 if not _tmpdir: 281 def cleanup(): 282 import shutil 283 shutil.rmtree(_tmpdir) 284 import atexit 285 atexit.register(cleanup) 286 _tmpdir = os.path.join(tempfile.gettempdir(), "anki_temp") 287 if not os.path.exists(_tmpdir): 288 os.mkdir(_tmpdir) 289 return _tmpdir 290 291def tmpfile(prefix="", suffix=""): 292 (fd, name) = tempfile.mkstemp(dir=tmpdir(), prefix=prefix, suffix=suffix) 293 os.close(fd) 294 return name 295 296def namedtmp(name, rm=True): 297 "Return tmpdir+name. Deletes any existing file." 298 path = os.path.join(tmpdir(), name) 299 if rm: 300 try: 301 os.unlink(path) 302 except (OSError, IOError): 303 pass 304 return path 305 306# Cmd invocation 307############################################################################## 308 309def call(argv, wait=True, **kwargs): 310 "Execute a command. If WAIT, return exit code." 311 # ensure we don't open a separate window for forking process on windows 312 if isWin: 313 si = subprocess.STARTUPINFO() 314 try: 315 si.dwFlags |= subprocess.STARTF_USESHOWWINDOW 316 except: 317 si.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW 318 else: 319 si = None 320 # run 321 try: 322 o = subprocess.Popen(argv, startupinfo=si, **kwargs) 323 except OSError: 324 # command not found 325 return -1 326 # wait for command to finish 327 if wait: 328 while 1: 329 try: 330 ret = o.wait() 331 except OSError: 332 # interrupted system call 333 continue 334 break 335 else: 336 ret = 0 337 return ret 338 339# OS helpers 340############################################################################## 341 342isMac = sys.platform.startswith("darwin") 343isWin = sys.platform.startswith("win32") 344isLin = not isMac and not isWin 345 346invalidFilenameChars = ":*?\"<>|" 347 348def invalidFilename(str, dirsep=True): 349 for c in invalidFilenameChars: 350 if c in str: 351 return c 352 if (dirsep or isWin) and "/" in str: 353 return "/" 354 elif (dirsep or not isWin) and "\\" in str: 355 return "\\" 356 elif str.strip().startswith("."): 357 return "." 358 359def platDesc(): 360 # we may get an interrupted system call, so try this in a loop 361 n = 0 362 theos = "unknown" 363 while n < 100: 364 n += 1 365 try: 366 system = platform.system() 367 if isMac: 368 theos = "mac:%s" % (platform.mac_ver()[0]) 369 elif isWin: 370 theos = "win:%s" % (platform.win32_ver()[0]) 371 elif system == "Linux": 372 dist = platform.dist() 373 theos = "lin:%s:%s" % (dist[0], dist[1]) 374 else: 375 theos = system 376 break 377 except: 378 continue 379 return theos 380 381# Debugging 382############################################################################## 383 384class TimedLog: 385 def __init__(self): 386 self._last = time.time() 387 def log(self, s): 388 path, num, fn, y = traceback.extract_stack(limit=2)[0] 389 sys.stderr.write("%5dms: %s(): %s\n" % ((time.time() - self._last)*1000, fn, s)) 390 self._last = time.time() 391 392