1#!/usr/bin/python3 -OO 2# Copyright 2007-2021 The SABnzbd-Team <team@sabnzbd.org> 3# 4# This program is free software; you can redistribute it and/or 5# modify it under the terms of the GNU General Public License 6# as published by the Free Software Foundation; either version 2 7# of the License, or (at your option) any later version. 8# 9# This program is distributed in the hope that it will be useful, 10# but WITHOUT ANY WARRANTY; without even the implied warranty of 11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12# GNU General Public License for more details. 13# 14# You should have received a copy of the GNU General Public License 15# along with this program; if not, write to the Free Software 16# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17 18""" 19sabnzbd.misc - misc classes 20""" 21import os 22import sys 23import logging 24import urllib.request 25import urllib.parse 26import re 27import subprocess 28import socket 29import time 30import datetime 31import inspect 32import ctypes 33import ipaddress 34from typing import Union, Tuple, Any, AnyStr, Optional, List 35 36import sabnzbd 37from sabnzbd.constants import DEFAULT_PRIORITY, MEBI, DEF_ARTICLE_CACHE_DEFAULT, DEF_ARTICLE_CACHE_MAX 38import sabnzbd.config as config 39import sabnzbd.cfg as cfg 40from sabnzbd.encoding import ubtou, platform_btou 41from sabnzbd.filesystem import userxbit 42 43TAB_UNITS = ("", "K", "M", "G", "T", "P") 44RE_UNITS = re.compile(r"(\d+\.*\d*)\s*([KMGTP]?)", re.I) 45RE_VERSION = re.compile(r"(\d+)\.(\d+)\.(\d+)([a-zA-Z]*)(\d*)") 46RE_IP4 = re.compile(r"inet\s+(addr:\s*)?(\d+\.\d+\.\d+\.\d+)") 47RE_IP6 = re.compile(r"inet6\s+(addr:\s*)?([0-9a-f:]+)", re.I) 48 49# Check if strings are defined for AM and PM 50HAVE_AMPM = bool(time.strftime("%p", time.localtime())) 51 52if sabnzbd.WIN32: 53 try: 54 import win32process 55 import win32con 56 57 # Define scheduling priorities 58 WIN_SCHED_PRIOS = { 59 1: win32process.IDLE_PRIORITY_CLASS, 60 2: win32process.BELOW_NORMAL_PRIORITY_CLASS, 61 3: win32process.NORMAL_PRIORITY_CLASS, 62 4: win32process.ABOVE_NORMAL_PRIORITY_CLASS, 63 } 64 except ImportError: 65 pass 66 67 68def time_format(fmt): 69 """Return time-format string adjusted for 12/24 hour clock setting""" 70 if cfg.ampm() and HAVE_AMPM: 71 return fmt.replace("%H:%M:%S", "%I:%M:%S %p").replace("%H:%M", "%I:%M %p") 72 else: 73 return fmt 74 75 76def calc_age(date: datetime.datetime, trans=False) -> str: 77 """Calculate the age difference between now and date. 78 Value is returned as either days, hours, or minutes. 79 When 'trans' is True, time symbols will be translated. 80 """ 81 if trans: 82 d = T("d") # : Single letter abbreviation of day 83 h = T("h") # : Single letter abbreviation of hour 84 m = T("m") # : Single letter abbreviation of minute 85 else: 86 d = "d" 87 h = "h" 88 m = "m" 89 try: 90 now = datetime.datetime.now() 91 # age = str(now - date).split(".")[0] #old calc_age 92 93 # time difference 94 dage = now - date 95 seconds = dage.seconds 96 # only one value should be returned 97 # if it is less than 1 day then it returns in hours, unless it is less than one hour where it returns in minutes 98 if dage.days: 99 age = "%d%s" % (dage.days, d) 100 elif int(seconds / 3600): 101 age = "%d%s" % (seconds / 3600, h) 102 else: 103 age = "%d%s" % (seconds / 60, m) 104 except: 105 age = "-" 106 107 return age 108 109 110def safe_lower(txt: Any) -> str: 111 """Return lowercased string. Return '' for None""" 112 if txt: 113 return txt.lower() 114 else: 115 return "" 116 117 118def cmp(x, y): 119 """ 120 Replacement for built-in funciton cmp that was removed in Python 3 121 122 Compare the two objects x and y and return an integer according to 123 the outcome. The return value is negative if x < y, zero if x == y 124 and strictly positive if x > y. 125 """ 126 127 return (x > y) - (x < y) 128 129 130def name_to_cat(fname, cat=None): 131 """Retrieve category from file name, but only if "cat" is None.""" 132 if cat is None and fname.startswith("{{"): 133 n = fname.find("}}") 134 if n > 2: 135 cat = fname[2:n].strip() 136 fname = fname[n + 2 :].strip() 137 logging.debug("Job %s has category %s", fname, cat) 138 139 return fname, cat 140 141 142def cat_to_opts(cat, pp=None, script=None, priority=None) -> Tuple[str, int, str, int]: 143 """Derive options from category, if options not already defined. 144 Specified options have priority over category-options. 145 If no valid category is given, special category '*' will supply default values 146 """ 147 def_cat = config.get_category() 148 cat = safe_lower(cat) 149 if cat in ("", "none", "default"): 150 cat = "*" 151 my_cat = config.get_category(cat) 152 # Ignore the input category if we don't know it 153 if my_cat == def_cat: 154 cat = "*" 155 156 if pp is None: 157 pp = my_cat.pp() 158 if pp == "": 159 pp = def_cat.pp() 160 161 if not script: 162 script = my_cat.script() 163 if safe_lower(script) in ("", "default"): 164 script = def_cat.script() 165 166 if priority is None or priority == "" or priority == DEFAULT_PRIORITY: 167 priority = my_cat.priority() 168 if priority == DEFAULT_PRIORITY: 169 priority = def_cat.priority() 170 171 logging.debug("Parsing category %s to attributes: pp=%s script=%s prio=%s", cat, pp, script, priority) 172 return cat, pp, script, priority 173 174 175def pp_to_opts(pp: int) -> Tuple[bool, bool, bool]: 176 """Convert numeric processing options to (repair, unpack, delete)""" 177 # Convert the pp to an int 178 pp = sabnzbd.interface.int_conv(pp) 179 if pp == 0: 180 return False, False, False 181 if pp == 1: 182 return True, False, False 183 if pp == 2: 184 return True, True, False 185 return True, True, True 186 187 188def opts_to_pp(repair: bool, unpack: bool, delete: bool) -> int: 189 """Convert (repair, unpack, delete) to numeric process options""" 190 pp = 0 191 if repair: 192 pp = 1 193 if unpack: 194 pp = 2 195 if delete: 196 pp = 3 197 return pp 198 199 200_wildcard_to_regex = { 201 "\\": r"\\", 202 "^": r"\^", 203 "$": r"\$", 204 ".": r"\.", 205 "[": r"\[", 206 "]": r"\]", 207 "(": r"\(", 208 ")": r"\)", 209 "+": r"\+", 210 "?": r".", 211 "|": r"\|", 212 "{": r"\{", 213 "}": r"\}", 214 "*": r".*", 215} 216 217 218def wildcard_to_re(text): 219 """Convert plain wildcard string (with '*' and '?') to regex.""" 220 return "".join([_wildcard_to_regex.get(ch, ch) for ch in text]) 221 222 223def cat_convert(cat): 224 """Convert indexer's category/group-name to user categories. 225 If no match found, but indexer-cat equals user-cat, then return user-cat 226 If no match found, but the indexer-cat starts with the user-cat, return user-cat 227 If no match found, return None 228 """ 229 if cat and cat.lower() != "none": 230 cats = config.get_ordered_categories() 231 raw_cats = config.get_categories() 232 for ucat in cats: 233 try: 234 # Ordered cat-list has tags only as string 235 indexer = raw_cats[ucat["name"]].newzbin() 236 if not isinstance(indexer, list): 237 indexer = [indexer] 238 except: 239 indexer = [] 240 for name in indexer: 241 if re.search("^%s$" % wildcard_to_re(name), cat, re.I): 242 if "." in name: 243 logging.debug('Convert group "%s" to user-cat "%s"', cat, ucat["name"]) 244 else: 245 logging.debug('Convert index site category "%s" to user-cat "%s"', cat, ucat["name"]) 246 return ucat["name"] 247 248 # Try to find full match between user category and indexer category 249 for ucat in cats: 250 if cat.lower() == ucat["name"].lower(): 251 logging.debug('Convert index site category "%s" to user-cat "%s"', cat, ucat["name"]) 252 return ucat["name"] 253 254 # Try to find partial match between user category and indexer category 255 for ucat in cats: 256 if cat.lower().startswith(ucat["name"].lower()): 257 logging.debug('Convert index site category "%s" to user-cat "%s"', cat, ucat["name"]) 258 return ucat["name"] 259 260 return None 261 262 263_SERVICE_KEY = "SYSTEM\\CurrentControlSet\\services\\" 264_SERVICE_PARM = "CommandLine" 265 266 267def get_serv_parms(service): 268 """Get the service command line parameters from Registry""" 269 import winreg 270 271 service_parms = [] 272 try: 273 key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, _SERVICE_KEY + service) 274 for n in range(winreg.QueryInfoKey(key)[1]): 275 name, service_parms, _val_type = winreg.EnumValue(key, n) 276 if name == _SERVICE_PARM: 277 break 278 winreg.CloseKey(key) 279 except OSError: 280 pass 281 282 # Always add the base program 283 service_parms.insert(0, os.path.normpath(os.path.abspath(sys.argv[0]))) 284 285 return service_parms 286 287 288def set_serv_parms(service, args): 289 """Set the service command line parameters in Registry""" 290 import winreg 291 292 serv = [] 293 for arg in args: 294 serv.append(arg[0]) 295 if arg[1]: 296 serv.append(arg[1]) 297 298 try: 299 key = winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, _SERVICE_KEY + service) 300 winreg.SetValueEx(key, _SERVICE_PARM, None, winreg.REG_MULTI_SZ, serv) 301 winreg.CloseKey(key) 302 except OSError: 303 return False 304 return True 305 306 307def get_from_url(url: str) -> Optional[str]: 308 """Retrieve URL and return content""" 309 try: 310 req = urllib.request.Request(url) 311 req.add_header("User-Agent", "SABnzbd/%s" % sabnzbd.__version__) 312 with urllib.request.urlopen(req) as response: 313 return ubtou(response.read()) 314 except: 315 return None 316 317 318def convert_version(text): 319 """Convert version string to numerical value and a testversion indicator""" 320 version = 0 321 test = True 322 m = RE_VERSION.search(ubtou(text)) 323 if m: 324 version = int(m.group(1)) * 1000000 + int(m.group(2)) * 10000 + int(m.group(3)) * 100 325 try: 326 if m.group(4).lower() == "rc": 327 version = version + 80 328 elif m.group(4).lower() == "beta": 329 version = version + 40 330 version = version + int(m.group(5)) 331 except: 332 version = version + 99 333 test = False 334 return version, test 335 336 337def check_latest_version(): 338 """Do an online check for the latest version 339 340 Perform an online version check 341 Syntax of online version file: 342 <current-final-release> 343 <url-of-current-final-release> 344 <latest-alpha/beta-or-rc> 345 <url-of-latest-alpha/beta/rc-release> 346 The latter two lines are only present when an alpha/beta/rc is available. 347 Formula for the version numbers (line 1 and 3). 348 <major>.<minor>.<bugfix>[rc|beta|alpha]<cand> 349 350 The <cand> value for a final version is assumned to be 99. 351 The <cand> value for the beta/rc version is 1..98, with RC getting 352 a boost of 80 and Beta of 40. 353 This is done to signal alpha/beta/rc users of availability of the final 354 version (which is implicitly 99). 355 People will only be informed to upgrade to a higher alpha/beta/rc version, if 356 they are already using an alpha/beta/rc. 357 RC's are valued higher than Beta's, which are valued higher than Alpha's. 358 """ 359 360 if not cfg.version_check(): 361 return 362 363 current, testver = convert_version(sabnzbd.__version__) 364 if not current: 365 logging.debug("Unsupported release number (%s), will not check", sabnzbd.__version__) 366 return 367 368 # Fetch version info 369 data = get_from_url("https://sabnzbd.org/latest.txt") 370 if not data: 371 logging.info("Cannot retrieve version information from GitHub.com") 372 logging.debug("Traceback: ", exc_info=True) 373 return 374 375 version_data = data.split() 376 try: 377 latest_label = version_data[0] 378 url = version_data[1] 379 except: 380 latest_label = "" 381 url = "" 382 383 try: 384 latest_testlabel = version_data[2] 385 url_beta = version_data[3] 386 except: 387 latest_testlabel = "" 388 url_beta = "" 389 390 latest = convert_version(latest_label)[0] 391 latest_test = convert_version(latest_testlabel)[0] 392 393 logging.debug( 394 "Checked for a new release, cur=%s, latest=%s (on %s), latest_test=%s (on %s)", 395 current, 396 latest, 397 url, 398 latest_test, 399 url_beta, 400 ) 401 402 if latest_test and cfg.version_check() > 1: 403 # User always wants to see the latest test release 404 latest = latest_test 405 latest_label = latest_testlabel 406 url = url_beta 407 408 notify_version = None 409 if current < latest: 410 # This is a test version, but user hasn't seen the 411 # "Final" of this one yet, so show the Final 412 # Or this one is behind, show latest final 413 sabnzbd.NEW_VERSION = (latest_label, url) 414 notify_version = latest_label 415 elif testver and current < latest_test: 416 # This is a test version beyond the latest Final, so show latest Alpha/Beta/RC 417 sabnzbd.NEW_VERSION = (latest_testlabel, url_beta) 418 notify_version = latest_testlabel 419 420 if notify_version: 421 sabnzbd.notifier.send_notification(T("Update Available!"), "SABnzbd %s" % notify_version, "other") 422 423 424def upload_file_to_sabnzbd(url, fp): 425 """Function for uploading nzbs to a running SABnzbd instance""" 426 try: 427 fp = urllib.parse.quote_plus(fp) 428 url = "%s&mode=addlocalfile&name=%s" % (url, fp) 429 # Add local API-key if it wasn't already in the registered URL 430 apikey = cfg.api_key() 431 if apikey and "apikey" not in url: 432 url = "%s&apikey=%s" % (url, apikey) 433 if "apikey" not in url: 434 # Use alternative login method 435 username = cfg.username() 436 password = cfg.password() 437 if username and password: 438 url = "%s&ma_username=%s&ma_password=%s" % (url, username, password) 439 get_from_url(url) 440 except: 441 logging.error(T("Failed to upload file: %s"), fp) 442 logging.info("Traceback: ", exc_info=True) 443 444 445def from_units(val: str) -> float: 446 """Convert K/M/G/T/P notation to float""" 447 val = str(val).strip().upper() 448 if val == "-1": 449 return float(val) 450 m = RE_UNITS.search(val) 451 if m: 452 if m.group(2): 453 val = float(m.group(1)) 454 unit = m.group(2) 455 n = 0 456 while unit != TAB_UNITS[n]: 457 val = val * 1024.0 458 n = n + 1 459 else: 460 val = m.group(1) 461 try: 462 return float(val) 463 except: 464 return 0.0 465 else: 466 return 0.0 467 468 469def to_units(val: Union[int, float], postfix="") -> str: 470 """Convert number to K/M/G/T/P notation 471 Show single decimal for M and higher 472 """ 473 dec_limit = 1 474 if val < 0: 475 sign = "-" 476 else: 477 sign = "" 478 val = str(abs(val)).strip() 479 480 n = 0 481 try: 482 val = float(val) 483 except: 484 return "" 485 while (val > 1023.0) and (n < 5): 486 val = val / 1024.0 487 n = n + 1 488 unit = TAB_UNITS[n] 489 if n > dec_limit: 490 decimals = 1 491 else: 492 decimals = 0 493 494 fmt = "%%s%%.%sf %%s%%s" % decimals 495 return fmt % (sign, val, unit, postfix) 496 497 498def caller_name(skip=2): 499 """Get a name of a caller in the format module.method 500 Originally used: https://gist.github.com/techtonik/2151727 501 Adapted for speed by using sys calls directly 502 """ 503 # Only do the tracing on Debug (function is always called) 504 if cfg.log_level() != 2: 505 return "N/A" 506 507 parentframe = sys._getframe(skip) 508 function_name = parentframe.f_code.co_name 509 510 # Module name is not available in the binaries, we can use the filename instead 511 if hasattr(sys, "frozen"): 512 module_name = inspect.getfile(parentframe) 513 else: 514 module_name = inspect.getmodule(parentframe).__name__ 515 516 # For decorated functions we have to go deeper 517 if function_name in ("call_func", "wrap") and skip == 2: 518 return caller_name(4) 519 520 return ".".join([module_name, function_name]) 521 522 523def exit_sab(value: int): 524 """Leave the program after flushing stderr/stdout""" 525 sys.stderr.flush() 526 sys.stdout.flush() 527 # Cannot use sys.exit as it will not work inside the macOS-runner-thread 528 os._exit(value) 529 530 531def split_host(srv): 532 """Split host:port notation, allowing for IPV6""" 533 if not srv: 534 return None, None 535 536 # IPV6 literal (with no port) 537 if srv[-1] == "]": 538 return srv, None 539 540 out = srv.rsplit(":", 1) 541 if len(out) == 1: 542 # No port 543 port = None 544 else: 545 try: 546 port = int(out[1]) 547 except ValueError: 548 return srv, None 549 550 return out[0], port 551 552 553def get_cache_limit(): 554 """Depending on OS, calculate cache limits. 555 In ArticleCache it will make sure we stay 556 within system limits for 32/64 bit 557 """ 558 # Calculate, if possible 559 try: 560 if sabnzbd.WIN32: 561 # Windows 562 mem_bytes = get_windows_memory() 563 elif sabnzbd.DARWIN: 564 # macOS 565 mem_bytes = get_darwin_memory() 566 else: 567 # Linux 568 mem_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") 569 570 # Use 1/4th of available memory 571 mem_bytes = mem_bytes / 4 572 573 # We don't want to set a value that's too high 574 if mem_bytes > from_units(DEF_ARTICLE_CACHE_MAX): 575 return DEF_ARTICLE_CACHE_MAX 576 577 # We make sure it's at least a valid value 578 if mem_bytes > from_units("32M"): 579 return to_units(mem_bytes) 580 except: 581 pass 582 583 # Always at least minimum on Windows/macOS 584 if sabnzbd.WIN32 and sabnzbd.DARWIN: 585 return DEF_ARTICLE_CACHE_DEFAULT 586 587 # If failed, leave empty for Linux so user needs to decide 588 return "" 589 590 591def get_windows_memory(): 592 """Use ctypes to extract available memory""" 593 594 class MEMORYSTATUSEX(ctypes.Structure): 595 _fields_ = [ 596 ("dwLength", ctypes.c_ulong), 597 ("dwMemoryLoad", ctypes.c_ulong), 598 ("ullTotalPhys", ctypes.c_ulonglong), 599 ("ullAvailPhys", ctypes.c_ulonglong), 600 ("ullTotalPageFile", ctypes.c_ulonglong), 601 ("ullAvailPageFile", ctypes.c_ulonglong), 602 ("ullTotalVirtual", ctypes.c_ulonglong), 603 ("ullAvailVirtual", ctypes.c_ulonglong), 604 ("sullAvailExtendedVirtual", ctypes.c_ulonglong), 605 ] 606 607 def __init__(self): 608 # have to initialize this to the size of MEMORYSTATUSEX 609 self.dwLength = ctypes.sizeof(self) 610 super(MEMORYSTATUSEX, self).__init__() 611 612 stat = MEMORYSTATUSEX() 613 ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) 614 return stat.ullTotalPhys 615 616 617def get_darwin_memory(): 618 """Use system-call to extract total memory on macOS""" 619 system_output = run_command(["sysctl", "hw.memsize"]) 620 return float(system_output.split()[1]) 621 622 623def on_cleanup_list(filename, skip_nzb=False): 624 """Return True if a filename matches the clean-up list""" 625 lst = cfg.cleanup_list() 626 if lst: 627 name, ext = os.path.splitext(filename) 628 ext = ext.strip().lower() 629 name = name.strip() 630 for k in lst: 631 item = k.strip().strip(".").lower() 632 item = "." + item 633 if (item == ext or (ext == "" and item == name)) and not (skip_nzb and item == ".nzb"): 634 return True 635 return False 636 637 638def memory_usage(): 639 try: 640 # Probably only works on Linux because it uses /proc/<pid>/statm 641 with open("/proc/%d/statm" % os.getpid()) as t: 642 v = t.read().split() 643 virt = int(_PAGE_SIZE * int(v[0]) / MEBI) 644 res = int(_PAGE_SIZE * int(v[1]) / MEBI) 645 return "V=%sM R=%sM" % (virt, res) 646 except IOError: 647 pass 648 except: 649 logging.debug("Error retrieving memory usage") 650 logging.info("Traceback: ", exc_info=True) 651 652 653try: 654 _PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") 655except: 656 _PAGE_SIZE = 0 657_HAVE_STATM = _PAGE_SIZE and memory_usage() 658 659 660def loadavg(): 661 """Return 1, 5 and 15 minute load average of host or "" if not supported""" 662 p = "" 663 if not sabnzbd.WIN32 and not sabnzbd.DARWIN: 664 opt = cfg.show_sysload() 665 if opt: 666 try: 667 p = "%.2f | %.2f | %.2f" % os.getloadavg() 668 except: 669 pass 670 if opt > 1 and _HAVE_STATM: 671 p = "%s | %s" % (p, memory_usage()) 672 return p 673 674 675def format_time_string(seconds): 676 """Return a formatted and translated time string""" 677 678 def unit(single, n): 679 # Seconds and minutes are special due to historical reasons 680 if single == "minute" or (single == "second" and n == 1): 681 single = single[:3] 682 if n == 1: 683 return T(single) 684 return T(single + "s") 685 686 # Format the string, size by size 687 seconds = int_conv(seconds) 688 completestr = [] 689 days = seconds // 86400 690 if days >= 1: 691 completestr.append("%s %s" % (days, unit("day", days))) 692 seconds -= days * 86400 693 hours = seconds // 3600 694 if hours >= 1: 695 completestr.append("%s %s" % (hours, unit("hour", hours))) 696 seconds -= hours * 3600 697 minutes = seconds // 60 698 if minutes >= 1: 699 completestr.append("%s %s" % (minutes, unit("minute", minutes))) 700 seconds -= minutes * 60 701 if seconds > 0: 702 completestr.append("%s %s" % (seconds, unit("second", seconds))) 703 704 # Zero or invalid integer 705 if not completestr: 706 completestr.append("0 %s" % unit("second", 0)) 707 708 return " ".join(completestr) 709 710 711def int_conv(value: Any) -> int: 712 """Safe conversion to int (can handle None)""" 713 try: 714 value = int(value) 715 except: 716 value = 0 717 return value 718 719 720def create_https_certificates(ssl_cert, ssl_key): 721 """Create self-signed HTTPS certificates and store in paths 'ssl_cert' and 'ssl_key'""" 722 try: 723 from sabnzbd.utils.certgen import generate_key, generate_local_cert 724 725 private_key = generate_key(key_size=2048, output_file=ssl_key) 726 generate_local_cert(private_key, days_valid=3560, output_file=ssl_cert, LN="SABnzbd", ON="SABnzbd") 727 logging.info("Self-signed certificates generated successfully") 728 except: 729 logging.error(T("Error creating SSL key and certificate")) 730 logging.info("Traceback: ", exc_info=True) 731 return False 732 733 return True 734 735 736def get_all_passwords(nzo): 737 """Get all passwords, from the NZB, meta and password file""" 738 if nzo.password: 739 logging.info("Found a password that was set by the user: %s", nzo.password) 740 passwords = [nzo.password.strip()] 741 else: 742 passwords = [] 743 744 meta_passwords = nzo.meta.get("password", []) 745 pw = nzo.nzo_info.get("password") 746 if pw: 747 meta_passwords.append(pw) 748 749 if meta_passwords: 750 passwords.extend(meta_passwords) 751 logging.info("Read %s passwords from meta data in NZB: %s", len(meta_passwords), meta_passwords) 752 753 pw_file = cfg.password_file.get_path() 754 if pw_file: 755 try: 756 with open(pw_file, "r") as pwf: 757 lines = pwf.read().split("\n") 758 # Remove empty lines and space-only passwords and remove surrounding spaces 759 pws = [pw.strip("\r\n ") for pw in lines if pw.strip("\r\n ")] 760 logging.debug("Read these passwords from file: %s", pws) 761 passwords.extend(pws) 762 logging.info("Read %s passwords from file %s", len(pws), pw_file) 763 764 # Check size 765 if len(pws) > 30: 766 logging.warning_helpful( 767 T( 768 "Your password file contains more than 30 passwords, testing all these passwords takes a lot of time. Try to only list useful passwords." 769 ) 770 ) 771 except: 772 logging.warning(T("Failed to read the password file %s"), pw_file) 773 logging.info("Traceback: ", exc_info=True) 774 775 if nzo.password: 776 # If an explicit password was set, add a retry without password, just in case. 777 passwords.append("") 778 elif not passwords or nzo.encrypted < 1: 779 # If we're not sure about encryption, start with empty password 780 # and make sure we have at least the empty password 781 passwords.insert(0, "") 782 return set(passwords) 783 784 785def find_on_path(targets): 786 """Search the PATH for a program and return full path""" 787 if sabnzbd.WIN32: 788 paths = os.getenv("PATH").split(";") 789 else: 790 paths = os.getenv("PATH").split(":") 791 792 if isinstance(targets, str): 793 targets = (targets,) 794 795 for path in paths: 796 for target in targets: 797 target_path = os.path.abspath(os.path.join(path, target)) 798 if os.path.isfile(target_path) and os.access(target_path, os.X_OK): 799 return target_path 800 return None 801 802 803def strip_ipv4_mapped_notation(ip: str) -> str: 804 """Convert an IP address in IPv4-mapped IPv6 notation (e.g. ::ffff:192.168.0.10) to its regular 805 IPv4 form. Any value of ip that doesn't use the relevant notation is returned unchanged. 806 807 CherryPy may report remote IP addresses in this notation. While the ipaddress module should be 808 able to handle that, the latter has issues with the is_private/is_loopback properties for these 809 addresses. See https://bugs.python.org/issue33433""" 810 try: 811 # Keep the original if ipv4_mapped is None 812 ip = ipaddress.ip_address(ip).ipv4_mapped or ip 813 except (AttributeError, ValueError): 814 pass 815 return str(ip) 816 817 818def ip_in_subnet(ip: str, subnet: str) -> bool: 819 """Determine whether ip is part of subnet. For the latter, the standard form with a prefix or 820 netmask (e.g. "192.168.1.0/24" or "10.42.0.0/255.255.0.0") is expected. Input in SABnzbd's old 821 cfg.local_ranges() settings style (e.g. "192.168.1."), intended for use with str.startswith(), 822 is also accepted and internally converted to address/prefix form.""" 823 if not ip or not subnet: 824 return False 825 826 try: 827 if subnet.find("/") < 0 and subnet.find("::") < 0: 828 # The subnet doesn't include a prefix or netmask, or represent a single (compressed) 829 # IPv6 address; try converting from the older local_ranges settings style. 830 831 # Take the IP version of the subnet into account 832 IP_LEN, IP_BITS, IP_SEP = (8, 16, ":") if subnet.find(":") >= 0 else (4, 8, ".") 833 834 subnet = subnet.rstrip(IP_SEP).split(IP_SEP) 835 prefix = IP_BITS * len(subnet) 836 # Append as many zeros as needed 837 subnet.extend(["0"] * (IP_LEN - len(subnet))) 838 # Store in address/prefix form 839 subnet = "%s/%s" % (IP_SEP.join(subnet), prefix) 840 841 ip = strip_ipv4_mapped_notation(ip) 842 return ipaddress.ip_address(ip) in ipaddress.ip_network(subnet, strict=True) 843 except Exception: 844 # Probably an invalid range 845 return False 846 847 848def is_ipv4_addr(ip: str) -> bool: 849 """Determine if the ip is an IPv4 address""" 850 try: 851 return ipaddress.ip_address(ip).version == 4 852 except ValueError: 853 return False 854 855 856def is_ipv6_addr(ip: str) -> bool: 857 """Determine if the ip is an IPv6 address; square brackets ([2001::1]) are OK""" 858 try: 859 return ipaddress.ip_address(ip.strip("[]")).version == 6 860 except (ValueError, AttributeError): 861 return False 862 863 864def is_loopback_addr(ip: str) -> bool: 865 """Determine if the ip is an IPv4 or IPv6 local loopback address""" 866 try: 867 if ip.find(".") < 0: 868 ip = ip.strip("[]") 869 ip = strip_ipv4_mapped_notation(ip) 870 return ipaddress.ip_address(ip).is_loopback 871 except (ValueError, AttributeError): 872 return False 873 874 875def is_localhost(value: str) -> bool: 876 """Determine if the input is some variety of 'localhost'""" 877 return (value == "localhost") or is_loopback_addr(value) 878 879 880def is_lan_addr(ip: str) -> bool: 881 """Determine if the ip is a local area network address""" 882 try: 883 ip = strip_ipv4_mapped_notation(ip) 884 return ( 885 # The ipaddress module considers these private, see https://bugs.python.org/issue38655 886 not ip in ("0.0.0.0", "255.255.255.255") 887 and not ip_in_subnet(ip, "::/128") # Also catch (partially) exploded forms of "::" 888 and ipaddress.ip_address(ip).is_private 889 and not is_loopback_addr(ip) 890 ) 891 except ValueError: 892 return False 893 894 895def ip_extract() -> List[str]: 896 """Return list of IP addresses of this system""" 897 ips = [] 898 program = find_on_path("ip") 899 if program: 900 program = [program, "a"] 901 else: 902 program = find_on_path("ifconfig") 903 if program: 904 program = [program] 905 906 if sabnzbd.WIN32 or not program: 907 try: 908 info = socket.getaddrinfo(socket.gethostname(), None) 909 except: 910 # Hostname does not resolve, use localhost 911 info = socket.getaddrinfo("localhost", None) 912 for item in info: 913 ips.append(item[4][0]) 914 else: 915 output = run_command(program) 916 for line in output.split("\n"): 917 m = RE_IP4.search(line) 918 if not (m and m.group(2)): 919 m = RE_IP6.search(line) 920 if m and m.group(2): 921 ips.append(m.group(2)) 922 return ips 923 924 925def get_server_addrinfo(host: str, port: int) -> socket.getaddrinfo: 926 """Return processed getaddrinfo()""" 927 try: 928 int(port) 929 except: 930 port = 119 931 opt = sabnzbd.cfg.ipv6_servers() 932 """ ... with the following meaning for 'opt': 933 Control the use of IPv6 Usenet server addresses. Meaning: 934 0 = don't use 935 1 = use when available and reachable (DEFAULT) 936 2 = force usage (when SABnzbd's detection fails) 937 """ 938 try: 939 # Standard IPV4 or IPV6 940 ips = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) 941 if opt == 2 or (opt == 1 and sabnzbd.EXTERNAL_IPV6) or (opt == 1 and sabnzbd.cfg.load_balancing() == 2): 942 # IPv6 forced by user, or IPv6 allowed and reachable, or IPv6 allowed and loadbalancing-with-IPv6 activated 943 # So return all IP addresses, no matter IPv4 or IPv6: 944 return ips 945 else: 946 # IPv6 unreachable or not allowed by user, so only return IPv4 address(es): 947 return [ip for ip in ips if ":" not in ip[4][0]] 948 except: 949 if opt == 2 or (opt == 1 and sabnzbd.EXTERNAL_IPV6) or (opt == 1 and sabnzbd.cfg.load_balancing() == 2): 950 try: 951 # Try IPV6 explicitly 952 return socket.getaddrinfo( 953 host, port, socket.AF_INET6, socket.SOCK_STREAM, socket.IPPROTO_IP, socket.AI_CANONNAME 954 ) 955 except: 956 # Nothing found! 957 pass 958 return [] 959 960 961def get_base_url(url: str) -> str: 962 """Return only the true root domain for the favicon, so api.oznzb.com -> oznzb.com 963 But also api.althub.co.za -> althub.co.za 964 """ 965 url_host = urllib.parse.urlparse(url).hostname 966 if url_host: 967 url_split = url_host.split(".") 968 # Exception for localhost and IPv6 addresses 969 if len(url_split) < 3: 970 return url_host 971 return ".".join(len(url_split[-2]) < 4 and url_split[-3:] or url_split[-2:]) 972 else: 973 return "" 974 975 976def match_str(text: AnyStr, matches: Tuple[AnyStr, ...]) -> Optional[AnyStr]: 977 """Return first matching element of list 'matches' in 'text', otherwise None""" 978 text = text.lower() 979 for match in matches: 980 if match.lower() in text: 981 return match 982 return None 983 984 985def nntp_to_msg(text: Union[List[AnyStr], str]) -> str: 986 """Format raw NNTP bytes data for display""" 987 if isinstance(text, list): 988 text = text[0] 989 990 # Only need to split if it was raw data 991 # Sometimes (failed login) we put our own texts 992 if not isinstance(text, bytes): 993 return text 994 else: 995 lines = text.split(b"\r\n") 996 return ubtou(lines[0]) 997 998 999def list2cmdline(lst: List[str]) -> str: 1000 """convert list to a cmd.exe-compatible command string""" 1001 nlst = [] 1002 for arg in lst: 1003 if not arg: 1004 nlst.append('""') 1005 else: 1006 nlst.append('"%s"' % arg) 1007 return " ".join(nlst) 1008 1009 1010def build_and_run_command(command: List[str], flatten_command=False, **kwargs): 1011 """Builds and then runs command with nessecary flags and optional 1012 IONice and Nice commands. Optional Popen arguments can be supplied. 1013 On Windows we need to run our own list2cmdline for Unrar. 1014 Returns the Popen-instance. 1015 """ 1016 # command[0] should be set, and thus not None 1017 if not command[0]: 1018 logging.error(T("[%s] The command in build_command is undefined."), caller_name()) 1019 raise IOError 1020 1021 if not sabnzbd.WIN32: 1022 if command[0].endswith(".py"): 1023 with open(command[0], "r") as script_file: 1024 if not userxbit(command[0]): 1025 # Inform user that Python scripts need x-bit and then stop 1026 logging.error(T('Python script "%s" does not have execute (+x) permission set'), command[0]) 1027 raise IOError 1028 elif script_file.read(2) != "#!": 1029 # No shebang (#!) defined, add default python 1030 command.insert(0, sys.executable if sys.executable else "python") 1031 1032 if sabnzbd.newsunpack.IONICE_COMMAND and cfg.ionice(): 1033 ionice = cfg.ionice().split() 1034 command = ionice + command 1035 command.insert(0, sabnzbd.newsunpack.IONICE_COMMAND) 1036 if sabnzbd.newsunpack.NICE_COMMAND and cfg.nice(): 1037 nice = cfg.nice().split() 1038 command = nice + command 1039 command.insert(0, sabnzbd.newsunpack.NICE_COMMAND) 1040 creationflags = 0 1041 startupinfo = None 1042 else: 1043 # For Windows we always need to add python interpreter 1044 if command[0].endswith(".py"): 1045 command.insert(0, "python.exe") 1046 if flatten_command: 1047 command = list2cmdline(command) 1048 # On some Windows platforms we need to supress a quick pop-up of the command window 1049 startupinfo = subprocess.STARTUPINFO() 1050 startupinfo.dwFlags = win32process.STARTF_USESHOWWINDOW 1051 startupinfo.wShowWindow = win32con.SW_HIDE 1052 creationflags = WIN_SCHED_PRIOS[cfg.win_process_prio()] 1053 1054 # Set the basic Popen arguments 1055 popen_kwargs = { 1056 "stdin": subprocess.PIPE, 1057 "stdout": subprocess.PIPE, 1058 "stderr": subprocess.STDOUT, 1059 "startupinfo": startupinfo, 1060 "creationflags": creationflags, 1061 } 1062 # Update with the supplied ones 1063 popen_kwargs.update(kwargs) 1064 1065 # Run the command 1066 logging.info("[%s] Running external command: %s", caller_name(), command) 1067 logging.debug("Popen arguments: %s", popen_kwargs) 1068 return subprocess.Popen(command, **popen_kwargs) 1069 1070 1071def run_command(cmd: List[str], **kwargs): 1072 """Run simple external command and return output as a string.""" 1073 with build_and_run_command(cmd, **kwargs) as p: 1074 txt = platform_btou(p.stdout.read()) 1075 p.wait() 1076 return txt 1077