1#!/usr/bin/python3 -OO
2# Copyright 2007-2021 The SABnzbd-Team <team@sabnzbd.org>
3#
4# This program is free software; you can redistribute it and/or
5# modify it under the terms of the GNU General Public License
6# as published by the Free Software Foundation; either version 2
7# of the License, or (at your option) any later version.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program; if not, write to the Free Software
16# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
17
18"""
19sabnzbd.newsunpack
20"""
21
22import os
23import sys
24import re
25import subprocess
26import logging
27import time
28import zlib
29import shutil
30import functools
31
32import sabnzbd
33from sabnzbd.encoding import platform_btou, correct_unknown_encoding, ubtou
34import sabnzbd.utils.rarfile as rarfile
35from sabnzbd.misc import (
36    format_time_string,
37    find_on_path,
38    int_conv,
39    get_all_passwords,
40    calc_age,
41    cmp,
42    run_command,
43    build_and_run_command,
44)
45from sabnzbd.filesystem import (
46    make_script_path,
47    real_path,
48    globber,
49    globber_full,
50    renamer,
51    clip_path,
52    long_path,
53    remove_file,
54    listdir_full,
55    setname_from_path,
56    get_ext,
57    get_filename,
58)
59from sabnzbd.nzbstuff import NzbObject, NzbFile
60from sabnzbd.sorting import SeriesSorter
61import sabnzbd.cfg as cfg
62from sabnzbd.constants import Status
63
64# Regex globals
65RAR_RE = re.compile(r"\.(?P<ext>part\d*\.rar|rar|r\d\d|s\d\d|t\d\d|u\d\d|v\d\d|\d\d\d?\d)$", re.I)
66RAR_RE_V3 = re.compile(r"\.(?P<ext>part\d*)$", re.I)
67
68LOADING_RE = re.compile(r'^Loading "(.+)"')
69TARGET_RE = re.compile(r'^(?:File|Target): "(.+)" -')
70EXTRACTFROM_RE = re.compile(r"^Extracting\sfrom\s(.+)")
71EXTRACTED_RE = re.compile(r"^(Extracting|Creating|...)\s+(.*?)\s+OK\s*$")
72SPLITFILE_RE = re.compile(r"\.(\d\d\d?\d$)", re.I)
73ZIP_RE = re.compile(r"\.(zip$)", re.I)
74SEVENZIP_RE = re.compile(r"\.7z$", re.I)
75SEVENMULTI_RE = re.compile(r"\.7z\.\d+$", re.I)
76TS_RE = re.compile(r"\.(\d+)\.(ts$)", re.I)
77
78PAR2_COMMAND = None
79MULTIPAR_COMMAND = None
80RAR_COMMAND = None
81NICE_COMMAND = None
82ZIP_COMMAND = None
83SEVEN_COMMAND = None
84IONICE_COMMAND = None
85RAR_PROBLEM = False
86PAR2_MT = True
87RAR_VERSION = 0
88
89
90def find_programs(curdir):
91    """Find external programs"""
92
93    def check(path, program):
94        p = os.path.abspath(os.path.join(path, program))
95        if os.access(p, os.X_OK):
96            return p
97        else:
98            return None
99
100    if sabnzbd.DARWIN:
101        sabnzbd.newsunpack.PAR2_COMMAND = check(curdir, "osx/par2/par2-sl64")
102        sabnzbd.newsunpack.RAR_COMMAND = check(curdir, "osx/unrar/unrar")
103        sabnzbd.newsunpack.SEVEN_COMMAND = check(curdir, "osx/7zip/7za")
104
105    if sabnzbd.WIN32:
106        if sabnzbd.WIN64:
107            # 64 bit versions
108            sabnzbd.newsunpack.MULTIPAR_COMMAND = check(curdir, "win/multipar/par2j64.exe")
109            sabnzbd.newsunpack.RAR_COMMAND = check(curdir, "win/unrar/x64/UnRAR.exe")
110        else:
111            # 32 bit versions
112            sabnzbd.newsunpack.MULTIPAR_COMMAND = check(curdir, "win/multipar/par2j.exe")
113            sabnzbd.newsunpack.RAR_COMMAND = check(curdir, "win/unrar/UnRAR.exe")
114        sabnzbd.newsunpack.SEVEN_COMMAND = check(curdir, "win/7zip/7za.exe")
115    else:
116        if not sabnzbd.newsunpack.PAR2_COMMAND:
117            sabnzbd.newsunpack.PAR2_COMMAND = find_on_path("par2")
118        if not sabnzbd.newsunpack.RAR_COMMAND:
119            sabnzbd.newsunpack.RAR_COMMAND = find_on_path(
120                (
121                    "unrar",
122                    "rar",
123                    "unrar3",
124                    "rar3",
125                )
126            )
127        sabnzbd.newsunpack.NICE_COMMAND = find_on_path("nice")
128        sabnzbd.newsunpack.IONICE_COMMAND = find_on_path("ionice")
129        if not sabnzbd.newsunpack.ZIP_COMMAND:
130            sabnzbd.newsunpack.ZIP_COMMAND = find_on_path("unzip")
131        if not sabnzbd.newsunpack.SEVEN_COMMAND:
132            sabnzbd.newsunpack.SEVEN_COMMAND = find_on_path("7za")
133        if not sabnzbd.newsunpack.SEVEN_COMMAND:
134            sabnzbd.newsunpack.SEVEN_COMMAND = find_on_path("7z")
135
136    if not (sabnzbd.WIN32 or sabnzbd.DARWIN):
137        # Run check on rar version
138        version, original = unrar_check(sabnzbd.newsunpack.RAR_COMMAND)
139        sabnzbd.newsunpack.RAR_PROBLEM = not original or version < sabnzbd.constants.REC_RAR_VERSION
140        sabnzbd.newsunpack.RAR_VERSION = version
141
142        # Run check on par2-multicore
143        sabnzbd.newsunpack.PAR2_MT = par2_mt_check(sabnzbd.newsunpack.PAR2_COMMAND)
144
145
146ENV_NZO_FIELDS = [
147    "bytes",
148    "bytes_downloaded",
149    "bytes_tried",
150    "cat",
151    "duplicate",
152    "encrypted",
153    "fail_msg",
154    "filename",
155    "final_name",
156    "group",
157    "nzo_id",
158    "oversized",
159    "password",
160    "pp",
161    "priority",
162    "repair",
163    "script",
164    "status",
165    "unpack",
166    "unwanted_ext",
167    "url",
168]
169
170
171def external_processing(extern_proc, nzo: NzbObject, complete_dir, nicename, status):
172    """Run a user postproc script, return console output and exit value"""
173    failure_url = nzo.nzo_info.get("failure", "")
174    # Items can be bool or null, causing POpen to fail
175    command = [
176        str(extern_proc),
177        str(complete_dir),
178        str(nzo.filename),
179        str(nicename),
180        "",
181        str(nzo.cat),
182        str(nzo.group),
183        str(status),
184        str(failure_url),
185    ]
186
187    # Add path to original NZB
188    nzb_paths = globber_full(nzo.admin_path, "*.gz")
189
190    # Fields not in the NZO directly
191    extra_env_fields = {
192        "failure_url": failure_url,
193        "complete_dir": complete_dir,
194        "pp_status": status,
195        "download_time": nzo.nzo_info.get("download_time", ""),
196        "avg_bps": int(nzo.avg_bps_total / nzo.avg_bps_freq) if nzo.avg_bps_freq else 0,
197        "age": calc_age(nzo.avg_date),
198        "orig_nzb_gz": clip_path(nzb_paths[0]) if nzb_paths else "",
199    }
200
201    try:
202        p = build_and_run_command(command, env=create_env(nzo, extra_env_fields))
203        sabnzbd.PostProcessor.external_process = p
204
205        # Follow the output, so we can abort it
206        proc = p.stdout
207        if p.stdin:
208            p.stdin.close()
209
210        lines = []
211        while 1:
212            line = platform_btou(proc.readline())
213            if not line:
214                break
215            line = line.strip()
216            lines.append(line)
217
218            # Show current line in history
219            nzo.set_action_line(T("Running script"), line)
220    except:
221        logging.debug("Failed script %s, Traceback: ", extern_proc, exc_info=True)
222        return "Cannot run script %s\r\n" % extern_proc, -1
223
224    output = "\n".join(lines)
225    ret = p.wait()
226    return output, ret
227
228
229def unpack_magic(
230    nzo: NzbObject, workdir, workdir_complete, dele, one_folder, joinables, zips, rars, sevens, ts, depth=0
231):
232    """Do a recursive unpack from all archives in 'workdir' to 'workdir_complete'"""
233    if depth > 5:
234        logging.warning(T("Unpack nesting too deep [%s]"), nzo.final_name)
235        return False, []
236    depth += 1
237
238    if depth == 1:
239        # First time, ignore anything in workdir_complete
240        xjoinables, xzips, xrars, xsevens, xts = build_filelists(workdir)
241    else:
242        xjoinables, xzips, xrars, xsevens, xts = build_filelists(workdir, workdir_complete, check_both=dele)
243
244    force_rerun = False
245    newfiles = []
246    error = None
247    new_joins = new_ts = None
248
249    if cfg.enable_filejoin():
250        new_joins = [jn for jn in xjoinables if jn not in joinables]
251        if new_joins:
252            logging.info("Filejoin starting on %s", workdir)
253            error, newf = file_join(nzo, workdir, workdir_complete, dele, new_joins)
254            if newf:
255                newfiles.extend(newf)
256            logging.info("Filejoin finished on %s", workdir)
257
258    if cfg.enable_unrar():
259        new_rars = [rar for rar in xrars if rar not in rars]
260        if new_rars:
261            logging.info("Unrar starting on %s", workdir)
262            error, newf = rar_unpack(nzo, workdir, workdir_complete, dele, one_folder, new_rars)
263            if newf:
264                newfiles.extend(newf)
265            logging.info("Unrar finished on %s", workdir)
266
267    if cfg.enable_7zip():
268        new_sevens = [seven for seven in xsevens if seven not in sevens]
269        if new_sevens:
270            logging.info("7za starting on %s", workdir)
271            error, newf = unseven(nzo, workdir, workdir_complete, dele, one_folder, new_sevens)
272            if newf:
273                newfiles.extend(newf)
274            logging.info("7za finished on %s", workdir)
275
276    if cfg.enable_unzip():
277        new_zips = [zip for zip in xzips if zip not in zips]
278        if new_zips:
279            logging.info("Unzip starting on %s", workdir)
280            if SEVEN_COMMAND:
281                error, newf = unseven(nzo, workdir, workdir_complete, dele, one_folder, new_zips)
282            else:
283                error, newf = unzip(nzo, workdir, workdir_complete, dele, one_folder, new_zips)
284            if newf:
285                newfiles.extend(newf)
286            logging.info("Unzip finished on %s", workdir)
287
288    if cfg.enable_tsjoin():
289        new_ts = [_ts for _ts in xts if _ts not in ts]
290        if new_ts:
291            logging.info("TS Joining starting on %s", workdir)
292            error, newf = file_join(nzo, workdir, workdir_complete, dele, new_ts)
293            if newf:
294                newfiles.extend(newf)
295            logging.info("TS Joining finished on %s", workdir)
296
297    # Refresh history and set output
298    nzo.set_action_line()
299
300    # Only re-run if something was unpacked and it was success
301    rerun = error in (False, 0)
302
303    # During a Retry we might miss files in the complete folder
304    # that failed during recursive unpack in the first run
305    if nzo.reuse and depth == 1 and any(build_filelists(workdir=None, workdir_complete=workdir_complete)):
306        rerun = True
307
308    # We can't recursive unpack on long paths on Windows
309    # See: https://github.com/sabnzbd/sabnzbd/pull/771
310    if sabnzbd.WIN32 and len(workdir_complete) > 256:
311        rerun = False
312
313    # Double-check that we didn't miss any files in workdir
314    # But only if dele=True, otherwise of course there will be files left
315    if rerun and dele and depth == 1 and any(build_filelists(workdir)):
316        force_rerun = True
317        # Clear lists to force re-scan of files
318        xjoinables, xzips, xrars, xsevens, xts = ([], [], [], [], [])
319
320    if rerun and (cfg.enable_recursive() or new_ts or new_joins or force_rerun):
321        z, y = unpack_magic(
322            nzo, workdir, workdir_complete, dele, one_folder, xjoinables, xzips, xrars, xsevens, xts, depth
323        )
324        if z:
325            error = z
326        if y:
327            newfiles.extend(y)
328
329    return error, newfiles
330
331
332##############################################################################
333# Filejoin Functions
334##############################################################################
335def match_ts(file):
336    """Return True if file is a joinable TS file"""
337    match = TS_RE.search(file)
338    if not match:
339        return False, "", 0
340
341    num = int(match.group(1))
342    try:
343        set = file[: match.start()]
344        set += ".ts"
345    except:
346        set = ""
347    return match, set, num
348
349
350def clean_up_joinables(names):
351    """Remove joinable files and their .1 backups"""
352    for name in names:
353        if os.path.exists(name):
354            try:
355                remove_file(name)
356            except:
357                pass
358        name1 = name + ".1"
359        if os.path.exists(name1):
360            try:
361                remove_file(name1)
362            except:
363                pass
364
365
366def get_seq_number(name):
367    """Return sequence number if name as an int"""
368    head, tail = os.path.splitext(name)
369    if tail == ".ts":
370        match, set, num = match_ts(name)
371    else:
372        num = tail[1:]
373    if num.isdigit():
374        return int(num)
375    else:
376        return 0
377
378
379def file_join(nzo: NzbObject, workdir, workdir_complete, delete, joinables):
380    """Join and joinable files in 'workdir' to 'workdir_complete' and
381    when successful, delete originals
382    """
383    newfiles = []
384    bufsize = 24 * 1024 * 1024
385
386    # Create matching sets from the list of files
387    joinable_sets = {}
388    joinable_set = None
389    for joinable in joinables:
390        head, tail = os.path.splitext(joinable)
391        if tail == ".ts":
392            head = match_ts(joinable)[1]
393        if head not in joinable_sets:
394            joinable_sets[head] = []
395        joinable_sets[head].append(joinable)
396    logging.debug("joinable_sets: %s", joinable_sets)
397
398    try:
399        # Handle each set
400        for joinable_set in joinable_sets:
401            current = joinable_sets[joinable_set]
402            joinable_sets[joinable_set].sort()
403
404            # If par2 already did the work, just remove the files
405            if os.path.exists(joinable_set):
406                logging.debug("file_join(): Skipping %s, (probably) joined by par2", joinable_set)
407                if delete:
408                    clean_up_joinables(current)
409                # done, go to next set
410                continue
411
412            # Only join when there is more than one file
413            size = len(current)
414            if size < 2:
415                continue
416
417            # Prepare joined file
418            filename = joinable_set
419            if workdir_complete:
420                filename = filename.replace(workdir, workdir_complete)
421            logging.debug("file_join(): Assembling %s", filename)
422
423            # Join the segments
424            with open(filename, "ab") as joined_file:
425                n = get_seq_number(current[0])
426                seq_error = n > 1
427                for joinable in current:
428                    if get_seq_number(joinable) != n:
429                        seq_error = True
430                    perc = (100.0 / size) * n
431                    logging.debug("Processing %s", joinable)
432                    nzo.set_action_line(T("Joining"), "%.0f%%" % perc)
433                    with open(joinable, "rb") as f:
434                        shutil.copyfileobj(f, joined_file, bufsize)
435                    if delete:
436                        remove_file(joinable)
437                    n += 1
438
439            # Remove any remaining .1 files
440            clean_up_joinables(current)
441
442            # Finish up
443            newfiles.append(filename)
444
445            setname = setname_from_path(joinable_set)
446            if seq_error:
447                msg = T("Incomplete sequence of joinable files")
448                nzo.fail_msg = T("File join of %s failed") % setname
449                nzo.set_unpack_info("Filejoin", T('[%s] Error "%s" while joining files') % (setname, msg))
450                logging.error(T('Error "%s" while running file_join on %s'), msg, nzo.final_name)
451                return True, []
452            else:
453                msg = T("[%s] Joined %s files") % (joinable_set, size)
454                nzo.set_unpack_info("Filejoin", msg, setname)
455    except:
456        msg = sys.exc_info()[1]
457        nzo.fail_msg = T("File join of %s failed") % msg
458        nzo.set_unpack_info(
459            "Filejoin", T('[%s] Error "%s" while joining files') % (setname_from_path(joinable_set), msg)
460        )
461        logging.error(T('Error "%s" while running file_join on %s'), msg, nzo.final_name)
462        return True, []
463
464    return False, newfiles
465
466
467##############################################################################
468# (Un)Rar Functions
469##############################################################################
470def rar_unpack(nzo: NzbObject, workdir, workdir_complete, delete, one_folder, rars):
471    """Unpack multiple sets 'rars' of RAR files from 'workdir' to 'workdir_complete.
472    When 'delete' is set, originals will be deleted.
473    When 'one_folder' is set, all files will be in a single folder
474    """
475    newfiles = extracted_files = []
476    rar_sets = {}
477    for rar in rars:
478        rar_set = setname_from_path(rar)
479        if RAR_RE_V3.search(rar_set):
480            # Remove the ".partXX" part
481            rar_set = os.path.splitext(rar_set)[0]
482        if rar_set not in rar_sets:
483            rar_sets[rar_set] = []
484        rar_sets[rar_set].append(rar)
485
486    logging.debug("Rar_sets: %s", rar_sets)
487
488    for rar_set in rar_sets:
489        # Run the RAR extractor
490        rar_sets[rar_set].sort(key=functools.cmp_to_key(rar_sort))
491
492        rarpath = rar_sets[rar_set][0]
493
494        if workdir_complete and rarpath.startswith(workdir):
495            extraction_path = workdir_complete
496        else:
497            extraction_path = os.path.split(rarpath)[0]
498
499        # Is the direct-unpacker still running? We wait for it
500        if nzo.direct_unpacker:
501            wait_count = 0
502            last_stats = nzo.direct_unpacker.get_formatted_stats()
503            while nzo.direct_unpacker.is_alive():
504                logging.debug("DirectUnpacker still alive for %s: %s", nzo.final_name, last_stats)
505
506                # Bump the file-lock in case it's stuck
507                with nzo.direct_unpacker.next_file_lock:
508                    nzo.direct_unpacker.next_file_lock.notify()
509                time.sleep(2)
510
511                # Did something change? Might be stuck
512                if last_stats == nzo.direct_unpacker.get_formatted_stats():
513                    wait_count += 1
514                    if wait_count > 60:
515                        # We abort after 2 minutes of no changes
516                        nzo.direct_unpacker.abort()
517                else:
518                    wait_count = 0
519                last_stats = nzo.direct_unpacker.get_formatted_stats()
520
521        # Did we already direct-unpack it? Not when recursive-unpacking
522        if nzo.direct_unpacker and rar_set in nzo.direct_unpacker.success_sets:
523            logging.info("Set %s completed by DirectUnpack", rar_set)
524            fail = False
525            success = True
526            rars, newfiles = nzo.direct_unpacker.success_sets.pop(rar_set)
527        else:
528            logging.info("Extracting rarfile %s (belonging to %s) to %s", rarpath, rar_set, extraction_path)
529            try:
530                fail, newfiles, rars = rar_extract(
531                    rarpath, len(rar_sets[rar_set]), one_folder, nzo, rar_set, extraction_path
532                )
533                success = not fail
534            except:
535                success = False
536                fail = True
537                msg = sys.exc_info()[1]
538                nzo.fail_msg = T("Unpacking failed, %s") % msg
539                setname = nzo.final_name
540                nzo.set_unpack_info("Unpack", T('[%s] Error "%s" while unpacking RAR files') % (setname, msg))
541
542                logging.error(T('Error "%s" while running rar_unpack on %s'), msg, setname)
543                logging.debug("Traceback: ", exc_info=True)
544
545        if success:
546            logging.debug("rar_unpack(): Rars: %s", rars)
547            logging.debug("rar_unpack(): Newfiles: %s", newfiles)
548            extracted_files.extend(newfiles)
549
550        # Do not fail if this was a recursive unpack
551        if fail and rarpath.startswith(workdir_complete):
552            # Do not delete the files, leave it to user!
553            logging.info("Ignoring failure to do recursive unpack of %s", rarpath)
554            fail = 0
555            success = True
556            newfiles = []
557
558        # Do not fail if this was maybe just some duplicate fileset
559        # Multipar and par2tbb will detect and log them, par2cmdline will not
560        if fail and rar_set.endswith((".1", ".2")):
561            # Just in case, we leave the raw files
562            logging.info("Ignoring failure of unpack for possible duplicate file %s", rarpath)
563            fail = 0
564            success = True
565            newfiles = []
566
567        # Delete the old files if we have to
568        if success and delete and newfiles:
569            for rar in rars:
570                try:
571                    remove_file(rar)
572                except OSError:
573                    if os.path.exists(rar):
574                        logging.warning(T("Deleting %s failed!"), rar)
575
576                brokenrar = "%s.1" % rar
577
578                if os.path.exists(brokenrar):
579                    logging.info("Deleting %s", brokenrar)
580                    try:
581                        remove_file(brokenrar)
582                    except OSError:
583                        if os.path.exists(brokenrar):
584                            logging.warning(T("Deleting %s failed!"), brokenrar)
585
586    return fail, extracted_files
587
588
589def rar_extract(rarfile_path, numrars, one_folder, nzo: NzbObject, setname, extraction_path):
590    """Unpack single rar set 'rarfile' to 'extraction_path',
591    with password tries
592    Return fail==0(ok)/fail==1(error)/fail==2(wrong password), new_files, rars
593    """
594    fail = 0
595    new_files = None
596    rars = []
597    passwords = get_all_passwords(nzo)
598
599    for password in passwords:
600        if password:
601            logging.debug('Trying unrar with password "%s"', password)
602            msg = T('Trying unrar with password "%s"') % password
603            nzo.fail_msg = msg
604            nzo.set_unpack_info("Unpack", msg, setname)
605        fail, new_files, rars = rar_extract_core(
606            rarfile_path, numrars, one_folder, nzo, setname, extraction_path, password
607        )
608        if fail != 2:
609            break
610
611    if fail == 2:
612        logging.error("%s (%s)", T("Unpacking failed, archive requires a password"), get_filename(rarfile_path))
613    return fail, new_files, rars
614
615
616def rar_extract_core(rarfile_path, numrars, one_folder, nzo: NzbObject, setname, extraction_path, password):
617    """Unpack single rar set 'rarfile_path' to 'extraction_path'
618    Return fail==0(ok)/fail==1(error)/fail==2(wrong password)/fail==3(crc-error), new_files, rars
619    """
620    start = time.time()
621
622    logging.debug("rar_extract(): Extractionpath: %s", extraction_path)
623
624    if password:
625        password_command = "-p%s" % password
626    else:
627        password_command = "-p-"
628
629    ############################################################################
630
631    if one_folder or cfg.flat_unpack():
632        action = "e"
633    else:
634        action = "x"
635    if cfg.overwrite_files():
636        overwrite = "-o+"  # Enable overwrite
637        rename = "-o+"  # Dummy
638    else:
639        overwrite = "-o-"  # Disable overwrite
640        rename = "-or"  # Auto renaming
641
642    if sabnzbd.WIN32:
643        # For Unrar to support long-path, we need to circumvent Python's list2cmdline
644        # See: https://github.com/sabnzbd/sabnzbd/issues/1043
645        # The -scf forces the output to be UTF8
646        command = [
647            "%s" % RAR_COMMAND,
648            action,
649            "-idp",
650            "-scf",
651            overwrite,
652            rename,
653            "-ai",
654            password_command,
655            rarfile_path,
656            "%s\\" % long_path(extraction_path),
657        ]
658
659    elif RAR_PROBLEM:
660        # Use only oldest options, specifically no "-or" or "-scf"
661        # Luckily platform_btou has a fallback for non-UTF-8
662        command = [
663            "%s" % RAR_COMMAND,
664            action,
665            "-idp",
666            overwrite,
667            password_command,
668            "%s" % rarfile_path,
669            "%s/" % extraction_path,
670        ]
671    else:
672        # Don't use "-ai" (not needed for non-Windows)
673        # The -scf forces the output to be UTF8
674        command = [
675            "%s" % RAR_COMMAND,
676            action,
677            "-idp",
678            "-scf",
679            overwrite,
680            rename,
681            password_command,
682            "%s" % rarfile_path,
683            "%s/" % extraction_path,
684        ]
685
686    if cfg.ignore_unrar_dates():
687        command.insert(3, "-tsm-")
688
689    # Get list of all the volumes part of this set
690    logging.debug("Analyzing rar file ... %s found", rarfile.is_rarfile(rarfile_path))
691    p = build_and_run_command(command, flatten_command=True)
692    sabnzbd.PostProcessor.external_process = p
693
694    proc = p.stdout
695    if p.stdin:
696        p.stdin.close()
697
698    nzo.set_action_line(T("Unpacking"), "00/%02d" % numrars)
699
700    # Loop over the output from rar!
701    curr = 0
702    extracted = []
703    rarfiles = []
704    fail = 0
705    inrecovery = False
706    lines = []
707
708    while 1:
709        line = platform_btou(proc.readline())
710        if not line:
711            break
712
713        line = line.strip()
714        lines.append(line)
715
716        if line.startswith("Extracting from"):
717            filename = re.search(EXTRACTFROM_RE, line).group(1)
718            if filename not in rarfiles:
719                rarfiles.append(filename)
720            curr += 1
721            nzo.set_action_line(T("Unpacking"), "%02d/%02d" % (curr, numrars))
722
723        elif line.find("recovery volumes found") > -1:
724            inrecovery = True  # and thus start ignoring "Cannot find volume" for a while
725            logging.debug("unrar recovery start: %s" % line)
726        elif line.startswith("Reconstruct"):
727            # end of reconstruction: 'Reconstructing... 100%' or 'Reconstructing... ' (both success), or 'Reconstruction impossible'
728            inrecovery = False
729            logging.debug("unrar recovery result: %s" % line)
730
731        elif line.startswith("Cannot find volume") and not inrecovery:
732            filename = os.path.basename(line[19:])
733            msg = T("Unpacking failed, unable to find %s") % filename
734            nzo.fail_msg = msg
735            nzo.set_unpack_info("Unpack", msg, setname)
736            logging.warning(T('ERROR: unable to find "%s"'), filename)
737            fail = 1
738
739        elif line.endswith("- CRC failed"):
740            msg = T("Unpacking failed, CRC error")
741            nzo.fail_msg = msg
742            nzo.set_unpack_info("Unpack", msg, setname)
743            logging.warning(T('ERROR: CRC failed in "%s"'), setname)
744            fail = 2  # Older unrar versions report a wrong password as a CRC error
745
746        elif line.startswith("File too large"):
747            msg = T("Unpacking failed, file too large for filesystem (FAT?)")
748            nzo.fail_msg = msg
749            nzo.set_unpack_info("Unpack", msg, setname)
750            # ERROR: File too large for file system (bigfile-5000MB)
751            logging.error(T("ERROR: File too large for filesystem (%s)"), setname)
752            fail = 1
753
754        elif line.startswith("Write error"):
755            msg = T("Unpacking failed, write error or disk is full?")
756            nzo.fail_msg = msg
757            nzo.set_unpack_info("Unpack", msg, setname)
758            logging.error(T("ERROR: write error (%s)"), line[11:])
759            fail = 1
760
761        elif line.startswith("Cannot create"):
762            line2 = platform_btou(proc.readline())
763            if "must not exceed 260" in line2:
764                msg = "%s: %s" % (T("Unpacking failed, path is too long"), line[13:])
765                nzo.fail_msg = msg
766                logging.error(T("ERROR: path too long (%s)"), line[13:])
767            else:
768                msg = "%s: %s" % (T("Unpacking failed, write error or disk is full?"), line[13:])
769                nzo.fail_msg = msg
770                logging.error(T("ERROR: write error (%s)"), line[13:])
771            nzo.set_unpack_info("Unpack", msg, setname)
772            fail = 1
773            # Kill the process (can stay in endless loop on Windows Server)
774            p.kill()
775
776        elif line.startswith("ERROR: "):
777            msg = T("ERROR: %s") % line[7:]
778            nzo.fail_msg = msg
779            logging.warning(msg)
780            nzo.set_unpack_info("Unpack", msg, setname)
781            fail = 1
782
783        elif (
784            "The specified password is incorrect" in line
785            or "Incorrect password" in line
786            or ("ncrypted file" in line and (("CRC failed" in line) or ("Checksum error" in line)))
787        ):
788            # unrar 3.x: "Encrypted file: CRC failed in oLKQfrcNVivzdzSG22a2xo7t001.part1.rar (password incorrect ?)"
789            # unrar 4.x: "CRC failed in the encrypted file oLKQfrcNVivzdzSG22a2xo7t001.part1.rar. Corrupt file or wrong password."
790            # unrar 5.x: "Checksum error in the encrypted file oLKQfrcNVivzdzSG22a2xo7t001.part1.rar. Corrupt file or wrong password."
791            # unrar 5.01: "The specified password is incorrect."
792            # unrar 5.80: "Incorrect password for oLKQfrcNVivzdzSG22a2xo7t001.part1.rar"
793            msg = T("Unpacking failed, archive requires a password")
794            nzo.fail_msg = msg
795            nzo.set_unpack_info("Unpack", msg, setname)
796            fail = 2
797
798        elif "is not RAR archive" in line:
799            # Unrecognizable RAR file
800            msg = T("Unusable RAR file")
801            nzo.fail_msg = msg
802            nzo.set_unpack_info("Unpack", msg, setname)
803            fail = 3
804
805        elif "checksum error" in line or "Unexpected end of archive" in line:
806            # Corrupt archive or passworded, we can't know
807            # packed data checksum error in volume FILE
808            msg = T("Corrupt RAR file")
809            nzo.fail_msg = msg
810            nzo.set_unpack_info("Unpack", msg, setname)
811            fail = 3
812
813        else:
814            m = re.search(EXTRACTED_RE, line)
815            if m:
816                # In case of flat-unpack, UnRar still prints the whole path (?!)
817                unpacked_file = m.group(2)
818                if cfg.flat_unpack():
819                    unpacked_file = os.path.basename(unpacked_file)
820                extracted.append(real_path(extraction_path, unpacked_file))
821
822        if fail:
823            if proc:
824                proc.close()
825            p.wait()
826            logging.debug("UNRAR output %s", "\n".join(lines))
827            return fail, (), ()
828
829    if proc:
830        proc.close()
831    p.wait()
832
833    # Which files did we use to extract this?
834    rarfiles = rar_volumelist(rarfile_path, password, rarfiles)
835
836    logging.debug("UNRAR output %s", "\n".join(lines))
837    nzo.fail_msg = ""
838    msg = T("Unpacked %s files/folders in %s") % (str(len(extracted)), format_time_string(time.time() - start))
839    nzo.set_unpack_info("Unpack", msg, setname)
840    logging.info("%s", msg)
841
842    return 0, extracted, rarfiles
843
844
845##############################################################################
846# (Un)Zip Functions
847##############################################################################
848def unzip(nzo: NzbObject, workdir, workdir_complete, delete, one_folder, zips):
849    """Unpack multiple sets 'zips' of ZIP files from 'workdir' to 'workdir_complete.
850    When 'delete' is ste, originals will be deleted.
851    """
852
853    try:
854        i = 0
855        unzip_failed = False
856        tms = time.time()
857
858        # For file-bookkeeping
859        orig_dir_content = listdir_full(workdir_complete)
860
861        for _zip in zips:
862            logging.info("Starting extract on zipfile: %s ", _zip)
863            nzo.set_action_line(T("Unpacking"), "%s" % setname_from_path(_zip))
864
865            if workdir_complete and _zip.startswith(workdir):
866                extraction_path = workdir_complete
867            else:
868                extraction_path = os.path.split(_zip)[0]
869
870            if ZIP_Extract(_zip, extraction_path, one_folder):
871                unzip_failed = True
872            else:
873                i += 1
874
875        msg = T("%s files in %s") % (str(i), format_time_string(time.time() - tms))
876        nzo.set_unpack_info("Unpack", msg)
877
878        # What's new?
879        new_files = list(set(orig_dir_content + listdir_full(workdir_complete)))
880
881        # Delete the old files if we have to
882        if delete and not unzip_failed:
883            i = 0
884
885            for _zip in zips:
886                try:
887                    remove_file(_zip)
888                    i += 1
889                except OSError:
890                    logging.warning(T("Deleting %s failed!"), _zip)
891
892                brokenzip = "%s.1" % _zip
893
894                if os.path.exists(brokenzip):
895                    try:
896                        remove_file(brokenzip)
897                        i += 1
898                    except OSError:
899                        logging.warning(T("Deleting %s failed!"), brokenzip)
900
901        return unzip_failed, new_files
902    except:
903        msg = sys.exc_info()[1]
904        nzo.fail_msg = T("Unpacking failed, %s") % msg
905        logging.error(T('Error "%s" while running unzip() on %s'), msg, nzo.final_name)
906        return True, []
907
908
909def ZIP_Extract(zipfile, extraction_path, one_folder):
910    """Unzip single zip set 'zipfile' to 'extraction_path'"""
911    command = ["%s" % ZIP_COMMAND, "-o", "-Pnone", "%s" % clip_path(zipfile), "-d%s" % extraction_path]
912
913    if one_folder or cfg.flat_unpack():
914        command.insert(3, "-j")  # Unpack without folders
915
916    p = build_and_run_command(command)
917    output = platform_btou(p.stdout.read())
918    ret = p.wait()
919    logging.debug("unzip output: \n%s", output)
920    return ret
921
922
923##############################################################################
924# 7Zip Functions
925##############################################################################
926def unseven(nzo: NzbObject, workdir, workdir_complete, delete, one_folder, sevens):
927    """Unpack multiple sets '7z' of 7Zip files from 'workdir' to 'workdir_complete.
928    When 'delete' is set, originals will be deleted.
929    """
930    i = 0
931    unseven_failed = False
932    new_files = []
933    tms = time.time()
934
935    # Find multi-volume sets, because 7zip will not provide actual set members
936    sets = {}
937    for seven in sevens:
938        name, ext = os.path.splitext(seven)
939        ext = ext.strip(".")
940        if not ext.isdigit():
941            name = seven
942            ext = None
943        if name not in sets:
944            sets[name] = []
945        if ext:
946            sets[name].append(ext)
947
948    # Unpack each set
949    for seven in sets:
950        extensions = sets[seven]
951        logging.info("Starting extract on 7zip set/file: %s ", seven)
952        nzo.set_action_line(T("Unpacking"), "%s" % setname_from_path(seven))
953
954        if workdir_complete and seven.startswith(workdir):
955            extraction_path = workdir_complete
956        else:
957            extraction_path = os.path.split(seven)[0]
958
959        res, new_files_set, msg = seven_extract(nzo, seven, extensions, extraction_path, one_folder, delete)
960        if res:
961            unseven_failed = True
962            nzo.set_unpack_info("Unpack", msg, setname_from_path(seven))
963        else:
964            i += 1
965        new_files.extend(new_files_set)
966
967    if not unseven_failed:
968        msg = T("%s files in %s") % (str(i), format_time_string(time.time() - tms))
969        nzo.set_unpack_info("Unpack", msg)
970
971    return unseven_failed, new_files
972
973
974def seven_extract(nzo: NzbObject, sevenset, extensions, extraction_path, one_folder, delete):
975    """Unpack single set 'sevenset' to 'extraction_path', with password tries
976    Return fail==0(ok)/fail==1(error)/fail==2(wrong password), new_files, sevens
977    """
978    # Before we start, make sure the 7z binary SEVEN_COMMAND is defined
979    if not SEVEN_COMMAND:
980        msg = T('No 7za binary found, cannot unpack "%s"') % os.path.basename(sevenset)
981        logging.error(msg)
982        return 1, [], msg
983
984    fail = 0
985    passwords = get_all_passwords(nzo)
986
987    for password in passwords:
988        if password:
989            msg = T('Trying 7zip with password "%s"') % password
990            logging.debug(msg)
991            nzo.fail_msg = msg
992            nzo.set_unpack_info("Unpack", msg, setname_from_path(sevenset))
993        fail, new_files, msg = seven_extract_core(sevenset, extensions, extraction_path, one_folder, delete, password)
994        if fail != 2:
995            break
996
997    nzo.fail_msg = ""
998    if fail == 2:
999        msg = "%s (%s)" % (T("Unpacking failed, archive requires a password"), os.path.basename(sevenset))
1000    if fail > 0:
1001        nzo.fail_msg = msg
1002        nzo.status = Status.FAILED
1003        logging.error(msg)
1004    return fail, new_files, msg
1005
1006
1007def seven_extract_core(sevenset, extensions, extraction_path, one_folder, delete, password):
1008    """Unpack single 7Z set 'sevenset' to 'extraction_path'
1009    Return fail==0(ok)/fail==1(error)/fail==2(wrong password), new_files, message
1010    """
1011    if one_folder:
1012        method = "e"  # Unpack without folders
1013    else:
1014        method = "x"  # Unpack with folders
1015    if sabnzbd.WIN32 or sabnzbd.DARWIN:
1016        case = "-ssc-"  # Case insensitive
1017    else:
1018        case = "-ssc"  # Case sensitive
1019    if cfg.overwrite_files():
1020        overwrite = "-aoa"
1021    else:
1022        overwrite = "-aou"
1023    if password:
1024        password = "-p%s" % password
1025    else:
1026        password = "-p"
1027
1028    if len(extensions) > 0:
1029        name = "%s.001" % sevenset
1030        parm = "-tsplit"
1031    else:
1032        name = sevenset
1033        parm = "-tzip" if sevenset.lower().endswith(".zip") else "-t7z"
1034
1035    if not os.path.exists(name):
1036        return 1, [], T('7ZIP set "%s" is incomplete, cannot unpack') % setname_from_path(sevenset)
1037
1038    # For file-bookkeeping
1039    orig_dir_content = listdir_full(extraction_path)
1040
1041    command = [SEVEN_COMMAND, method, "-y", overwrite, parm, case, password, "-o%s" % extraction_path, name]
1042    p = build_and_run_command(command)
1043    sabnzbd.PostProcessor.external_process = p
1044    output = platform_btou(p.stdout.read())
1045    logging.debug("7za output: %s", output)
1046
1047    ret = p.wait()
1048
1049    # Return-code for CRC and Password is the same
1050    if ret == 2 and "ERROR: CRC Failed" in output:
1051        # We can output a more general error
1052        ret = 1
1053        msg = T('ERROR: CRC failed in "%s"') % setname_from_path(sevenset)
1054    else:
1055        # Default message
1056        msg = T("Could not unpack %s") % setname_from_path(sevenset)
1057
1058    # What's new?
1059    new_files = list(set(orig_dir_content + listdir_full(extraction_path)))
1060
1061    if ret == 0 and delete:
1062        if extensions:
1063            for ext in extensions:
1064                path = "%s.%s" % (sevenset, ext)
1065                try:
1066                    remove_file(path)
1067                except:
1068                    logging.warning(T("Deleting %s failed!"), path)
1069        else:
1070            try:
1071                remove_file(sevenset)
1072            except:
1073                logging.warning(T("Deleting %s failed!"), sevenset)
1074
1075    # Always return an error message, even when return code is 0
1076    return ret, new_files, msg
1077
1078
1079##############################################################################
1080# PAR2 Functions
1081##############################################################################
1082def par2_repair(parfile_nzf: NzbFile, nzo: NzbObject, workdir, setname, single):
1083    """Try to repair a set, return readd or correctness"""
1084    # Check if file exists, otherwise see if another is done
1085    parfile_path = os.path.join(workdir, parfile_nzf.filename)
1086    if not os.path.exists(parfile_path) and nzo.extrapars[setname]:
1087        for new_par in nzo.extrapars[setname]:
1088            test_parfile = os.path.join(workdir, new_par.filename)
1089            if os.path.exists(test_parfile):
1090                parfile_nzf = new_par
1091                break
1092        else:
1093            # No file was found, we assume this set already finished
1094            return False, True
1095
1096    parfile = os.path.join(workdir, parfile_nzf.filename)
1097    old_dir_content = os.listdir(workdir)
1098    used_joinables = ()
1099    joinables = ()
1100    used_for_repair = ()
1101    result = readd = False
1102
1103    # Need to copy now, gets pop-ed during repair
1104    setpars = nzo.extrapars[setname][:]
1105
1106    # Start QuickCheck
1107    nzo.status = Status.QUICK_CHECK
1108    nzo.set_action_line(T("Repair"), T("Quick Checking"))
1109    qc_result = quick_check_set(setname, nzo)
1110    if qc_result:
1111        logging.info("Quick-check for %s is OK, skipping repair", setname)
1112        nzo.set_unpack_info("Repair", T("[%s] Quick Check OK") % setname)
1113        result = True
1114
1115    if not result and cfg.enable_all_par():
1116        # Download all par2 files that haven't been downloaded yet
1117        readd = False
1118        for extrapar in nzo.extrapars[setname][:]:
1119            # Make sure we only get new par2 files
1120            if extrapar not in nzo.finished_files and extrapar not in nzo.files:
1121                nzo.add_parfile(extrapar)
1122                readd = True
1123        if readd:
1124            return readd, result
1125
1126    if not result:
1127        nzo.status = Status.REPAIRING
1128        result = False
1129        readd = False
1130        try:
1131            nzo.set_action_line(T("Repair"), T("Starting Repair"))
1132            logging.info('Scanning "%s"', parfile)
1133
1134            joinables, zips, rars, sevens, ts = build_filelists(workdir, check_rar=False)
1135
1136            # Multipar on Windows, par2 on the other platforms
1137            if sabnzbd.WIN32:
1138                finished, readd, datafiles, used_joinables, used_for_repair = MultiPar_Verify(
1139                    parfile, nzo, setname, joinables, single=single
1140                )
1141            else:
1142                finished, readd, datafiles, used_joinables, used_for_repair = PAR_Verify(
1143                    parfile, nzo, setname, joinables, single=single
1144                )
1145
1146            if finished:
1147                result = True
1148                logging.info("Par verify finished ok on %s!", parfile)
1149
1150                # Remove this set so we don't try to check it again
1151                nzo.remove_parset(parfile_nzf.setname)
1152            else:
1153                logging.info("Par verify failed on %s!", parfile)
1154
1155                if not readd:
1156                    # Failed to repair -> remove this set
1157                    nzo.remove_parset(parfile_nzf.setname)
1158                return readd, False
1159        except:
1160            msg = sys.exc_info()[1]
1161            nzo.fail_msg = T("Repairing failed, %s") % msg
1162            logging.error(T("Error %s while running par2_repair on set %s"), msg, setname)
1163            logging.info("Traceback: ", exc_info=True)
1164            return readd, result
1165
1166    try:
1167        if cfg.enable_par_cleanup():
1168            deletables = []
1169            new_dir_content = os.listdir(workdir)
1170
1171            # Remove extra files created during repair and par2 base files
1172            for path in new_dir_content:
1173                if os.path.splitext(path)[1] == ".1" and path not in old_dir_content:
1174                    deletables.append(os.path.join(workdir, path))
1175            deletables.append(os.path.join(workdir, setname + ".par2"))
1176            deletables.append(os.path.join(workdir, setname + ".PAR2"))
1177            deletables.append(parfile)
1178
1179            # Add output of par2-repair to remove
1180            deletables.extend(used_joinables)
1181            deletables.extend([os.path.join(workdir, f) for f in used_for_repair])
1182
1183            # Delete pars of the set
1184            deletables.extend([os.path.join(workdir, nzf.filename) for nzf in setpars])
1185
1186            for filepath in deletables:
1187                if filepath in joinables:
1188                    joinables.remove(filepath)
1189                if os.path.exists(filepath):
1190                    try:
1191                        remove_file(filepath)
1192                    except OSError:
1193                        logging.warning(T("Deleting %s failed!"), filepath)
1194    except:
1195        msg = sys.exc_info()[1]
1196        nzo.fail_msg = T("Repairing failed, %s") % msg
1197        logging.error(T('Error "%s" while running par2_repair on set %s'), msg, setname, exc_info=True)
1198
1199    return readd, result
1200
1201
1202_RE_BLOCK_FOUND = re.compile(r'File: "([^"]+)" - found \d+ of \d+ data blocks from "([^"]+)"')
1203_RE_IS_MATCH_FOR = re.compile(r'File: "([^"]+)" - is a match for "([^"]+)"')
1204_RE_LOADING_PAR2 = re.compile(r'Loading "([^"]+)"\.')
1205_RE_LOADED_PAR2 = re.compile(r"Loaded (\d+) new packets")
1206
1207
1208def PAR_Verify(parfile, nzo: NzbObject, setname, joinables, single=False):
1209    """Run par2 on par-set"""
1210    used_joinables = []
1211    used_for_repair = []
1212    # set the current nzo status to "Verifying...". Used in History
1213    nzo.status = Status.VERIFYING
1214    start = time.time()
1215
1216    options = cfg.par_option().strip()
1217    command = [str(PAR2_COMMAND), "r", options, parfile]
1218
1219    # Append the wildcard for this set
1220    parfolder = os.path.split(parfile)[0]
1221    if single or len(globber(parfolder, setname + "*")) < 2:
1222        # Support bizarre naming conventions
1223        wildcard = "*"
1224    else:
1225        # Normal case, everything is named after set
1226        wildcard = setname + "*"
1227
1228    if sabnzbd.WIN32 or sabnzbd.DARWIN:
1229        command.append(os.path.join(parfolder, wildcard))
1230    else:
1231        # For Unix systems, remove folders, due to bug in some par2cmdline versions
1232        flist = [item for item in globber_full(parfolder, wildcard) if os.path.isfile(item)]
1233        command.extend(flist)
1234
1235    # We need to check for the bad par2cmdline that skips blocks
1236    # Or the one that complains about basepath
1237    # Only if we're not doing multicore
1238    if not sabnzbd.WIN32 and not sabnzbd.DARWIN:
1239        par2text = run_command([command[0], "-h"])
1240        if "No data skipping" in par2text:
1241            logging.info("Detected par2cmdline version that skips blocks, adding -N parameter")
1242            command.insert(2, "-N")
1243        if "Set the basepath" in par2text:
1244            logging.info("Detected par2cmdline version that needs basepath, adding -B<path> parameter")
1245            command.insert(2, "-B")
1246            command.insert(3, parfolder)
1247
1248    # par2multicore wants to see \\.\ paths on Windows
1249    # See: https://github.com/sabnzbd/sabnzbd/pull/771
1250    if sabnzbd.WIN32:
1251        command = [clip_path(x) if x.startswith("\\\\?\\") else x for x in command]
1252
1253    # Run the external command
1254    p = build_and_run_command(command)
1255    sabnzbd.PostProcessor.external_process = p
1256    proc = p.stdout
1257
1258    if p.stdin:
1259        p.stdin.close()
1260
1261    # Set up our variables
1262    lines = []
1263    datafiles = []
1264    renames = {}
1265    reconstructed = []
1266
1267    linebuf = ""
1268    finished = 0
1269    readd = False
1270
1271    verifynum = 1
1272    verifytotal = 0
1273    verified = 0
1274
1275    in_verify_repaired = False
1276
1277    # Loop over the output, whee
1278    while 1:
1279        char = platform_btou(proc.read(1))
1280        if not char:
1281            break
1282
1283        # Line not complete yet
1284        if char not in ("\n", "\r"):
1285            linebuf += char
1286            continue
1287
1288        line = linebuf.strip()
1289        linebuf = ""
1290
1291        # Skip empty lines
1292        if line == "":
1293            continue
1294
1295        if "Repairing:" not in line:
1296            lines.append(line)
1297
1298        if line.startswith(("Invalid option specified", "Invalid thread option", "Cannot specify recovery file count")):
1299            msg = T("[%s] PAR2 received incorrect options, check your Config->Switches settings") % setname
1300            nzo.set_unpack_info("Repair", msg)
1301            nzo.status = Status.FAILED
1302            logging.error(msg)
1303
1304        elif line.startswith("All files are correct"):
1305            msg = T("[%s] Verified in %s, all files correct") % (setname, format_time_string(time.time() - start))
1306            nzo.set_unpack_info("Repair", msg)
1307            logging.info("Verified in %s, all files correct", format_time_string(time.time() - start))
1308            finished = 1
1309
1310        elif line.startswith("Repair is required"):
1311            msg = T("[%s] Verified in %s, repair is required") % (setname, format_time_string(time.time() - start))
1312            nzo.set_unpack_info("Repair", msg)
1313            logging.info("Verified in %s, repair is required", format_time_string(time.time() - start))
1314            start = time.time()
1315            verified = 1
1316            # Reset to use them again for verification of repair
1317            verifytotal = 0
1318            verifynum = 0
1319
1320        elif line.startswith("Main packet not found") or "The recovery file does not exist" in line:
1321            # Initialparfile probably didn't decode properly or bad user parameters
1322            # We will try to get another par2 file, but 99% of time it's user parameters
1323            msg = T("Invalid par2 files or invalid PAR2 parameters, cannot verify or repair")
1324            logging.info(msg)
1325            logging.info("Extra pars = %s", nzo.extrapars[setname])
1326
1327            # Look for the smallest par2file
1328            block_table = {}
1329            for nzf in nzo.extrapars[setname]:
1330                if not nzf.completed:
1331                    block_table[nzf.blocks] = nzf
1332
1333            if block_table:
1334                nzf = block_table[min(block_table)]
1335                logging.info("Found new par2file %s", nzf.filename)
1336
1337                # Move from extrapar list to files to be downloaded
1338                # and remove it from the extrapars list
1339                nzo.add_parfile(nzf)
1340                readd = True
1341            else:
1342                nzo.fail_msg = msg
1343                nzo.set_unpack_info("Repair", msg, setname)
1344                nzo.status = Status.FAILED
1345
1346        elif line.startswith("You need"):
1347            # We need more blocks, but are they available?
1348            chunks = line.split()
1349            needed_blocks = int(chunks[2])
1350
1351            # Check if we have enough blocks
1352            added_blocks = nzo.get_extra_blocks(setname, needed_blocks)
1353            if added_blocks:
1354                msg = T("Fetching %s blocks...") % str(added_blocks)
1355                nzo.set_action_line(T("Fetching"), msg)
1356                readd = True
1357            else:
1358                # Failed
1359                msg = T("Repair failed, not enough repair blocks (%s short)") % str(needed_blocks)
1360                nzo.fail_msg = msg
1361                nzo.set_unpack_info("Repair", msg, setname)
1362                nzo.status = Status.FAILED
1363
1364        elif line.startswith("Repair is possible"):
1365            start = time.time()
1366            nzo.set_action_line(T("Repairing"), "%2d%%" % 0)
1367
1368        elif line.startswith("Repairing:"):
1369            chunks = line.split()
1370            per = float(chunks[-1][:-1])
1371            nzo.set_action_line(T("Repairing"), "%2d%%" % per)
1372            nzo.status = Status.REPAIRING
1373
1374        elif line.startswith("Repair complete"):
1375            msg = T("[%s] Repaired in %s") % (setname, format_time_string(time.time() - start))
1376            nzo.set_unpack_info("Repair", msg)
1377            logging.info("Repaired in %s", format_time_string(time.time() - start))
1378            finished = 1
1379
1380        elif verified and line.endswith(("are missing.", "exist but are damaged.")):
1381            # Files that will later be verified after repair
1382            chunks = line.split()
1383            verifytotal += int(chunks[0])
1384
1385        elif line.startswith("Verifying repaired files"):
1386            in_verify_repaired = True
1387            nzo.set_action_line(T("Verifying repair"), "%02d/%02d" % (verifynum, verifytotal))
1388
1389        elif in_verify_repaired and line.startswith("Target"):
1390            verifynum += 1
1391            if verifynum <= verifytotal:
1392                nzo.set_action_line(T("Verifying repair"), "%02d/%02d" % (verifynum, verifytotal))
1393
1394        elif line.startswith("File:") and line.find("data blocks from") > 0:
1395            m = _RE_BLOCK_FOUND.search(line)
1396            if m:
1397                workdir = os.path.split(parfile)[0]
1398                old_name = m.group(1)
1399                new_name = m.group(2)
1400                if joinables:
1401                    # Find out if a joinable file has been used for joining
1402                    for jn in joinables:
1403                        if line.find(os.path.split(jn)[1]) > 0:
1404                            used_joinables.append(jn)
1405                            break
1406                    # Special case of joined RAR files, the "of" and "from" must both be RAR files
1407                    # This prevents the joined rars files from being seen as an extra rar-set
1408                    if ".rar" in old_name.lower() and ".rar" in new_name.lower():
1409                        used_joinables.append(os.path.join(workdir, old_name))
1410                else:
1411                    logging.debug('PAR2 will reconstruct "%s" from "%s"', new_name, old_name)
1412                    reconstructed.append(os.path.join(workdir, old_name))
1413
1414        elif "Could not write" in line and "at offset 0:" in line:
1415            # If there are joinables, this error will only happen in case of 100% complete files
1416            # We can just skip the retry, because par2cmdline will fail in those cases
1417            # becauses it refuses to scan the ".001" file
1418            if joinables:
1419                finished = 1
1420                used_joinables = []
1421
1422        elif " cannot be renamed to " in line:
1423            msg = line.strip()
1424            nzo.fail_msg = msg
1425            nzo.set_unpack_info("Repair", msg, setname)
1426            nzo.status = Status.FAILED
1427
1428        elif "There is not enough space on the disk" in line:
1429            # Oops, disk is full!
1430            msg = T("Repairing failed, %s") % T("Disk full")
1431            nzo.fail_msg = msg
1432            nzo.set_unpack_info("Repair", msg, setname)
1433            nzo.status = Status.FAILED
1434
1435        # File: "oldname.rar" - is a match for "newname.rar".
1436        elif "is a match for" in line:
1437            m = _RE_IS_MATCH_FOR.search(line)
1438            if m:
1439                old_name = m.group(1)
1440                new_name = m.group(2)
1441                logging.debug('PAR2 will rename "%s" to "%s"', old_name, new_name)
1442                renames[new_name] = old_name
1443
1444                # Show progress
1445                if verifytotal == 0 or verifynum < verifytotal:
1446                    verifynum += 1
1447                    nzo.set_action_line(T("Verifying"), "%02d/%02d" % (verifynum, verifytotal))
1448
1449        elif "Scanning extra files" in line:
1450            # Obfuscated post most likely, so reset counter to show progress
1451            verifynum = 1
1452
1453        elif "No details available for recoverable file" in line:
1454            msg = line.strip()
1455            nzo.fail_msg = msg
1456            nzo.set_unpack_info("Repair", msg, setname)
1457            nzo.status = Status.FAILED
1458
1459        elif line.startswith("Repair Failed."):
1460            # Unknown repair problem
1461            msg = T("Repairing failed, %s") % line
1462            nzo.fail_msg = msg
1463            nzo.set_unpack_info("Repair", msg, setname)
1464            nzo.status = Status.FAILED
1465            finished = 0
1466
1467        elif not verified:
1468            if line.startswith("Verifying source files"):
1469                nzo.set_action_line(T("Verifying"), "01/%02d" % verifytotal)
1470                nzo.status = Status.VERIFYING
1471
1472            elif line.startswith("Scanning:"):
1473                pass
1474
1475            # Target files
1476            m = TARGET_RE.match(line)
1477            if m:
1478                nzo.status = Status.VERIFYING
1479                verifynum += 1
1480                if verifytotal == 0 or verifynum < verifytotal:
1481                    nzo.set_action_line(T("Verifying"), "%02d/%02d" % (verifynum, verifytotal))
1482                else:
1483                    nzo.set_action_line(T("Checking extra files"), "%02d" % verifynum)
1484
1485                # Remove redundant extra files that are just duplicates of original ones
1486                if "duplicate data blocks" in line:
1487                    used_for_repair.append(m.group(1))
1488                else:
1489                    datafiles.append(m.group(1))
1490                continue
1491
1492            # Verify done
1493            m = re.match(r"There are (\d+) recoverable files", line)
1494            if m:
1495                verifytotal = int(m.group(1))
1496
1497    p.wait()
1498
1499    # Also log what is shown to user in history
1500    if nzo.fail_msg:
1501        logging.info(nzo.fail_msg)
1502
1503    logging.debug("PAR2 output was\n%s", "\n".join(lines))
1504
1505    # If successful, add renamed files to the collection
1506    if finished and renames:
1507        nzo.renamed_file(renames)
1508
1509    # If successful and files were reconstructed, remove incomplete original files
1510    if finished and reconstructed:
1511        # Use 'used_joinables' as a vehicle to get rid of the files
1512        used_joinables.extend(reconstructed)
1513
1514    return finished, readd, datafiles, used_joinables, used_for_repair
1515
1516
1517_RE_FILENAME = re.compile(r'"([^"]+)"')
1518
1519
1520def MultiPar_Verify(parfile, nzo: NzbObject, setname, joinables, single=False):
1521    """Run par2 on par-set"""
1522    parfolder = os.path.split(parfile)[0]
1523    used_joinables = []
1524    used_for_repair = []
1525
1526    # set the current nzo status to "Verifying...". Used in History
1527    nzo.status = Status.VERIFYING
1528    start = time.time()
1529
1530    # Caching of verification implemented by adding -vs/-vd
1531    # Force output of utf-8 by adding -uo
1532    # But not really required due to prospective-par2
1533    command = [str(MULTIPAR_COMMAND), "r", "-uo", "-vs2", "-vd%s" % parfolder, parfile]
1534
1535    # Check if there are maybe par2cmdline/par2tbb commands supplied
1536    if "-t" in cfg.par_option() or "-p" in cfg.par_option():
1537        logging.info("Removing old par2cmdline/par2tbb options for MultiPar")
1538        cfg.par_option.set("")
1539
1540    # Only add user-options if supplied
1541    options = cfg.par_option().strip()
1542    if options:
1543        # We wrongly instructed users to use /x parameter style instead of -x
1544        options = options.replace("/", "-", 1)
1545        command.insert(2, options)
1546
1547    # Append the wildcard for this set
1548    if single or len(globber(parfolder, setname + "*")) < 2:
1549        # Support bizarre naming conventions
1550        wildcard = "*"
1551    else:
1552        # Normal case, everything is named after set
1553        wildcard = setname + "*"
1554    command.append(os.path.join(parfolder, wildcard))
1555
1556    # Run MultiPar
1557    p = build_and_run_command(command)
1558    sabnzbd.PostProcessor.external_process = p
1559    proc = p.stdout
1560    if p.stdin:
1561        p.stdin.close()
1562
1563    # Set up our variables
1564    lines = []
1565    datafiles = []
1566    renames = {}
1567    reconstructed = []
1568
1569    linebuf = b""
1570    finished = 0
1571    readd = False
1572
1573    verifynum = 0
1574    verifytotal = 0
1575
1576    in_check = False
1577    in_verify = False
1578    in_repair = False
1579    in_verify_repaired = False
1580    misnamed_files = False
1581    old_name = None
1582
1583    # Loop over the output, whee
1584    while 1:
1585        char = proc.read(1)
1586        if not char:
1587            break
1588
1589        # Line not complete yet
1590        if char not in (b"\n", b"\r"):
1591            linebuf += char
1592            continue
1593
1594        line = ubtou(linebuf).strip()
1595        linebuf = b""
1596
1597        # Skip empty lines
1598        if line == "":
1599            continue
1600
1601        # Save it all
1602        lines.append(line)
1603
1604        # ----------------- Startup
1605        if line.startswith("invalid option"):
1606            # Option error
1607            msg = T("[%s] PAR2 received incorrect options, check your Config->Switches settings") % setname
1608            nzo.set_unpack_info("Repair", msg)
1609            nzo.status = Status.FAILED
1610            logging.error(msg)
1611
1612        elif line.startswith("valid file is not found"):
1613            # Initialparfile probably didn't decode properly, or bad user parameters
1614            # We will try to get another par2 file, but 99% of time it's user parameters
1615            msg = T("Invalid par2 files or invalid PAR2 parameters, cannot verify or repair")
1616            logging.info(msg)
1617            logging.info("Extra pars = %s", nzo.extrapars[setname])
1618
1619            # Look for the smallest par2file
1620            block_table = {}
1621            for nzf in nzo.extrapars[setname]:
1622                if not nzf.completed:
1623                    block_table[nzf.blocks] = nzf
1624
1625            if block_table:
1626                nzf = block_table[min(block_table)]
1627                logging.info("Found new par2file %s", nzf.filename)
1628
1629                # Move from extrapar list to files to be downloaded
1630                # and remove it from the extrapars list
1631                nzo.add_parfile(nzf)
1632                readd = True
1633            else:
1634                nzo.fail_msg = msg
1635                nzo.set_unpack_info("Repair", msg, setname)
1636                nzo.status = Status.FAILED
1637
1638        elif line.startswith("There is not enough space on the disk"):
1639            msg = T("Repairing failed, %s") % T("Disk full")
1640            nzo.fail_msg = msg
1641            nzo.set_unpack_info("Repair", msg, setname)
1642            nzo.status = Status.FAILED
1643
1644        # ----------------- Start check/verify stage
1645        elif line.startswith("Recovery Set ID"):
1646            # Remove files were MultiPar stores verification result when repaired succesfull
1647            recovery_id = line.split()[-1]
1648            used_for_repair.append("2_%s.bin" % recovery_id)
1649            used_for_repair.append("2_%s.ini" % recovery_id)
1650
1651        elif line.startswith("Input File total count"):
1652            # How many files will it try to find?
1653            verifytotal = int(line.split()[-1])
1654
1655        # ----------------- Misnamed-detection stage
1656        # Misnamed files
1657        elif line.startswith("Searching misnamed file"):
1658            # We are in the misnamed files block
1659            misnamed_files = True
1660            verifynum = 0
1661        elif misnamed_files and "Found" in line:
1662            # First it reports the current filename
1663            m = _RE_FILENAME.search(line)
1664            if m:
1665                verifynum += 1
1666                nzo.set_action_line(T("Checking"), "%02d/%02d" % (verifynum, verifytotal))
1667                old_name = m.group(1)
1668        elif misnamed_files and "Misnamed" in line:
1669            # Then it finds the actual
1670            m = _RE_FILENAME.search(line)
1671            if m and old_name:
1672                new_name = m.group(1)
1673                logging.debug('MultiPar will rename "%s" to "%s"', old_name, new_name)
1674                renames[new_name] = old_name
1675                # New name is also part of data!
1676                datafiles.append(new_name)
1677                reconstructed.append(old_name)
1678
1679        # ----------------- Checking stage
1680        # Checking input files
1681        elif line.startswith("Complete file count"):
1682            in_check = False
1683            verifynum = 0
1684            old_name = None
1685        elif line.startswith("Verifying Input File"):
1686            in_check = True
1687            nzo.status = Status.VERIFYING
1688        elif in_check:
1689            m = _RE_FILENAME.search(line)
1690            if m:
1691                # Only increase counter if it was really the detection line
1692                if line.startswith("= ") or "%" not in line:
1693                    verifynum += 1
1694                nzo.set_action_line(T("Checking"), "%02d/%02d" % (verifynum, verifytotal))
1695                old_name = m.group(1)
1696
1697        # ----------------- Verify stage
1698        # Which files need extra verification?
1699        elif line.startswith("Damaged file count"):
1700            verifytotal = int(line.split()[-1])
1701
1702        elif line.startswith("Missing file count"):
1703            verifytotal += int(line.split()[-1])
1704
1705        # Actual verification
1706        elif line.startswith("Input File Slice found"):
1707            # End of verification AND end of misnamed file search
1708            in_verify = False
1709            misnamed_files = False
1710            old_name = None
1711        elif line.startswith("Finding available slice"):
1712            # The actual scanning of the files
1713            in_verify = True
1714            nzo.set_action_line(T("Verifying"), T("Checking"))
1715        elif in_verify:
1716            m = _RE_FILENAME.search(line)
1717            if m:
1718                # It prints the filename couple of times, so we save it to check
1719                # 'datafiles' will not contain all data-files in par-set, only the
1720                # ones that got scanned, but it's ouput is never used!
1721                nzo.status = Status.VERIFYING
1722                if line.split()[1] in ("Damaged", "Found"):
1723                    verifynum += 1
1724                    datafiles.append(m.group(1))
1725
1726                    # Set old_name in case it was misnamed and found (not when we are joining)
1727                    old_name = None
1728                    if line.split()[1] == "Found" and not joinables:
1729                        old_name = m.group(1)
1730
1731                    # Sometimes we don't know the total (filejoin)
1732                    if verifytotal <= 1:
1733                        nzo.set_action_line(T("Verifying"), "%02d" % verifynum)
1734                    else:
1735                        nzo.set_action_line(T("Verifying"), "%02d/%02d" % (verifynum, verifytotal))
1736
1737                elif old_name and old_name != m.group(1):
1738                    # Hey we found another misnamed one!
1739                    new_name = m.group(1)
1740                    logging.debug('MultiPar will rename "%s" to "%s"', old_name, new_name)
1741                    renames[new_name] = old_name
1742                    # Put it back with it's new name!
1743                    datafiles.pop()
1744                    datafiles.append(new_name)
1745                    # Need to remove the old file after repair (Multipar keeps it)
1746                    used_for_repair.append(old_name)
1747                    # Need to reset it to avoid collision
1748                    old_name = None
1749
1750                else:
1751                    # It's scanning extra files that don't belong to the set
1752                    # For damaged files it reports the filename twice, so only then start
1753                    verifynum += 1
1754                    if verifynum / 2 > verifytotal:
1755                        nzo.set_action_line(T("Checking extra files"), "%02d" % verifynum)
1756
1757                if joinables:
1758                    # Find out if a joinable file has been used for joining
1759                    for jn in joinables:
1760                        if line.find(os.path.split(jn)[1]) > 0:
1761                            used_joinables.append(jn)
1762                            datafiles.append(m.group(1))
1763                            break
1764
1765        elif line.startswith("Need"):
1766            # We need more blocks, but are they available?
1767            chunks = line.split()
1768            needed_blocks = int(chunks[1])
1769
1770            # Check if we have enough blocks
1771            added_blocks = nzo.get_extra_blocks(setname, needed_blocks)
1772            if added_blocks:
1773                msg = T("Fetching %s blocks...") % str(added_blocks)
1774                nzo.set_action_line(T("Fetching"), msg)
1775                readd = True
1776            else:
1777                # Failed
1778                msg = T("Repair failed, not enough repair blocks (%s short)") % str(needed_blocks)
1779                nzo.fail_msg = msg
1780                nzo.set_unpack_info("Repair", msg, setname)
1781                nzo.status = Status.FAILED
1782
1783            # MultiPar can say 'PAR File(s) Incomplete' also when it needs more blocks
1784            # But the Need-more-blocks message is always last, so force failure
1785            finished = 0
1786
1787        # Result of verification
1788        elif line.startswith("All Files Complete") or line.endswith("PAR File(s) Incomplete"):
1789            # Completed without damage!
1790            # 'PAR File(s) Incomplete' is reported for success
1791            # but when there are very similar filenames in the folder
1792            msg = T("[%s] Verified in %s, all files correct") % (setname, format_time_string(time.time() - start))
1793            nzo.set_unpack_info("Repair", msg)
1794            logging.info("Verified in %s, all files correct", format_time_string(time.time() - start))
1795            finished = 1
1796
1797        elif line.startswith(("Ready to repair", "Ready to rejoin")):
1798            # Ready to repair!
1799            # Or we are re-joining a split file when there's no damage but takes time
1800            msg = T("[%s] Verified in %s, repair is required") % (setname, format_time_string(time.time() - start))
1801            nzo.set_unpack_info("Repair", msg)
1802            logging.info("Verified in %s, repair is required", format_time_string(time.time() - start))
1803            start = time.time()
1804
1805            # Set message for user in case of joining
1806            if line.startswith("Ready to rejoin"):
1807                nzo.set_action_line(T("Joining"), "%2d" % len(used_joinables))
1808            else:
1809                # If we are repairing a joinable set, it won't actually
1810                # do the joining. So we can't remove those files!
1811                used_joinables = []
1812
1813        # ----------------- Repair stage
1814        elif "Recovering slice" in line:
1815            # Before this it will calculate matrix, here is where it starts
1816            start = time.time()
1817            in_repair = True
1818            nzo.set_action_line(T("Repairing"), "%2d%%" % 0)
1819
1820        elif in_repair and line.startswith("Verifying repair"):
1821            in_repair = False
1822            in_verify_repaired = True
1823            # How many will be checked?
1824            verifytotal = int(line.split()[-1])
1825            verifynum = 0
1826
1827        elif in_repair:
1828            try:
1829                # Line with percentage of repair (nothing else)
1830                per = float(line[:-1])
1831                nzo.set_action_line(T("Repairing"), "%2d%%" % per)
1832                nzo.status = Status.REPAIRING
1833            except:
1834                # Checksum error
1835                if "checksum" in line:
1836                    # Failed due to checksum error of multipar
1837                    msg = T("Repairing failed, %s") % line
1838                    nzo.fail_msg = msg
1839                    nzo.set_unpack_info("Repair", msg, setname)
1840                    nzo.status = Status.FAILED
1841                else:
1842                    # Not sure, log error
1843                    logging.info("Traceback: ", exc_info=True)
1844
1845        elif line.startswith("Repaired successfully"):
1846            msg = T("[%s] Repaired in %s") % (setname, format_time_string(time.time() - start))
1847            nzo.set_unpack_info("Repair", msg)
1848            logging.info("Repaired in %s", format_time_string(time.time() - start))
1849            finished = 1
1850
1851        elif in_verify_repaired and line.startswith("Repaired :"):
1852            # Track verification of repaired files (can sometimes take a while)
1853            verifynum += 1
1854            nzo.set_action_line(T("Verifying repair"), "%02d/%02d" % (verifynum, verifytotal))
1855
1856        elif line.startswith("Failed to repair") and not readd:
1857            # Unknown repair problem
1858            msg = T("Repairing failed, %s") % line
1859            nzo.fail_msg = msg
1860            nzo.set_unpack_info("Repair", msg, setname)
1861            nzo.status = Status.FAILED
1862            finished = 0
1863
1864    p.wait()
1865
1866    # Also log what is shown to user in history
1867    if nzo.fail_msg:
1868        logging.info(nzo.fail_msg)
1869
1870    logging.debug("MultiPar output was\n%s", "\n".join(lines))
1871
1872    # Add renamed files to the collection
1873    # MultiPar always(!!) renames automatically whatever it can in the 'Searching misnamed file:'-section
1874    # Even if the repair did not complete fully it will rename those!
1875    # But the ones in 'Finding available slices'-section will only be renamed after succesfull repair
1876    if renames:
1877        # If succes, we also remove the possibly previously renamed ones
1878        if finished:
1879            reconstructed.extend(list(renames.values()))
1880
1881        # Adding to the collection
1882        nzo.renamed_file(renames)
1883
1884        # Remove renamed original files
1885        workdir = os.path.split(parfile)[0]
1886        used_joinables.extend([os.path.join(workdir, name) for name in reconstructed])
1887
1888    return finished, readd, datafiles, used_joinables, used_for_repair
1889
1890
1891def create_env(nzo=None, extra_env_fields={}):
1892    """Modify the environment for pp-scripts with extra information
1893    macOS: Return copy of environment without PYTHONPATH and PYTHONHOME
1894    other: return None
1895    """
1896    env = os.environ.copy()
1897
1898    # Are we adding things?
1899    if nzo:
1900        # Add basic info
1901        for field in ENV_NZO_FIELDS:
1902            try:
1903                field_value = getattr(nzo, field)
1904                # Special filters for Python types
1905                if field_value is None:
1906                    env["SAB_" + field.upper()] = ""
1907                elif isinstance(field_value, bool):
1908                    env["SAB_" + field.upper()] = str(field_value * 1)
1909                else:
1910                    env["SAB_" + field.upper()] = str(field_value)
1911            except:
1912                # Catch key errors
1913                pass
1914
1915    # Always supply basic info
1916    extra_env_fields.update(
1917        {
1918            "program_dir": sabnzbd.DIR_PROG,
1919            "par2_command": sabnzbd.newsunpack.PAR2_COMMAND,
1920            "multipar_command": sabnzbd.newsunpack.MULTIPAR_COMMAND,
1921            "rar_command": sabnzbd.newsunpack.RAR_COMMAND,
1922            "zip_command": sabnzbd.newsunpack.ZIP_COMMAND,
1923            "7zip_command": sabnzbd.newsunpack.SEVEN_COMMAND,
1924            "version": sabnzbd.__version__,
1925        }
1926    )
1927
1928    # Add extra fields
1929    for field in extra_env_fields:
1930        try:
1931            if extra_env_fields[field] is not None:
1932                env["SAB_" + field.upper()] = str(extra_env_fields[field])
1933            else:
1934                env["SAB_" + field.upper()] = ""
1935        except:
1936            # Catch key errors
1937            pass
1938
1939    if sabnzbd.DARWIN:
1940        if "PYTHONPATH" in env:
1941            del env["PYTHONPATH"]
1942        if "PYTHONHOME" in env:
1943            del env["PYTHONHOME"]
1944    elif not nzo:
1945        # No modification
1946        return None
1947
1948    return env
1949
1950
1951def rar_volumelist(rarfile_path, password, known_volumes):
1952    """List volumes that are part of this rarset
1953    and merge them with parsed paths list, removing duplicates.
1954    We assume RarFile is right and use parsed paths as backup.
1955    """
1956    # UnRar is required to read some RAR files
1957    # RarFile can fail in special cases
1958    try:
1959        rarfile.UNRAR_TOOL = RAR_COMMAND
1960        zf = rarfile.RarFile(rarfile_path)
1961
1962        # setpassword can fail due to bugs in RarFile
1963        if password:
1964            try:
1965                zf.setpassword(password)
1966            except:
1967                pass
1968        zf_volumes = zf.volumelist()
1969    except:
1970        zf_volumes = []
1971
1972    # Remove duplicates
1973    zf_volumes_base = [os.path.basename(vol) for vol in zf_volumes]
1974    for known_volume in known_volumes:
1975        if os.path.basename(known_volume) not in zf_volumes_base:
1976            # Long-path notation just to be sure
1977            zf_volumes.append(long_path(known_volume))
1978    return zf_volumes
1979
1980
1981# Sort the various RAR filename formats properly :\
1982def rar_sort(a, b):
1983    """Define sort method for rar file names"""
1984    aext = a.split(".")[-1]
1985    bext = b.split(".")[-1]
1986
1987    if aext == "rar" and bext == "rar":
1988        return cmp(a, b)
1989    elif aext == "rar":
1990        return -1
1991    elif bext == "rar":
1992        return 1
1993    else:
1994        return cmp(a, b)
1995
1996
1997def build_filelists(workdir, workdir_complete=None, check_both=False, check_rar=True):
1998    """Build filelists, if workdir_complete has files, ignore workdir.
1999    Optionally scan both directories.
2000    Optionally test content to establish RAR-ness
2001    """
2002    sevens, joinables, zips, rars, ts, filelist = ([], [], [], [], [], [])
2003
2004    if workdir_complete:
2005        filelist.extend(listdir_full(workdir_complete))
2006
2007    if workdir and (not filelist or check_both):
2008        filelist.extend(listdir_full(workdir, recursive=False))
2009
2010    for file in filelist:
2011        # Extra check for rar (takes CPU/disk)
2012        file_is_rar = False
2013        if check_rar:
2014            file_is_rar = rarfile.is_rarfile(file)
2015
2016        # Run through all the checks
2017        if SEVENZIP_RE.search(file) or SEVENMULTI_RE.search(file):
2018            # 7zip
2019            sevens.append(file)
2020        elif SPLITFILE_RE.search(file) and not file_is_rar:
2021            # Joinables, optional with RAR check
2022            joinables.append(file)
2023        elif ZIP_RE.search(file):
2024            # ZIP files
2025            zips.append(file)
2026        elif RAR_RE.search(file):
2027            # RAR files
2028            rars.append(file)
2029        elif TS_RE.search(file):
2030            # TS split files
2031            ts.append(file)
2032
2033    logging.debug("build_filelists(): joinables: %s", joinables)
2034    logging.debug("build_filelists(): zips: %s", zips)
2035    logging.debug("build_filelists(): rars: %s", rars)
2036    logging.debug("build_filelists(): 7zips: %s", sevens)
2037    logging.debug("build_filelists(): ts: %s", ts)
2038
2039    return joinables, zips, rars, sevens, ts
2040
2041
2042def quick_check_set(set, nzo):
2043    """Check all on-the-fly md5sums of a set"""
2044    md5pack = nzo.md5packs.get(set)
2045    if md5pack is None:
2046        return False
2047
2048    # We use bitwise assigment (&=) so False always wins in case of failure
2049    # This way the renames always get saved!
2050    result = True
2051    nzf_list = nzo.finished_files
2052    renames = {}
2053
2054    # Files to ignore
2055    ignore_ext = cfg.quick_check_ext_ignore()
2056
2057    for file in md5pack:
2058        found = False
2059        file_to_ignore = get_ext(file).replace(".", "") in ignore_ext
2060        for nzf in nzf_list:
2061            # Do a simple filename based check
2062            if file == nzf.filename:
2063                found = True
2064                if (nzf.md5sum is not None) and nzf.md5sum == md5pack[file]:
2065                    logging.debug("Quick-check of file %s OK", file)
2066                    result &= True
2067                elif file_to_ignore:
2068                    # We don't care about these files
2069                    logging.debug("Quick-check ignoring file %s", file)
2070                    result &= True
2071                else:
2072                    logging.info("Quick-check of file %s failed!", file)
2073                    result = False
2074                break
2075
2076            # Now lets do obfuscation check
2077            if nzf.md5sum == md5pack[file]:
2078                try:
2079                    logging.debug("Quick-check will rename %s to %s", nzf.filename, file)
2080
2081                    # Note: file can and is allowed to be in a subdirectory.
2082                    # Subdirectories in par2 always contain "/", not "\"
2083                    renamer(
2084                        os.path.join(nzo.download_path, nzf.filename),
2085                        os.path.join(nzo.download_path, file),
2086                        create_local_directories=True,
2087                    )
2088                    renames[file] = nzf.filename
2089                    nzf.filename = file
2090                    result &= True
2091                    found = True
2092                    break
2093                except IOError:
2094                    # Renamed failed for some reason, probably already done
2095                    break
2096
2097        if not found:
2098            if file_to_ignore:
2099                # We don't care about these files
2100                logging.debug("Quick-check ignoring missing file %s", file)
2101                continue
2102
2103            logging.info("Cannot Quick-check missing file %s!", file)
2104            result = False
2105
2106    # Save renames
2107    if renames:
2108        nzo.renamed_file(renames)
2109
2110    return result
2111
2112
2113def unrar_check(rar):
2114    """Return version number of unrar, where "5.01" returns 501
2115    Also return whether an original version is found
2116    (version, original)
2117    """
2118    version = 0
2119    original = ""
2120    if rar:
2121        try:
2122            version = run_command([rar])
2123        except:
2124            return version, original
2125        original = "Alexander Roshal" in version
2126        m = re.search(r"RAR\s(\d+)\.(\d+)", version)
2127        if m:
2128            version = int(m.group(1)) * 100 + int(m.group(2))
2129        else:
2130            version = 0
2131    return version, original
2132
2133
2134def par2_mt_check(par2_path):
2135    """Detect if we have multicore par2 variants"""
2136    try:
2137        par2_version = run_command([par2_path, "-h"])
2138        # Look for a threads option
2139        if "-t<" in par2_version:
2140            return True
2141    except:
2142        pass
2143    return False
2144
2145
2146def is_sfv_file(myfile):
2147    """Checks if given file is a SFV file, and returns result as boolean"""
2148    # based on https://stackoverflow.com/a/7392391/5235502
2149    textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7F})
2150    is_ascii_string = lambda input_bytes: not bool(input_bytes.translate(None, textchars))
2151
2152    # first check if it's plain text (ASCII or Unicode)
2153    try:
2154        with open(myfile, "rb") as f:
2155            # get first 10000 bytes to check
2156            myblock = f.read(10000)
2157            if is_ascii_string(myblock):
2158                # ASCII, so store lines for further inspection
2159                try:
2160                    lines = ubtou(myblock).split("\n")
2161                except UnicodeDecodeError:
2162                    return False
2163            else:
2164                # non-ASCII, so not SFV
2165                return False
2166    except:
2167        # the with-open() went wrong, so not an existing file, so certainly not a SFV file
2168        return False
2169
2170    sfv_info_line_counter = 0
2171    for line in lines:
2172        line = line.strip()
2173        if re.search(r"^[^;].*\ +[A-Fa-f0-9]{8}$", line):
2174            # valid, useful SFV line: some text, then one or more space, and a 8-digit hex number
2175            sfv_info_line_counter += 1
2176            if sfv_info_line_counter >= 10:
2177                # with 10 valid, useful lines we're confident enough
2178                # (note: if we find less lines (even just 1 line), with no negatives, it is OK. See below)
2179                break
2180        elif not line or line.startswith(";"):
2181            # comment line or just spaces, so continue to next line
2182            continue
2183        else:
2184            # not a valid SFV line, so not a SFV file:
2185            return False
2186    # if we get here, no negatives were found, and at least 1 valid line is OK
2187    return sfv_info_line_counter >= 1
2188
2189
2190def sfv_check(sfvs, nzo: NzbObject, workdir):
2191    """Verify files using SFV files"""
2192    # Update status
2193    nzo.status = Status.VERIFYING
2194    nzo.set_action_line(T("Trying SFV verification"), "...")
2195
2196    # We use bitwise assigment (&=) so False always wins in case of failure
2197    # This way the renames always get saved!
2198    result = True
2199    nzf_list = nzo.finished_files
2200    renames = {}
2201
2202    # Files to ignore
2203    ignore_ext = cfg.quick_check_ext_ignore()
2204
2205    # We need the crc32 of all files
2206    calculated_crc32 = {}
2207    verifytotal = len(nzo.finished_files)
2208    verifynum = 0
2209    for nzf in nzf_list:
2210        verifynum += 1
2211        nzo.set_action_line(T("Verifying"), "%02d/%02d" % (verifynum, verifytotal))
2212        calculated_crc32[nzf.filename] = crc_calculate(os.path.join(workdir, nzf.filename))
2213
2214    sfv_parse_results = {}
2215    nzo.set_action_line(T("Trying SFV verification"), "...")
2216    for sfv in sfvs:
2217        setname = setname_from_path(sfv)
2218        nzo.set_unpack_info("Repair", T("Trying SFV verification"), setname)
2219
2220        # Parse the sfv and add to the already found results
2221        # Duplicates will be replaced
2222        sfv_parse_results.update(parse_sfv(sfv))
2223
2224    for file in sfv_parse_results:
2225        found = False
2226        file_to_ignore = get_ext(file).replace(".", "") in ignore_ext
2227        for nzf in nzf_list:
2228            # Do a simple filename based check
2229            if file == nzf.filename:
2230                found = True
2231                if nzf.filename in calculated_crc32 and calculated_crc32[nzf.filename] == sfv_parse_results[file]:
2232                    logging.debug("SFV-check of file %s OK", file)
2233                    result &= True
2234                elif file_to_ignore:
2235                    # We don't care about these files
2236                    logging.debug("SFV-check ignoring file %s", file)
2237                    result &= True
2238                else:
2239                    logging.info("SFV-check of file %s failed!", file)
2240                    result = False
2241                break
2242
2243            # Now lets do obfuscation check
2244            if nzf.filename in calculated_crc32 and calculated_crc32[nzf.filename] == sfv_parse_results[file]:
2245                try:
2246                    logging.debug("SFV-check will rename %s to %s", nzf.filename, file)
2247                    renamer(os.path.join(nzo.download_path, nzf.filename), os.path.join(nzo.download_path, file))
2248                    renames[file] = nzf.filename
2249                    nzf.filename = file
2250                    result &= True
2251                    found = True
2252                    break
2253                except IOError:
2254                    # Renamed failed for some reason, probably already done
2255                    break
2256
2257        if not found:
2258            if file_to_ignore:
2259                # We don't care about these files
2260                logging.debug("SVF-check ignoring missing file %s", file)
2261                continue
2262
2263            logging.info("Cannot SFV-check missing file %s!", file)
2264            result = False
2265
2266    # Save renames
2267    if renames:
2268        nzo.renamed_file(renames)
2269
2270    return result
2271
2272
2273def parse_sfv(sfv_filename):
2274    """Parse SFV file and return dictonary of crc32's and filenames"""
2275    results = {}
2276    with open(sfv_filename, mode="rb") as sfv_list:
2277        for sfv_item in sfv_list:
2278            sfv_item = sfv_item.strip()
2279            # Ignore comment-lines
2280            if sfv_item.startswith(b";"):
2281                continue
2282            # Parse out the filename and crc32
2283            filename, expected_crc32 = sfv_item.strip().rsplit(maxsplit=1)
2284            # We don't know what encoding is used when it was created
2285            results[correct_unknown_encoding(filename)] = expected_crc32.lower()
2286    return results
2287
2288
2289def crc_calculate(path):
2290    """Calculate crc32 of the given file"""
2291    crc = 0
2292    with open(path, "rb") as fp:
2293        while 1:
2294            data = fp.read(4096)
2295            if not data:
2296                break
2297            crc = zlib.crc32(data, crc)
2298    return b"%08x" % (crc & 0xFFFFFFFF)
2299
2300
2301def analyse_show(name):
2302    """Do a quick SeasonSort check and return basic facts"""
2303    job = SeriesSorter(None, name, None, None)
2304    job.match(force=True)
2305    if job.is_match():
2306        job.get_values()
2307    info = job.show_info
2308    show_name = info.get("show_name", "").replace(".", " ").replace("_", " ")
2309    show_name = show_name.replace("  ", " ")
2310    return show_name, info.get("season_num", ""), info.get("episode_num", ""), info.get("ep_name", "")
2311
2312
2313def pre_queue(nzo: NzbObject, pp, cat):
2314    """Run pre-queue script (if any) and process results.
2315    pp and cat are supplied seperate since they can change.
2316    """
2317
2318    def fix(p):
2319        # If added via API, some items can still be "None" (as a string)
2320        if not p or str(p).lower() == "none":
2321            return ""
2322        return str(p)
2323
2324    values = [1, nzo.final_name_with_password, pp, cat, nzo.script, nzo.priority, None]
2325    script_path = make_script_path(cfg.pre_script())
2326    if script_path:
2327        # Basic command-line parameters
2328        command = [
2329            script_path,
2330            nzo.final_name_with_password,
2331            pp,
2332            cat,
2333            nzo.script,
2334            nzo.priority,
2335            str(nzo.bytes),
2336            " ".join(nzo.groups),
2337        ]
2338        command.extend(analyse_show(nzo.final_name_with_password))
2339        command = [fix(arg) for arg in command]
2340
2341        # Fields not in the NZO directly
2342        extra_env_fields = {
2343            "groups": " ".join(nzo.groups),
2344            "show_name": command[8],
2345            "show_season": command[9],
2346            "show_episode": command[10],
2347            "show_episode_name": command[11],
2348        }
2349
2350        try:
2351            p = build_and_run_command(command, env=create_env(nzo, extra_env_fields))
2352        except:
2353            logging.debug("Failed script %s, Traceback: ", script_path, exc_info=True)
2354            return values
2355
2356        output = platform_btou(p.stdout.read())
2357        ret = p.wait()
2358        logging.info("Pre-queue script returned %s and output=\n%s", ret, output)
2359        if ret == 0:
2360            split_output = output.splitlines()
2361            try:
2362                # Extract category line from pre-queue output
2363                pre_queue_category = split_output[3].strip(" '\"")
2364            except IndexError:
2365                pre_queue_category = None
2366
2367            for index, line in enumerate(split_output):
2368                line = line.strip(" '\"")
2369                if index < len(values):
2370                    if line:
2371                        values[index] = line
2372                    elif pre_queue_category and index in (2, 4, 5):
2373                        # Preserve empty pp, script, and priority lines to prevent
2374                        # pre-existing values from overriding category-based settings
2375                        values[index] = ""
2376
2377        accept = int_conv(values[0])
2378        if accept < 1:
2379            logging.info("Pre-Q refuses %s", nzo.final_name)
2380        elif accept == 2:
2381            logging.info("Pre-Q accepts&fails %s", nzo.final_name)
2382        else:
2383            logging.info("Pre-Q accepts %s", nzo.final_name)
2384
2385    return values
2386
2387
2388def is_sevenfile(path):
2389    """Return True if path has proper extension and 7Zip is installed"""
2390    return SEVEN_COMMAND and os.path.splitext(path)[1].lower() == ".7z"
2391
2392
2393class SevenZip:
2394    """Minimal emulation of ZipFile class for 7Zip"""
2395
2396    def __init__(self, path):
2397        self.path = path
2398
2399    def namelist(self):
2400        """Return list of names in 7Zip"""
2401        names = []
2402        # Future extension: use '-sccUTF-8' to get names in UTF8 encoding
2403        command = [SEVEN_COMMAND, "l", "-p", "-y", "-slt", self.path]
2404        output = run_command(command)
2405
2406        re_path = re.compile("^Path = (.+)")
2407        for line in output.split("\n"):
2408            m = re_path.search(line)
2409            if m:
2410                names.append(m.group(1).strip("\r"))
2411        if names:
2412            # Remove name of archive itself
2413            del names[0]
2414        return names
2415
2416    def read(self, name):
2417        """Read named file from 7Zip and return data"""
2418        command = [SEVEN_COMMAND, "e", "-p", "-y", "-so", self.path, name]
2419        # Ignore diagnostic output, otherwise it will be appended to content
2420        return run_command(command, stderr=subprocess.DEVNULL)
2421
2422    def close(self):
2423        """Close file"""
2424        pass
2425