1#!/usr/local/bin/python3.8
2# vim:fileencoding=utf-8
3# License: GPLv3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
4
5
6import os
7import shutil
8import tempfile
9from threading import Lock
10
11from calibre.customize.ui import input_profiles, output_profiles
12from calibre.db.errors import NoSuchBook
13from calibre.srv.changes import formats_added
14from calibre.srv.errors import BookNotFound, HTTPNotFound
15from calibre.srv.routes import endpoint, json
16from calibre.srv.utils import get_library_data
17from calibre.utils.monotonic import monotonic
18from calibre.utils.shared_file import share_open
19from polyglot.builtins import iteritems
20
21receive_data_methods = {'GET', 'POST'}
22conversion_jobs = {}
23cache_lock = Lock()
24
25
26class JobStatus:
27
28    def __init__(self, job_id, book_id, tdir, library_id, pathtoebook, conversion_data):
29        self.job_id = job_id
30        self.log = self.traceback = ''
31        self.book_id = book_id
32        self.output_path = os.path.join(
33            tdir, 'output.' + conversion_data['output_fmt'].lower())
34        self.tdir = tdir
35        self.library_id, self.pathtoebook = library_id, pathtoebook
36        self.conversion_data = conversion_data
37        self.running = self.ok = True
38        self.last_check_at = monotonic()
39        self.was_aborted = False
40
41    def cleanup(self):
42        safe_delete_tree(self.tdir)
43        self.log = self.traceback = ''
44
45    @property
46    def current_status(self):
47        try:
48            with share_open(os.path.join(self.tdir, 'status'), 'rb') as f:
49                lines = f.read().decode('utf-8').splitlines()
50        except Exception:
51            lines = ()
52        for line in reversed(lines):
53            if line.endswith('|||'):
54                p, msg = line.partition(':')[::2]
55                percent = float(p)
56                msg = msg[:-3]
57                return percent, msg
58        return 0, ''
59
60
61def expire_old_jobs():
62    now = monotonic()
63    with cache_lock:
64        remove = [job_id for job_id, job_status in iteritems(conversion_jobs) if now - job_status.last_check_at >= 360]
65        for job_id in remove:
66            job_status = conversion_jobs.pop(job_id)
67            job_status.cleanup()
68
69
70def safe_delete_file(path):
71    try:
72        os.remove(path)
73    except OSError:
74        pass
75
76
77def safe_delete_tree(path):
78    try:
79        shutil.rmtree(path, ignore_errors=True)
80    except OSError:
81        pass
82
83
84def job_done(job):
85    with cache_lock:
86        try:
87            job_status = conversion_jobs[job.job_id]
88        except KeyError:
89            return
90        job_status.running = False
91        if job.failed:
92            job_status.ok = False
93            job_status.log = job.read_log()
94            job_status.was_aborted = job.was_aborted
95            job_status.traceback = job.traceback
96    safe_delete_file(job_status.pathtoebook)
97
98
99def convert_book(path_to_ebook, opf_path, cover_path, output_fmt, recs):
100    from calibre.customize.conversion import OptionRecommendation
101    from calibre.ebooks.conversion.plumber import Plumber
102    from calibre.utils.logging import Log
103    recs.append(('verbose', 2, OptionRecommendation.HIGH))
104    recs.append(('read_metadata_from_opf', opf_path,
105                OptionRecommendation.HIGH))
106    if cover_path:
107        recs.append(('cover', cover_path, OptionRecommendation.HIGH))
108    log = Log()
109    os.chdir(os.path.dirname(path_to_ebook))
110    status_file = share_open('status', 'wb')
111
112    def notification(percent, msg=''):
113        status_file.write('{}:{}|||\n'.format(percent, msg).encode('utf-8'))
114        status_file.flush()
115
116    output_path = os.path.abspath('output.' + output_fmt.lower())
117    plumber = Plumber(path_to_ebook, output_path, log,
118                      report_progress=notification, override_input_metadata=True)
119    plumber.merge_ui_recommendations(recs)
120    plumber.run()
121
122
123def queue_job(ctx, rd, library_id, db, fmt, book_id, conversion_data):
124    from calibre.ebooks.metadata.opf2 import metadata_to_opf
125    from calibre.ebooks.conversion.config import GuiRecommendations, save_specifics
126    from calibre.customize.conversion import OptionRecommendation
127    tdir = tempfile.mkdtemp(dir=rd.tdir)
128    with tempfile.NamedTemporaryFile(prefix='', suffix=('.' + fmt.lower()), dir=tdir, delete=False) as src_file:
129        db.copy_format_to(book_id, fmt, src_file)
130    with tempfile.NamedTemporaryFile(prefix='', suffix='.jpg', dir=tdir, delete=False) as cover_file:
131        cover_copied = db.copy_cover_to(book_id, cover_file)
132    cover_path = cover_file.name if cover_copied else None
133    mi = db.get_metadata(book_id)
134    mi.application_id = mi.uuid
135    raw = metadata_to_opf(mi)
136    with tempfile.NamedTemporaryFile(prefix='', suffix='.opf', dir=tdir, delete=False) as opf_file:
137        opf_file.write(raw)
138    recs = GuiRecommendations()
139    recs.update(conversion_data['options'])
140    recs['gui_preferred_input_format'] = conversion_data['input_fmt'].lower()
141    save_specifics(db, book_id, recs)
142    recs = [(k, v, OptionRecommendation.HIGH) for k, v in iteritems(recs)]
143
144    job_id = ctx.start_job(
145        'Convert book %s (%s)' % (book_id, fmt), 'calibre.srv.convert',
146        'convert_book', args=(
147            src_file.name, opf_file.name, cover_path, conversion_data['output_fmt'], recs),
148        job_done_callback=job_done
149    )
150    expire_old_jobs()
151    with cache_lock:
152        conversion_jobs[job_id] = JobStatus(
153            job_id, book_id, tdir, library_id, src_file.name, conversion_data)
154    return job_id
155
156
157@endpoint('/conversion/start/{book_id}', postprocess=json, needs_db_write=True, types={'book_id': int}, methods=receive_data_methods)
158def start_conversion(ctx, rd, book_id):
159    db, library_id = get_library_data(ctx, rd)[:2]
160    if not ctx.has_id(rd, db, book_id):
161        raise BookNotFound(book_id, db)
162    data = json.loads(rd.request_body_file.read())
163    input_fmt = data['input_fmt']
164    job_id = queue_job(ctx, rd, library_id, db, input_fmt, book_id, data)
165    return job_id
166
167
168@endpoint('/conversion/status/{job_id}', postprocess=json, needs_db_write=True, types={'job_id': int}, methods=receive_data_methods)
169def conversion_status(ctx, rd, job_id):
170    with cache_lock:
171        job_status = conversion_jobs.get(job_id)
172        if job_status is None:
173            raise HTTPNotFound('No job with id: {}'.format(job_id))
174        job_status.last_check_at = monotonic()
175        if job_status.running:
176            percent, msg = job_status.current_status
177            if rd.query.get('abort_job'):
178                ctx.abort_job(job_id)
179            return {'running': True, 'percent': percent, 'msg': msg}
180
181        del conversion_jobs[job_id]
182
183    try:
184        ans = {'running': False, 'ok': job_status.ok, 'was_aborted':
185               job_status.was_aborted, 'traceback': job_status.traceback,
186               'log': job_status.log}
187        if job_status.ok:
188            db, library_id = get_library_data(ctx, rd)[:2]
189            if library_id != job_status.library_id:
190                raise HTTPNotFound('job library_id does not match')
191            fmt = job_status.output_path.rpartition('.')[-1]
192            try:
193                db.add_format(job_status.book_id, fmt, job_status.output_path)
194            except NoSuchBook:
195                raise HTTPNotFound(
196                    'book_id {} not found in library'.format(job_status.book_id))
197            formats_added({job_status.book_id: (fmt,)})
198            ans['size'] = os.path.getsize(job_status.output_path)
199            ans['fmt'] = fmt
200        return ans
201    finally:
202        job_status.cleanup()
203
204
205def get_conversion_options(input_fmt, output_fmt, book_id, db):
206    from calibre.ebooks.conversion.plumber import create_dummy_plumber
207    from calibre.ebooks.conversion.config import (
208        load_specifics, load_defaults, OPTIONS, options_for_input_fmt, options_for_output_fmt)
209    from calibre.customize.conversion import OptionRecommendation
210    plumber = create_dummy_plumber(input_fmt, output_fmt)
211    specifics = load_specifics(db, book_id)
212    ans = {'options': {}, 'disabled': set(), 'defaults': {}, 'help': {}}
213    ans['input_plugin_name'] = plumber.input_plugin.commit_name
214    ans['output_plugin_name'] = plumber.output_plugin.commit_name
215    ans['input_ui_data'] = plumber.input_plugin.ui_data
216    ans['output_ui_data'] = plumber.output_plugin.ui_data
217
218    def merge_group(group_name, option_names):
219        if not group_name or group_name in ('debug', 'metadata'):
220            return
221        defs = load_defaults(group_name)
222        defs.merge_recommendations(
223            plumber.get_option_by_name, OptionRecommendation.LOW, option_names)
224        specifics.merge_recommendations(
225            plumber.get_option_by_name, OptionRecommendation.HIGH, option_names, only_existing=True)
226        defaults = defs.as_dict()['options']
227        for k in defs:
228            if k in specifics:
229                defs[k] = specifics[k]
230        defs = defs.as_dict()
231        ans['options'].update(defs['options'])
232        ans['disabled'] |= set(defs['disabled'])
233        ans['defaults'].update(defaults)
234        ans['help'] = plumber.get_all_help()
235
236    for group_name, option_names in iteritems(OPTIONS['pipe']):
237        merge_group(group_name, option_names)
238
239    group_name, option_names = options_for_input_fmt(input_fmt)
240    merge_group(group_name, option_names)
241    group_name, option_names = options_for_output_fmt(output_fmt)
242    merge_group(group_name, option_names)
243
244    ans['disabled'] = tuple(ans['disabled'])
245    return ans
246
247
248def profiles():
249    ans = getattr(profiles, 'ans', None)
250    if ans is None:
251        def desc(profile):
252            w, h = profile.screen_size
253            if w >= 10000:
254                ss = _('unlimited')
255            else:
256                ss = _('%(width)d x %(height)d pixels') % dict(width=w, height=h)
257            ss = _('Screen size: %s') % ss
258            return {'name': profile.name, 'description': ('%s [%s]' % (profile.description, ss))}
259
260        ans = profiles.ans = {}
261        ans['input'] = {p.short_name: desc(p) for p in input_profiles()}
262        ans['output'] = {p.short_name: desc(p) for p in output_profiles()}
263    return ans
264
265
266@endpoint('/conversion/book-data/{book_id}', postprocess=json, types={'book_id': int})
267def conversion_data(ctx, rd, book_id):
268    from calibre.ebooks.conversion.config import (
269        NoSupportedInputFormats, get_input_format_for_book, get_sorted_output_formats)
270    db = get_library_data(ctx, rd)[0]
271    if not ctx.has_id(rd, db, book_id):
272        raise BookNotFound(book_id, db)
273    try:
274        input_format, input_formats = get_input_format_for_book(db, book_id)
275    except NoSupportedInputFormats:
276        input_formats = []
277    else:
278        if rd.query.get('input_fmt') and rd.query.get('input_fmt').lower() in input_formats:
279            input_format = rd.query.get('input_fmt').lower()
280        if input_format in input_formats:
281            input_formats.remove(input_format)
282            input_formats.insert(0, input_format)
283    input_fmt = input_formats[0] if input_formats else 'epub'
284    output_formats = get_sorted_output_formats(rd.query.get('output_fmt'))
285    ans = {
286        'input_formats': [x.upper() for x in input_formats],
287        'output_formats': output_formats,
288        'profiles': profiles(),
289        'conversion_options': get_conversion_options(input_fmt, output_formats[0], book_id, db),
290        'title': db.field_for('title', book_id),
291        'authors': db.field_for('authors', book_id),
292        'book_id': book_id
293    }
294    return ans
295