1#!@TARGET_PYTHON@
2# -*- coding: utf-8 -*-
3
4# This file is part of LilyPond, the GNU music typesetter.
5#
6# Copyright (C) 1998--2021  Han-Wen Nienhuys <hanwen@xs4all.nl>
7#                           Jan Nieuwenhuizen <janneke@gnu.org>
8#
9# LilyPond is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# LilyPond is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with LilyPond.  If not, see <http://www.gnu.org/licenses/>.
21
22r'''
23Example usage:
24
25test:
26  lilypond-book --filter="tr '[a-z]' '[A-Z]'" BOOK
27
28convert-ly on book:
29  lilypond-book --filter="convert-ly --no-version --from=1.6.11 -" BOOK
30
31classic lilypond-book:
32  lilypond-book --process="lilypond" BOOK.tely
33
34TODO:
35
36  *  ly-options: intertext?
37  *  --line-width?
38  *  eps in latex / eps by lilypond -b ps?
39  *  check latex parameters, twocolumn, multicolumn?
40  *  use --png --ps --pdf for making images?
41
42  *  Converting from lilypond-book source, substitute:
43   @mbinclude foo.itely -> @include foo.itely
44   \mbinput -> \input
45
46'''
47
48
49# TODO: Better solve the global_options copying to the snippets...
50
51import gettext
52import glob
53import hashlib
54from optparse import OptionGroup
55import os
56import re
57import shlex
58import stat
59import subprocess
60import sys
61import tempfile
62import typing
63
64# See lock_path and unlock_path; this module is not available at all on Windows.
65if os.name == 'posix':
66    import fcntl
67
68"""
69@relocate-preamble@
70"""
71
72import book_base
73import book_docbook
74import book_html
75import book_latex
76import book_texinfo
77import book_snippets
78
79# Load translation and install _() into Python's builtins namespace.
80gettext.install('lilypond', '@localedir@')
81
82import lilylib as ly
83
84backend = 'ps'
85
86help_summary = (
87    _("Process LilyPond snippets in hybrid HTML, LaTeX, texinfo or DocBook document.")
88    + '\n\n'
89    + _("Examples:")
90    + '''
91 $ lilypond-book --filter="tr '[a-z]' '[A-Z]'" %(BOOK)s
92 $ lilypond-book -F "convert-ly --no-version --from=2.0.0 -" %(BOOK)s
93 $ lilypond-book --process='lilypond -I include' %(BOOK)s
94''' % {'BOOK': _("BOOK")})
95
96authors = ('Jan Nieuwenhuizen <janneke@gnu.org>',
97           'Han-Wen Nienhuys <hanwen@xs4all.nl>')
98
99################################################################
100
101
102def exit(i):
103    if ly.is_verbose():
104        raise Exception(_('Exiting (%d)...') % i)
105    else:
106        sys.exit(i)
107
108
109progress = ly.progress
110warning = ly.warning
111error = ly.error
112
113program_version = '@TOPLEVEL_VERSION@'
114if program_version.startswith("@"):
115    # '@' in lilypond-book output confuses texinfo
116    program_version = "dev"
117
118
119def identify():
120    progress('%s (GNU LilyPond) %s' % (ly.program_name, program_version))
121
122
123def warranty():
124    identify()
125    sys.stdout.write('''
126%s
127
128  %s
129
130%s
131%s
132''' % (_('Copyright (c) %s by') % '2001--2021',
133        '\n  '.join(authors),
134        _("Distributed under terms of the GNU General Public License."),
135        _("It comes with NO WARRANTY.")))
136
137
138def get_option_parser():
139    p = ly.get_option_parser(usage=_("%s [OPTION]... FILE") % 'lilypond-book',
140                             description=help_summary,
141                             conflict_handler="resolve",
142                             add_help_option=False)
143
144    p.add_option('-F', '--filter',
145                 help=_("pipe snippets through FILTER "
146                        "[default: `convert-ly -n -']"),
147                 metavar=_("FILTER"),
148                 action="store",
149                 dest="filter_cmd",
150                 default=None)
151
152    p.add_option('-f', '--format',
153                 help=_("use output format FORMAT (texi [default], "
154                        "texi-html, latex, html, docbook)"),
155                 metavar=_("FORMAT"),
156                 action='store')
157
158    p.add_option("-h", "--help",
159                 action="help",
160                 help=_("show this help and exit"))
161
162    p.add_option("-I", '--include',
163                 help=_("add DIR to include path"),
164                 metavar=_("DIR"),
165                 action='append',
166                 dest='include_path',
167                 default=[])
168
169    p.add_option('--info-images-dir',
170                 help=_("format Texinfo output so that Info will "
171                        "look for images of music in DIR"),
172                 metavar=_("DIR"),
173                 action='store',
174                 dest='info_images_dir',
175                 default='')
176
177    p.add_option('--left-padding',
178                 help=_("pad left side of music to align music in spite "
179                        "of uneven bar numbers (in mm) [default: %default]"),
180                 metavar=_("PAD"),
181                 dest="padding_mm",
182                 type="float",
183                 default=3.0)
184
185    p.add_option('--lily-loglevel',
186                 help=_("print lilypond log messages according to LOGLEVEL "
187                        "[default: %default]"),
188                 metavar=_("LOGLEVEL"),
189                 action='store',
190                 dest='lily_loglevel',
191                 default=os.environ.get("LILYPOND_LOGLEVEL", None))
192
193    p.add_option('--lily-output-dir',
194                 help=_("write lily-XXX files to DIR, "
195                        "link into --output dir"),
196                 metavar=_("DIR"),
197                 action='store',
198                 dest='lily_output_dir',
199                 default=None)
200
201    p.add_option("-l", "--loglevel",
202                 help=_("print log messages according to LOGLEVEL "
203                        "(NONE, ERROR, WARNING, PROGRESS [default], DEBUG)"),
204                 metavar=_("LOGLEVEL"),
205                 action='callback',
206                 callback=ly.handle_loglevel_option,
207                 type='string')
208
209    p.add_option("-o", '--output',
210                 help=_("write output to DIR"),
211                 metavar=_("DIR"),
212                 action='store',
213                 dest='output_dir',
214                 default='')
215
216    p.add_option('-P', '--process',
217                 help=_("process ly_files using COMMAND FILE..."),
218                 metavar=_("COMMAND"),
219                 action='store',
220                 dest='process_cmd',
221                 default='')
222
223    p.add_option('--redirect-lilypond-output',
224                 help=_("redirect the lilypond output"),
225                 action='store_true',
226                 dest='redirect_output',
227                 default=False)
228
229    p.add_option('-s', '--safe',
230                 help=_("compile snippets in safe mode"),
231                 action="store_true",
232                 dest="safe_mode",
233                 default=False)
234
235    p.add_option('--skip-lily-check',
236                 help=_("do not fail if no lilypond output is found"),
237                 metavar=_("DIR"),
238                 action='store_true',
239                 dest='skip_lilypond_run',
240                 default=False)
241
242    p.add_option('--skip-png-check',
243                 help=_("do not fail if no PNG images "
244                        "are found for EPS files"),
245                 metavar=_("DIR"),
246                 action='store_true',
247                 dest='skip_png_check',
248                 default=False)
249
250    p.add_option('--use-source-file-names',
251                 help=_("write snippet output files with the same "
252                        "base name as their source file"),
253                 action='store_true',
254                 dest='use_source_file_names',
255                 default=False)
256
257    p.add_option('-V', '--verbose',
258                 help=_("be verbose"),
259                 action="callback",
260                 callback=ly.handle_loglevel_option,
261                 callback_args=("DEBUG",))
262
263    p.version = "@TOPLEVEL_VERSION@"
264    p.add_option("--version",
265                 help=_("show version number and exit"),
266                 action="version")
267
268    p.add_option('-w', '--warranty',
269                 help=_("show warranty and copyright"),
270                 action='store_true')
271
272    group = OptionGroup(p, "Options only for the latex and texinfo backends")
273    group.add_option('--latex-program',
274                     help=_("run executable PROG instead of latex or, "
275                            "in case --pdf option is set, "
276                            "instead of pdflatex"),
277                     metavar=_("PROG"),
278                     action='store',
279                     dest='latex_program',
280                     default='latex')
281    group.add_option('--texinfo-program',
282                     help=_("run executable PROG instead of texi2pdf"),
283                     metavar=_("PROG"),
284                     action='store',
285                     dest='texinfo_program',
286                     default='texi2pdf')
287    group.add_option('--pdf',
288                     help=_("create PDF files for use with pdftex"),
289                     action="store_true",
290                     dest="create_pdf",
291                     default=False)
292    p.add_option_group(group)
293
294    p.add_option_group('',
295                       description=(
296                           _("Report bugs via %s")
297                           % 'bug-lilypond@gnu.org') + '\n')
298
299    for formatter in book_base.all_formats:
300        formatter.add_options(p)
301
302    return p
303
304
305lilypond_binary = os.path.join('@bindir@', 'lilypond')
306
307# If we are called with full path, try to use lilypond binary
308# installed in the same path; this is needed in GUB binaries, where
309# @bindir is always different from the installed binary path.
310if 'bindir' in globals() and bindir:
311    lilypond_binary = os.path.join(bindir, 'lilypond')
312
313# Only use installed binary when we are installed too.
314if '@bindir@' == ('@' + 'bindir@') or not os.path.exists(lilypond_binary):
315    lilypond_binary = 'lilypond'
316
317# Need to shell-quote, issue 3468
318# FIXME: we should really pass argument lists
319# everywhere instead of playing with shell syntax.
320lilypond_binary = shlex.quote(lilypond_binary)
321
322global_options = None
323
324
325def command_name(cmd):
326    # Strip all stuf after command,
327    # deal with "((latex ) >& 1 ) .." too
328    cmd = re.match(r'([\(\)]*)([^\\ ]*)', cmd).group(2)
329    return os.path.basename(cmd)
330
331
332def system_in_directory(cmd_str, directory, log_file):
333    """Execute a command in a different directory."""
334
335    if ly.is_verbose():
336        ly.progress(_("Invoking `%s\'") % cmd_str)
337    elif global_options.redirect_output:
338        ly.progress(_("Processing %s.ly") % log_file)
339    else:
340        name = command_name(cmd_str)
341        ly.progress(_("Running %s...") % name)
342
343    output_location = None
344    if global_options.redirect_output:
345        output_location = open(log_file + '.log', 'w', encoding='utf8')
346
347    try:
348        subprocess.run(cmd_str, stdout=output_location,
349                       stderr=output_location, cwd=directory,
350                       shell=True, check=True)
351    except subprocess.CalledProcessError as e:
352        sys.stderr.write("%s\n" % e)
353        sys.exit(1)
354
355
356def process_snippets(cmd, outdated_dict,
357                     formatter, lily_output_dir):
358    """Run cmd on all of the .ly files from snippets."""
359    basenames = sorted(outdated_dict.keys())
360
361    # No need for a secure hash function, just need a digest.
362    checksum = hashlib.md5()
363    for name in basenames:
364        checksum.update(name.encode('ascii'))
365    checksum = checksum.hexdigest()
366
367    lily_output_dir = global_options.lily_output_dir
368
369    # Write list of snippet names.
370    snippet_names_file = 'snippet-names-%s.ly' % checksum
371    snippet_names_path = os.path.join(lily_output_dir, snippet_names_file)
372    with open(snippet_names_path, 'w', encoding='utf8') as snippet_names:
373        snippet_names.write('\n'.join([name + '.ly' for name in basenames]))
374
375    # Run command.
376    cmd = formatter.adjust_snippet_command(cmd)
377    # Remove .ly ending.
378    logfile = os.path.splitext(snippet_names_path)[0]
379    snippet_names_arg = mkarg(snippet_names_path.replace(os.path.sep, '/'))
380    system_in_directory(' '.join([cmd, snippet_names_arg]),
381                        lily_output_dir,
382                        logfile)
383    os.unlink(snippet_names_path)
384
385
386def lock_path(name):
387    if os.name != 'posix':
388        return None
389
390    fp = open(name, 'w', encoding='utf8')
391    fcntl.lockf(fp, fcntl.LOCK_EX)
392    return fp
393
394
395def unlock_path(lock):
396    if os.name != 'posix':
397        return None
398    fcntl.lockf(lock, fcntl.LOCK_UN)
399    lock.close()
400
401
402def do_process_cmd(chunks, options):
403    """Wrap do_process_cmd_locked in a filesystem lock"""
404    snippets = [c for c in chunks if isinstance(
405        c, book_snippets.LilypondSnippet)]
406
407    # calculate checksums eagerly
408    for s in snippets:
409        s.get_checksum()
410
411    os.makedirs(options.lily_output_dir, exist_ok=True)
412    lock_file = os.path.join(options.lily_output_dir, "lock")
413    lock = None
414    try:
415        lock = lock_path(lock_file)
416        do_process_cmd_locked(snippets, options)
417    finally:
418        if lock:
419            unlock_path(lock)
420
421
422def do_process_cmd_locked(snippets, options):
423    """Look at all snippets, write the outdated ones, and compile them."""
424    outdated = [c for c in snippets if c.is_outdated(options.lily_output_dir)]
425
426    if outdated:
427        # First unique the list based on the basename, by using them as keys
428        # in a dict.
429        outdated_dict = dict()
430        for snippet in outdated:
431            outdated_dict[snippet.basename()] = snippet
432
433        # Next call write_ly() for each snippet once.
434        progress(_("Writing snippets..."))
435        for snippet in outdated_dict.values():
436            snippet.write_ly()
437
438        progress(_("Processing..."))
439        process_snippets(options.process_cmd, outdated_dict,
440                         options.formatter, options.lily_output_dir)
441
442    else:
443        progress(_("All snippets are up to date..."))
444
445    progress(_("Linking files..."))
446    if options.lily_output_dir != options.output_dir:
447        for snippet in snippets:
448            snippet.link_all_output_files(options.lily_output_dir,
449                                          options.output_dir)
450
451
452###
453# Format guessing data
454
455def guess_format(input_filename):
456    format = None
457    e = os.path.splitext(input_filename)[1]
458    for formatter in book_base.all_formats:
459        if formatter.can_handle_extension(e):
460            return formatter
461    error(_("cannot determine format for: %s" % input_filename))
462    exit(1)
463
464def write_if_updated(file_name, lines):
465    try:
466        with open(file_name, encoding='utf-8') as file:
467            old_str = file.read()
468    except FileNotFoundError:
469        pass
470    else:
471        new_str = ''.join(lines)
472        if old_str == new_str:
473            progress(_("%s is up to date.") % file_name)
474
475            # this prevents make from always rerunning lilypond-book:
476            # output file must be touched in order to be up to date
477            os.utime(file_name, None)
478            return
479
480    output_dir = os.path.dirname(file_name)
481    os.makedirs(output_dir, exist_ok=True)
482
483    progress(_("Writing `%s'...") % file_name)
484    open(file_name, 'w', encoding='utf-8').writelines(lines)
485
486
487def note_input_file(name, inputs=[]):
488    # hack: inputs is mutable!
489    inputs.append(name)
490    return inputs
491
492
493def samefile(f1, f2):
494    try:
495        return os.path.samefile(f1, f2)
496    except AttributeError:                # Windoze
497        f1 = re.sub("//*", "/", f1)
498        f2 = re.sub("//*", "/", f2)
499        return f1 == f2
500
501
502def do_file(input_filename, included=False):
503    # Ugh.
504    input_absname = input_filename
505    if not input_filename or input_filename == '-':
506        in_handle = sys.stdin
507    else:
508        if os.path.exists(input_filename):
509            input_fullname = input_filename
510        else:
511            input_fullname = global_options.formatter.input_fullname(
512                input_filename)
513        # Normalize path to absolute path, since we will change cwd to the output dir!
514        # Otherwise, "lilypond-book -o out test.tex" will complain that it is
515        # overwriting the input file (which it is actually not), since the
516        # input filename is relative to the CWD...
517        input_absname = os.path.abspath(input_fullname)
518
519        note_input_file(input_fullname)
520        in_handle = open(input_fullname, 'r', encoding='utf-8')
521
522    if input_filename == '-':
523        global_options.input_dir = os.getcwd()
524        input_base = 'stdin'
525    elif included:
526        input_base = os.path.splitext(input_filename)[0]
527    else:
528        global_options.input_dir = os.path.split(input_absname)[0]
529        input_base = os.path.basename(
530            os.path.splitext(input_filename)[0])
531
532    output_filename = os.path.join(global_options.output_dir,
533                                   input_base + global_options.formatter.default_extension)
534    if (os.path.exists(input_filename)
535        and os.path.exists(output_filename)
536            and samefile(output_filename, input_absname)):
537        error(
538            _("Output would overwrite input file; use --output."))
539        exit(2)
540
541    try:
542        progress(_("Reading `%s'") % input_absname)
543        source = in_handle.read()
544
545        if not included:
546            global_options.formatter.init_default_snippet_options(source)
547
548        progress(_("Dissecting..."))
549        chunks = book_base.find_toplevel_snippets(
550            source, global_options.formatter, global_options)
551        for c in chunks:
552            c.set_output_fullpath(output_filename)
553
554        # Let the formatter modify the chunks before further processing
555        chunks = global_options.formatter.process_chunks(chunks)
556
557        def process_include(snippet):
558            name = snippet.substring('filename')
559            progress(_("Processing include `%s'") % name)
560            return do_file(name, included=True)
561
562        include_chunks = []
563        for x in chunks:
564            if isinstance(x, book_snippets.IncludeSnippet):
565               include_chunks += process_include(x)
566
567        return chunks + include_chunks
568
569    except book_snippets.CompileError:
570        progress(_("Removing `%s'") % output_filename)
571        raise book_snippets.CompileError
572
573
574def do_options():
575    global global_options
576
577    opt_parser = get_option_parser()
578    (global_options, args) = opt_parser.parse_args()
579
580    global_options.information = {
581        'program_version': program_version, 'program_name': ly.program_name}
582
583    if global_options.lily_output_dir:
584        global_options.lily_output_dir = os.path.expanduser(
585            global_options.lily_output_dir)
586    if global_options.output_dir:
587        global_options.output_dir = os.path.expanduser(
588            global_options.output_dir)
589
590    # Compute absolute paths of include directories.
591    for i, path in enumerate(global_options.include_path):
592        global_options.include_path[i] = os.path.abspath(path)
593
594    # Append the current directory.
595    global_options.include_path.append(os.getcwd())
596
597    if global_options.warranty:
598        warranty()
599        exit(0)
600    if not args or len(args) > 1:
601        opt_parser.print_help()
602        exit(2)
603
604    return args
605
606
607def mkarg(x):
608    r"""
609    A modified version of the commands.mkarg(x)
610
611    Uses double quotes (since Windows can't handle the single quotes)
612    and escapes the characters \, $, ", and ` for unix shells.
613    """
614    if os.name == 'nt':
615        return ' "%s"' % x
616    s = ' "'
617    for c in x:
618        if c in '\\$"`':
619            s = s + '\\'
620        s = s + c
621    s = s + '"'
622    return s
623
624
625def write_output_documents(chunks: typing.List[book_snippets.Chunk], is_filter: bool):
626    text_by_path = {}
627    for ch in chunks:
628        path = ch.output_fullpath()
629        if path not in text_by_path:
630            text_by_path[path] = []
631
632        if is_filter:
633            s = ch.filter_text()
634        else:
635            s = ch.replacement_text()
636
637        text_by_path[path].append(s)
638
639    for path in text_by_path:
640        write_if_updated(path, text_by_path[path])
641
642
643def main():
644    if "LILYPOND_BOOK_LOGLEVEL" in os.environ:
645        ly.set_loglevel(os.environ["LILYPOND_BOOK_LOGLEVEL"])
646    files = do_options()
647
648    basename = os.path.splitext(files[0])[0]
649    basename = os.path.split(basename)[1]
650
651    if global_options.format:
652        # Retrieve the formatter for the given format
653        for formatter in book_base.all_formats:
654            if formatter.can_handle_format(global_options.format):
655                global_options.formatter = formatter
656    else:
657        global_options.formatter = guess_format(files[0])
658        global_options.format = global_options.formatter.format
659
660    # make the global options available to the formatters:
661    global_options.formatter.global_options = global_options
662    formats = global_options.formatter.image_formats
663
664    if global_options.process_cmd == '':
665        global_options.process_cmd = (
666            lilypond_binary + ' --formats=%s ' % formats)
667
668    global_options.process_cmd += (
669        ' '.join([' -I %s' % mkarg(p) for p in global_options.include_path])
670        + ' -daux-files ')
671
672    global_options.formatter.process_options(global_options)
673
674    if global_options.lily_loglevel:
675        ly.debug_output(_("Setting LilyPond's loglevel to %s") %
676                        global_options.lily_loglevel, True)
677        global_options.process_cmd += " --loglevel=%s" % global_options.lily_loglevel
678    elif ly.is_verbose():
679        if os.environ.get("LILYPOND_LOGLEVEL", None):
680            ly.debug_output(_("Setting LilyPond's loglevel to %s (from environment variable LILYPOND_LOGLEVEL)") %
681                            os.environ.get("LILYPOND_LOGLEVEL", None), True)
682            global_options.process_cmd += " --loglevel=%s" % os.environ.get(
683                "LILYPOND_LOGLEVEL", None)
684        else:
685            ly.debug_output(
686                _("Setting LilyPond's output to --verbose, implied by lilypond-book's setting"), True)
687            global_options.process_cmd += " --verbose"
688
689    global_options.process_cmd += " -dread-file-list -dno-strip-output-dir"
690
691    # Store the original argument to construct the dependency file below.
692    relative_output_dir = global_options.output_dir
693
694    if global_options.output_dir:
695        global_options.output_dir = os.path.abspath(global_options.output_dir)
696        # Create the directory, but do not complain if it already exists.
697        os.makedirs(global_options.output_dir, exist_ok=True)
698    else:
699        global_options.output_dir = os.getcwd()
700
701    if global_options.lily_output_dir:
702        global_options.lily_output_dir = os.path.abspath(
703            global_options.lily_output_dir)
704    else:
705        global_options.lily_output_dir = global_options.output_dir
706
707    identify()
708    try:
709        chunks = do_file(files[0])
710        if global_options.filter_cmd:
711            write_output_documents(chunks, is_filter=True)
712        elif global_options.process_cmd:
713            do_process_cmd(chunks, global_options)
714            progress(_("Compiling `%s'...") % files[0])
715            write_output_documents(chunks, is_filter=False)
716    except book_snippets.CompileError:
717        exit(1)
718
719    inputs = note_input_file('')
720    inputs.pop()
721
722    base_file_name = os.path.splitext(os.path.basename(files[0]))[0]
723    dep_file = os.path.join(global_options.output_dir, base_file_name + '.dep')
724    final_output_file = os.path.join(relative_output_dir,
725                                     base_file_name + global_options.formatter.default_extension)
726    open(dep_file, 'w', encoding='utf8').write('%s: %s\n'
727                              % (final_output_file, ' '.join(inputs)))
728
729
730if __name__ == '__main__':
731    main()
732