1#!@TARGET_PYTHON@ 2# -*- coding: utf-8 -*- 3 4# This file is part of LilyPond, the GNU music typesetter. 5# 6# Copyright (C) 1998--2021 Han-Wen Nienhuys <hanwen@xs4all.nl> 7# Jan Nieuwenhuizen <janneke@gnu.org> 8# 9# LilyPond is free software: you can redistribute it and/or modify 10# it under the terms of the GNU General Public License as published by 11# the Free Software Foundation, either version 3 of the License, or 12# (at your option) any later version. 13# 14# LilyPond is distributed in the hope that it will be useful, 15# but WITHOUT ANY WARRANTY; without even the implied warranty of 16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17# GNU General Public License for more details. 18# 19# You should have received a copy of the GNU General Public License 20# along with LilyPond. If not, see <http://www.gnu.org/licenses/>. 21 22r''' 23Example usage: 24 25test: 26 lilypond-book --filter="tr '[a-z]' '[A-Z]'" BOOK 27 28convert-ly on book: 29 lilypond-book --filter="convert-ly --no-version --from=1.6.11 -" BOOK 30 31classic lilypond-book: 32 lilypond-book --process="lilypond" BOOK.tely 33 34TODO: 35 36 * ly-options: intertext? 37 * --line-width? 38 * eps in latex / eps by lilypond -b ps? 39 * check latex parameters, twocolumn, multicolumn? 40 * use --png --ps --pdf for making images? 41 42 * Converting from lilypond-book source, substitute: 43 @mbinclude foo.itely -> @include foo.itely 44 \mbinput -> \input 45 46''' 47 48 49# TODO: Better solve the global_options copying to the snippets... 50 51import gettext 52import glob 53import hashlib 54from optparse import OptionGroup 55import os 56import re 57import shlex 58import stat 59import subprocess 60import sys 61import tempfile 62import typing 63 64# See lock_path and unlock_path; this module is not available at all on Windows. 65if os.name == 'posix': 66 import fcntl 67 68""" 69@relocate-preamble@ 70""" 71 72import book_base 73import book_docbook 74import book_html 75import book_latex 76import book_texinfo 77import book_snippets 78 79# Load translation and install _() into Python's builtins namespace. 80gettext.install('lilypond', '@localedir@') 81 82import lilylib as ly 83 84backend = 'ps' 85 86help_summary = ( 87 _("Process LilyPond snippets in hybrid HTML, LaTeX, texinfo or DocBook document.") 88 + '\n\n' 89 + _("Examples:") 90 + ''' 91 $ lilypond-book --filter="tr '[a-z]' '[A-Z]'" %(BOOK)s 92 $ lilypond-book -F "convert-ly --no-version --from=2.0.0 -" %(BOOK)s 93 $ lilypond-book --process='lilypond -I include' %(BOOK)s 94''' % {'BOOK': _("BOOK")}) 95 96authors = ('Jan Nieuwenhuizen <janneke@gnu.org>', 97 'Han-Wen Nienhuys <hanwen@xs4all.nl>') 98 99################################################################ 100 101 102def exit(i): 103 if ly.is_verbose(): 104 raise Exception(_('Exiting (%d)...') % i) 105 else: 106 sys.exit(i) 107 108 109progress = ly.progress 110warning = ly.warning 111error = ly.error 112 113program_version = '@TOPLEVEL_VERSION@' 114if program_version.startswith("@"): 115 # '@' in lilypond-book output confuses texinfo 116 program_version = "dev" 117 118 119def identify(): 120 progress('%s (GNU LilyPond) %s' % (ly.program_name, program_version)) 121 122 123def warranty(): 124 identify() 125 sys.stdout.write(''' 126%s 127 128 %s 129 130%s 131%s 132''' % (_('Copyright (c) %s by') % '2001--2021', 133 '\n '.join(authors), 134 _("Distributed under terms of the GNU General Public License."), 135 _("It comes with NO WARRANTY."))) 136 137 138def get_option_parser(): 139 p = ly.get_option_parser(usage=_("%s [OPTION]... FILE") % 'lilypond-book', 140 description=help_summary, 141 conflict_handler="resolve", 142 add_help_option=False) 143 144 p.add_option('-F', '--filter', 145 help=_("pipe snippets through FILTER " 146 "[default: `convert-ly -n -']"), 147 metavar=_("FILTER"), 148 action="store", 149 dest="filter_cmd", 150 default=None) 151 152 p.add_option('-f', '--format', 153 help=_("use output format FORMAT (texi [default], " 154 "texi-html, latex, html, docbook)"), 155 metavar=_("FORMAT"), 156 action='store') 157 158 p.add_option("-h", "--help", 159 action="help", 160 help=_("show this help and exit")) 161 162 p.add_option("-I", '--include', 163 help=_("add DIR to include path"), 164 metavar=_("DIR"), 165 action='append', 166 dest='include_path', 167 default=[]) 168 169 p.add_option('--info-images-dir', 170 help=_("format Texinfo output so that Info will " 171 "look for images of music in DIR"), 172 metavar=_("DIR"), 173 action='store', 174 dest='info_images_dir', 175 default='') 176 177 p.add_option('--left-padding', 178 help=_("pad left side of music to align music in spite " 179 "of uneven bar numbers (in mm) [default: %default]"), 180 metavar=_("PAD"), 181 dest="padding_mm", 182 type="float", 183 default=3.0) 184 185 p.add_option('--lily-loglevel', 186 help=_("print lilypond log messages according to LOGLEVEL " 187 "[default: %default]"), 188 metavar=_("LOGLEVEL"), 189 action='store', 190 dest='lily_loglevel', 191 default=os.environ.get("LILYPOND_LOGLEVEL", None)) 192 193 p.add_option('--lily-output-dir', 194 help=_("write lily-XXX files to DIR, " 195 "link into --output dir"), 196 metavar=_("DIR"), 197 action='store', 198 dest='lily_output_dir', 199 default=None) 200 201 p.add_option("-l", "--loglevel", 202 help=_("print log messages according to LOGLEVEL " 203 "(NONE, ERROR, WARNING, PROGRESS [default], DEBUG)"), 204 metavar=_("LOGLEVEL"), 205 action='callback', 206 callback=ly.handle_loglevel_option, 207 type='string') 208 209 p.add_option("-o", '--output', 210 help=_("write output to DIR"), 211 metavar=_("DIR"), 212 action='store', 213 dest='output_dir', 214 default='') 215 216 p.add_option('-P', '--process', 217 help=_("process ly_files using COMMAND FILE..."), 218 metavar=_("COMMAND"), 219 action='store', 220 dest='process_cmd', 221 default='') 222 223 p.add_option('--redirect-lilypond-output', 224 help=_("redirect the lilypond output"), 225 action='store_true', 226 dest='redirect_output', 227 default=False) 228 229 p.add_option('-s', '--safe', 230 help=_("compile snippets in safe mode"), 231 action="store_true", 232 dest="safe_mode", 233 default=False) 234 235 p.add_option('--skip-lily-check', 236 help=_("do not fail if no lilypond output is found"), 237 metavar=_("DIR"), 238 action='store_true', 239 dest='skip_lilypond_run', 240 default=False) 241 242 p.add_option('--skip-png-check', 243 help=_("do not fail if no PNG images " 244 "are found for EPS files"), 245 metavar=_("DIR"), 246 action='store_true', 247 dest='skip_png_check', 248 default=False) 249 250 p.add_option('--use-source-file-names', 251 help=_("write snippet output files with the same " 252 "base name as their source file"), 253 action='store_true', 254 dest='use_source_file_names', 255 default=False) 256 257 p.add_option('-V', '--verbose', 258 help=_("be verbose"), 259 action="callback", 260 callback=ly.handle_loglevel_option, 261 callback_args=("DEBUG",)) 262 263 p.version = "@TOPLEVEL_VERSION@" 264 p.add_option("--version", 265 help=_("show version number and exit"), 266 action="version") 267 268 p.add_option('-w', '--warranty', 269 help=_("show warranty and copyright"), 270 action='store_true') 271 272 group = OptionGroup(p, "Options only for the latex and texinfo backends") 273 group.add_option('--latex-program', 274 help=_("run executable PROG instead of latex or, " 275 "in case --pdf option is set, " 276 "instead of pdflatex"), 277 metavar=_("PROG"), 278 action='store', 279 dest='latex_program', 280 default='latex') 281 group.add_option('--texinfo-program', 282 help=_("run executable PROG instead of texi2pdf"), 283 metavar=_("PROG"), 284 action='store', 285 dest='texinfo_program', 286 default='texi2pdf') 287 group.add_option('--pdf', 288 help=_("create PDF files for use with pdftex"), 289 action="store_true", 290 dest="create_pdf", 291 default=False) 292 p.add_option_group(group) 293 294 p.add_option_group('', 295 description=( 296 _("Report bugs via %s") 297 % 'bug-lilypond@gnu.org') + '\n') 298 299 for formatter in book_base.all_formats: 300 formatter.add_options(p) 301 302 return p 303 304 305lilypond_binary = os.path.join('@bindir@', 'lilypond') 306 307# If we are called with full path, try to use lilypond binary 308# installed in the same path; this is needed in GUB binaries, where 309# @bindir is always different from the installed binary path. 310if 'bindir' in globals() and bindir: 311 lilypond_binary = os.path.join(bindir, 'lilypond') 312 313# Only use installed binary when we are installed too. 314if '@bindir@' == ('@' + 'bindir@') or not os.path.exists(lilypond_binary): 315 lilypond_binary = 'lilypond' 316 317# Need to shell-quote, issue 3468 318# FIXME: we should really pass argument lists 319# everywhere instead of playing with shell syntax. 320lilypond_binary = shlex.quote(lilypond_binary) 321 322global_options = None 323 324 325def command_name(cmd): 326 # Strip all stuf after command, 327 # deal with "((latex ) >& 1 ) .." too 328 cmd = re.match(r'([\(\)]*)([^\\ ]*)', cmd).group(2) 329 return os.path.basename(cmd) 330 331 332def system_in_directory(cmd_str, directory, log_file): 333 """Execute a command in a different directory.""" 334 335 if ly.is_verbose(): 336 ly.progress(_("Invoking `%s\'") % cmd_str) 337 elif global_options.redirect_output: 338 ly.progress(_("Processing %s.ly") % log_file) 339 else: 340 name = command_name(cmd_str) 341 ly.progress(_("Running %s...") % name) 342 343 output_location = None 344 if global_options.redirect_output: 345 output_location = open(log_file + '.log', 'w', encoding='utf8') 346 347 try: 348 subprocess.run(cmd_str, stdout=output_location, 349 stderr=output_location, cwd=directory, 350 shell=True, check=True) 351 except subprocess.CalledProcessError as e: 352 sys.stderr.write("%s\n" % e) 353 sys.exit(1) 354 355 356def process_snippets(cmd, outdated_dict, 357 formatter, lily_output_dir): 358 """Run cmd on all of the .ly files from snippets.""" 359 basenames = sorted(outdated_dict.keys()) 360 361 # No need for a secure hash function, just need a digest. 362 checksum = hashlib.md5() 363 for name in basenames: 364 checksum.update(name.encode('ascii')) 365 checksum = checksum.hexdigest() 366 367 lily_output_dir = global_options.lily_output_dir 368 369 # Write list of snippet names. 370 snippet_names_file = 'snippet-names-%s.ly' % checksum 371 snippet_names_path = os.path.join(lily_output_dir, snippet_names_file) 372 with open(snippet_names_path, 'w', encoding='utf8') as snippet_names: 373 snippet_names.write('\n'.join([name + '.ly' for name in basenames])) 374 375 # Run command. 376 cmd = formatter.adjust_snippet_command(cmd) 377 # Remove .ly ending. 378 logfile = os.path.splitext(snippet_names_path)[0] 379 snippet_names_arg = mkarg(snippet_names_path.replace(os.path.sep, '/')) 380 system_in_directory(' '.join([cmd, snippet_names_arg]), 381 lily_output_dir, 382 logfile) 383 os.unlink(snippet_names_path) 384 385 386def lock_path(name): 387 if os.name != 'posix': 388 return None 389 390 fp = open(name, 'w', encoding='utf8') 391 fcntl.lockf(fp, fcntl.LOCK_EX) 392 return fp 393 394 395def unlock_path(lock): 396 if os.name != 'posix': 397 return None 398 fcntl.lockf(lock, fcntl.LOCK_UN) 399 lock.close() 400 401 402def do_process_cmd(chunks, options): 403 """Wrap do_process_cmd_locked in a filesystem lock""" 404 snippets = [c for c in chunks if isinstance( 405 c, book_snippets.LilypondSnippet)] 406 407 # calculate checksums eagerly 408 for s in snippets: 409 s.get_checksum() 410 411 os.makedirs(options.lily_output_dir, exist_ok=True) 412 lock_file = os.path.join(options.lily_output_dir, "lock") 413 lock = None 414 try: 415 lock = lock_path(lock_file) 416 do_process_cmd_locked(snippets, options) 417 finally: 418 if lock: 419 unlock_path(lock) 420 421 422def do_process_cmd_locked(snippets, options): 423 """Look at all snippets, write the outdated ones, and compile them.""" 424 outdated = [c for c in snippets if c.is_outdated(options.lily_output_dir)] 425 426 if outdated: 427 # First unique the list based on the basename, by using them as keys 428 # in a dict. 429 outdated_dict = dict() 430 for snippet in outdated: 431 outdated_dict[snippet.basename()] = snippet 432 433 # Next call write_ly() for each snippet once. 434 progress(_("Writing snippets...")) 435 for snippet in outdated_dict.values(): 436 snippet.write_ly() 437 438 progress(_("Processing...")) 439 process_snippets(options.process_cmd, outdated_dict, 440 options.formatter, options.lily_output_dir) 441 442 else: 443 progress(_("All snippets are up to date...")) 444 445 progress(_("Linking files...")) 446 if options.lily_output_dir != options.output_dir: 447 for snippet in snippets: 448 snippet.link_all_output_files(options.lily_output_dir, 449 options.output_dir) 450 451 452### 453# Format guessing data 454 455def guess_format(input_filename): 456 format = None 457 e = os.path.splitext(input_filename)[1] 458 for formatter in book_base.all_formats: 459 if formatter.can_handle_extension(e): 460 return formatter 461 error(_("cannot determine format for: %s" % input_filename)) 462 exit(1) 463 464def write_if_updated(file_name, lines): 465 try: 466 with open(file_name, encoding='utf-8') as file: 467 old_str = file.read() 468 except FileNotFoundError: 469 pass 470 else: 471 new_str = ''.join(lines) 472 if old_str == new_str: 473 progress(_("%s is up to date.") % file_name) 474 475 # this prevents make from always rerunning lilypond-book: 476 # output file must be touched in order to be up to date 477 os.utime(file_name, None) 478 return 479 480 output_dir = os.path.dirname(file_name) 481 os.makedirs(output_dir, exist_ok=True) 482 483 progress(_("Writing `%s'...") % file_name) 484 open(file_name, 'w', encoding='utf-8').writelines(lines) 485 486 487def note_input_file(name, inputs=[]): 488 # hack: inputs is mutable! 489 inputs.append(name) 490 return inputs 491 492 493def samefile(f1, f2): 494 try: 495 return os.path.samefile(f1, f2) 496 except AttributeError: # Windoze 497 f1 = re.sub("//*", "/", f1) 498 f2 = re.sub("//*", "/", f2) 499 return f1 == f2 500 501 502def do_file(input_filename, included=False): 503 # Ugh. 504 input_absname = input_filename 505 if not input_filename or input_filename == '-': 506 in_handle = sys.stdin 507 else: 508 if os.path.exists(input_filename): 509 input_fullname = input_filename 510 else: 511 input_fullname = global_options.formatter.input_fullname( 512 input_filename) 513 # Normalize path to absolute path, since we will change cwd to the output dir! 514 # Otherwise, "lilypond-book -o out test.tex" will complain that it is 515 # overwriting the input file (which it is actually not), since the 516 # input filename is relative to the CWD... 517 input_absname = os.path.abspath(input_fullname) 518 519 note_input_file(input_fullname) 520 in_handle = open(input_fullname, 'r', encoding='utf-8') 521 522 if input_filename == '-': 523 global_options.input_dir = os.getcwd() 524 input_base = 'stdin' 525 elif included: 526 input_base = os.path.splitext(input_filename)[0] 527 else: 528 global_options.input_dir = os.path.split(input_absname)[0] 529 input_base = os.path.basename( 530 os.path.splitext(input_filename)[0]) 531 532 output_filename = os.path.join(global_options.output_dir, 533 input_base + global_options.formatter.default_extension) 534 if (os.path.exists(input_filename) 535 and os.path.exists(output_filename) 536 and samefile(output_filename, input_absname)): 537 error( 538 _("Output would overwrite input file; use --output.")) 539 exit(2) 540 541 try: 542 progress(_("Reading `%s'") % input_absname) 543 source = in_handle.read() 544 545 if not included: 546 global_options.formatter.init_default_snippet_options(source) 547 548 progress(_("Dissecting...")) 549 chunks = book_base.find_toplevel_snippets( 550 source, global_options.formatter, global_options) 551 for c in chunks: 552 c.set_output_fullpath(output_filename) 553 554 # Let the formatter modify the chunks before further processing 555 chunks = global_options.formatter.process_chunks(chunks) 556 557 def process_include(snippet): 558 name = snippet.substring('filename') 559 progress(_("Processing include `%s'") % name) 560 return do_file(name, included=True) 561 562 include_chunks = [] 563 for x in chunks: 564 if isinstance(x, book_snippets.IncludeSnippet): 565 include_chunks += process_include(x) 566 567 return chunks + include_chunks 568 569 except book_snippets.CompileError: 570 progress(_("Removing `%s'") % output_filename) 571 raise book_snippets.CompileError 572 573 574def do_options(): 575 global global_options 576 577 opt_parser = get_option_parser() 578 (global_options, args) = opt_parser.parse_args() 579 580 global_options.information = { 581 'program_version': program_version, 'program_name': ly.program_name} 582 583 if global_options.lily_output_dir: 584 global_options.lily_output_dir = os.path.expanduser( 585 global_options.lily_output_dir) 586 if global_options.output_dir: 587 global_options.output_dir = os.path.expanduser( 588 global_options.output_dir) 589 590 # Compute absolute paths of include directories. 591 for i, path in enumerate(global_options.include_path): 592 global_options.include_path[i] = os.path.abspath(path) 593 594 # Append the current directory. 595 global_options.include_path.append(os.getcwd()) 596 597 if global_options.warranty: 598 warranty() 599 exit(0) 600 if not args or len(args) > 1: 601 opt_parser.print_help() 602 exit(2) 603 604 return args 605 606 607def mkarg(x): 608 r""" 609 A modified version of the commands.mkarg(x) 610 611 Uses double quotes (since Windows can't handle the single quotes) 612 and escapes the characters \, $, ", and ` for unix shells. 613 """ 614 if os.name == 'nt': 615 return ' "%s"' % x 616 s = ' "' 617 for c in x: 618 if c in '\\$"`': 619 s = s + '\\' 620 s = s + c 621 s = s + '"' 622 return s 623 624 625def write_output_documents(chunks: typing.List[book_snippets.Chunk], is_filter: bool): 626 text_by_path = {} 627 for ch in chunks: 628 path = ch.output_fullpath() 629 if path not in text_by_path: 630 text_by_path[path] = [] 631 632 if is_filter: 633 s = ch.filter_text() 634 else: 635 s = ch.replacement_text() 636 637 text_by_path[path].append(s) 638 639 for path in text_by_path: 640 write_if_updated(path, text_by_path[path]) 641 642 643def main(): 644 if "LILYPOND_BOOK_LOGLEVEL" in os.environ: 645 ly.set_loglevel(os.environ["LILYPOND_BOOK_LOGLEVEL"]) 646 files = do_options() 647 648 basename = os.path.splitext(files[0])[0] 649 basename = os.path.split(basename)[1] 650 651 if global_options.format: 652 # Retrieve the formatter for the given format 653 for formatter in book_base.all_formats: 654 if formatter.can_handle_format(global_options.format): 655 global_options.formatter = formatter 656 else: 657 global_options.formatter = guess_format(files[0]) 658 global_options.format = global_options.formatter.format 659 660 # make the global options available to the formatters: 661 global_options.formatter.global_options = global_options 662 formats = global_options.formatter.image_formats 663 664 if global_options.process_cmd == '': 665 global_options.process_cmd = ( 666 lilypond_binary + ' --formats=%s ' % formats) 667 668 global_options.process_cmd += ( 669 ' '.join([' -I %s' % mkarg(p) for p in global_options.include_path]) 670 + ' -daux-files ') 671 672 global_options.formatter.process_options(global_options) 673 674 if global_options.lily_loglevel: 675 ly.debug_output(_("Setting LilyPond's loglevel to %s") % 676 global_options.lily_loglevel, True) 677 global_options.process_cmd += " --loglevel=%s" % global_options.lily_loglevel 678 elif ly.is_verbose(): 679 if os.environ.get("LILYPOND_LOGLEVEL", None): 680 ly.debug_output(_("Setting LilyPond's loglevel to %s (from environment variable LILYPOND_LOGLEVEL)") % 681 os.environ.get("LILYPOND_LOGLEVEL", None), True) 682 global_options.process_cmd += " --loglevel=%s" % os.environ.get( 683 "LILYPOND_LOGLEVEL", None) 684 else: 685 ly.debug_output( 686 _("Setting LilyPond's output to --verbose, implied by lilypond-book's setting"), True) 687 global_options.process_cmd += " --verbose" 688 689 global_options.process_cmd += " -dread-file-list -dno-strip-output-dir" 690 691 # Store the original argument to construct the dependency file below. 692 relative_output_dir = global_options.output_dir 693 694 if global_options.output_dir: 695 global_options.output_dir = os.path.abspath(global_options.output_dir) 696 # Create the directory, but do not complain if it already exists. 697 os.makedirs(global_options.output_dir, exist_ok=True) 698 else: 699 global_options.output_dir = os.getcwd() 700 701 if global_options.lily_output_dir: 702 global_options.lily_output_dir = os.path.abspath( 703 global_options.lily_output_dir) 704 else: 705 global_options.lily_output_dir = global_options.output_dir 706 707 identify() 708 try: 709 chunks = do_file(files[0]) 710 if global_options.filter_cmd: 711 write_output_documents(chunks, is_filter=True) 712 elif global_options.process_cmd: 713 do_process_cmd(chunks, global_options) 714 progress(_("Compiling `%s'...") % files[0]) 715 write_output_documents(chunks, is_filter=False) 716 except book_snippets.CompileError: 717 exit(1) 718 719 inputs = note_input_file('') 720 inputs.pop() 721 722 base_file_name = os.path.splitext(os.path.basename(files[0]))[0] 723 dep_file = os.path.join(global_options.output_dir, base_file_name + '.dep') 724 final_output_file = os.path.join(relative_output_dir, 725 base_file_name + global_options.formatter.default_extension) 726 open(dep_file, 'w', encoding='utf8').write('%s: %s\n' 727 % (final_output_file, ' '.join(inputs))) 728 729 730if __name__ == '__main__': 731 main() 732