1#!/usr/bin/env python 2 3import sys 4import os 5 6 7def report(msg): 8 sys.stderr.write(msg + "\n") 9 10 11def warn(msg): 12 sys.stderr.write("warning: %s\n" % msg) 13 14 15def ensure(condition, message): 16 if not condition: 17 raise Exception(message) 18 19 20def get_project_info(top=None): 21 """Load wscript to get project information (name, version, and so on)""" 22 23 import importlib 24 import importlib.machinery 25 import importlib.util 26 27 project_dir = top or os.getcwd() 28 wscript_path = os.path.join(project_dir, "wscript") 29 sys.path.insert(0, os.path.dirname(wscript_path)) 30 31 loader = importlib.machinery.SourceFileLoader("wscript", wscript_path) 32 spec = importlib.util.spec_from_loader("wscript", loader) 33 wscript = importlib.util.module_from_spec(spec) 34 35 try: 36 spec.loader.exec_module(wscript) 37 38 info = {"name": wscript.APPNAME, "version": wscript.VERSION} 39 40 for key in ["uri", "title", "dist_pattern", "post_tags"]: 41 value = getattr(wscript, key, None) 42 if value is not None: 43 info[key] = value 44 45 if "title" not in info: 46 info["title"] = wscript.APPNAME.title() 47 48 return info 49 50 except Exception: 51 return {} 52 53 54def parse_version(revision): 55 """Convert semver string `revision` to a tuple of integers""" 56 return tuple(map(int, revision.split("."))) 57 58 59def is_release_version(version): 60 """Return true if `version` is a stable version number""" 61 if isinstance(version, tuple): 62 return version[len(version) - 1] % 2 == 0 63 64 return is_release_version(parse_version(version)) 65 66 67def get_blurb(in_file): 68 """Get the first paragraph of a Markdown file""" 69 with open(in_file, "r") as f: 70 f.readline() # Title 71 f.readline() # Title underline 72 f.readline() # Blank 73 74 out = "" 75 line = f.readline() 76 while len(line) > 0 and line != "\n": 77 out += line.replace("\n", " ") 78 line = f.readline() 79 80 return out.strip() 81 82 83def get_items_markdown(items, indent=""): 84 """Return a list of NEWS entries as a Markdown list""" 85 return "".join([indent + "* %s\n" % item for item in items]) 86 87 88def get_release_json(title, entry): 89 """Return a release description in Gitlab JSON format""" 90 import json 91 92 version = entry["revision"] 93 desc = { 94 "name": "%s %s" % (title, version), 95 "tag_name": "v%s" % version, 96 "description": get_items_markdown(entry["items"]), 97 "released_at": entry["date"].isoformat(), 98 } 99 100 return json.dumps(desc) 101 102 103def read_text_news(in_file, preserve_timezones=False, dist_pattern=None): 104 """Read NEWS entries""" 105 106 import datetime 107 import email.utils 108 import re 109 110 entries = {} 111 with open(in_file, "r") as f: 112 while True: 113 # Read header line 114 head = f.readline() 115 matches = re.match(r"([^(]*) \(([0-9.]*)\) ([a-zA-z]*)", head) 116 if matches is None: 117 break 118 119 e = { 120 "name": matches.group(1), 121 "revision": matches.group(2), 122 "status": matches.group(3), 123 "items": [], 124 } 125 126 semver = parse_version(e["revision"]) 127 if is_release_version(semver) and dist_pattern is not None: 128 e["dist"] = dist_pattern % semver 129 130 # Read blank line after header 131 if f.readline() != "\n": 132 raise SyntaxError("expected blank line after NEWS header") 133 134 def add_item(item): 135 if len(item) > 0: 136 e["items"] += [item.replace("\n", " ").strip()] 137 138 # Read entries for this revision 139 item = "" 140 line = f.readline() 141 while line: 142 if line.startswith(" * "): 143 add_item(item) 144 item = line[3:].lstrip() 145 elif line == "\n": 146 add_item(item) 147 break 148 else: 149 item += line.lstrip() 150 151 line = f.readline() 152 153 matches = re.match(r" -- (.*) <(.*)> (.*)", f.readline()) 154 date = email.utils.parsedate_to_datetime(matches.group(3)) 155 if not preserve_timezones: 156 date = date.astimezone(datetime.timezone.utc) 157 158 e.update( 159 { 160 "date": date, 161 "blamee_name": matches.group(1), 162 "blamee_mbox": matches.group(2), 163 } 164 ) 165 166 entries[semver] = e 167 168 # Skip trailing blank line before next entry 169 space = f.readline() 170 if space != "\n" and space != "": 171 raise SyntaxError("expected blank line, not '%s'" % space) 172 173 return entries 174 175 176def write_text_news(entries, news): 177 """Write NEWS in standard Debian changelog format""" 178 import textwrap 179 180 revisions = sorted(entries.keys(), reverse=True) 181 for r in revisions: 182 e = entries[r] 183 summary = "%s (%s) %s" % (e["name"], e["revision"], e["status"]) 184 news.write("\n" if r != revisions[0] else "") 185 news.write("%s;\n" % summary) 186 187 for item in e["items"]: 188 wrapped = textwrap.wrap(item, width=74) 189 news.write("\n * " + "\n ".join(wrapped)) 190 191 email = e["blamee_mbox"].replace("mailto:", "") 192 author = "%s <%s>" % (e["blamee_name"], email) 193 date = e["date"].strftime("%a, %d %b %Y %H:%M:%S %z") 194 news.write("\n\n -- %s %s\n" % (author, date)) 195 196 197def read_ttl_news(name, in_files, dist_pattern=None): 198 """Read news entries from Turtle""" 199 200 import rdflib 201 202 doap = rdflib.Namespace("http://usefulinc.com/ns/doap#") 203 dcs = rdflib.Namespace("http://ontologi.es/doap-changeset#") 204 rdfs = rdflib.Namespace("http://www.w3.org/2000/01/rdf-schema#") 205 foaf = rdflib.Namespace("http://xmlns.com/foaf/0.1/") 206 rdf = rdflib.Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#") 207 g = rdflib.ConjunctiveGraph() 208 209 # Parse input files 210 for i in in_files: 211 g.parse(i, format="turtle") 212 213 proj = g.value(None, rdf.type, doap.Project) 214 for f in g.triples([proj, rdfs.seeAlso, None]): 215 if f[2].endswith(".ttl"): 216 g.parse(f[2], format="turtle") 217 218 def parse_datetime(date): 219 import datetime 220 221 try: 222 return datetime.datetime.strptime(date, "%Y-%m-%dT%H:%M:%S%z") 223 except Exception: 224 return datetime.datetime.strptime(date, "%Y-%m-%d") 225 226 entries = {} 227 for r in g.triples([proj, doap.release, None]): 228 release = r[2] 229 revision = g.value(release, doap.revision, None) 230 date = g.value(release, doap.created, None) 231 blamee = g.value(release, dcs.blame, None) 232 changeset = g.value(release, dcs.changeset, None) 233 dist = g.value(release, doap["file-release"], None) 234 235 semver = parse_version(revision) 236 if not dist: 237 if dist_pattern is not None: 238 dist = dist_pattern % semver 239 else: 240 warn("No file release for %s %s" % (proj, revision)) 241 242 if revision and date and blamee and changeset: 243 status = "stable" if is_release_version(revision) else "unstable" 244 iso_date = parse_datetime(date) 245 246 e = { 247 "name": name, 248 "revision": str(revision), 249 "date": iso_date, 250 "status": status, 251 "items": [], 252 } 253 254 if dist is not None: 255 e["dist"] = dist 256 257 for i in g.triples([changeset, dcs.item, None]): 258 item = str(g.value(i[2], rdfs.label, None)) 259 e["items"] += [item] 260 261 e["blamee_name"] = str(g.value(blamee, foaf.name, None)) 262 e["blamee_mbox"] = str(g.value(blamee, foaf.mbox, None)) 263 264 entries[semver] = e 265 else: 266 warn("Ignored incomplete %s release description" % name) 267 268 return entries 269 270 271def write_ttl_news(entries, out_file, template=None, subject_uri=None): 272 """Write NEWS in Turtle format""" 273 import rdflib 274 import rdflib.namespace 275 import rdflib.resource 276 import datetime 277 278 # Set up namespaces and make a graph for the output 279 doap = rdflib.Namespace("http://usefulinc.com/ns/doap#") 280 dcs = rdflib.Namespace("http://ontologi.es/doap-changeset#") 281 rdfs = rdflib.Namespace("http://www.w3.org/2000/01/rdf-schema#") 282 rdf = rdflib.Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#") 283 xsd = rdflib.Namespace("http://www.w3.org/2001/XMLSchema#") 284 g = rdflib.ConjunctiveGraph() 285 ns = rdflib.namespace.NamespaceManager(g) 286 ns.bind("doap", doap) 287 ns.bind("dcs", dcs) 288 289 # Load given template file 290 if template is not None: 291 g.load(template, format="turtle") 292 293 if subject_uri is not None: 294 # Use given subject uri 295 subject = rdflib.URIRef(subject_uri) 296 g.add((subject, rdf.type, doap.Project)) 297 else: 298 # Find project URI to use as subject 299 subject = g.value(None, rdf.type, doap.Project) 300 ensure(subject is not None, "Unable to find project URI for subject") 301 302 # Get doap:name from first NEWS entry if it is not already present 303 if g.value(subject, doap.name, None) is None: 304 first_entry = next(iter(entries.values())) 305 g.add((subject, doap.name, rdflib.Literal(first_entry["name"]))) 306 307 # Get maintainer 308 maintainer = g.value(subject, doap.maintainer, None) 309 if not maintainer: 310 maintainer = g.value(subject, doap.developer, None) 311 312 revisions = sorted(entries.keys(), reverse=True) 313 for r in revisions: 314 e = entries[r] 315 semver = parse_version(e["revision"]) 316 ver_string = ("%03d" * len(semver)) % semver 317 318 release = rdflib.BNode("r%s" % ver_string) 319 g.add((subject, doap.release, release)) 320 g.add((release, doap.revision, rdflib.Literal(e["revision"]))) 321 322 if "dist" in e: 323 g.add((release, doap["file-release"], rdflib.URIRef(e["dist"]))) 324 325 utc_date = e["date"].astimezone(datetime.timezone.utc) 326 date_str = utc_date.strftime("%Y-%m-%dT%H:%M:%S") + "Z" 327 time = rdflib.Literal(date_str, datatype=xsd.dateTime, normalize=False) 328 g.add((release, doap.created, time)) 329 330 if maintainer is not None: 331 g.add((release, dcs.blame, maintainer)) 332 333 changeset = rdflib.BNode("c%s" % ver_string) 334 g.add((release, dcs.changeset, changeset)) 335 for index, item in enumerate(e["items"]): 336 item_node = rdflib.BNode("i%s%08d" % (ver_string, index)) 337 g.add((changeset, dcs.item, item_node)) 338 g.add((item_node, rdfs.label, rdflib.Literal(item))) 339 340 g.serialize(out_file, format="turtle") 341 342 343def read_news(path=None, format="NEWS", unsorted=False, utc=True, top=None): 344 """Read news in either text changelog or Turtle format""" 345 346 if format == "NEWS" and path is None: 347 path = os.path.join(top or "", "NEWS") 348 349 top = top or os.path.dirname(path) 350 info = get_project_info(top) 351 dist_pattern = info.get("dist_pattern", None) 352 353 if format == "NEWS": 354 entries = read_text_news(path, not utc, dist_pattern) 355 else: 356 ensure(path is not None, "Input path must be given for Turtle input") 357 entries = read_ttl_news(info["name"], [path]) 358 359 if not unsorted: 360 for r, e in entries.items(): 361 e["items"] = list(sorted(e["items"])) 362 363 return entries 364 365 366def write_news_file(entries, news, format, template, subject): 367 """Write news entries to a file object""" 368 if format == "NEWS": 369 write_text_news(entries, news) 370 else: 371 write_ttl_news(entries, news, template, subject) 372 373 374def write_news(entries, news, format="NEWS", template=None, subject=None): 375 """Write news entries to a file object or path""" 376 if isinstance(news, str): 377 with open(news, "w" if format == "NEWS" else "wb") as f: 378 write_news_file(entries, f, format, template, subject) 379 else: 380 write_news_file(entries, news, format, template, subject) 381 382 383def news_command(): 384 ap = argparse.ArgumentParser(description="Generate NEWS file") 385 ap.add_argument("out_path", help="news output file") 386 ap.add_argument("--in-path", help="input file") 387 ap.add_argument("--unsorted", action="store_true", help="don't sort items") 388 ap.add_argument("--in-format", default="NEWS", choices=["NEWS", "turtle"]) 389 ap.add_argument("--timezones", action="store_true", help="keep timezones") 390 391 args = ap.parse_args(sys.argv[2:]) 392 entries = read_news( 393 args.in_path, args.in_format, args.unsorted, not args.timezones 394 ) 395 396 with open(args.out_path, "w") as news: 397 write_news(entries, news) 398 399 400def ttl_news_command(): 401 ap = argparse.ArgumentParser(description="Generate Turtle changeset") 402 ap.add_argument("--in-path", help="news input file") 403 ap.add_argument("out_path", help="news output file") 404 ap.add_argument("--template") 405 ap.add_argument("--unsorted", action="store_true", help="don't sort items") 406 ap.add_argument("--in-format", default="NEWS", choices=["NEWS", "turtle"]) 407 ap.add_argument("--uri", help="project URI") 408 409 args = ap.parse_args(sys.argv[2:]) 410 entries = read_news(args.in_path, args.in_format, args.unsorted, True) 411 412 uri = args.uri if args.uri else get_project_info()["uri"] 413 write_ttl_news( 414 entries, args.out_path, template=args.template, subject_uri=uri 415 ) 416 417 418def write_posts(entries, out_dir, meta={}): 419 """Write news posts in Pelican Markdown format""" 420 import datetime 421 422 report("Writing posts to %s" % out_dir) 423 424 info = get_project_info() 425 description = get_blurb("README.md") 426 title = info["title"] 427 meta["Author"] = meta.get("Author", os.getenv("USER")) 428 if info["post_tags"]: 429 meta["Tags"] = ", ".join(info["post_tags"]) 430 431 try: 432 os.mkdir(out_dir) 433 except Exception: 434 pass 435 436 for r, e in entries.items(): 437 name = e["name"] 438 revision = e["revision"] 439 if "dist" not in e: 440 warn("No file release for %s %s" % (name, revision)) 441 continue 442 443 date = e["date"].astimezone(datetime.timezone.utc) 444 date_str = date.strftime("%Y-%m-%d") 445 datetime_str = date.strftime("%Y-%m-%d %H:%M") 446 slug_version = revision.replace(".", "-") 447 filename = "%s-%s-%s.md" % (date_str, name, slug_version) 448 449 with open(os.path.join(out_dir, filename), "w") as post: 450 slug = "%s-%s" % (name, slug_version) 451 post.write("Title: %s %s\n" % (title, revision)) 452 post.write("Date: %s\n" % datetime_str) 453 post.write("Slug: %s\n" % slug) 454 for k in sorted(meta.keys()): 455 post.write("%s: %s\n" % (k, meta[k])) 456 457 url = e["dist"] 458 link = "[%s %s](%s)" % (title, revision, url) 459 post.write("\n%s has been released." % link) 460 post.write(" " + description + "\n") 461 462 if e["items"] != ["Initial release"]: 463 post.write("\nChanges:\n\n") 464 post.write(get_items_markdown(e["items"], indent=" ")) 465 466 467def posts_command(): 468 ap = argparse.ArgumentParser(description="Generate Pelican posts") 469 ap.add_argument("out_dir", help="output directory") 470 ap.add_argument("--author", help="post author") 471 ap.add_argument("--in-path", help="input file") 472 ap.add_argument("--in-format", default="NEWS", choices=["NEWS", "turtle"]) 473 ap.add_argument("--title", help="Title for posts") 474 475 args = ap.parse_args(sys.argv[2:]) 476 entries = read_news(args.in_path, args.in_format) 477 meta = {"Author": args.author} if args.author else {} 478 479 write_posts(entries, args.out_dir, meta) 480 481 482def json_command(): 483 ap = argparse.ArgumentParser(description="Get release description in JSON") 484 ap.add_argument("version", help="Version number") 485 ap.add_argument("--in-path", default="NEWS", help="input file") 486 ap.add_argument("--in-format", default="NEWS", choices=["NEWS", "turtle"]) 487 488 args = ap.parse_args(sys.argv[2:]) 489 info = get_project_info() 490 semver = parse_version(args.version) 491 entries = read_news(args.in_path, args.in_format) 492 493 print(get_release_json(info["title"], entries[semver])) 494 495 496def post_lab_release(version, lab, group, token, dry_run=False): 497 import shlex 498 import subprocess 499 500 def run_cmd(cmd): 501 if dry_run: 502 print(" ".join([shlex.quote(i) for i in cmd])) 503 else: 504 subprocess.check_call(cmd) 505 506 info = get_project_info() 507 name = info["name"] 508 title = info["title"] 509 semver = parse_version(version) 510 entries = read_news() 511 url = "https://%s/api/v4/projects/%s%%2F%s" % (lab, group, name) 512 dry_run = dry_run 513 514 # Check that this is a release version 515 ensure(is_release_version(semver), "%s is an unstable version" % version) 516 517 # Post Gitlab release 518 post_cmd = [ 519 "curl", 520 "-XPOST", 521 "-HContent-Type: application/json", 522 "-HPRIVATE-TOKEN: " + token, 523 "-d" + get_release_json(title, entries[semver]), 524 "%s/releases" % url, 525 ] 526 run_cmd(post_cmd) 527 528 report("Posted Gitlab release %s %s" % (name, version)) 529 530 531def post_lab_release_command(): 532 ap = argparse.ArgumentParser(description="Post Gitlab release") 533 ap.add_argument("version", help="Version number") 534 ap.add_argument("group", help="Gitlab user or group for project") 535 ap.add_argument("token", help="Gitlab access token") 536 ap.add_argument("--lab", default="gitlab.com", help="Gitlab instance") 537 ap.add_argument("--dry-run", action="store_true", help="do nothing") 538 args = ap.parse_args(sys.argv[2:]) 539 540 post_lab_release( 541 args.version, args.lab, args.group, args.token, args.dry_run 542 ) 543 544 545def release(args, posts_dir=None, remote_dist_dir=None, dist_name=None): 546 import json 547 import os 548 import shlex 549 import subprocess 550 551 def run_cmd(cmd): 552 if args.dry_run: 553 print(" ".join([shlex.quote(i) for i in cmd])) 554 else: 555 subprocess.check_call(cmd) 556 557 info = get_project_info() 558 name = info["name"] 559 title = info["title"] 560 version = info["version"] 561 semver = parse_version(version) 562 dry_run = args.dry_run 563 564 # Check that this is a release version first of all 565 ensure(is_release_version(semver), "%s is an unstable version" % version) 566 report("Releasing %s %s" % (name, version)) 567 568 # Check that NEWS is up to date 569 entries = read_news() 570 ensure(semver in entries, "%s has no NEWS entries" % version) 571 572 # Check that working copy is up to date 573 fetch_cmd = ["git", "fetch", "--dry-run"] 574 fetch_status = subprocess.check_output(fetch_cmd).decode("utf-8") 575 ensure(len(fetch_status) == 0, "Local copy is out of date") 576 577 # Remove distribution if one was already built 578 dist = "%s-%s.tar.bz2" % (dist_name or name.lower(), version) 579 sig = dist + ".sig" 580 try: 581 os.remove(dist) 582 os.remove(sig) 583 except Exception: 584 pass 585 586 # Check that working copy is clean 587 branch_cmd = ["git", "rev-parse", "--abbrev-ref", "HEAD"] 588 branch = subprocess.check_output(branch_cmd).decode("ascii").strip() 589 status_cmd = ["git", "status", "--porcelain", "-b"] 590 status = subprocess.check_output(status_cmd).decode("utf-8") 591 sys.stdout.write(status) 592 expected_status = "## %s...origin/%s\n" % (branch, branch) 593 ensure(status == expected_status, "Working copy is dirty") 594 595 # Fetch project description and ensure it matches 596 url = "https://%s/api/v4/projects/%s%%2F%s" % (args.lab, args.group, name) 597 desc_cmd = ["curl", "-HPRIVATE-TOKEN: " + args.token, url] 598 desc = json.loads(subprocess.check_output(desc_cmd)) 599 proj_name = desc["name"] 600 ensure(proj_name == name, "Project name '%s' != '%s'" % (proj_name, name)) 601 602 # Build distribution 603 run_cmd(["./waf", "configure", "--docs"]) 604 run_cmd(["./waf", "build"]) 605 run_cmd(["./waf", "distcheck"]) 606 ensure(dry_run or os.path.exists(dist), "%s was not created" % dist) 607 608 # Sign distribution 609 run_cmd(["gpg", "-b", dist]) 610 ensure(dry_run or os.path.exists(sig), "%s.sig was not created" % dist) 611 run_cmd(["gpg", "--verify", sig]) 612 613 # Tag release 614 tag = "v" + version 615 run_cmd(["git", "tag", "-s", tag, "-m", "%s %s" % (title, version)]) 616 run_cmd(["git", "push", "--tags"]) 617 618 # Generate posts 619 if posts_dir is not None: 620 write_posts(entries, posts_dir) 621 622 # Upload distribution and signature 623 if remote_dist_dir is not None: 624 run_cmd(["scp", dist, os.path.join(remote_dist_dir, dist)]) 625 run_cmd(["scp", sig, os.path.join(remote_dist_dir, sig)]) 626 627 # Post Gitlab release 628 post_lab_release(version, args.lab, args.group, args.token, dry_run) 629 630 report("Released %s %s" % (name, version)) 631 report("Remember to upload posts and push to other remotes!") 632 633 634def release_command(): 635 ap = argparse.ArgumentParser(description="Release project") 636 ap.add_argument("group", help="Gitlab user or group for project") 637 ap.add_argument("token", help="Gitlab access token") 638 ap.add_argument("--lab", default="gitlab.com", help="Gitlab instance") 639 ap.add_argument("--dry-run", action="store_true", help="do nothing") 640 ap.add_argument("--posts", help="Pelican posts directory") 641 ap.add_argument("--scp", help="SSH path to distribution directory") 642 args = ap.parse_args(sys.argv[2:]) 643 644 release(args, posts_dir=args.posts, remote_dist_dir=args.scp) 645 646 647if __name__ == "__main__": 648 import argparse 649 650 # Get list of command names from handler functions for help text 651 global_names = list(globals().keys()) 652 handlers = [k[0:-8] for k in global_names if k.endswith("_command")] 653 654 # Run simple top level argument parser to get command name 655 ap = argparse.ArgumentParser( 656 description="Automatic release building", 657 epilog="commands: " + " ".join(handlers), 658 ) 659 ap.add_argument("command", help="Subcommand to run") 660 args = ap.parse_args(sys.argv[1:2]) 661 662 # Check that a handler is defined for the given command 663 function_name = args.command + "_command" 664 if function_name not in globals(): 665 sys.stderr.write("error: Unknown command '%s'\n" % args.command) 666 ap.print_help() 667 sys.exit(1) 668 669 # Dispatch to command handler 670 globals()[function_name]() 671 sys.exit(0) 672