1#!/usr/bin/env python
2
3import sys
4import os
5
6
7def report(msg):
8    sys.stderr.write(msg + "\n")
9
10
11def warn(msg):
12    sys.stderr.write("warning: %s\n" % msg)
13
14
15def ensure(condition, message):
16    if not condition:
17        raise Exception(message)
18
19
20def get_project_info(top=None):
21    """Load wscript to get project information (name, version, and so on)"""
22
23    import importlib
24    import importlib.machinery
25    import importlib.util
26
27    project_dir = top or os.getcwd()
28    wscript_path = os.path.join(project_dir, "wscript")
29    sys.path.insert(0, os.path.dirname(wscript_path))
30
31    loader = importlib.machinery.SourceFileLoader("wscript", wscript_path)
32    spec = importlib.util.spec_from_loader("wscript", loader)
33    wscript = importlib.util.module_from_spec(spec)
34
35    try:
36        spec.loader.exec_module(wscript)
37
38        info = {"name": wscript.APPNAME, "version": wscript.VERSION}
39
40        for key in ["uri", "title", "dist_pattern", "post_tags"]:
41            value = getattr(wscript, key, None)
42            if value is not None:
43                info[key] = value
44
45        if "title" not in info:
46            info["title"] = wscript.APPNAME.title()
47
48        return info
49
50    except Exception:
51        return {}
52
53
54def parse_version(revision):
55    """Convert semver string `revision` to a tuple of integers"""
56    return tuple(map(int, revision.split(".")))
57
58
59def is_release_version(version):
60    """Return true if `version` is a stable version number"""
61    if isinstance(version, tuple):
62        return version[len(version) - 1] % 2 == 0
63
64    return is_release_version(parse_version(version))
65
66
67def get_blurb(in_file):
68    """Get the first paragraph of a Markdown file"""
69    with open(in_file, "r") as f:
70        f.readline()  # Title
71        f.readline()  # Title underline
72        f.readline()  # Blank
73
74        out = ""
75        line = f.readline()
76        while len(line) > 0 and line != "\n":
77            out += line.replace("\n", " ")
78            line = f.readline()
79
80        return out.strip()
81
82
83def get_items_markdown(items, indent=""):
84    """Return a list of NEWS entries as a Markdown list"""
85    return "".join([indent + "* %s\n" % item for item in items])
86
87
88def get_release_json(title, entry):
89    """Return a release description in Gitlab JSON format"""
90    import json
91
92    version = entry["revision"]
93    desc = {
94        "name": "%s %s" % (title, version),
95        "tag_name": "v%s" % version,
96        "description": get_items_markdown(entry["items"]),
97        "released_at": entry["date"].isoformat(),
98    }
99
100    return json.dumps(desc)
101
102
103def read_text_news(in_file, preserve_timezones=False, dist_pattern=None):
104    """Read NEWS entries"""
105
106    import datetime
107    import email.utils
108    import re
109
110    entries = {}
111    with open(in_file, "r") as f:
112        while True:
113            # Read header line
114            head = f.readline()
115            matches = re.match(r"([^(]*) \(([0-9.]*)\) ([a-zA-z]*)", head)
116            if matches is None:
117                break
118
119            e = {
120                "name": matches.group(1),
121                "revision": matches.group(2),
122                "status": matches.group(3),
123                "items": [],
124            }
125
126            semver = parse_version(e["revision"])
127            if is_release_version(semver) and dist_pattern is not None:
128                e["dist"] = dist_pattern % semver
129
130            # Read blank line after header
131            if f.readline() != "\n":
132                raise SyntaxError("expected blank line after NEWS header")
133
134            def add_item(item):
135                if len(item) > 0:
136                    e["items"] += [item.replace("\n", " ").strip()]
137
138            # Read entries for this revision
139            item = ""
140            line = f.readline()
141            while line:
142                if line.startswith("  * "):
143                    add_item(item)
144                    item = line[3:].lstrip()
145                elif line == "\n":
146                    add_item(item)
147                    break
148                else:
149                    item += line.lstrip()
150
151                line = f.readline()
152
153            matches = re.match(r" -- (.*) <(.*)>  (.*)", f.readline())
154            date = email.utils.parsedate_to_datetime(matches.group(3))
155            if not preserve_timezones:
156                date = date.astimezone(datetime.timezone.utc)
157
158            e.update(
159                {
160                    "date": date,
161                    "blamee_name": matches.group(1),
162                    "blamee_mbox": matches.group(2),
163                }
164            )
165
166            entries[semver] = e
167
168            # Skip trailing blank line before next entry
169            space = f.readline()
170            if space != "\n" and space != "":
171                raise SyntaxError("expected blank line, not '%s'" % space)
172
173    return entries
174
175
176def write_text_news(entries, news):
177    """Write NEWS in standard Debian changelog format"""
178    import textwrap
179
180    revisions = sorted(entries.keys(), reverse=True)
181    for r in revisions:
182        e = entries[r]
183        summary = "%s (%s) %s" % (e["name"], e["revision"], e["status"])
184        news.write("\n" if r != revisions[0] else "")
185        news.write("%s;\n" % summary)
186
187        for item in e["items"]:
188            wrapped = textwrap.wrap(item, width=74)
189            news.write("\n  * " + "\n    ".join(wrapped))
190
191        email = e["blamee_mbox"].replace("mailto:", "")
192        author = "%s <%s>" % (e["blamee_name"], email)
193        date = e["date"].strftime("%a, %d %b %Y %H:%M:%S %z")
194        news.write("\n\n -- %s  %s\n" % (author, date))
195
196
197def read_ttl_news(name, in_files, dist_pattern=None):
198    """Read news entries from Turtle"""
199
200    import rdflib
201
202    doap = rdflib.Namespace("http://usefulinc.com/ns/doap#")
203    dcs = rdflib.Namespace("http://ontologi.es/doap-changeset#")
204    rdfs = rdflib.Namespace("http://www.w3.org/2000/01/rdf-schema#")
205    foaf = rdflib.Namespace("http://xmlns.com/foaf/0.1/")
206    rdf = rdflib.Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
207    g = rdflib.ConjunctiveGraph()
208
209    # Parse input files
210    for i in in_files:
211        g.parse(i, format="turtle")
212
213    proj = g.value(None, rdf.type, doap.Project)
214    for f in g.triples([proj, rdfs.seeAlso, None]):
215        if f[2].endswith(".ttl"):
216            g.parse(f[2], format="turtle")
217
218    def parse_datetime(date):
219        import datetime
220
221        try:
222            return datetime.datetime.strptime(date, "%Y-%m-%dT%H:%M:%S%z")
223        except Exception:
224            return datetime.datetime.strptime(date, "%Y-%m-%d")
225
226    entries = {}
227    for r in g.triples([proj, doap.release, None]):
228        release = r[2]
229        revision = g.value(release, doap.revision, None)
230        date = g.value(release, doap.created, None)
231        blamee = g.value(release, dcs.blame, None)
232        changeset = g.value(release, dcs.changeset, None)
233        dist = g.value(release, doap["file-release"], None)
234
235        semver = parse_version(revision)
236        if not dist:
237            if dist_pattern is not None:
238                dist = dist_pattern % semver
239            else:
240                warn("No file release for %s %s" % (proj, revision))
241
242        if revision and date and blamee and changeset:
243            status = "stable" if is_release_version(revision) else "unstable"
244            iso_date = parse_datetime(date)
245
246            e = {
247                "name": name,
248                "revision": str(revision),
249                "date": iso_date,
250                "status": status,
251                "items": [],
252            }
253
254            if dist is not None:
255                e["dist"] = dist
256
257            for i in g.triples([changeset, dcs.item, None]):
258                item = str(g.value(i[2], rdfs.label, None))
259                e["items"] += [item]
260
261            e["blamee_name"] = str(g.value(blamee, foaf.name, None))
262            e["blamee_mbox"] = str(g.value(blamee, foaf.mbox, None))
263
264            entries[semver] = e
265        else:
266            warn("Ignored incomplete %s release description" % name)
267
268    return entries
269
270
271def write_ttl_news(entries, out_file, template=None, subject_uri=None):
272    """Write NEWS in Turtle format"""
273    import rdflib
274    import rdflib.namespace
275    import rdflib.resource
276    import datetime
277
278    # Set up namespaces and make a graph for the output
279    doap = rdflib.Namespace("http://usefulinc.com/ns/doap#")
280    dcs = rdflib.Namespace("http://ontologi.es/doap-changeset#")
281    rdfs = rdflib.Namespace("http://www.w3.org/2000/01/rdf-schema#")
282    rdf = rdflib.Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
283    xsd = rdflib.Namespace("http://www.w3.org/2001/XMLSchema#")
284    g = rdflib.ConjunctiveGraph()
285    ns = rdflib.namespace.NamespaceManager(g)
286    ns.bind("doap", doap)
287    ns.bind("dcs", dcs)
288
289    # Load given template file
290    if template is not None:
291        g.load(template, format="turtle")
292
293    if subject_uri is not None:
294        # Use given subject uri
295        subject = rdflib.URIRef(subject_uri)
296        g.add((subject, rdf.type, doap.Project))
297    else:
298        # Find project URI to use as subject
299        subject = g.value(None, rdf.type, doap.Project)
300        ensure(subject is not None, "Unable to find project URI for subject")
301
302    # Get doap:name from first NEWS entry if it is not already present
303    if g.value(subject, doap.name, None) is None:
304        first_entry = next(iter(entries.values()))
305        g.add((subject, doap.name, rdflib.Literal(first_entry["name"])))
306
307    # Get maintainer
308    maintainer = g.value(subject, doap.maintainer, None)
309    if not maintainer:
310        maintainer = g.value(subject, doap.developer, None)
311
312    revisions = sorted(entries.keys(), reverse=True)
313    for r in revisions:
314        e = entries[r]
315        semver = parse_version(e["revision"])
316        ver_string = ("%03d" * len(semver)) % semver
317
318        release = rdflib.BNode("r%s" % ver_string)
319        g.add((subject, doap.release, release))
320        g.add((release, doap.revision, rdflib.Literal(e["revision"])))
321
322        if "dist" in e:
323            g.add((release, doap["file-release"], rdflib.URIRef(e["dist"])))
324
325        utc_date = e["date"].astimezone(datetime.timezone.utc)
326        date_str = utc_date.strftime("%Y-%m-%dT%H:%M:%S") + "Z"
327        time = rdflib.Literal(date_str, datatype=xsd.dateTime, normalize=False)
328        g.add((release, doap.created, time))
329
330        if maintainer is not None:
331            g.add((release, dcs.blame, maintainer))
332
333        changeset = rdflib.BNode("c%s" % ver_string)
334        g.add((release, dcs.changeset, changeset))
335        for index, item in enumerate(e["items"]):
336            item_node = rdflib.BNode("i%s%08d" % (ver_string, index))
337            g.add((changeset, dcs.item, item_node))
338            g.add((item_node, rdfs.label, rdflib.Literal(item)))
339
340    g.serialize(out_file, format="turtle")
341
342
343def read_news(path=None, format="NEWS", unsorted=False, utc=True, top=None):
344    """Read news in either text changelog or Turtle format"""
345
346    if format == "NEWS" and path is None:
347        path = os.path.join(top or "", "NEWS")
348
349    top = top or os.path.dirname(path)
350    info = get_project_info(top)
351    dist_pattern = info.get("dist_pattern", None)
352
353    if format == "NEWS":
354        entries = read_text_news(path, not utc, dist_pattern)
355    else:
356        ensure(path is not None, "Input path must be given for Turtle input")
357        entries = read_ttl_news(info["name"], [path])
358
359    if not unsorted:
360        for r, e in entries.items():
361            e["items"] = list(sorted(e["items"]))
362
363    return entries
364
365
366def write_news_file(entries, news, format, template, subject):
367    """Write news entries to a file object"""
368    if format == "NEWS":
369        write_text_news(entries, news)
370    else:
371        write_ttl_news(entries, news, template, subject)
372
373
374def write_news(entries, news, format="NEWS", template=None, subject=None):
375    """Write news entries to a file object or path"""
376    if isinstance(news, str):
377        with open(news, "w" if format == "NEWS" else "wb") as f:
378            write_news_file(entries, f, format, template, subject)
379    else:
380        write_news_file(entries, news, format, template, subject)
381
382
383def news_command():
384    ap = argparse.ArgumentParser(description="Generate NEWS file")
385    ap.add_argument("out_path", help="news output file")
386    ap.add_argument("--in-path", help="input file")
387    ap.add_argument("--unsorted", action="store_true", help="don't sort items")
388    ap.add_argument("--in-format", default="NEWS", choices=["NEWS", "turtle"])
389    ap.add_argument("--timezones", action="store_true", help="keep timezones")
390
391    args = ap.parse_args(sys.argv[2:])
392    entries = read_news(
393        args.in_path, args.in_format, args.unsorted, not args.timezones
394    )
395
396    with open(args.out_path, "w") as news:
397        write_news(entries, news)
398
399
400def ttl_news_command():
401    ap = argparse.ArgumentParser(description="Generate Turtle changeset")
402    ap.add_argument("--in-path", help="news input file")
403    ap.add_argument("out_path", help="news output file")
404    ap.add_argument("--template")
405    ap.add_argument("--unsorted", action="store_true", help="don't sort items")
406    ap.add_argument("--in-format", default="NEWS", choices=["NEWS", "turtle"])
407    ap.add_argument("--uri", help="project URI")
408
409    args = ap.parse_args(sys.argv[2:])
410    entries = read_news(args.in_path, args.in_format, args.unsorted, True)
411
412    uri = args.uri if args.uri else get_project_info()["uri"]
413    write_ttl_news(
414        entries, args.out_path, template=args.template, subject_uri=uri
415    )
416
417
418def write_posts(entries, out_dir, meta={}):
419    """Write news posts in Pelican Markdown format"""
420    import datetime
421
422    report("Writing posts to %s" % out_dir)
423
424    info = get_project_info()
425    description = get_blurb("README.md")
426    title = info["title"]
427    meta["Author"] = meta.get("Author", os.getenv("USER"))
428    if info["post_tags"]:
429        meta["Tags"] = ", ".join(info["post_tags"])
430
431    try:
432        os.mkdir(out_dir)
433    except Exception:
434        pass
435
436    for r, e in entries.items():
437        name = e["name"]
438        revision = e["revision"]
439        if "dist" not in e:
440            warn("No file release for %s %s" % (name, revision))
441            continue
442
443        date = e["date"].astimezone(datetime.timezone.utc)
444        date_str = date.strftime("%Y-%m-%d")
445        datetime_str = date.strftime("%Y-%m-%d %H:%M")
446        slug_version = revision.replace(".", "-")
447        filename = "%s-%s-%s.md" % (date_str, name, slug_version)
448
449        with open(os.path.join(out_dir, filename), "w") as post:
450            slug = "%s-%s" % (name, slug_version)
451            post.write("Title: %s %s\n" % (title, revision))
452            post.write("Date: %s\n" % datetime_str)
453            post.write("Slug: %s\n" % slug)
454            for k in sorted(meta.keys()):
455                post.write("%s: %s\n" % (k, meta[k]))
456
457            url = e["dist"]
458            link = "[%s %s](%s)" % (title, revision, url)
459            post.write("\n%s has been released." % link)
460            post.write("  " + description + "\n")
461
462            if e["items"] != ["Initial release"]:
463                post.write("\nChanges:\n\n")
464                post.write(get_items_markdown(e["items"], indent=" "))
465
466
467def posts_command():
468    ap = argparse.ArgumentParser(description="Generate Pelican posts")
469    ap.add_argument("out_dir", help="output directory")
470    ap.add_argument("--author", help="post author")
471    ap.add_argument("--in-path", help="input file")
472    ap.add_argument("--in-format", default="NEWS", choices=["NEWS", "turtle"])
473    ap.add_argument("--title", help="Title for posts")
474
475    args = ap.parse_args(sys.argv[2:])
476    entries = read_news(args.in_path, args.in_format)
477    meta = {"Author": args.author} if args.author else {}
478
479    write_posts(entries, args.out_dir, meta)
480
481
482def json_command():
483    ap = argparse.ArgumentParser(description="Get release description in JSON")
484    ap.add_argument("version", help="Version number")
485    ap.add_argument("--in-path", default="NEWS", help="input file")
486    ap.add_argument("--in-format", default="NEWS", choices=["NEWS", "turtle"])
487
488    args = ap.parse_args(sys.argv[2:])
489    info = get_project_info()
490    semver = parse_version(args.version)
491    entries = read_news(args.in_path, args.in_format)
492
493    print(get_release_json(info["title"], entries[semver]))
494
495
496def post_lab_release(version, lab, group, token, dry_run=False):
497    import shlex
498    import subprocess
499
500    def run_cmd(cmd):
501        if dry_run:
502            print(" ".join([shlex.quote(i) for i in cmd]))
503        else:
504            subprocess.check_call(cmd)
505
506    info = get_project_info()
507    name = info["name"]
508    title = info["title"]
509    semver = parse_version(version)
510    entries = read_news()
511    url = "https://%s/api/v4/projects/%s%%2F%s" % (lab, group, name)
512    dry_run = dry_run
513
514    # Check that this is a release version
515    ensure(is_release_version(semver), "%s is an unstable version" % version)
516
517    # Post Gitlab release
518    post_cmd = [
519        "curl",
520        "-XPOST",
521        "-HContent-Type: application/json",
522        "-HPRIVATE-TOKEN: " + token,
523        "-d" + get_release_json(title, entries[semver]),
524        "%s/releases" % url,
525    ]
526    run_cmd(post_cmd)
527
528    report("Posted Gitlab release %s %s" % (name, version))
529
530
531def post_lab_release_command():
532    ap = argparse.ArgumentParser(description="Post Gitlab release")
533    ap.add_argument("version", help="Version number")
534    ap.add_argument("group", help="Gitlab user or group for project")
535    ap.add_argument("token", help="Gitlab access token")
536    ap.add_argument("--lab", default="gitlab.com", help="Gitlab instance")
537    ap.add_argument("--dry-run", action="store_true", help="do nothing")
538    args = ap.parse_args(sys.argv[2:])
539
540    post_lab_release(
541        args.version, args.lab, args.group, args.token, args.dry_run
542    )
543
544
545def release(args, posts_dir=None, remote_dist_dir=None, dist_name=None):
546    import json
547    import os
548    import shlex
549    import subprocess
550
551    def run_cmd(cmd):
552        if args.dry_run:
553            print(" ".join([shlex.quote(i) for i in cmd]))
554        else:
555            subprocess.check_call(cmd)
556
557    info = get_project_info()
558    name = info["name"]
559    title = info["title"]
560    version = info["version"]
561    semver = parse_version(version)
562    dry_run = args.dry_run
563
564    # Check that this is a release version first of all
565    ensure(is_release_version(semver), "%s is an unstable version" % version)
566    report("Releasing %s %s" % (name, version))
567
568    # Check that NEWS is up to date
569    entries = read_news()
570    ensure(semver in entries, "%s has no NEWS entries" % version)
571
572    # Check that working copy is up to date
573    fetch_cmd = ["git", "fetch", "--dry-run"]
574    fetch_status = subprocess.check_output(fetch_cmd).decode("utf-8")
575    ensure(len(fetch_status) == 0, "Local copy is out of date")
576
577    # Remove distribution if one was already built
578    dist = "%s-%s.tar.bz2" % (dist_name or name.lower(), version)
579    sig = dist + ".sig"
580    try:
581        os.remove(dist)
582        os.remove(sig)
583    except Exception:
584        pass
585
586    # Check that working copy is clean
587    branch_cmd = ["git", "rev-parse", "--abbrev-ref", "HEAD"]
588    branch = subprocess.check_output(branch_cmd).decode("ascii").strip()
589    status_cmd = ["git", "status", "--porcelain", "-b"]
590    status = subprocess.check_output(status_cmd).decode("utf-8")
591    sys.stdout.write(status)
592    expected_status = "## %s...origin/%s\n" % (branch, branch)
593    ensure(status == expected_status, "Working copy is dirty")
594
595    # Fetch project description and ensure it matches
596    url = "https://%s/api/v4/projects/%s%%2F%s" % (args.lab, args.group, name)
597    desc_cmd = ["curl", "-HPRIVATE-TOKEN: " + args.token, url]
598    desc = json.loads(subprocess.check_output(desc_cmd))
599    proj_name = desc["name"]
600    ensure(proj_name == name, "Project name '%s' != '%s'" % (proj_name, name))
601
602    # Build distribution
603    run_cmd(["./waf", "configure", "--docs"])
604    run_cmd(["./waf", "build"])
605    run_cmd(["./waf", "distcheck"])
606    ensure(dry_run or os.path.exists(dist), "%s was not created" % dist)
607
608    # Sign distribution
609    run_cmd(["gpg", "-b", dist])
610    ensure(dry_run or os.path.exists(sig), "%s.sig was not created" % dist)
611    run_cmd(["gpg", "--verify", sig])
612
613    # Tag release
614    tag = "v" + version
615    run_cmd(["git", "tag", "-s", tag, "-m", "%s %s" % (title, version)])
616    run_cmd(["git", "push", "--tags"])
617
618    # Generate posts
619    if posts_dir is not None:
620        write_posts(entries, posts_dir)
621
622    # Upload distribution and signature
623    if remote_dist_dir is not None:
624        run_cmd(["scp", dist, os.path.join(remote_dist_dir, dist)])
625        run_cmd(["scp", sig, os.path.join(remote_dist_dir, sig)])
626
627    # Post Gitlab release
628    post_lab_release(version, args.lab, args.group, args.token, dry_run)
629
630    report("Released %s %s" % (name, version))
631    report("Remember to upload posts and push to other remotes!")
632
633
634def release_command():
635    ap = argparse.ArgumentParser(description="Release project")
636    ap.add_argument("group", help="Gitlab user or group for project")
637    ap.add_argument("token", help="Gitlab access token")
638    ap.add_argument("--lab", default="gitlab.com", help="Gitlab instance")
639    ap.add_argument("--dry-run", action="store_true", help="do nothing")
640    ap.add_argument("--posts", help="Pelican posts directory")
641    ap.add_argument("--scp", help="SSH path to distribution directory")
642    args = ap.parse_args(sys.argv[2:])
643
644    release(args, posts_dir=args.posts, remote_dist_dir=args.scp)
645
646
647if __name__ == "__main__":
648    import argparse
649
650    # Get list of command names from handler functions for help text
651    global_names = list(globals().keys())
652    handlers = [k[0:-8] for k in global_names if k.endswith("_command")]
653
654    # Run simple top level argument parser to get command name
655    ap = argparse.ArgumentParser(
656        description="Automatic release building",
657        epilog="commands: " + " ".join(handlers),
658    )
659    ap.add_argument("command", help="Subcommand to run")
660    args = ap.parse_args(sys.argv[1:2])
661
662    # Check that a handler is defined for the given command
663    function_name = args.command + "_command"
664    if function_name not in globals():
665        sys.stderr.write("error: Unknown command '%s'\n" % args.command)
666        ap.print_help()
667        sys.exit(1)
668
669    # Dispatch to command handler
670    globals()[function_name]()
671    sys.exit(0)
672