1#!/usr/bin/python3 2# -*- coding: utf-8 -*- 3 4# Generates the CP2K Dashboard html page 5# Inspired by Iain's cp2k_page_update.sh 6# 7# author: Ole Schuett 8 9import sys 10import os 11import smtplib 12from email.mime.text import MIMEText 13import html 14import re 15import gzip 16from datetime import datetime, timedelta 17from subprocess import check_output 18import traceback 19from os import path 20from pprint import pformat 21from glob import glob 22import itertools 23import configparser 24import pickle 25from pathlib import Path 26import requests 27import hashlib 28from collections import OrderedDict 29 30import matplotlib as mpl 31 32mpl.use("Agg") # change backend, to run without X11 33import matplotlib.pyplot as plt 34from matplotlib.ticker import AutoMinorLocator 35 36# =============================================================================== 37class GitLog(list): 38 def __init__(self): 39 cmd = ["git", "log", "--pretty=format:%H%n%ct%n%an%n%ae%n%s%n%b<--separator-->"] 40 outbytes = check_output(cmd) 41 output = outbytes.decode("utf-8", errors="replace") 42 for entry in output.split("<--separator-->")[:-1]: 43 lines = entry.strip().split("\n") 44 commit = dict() 45 commit["git-sha"] = lines[0] 46 commit["date"] = datetime.fromtimestamp(float(lines[1])) 47 commit["author-name"] = lines[2] 48 commit["author-email"] = lines[3] 49 commit["msg"] = lines[4] 50 self.append(commit) 51 52 # git-log outputs entries from new to old. 53 self.index = {c["git-sha"]: i for i, c in enumerate(self)} 54 55 56# =============================================================================== 57def main(): 58 if len(sys.argv) < 4: 59 print( 60 "Usage update_dashboard.py <config-file> <status-file> <output-dir> [--send-emails]" 61 ) 62 sys.exit(1) 63 64 config_fn, status_fn, outdir = sys.argv[1:4] 65 assert outdir.endswith("/") 66 assert path.exists(config_fn) 67 68 global send_emails 69 if len(sys.argv) == 5: 70 assert sys.argv[4] == "--send-emails" 71 send_emails = True 72 else: 73 send_emails = False 74 75 config = configparser.ConfigParser() 76 config.read(config_fn) 77 log = GitLog() # Reads history from local git repo. 78 79 gen_frontpage(config, log, status_fn, outdir) 80 gen_archive(config, log, outdir) 81 gen_url_list(config, outdir) 82 83 84# =============================================================================== 85def gen_frontpage(config, log, status_fn, outdir): 86 if path.exists(status_fn): 87 status = eval(open(status_fn).read()) 88 else: 89 status = dict() 90 91 output = html_header(title="CP2K Dashboard") 92 output += '<div id="flex-container"><div>\n' 93 output += html_gitbox(log) 94 output += html_linkbox() 95 output += "</div>\n" 96 output += '<table border="1" cellspacing="3" cellpadding="5">\n' 97 output += "<tr><th>Name</th><th>Host</th><th>Status</th>" 98 output += "<th>Commit</th><th>Summary</th><th>Last OK</th></tr>\n\n" 99 100 def get_sortkey(s): 101 return config.getint(s, "sortkey") 102 103 now = datetime.utcnow().replace(microsecond=0) 104 105 for s in sorted(config.sections(), key=get_sortkey): 106 print("Working on summary entry of: " + s) 107 name = config.get(s, "name") 108 host = config.get(s, "host") 109 report_url = config.get(s, "report_url") 110 do_notify = ( 111 config.getboolean(s, "notify") if (config.has_option(s, "notify")) else True 112 ) 113 timeout = ( 114 config.getint(s, "timeout") if (config.has_option(s, "timeout")) else 24 115 ) 116 117 # find latest commit that should have been tested by now 118 freshness_threshold = now - timedelta(hours=timeout) 119 ages_beyond_threshold = [ 120 i for i, c in enumerate(log) if c["date"] < freshness_threshold 121 ] 122 threshold_age = ages_beyond_threshold[0] 123 124 # get and parse report 125 report_txt = retrieve_report(report_url) 126 report = parse_report(report_txt, log) 127 128 if s not in status: 129 status[s] = {"last_ok": None, "notified": False} 130 131 if report["status"] == "OK": 132 status[s]["last_ok"] = report["git-sha"] 133 status[s]["notified"] = False 134 elif do_notify and not status[s]["notified"]: 135 send_notification(report, status[s]["last_ok"], log, name, s) 136 status[s]["notified"] = True 137 138 if report["git-sha"]: 139 age = log.index[report["git-sha"]] 140 if age > threshold_age: 141 report["status"] = "OUTDATED" 142 elif report["status"] in ("OK", "FAILED"): 143 # store only useful and fresh reports, prevents overwriting archive 144 store_report(report, report_txt, s, outdir) 145 146 uptodate = report["git-sha"] == log[0]["git-sha"] # report from latest commit? 147 output += '<tr align="center">' 148 output += '<td align="left"><a href="archive/%s/index.html">%s</a></td>' % ( 149 s, 150 name, 151 ) 152 output += '<td align="left">%s</td>' % host 153 output += status_cell(report["status"], report_url, uptodate) 154 155 # Commit 156 output += commit_cell(report["git-sha"], log) 157 158 # Summary 159 output += '<td align="left">%s</td>' % report["summary"] 160 161 # Last OK 162 if report["status"] != "OK": 163 output += commit_cell(status[s]["last_ok"], log) 164 else: 165 output += "<td></td>" 166 167 output += "</tr>\n\n" 168 169 output += "</table>\n" 170 output += '<div id="dummybox"></div></div>\n' # complete flex-container 171 output += html_footer() 172 write_file(outdir + "index.html", output) 173 write_file(status_fn, pformat(status)) 174 175 176# =============================================================================== 177def gen_archive(config, log, outdir): 178 179 for s in config.sections(): 180 print("Working on archive page of: " + s) 181 name = config.get(s, "name") 182 info_url = ( 183 config.get(s, "info_url") if (config.has_option(s, "info_url")) else None 184 ) 185 archive_files = glob(outdir + "archive/%s/rev_*.txt.gz" % s) + glob( 186 outdir + "archive/%s/commit_*.txt.gz" % s 187 ) 188 189 # read cache 190 cache_fn = outdir + "archive/%s/reports.cache" % s 191 if not path.exists(cache_fn): 192 reports_cache = dict() 193 else: 194 reports_cache = pickle.load(open(cache_fn, "rb")) 195 cache_age = path.getmtime(cache_fn) 196 # remove outdated cache entries 197 reports_cache = { 198 k: v for k, v in reports_cache.items() if path.getmtime(k) < cache_age 199 } 200 201 # read all archived reports 202 archive_reports = dict() 203 for fn in archive_files: 204 if fn in reports_cache: 205 report = reports_cache[fn] 206 else: 207 report_txt = ( 208 gzip.open(fn, "rb").read().decode("utf-8", errors="replace") 209 ) 210 report = parse_report(report_txt, log) 211 report["url"] = path.basename(fn)[:-3] 212 reports_cache[fn] = report 213 sha = report["git-sha"] 214 assert sha not in archive_reports 215 archive_reports[sha] = report 216 217 # write cache 218 if reports_cache: 219 pickle.dump(reports_cache, open(cache_fn, "wb")) 220 221 # loop over all relevant commits 222 all_url_rows = [] 223 all_html_rows = [] 224 max_age = 1 + max([-1] + [log.index[sha] for sha in archive_reports.keys()]) 225 for commit in log[:max_age]: 226 sha = commit["git-sha"] 227 html_row = "<tr>" 228 html_row += commit_cell(sha, log) 229 if sha in archive_reports: 230 report = archive_reports[sha] 231 html_row += status_cell(report["status"], report["url"]) 232 html_row += '<td align="left">%s</td>' % report["summary"] 233 url_row = "https://dashboard.cp2k.org/archive/%s/%s.gz\n" % ( 234 s, 235 report["url"], 236 ) 237 else: 238 html_row += 2 * "<td></td>" 239 url_row = "" 240 html_row += '<td align="left">%s</td>' % html.escape(commit["author-name"]) 241 html_row += '<td align="left">%s</td>' % html.escape(commit["msg"]) 242 html_row += "</tr>\n\n" 243 all_html_rows.append(html_row) 244 all_url_rows.append(url_row) 245 246 # generate html pages 247 for full_archive in (False, True): 248 if full_archive: 249 html_out_postfix = "index_full.html" 250 urls_out_postfix = "list_full.txt" 251 other_index_link = '<p>View <a href="index.html">recent archive</a></p>' 252 max_age = None # output all 253 else: 254 html_out_postfix = "index.html" 255 urls_out_postfix = "list_recent.txt" 256 other_index_link = ( 257 '<p>View <a href="index_full.html">full archive</a></p>' 258 ) 259 max_age = 100 260 261 # generate archive index 262 output = html_header(title=name) 263 output += '<p>Go back to <a href="../../index.html">main page</a></p>\n' 264 if info_url: 265 output += '<p>Get <a href="%s">more information</a></p>\n' % info_url 266 output += gen_plots( 267 archive_reports, log, outdir + "archive/" + s + "/", full_archive 268 ) 269 output += other_index_link 270 output += '<table border="1" cellspacing="3" cellpadding="5">\n' 271 output += "<tr><th>Commit</th><th>Status</th><th>Summary</th><th>Author</th><th>Commit Message</th></tr>\n\n" 272 output += "".join(all_html_rows[:max_age]) 273 output += "</table>\n" 274 output += other_index_link 275 output += html_footer() 276 html_out_fn = outdir + "archive/%s/%s" % (s, html_out_postfix) 277 write_file(html_out_fn, output) 278 279 url_list = "".join(all_url_rows[:max_age]) 280 urls_out_fn = outdir + "archive/%s/%s" % (s, urls_out_postfix) 281 write_file(urls_out_fn, url_list) 282 283 284# =============================================================================== 285def gen_url_list(config, outdir): 286 print("Working on url lists.") 287 for postfix in ("list_full.txt", "list_recent.txt"): 288 url_list = "" 289 for s in config.sections(): 290 fn = outdir + "archive/%s/%s" % (s, postfix) 291 if not path.exists(fn): 292 continue 293 url_list += open(fn).read() 294 write_file(outdir + "archive/" + postfix, url_list) 295 296 297# =============================================================================== 298def gen_plots(archive_reports, log, outdir, full_archive): 299 300 ordered_shas = [commit["git-sha"] for commit in log] 301 ordered_reports = [ 302 archive_reports[sha] for sha in ordered_shas if sha in archive_reports 303 ] 304 305 # collect plot data 306 plots = OrderedDict() 307 for report in ordered_reports: 308 for p in report["plots"]: 309 if p["name"] not in plots.keys(): 310 plots[p["name"]] = { 311 "curves": OrderedDict(), 312 "title": p["title"], 313 "ylabel": p["ylabel"], 314 } 315 for pp in report["plotpoints"]: 316 p = plots[pp["plot"]] 317 if pp["name"] not in p["curves"].keys(): 318 p["curves"][pp["name"]] = { 319 "x": [], 320 "y": [], 321 "yerr": [], 322 "label": pp["label"], 323 } 324 c = p["curves"][pp["name"]] 325 age = log.index[report["git-sha"]] 326 c["x"].append(-age) 327 c["y"].append(pp["y"]) 328 c["yerr"].append(pp["yerr"]) 329 330 # write raw data 331 tags = sorted( 332 [(pname, cname) for pname, p in plots.items() for cname in p["curves"].keys()] 333 ) 334 if tags: 335 raw_output = "# %6s %40s" % ("age", "commit") 336 for pname, cname in tags: 337 raw_output += " %18s %22s" % ( 338 pname + "/" + cname, 339 pname + "/" + cname + "_err", 340 ) 341 raw_output += "\n" 342 for report in reversed(ordered_reports): 343 age = log.index[report["git-sha"]] 344 raw_output += "%8d %40s" % (-age, report["git-sha"]) 345 for pname, cname in tags: 346 pp = [ 347 pp 348 for pp in report["plotpoints"] 349 if (pp["plot"] == pname and pp["name"] == cname) 350 ] 351 if len(pp) > 1: 352 print("Warning: Found redundant plot points.") 353 if pp: 354 raw_output += " %18f %22f" % (pp[-1]["y"], pp[-1]["yerr"]) 355 else: 356 raw_output += " %18s %22s" % ("?", "?") 357 raw_output += "\n" 358 write_file(outdir + "plot_data.txt", raw_output) 359 360 # create png images 361 if full_archive: 362 fig_ext = "_full.png" 363 max_age = max([-1] + [log.index[sha] for sha in archive_reports.keys()]) 364 else: 365 fig_ext = ".png" 366 max_age = 100 367 368 for pname, p in plots.items(): 369 print("Working on plot: " + pname) 370 fig = plt.figure(figsize=(12, 4)) 371 fig.subplots_adjust(bottom=0.18, left=0.06, right=0.70) 372 fig.suptitle(p["title"], fontsize=14, fontweight="bold", x=0.4) 373 ax = fig.add_subplot(111) 374 ax.set_xlabel("Commit Age") 375 ax.set_ylabel(p["ylabel"]) 376 markers = itertools.cycle("os>^*") 377 for cname in p["curves"].keys(): 378 c = p["curves"][cname] 379 if full_archive: 380 ax.plot(c["x"], c["y"], label=c["label"], linewidth=2) # less crowded 381 else: 382 ax.errorbar( 383 c["x"], 384 c["y"], 385 yerr=c["yerr"], 386 label=c["label"], 387 marker=next(markers), 388 linewidth=2, 389 markersize=6, 390 ) 391 ax.set_xlim(-max_age - 1, 0) 392 ax.xaxis.set_minor_locator(AutoMinorLocator()) 393 ax.legend( 394 bbox_to_anchor=(1.01, 1), 395 loc="upper left", 396 numpoints=1, 397 fancybox=True, 398 shadow=True, 399 borderaxespad=0.0, 400 ) 401 visibles = [ 402 [y for x, y in zip(c["x"], c["y"]) if x >= -max_age] 403 for c in p["curves"].values() 404 ] # visible y-values 405 visibles = [ys for ys in visibles if ys] # remove completely invisible curves 406 if not visibles: 407 print("Warning: Found no visible plot curve.") 408 else: 409 ymin = min([min(ys) for ys in visibles]) # lowest point from lowest curve 410 ymax = max([max(ys) for ys in visibles]) # highest point from highest curve 411 if full_archive: 412 ax.set_ylim(0.98 * ymin, 1.02 * ymax) 413 else: 414 ymax2 = max( 415 [min(ys) for ys in visibles] 416 ) # lowest point from highest curve 417 ax.set_ylim( 418 0.98 * ymin, min(1.02 * ymax, 1.3 * ymax2) 419 ) # protect against outlayers 420 fig.savefig(outdir + pname + fig_ext) 421 plt.close(fig) 422 423 # write html output 424 html_output = "" 425 for pname in sorted(plots.keys()): 426 html_output += '<a href="plot_data.txt"><img src="%s" alt="%s"></a>\n' % ( 427 pname + fig_ext, 428 plots[pname]["title"], 429 ) 430 return html_output 431 432 433# =============================================================================== 434def send_notification(report, last_ok, log, name, s): 435 idx_end = log.index[report["git-sha"]] if (report["git-sha"]) else 0 436 if not last_ok: 437 return # we don't know when this started 438 idx_last_ok = log.index[last_ok] 439 if idx_end == idx_last_ok: 440 return # probably a flapping tester 441 emails = set([log[i]["author-email"] for i in range(idx_end, idx_last_ok)]) 442 emails = [e for e in emails if "noreply" not in e] 443 emails_str = ", ".join(emails) 444 if not emails: 445 return # no author emails found 446 if len(emails) > 3: 447 print("Spam protection, found more than three authors: " + emails_str) 448 return 449 if not send_emails: 450 print("Email sending disabled, would otherwise send to: " + emails_str) 451 return 452 453 print("Sending email to: " + emails_str) 454 455 msg_txt = "Dear CP2K developer,\n\n" 456 msg_txt += "the dashboard has detected a problem that one of your recent commits might have introduced.\n\n" 457 msg_txt += " test name: %s\n" % name 458 msg_txt += " report state: %s\n" % report["status"] 459 msg_txt += " report summary: %s\n" % report["summary"] 460 msg_txt += " last OK commit: %s\n\n" % last_ok[:7] 461 msg_txt += "For more information visit:\n" 462 msg_txt += " https://dashboard.cp2k.org/archive/%s/index.html \n\n" % s 463 msg_txt += "Sincerely,\n" 464 msg_txt += " your CP2K Dashboard ;-)\n" 465 466 msg = MIMEText(msg_txt) 467 msg["Subject"] = "Problem with " + name 468 msg["From"] = "CP2K Dashboard <dashboard@cp2k.org>" 469 msg["To"] = ", ".join(emails) 470 471 smtp_conn = smtplib.SMTP("localhost") 472 smtp_conn.sendmail(msg["From"], emails, msg.as_string()) 473 smtp_conn.quit() 474 475 476# =============================================================================== 477def html_header(title): 478 output = '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">\n' 479 output += "<html><head>\n" 480 output += '<meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n' 481 output += '<meta http-equiv="refresh" content="200">\n' 482 output += '<link rel="icon" type="image/x-icon" href="data:image/x-icon;base64,AAABAAEAEBAQAAAAAAAoAQAAFgAAACgAAAAQAAAAIAAAAAEABAAAAAAAgAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAD/AAmRCQAAb/8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAiIgAAAAAAACIiAAAAAAAAIiIAAAAAAAAAAAAAAAAAADMzAAAAAAAAMzMAAAAAAAAzMwAAAAAAAAAAAAAAAAAAEREAAAAAAAAREQAAAAAAABERAAAAAAAAAAAAAAD+fwAA/n8AAPw/AAD4HwAA+B8AAPgfAAD4HwAA+B8AAPgfAAD4HwAA+B8AAPgfAAD4HwAA+B8AAPgfAAD4HwAA">\n' 483 output += '<style type="text/css">\n' 484 output += ".ribbon {\n" 485 output += " overflow: hidden;\n" 486 output += " position: absolute;\n" 487 output += " right:0px;\n" 488 output += " top: 0px;\n" 489 output += " width: 200px;\n" 490 output += " height: 200px;\n" 491 output += "}\n" 492 output += ".ribbon a {\n" 493 output += " position: relative;\n" 494 output += " white-space: nowrap;\n" 495 output += " background-color: #a00;\n" 496 output += " border: 1px solid #faa;\n" 497 output += " color: #fff;\n" 498 output += " display: block;\n" 499 output += " font: bold 11pt sans-serif;\n" 500 output += " padding: 7px;\n" 501 output += " top: 35px;\n" 502 output += " right: 10px;\n" 503 output += " width: 300px;\n" 504 output += " text-align: center;\n" 505 output += " text-decoration: none;\n" 506 output += " transform: rotate(45deg);\n" 507 output += " box-shadow: 0 0 10px #888;\n" 508 output += "}\n" 509 output += "#flex-container {\n" 510 output += " display: -webkit-flex; /* Safari */\n" 511 output += " display: flex;\n" 512 output += " -webkit-flex-flow: row wrap-reverse; /* Safari */\n" 513 output += " flex-flow: row wrap-reverse;\n" 514 output += " -webkit-justify-content: space-around; /* Safari */\n" 515 output += " justify-content: space-around;\n" 516 output += " -webkit-align-items: flex-end; /* Safari */\n" 517 output += " align-items: flex-end;\n" 518 output += "}\n" 519 output += ".sidebox {\n" 520 output += " width: 15em;\n" 521 output += " border-radius: 1em;\n" 522 output += " box-shadow: .2em .2em .7em 0 #777;\n" 523 output += " background: #f7f7f0;\n" 524 output += " padding: 1em;\n" 525 output += " margin: 40px 20px;\n" 526 output += "}\n" 527 output += ".sidebox h2 {\n" 528 output += " margin: 0 0 0.5em 0;\n" 529 output += "}\n" 530 output += ".sidebox p {\n" 531 output += " margin: 0.5em;\n" 532 output += "}\n" 533 output += "#dummybox {\n" 534 output += " width: 15em;\n" 535 output += "}\n" 536 output += "</style>\n" 537 output += "<title>%s</title>\n" % title 538 output += "</head><body>\n" 539 output += '<div class="ribbon"><a href="https://cp2k.org/dev:dashboard">Need Help?</a></div>\n' 540 output += "<center><h1>%s</h1></center>\n" % title.upper() 541 return output 542 543 544# =============================================================================== 545def html_linkbox(): 546 output = '<div class="sidebox">\n' 547 output += "<h2>More...</h2>\n" 548 output += '<a href="regtest_survey.html">Regtest Survey</a><br>\n' 549 output += '<a href="https://www.cp2k.org/static/coverage/">Test Coverage</a><br>\n' 550 output += '<a href="discontinued_tests.html">Discontinued Tests</a><br>\n' 551 output += "</div>\n" 552 return output 553 554 555# =============================================================================== 556def html_gitbox(log): 557 now = datetime.utcnow() 558 output = '<div class="sidebox">\n' 559 output += "<h2>Recent Commits</h2>\n" 560 for commit in log[0:10]: 561 url = "https://github.com/cp2k/cp2k/commit/" + commit["git-sha"] 562 msg = commit["msg"] 563 if len(msg) > 27: 564 msg = msg[:26] + "..." 565 output += '<p><a title="%s" href="%s">%s</a><br>\n' % ( 566 html.escape(commit["msg"]), 567 url, 568 html.escape(msg), 569 ) 570 delta = now - commit["date"] 571 age = delta.days * 24.0 + delta.seconds / 3600.0 572 output += "<small>git:" + commit["git-sha"][:7] 573 output += "<br>\n%s %.1fh ago.</small></p>\n" % ( 574 html.escape(commit["author-name"]), 575 age, 576 ) 577 output += "</div>\n" 578 return output 579 580 581# =============================================================================== 582def html_footer(): 583 now = datetime.utcnow().replace(microsecond=0) 584 output = "<p><small>Page last updated: %s</small></p>\n" % now.isoformat() 585 output += "</body></html>" 586 return output 587 588 589# =============================================================================== 590def write_file(fn, content, gz=False): 591 d = path.dirname(fn) 592 if len(d) > 0 and not path.exists(d): 593 os.makedirs(d) 594 print("Created dir: " + d) 595 if path.exists(fn): 596 old_bytes = gzip.open(fn, "rb").read() if (gz) else open(fn, "rb").read() 597 old_content = old_bytes.decode("utf-8", errors="replace") 598 if old_content == content: 599 print("File did not change: " + fn) 600 return 601 f = gzip.open(fn, "wb") if (gz) else open(fn, "wb") 602 f.write(content.encode("utf-8")) 603 f.close() 604 print("Wrote: " + fn) 605 606 607# =============================================================================== 608def status_cell(status, report_url, uptodate=True): 609 if status == "OK": 610 bgcolor = "#00FF00" if (uptodate) else "#8CE18C" 611 elif status == "FAILED": 612 bgcolor = "#FF0000" if (uptodate) else "#E18C8C" 613 else: 614 bgcolor = "#d3d3d3" 615 return '<td bgcolor="%s"><a href="%s">%s</a></td>' % (bgcolor, report_url, status) 616 617 618# =============================================================================== 619def commit_cell(git_sha, log): 620 if git_sha is None: 621 return "<td>N/A</td>" 622 idx = log.index[git_sha] 623 commit = log[idx] 624 git_url = "https://github.com/cp2k/cp2k/commit/" + git_sha 625 output = '<td align="left"><a href="%s">%s</a>' % (git_url, git_sha[:7]) 626 output += " (%d)</td>" % (-idx) 627 return output 628 629 630# =============================================================================== 631def retrieve_report(url): 632 try: 633 # see if we have a cached entry 634 h = hashlib.md5(url.encode("utf8")).hexdigest() 635 etag_file = Path("/tmp/dashboard_retrieval_cache_" + h + ".etag") 636 data_file = Path("/tmp/dashboard_retrieval_cache_" + h + ".data") 637 etag = etag_file.read_text() if etag_file.exists() else "" 638 639 # make conditional http request 640 r = requests.get(url, headers={"If-None-Match": etag}, timeout=5) 641 r.raise_for_status() 642 if r.status_code == 304: # Not Modified - cache hit 643 return data_file.read_text() 644 645 # check report size 646 report_size = int(r.headers["Content-Length"]) 647 assert report_size < 3 * 1024 * 1024 # 3 MB 648 649 # cache miss - store response 650 if "ETag" in r.headers: 651 data_file.write_text(r.text) 652 etag_file.write_text(r.headers["ETag"]) 653 return r.text 654 655 except: 656 print(traceback.print_exc()) 657 return None 658 659 660# =============================================================================== 661def store_report(report, report_txt, section, outdir): 662 fn = outdir + "archive/%s/commit_%s.txt.gz" % (section, report["git-sha"]) 663 write_file(fn, report_txt, gz=True) 664 665 666# =============================================================================== 667def parse_report(report_txt, log): 668 if report_txt is None: 669 return { 670 "status": "UNKNOWN", 671 "summary": "Error while retrieving report.", 672 "git-sha": None, 673 } 674 try: 675 report = dict() 676 report["git-sha"] = re.search("(^|\n)CommitSHA: (\w{40})\n", report_txt).group( 677 2 678 ) 679 report["summary"] = re.findall("(^|\n)Summary: (.+)\n", report_txt)[-1][1] 680 report["status"] = re.findall("(^|\n)Status: (.+)\n", report_txt)[-1][1] 681 report["plots"] = [ 682 eval("dict(%s)" % m[1]) 683 for m in re.findall("(^|\n)Plot: (.+)(?=\n)", report_txt) 684 ] 685 report["plotpoints"] = [ 686 eval("dict(%s)" % m[1]) 687 for m in re.findall("(^|\n)PlotPoint: (.+)(?=\n)", report_txt) 688 ] 689 690 # Check that every plot has at least one PlotPoint 691 for plot in report["plots"]: 692 points = [pp for pp in report["plotpoints"] if pp["plot"] == plot["name"]] 693 if not points: 694 report["status"] = "FAILED" 695 report["summary"] = 'Plot "%s" has no PlotPoints.' % plot["name"] 696 697 # Check that CommitSHA belongs to the master branch. 698 if report["git-sha"] not in log.index: 699 report["git-sha"] = None 700 report["status"] = "FAILED" 701 report["summary"] = "Unknown CommitSHA." 702 703 return report 704 except: 705 print(traceback.print_exc()) 706 return { 707 "status": "UNKNOWN", 708 "summary": "Error while parsing report.", 709 "git-sha": None, 710 } 711 712 713# =============================================================================== 714main() 715