1#!/usr/bin/env python3 2 3# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7""" 8Script which downloads exe and wheel files hosted on AppVeyor: 9https://ci.appveyor.com/project/giampaolo/psutil 10Readapted from the original recipe of Ibarra Corretge' 11<saghul@gmail.com>: 12http://code.saghul.net/index.php/2015/09/09/ 13""" 14 15from __future__ import print_function 16import argparse 17import concurrent.futures 18import errno 19import os 20import requests 21import shutil 22import sys 23 24from psutil import __version__ as PSUTIL_VERSION 25from psutil._common import bytes2human 26from psutil._common import print_color 27 28 29BASE_URL = 'https://ci.appveyor.com/api' 30PY_VERSIONS = ['2.7', '3.5', '3.6', '3.7', '3.8'] 31TIMEOUT = 30 32COLORS = True 33 34 35def safe_makedirs(path): 36 try: 37 os.makedirs(path) 38 except OSError as err: 39 if err.errno == errno.EEXIST: 40 if not os.path.isdir(path): 41 raise 42 else: 43 raise 44 45 46def safe_rmtree(path): 47 def onerror(fun, path, excinfo): 48 exc = excinfo[1] 49 if exc.errno != errno.ENOENT: 50 raise 51 52 shutil.rmtree(path, onerror=onerror) 53 54 55def download_file(url): 56 local_fname = url.split('/')[-1] 57 local_fname = os.path.join('dist', local_fname) 58 safe_makedirs('dist') 59 r = requests.get(url, stream=True, timeout=TIMEOUT) 60 tot_bytes = 0 61 with open(local_fname, 'wb') as f: 62 for chunk in r.iter_content(chunk_size=16384): 63 if chunk: # filter out keep-alive new chunks 64 f.write(chunk) 65 tot_bytes += len(chunk) 66 return local_fname 67 68 69def get_file_urls(options): 70 with requests.Session() as session: 71 data = session.get( 72 BASE_URL + '/projects/' + options.user + '/' + options.project, 73 timeout=TIMEOUT) 74 data = data.json() 75 76 urls = [] 77 for job in (job['jobId'] for job in data['build']['jobs']): 78 job_url = BASE_URL + '/buildjobs/' + job + '/artifacts' 79 data = session.get(job_url, timeout=TIMEOUT) 80 data = data.json() 81 for item in data: 82 file_url = job_url + '/' + item['fileName'] 83 urls.append(file_url) 84 if not urls: 85 print_color("no artifacts found", 'ret') 86 sys.exit(1) 87 else: 88 for url in sorted(urls, key=lambda x: os.path.basename(x)): 89 yield url 90 91 92def rename_27_wheels(): 93 # See: https://github.com/giampaolo/psutil/issues/810 94 src = 'dist/psutil-%s-cp27-cp27m-win32.whl' % PSUTIL_VERSION 95 dst = 'dist/psutil-%s-cp27-none-win32.whl' % PSUTIL_VERSION 96 print("rename: %s\n %s" % (src, dst)) 97 os.rename(src, dst) 98 src = 'dist/psutil-%s-cp27-cp27m-win_amd64.whl' % PSUTIL_VERSION 99 dst = 'dist/psutil-%s-cp27-none-win_amd64.whl' % PSUTIL_VERSION 100 print("rename: %s\n %s" % (src, dst)) 101 os.rename(src, dst) 102 103 104def run(options): 105 safe_rmtree('dist') 106 urls = get_file_urls(options) 107 completed = 0 108 exc = None 109 with concurrent.futures.ThreadPoolExecutor() as e: 110 fut_to_url = {e.submit(download_file, url): url for url in urls} 111 for fut in concurrent.futures.as_completed(fut_to_url): 112 url = fut_to_url[fut] 113 try: 114 local_fname = fut.result() 115 except Exception: 116 print_color("error while downloading %s" % (url), 'red') 117 raise 118 else: 119 completed += 1 120 print("downloaded %-45s %s" % ( 121 local_fname, bytes2human(os.path.getsize(local_fname)))) 122 # 2 wheels (32 and 64 bit) per supported python version 123 expected = len(PY_VERSIONS) * 2 124 if expected != completed: 125 return exit("expected %s files, got %s" % (expected, completed)) 126 if exc: 127 return exit() 128 rename_27_wheels() 129 130 131def main(): 132 parser = argparse.ArgumentParser( 133 description='AppVeyor artifact downloader') 134 parser.add_argument('--user', required=True) 135 parser.add_argument('--project', required=True) 136 args = parser.parse_args() 137 run(args) 138 139 140if __name__ == '__main__': 141 main() 142