1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3 4# Copyright (C) 2018 Mate Soos 5# 6# This program is free software; you can redistribute it and/or 7# modify it under the terms of the GNU General Public License 8# as published by the Free Software Foundation; version 2 9# of the License. 10# 11# This program is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License 17# along with this program; if not, write to the Free Software 18# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 19# 02110-1301, USA. 20 21from __future__ import print_function 22import os 23import socket 24import sys 25import optparse 26import struct 27import pickle 28import threading 29import random 30import time 31import subprocess 32import resource 33import pprint 34import traceback 35import boto 36import boto.utils 37import boto.ec2 38import logging 39import functools 40import string 41 42# for importing in systems where "." is not in the PATH 43sys.path.append(os.getcwd()+"/scripts/learn/") 44from common_aws import * 45import add_lemma_ind as addlemm 46 47pp = pprint.PrettyPrinter(depth=6) 48 49 50class PlainHelpFormatter(optparse.IndentedHelpFormatter): 51 52 def format_description(self, description): 53 if description: 54 return description + "\n" 55 else: 56 return "" 57 58 59def uptime(): 60 with open('/proc/uptime', 'r') as f: 61 return float(f.readline().split()[0]) 62 63 return None 64 65 66def get_n_bytes_from_connection(sock, MSGLEN): 67 chunks = [] 68 bytes_recd = 0 69 while bytes_recd < MSGLEN: 70 chunk = sock.recv(min(MSGLEN - bytes_recd, 2048)) 71 if chunk == '': 72 raise RuntimeError("socket connection broken") 73 chunks.append(chunk) 74 bytes_recd = bytes_recd + len(chunk) 75 76 return ''.join(chunks) 77 78 79def connect_client(threadID): 80 # Create a socket object 81 sock = socket.socket() 82 83 # Get local machine name 84 if options.host is None: 85 print("You must supply the host to connect to as a client") 86 exit(-1) 87 88 logging.info("Getting host by name %s", options.host, 89 extra={"threadid": threadID}) 90 host = socket.gethostbyname_ex(options.host) 91 logging.info("Connecting to host %s", host, 92 extra={"threadid": threadID}) 93 sock.connect((host[2][0], options.port)) 94 95 return sock 96 97 98def send_command(sock, command, tosend=None): 99 # can't set tosend={} because of Python weirdness 100 tosend = tosend or {} 101 102 tosend["command"] = command 103 tosend = pickle.dumps(tosend) 104 tosend = struct.pack('!q', len(tosend)) + tosend 105 sock.sendall(tosend) 106 107 108def ask_for_data(sock, command, threadID): 109 logging.info("Asking for %s", command, extra={"threadid": threadID}) 110 tosend = {} 111 tosend["uptime"] = uptime() 112 send_command(sock, command, tosend) 113 114 # get stuff to solve 115 data = get_n_bytes_from_connection(sock, 8) 116 length = struct.unpack('!q', data)[0] 117 data = get_n_bytes_from_connection(sock, length) 118 indata = pickle.loads(data) 119 return indata 120 121 122def signal_error_to_master(): 123 sock = connect_client(100) 124 send_command(sock, "error") 125 sock.close() 126 127 128def setlimits(time_limit, mem_limit): 129 #logging.info( 130 #"Setting resource limit in child (pid %d). Time %d s" 131 #"Mem %d MB\n", os.getpid(), time_limit, 132 #mem_limit, 133 #extra=self.logextra) 134 135 resource.setrlimit(resource.RLIMIT_CPU, ( 136 time_limit, 137 time_limit)) 138 139 resource.setrlimit(resource.RLIMIT_DATA, ( 140 mem_limit * 1024 * 1024, 141 mem_limit * 1024 * 1024)) 142 143 144class solverThread (threading.Thread): 145 146 def __init__(self, threadID): 147 threading.Thread.__init__(self) 148 self.threadID = threadID 149 self.logextra = {'threadid': self.threadID} 150 self.temp_space = self.create_temp_space() 151 logging.info("Initializing thread", extra=self.logextra) 152 153 def create_temp_space(self): 154 newdir = options.temp_space + "/thread-%s" % self.threadID 155 try: 156 os.system("sudo mkdir %s" % newdir) 157 os.system("sudo chown ubuntu:ubuntu %s" % newdir) 158 except: 159 exc_type, exc_value, exc_traceback = sys.exc_info() 160 the_trace = traceback.format_exc().rstrip().replace("\n", " || ") 161 logging.warning("Error creating directory: %s", 162 the_trace, extra={"threadid": -1}) 163 164 return newdir 165 166 def get_fname_no_dir(self): 167 fname = self.indata["cnf_filename"] 168 slash_at = fname.find("/") 169 return fname[slash_at+1:] 170 171 def get_tmp_cnf_fname(self): 172 return "%s/%s" % ( 173 self.temp_space, 174 self.get_fname_no_dir() 175 ) 176 177 def get_stdout_fname(self): 178 return self.get_tmp_cnf_fname() + "-" + self.indata["uniq_cnt"] + ".stdout" 179 180 def get_stderr_fname(self): 181 return self.get_tmp_cnf_fname() + "-" + self.indata["uniq_cnt"] + ".stderr" 182 183 def get_sqlite_fname(self): 184 return self.get_tmp_cnf_fname() + "-" + self.indata["uniq_cnt"] + ".sqlite" 185 186 def get_lemmas_fname(self): 187 return "%s/lemmas" % self.temp_space 188 189 def get_drat_fname(self): 190 return "%s/drat" % self.temp_space 191 192 def get_toexec(self): 193 logging.info("Getting file to solve {cnf}".format(cnf=self.indata["cnf_filename"]), 194 extra=self.logextra) 195 196 key = boto.connect_s3().get_bucket("msoos-solve-data").get_key(self.indata["cnf_filename"]) 197 key.get_contents_to_filename(self.get_tmp_cnf_fname()) 198 199 toexec = [] 200 toexec.append("%s/%s" % (options.base_dir, self.indata["solver"])) 201 toexec.append(self.indata["extra_opts"].replace(",", " ")) 202 if "cryptominisat5" in self.indata["solver"]: 203 toexec.append("--printsol 0") 204 if self.indata["stats"]: 205 toexec.append("--sql 2") 206 toexec.append("--sqlitedb %s" % self.get_sqlite_fname()) 207 208 toexec.append(self.get_tmp_cnf_fname()) 209 if self.indata["drat"]: 210 if "Maple" in self.indata["solver"]: 211 toexec.extend(["-drup-file=%s" % self.get_drat_fname()]) 212 if "cryptominisat5" in self.indata["solver"]: 213 toexec.extend(["--drat", self.get_drat_fname()]) 214 # never stop search() to simplify anything 215 # toexec.append("-n 1") 216 # toexec.append("--ml 0") 217 # toexec.append("--gluecut0 100") 218 # toexec.append("--otfsubsume 0") 219 if self.indata["stats"]: 220 toexec.append("--clid") 221 else: 222 if "cryptominisat5" in self.indata["solver"] and self.indata["stats"]: 223 toexec.append("--sqlfull 0") 224 225 return " ".join(toexec) 226 227 def execute_solver(self): 228 toexec = self.get_toexec() 229 stdout_file = open(self.get_stdout_fname(), "w") 230 stderr_file = open(self.get_stderr_fname(), "w") 231 232 # limit time 233 limits_printed = "Thread %d executing '%s' with timeout %d s and memout %d MB" % ( 234 self.threadID, 235 toexec, 236 self.indata["timeout_in_secs"], 237 self.indata["mem_limit_in_mb"] 238 ) 239 logging.info(limits_printed, extra=self.logextra) 240 stderr_file.write(limits_printed + "\n") 241 stderr_file.flush() 242 stdout_file.write(limits_printed + "\n") 243 stdout_file.flush() 244 245 tstart = time.time() 246 p = subprocess.Popen( 247 toexec.rsplit(), stderr=stderr_file, stdout=stdout_file, 248 preexec_fn=functools.partial( 249 setlimits, 250 self.indata["timeout_in_secs"], 251 self.indata["mem_limit_in_mb"])) 252 p.wait() 253 tend = time.time() 254 255 towrite = "Finished in %f seconds by thread %s return code: %d\n" % ( 256 tend - tstart, self.threadID, p.returncode) 257 stderr_file.write(towrite) 258 stdout_file.write(towrite) 259 stderr_file.close() 260 stdout_file.close() 261 logging.info(towrite.strip(), extra=self.logextra) 262 263 return p.returncode, toexec 264 265 def run_drat_trim(self): 266 toexec = "%s/drat-trim/drat-trim %s %s -x %s" % ( 267 options.base_dir, 268 self.get_tmp_cnf_fname(), 269 self.get_drat_fname(), 270 self.get_lemmas_fname()) 271 logging.info("Current working dir: %s", os.getcwd(), extra=self.logextra) 272 logging.info("Executing %s", toexec, extra=self.logextra) 273 274 stdout_file = open(self.get_stdout_fname(), "a") 275 stderr_file = open(self.get_stderr_fname(), "a") 276 tstart = time.time() 277 p = subprocess.Popen( 278 toexec.rsplit(), stderr=stderr_file, stdout=stdout_file, 279 preexec_fn=functools.partial( 280 setlimits, 281 10*self.indata["timeout_in_secs"], 282 2*self.indata["mem_limit_in_mb"])) 283 p.wait() 284 tend = time.time() 285 286 towrite = "Finished DRAT-TRIM2 in %f seconds by thread %s return code: %d\n" % ( 287 tend - tstart, self.threadID, p.returncode) 288 stderr_file.write(towrite) 289 stdout_file.write(towrite) 290 stderr_file.close() 291 stdout_file.close() 292 293 return p.returncode 294 295 def add_lemma_idx_to_sqlite(self, lemmafname, dbfname): 296 logging.info("Updating sqlite with DRAT info." 297 "Using sqlite3db file %s. Using lemma file %s", 298 dbfname, lemmafname, extra=self.logextra) 299 300 useful_lemma_ids = [] 301 with addlemm.Query(dbfname) as q: 302 useful_lemma_ids = addlemm.parse_lemmas(lemmafname) 303 q.add_goods(useful_lemma_ids) 304 305 logging.info("Num good IDs: %d", 306 len(useful_lemma_ids), extra=self.logextra) 307 308 os.unlink(self.get_lemmas_fname()) 309 310 def create_url(self, bucket, folder, key): 311 return 'https://%s.s3.amazonaws.com/%s/%s' % (bucket, folder, key) 312 313 def rnd_id(self): 314 return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5)) 315 316 def copy_solution_to_s3(self): 317 exists = boto_conn.lookup(self.indata["s3_bucket"]) 318 if not exists: 319 boto_conn.create_bucket(self.indata["s3_bucket"]) 320 boto_bucket = boto_conn.get_bucket(self.indata["s3_bucket"]) 321 k = boto.s3.key.Key(boto_bucket) 322 323 s3_folder = get_s3_folder(self.indata["given_folder"], 324 self.indata["git_rev"], 325 self.indata["solver"], 326 self.indata["timeout_in_secs"], 327 self.indata["mem_limit_in_mb"]) 328 329 s3_folder_and_fname = s3_folder + "/" + self.get_fname_no_dir() + "-" + self.indata["uniq_cnt"] 330 s3_folder_and_fname_clean = s3_folder + "/" + self.get_fname_no_dir() 331 332 toreturn = [] 333 334 # stdout 335 ret = os.system("gzip -f %s" % self.get_stdout_fname()) 336 logging.info("Return from gzip '%s': %s", self.get_stdout_fname(), 337 ret, extra=self.logextra) 338 fname = s3_folder_and_fname + ".stdout.gz-tmp" + self.rnd_id() 339 fname_clean = s3_folder_and_fname_clean + ".stdout.gz" 340 k.key = fname 341 boto_bucket.delete_key(k) 342 ret = k.set_contents_from_filename(self.get_stdout_fname() + ".gz") 343 logging.info("Return from S3 writing file '%s': %s", 344 fname, ret, extra=self.logextra) 345 toreturn.append([fname, fname_clean]) 346 347 # stderr 348 ret = os.system("gzip -f %s" % self.get_stderr_fname()) 349 logging.info("Return from gzip '%s': %s", self.get_stderr_fname(), 350 ret, extra=self.logextra) 351 fname = s3_folder_and_fname + ".stderr.gz-tmp" + self.rnd_id() 352 fname_clean = s3_folder_and_fname_clean + ".stderr.gz" 353 k.key = fname 354 boto_bucket.delete_key(k) 355 ret = k.set_contents_from_filename(self.get_stderr_fname() + ".gz") 356 logging.info("Return from S3 writing file '%s': %s", 357 fname, ret, extra=self.logextra) 358 toreturn.append([fname, fname_clean]) 359 360 # sqlite 361 if "cryptominisat5" in self.indata["solver"] and self.indata["stats"]: 362 ret = os.system("gzip -f %s" % self.get_sqlite_fname()) 363 logging.info("Return from gzip '%s': %s", self.get_sqlite_fname(), 364 ret, extra=self.logextra) 365 fname = s3_folder_and_fname + ".sqlite.gz-tmp" + self.rnd_id() 366 fname_clean = s3_folder_and_fname_clean + ".sqlite.gz" 367 k.key = fname 368 boto_bucket.delete_key(k) 369 ret = k.set_contents_from_filename(self.get_sqlite_fname() + ".gz") 370 logging.info("Return from S3 writing file '%s': %s", 371 fname, ret, extra=self.logextra) 372 toreturn.append([fname, fname_clean]) 373 374 logging.info("Uploaded stdout+stderr+sqlite files: %s", 375 toreturn, extra=self.logextra) 376 377 os.unlink(self.get_stdout_fname() + ".gz") 378 os.unlink(self.get_stderr_fname() + ".gz") 379 if "cryptominisat5" in self.indata["solver"] and self.indata["stats"]: 380 os.unlink(self.get_sqlite_fname() + ".gz") 381 382 return toreturn 383 384 def run_loop(self): 385 global exitapp 386 num_connect_problems = 0 387 while not exitapp: 388 if (num_connect_problems >= 20): 389 logging.error("Too many connection problems, exiting.", 390 extra=self.logextra) 391 exitapp = True 392 return 393 394 time.sleep(random.randint(0, 100) / 20.0) 395 try: 396 sock = connect_client(self.threadID) 397 except: 398 exc_type, exc_value, exc_traceback = sys.exc_info() 399 the_trace = traceback.format_exc().rstrip().replace("\n", " || ") 400 logging.warn("Problem trying to connect" 401 "waiting and re-connecting." 402 " Trace: %s", the_trace, 403 extra=self.logextra) 404 time.sleep(3) 405 num_connect_problems += 1 406 continue 407 408 self.indata = ask_for_data(sock, "need", self.threadID) 409 sock.close() 410 411 logging.info("Got data from server %s", 412 pprint.pformat(self.indata, indent=4).replace("\n", " || "), 413 extra=self.logextra) 414 options.noshutdown |= self.indata["noshutdown"] 415 416 # handle 'finish' 417 if self.indata["command"] == "finish": 418 logging.warn("Client received that there is nothing more" 419 " to solve, exiting this thread", 420 extra=self.logextra) 421 return 422 423 # handle 'wait' 424 if self.indata["command"] == "wait": 425 time.sleep(20) 426 continue 427 428 # handle 'solve' 429 if self.indata["command"] == "solve": 430 returncode, executed = self.execute_solver() 431 if returncode == 20 and self.indata["drat"] and self.indata["stats"]: 432 if self.run_drat_trim() == 0: 433 self.add_lemma_idx_to_sqlite( 434 self.get_lemmas_fname(), 435 self.get_sqlite_fname()) 436 os.unlink(self.get_tmp_cnf_fname()) 437 if self.indata["drat"]: 438 os.unlink(self.get_drat_fname()) 439 files = self.copy_solution_to_s3() 440 self.send_back_that_we_solved(returncode, files) 441 continue 442 443 logging.error("Data unrecognised by client: %s, exiting", 444 self.logextra) 445 return 446 447 logging.info("Exit asked for by another thread. Exiting", 448 extra=self.logextra) 449 450 def send_back_that_we_solved(self, returncode, files): 451 logging.info("Trying to send to server that we are done", 452 extra=self.logextra) 453 fail_connect = 0 454 while True: 455 if fail_connect > 5: 456 logging.error("Too many errors connecting to server to" 457 " send results. Shutting down", 458 extra=self.logextra) 459 shutdown(-1) 460 try: 461 sock = connect_client(self.threadID) 462 break 463 except: 464 exc_type, exc_value, exc_traceback = sys.exc_info() 465 the_trace = traceback.format_exc().rstrip().replace("\n", " || ") 466 467 logging.warn("Problem, waiting and re-connecting." 468 " Trace: %s", the_trace, 469 extra=self.logextra) 470 time.sleep(random.randint(0, 5) / 10.0) 471 fail_connect += 1 472 473 tosend = {} 474 tosend["file_num"] = self.indata["file_num"] 475 tosend["returncode"] = returncode 476 tosend["files"] = files 477 send_command(sock, "done", tosend) 478 logging.info("Sent that we finished %s with retcode %d", 479 self.indata["file_num"], returncode, extra=self.logextra) 480 481 sock.close() 482 483 def run(self): 484 logging.info("Starting thread", extra=self.logextra) 485 global exitapp 486 487 try: 488 self.run_loop() 489 except KeyboardInterrupt: 490 exitapp = True 491 raise 492 except: 493 exc_type, exc_value, exc_traceback = sys.exc_info() 494 the_trace = traceback.format_exc().rstrip().replace("\n", " || ") 495 496 exitapp = True 497 logging.error("Unexpected error in thread: %s", the_trace, 498 extra=self.logextra) 499 shutdown(-1) 500 raise 501 502 503def build_cryptominisat(indata): 504 opts = [] 505 opts.append(indata["git_rev"]) 506 opts.append(str(options.num_threads)) 507 if indata["stats"]: 508 opts.append("-DSTATS=ON") 509 510 if indata["gauss"]: 511 opts.append("-DUSE_GAUSS=ON") 512 513 ret = os.system('%s/cryptominisat/scripts/aws/build_cryptominisat.sh %s >> %s/build.log 2>&1' % 514 (options.base_dir, 515 " ".join(opts), 516 options.base_dir)) 517 global s3_folder 518 s3_folder = get_s3_folder(indata["given_folder"], 519 indata["git_rev"], 520 indata["solver"], 521 indata["timeout_in_secs"], 522 indata["mem_limit_in_mb"] 523 ) 524 global s3_bucket 525 s3_bucket = indata["s3_bucket"] 526 logging.info("s3 bucket: %, s3 folder: %", s3_bucket, s3_folder, extra={"threadid": "-1"}) 527 if ret != 0: 528 logging.error("Error building cryptominisat, shutting down!", 529 extra={"threadid": -1} 530 ) 531 shutdown(-1) 532 533 534def build_system(): 535 built_system = False 536 logging.info("Building system", extra={"threadid": -1}) 537 tries = 0 538 while not built_system and tries < 10: 539 try: 540 tries += 1 541 sock = connect_client(-1) 542 except Exception: 543 exc_type, exc_value, exc_traceback = sys.exc_info() 544 the_trace = traceback.format_exc().rstrip().replace("\n", " || ") 545 logging.warning("Problem, waiting and re-connecting. Error: %s", 546 the_trace, 547 extra={"threadid": -1}) 548 time.sleep(3) 549 continue 550 551 indata = ask_for_data(sock, "build", -1) 552 options.noshutdown |= indata["noshutdown"] 553 sock.close() 554 555 if "cryptominisat5" in indata["solver"]: 556 build_cryptominisat(indata) 557 558 built_system = True 559 560 if not built_system: 561 shutdown(-1) 562 563 564def num_cpus(): 565 num_cpu = 0 566 cpuinfo = open("/proc/cpuinfo", "r") 567 for line in cpuinfo: 568 if "processor" in line: 569 num_cpu += 1 570 571 cpuinfo.close() 572 return num_cpu 573 574 575def shutdown(exitval=0): 576 toexec = "sudo shutdown -h now" 577 logging.info("SHUTTING DOWN", extra={"threadid": -1}) 578 579 # signal error to master 580 if exitval != 0: 581 try: 582 signal_error_to_master() 583 except: 584 pass 585 586 # send email 587 if exitval == 0: 588 reason = "OK" 589 else: 590 reason = "FAIL" 591 592 try: 593 send_email("Client shutting down %s" % reason, 594 "Client finished.", options.logfile_name) 595 except: 596 exc_type, exc_value, exc_traceback = sys.exc_info() 597 the_trace = traceback.format_exc().rstrip().replace("\n", " || ") 598 logging.error("Cannot send email! Traceback: %s", the_trace, 599 extra={"threadid": -1}) 600 601 if not options.noshutdown: 602 os.system(toexec) 603 604 exit(exitval) 605 606 607def set_up_logging(): 608 form = '[ %(asctime)-15s thread %(threadid)s ' 609 form += get_ip_address(options.network_device) 610 form += " %(levelname)s %(message)s ]" 611 612 logformatter = logging.Formatter(form) 613 614 consoleHandler = logging.StreamHandler() 615 consoleHandler.setFormatter(logformatter) 616 logging.getLogger().addHandler(consoleHandler) 617 618 try: 619 os.unlink(options.logfile_name) 620 except: 621 pass 622 fileHandler = logging.FileHandler(options.logfile_name) 623 fileHandler.setFormatter(logformatter) 624 logging.getLogger().addHandler(fileHandler) 625 626 logging.getLogger().setLevel(logging.INFO) 627 628 629def update_num_threads(): 630 if options.num_threads is None: 631 options.num_threads = num_cpus()/2 632 options.num_threads = max(options.num_threads, 1) 633 634 logging.info("Running with %d threads", options.num_threads, 635 extra={"threadid": -1}) 636 637 638def build_system_full(): 639 try: 640 build_system() 641 except: 642 exc_type, exc_value, exc_traceback = sys.exc_info() 643 the_trace = traceback.format_exc().rstrip().replace("\n", " || ") 644 logging.error("Error getting data for building system: %s", 645 the_trace, extra={"threadid": -1}) 646 shutdown(-1) 647 648 649def start_threads(): 650 threads = [] 651 # we should test at least 2 threads, it's only used during testing anyway 652 options.num_threads = max(options.num_threads, 2) 653 for i in range(options.num_threads): 654 threads.append(solverThread(i)) 655 656 for t in threads: 657 t.setDaemon(True) 658 t.start() 659 660 661def print_to_log_local_setup(): 662 data = boto.utils.get_instance_metadata() 663 for a, b in data.items(): 664 logging.info("%s -- %s", a, b, extra={"threadid": -1}) 665 666 667class VolumeAdderMount(): 668 def __init__(self): 669 pass 670 671 def add_volume(self): 672 os.system("sudo mkfs.ext3 /dev/xvdb") 673 os.system("sudo mkdir %s" % options.temp_space) 674 os.system("sudo mount /dev/xvdb %s" % options.temp_space) 675 676 def delete_volume(self): 677 pass 678 679 680class VolumeAdder(): 681 def __init__(self): 682 self.conn = boto.ec2.connect_to_region(self._get_region()) 683 684 def _get_instance_id(self): 685 instance_id = boto.utils.get_instance_metadata() 686 return instance_id['instance-id'] 687 688 def _get_availability_zone(self): 689 dat = boto.utils.get_instance_metadata() 690 return dat["placement"]["availability-zone"] 691 692 def _get_region(self): 693 region = boto.utils.get_instance_metadata() 694 return region['local-hostname'].split('.')[1] 695 696 def add_volume(self): 697 self.vol = self.conn.create_volume(50, self._get_availability_zone()) 698 while self.vol.status != 'available': 699 print('Vol state: ', self.vol.status) 700 time.sleep(5) 701 self.vol.update() 702 703 dev = "xvdc" 704 logging.info("Created volume, attaching... %s", self.vol, 705 extra={"threadid": -1}) 706 self.conn.attach_volume(self.vol.id, self._get_instance_id(), dev) 707 logging.info("Waiting for volume to show up...", extra={"threadid": -1}) 708 time.sleep(10) 709 710 logging.info("Trying to mkfs, mkdir and mount", extra={"threadid": -1}) 711 os.system("sudo mkfs.ext3 /dev/%s" % dev) 712 os.system("sudo mkdir %s" % options.temp_space) 713 os.system("sudo chown ubuntu:ubuntu %s" % options.temp_space) 714 os.system("sudo mount /dev/%s %s" % (dev, options.temp_space)) 715 716 return self.vol.id 717 718 def delete_volume(self): 719 try: 720 os.system("sudo umount /mnt2") 721 time.sleep(2) 722 except: 723 logging.error("Issue with unmounting, but ignored", 724 extra={"threadid": -1}) 725 726 self.conn.detach_volume(self.vol.id, force=True) 727 time.sleep(1) 728 self.conn.delete_volume(self.vol.id) 729 730 731def parse_command_line(): 732 usage = "usage: %prog" 733 parser = optparse.OptionParser(usage=usage, formatter=PlainHelpFormatter()) 734 parser.add_option("--verbose", "-v", action="store_true", default=False, 735 dest="verbose", help="Be more verbose" 736 ) 737 738 parser.add_option("--host", dest="host", 739 help="Host to connect to as a client") 740 parser.add_option("--port", "-p", default=10000, dest="port", 741 type="int", help="Port to use" 742 " [default: %default]", 743 ) 744 745 parser.add_option("--temp", default="/mnt2", dest="temp_space", type=str, 746 help="Temporary space to use" 747 " [default: %default]", 748 ) 749 750 parser.add_option("--noshutdown", "-n", default=False, dest="noshutdown", 751 action="store_true", help="Do not shut down" 752 ) 753 754 parser.add_option("--dir", default="/home/ubuntu/", dest="base_dir", type=str, 755 help="The home dir of cryptominisat" 756 " [default: %default]", 757 ) 758 parser.add_option("--net", default="ens3", dest="network_device", type=str, 759 help="The network device we will be using" 760 " [default: %default]", 761 ) 762 763 parser.add_option("--threads", dest="num_threads", type=int, 764 help="Force using this many threads") 765 766 parser.add_option("--dev", dest="dev", type=str, default="xvdc", 767 help="Device name") 768 769 parser.add_option("--logfile", dest="logfile_name", type=str, 770 default="python_log.log", help="Name of LOG file") 771 772 (options, args) = parser.parse_args() 773 774 return options, args 775 776 777if __name__ == "__main__": 778 global s3_bucket 779 global s3_folder 780 s3_bucket = "msoos-no-bucket" 781 s3_folder = "no_s3_folder" 782 options, args = parse_command_line() 783 784 exitapp = False 785 options.logfile_name = options.base_dir + options.logfile_name 786 787 # get host 788 if options.host is None: 789 for line in boto.utils.get_instance_userdata().split("\n"): 790 if "DATA" in line: 791 options.host = line.split("=")[1].strip().strip('"') 792 793 print("HOST has beeen set to %s" % options.host) 794 795 try: 796 set_up_logging() 797 logging.info("Client called with parameters: %s", 798 pprint.pformat(options, indent=4).replace("\n", " || "), 799 extra={"threadid": -1}) 800 print_to_log_local_setup() 801 v = VolumeAdderMount() 802 v.add_volume() 803 804 boto_conn = boto.connect_s3() 805 update_num_threads() 806 build_system_full() 807 start_threads() 808 while threading.active_count() > 1: 809 time.sleep(0.1) 810 811 # finish up 812 logging.info("Exiting Main Thread, shutting down", extra={"threadid": -1}) 813 v.delete_volume() 814 except: 815 exc_type, exc_value, exc_traceback = sys.exc_info() 816 the_trace = traceback.format_exc().rstrip().replace("\n", " || ") 817 logging.error("Problem in __main__" 818 "Trace: %s", the_trace, extra={"threadid": -1}) 819 shutdown(-1) 820 821 shutdown() 822