mirror of
https://github.com/certbot/certbot.git
synced 2026-06-03 13:59:02 -04:00
Try to finally prevent dangling AWS volumes
This commit is contained in:
Peter Eckersley
parent
77d606d8aa
commit
f4c33656a2
1 changed files with 119 additions and 109 deletions
|
|
@ -333,6 +333,55 @@ def create_client_instances(targetlist):
|
|||
print()
|
||||
return instances
|
||||
|
||||
|
||||
def test_client_process(inqueue, outqueue):
|
||||
cur_proc = mp.current_process()
|
||||
for inreq in iter(inqueue.get, SENTINEL):
|
||||
ii, target = inreq
|
||||
|
||||
#save all stdout to log file
|
||||
sys.stdout = open(LOGDIR+'/'+'%d_%s.log'%(ii,target['name']), 'w')
|
||||
|
||||
print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name']))
|
||||
instances[ii] = block_until_instance_ready(instances[ii])
|
||||
print("server %s at %s"%(instances[ii], instances[ii].public_ip_address))
|
||||
env.host_string = "%s@%s"%(target['user'], instances[ii].public_ip_address)
|
||||
print(env.host_string)
|
||||
|
||||
try:
|
||||
install_and_launch_letsencrypt(instances[ii], boulder_url, target)
|
||||
outqueue.put((ii, target, 'pass'))
|
||||
print("%s - %s SUCCESS"%(target['ami'], target['name']))
|
||||
except:
|
||||
outqueue.put((ii, target, 'fail'))
|
||||
print("%s - %s FAIL"%(target['ami'], target['name']))
|
||||
pass
|
||||
|
||||
# append server letsencrypt.log to each per-machine output log
|
||||
print("\n\nletsencrypt.log\n" + "-"*80 + "\n")
|
||||
try:
|
||||
execute(grab_letsencrypt_log)
|
||||
except:
|
||||
print("log fail\n")
|
||||
pass
|
||||
|
||||
|
||||
def cleanup(cl_args, instances, targetlist):
|
||||
print('Logs in ', LOGDIR)
|
||||
if not cl_args.saveinstances:
|
||||
print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes')
|
||||
if cl_args.killboulder:
|
||||
boulder_server.terminate()
|
||||
terminate_and_clean(instances)
|
||||
else:
|
||||
# print login information for the boxes for debugging
|
||||
for ii, target in enumerate(targetlist):
|
||||
print(target['name'],
|
||||
target['ami'],
|
||||
"%s@%s"%(target['user'], instances[ii].public_ip_address))
|
||||
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# SCRIPT BEGINS
|
||||
#-------------------------------------------------------------------------------
|
||||
|
|
@ -413,124 +462,85 @@ else:
|
|||
#machine_type='t2.medium',
|
||||
security_groups=['letsencrypt_test'])
|
||||
|
||||
if not cl_args.boulderonly:
|
||||
instances = create_client_instances(targetlist)
|
||||
try:
|
||||
if not cl_args.boulderonly:
|
||||
instances = create_client_instances(targetlist)
|
||||
|
||||
# Configure and launch boulder server
|
||||
#-------------------------------------------------------------------------------
|
||||
print("Waiting on Boulder Server")
|
||||
boulder_server = block_until_instance_ready(boulder_server)
|
||||
print(" server %s"%boulder_server)
|
||||
# Configure and launch boulder server
|
||||
#-------------------------------------------------------------------------------
|
||||
print("Waiting on Boulder Server")
|
||||
boulder_server = block_until_instance_ready(boulder_server)
|
||||
print(" server %s"%boulder_server)
|
||||
|
||||
|
||||
# env.host_string defines the ssh user and host for connection
|
||||
env.host_string = "ubuntu@%s"%boulder_server.public_ip_address
|
||||
print("Boulder Server at (SSH):", env.host_string)
|
||||
if not boulder_preexists:
|
||||
print("Configuring and Launching Boulder")
|
||||
config_and_launch_boulder(boulder_server)
|
||||
# blocking often unnecessary, but cheap EC2 VMs can get very slow
|
||||
block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address,
|
||||
wait_time=10, timeout=500)
|
||||
# env.host_string defines the ssh user and host for connection
|
||||
env.host_string = "ubuntu@%s"%boulder_server.public_ip_address
|
||||
print("Boulder Server at (SSH):", env.host_string)
|
||||
if not boulder_preexists:
|
||||
print("Configuring and Launching Boulder")
|
||||
config_and_launch_boulder(boulder_server)
|
||||
# blocking often unnecessary, but cheap EC2 VMs can get very slow
|
||||
block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address,
|
||||
wait_time=10, timeout=500)
|
||||
|
||||
boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address
|
||||
print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address)
|
||||
print("Boulder Server at (EC2 private ip): %s"%boulder_url)
|
||||
boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address
|
||||
print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address)
|
||||
print("Boulder Server at (EC2 private ip): %s"%boulder_url)
|
||||
|
||||
if cl_args.boulderonly:
|
||||
sys.exit(0)
|
||||
if cl_args.boulderonly:
|
||||
sys.exit(0)
|
||||
|
||||
# Install and launch client scripts in parallel
|
||||
#-------------------------------------------------------------------------------
|
||||
print("Uploading and running test script in parallel: %s"%cl_args.test_script)
|
||||
print("Output routed to log files in %s"%LOGDIR)
|
||||
# (Advice: always use Manager.Queue, never regular multiprocessing.Queue
|
||||
# the latter has implementation flaws that deadlock it in some circumstances)
|
||||
manager = Manager()
|
||||
outqueue = manager.Queue()
|
||||
inqueue = manager.Queue()
|
||||
SENTINEL = None #queue kill signal
|
||||
# Install and launch client scripts in parallel
|
||||
#-------------------------------------------------------------------------------
|
||||
print("Uploading and running test script in parallel: %s"%cl_args.test_script)
|
||||
print("Output routed to log files in %s"%LOGDIR)
|
||||
# (Advice: always use Manager.Queue, never regular multiprocessing.Queue
|
||||
# the latter has implementation flaws that deadlock it in some circumstances)
|
||||
manager = Manager()
|
||||
outqueue = manager.Queue()
|
||||
inqueue = manager.Queue()
|
||||
SENTINEL = None #queue kill signal
|
||||
|
||||
# launch as many processes as clients to test
|
||||
num_processes = len(targetlist)
|
||||
jobs = [] #keep a reference to current procs
|
||||
# launch as many processes as clients to test
|
||||
num_processes = len(targetlist)
|
||||
jobs = [] #keep a reference to current procs
|
||||
|
||||
def test_client_process(inqueue, outqueue):
|
||||
cur_proc = mp.current_process()
|
||||
for inreq in iter(inqueue.get, SENTINEL):
|
||||
ii, target = inreq
|
||||
|
||||
#save all stdout to log file
|
||||
sys.stdout = open(LOGDIR+'/'+'%d_%s.log'%(ii,target['name']), 'w')
|
||||
# initiate process execution
|
||||
for i in range(num_processes):
|
||||
p = mp.Process(target=test_client_process, args=(inqueue, outqueue))
|
||||
jobs.append(p)
|
||||
p.daemon = True # kills subprocesses if parent is killed
|
||||
p.start()
|
||||
|
||||
print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name']))
|
||||
instances[ii] = block_until_instance_ready(instances[ii])
|
||||
print("server %s at %s"%(instances[ii], instances[ii].public_ip_address))
|
||||
env.host_string = "%s@%s"%(target['user'], instances[ii].public_ip_address)
|
||||
print(env.host_string)
|
||||
|
||||
try:
|
||||
install_and_launch_letsencrypt(instances[ii], boulder_url, target)
|
||||
outqueue.put((ii, target, 'pass'))
|
||||
print("%s - %s SUCCESS"%(target['ami'], target['name']))
|
||||
except:
|
||||
outqueue.put((ii, target, 'fail'))
|
||||
print("%s - %s FAIL"%(target['ami'], target['name']))
|
||||
pass
|
||||
|
||||
# append server letsencrypt.log to each per-machine output log
|
||||
print("\n\nletsencrypt.log\n" + "-"*80 + "\n")
|
||||
try:
|
||||
execute(grab_letsencrypt_log)
|
||||
except:
|
||||
print("log fail\n")
|
||||
pass
|
||||
|
||||
# initiate process execution
|
||||
for i in range(num_processes):
|
||||
p = mp.Process(target=test_client_process, args=(inqueue, outqueue))
|
||||
jobs.append(p)
|
||||
p.daemon = True # kills subprocesses if parent is killed
|
||||
p.start()
|
||||
|
||||
# fill up work queue
|
||||
for ii, target in enumerate(targetlist):
|
||||
inqueue.put((ii, target))
|
||||
|
||||
# add SENTINELs to end client processes
|
||||
for i in range(num_processes):
|
||||
inqueue.put(SENTINEL)
|
||||
# wait on termination of client processes
|
||||
for p in jobs:
|
||||
p.join()
|
||||
# add SENTINEL to output queue
|
||||
outqueue.put(SENTINEL)
|
||||
|
||||
# clean up
|
||||
execute(local_repo_clean)
|
||||
|
||||
# print and save summary results
|
||||
results_file = open(LOGDIR+'/results', 'w')
|
||||
outputs = [outq for outq in iter(outqueue.get, SENTINEL)]
|
||||
outputs.sort(key=lambda x: x[0])
|
||||
for outq in outputs:
|
||||
ii, target, status = outq
|
||||
print('%d %s %s'%(ii, target['name'], status))
|
||||
results_file.write('%d %s %s\n'%(ii, target['name'], status))
|
||||
results_file.close()
|
||||
|
||||
if not cl_args.saveinstances:
|
||||
print('Logs in ', LOGDIR)
|
||||
print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes')
|
||||
if cl_args.killboulder:
|
||||
boulder_server.terminate()
|
||||
terminate_and_clean(instances)
|
||||
else:
|
||||
# print login information for the boxes for debugging
|
||||
# fill up work queue
|
||||
for ii, target in enumerate(targetlist):
|
||||
print(target['name'],
|
||||
target['ami'],
|
||||
"%s@%s"%(target['user'], instances[ii].public_ip_address))
|
||||
inqueue.put((ii, target))
|
||||
|
||||
# kill any connections
|
||||
fabric.network.disconnect_all()
|
||||
# add SENTINELs to end client processes
|
||||
for i in range(num_processes):
|
||||
inqueue.put(SENTINEL)
|
||||
# wait on termination of client processes
|
||||
for p in jobs:
|
||||
p.join()
|
||||
# add SENTINEL to output queue
|
||||
outqueue.put(SENTINEL)
|
||||
|
||||
# clean up
|
||||
execute(local_repo_clean)
|
||||
|
||||
# print and save summary results
|
||||
results_file = open(LOGDIR+'/results', 'w')
|
||||
outputs = [outq for outq in iter(outqueue.get, SENTINEL)]
|
||||
outputs.sort(key=lambda x: x[0])
|
||||
for outq in outputs:
|
||||
ii, target, status = outq
|
||||
print('%d %s %s'%(ii, target['name'], status))
|
||||
results_file.write('%d %s %s\n'%(ii, target['name'], status))
|
||||
results_file.close()
|
||||
|
||||
finally:
|
||||
cleanup(cl_args, instances, targetlist)
|
||||
|
||||
# kill any connections
|
||||
fabric.network.disconnect_all()
|
||||
|
|
|
|||
Loading…
Reference in a new issue