mirror of
https://github.com/certbot/certbot.git
synced 2026-06-04 22:33:00 -04:00
Create main function.
This commit is contained in:
Brad Warren
parent
a7fd7cdd87
commit
3c041f1655
1 changed files with 164 additions and 164 deletions
|
|
@ -394,186 +394,186 @@ def cleanup(cl_args, instances, targetlist):
|
|||
"%s@%s"%(target['user'], instances[ii].public_ip_address))
|
||||
|
||||
|
||||
def main():
|
||||
# Fabric library controlled through global env parameters
|
||||
env.key_filename = KEYFILE
|
||||
env.shell = '/bin/bash -l -i -c'
|
||||
env.connection_attempts = 5
|
||||
env.timeout = 10
|
||||
# replace default SystemExit thrown by fabric during trouble
|
||||
class FabricException(Exception):
|
||||
pass
|
||||
env['abort_exception'] = FabricException
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# SCRIPT BEGINS
|
||||
#-------------------------------------------------------------------------------
|
||||
# Set up local copy of git repo
|
||||
#-------------------------------------------------------------------------------
|
||||
print("Making local dir for test repo and logs: %s"%LOGDIR)
|
||||
local('mkdir %s'%LOGDIR)
|
||||
|
||||
# Fabric library controlled through global env parameters
|
||||
env.key_filename = KEYFILE
|
||||
env.shell = '/bin/bash -l -i -c'
|
||||
env.connection_attempts = 5
|
||||
env.timeout = 10
|
||||
# replace default SystemExit thrown by fabric during trouble
|
||||
class FabricException(Exception):
|
||||
pass
|
||||
env['abort_exception'] = FabricException
|
||||
# figure out what git object to test and locally create it in LOGDIR
|
||||
print("Making local git repo")
|
||||
try:
|
||||
if cl_args.pull_request != '~':
|
||||
print('Testing PR %s '%cl_args.pull_request,
|
||||
"MERGING into master" if cl_args.merge_master else "")
|
||||
execute(local_git_PR, cl_args.repo, cl_args.pull_request, cl_args.merge_master)
|
||||
elif cl_args.branch != '~':
|
||||
print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo))
|
||||
execute(local_git_branch, cl_args.repo, cl_args.branch)
|
||||
else:
|
||||
print('Testing master of %s'%cl_args.repo)
|
||||
execute(local_git_clone, cl_args.repo)
|
||||
except FabricException:
|
||||
print("FAIL: trouble with git repo")
|
||||
traceback.print_exc()
|
||||
exit()
|
||||
|
||||
# Set up local copy of git repo
|
||||
#-------------------------------------------------------------------------------
|
||||
print("Making local dir for test repo and logs: %s"%LOGDIR)
|
||||
local('mkdir %s'%LOGDIR)
|
||||
|
||||
# figure out what git object to test and locally create it in LOGDIR
|
||||
print("Making local git repo")
|
||||
try:
|
||||
if cl_args.pull_request != '~':
|
||||
print('Testing PR %s '%cl_args.pull_request,
|
||||
"MERGING into master" if cl_args.merge_master else "")
|
||||
execute(local_git_PR, cl_args.repo, cl_args.pull_request, cl_args.merge_master)
|
||||
elif cl_args.branch != '~':
|
||||
print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo))
|
||||
execute(local_git_branch, cl_args.repo, cl_args.branch)
|
||||
# Set up EC2 instances
|
||||
#-------------------------------------------------------------------------------
|
||||
configdata = yaml.load(open(cl_args.config_file, 'r'))
|
||||
targetlist = configdata['targets']
|
||||
print('Testing against these images: [%d total]'%len(targetlist))
|
||||
for target in targetlist:
|
||||
print(target['ami'], target['name'])
|
||||
|
||||
print("Connecting to EC2 using\n profile %s\n keyname %s\n keyfile %s"%(PROFILE, KEYNAME, KEYFILE))
|
||||
aws_session = boto3.session.Session(profile_name=PROFILE)
|
||||
ec2_client = aws_session.resource('ec2')
|
||||
|
||||
print("Determining Subnet")
|
||||
for subnet in ec2_client.subnets.all():
|
||||
if should_use_subnet(subnet):
|
||||
subnet_id = subnet.id
|
||||
vpc_id = subnet.vpc.id
|
||||
break
|
||||
else:
|
||||
print('Testing master of %s'%cl_args.repo)
|
||||
execute(local_git_clone, cl_args.repo)
|
||||
except FabricException:
|
||||
print("FAIL: trouble with git repo")
|
||||
traceback.print_exc()
|
||||
exit()
|
||||
print("No usable subnet exists!")
|
||||
print("Please create a VPC with a subnet named {0}".format(SUBNET_NAME))
|
||||
print("that maps public IPv4 addresses to instances launched in the subnet.")
|
||||
sys.exit(1)
|
||||
|
||||
print("Making Security Group")
|
||||
vpc = ec2_client.Vpc(vpc_id)
|
||||
sg_exists = False
|
||||
for sg in vpc.security_groups.all():
|
||||
if sg.group_name == SECURITY_GROUP_NAME:
|
||||
security_group_id = sg.id
|
||||
sg_exists = True
|
||||
print(" %s already exists"%SECURITY_GROUP_NAME)
|
||||
if not sg_exists:
|
||||
security_group_id = make_security_group(vpc).id
|
||||
time.sleep(30)
|
||||
|
||||
boulder_preexists = False
|
||||
boulder_servers = ec2_client.instances.filter(Filters=[
|
||||
{'Name': 'tag:Name', 'Values': ['le-boulderserver']},
|
||||
{'Name': 'instance-state-name', 'Values': ['running']}])
|
||||
|
||||
boulder_server = next(iter(boulder_servers), None)
|
||||
|
||||
print("Requesting Instances...")
|
||||
if boulder_server:
|
||||
print("Found existing boulder server:", boulder_server)
|
||||
boulder_preexists = True
|
||||
else:
|
||||
print("Can't find a boulder server, starting one...")
|
||||
boulder_server = make_instance(ec2_client,
|
||||
'le-boulderserver',
|
||||
BOULDER_AMI,
|
||||
KEYNAME,
|
||||
machine_type='t2.micro',
|
||||
#machine_type='t2.medium',
|
||||
security_group_id=security_group_id,
|
||||
subnet_id=subnet_id)
|
||||
|
||||
try:
|
||||
if not cl_args.boulderonly:
|
||||
instances = create_client_instances(ec2_client, targetlist, security_group_id, subnet_id)
|
||||
|
||||
# Configure and launch boulder server
|
||||
#-------------------------------------------------------------------------------
|
||||
print("Waiting on Boulder Server")
|
||||
boulder_server = block_until_instance_ready(boulder_server)
|
||||
print(" server %s"%boulder_server)
|
||||
|
||||
|
||||
# Set up EC2 instances
|
||||
#-------------------------------------------------------------------------------
|
||||
configdata = yaml.load(open(cl_args.config_file, 'r'))
|
||||
targetlist = configdata['targets']
|
||||
print('Testing against these images: [%d total]'%len(targetlist))
|
||||
for target in targetlist:
|
||||
print(target['ami'], target['name'])
|
||||
# env.host_string defines the ssh user and host for connection
|
||||
env.host_string = "ubuntu@%s"%boulder_server.public_ip_address
|
||||
print("Boulder Server at (SSH):", env.host_string)
|
||||
if not boulder_preexists:
|
||||
print("Configuring and Launching Boulder")
|
||||
config_and_launch_boulder(boulder_server)
|
||||
# blocking often unnecessary, but cheap EC2 VMs can get very slow
|
||||
block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address,
|
||||
wait_time=10, timeout=500)
|
||||
|
||||
print("Connecting to EC2 using\n profile %s\n keyname %s\n keyfile %s"%(PROFILE, KEYNAME, KEYFILE))
|
||||
aws_session = boto3.session.Session(profile_name=PROFILE)
|
||||
ec2_client = aws_session.resource('ec2')
|
||||
boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address
|
||||
print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address)
|
||||
print("Boulder Server at (EC2 private ip): %s"%boulder_url)
|
||||
|
||||
print("Determining Subnet")
|
||||
for subnet in ec2_client.subnets.all():
|
||||
if should_use_subnet(subnet):
|
||||
subnet_id = subnet.id
|
||||
vpc_id = subnet.vpc.id
|
||||
break
|
||||
else:
|
||||
print("No usable subnet exists!")
|
||||
print("Please create a VPC with a subnet named {0}".format(SUBNET_NAME))
|
||||
print("that maps public IPv4 addresses to instances launched in the subnet.")
|
||||
sys.exit(1)
|
||||
if cl_args.boulderonly:
|
||||
sys.exit(0)
|
||||
|
||||
print("Making Security Group")
|
||||
vpc = ec2_client.Vpc(vpc_id)
|
||||
sg_exists = False
|
||||
for sg in vpc.security_groups.all():
|
||||
if sg.group_name == SECURITY_GROUP_NAME:
|
||||
security_group_id = sg.id
|
||||
sg_exists = True
|
||||
print(" %s already exists"%SECURITY_GROUP_NAME)
|
||||
if not sg_exists:
|
||||
security_group_id = make_security_group(vpc).id
|
||||
time.sleep(30)
|
||||
# Install and launch client scripts in parallel
|
||||
#-------------------------------------------------------------------------------
|
||||
print("Uploading and running test script in parallel: %s"%cl_args.test_script)
|
||||
print("Output routed to log files in %s"%LOGDIR)
|
||||
# (Advice: always use Manager.Queue, never regular multiprocessing.Queue
|
||||
# the latter has implementation flaws that deadlock it in some circumstances)
|
||||
manager = Manager()
|
||||
outqueue = manager.Queue()
|
||||
inqueue = manager.Queue()
|
||||
|
||||
boulder_preexists = False
|
||||
boulder_servers = ec2_client.instances.filter(Filters=[
|
||||
{'Name': 'tag:Name', 'Values': ['le-boulderserver']},
|
||||
{'Name': 'instance-state-name', 'Values': ['running']}])
|
||||
|
||||
boulder_server = next(iter(boulder_servers), None)
|
||||
|
||||
print("Requesting Instances...")
|
||||
if boulder_server:
|
||||
print("Found existing boulder server:", boulder_server)
|
||||
boulder_preexists = True
|
||||
else:
|
||||
print("Can't find a boulder server, starting one...")
|
||||
boulder_server = make_instance(ec2_client,
|
||||
'le-boulderserver',
|
||||
BOULDER_AMI,
|
||||
KEYNAME,
|
||||
machine_type='t2.micro',
|
||||
#machine_type='t2.medium',
|
||||
security_group_id=security_group_id,
|
||||
subnet_id=subnet_id)
|
||||
|
||||
try:
|
||||
if not cl_args.boulderonly:
|
||||
instances = create_client_instances(ec2_client, targetlist, security_group_id, subnet_id)
|
||||
|
||||
# Configure and launch boulder server
|
||||
#-------------------------------------------------------------------------------
|
||||
print("Waiting on Boulder Server")
|
||||
boulder_server = block_until_instance_ready(boulder_server)
|
||||
print(" server %s"%boulder_server)
|
||||
# launch as many processes as clients to test
|
||||
num_processes = len(targetlist)
|
||||
jobs = [] #keep a reference to current procs
|
||||
|
||||
|
||||
# env.host_string defines the ssh user and host for connection
|
||||
env.host_string = "ubuntu@%s"%boulder_server.public_ip_address
|
||||
print("Boulder Server at (SSH):", env.host_string)
|
||||
if not boulder_preexists:
|
||||
print("Configuring and Launching Boulder")
|
||||
config_and_launch_boulder(boulder_server)
|
||||
# blocking often unnecessary, but cheap EC2 VMs can get very slow
|
||||
block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address,
|
||||
wait_time=10, timeout=500)
|
||||
# initiate process execution
|
||||
for i in range(num_processes):
|
||||
p = mp.Process(target=test_client_process, args=(inqueue, outqueue, boulder_url))
|
||||
jobs.append(p)
|
||||
p.daemon = True # kills subprocesses if parent is killed
|
||||
p.start()
|
||||
|
||||
boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address
|
||||
print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address)
|
||||
print("Boulder Server at (EC2 private ip): %s"%boulder_url)
|
||||
# fill up work queue
|
||||
for ii, target in enumerate(targetlist):
|
||||
inqueue.put((ii, instances[ii].id, target))
|
||||
|
||||
if cl_args.boulderonly:
|
||||
sys.exit(0)
|
||||
# add SENTINELs to end client processes
|
||||
for i in range(num_processes):
|
||||
inqueue.put(SENTINEL)
|
||||
# wait on termination of client processes
|
||||
for p in jobs:
|
||||
p.join()
|
||||
# add SENTINEL to output queue
|
||||
outqueue.put(SENTINEL)
|
||||
|
||||
# Install and launch client scripts in parallel
|
||||
#-------------------------------------------------------------------------------
|
||||
print("Uploading and running test script in parallel: %s"%cl_args.test_script)
|
||||
print("Output routed to log files in %s"%LOGDIR)
|
||||
# (Advice: always use Manager.Queue, never regular multiprocessing.Queue
|
||||
# the latter has implementation flaws that deadlock it in some circumstances)
|
||||
manager = Manager()
|
||||
outqueue = manager.Queue()
|
||||
inqueue = manager.Queue()
|
||||
# clean up
|
||||
execute(local_repo_clean)
|
||||
|
||||
# launch as many processes as clients to test
|
||||
num_processes = len(targetlist)
|
||||
jobs = [] #keep a reference to current procs
|
||||
# print and save summary results
|
||||
results_file = open(LOGDIR+'/results', 'w')
|
||||
outputs = [outq for outq in iter(outqueue.get, SENTINEL)]
|
||||
outputs.sort(key=lambda x: x[0])
|
||||
for outq in outputs:
|
||||
ii, target, status = outq
|
||||
print('%d %s %s'%(ii, target['name'], status))
|
||||
results_file.write('%d %s %s\n'%(ii, target['name'], status))
|
||||
if len(outputs) != num_processes:
|
||||
failure_message = 'FAILURE: Some target machines failed to run and were not tested. ' +\
|
||||
'Tests should be rerun.'
|
||||
print(failure_message)
|
||||
results_file.write(failure_message + '\n')
|
||||
results_file.close()
|
||||
|
||||
finally:
|
||||
cleanup(cl_args, instances, targetlist)
|
||||
|
||||
# kill any connections
|
||||
fabric.network.disconnect_all()
|
||||
|
||||
|
||||
# initiate process execution
|
||||
for i in range(num_processes):
|
||||
p = mp.Process(target=test_client_process, args=(inqueue, outqueue, boulder_url))
|
||||
jobs.append(p)
|
||||
p.daemon = True # kills subprocesses if parent is killed
|
||||
p.start()
|
||||
|
||||
# fill up work queue
|
||||
for ii, target in enumerate(targetlist):
|
||||
inqueue.put((ii, instances[ii].id, target))
|
||||
|
||||
# add SENTINELs to end client processes
|
||||
for i in range(num_processes):
|
||||
inqueue.put(SENTINEL)
|
||||
# wait on termination of client processes
|
||||
for p in jobs:
|
||||
p.join()
|
||||
# add SENTINEL to output queue
|
||||
outqueue.put(SENTINEL)
|
||||
|
||||
# clean up
|
||||
execute(local_repo_clean)
|
||||
|
||||
# print and save summary results
|
||||
results_file = open(LOGDIR+'/results', 'w')
|
||||
outputs = [outq for outq in iter(outqueue.get, SENTINEL)]
|
||||
outputs.sort(key=lambda x: x[0])
|
||||
for outq in outputs:
|
||||
ii, target, status = outq
|
||||
print('%d %s %s'%(ii, target['name'], status))
|
||||
results_file.write('%d %s %s\n'%(ii, target['name'], status))
|
||||
if len(outputs) != num_processes:
|
||||
failure_message = 'FAILURE: Some target machines failed to run and were not tested. ' +\
|
||||
'Tests should be rerun.'
|
||||
print(failure_message)
|
||||
results_file.write(failure_message + '\n')
|
||||
results_file.close()
|
||||
|
||||
finally:
|
||||
cleanup(cl_args, instances, targetlist)
|
||||
|
||||
# kill any connections
|
||||
fabric.network.disconnect_all()
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
Loading…
Reference in a new issue