From 34deb466255f6ee1669ae2629bbf8f334b6c9079 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 27 May 2021 11:07:24 -0700 Subject: [PATCH] capture stdout and stderr separately --- .../certbot_tests/context.py | 2 +- .../certbot_tests/test_main.py | 42 ++++++++----------- .../utils/certbot_call.py | 12 ++++-- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index 9af1bc15c..c86a06754 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -61,7 +61,7 @@ class IntegrationTestsContext: Execute certbot with given args, not renewing certificates by default. :param args: args to pass to certbot :param force_renew: set to False to not renew by default - :return: output of certbot execution + :return: stdout and stderr from certbot execution """ command = ['--authenticator', 'standalone', '--installer', 'null'] command.extend(args) diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index bc1e5997a..96566e377 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -78,9 +78,9 @@ def test_registration_override(context): def test_prepare_plugins(context): """Test that plugins are correctly instantiated and displayed.""" - output = context.certbot(['plugins', '--init', '--prepare']) + stdout, _ = context.certbot(['plugins', '--init', '--prepare']) - assert 'webroot' in output + assert 'webroot' in stdout def test_http_01(context): @@ -407,9 +407,9 @@ def test_invalid_domain_with_dns_challenge(context): '--manual-cleanup-hook', context.manual_dns_cleanup_hook ]) - output = context.certbot(['certificates']) + stdout, _ = context.certbot(['certificates']) - assert context.get_domain('fail-dns1') not in output + assert context.get_domain('fail-dns1') not in stdout def test_reuse_key(context): @@ -614,11 +614,11 @@ def test_revoke_and_unregister(context): context.certbot(['unregister']) - output = context.certbot(['certificates']) + stdout, _ = context.certbot(['certificates']) - assert cert1 not in output - assert cert2 not in output - assert cert3 in output + assert cert1 not in stdout + assert cert2 not in stdout + assert cert3 in stdout def test_revoke_mutual_exclusive_flags(context): @@ -630,7 +630,7 @@ def test_revoke_mutual_exclusive_flags(context): 'revoke', '--cert-name', cert, '--cert-path', join(context.config_dir, 'live', cert, 'fullchain.pem') ]) - assert 'Exactly one of --cert-path or --cert-name must be specified' in error.value.output + assert 'Exactly one of --cert-path or --cert-name must be specified' in error.value.stderr def test_revoke_multiple_lineages(context): @@ -685,13 +685,11 @@ def test_wildcard_certificates(context): def test_ocsp_status_stale(context): """Test retrieval of OCSP statuses for staled config""" sample_data_path = misc.load_sample_data_path(context.workspace) - output = context.certbot(['certificates', '--config-dir', sample_data_path]) + stdout, _ = context.certbot(['certificates', '--config-dir', sample_data_path]) - # Output counts are doubled because context.certbot sets --verbose and - # messages are logged before they are printed to the user. - assert output.count('TEST_CERT') == 4, ('Did not find two test certs as expected ({0})' + assert stdout.count('TEST_CERT') == 4, ('Did not find two test certs as expected ({0})' .format(output.count('TEST_CERT'))) - assert output.count('EXPIRED') == 4, ('Did not find two expired certs as expected ({0})' + assert stdout.count('EXPIRED') == 4, ('Did not find two expired certs as expected ({0})' .format(output.count('EXPIRED'))) @@ -701,24 +699,20 @@ def test_ocsp_status_live(context): # OSCP 1: Check live certificate OCSP status (VALID) context.certbot(['--domains', cert]) - output = context.certbot(['certificates']) + stdout, _ = context.certbot(['certificates']) - # Output counts are doubled because context.certbot sets --verbose and - # messages are logged before they are printed to the user. - assert output.count('VALID') == 2, 'Expected {0} to be VALID'.format(cert) - assert output.count('EXPIRED') == 0, 'Did not expect {0} to be EXPIRED'.format(cert) + assert stdout.count('VALID') == 2, 'Expected {0} to be VALID'.format(cert) + assert stdout.count('EXPIRED') == 0, 'Did not expect {0} to be EXPIRED'.format(cert) # OSCP 2: Check live certificate OCSP status (REVOKED) context.certbot(['revoke', '--cert-name', cert, '--no-delete-after-revoke']) # Sometimes in oldest tests (using openssl binary and not cryptography), the OCSP status is # not seen immediately by Certbot as invalid. Waiting few seconds solves this transient issue. time.sleep(5) - output = context.certbot(['certificates']) + stdout, _ = context.certbot(['certificates']) - # Output counts are doubled because context.certbot sets --verbose and - # messages are logged before they are printed to the user. - assert output.count('INVALID') == 2, 'Expected {0} to be INVALID'.format(cert) - assert output.count('REVOKED') == 2, 'Expected {0} to be REVOKED'.format(cert) + assert stdout.count('INVALID') == 2, 'Expected {0} to be INVALID'.format(cert) + assert stdout.count('REVOKED') == 2, 'Expected {0} to be REVOKED'.format(cert) def test_ocsp_renew(context): diff --git a/certbot-ci/certbot_integration_tests/utils/certbot_call.py b/certbot-ci/certbot_integration_tests/utils/certbot_call.py index 919deeb7e..32f13ac37 100755 --- a/certbot-ci/certbot_integration_tests/utils/certbot_call.py +++ b/certbot-ci/certbot_integration_tests/utils/certbot_call.py @@ -25,14 +25,18 @@ def certbot_test(certbot_args, directory_url, http_01_port, tls_alpn_01_port, :param str config_dir: certbot configuration directory to use :param str workspace: certbot current directory to use :param bool force_renew: set False to not force renew existing certificates (default: True) - :return: stdout as string - :rtype: str + :return: stdout and stderr as strings + :rtype: `tuple` of `str` """ command, env = _prepare_args_env(certbot_args, directory_url, http_01_port, tls_alpn_01_port, config_dir, workspace, force_renew) - return subprocess.check_output(command, stderr=subprocess.STDOUT, - universal_newlines=True, cwd=workspace, env=env) + proc = subprocess.run(command, capture_output=True, check=False, + universal_newlines=True, cwd=workspace, env=env) + print('--> Certbot log output was:') + print(proc.stderr) + proc.check_returncode() + return proc.stdout, proc.stderr def _prepare_environ(workspace):