From b0c72410bacc6c443b34dadbef179bb94fe6946f Mon Sep 17 00:00:00 2001
From: Jakub Warmuz <jakub@warmuz.org>
Date: Fri, 10 Jul 2015 13:40:51 +0000
Subject: [PATCH] Unified vector loading in letsencrypt.

---
 acme/test_util.py                             |  2 +
 letsencrypt/tests/account_test.py             |  8 +-
 letsencrypt/tests/achallenges_test.py         | 13 +--
 letsencrypt/tests/acme_util.py                | 12 +--
 letsencrypt/tests/client_test.py              | 10 +--
 letsencrypt/tests/crypto_util_test.py         | 79 ++++++++-----------
 letsencrypt/tests/display/ops_test.py         |  6 +-
 letsencrypt/tests/display/revocation_test.py  | 18 ++---
 letsencrypt/tests/proof_of_possession_test.py | 26 ++----
 letsencrypt/tests/renewer_test.py             | 17 ++--
 letsencrypt/tests/revoker_test.py             | 25 ++----
 letsencrypt/tests/test_util.py                |  1 +
 setup.py                                      |  2 +-
 tox.ini                                       |  8 +-
 14 files changed, 88 insertions(+), 139 deletions(-)
 create mode 120000 letsencrypt/tests/test_util.py

diff --git a/acme/test_util.py b/acme/test_util.py
index cec732625..85289a978 100644
--- a/acme/test_util.py
+++ b/acme/test_util.py
@@ -1,3 +1,5 @@
+# Symlinked in letsencrypt/tests/test_util.py, casues duplicate-code
+# warning that cannot be disabled locally.
 """Test utilities.
 
 .. warning:: This module is not part of the public API.
diff --git a/letsencrypt/tests/account_test.py b/letsencrypt/tests/account_test.py
index 9a129dbe6..e19940fe8 100644
--- a/letsencrypt/tests/account_test.py
+++ b/letsencrypt/tests/account_test.py
@@ -1,7 +1,6 @@
 """Tests for letsencrypt.account."""
 import datetime
 import os
-import pkg_resources
 import shutil
 import stat
 import tempfile
@@ -15,9 +14,10 @@ from acme import messages
 
 from letsencrypt import errors
 
+from letsencrypt.tests import test_util
 
-KEY = jose.JWKRSA.load(pkg_resources.resource_string(
-    "letsencrypt.tests", os.path.join("testdata", "rsa512_key_2.pem")))
+
+KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key_2.pem"))
 
 
 class AccountTest(unittest.TestCase):
@@ -61,7 +61,7 @@ class ReportNewAccountTest(unittest.TestCase):
     """Tests for letsencrypt.account.report_new_account."""
 
     def setUp(self):
-        self.config = mock.MagicMock(config_dir='/etc/letsencrypt')
+        self.config = mock.MagicMock(config_dir="/etc/letsencrypt")
         reg = messages.Registration.from_data(email="rhino@jungle.io")
         reg = reg.update(recovery_token="ECCENTRIC INVISIBILITY RHINOCEROS")
         self.acc = mock.MagicMock(regr=messages.RegistrationResource(
diff --git a/letsencrypt/tests/achallenges_test.py b/letsencrypt/tests/achallenges_test.py
index 4faa34d12..253cc20d7 100644
--- a/letsencrypt/tests/achallenges_test.py
+++ b/letsencrypt/tests/achallenges_test.py
@@ -1,17 +1,15 @@
 """Tests for letsencrypt.achallenges."""
-import os
-import pkg_resources
 import unittest
 
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization
 import OpenSSL
 
 from acme import challenges
 from acme import jose
 
 from letsencrypt import crypto_util
+
 from letsencrypt.tests import acme_util
+from letsencrypt.tests import test_util
 
 
 class DVSNITest(unittest.TestCase):
@@ -21,12 +19,7 @@ class DVSNITest(unittest.TestCase):
         self.chall = acme_util.chall_to_challb(
             challenges.DVSNI(r="r_value", nonce="12345ABCDE"), "pending")
         self.response = challenges.DVSNIResponse()
-        key = jose.JWKRSA(key=jose.ComparableRSAKey(
-            serialization.load_pem_private_key(
-                pkg_resources.resource_string(
-                    "letsencrypt.tests", os.path.join(
-                        "testdata", "rsa512_key.pem")),
-                password=None, backend=default_backend())))
+        key = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem"))
 
         from letsencrypt.achallenges import DVSNI
         self.achall = DVSNI(challb=self.chall, domain="example.com", key=key)
diff --git a/letsencrypt/tests/acme_util.py b/letsencrypt/tests/acme_util.py
index 8e19a9ca8..0f504fde3 100644
--- a/letsencrypt/tests/acme_util.py
+++ b/letsencrypt/tests/acme_util.py
@@ -1,21 +1,15 @@
 """ACME utilities for testing."""
 import datetime
 import itertools
-import os
-import pkg_resources
-
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization
 
 from acme import challenges
 from acme import jose
 from acme import messages
 
+from letsencrypt.tests import test_util
 
-KEY = serialization.load_pem_private_key(
-    pkg_resources.resource_string(
-        __name__, os.path.join('testdata', 'rsa512_key.pem')),
-    password=None, backend=default_backend())
+
+KEY = test_util.load_rsa_private_key('rsa512_key.pem')
 
 # Challenges
 SIMPLE_HTTP = challenges.SimpleHTTP(
diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py
index 2d1f1d2fd..b992089cc 100644
--- a/letsencrypt/tests/client_test.py
+++ b/letsencrypt/tests/client_test.py
@@ -1,7 +1,5 @@
 """Tests for letsencrypt.client."""
-import os
 import unittest
-import pkg_resources
 
 import configobj
 import OpenSSL
@@ -14,11 +12,11 @@ from letsencrypt import configuration
 from letsencrypt import errors
 from letsencrypt import le_util
 
+from letsencrypt.tests import test_util
 
-KEY = pkg_resources.resource_string(
-    __name__, os.path.join("testdata", "rsa512_key.pem"))
-CSR_SAN = pkg_resources.resource_string(
-    __name__, os.path.join("testdata", "csr-san.der"))
+
+KEY = test_util.load_vector("rsa512_key.pem")
+CSR_SAN = test_util.load_vector("csr-san.der")
 
 
 class RegisterTest(unittest.TestCase):
diff --git a/letsencrypt/tests/crypto_util_test.py b/letsencrypt/tests/crypto_util_test.py
index 2e1e62797..fd3b60c60 100644
--- a/letsencrypt/tests/crypto_util_test.py
+++ b/letsencrypt/tests/crypto_util_test.py
@@ -1,7 +1,5 @@
 """Tests for letsencrypt.crypto_util."""
 import logging
-import os
-import pkg_resources
 import shutil
 import tempfile
 import unittest
@@ -9,15 +7,13 @@ import unittest
 import OpenSSL
 import mock
 
+from letsencrypt.tests import test_util
 
-RSA256_KEY = pkg_resources.resource_string(
-    'letsencrypt.tests', os.path.join('testdata', 'rsa256_key.pem'))
-RSA512_KEY = pkg_resources.resource_string(
-    'letsencrypt.tests', os.path.join('testdata', 'rsa512_key.pem'))
-CERT = pkg_resources.resource_string(
-    'letsencrypt.tests', os.path.join('testdata', 'cert.pem'))
-SAN_CERT = pkg_resources.resource_string(
-    'letsencrypt.tests', os.path.join('testdata', 'cert-san.pem'))
+
+RSA256_KEY = test_util.load_vector('rsa256_key.pem')
+RSA512_KEY = test_util.load_vector('rsa512_key.pem')
+CERT = test_util.load_vector('cert.pem')
+SAN_CERT = test_util.load_vector('cert-san.pem')
 
 
 class InitSaveKeyTest(unittest.TestCase):
@@ -100,21 +96,17 @@ class ValidCSRTest(unittest.TestCase):
         from letsencrypt.crypto_util import valid_csr
         return valid_csr(csr)
 
-    def _call_testdata(self, name):
-        return self._call(pkg_resources.resource_string(
-            __name__, os.path.join('testdata', name)))
-
     def test_valid_pem_true(self):
-        self.assertTrue(self._call_testdata('csr.pem'))
+        self.assertTrue(self._call(test_util.load_vector('csr.pem')))
 
     def test_valid_pem_san_true(self):
-        self.assertTrue(self._call_testdata('csr-san.pem'))
+        self.assertTrue(self._call(test_util.load_vector('csr-san.pem')))
 
     def test_valid_der_false(self):
-        self.assertFalse(self._call_testdata('csr.der'))
+        self.assertFalse(self._call(test_util.load_vector('csr.der')))
 
     def test_valid_der_san_false(self):
-        self.assertFalse(self._call_testdata('csr-san.der'))
+        self.assertFalse(self._call(test_util.load_vector('csr-san.der')))
 
     def test_empty_false(self):
         self.assertFalse(self._call(''))
@@ -127,16 +119,17 @@ class CSRMatchesPubkeyTest(unittest.TestCase):
     """Tests for letsencrypt.crypto_util.csr_matches_pubkey."""
 
     @classmethod
-    def _call_testdata(cls, name, privkey):
+    def _call(cls, *args, **kwargs):
         from letsencrypt.crypto_util import csr_matches_pubkey
-        return csr_matches_pubkey(pkg_resources.resource_string(
-            __name__, os.path.join('testdata', name)), privkey)
+        return csr_matches_pubkey(*args, **kwargs)
 
     def test_valid_true(self):
-        self.assertTrue(self._call_testdata('csr.pem', RSA512_KEY))
+        self.assertTrue(self._call(
+            test_util.load_vector('csr.pem'), RSA512_KEY))
 
     def test_invalid_false(self):
-        self.assertFalse(self._call_testdata('csr.pem', RSA256_KEY))
+        self.assertFalse(self._call(
+            test_util.load_vector('csr.pem'), RSA256_KEY))
 
 
 class MakeKeyTest(unittest.TestCase):  # pylint: disable=too-few-public-methods
@@ -185,50 +178,42 @@ class GetSANsFromCertTest(unittest.TestCase):
         return get_sans_from_cert(*args, **kwargs)
 
     def test_single(self):
-        self.assertEqual([], self._call(pkg_resources.resource_string(
-            __name__, os.path.join('testdata', 'cert.pem'))))
+        self.assertEqual([], self._call(test_util.load_vector('cert.pem')))
 
     def test_san(self):
         self.assertEqual(
             ['example.com', 'www.example.com'],
-            self._call(pkg_resources.resource_string(
-                __name__, os.path.join('testdata', 'cert-san.pem'))))
+            self._call(test_util.load_vector('cert-san.pem')))
 
 
 class GetSANsFromCSRTest(unittest.TestCase):
     """Tests for letsencrypt.crypto_util.get_sans_from_csr."""
-    def test_extract_one_san(self):
+
+    @classmethod
+    def _call(cls, *args, **kwargs):
         from letsencrypt.crypto_util import get_sans_from_csr
-        csr = pkg_resources.resource_string(
-            __name__, os.path.join('testdata', 'csr.pem'))
-        self.assertEqual(get_sans_from_csr(csr), ['example.com'])
+        return get_sans_from_csr(*args, **kwargs)
+
+    def test_extract_one_san(self):
+        self.assertEqual(['example.com'], self._call(
+            test_util.load_vector('csr.pem')))
 
     def test_extract_two_sans(self):
-        from letsencrypt.crypto_util import get_sans_from_csr
-        csr = pkg_resources.resource_string(
-            __name__, os.path.join('testdata', 'csr-san.pem'))
-        self.assertEqual(get_sans_from_csr(csr), ['example.com',
-                                                  'www.example.com'])
+        self.assertEqual(['example.com', 'www.example.com'], self._call(
+            test_util.load_vector('csr-san.pem')))
 
     def test_extract_six_sans(self):
-        from letsencrypt.crypto_util import get_sans_from_csr
-        csr = pkg_resources.resource_string(
-            __name__, os.path.join('testdata', 'csr-6sans.pem'))
-        self.assertEqual(get_sans_from_csr(csr),
+        self.assertEqual(self._call(test_util.load_vector('csr-6sans.pem')),
                          ["example.com", "example.org", "example.net",
                           "example.info", "subdomain.example.com",
                           "other.subdomain.example.com"])
 
     def test_parse_non_csr(self):
-        from letsencrypt.crypto_util import get_sans_from_csr
-        self.assertRaises(OpenSSL.crypto.Error, get_sans_from_csr,
-                          "hello there")
+        self.assertRaises(OpenSSL.crypto.Error, self._call, "hello there")
 
     def test_parse_no_sans(self):
-        from letsencrypt.crypto_util import get_sans_from_csr
-        csr = pkg_resources.resource_string(
-            __name__, os.path.join('testdata', 'csr-nosans.pem'))
-        self.assertEqual([], get_sans_from_csr(csr))
+        self.assertEqual(
+            [], self._call(test_util.load_vector('csr-nosans.pem')))
 
 
 if __name__ == '__main__':
diff --git a/letsencrypt/tests/display/ops_test.py b/letsencrypt/tests/display/ops_test.py
index 59b8d9b5c..3a0c627ce 100644
--- a/letsencrypt/tests/display/ops_test.py
+++ b/letsencrypt/tests/display/ops_test.py
@@ -1,6 +1,5 @@
 """Test letsencrypt.display.ops."""
 import os
-import pkg_resources
 import sys
 import tempfile
 import unittest
@@ -16,9 +15,10 @@ from letsencrypt import interfaces
 
 from letsencrypt.display import util as display_util
 
+from letsencrypt.tests import test_util
 
-KEY = jose.JWKRSA.load(pkg_resources.resource_string(
-    "letsencrypt.tests", os.path.join("testdata", "rsa512_key.pem")))
+
+KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem"))
 
 
 class ChoosePluginTest(unittest.TestCase):
diff --git a/letsencrypt/tests/display/revocation_test.py b/letsencrypt/tests/display/revocation_test.py
index cb877499a..6e9763006 100644
--- a/letsencrypt/tests/display/revocation_test.py
+++ b/letsencrypt/tests/display/revocation_test.py
@@ -1,6 +1,4 @@
 """Test :mod:`letsencrypt.display.revocation`."""
-import os
-import pkg_resources
 import sys
 import unittest
 
@@ -9,15 +7,14 @@ import zope.component
 
 from letsencrypt.display import util as display_util
 
+from letsencrypt.tests import test_util
+
 
 class DisplayCertsTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.revoker import Cert
-        base_package = "letsencrypt.tests"
-        self.cert0 = Cert(pkg_resources.resource_filename(
-            base_package, os.path.join("testdata", "cert.pem")))
-        self.cert1 = Cert(pkg_resources.resource_filename(
-            base_package, os.path.join("testdata", "cert-san.pem")))
+        self.cert0 = Cert(test_util.vector_path("cert.pem"))
+        self.cert1 = Cert(test_util.vector_path("cert-san.pem"))
 
         self.certs = [self.cert0, self.cert1]
 
@@ -62,9 +59,7 @@ class MoreInfoCertTest(unittest.TestCase):
 class SuccessRevocationTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.revoker import Cert
-        base_package = "letsencrypt.tests"
-        self.cert = Cert(pkg_resources.resource_filename(
-            base_package, os.path.join("testdata", "cert.pem")))
+        self.cert = Cert(test_util.vector_path("cert.pem"))
 
     @classmethod
     def _call(cls, cert):
@@ -82,8 +77,7 @@ class SuccessRevocationTest(unittest.TestCase):
 class ConfirmRevocationTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.revoker import Cert
-        self.cert = Cert(pkg_resources.resource_filename(
-            "letsencrypt.tests", os.path.join("testdata", "cert.pem")))
+        self.cert = Cert(test_util.vector_path("cert.pem"))
 
     @classmethod
     def _call(cls, cert):
diff --git a/letsencrypt/tests/proof_of_possession_test.py b/letsencrypt/tests/proof_of_possession_test.py
index ef0ffc21e..bfe3478d1 100644
--- a/letsencrypt/tests/proof_of_possession_test.py
+++ b/letsencrypt/tests/proof_of_possession_test.py
@@ -1,11 +1,8 @@
 """Tests for letsencrypt.proof_of_possession."""
 import os
-import pkg_resources
 import tempfile
 import unittest
 
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization
 import mock
 
 from acme import challenges
@@ -16,22 +13,15 @@ from letsencrypt import achallenges
 from letsencrypt import proof_of_possession
 from letsencrypt.display import util as display_util
 
+from letsencrypt.tests import test_util
 
-BASE_PACKAGE = "letsencrypt.tests"
-CERT0_PATH = pkg_resources.resource_filename(
-    BASE_PACKAGE, os.path.join("testdata", "cert.der"))
-CERT2_PATH = pkg_resources.resource_filename(
-    BASE_PACKAGE, os.path.join("testdata", "dsa_cert.pem"))
-CERT2_KEY_PATH = pkg_resources.resource_filename(
-    BASE_PACKAGE, os.path.join("testdata", "dsa512_key.pem"))
-CERT3_PATH = pkg_resources.resource_filename(
-    BASE_PACKAGE, os.path.join("testdata", "matching_cert.pem"))
-CERT3_KEY_PATH = pkg_resources.resource_filename(
-    BASE_PACKAGE, os.path.join("testdata", "rsa512_key_2.pem"))
-with open(CERT3_KEY_PATH) as cert3_file:
-    CERT3_KEY = serialization.load_pem_private_key(
-        cert3_file.read(), password=None,
-        backend=default_backend()).public_key()
+
+CERT0_PATH = test_util.vector_path("cert.der")
+CERT2_PATH = test_util.vector_path("dsa_cert.pem")
+CERT2_KEY_PATH = test_util.vector_path("dsa512_key.pem")
+CERT3_PATH = test_util.vector_path("matching_cert.pem")
+CERT3_KEY_PATH = test_util.vector_path("rsa512_key_2.pem")
+CERT3_KEY = test_util.load_rsa_private_key("rsa512_key_2.pem").public_key()
 
 
 class ProofOfPossessionTest(unittest.TestCase):
diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py
index 4ee7cc0e8..65bfce314 100644
--- a/letsencrypt/tests/renewer_test.py
+++ b/letsencrypt/tests/renewer_test.py
@@ -2,22 +2,20 @@
 import datetime
 import os
 import tempfile
-import pkg_resources
 import shutil
 import unittest
 
 import configobj
 import mock
-import OpenSSL
 import pytz
 
 from letsencrypt import configuration
 from letsencrypt.storage import ALL_FOUR
 
+from letsencrypt.tests import test_util
 
-CERT = OpenSSL.crypto.load_certificate(
-    OpenSSL.crypto.FILETYPE_PEM, pkg_resources.resource_string(
-        'letsencrypt.tests', os.path.join('testdata', 'cert.pem')))
+
+CERT = test_util.load_cert('cert.pem')
 
 
 def unlink_all(rc_object):
@@ -295,8 +293,7 @@ class RenewableCertTests(unittest.TestCase):
                 self.assertFalse(self.test_rc.has_pending_deployment())
 
     def _test_notafterbefore(self, function, timestamp):
-        test_cert = pkg_resources.resource_string(
-            "letsencrypt.tests", os.path.join("testdata", "cert.pem"))
+        test_cert = test_util.load_vector("cert.pem")
         os.symlink(os.path.join("..", "..", "archive", "example.org",
                                 "cert12.pem"), self.test_rc.cert)
         with open(self.test_rc.cert, "w") as f:
@@ -319,8 +316,7 @@ class RenewableCertTests(unittest.TestCase):
     def test_time_interval_judgments(self, mock_datetime):
         """Test should_autodeploy() and should_autorenew() on the basis
         of expiry time windows."""
-        test_cert = pkg_resources.resource_string(
-            "letsencrypt.tests", os.path.join("testdata", "cert.pem"))
+        test_cert = test_util.load_vector("cert.pem")
         for kind in ALL_FOUR:
             where = getattr(self.test_rc, kind)
             os.symlink(os.path.join("..", "..", "archive", "example.org",
@@ -561,8 +557,7 @@ class RenewableCertTests(unittest.TestCase):
     def test_renew(self, mock_c, mock_acc_storage, mock_pd):
         from letsencrypt import renewer
 
-        test_cert = pkg_resources.resource_string(
-            "letsencrypt.tests", os.path.join("testdata", "cert-san.pem"))
+        test_cert = test_util.load_vector("cert-san.pem")
         for kind in ALL_FOUR:
             os.symlink(os.path.join("..", "..", "archive", "example.org",
                                     kind + "1.pem"),
diff --git a/letsencrypt/tests/revoker_test.py b/letsencrypt/tests/revoker_test.py
index 92eeaf92d..87dab4eb8 100644
--- a/letsencrypt/tests/revoker_test.py
+++ b/letsencrypt/tests/revoker_test.py
@@ -1,7 +1,6 @@
 """Test letsencrypt.revoker."""
 import csv
 import os
-import pkg_resources
 import shutil
 import tempfile
 import unittest
@@ -13,10 +12,11 @@ from letsencrypt import errors
 from letsencrypt import le_util
 from letsencrypt.display import util as display_util
 
+from letsencrypt.tests import test_util
+
 
 KEY = OpenSSL.crypto.load_privatekey(
-    OpenSSL.crypto.FILETYPE_PEM, pkg_resources.resource_string(
-        __name__, os.path.join("testdata", "rsa512_key.pem")))
+    OpenSSL.crypto.FILETYPE_PEM, test_util.load_vector("rsa512_key.pem"))
 
 
 class RevokerBase(unittest.TestCase):  # pylint: disable=too-few-public-methods
@@ -98,8 +98,7 @@ class RevokerTest(RevokerBase):
     def test_revoke_by_wrong_key(self, mock_display, mock_acme):
         mock_display().confirm_revocation.return_value = True
 
-        key_path = pkg_resources.resource_filename(
-            "letsencrypt.tests", os.path.join("testdata", "rsa256_key.pem"))
+        key_path = test_util.vector_path("rsa256_key.pem")
 
         wrong_key = le_util.Key(key_path, open(key_path).read())
         self.revoker.revoke_from_key(wrong_key)
@@ -395,22 +394,14 @@ class CertTest(unittest.TestCase):
 
 def create_revoker_certs():
     """Create a few revoker.Cert objects."""
+    cert0_path = test_util.vector_path("cert.pem")
+    cert1_path = test_util.vector_path("cert-san.pem")
+    key_path = test_util.vector_path("rsa512_key.pem")
+
     from letsencrypt.revoker import Cert
-
-    base_package = "letsencrypt.tests"
-
-    cert0_path = pkg_resources.resource_filename(
-        base_package, os.path.join("testdata", "cert.pem"))
-
-    cert1_path = pkg_resources.resource_filename(
-        base_package, os.path.join("testdata", "cert-san.pem"))
-
     cert0 = Cert(cert0_path)
     cert1 = Cert(cert1_path)
 
-    key_path = pkg_resources.resource_filename(
-        base_package, os.path.join("testdata", "rsa512_key.pem"))
-
     return [cert0_path, cert1_path], [cert0, cert1], key_path
 
 
diff --git a/letsencrypt/tests/test_util.py b/letsencrypt/tests/test_util.py
new file mode 120000
index 000000000..d46ad3bd4
--- /dev/null
+++ b/letsencrypt/tests/test_util.py
@@ -0,0 +1 @@
+../../acme/test_util.py
\ No newline at end of file
diff --git a/setup.py b/setup.py
index b2bd0e652..0c74b296c 100644
--- a/setup.py
+++ b/setup.py
@@ -187,6 +187,6 @@ setup(
         ],
     },
 
-    zip_safe=False,
+    zip_safe=False,  # letsencrypt/tests/test_util.py is a symlink!
     include_package_data=True,
 )
diff --git a/tox.ini b/tox.ini
index aed60f454..2f20e5799 100644
--- a/tox.ini
+++ b/tox.ini
@@ -28,6 +28,12 @@ commands =
 [testenv:lint]
 # recent versions of pylint do not support Python 2.6 (#97, #187)
 basepython = python2.7
+# separating into multiple invocations disables cross package
+# duplicate code checking; if one of the commands fails, others will
+# continue, but tox return code will reflect previous error
 commands =
     pip install -r requirements.txt -e .[dev]
-    pylint --rcfile=.pylintrc letsencrypt acme letsencrypt_apache letsencrypt_nginx
+    pylint --rcfile=.pylintrc letsencrypt
+    pylint --rcfile=.pylintrc acme
+    pylint --rcfile=.pylintrc letsencrypt_apache
+    pylint --rcfile=.pylintrc letsencrypt_nginx