diff --git a/letsencrypt-compatibility-test/MANIFEST.in b/letsencrypt-compatibility-test/MANIFEST.in
new file mode 100644
index 000000000..29862fc12
--- /dev/null
+++ b/letsencrypt-compatibility-test/MANIFEST.in
@@ -0,0 +1 @@
+include letsencrypt_compatibility_test/testdata/rsa1024_key.pem
diff --git a/tests/compatibility/__init__.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/__init__.py
similarity index 100%
rename from tests/compatibility/__init__.py
rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/__init__.py
diff --git a/tests/compatibility/configurators/__init__.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/__init__.py
similarity index 100%
rename from tests/compatibility/configurators/__init__.py
rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/__init__.py
diff --git a/tests/compatibility/configurators/apache/Dockerfile b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/Dockerfile
similarity index 100%
rename from tests/compatibility/configurators/apache/Dockerfile
rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/Dockerfile
diff --git a/tests/compatibility/configurators/apache/__init__.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/__init__.py
similarity index 100%
rename from tests/compatibility/configurators/apache/__init__.py
rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/__init__.py
diff --git a/tests/compatibility/configurators/apache/a2enmod.sh b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh
similarity index 100%
rename from tests/compatibility/configurators/apache/a2enmod.sh
rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh
diff --git a/tests/compatibility/configurators/apache/apache24.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/apache24.py
similarity index 93%
rename from tests/compatibility/configurators/apache/apache24.py
rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/apache24.py
index 44150e7fe..2ffc44976 100644
--- a/tests/compatibility/configurators/apache/apache24.py
+++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/apache24.py
@@ -2,9 +2,9 @@
 
 import zope.interface
 
-from tests.compatibility import errors
-from tests.compatibility import interfaces
-from tests.compatibility.configurators.apache import common as apache_common
+from letsencrypt_compatibility_test import errors
+from letsencrypt_compatibility_test import interfaces
+from letsencrypt_compatibility_test.configurators.apache import common as apache_common
 
 
 # The docker image doesn't actually have the watchdog module, but unless the
diff --git a/tests/compatibility/configurators/apache/common.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py
similarity index 97%
rename from tests/compatibility/configurators/apache/common.py
rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py
index e21ac61d8..518bb52d7 100644
--- a/tests/compatibility/configurators/apache/common.py
+++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py
@@ -9,10 +9,10 @@ import zope.interface
 from letsencrypt import configuration
 from letsencrypt import errors as le_errors
 from letsencrypt_apache import configurator
-from tests.compatibility import errors
-from tests.compatibility import interfaces
-from tests.compatibility import util
-from tests.compatibility.configurators import common as configurators_common
+from letsencrypt_compatibility_test import errors
+from letsencrypt_compatibility_test import interfaces
+from letsencrypt_compatibility_test import util
+from letsencrypt_compatibility_test.configurators import common as configurators_common
 
 
 APACHE_VERSION_REGEX = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE)
diff --git a/tests/compatibility/configurators/common.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/common.py
similarity index 98%
rename from tests/compatibility/configurators/common.py
rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/common.py
index 3e7863c0f..3aa5c62e3 100644
--- a/tests/compatibility/configurators/common.py
+++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/common.py
@@ -8,8 +8,8 @@ import threading
 import docker
 
 from letsencrypt import constants
-from tests.compatibility import errors
-from tests.compatibility import util
+from letsencrypt_compatibility_test import errors
+from letsencrypt_compatibility_test import util
 
 
 logger = logging.getLogger(__name__)
diff --git a/tests/compatibility/errors.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/errors.py
similarity index 100%
rename from tests/compatibility/errors.py
rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/errors.py
diff --git a/tests/compatibility/interfaces.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/interfaces.py
similarity index 100%
rename from tests/compatibility/interfaces.py
rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/interfaces.py
diff --git a/tests/compatibility/test_driver.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/test_driver.py
similarity index 98%
rename from tests/compatibility/test_driver.py
rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/test_driver.py
index 90ad5ec39..e50335b4a 100644
--- a/tests/compatibility/test_driver.py
+++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/test_driver.py
@@ -16,9 +16,10 @@ from letsencrypt import achallenges
 from letsencrypt import errors as le_errors
 from letsencrypt import validator
 from letsencrypt.tests import acme_util
-from tests.compatibility import errors
-from tests.compatibility import util
-from tests.compatibility.configurators.apache import apache24
+
+from letsencrypt_compatibility_test import errors
+from letsencrypt_compatibility_test import util
+from letsencrypt_compatibility_test.configurators.apache import apache24
 
 
 DESCRIPTION = """
diff --git a/tests/compatibility/testdata/rsa1024_key.pem b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/testdata/rsa1024_key.pem
similarity index 100%
rename from tests/compatibility/testdata/rsa1024_key.pem
rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/testdata/rsa1024_key.pem
diff --git a/tests/compatibility/util.py b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py
similarity index 97%
rename from tests/compatibility/util.py
rename to letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py
index 728850c75..03b15d217 100644
--- a/tests/compatibility/util.py
+++ b/letsencrypt-compatibility-test/letsencrypt_compatibility_test/util.py
@@ -11,7 +11,8 @@ import tarfile
 from acme import jose
 from acme import test_util
 from letsencrypt import constants
-from tests.compatibility import errors
+
+from letsencrypt_compatibility_test import errors
 
 
 _KEY_BASE = "rsa1024_key.pem"
diff --git a/tests/setup.py b/letsencrypt-compatibility-test/setup.py
similarity index 72%
rename from tests/setup.py
rename to letsencrypt-compatibility-test/setup.py
index 5810b66f9..f02041e55 100644
--- a/tests/setup.py
+++ b/letsencrypt-compatibility-test/setup.py
@@ -12,12 +12,12 @@ install_requires = [
 ]
 
 setup(
-    name='compatibility-test',
+    name='letsencrypt-compatibility-test',
     packages=find_packages(),
     install_requires=install_requires,
     entry_points={
         'console_scripts': [
-            'compatibility-test = compatibility.test_driver:main',
+            'letsencrypt-compatibility-test = letsencrypt_compatibility_test.test_driver:main',
         ],
     },
 )
diff --git a/tests/MANIFEST.in b/tests/MANIFEST.in
deleted file mode 100644
index be22c3801..000000000
--- a/tests/MANIFEST.in
+++ /dev/null
@@ -1 +0,0 @@
-recursive-include tests/compatibility/testdata/ *
diff --git a/tests/__init__.py b/tests/__init__.py
deleted file mode 100644
index ea250d700..000000000
--- a/tests/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-"""Let's Encrypt tests"""