diff --git a/letsencrypt-auto-source/README.md b/letsencrypt-auto-source/README.md
new file mode 100644
index 000000000..f37f3dfbd
--- /dev/null
+++ b/letsencrypt-auto-source/README.md
@@ -0,0 +1,12 @@
+# Legacy letsencrypt-auto files
+
+`certbot-auto` and `letsencrypt-auto` were two names for the same self-updating
+shell script that wrapped Certbot. Old versions of the script continue to rely
+on pulling `letsencrypt-auto` and `letsencrypt-auto.sig` from this directory hosted on Github to download and
+verify updates. We're keeping these files and the tests for them around to
+prevent these old scripts from breaking.
+
+If we need or want to remove these files and tests in the future, we can, but
+before we do, we should write a Let's Encrypt forum post describing the error
+message users will see and how they can work around the problem. See
+https://github.com/certbot/certbot/issues/8812 for more info.
diff --git a/tests/modification-check.py b/tests/modification-check.py
index c1530d1f9..d685225a8 100755
--- a/tests/modification-check.py
+++ b/tests/modification-check.py
@@ -10,12 +10,11 @@ import os
 # taken from our v1.14.0 tag which was the last release we intended to make
 # changes to certbot-auto.
 #
-# Deleting letsencrypt-auto-source/letsencrypt-auto and
-# letsencrypt-auto-source/letsencrypt-auto.sig can be done once we're
-# comfortable breaking any certbot-auto scripts that haven't already updated to
-# the last version. See
-# https://opensource.eff.org/eff-open-source/pl/65geri7c4tr6iqunc1rpb3mpna for
-# more info.
+# We can delete this script and the files under letsencrypt-auto-source when
+# we're comfortable breaking any old certbot-auto scripts that haven't updated
+# to the last version of the script yet.  See
+# https://opensource.eff.org/eff-open-source/pl/65geri7c4tr6iqunc1rpb3mpna and
+# letsencrypt-auto-source/README.md for more info.
 EXPECTED_FILES = {
     os.path.join('letsencrypt-auto-source', 'letsencrypt-auto'):
         'b997e3608526650a08e36e682fc3bf0c29903c06fa5ba4cc49308c43832450c2',