diff --git a/builtin/logical/mssql/backend_test.go b/builtin/logical/mssql/backend_test.go
index 438ee5f374..9e6b3f9604 100644
--- a/builtin/logical/mssql/backend_test.go
+++ b/builtin/logical/mssql/backend_test.go
@@ -104,6 +104,10 @@ func TestBackend_config_connection(t *testing.T) {
 }
 
 func TestBackend_basic(t *testing.T) {
+	if os.Getenv(logicaltest.TestEnvVar) == "" {
+		t.Skip(fmt.Sprintf("Acceptance tests skipped unless env '%s' set", logicaltest.TestEnvVar))
+	}
+
 	b, _ := Factory(context.Background(), logical.TestBackendConfig())
 
 	cleanup, connURL := prepareMSSQLTestContainer(t)
@@ -122,6 +126,10 @@ func TestBackend_basic(t *testing.T) {
 }
 
 func TestBackend_roleCrud(t *testing.T) {
+	if os.Getenv(logicaltest.TestEnvVar) == "" {
+		t.Skip(fmt.Sprintf("Acceptance tests skipped unless env '%s' set", logicaltest.TestEnvVar))
+	}
+
 	b := Backend()
 
 	cleanup, connURL := prepareMSSQLTestContainer(t)
@@ -142,6 +150,10 @@ func TestBackend_roleCrud(t *testing.T) {
 }
 
 func TestBackend_leaseWriteRead(t *testing.T) {
+	if os.Getenv(logicaltest.TestEnvVar) == "" {
+		t.Skip(fmt.Sprintf("Acceptance tests skipped unless env '%s' set", logicaltest.TestEnvVar))
+	}
+
 	b := Backend()
 
 	cleanup, connURL := prepareMSSQLTestContainer(t)
diff --git a/logical/testing/testing.go b/logical/testing/testing.go
index d9da151f11..0f8a9b28d3 100644
--- a/logical/testing/testing.go
+++ b/logical/testing/testing.go
@@ -128,6 +128,11 @@ func Test(tt TestT, c TestCase) {
 		c.PreCheck()
 	}
 
+	// Defer on the teardown, regardless of pass/fail at this point
+	if c.Teardown != nil {
+		defer c.Teardown()
+	}
+
 	// Check that something is provided
 	if c.Backend == nil && c.Factory == nil {
 		tt.Fatal("Must provide either Backend or Factory")
@@ -334,11 +339,6 @@ func Test(tt TestT, c TestCase) {
 				s))
 		}
 	}
-
-	// Cleanup
-	if c.Teardown != nil {
-		c.Teardown()
-	}
 }
 
 // TestCheckMulti is a helper to have multiple checks.