enos: poll for LDAP server readiness when populating org, groups, and users

* enos: poll for LDAP server readiness when populating org, groups, and users

The prior implementation had a hard 10 second sleep waiting for the
container to start up. That is not enough time as we see regular
failures in CI:

```
│ Error: exit status 1
│
│ Error: Execution Error
│
│   with module.set_up_external_integration_target.enos_remote_exec.populate_ldap,
│   on ../../modules/set_up_external_integration_target/main.tf line 70, in resource "enos_remote_exec" "populate_ldap":
│   70: resource "enos_remote_exec" "populate_ldap" {
│
│ failed to execute commands due to: running script:
│ [/home/runner/actions-runner/_work/vault-enterprise/vault-enterprise/enos/modules/set_up_external_integration_target/scripts/populate-ldap.sh]
│ failed, due to: 1 error occurred:
│ 	* executing script: Process exited with status 255: ldap_sasl_bind(SIMPLE):
│ Can't contact LDAP server (-1)
```

Signed-off-by: Ryan Cragun <me@ryan.ec>
Co-authored-by: Ryan Cragun <me@ryan.ec>

enos/modules/set_up_external_integration_target/main.tf

@@ -73,10 +73,12 @@ resource "enos_remote_exec" "populate_ldap" {
   scripts = [abspath("${path.module}/scripts/populate-ldap.sh")]
 
   environment = {
-    LDAP_SERVER   = local.ldap_server.host.private_ip
-    LDAP_PORT     = local.ldap_server.port
-    LDAP_ADMIN_PW = local.ldap_server.admin_pw
-    LDAP_DOMAIN   = local.ldap_server.domain
+    LDAP_SERVER     = local.ldap_server.host.private_ip
+    LDAP_PORT       = local.ldap_server.port
+    LDAP_ADMIN_PW   = local.ldap_server.admin_pw
+    LDAP_DOMAIN     = local.ldap_server.domain
+    RETRY_INTERVAL  = var.retry_interval
+    TIMEOUT_SECONDS = var.timeout
   }
 
   transport = {

enos/modules/set_up_external_integration_target/scripts/populate-ldap.sh

@@ -13,10 +13,8 @@ fail() {
 [[ -z "$LDAP_PORT" ]] && fail "LDAP_PORT env variable has not been set"
 [[ -z "$LDAP_ADMIN_PW" ]] && fail "LDAP_ADMIN_PW env variable has not been set"
 [[ -z "$LDAP_DOMAIN" ]] && fail "LDAP_DOMAIN env variable has not been set"
-
-echo "OpenLDAP: Checking for OpenLDAP Server Connection: ${LDAP_SERVER}:${LDAP_PORT}"
-# Wait for LDAP server to be ready
-sleep 10
+[[ -z "$RETRY_INTERVAL" ]] && fail "RETRY_INTERVAL env variable has not been set"
+[[ -z "$TIMEOUT_SECONDS" ]] && fail "TIMEOUT_SECONDS env variable has not been set"
 
 # Extract domain components from LDAP_DOMAIN (e.g., "enos.com" -> "dc=enos,dc=com")
 IFS='.' read -ra DOMAIN_PARTS <<< "$LDAP_DOMAIN"
@@ -29,16 +27,37 @@ for part in "${DOMAIN_PARTS[@]}"; do
   fi
 done
 
+echo "OpenLDAP: Checking for OpenLDAP Server Connection: ${LDAP_SERVER}:${LDAP_PORT}"
 echo "OpenLDAP: Using domain DN: ${DOMAIN_DN}"
 echo "OpenLDAP: Testing connection with admin credentials"
 
-# Test connection
-ldapsearch -x -H "ldap://${LDAP_SERVER}:${LDAP_PORT}" -b "${DOMAIN_DN}" -D "cn=admin,${DOMAIN_DN}" -w "${LDAP_ADMIN_PW}" -s base
+begin_time=$(date +%s)
+end_time=$((begin_time + TIMEOUT_SECONDS))
+test_conn_out=""
+test_conn_res=""
+declare -i tries=0
+while [ "$(date +%s)" -lt "$end_time" ]; do
+  # Test connection
+  tries+=1
+  test_conn_out=$(ldapsearch -x -H "ldap://${LDAP_SERVER}:${LDAP_PORT}" -b "${DOMAIN_DN}" -D "cn=admin,${DOMAIN_DN}" -w "${LDAP_ADMIN_PW}" -s base 2>&1)
+  test_conn_res=$?
+  if [ "$test_conn_res" -eq 0 ]; then
+    break
+  fi
+
+  echo "Unable to connect to ldap://${LDAP_SERVER}:${LDAP_PORT} cn=admin,${DOMAIN_DN}, attempt: ${tries}, exit code: ${test_conn_res}, error: ${test_conn_out}, retrying..."
+  sleep "$RETRY_INTERVAL"
+done
+
+if [ "$test_conn_res" -ne 0 ]; then
+  echo "Timed out waiting to connect to ldap://${LDAP_SERVER}:${LDAP_PORT} cn=admin,${DOMAIN_DN}, attempt: ${tries}, exit code: ${test_conn_res}, error: ${test_conn_out}" 2>&1
+  exit "$test_conn_res"
+fi
 
 echo "OpenLDAP: Creating organizational units"
 # Creating Users and Groups Org Units LDIF file
 OU_LDIF="ou.ldif"
-cat << EOF > ${OU_LDIF}
+cat << EOF > "${OU_LDIF}"
 dn: ou=users,${DOMAIN_DN}
 objectClass: organizationalUnit
 ou: users
@@ -48,11 +67,11 @@ objectClass: organizationalUnit
 ou: groups
 EOF
 
-ldapadd -x -H "ldap://${LDAP_SERVER}:${LDAP_PORT}" -D "cn=admin,${DOMAIN_DN}" -w "${LDAP_ADMIN_PW}" -f ${OU_LDIF} || echo "OUs may already exist"
+ldapadd -x -H "ldap://${LDAP_SERVER}:${LDAP_PORT}" -D "cn=admin,${DOMAIN_DN}" -w "${LDAP_ADMIN_PW}" -f "${OU_LDIF}" || echo "OUs may already exist"
 
 echo "OpenLDAP: Creating test users"
 USER_LDIF="users.ldif"
-cat << EOF > ${USER_LDIF}
+cat << EOF > "${USER_LDIF}"
 # User: enos
 dn: uid=enos,ou=users,${DOMAIN_DN}
 objectClass: inetOrgPerson
@@ -92,6 +111,6 @@ uid: svc-delete
 userPassword: ${LDAP_ADMIN_PW}
 EOF
 
-ldapadd -x -H "ldap://${LDAP_SERVER}:${LDAP_PORT}" -D "cn=admin,${DOMAIN_DN}" -w "${LDAP_ADMIN_PW}" -f ${USER_LDIF} || echo "Users may already exist"
+ldapadd -x -H "ldap://${LDAP_SERVER}:${LDAP_PORT}" -D "cn=admin,${DOMAIN_DN}" -w "${LDAP_ADMIN_PW}" -f "${USER_LDIF}" || echo "Users may already exist"
 
 echo "LDAP population completed successfully."

enos/modules/set_up_external_integration_target/variables.tf

@@ -16,14 +16,6 @@ variable "ip_version" {
   default     = "4"
 }
 
-variable "ports" {
-  description = "Port configuration for services"
-  type = map(object({
-    port        = string
-    description = string
-  }))
-}
-
 variable "ldap_version" {
   type        = string
   description = "OpenLDAP Server Version to use"
@@ -35,3 +27,23 @@ variable "packages" {
   description = "A list of packages to install via the target host package manager"
   default     = []
 }
+
+variable "ports" {
+  description = "Port configuration for services"
+  type = map(object({
+    port        = string
+    description = string
+  }))
+}
+
+variable "retry_interval" {
+  type        = number
+  description = "How many seconds to wait between each retry"
+  default     = 2
+}
+
+variable "timeout" {
+  type        = number
+  description = "The max number of seconds to wait before timing out"
+  default     = 60
+}

enos/modules/verify_secrets_engines/scripts/ldap/Dynamic-roles/dynamic-roles-deletion.sh

@@ -119,6 +119,7 @@ test_role_deletion_with_active_leases() {
   "$binpath" lease revoke -prefix "${MOUNT}/creds/${lease_role}" > /dev/null 2>&1
 
   # Define the check function
+  # shellcheck disable=SC2329
   wait_for_user_deletion() {
     check_user=$(ldapsearch -x -H "ldap://${LDAP_SERVER}:${LDAP_PORT}" \
         -b "dc=${LDAP_USERNAME},dc=com" \