From 3dae110c828fddf4fd6407060efbaa6e6d10e11d Mon Sep 17 00:00:00 2001 From: Vault Automation Date: Mon, 18 May 2026 10:54:15 -0600 Subject: [PATCH] [VAULT-44431] enos: merge changes for enterprise zap scenario into ce/main (#14849) Backport community files that changed as part the enterprise only zap scenarios. This mostly includes fixes to scenario execution, retries, and blackbox SDK tests that were broken. Signed-off-by: Ryan Cragun Co-authored-by: Ryan Cragun --- .copywrite.hcl | 4 +- .github/actions/build-ui/action.yml | 57 ++++ .github/actions/build-vault/action.yml | 2 +- .github/actions/run-enos-scenario/README.md | 114 +++++++ .github/actions/run-enos-scenario/action.yml | 266 ++++++++++++++++ .github/workflows/build.yml | 33 +- .../test-run-enos-scenario-containers.yml | 45 +-- .../test-run-enos-scenario-matrix.yml | 83 ++--- .github/workflows/test-run-enos-scenario.yml | 30 +- .gitignore | 2 + .release/pipeline.hcl | 32 +- enos/enos-descriptions.hcl | 6 - enos/enos-providers.hcl | 3 + enos/enos-scenario-agent.hcl | 52 +--- enos/enos-scenario-autopilot.hcl | 28 +- enos/enos-scenario-dr-replication.hcl | 57 ++-- enos/enos-scenario-plugin.hcl | 51 +-- enos/enos-scenario-pr-replication.hcl | 52 +--- enos/enos-scenario-proxy.hcl | 51 +-- enos/enos-scenario-seal-ha.hcl | 51 +-- enos/enos-scenario-smoke-sdk.hcl | 15 +- enos/enos-scenario-smoke.hcl | 54 +--- enos/enos-scenario-upgrade.hcl | 51 +-- enos/modules/build_local/scripts/build.sh | 4 +- .../cloud_docker_vault_cluster/main.tf | 2 + enos/modules/ec2_info/main.tf | 2 +- enos/modules/vault_run_blackbox_test/main.tf | 32 +- .../vault_run_blackbox_test/variables.tf | 24 ++ sdk/helper/testcluster/blackbox/assertions.go | 11 + sdk/helper/testcluster/blackbox/session.go | 9 +- .../testcluster/blackbox/session_autopilot.go | 58 ++++ .../testcluster/blackbox/session_client.go | 67 ++++ .../testcluster/blackbox/session_cluster.go | 89 ++++++ .../testcluster/blackbox/session_config.go | 89 ++++++ sdk/helper/testcluster/blackbox/session_ha.go | 85 +++++ .../testcluster/blackbox/session_logical.go | 21 ++ .../testcluster/blackbox/session_raft.go | 281 +++++------------ .../blackbox/session_replication.go | 63 ++++ .../testcluster/blackbox/session_seal.go | 27 ++ .../testcluster/blackbox/session_status.go | 32 -- .../testcluster/blackbox/session_util.go | 18 -- vault/external_tests/blackbox/doc.go | 6 + .../integration/external_secrets_test.go | 63 ---- .../blackbox/integration/smoke_test.go | 55 ++-- .../blackbox/plugins/aws/helpers.go | 290 ++++++++++++++++++ .../blackbox/plugins/aws/secrets_aws_test.go | 113 +++++++ .../secret_mongodb_connection_config_test.go | 2 +- ...secret_mongodb_static_roles_delete_test.go | 76 +++-- .../secret_mongodb_static_roles_read_test.go | 35 ++- .../secret_mongodb_static_roles_test.go | 219 +------------ .../mongodb/secret_mongodb_test_helper.go | 273 +++++++++++++++-- ...tgresql_connection_config_failover_test.go | 4 - ...ecret_postgresql_connection_test_helper.go | 4 + .../blackbox/plugins/ldap/helpers.go | 60 ++-- ...rets_ldap_root_credential_rollback_test.go | 53 ++-- .../plugins/ldap/secrets_ldap_test.go | 6 +- .../blackbox/raft/voters_test.go | 13 +- .../blackbox/secrets/secrets_pki_test.go | 6 +- .../blackbox/system/billing_test.go | 19 +- .../verify/version_verification_test.go | 13 +- 60 files changed, 2190 insertions(+), 1173 deletions(-) create mode 100644 .github/actions/build-ui/action.yml create mode 100644 .github/actions/run-enos-scenario/README.md create mode 100644 .github/actions/run-enos-scenario/action.yml create mode 100644 sdk/helper/testcluster/blackbox/session_autopilot.go create mode 100644 sdk/helper/testcluster/blackbox/session_client.go create mode 100644 sdk/helper/testcluster/blackbox/session_cluster.go create mode 100644 sdk/helper/testcluster/blackbox/session_config.go create mode 100644 sdk/helper/testcluster/blackbox/session_ha.go create mode 100644 sdk/helper/testcluster/blackbox/session_replication.go create mode 100644 sdk/helper/testcluster/blackbox/session_seal.go create mode 100644 vault/external_tests/blackbox/doc.go delete mode 100644 vault/external_tests/blackbox/integration/external_secrets_test.go create mode 100644 vault/external_tests/blackbox/plugins/aws/helpers.go create mode 100644 vault/external_tests/blackbox/plugins/aws/secrets_aws_test.go diff --git a/.copywrite.hcl b/.copywrite.hcl index 126b0188df..dac6e0ced1 100644 --- a/.copywrite.hcl +++ b/.copywrite.hcl @@ -2,8 +2,9 @@ schema_version = 1 project { license = "BUSL-1.1" - copyright_year = 2026 + copyright_year = 2016 copyright_holder = "IBM Corp." + ignore_year1 = true # (OPTIONAL) A list of globs that should not have copyright/license headers. # Supports doublestar glob patterns for more flexibility in defining which @@ -14,6 +15,7 @@ project { "enos/.terraform/**", "enos/k8s/.enos/**", "enos/modules/k8s_deploy_vault/raft-config.hcl", + "enos/modules/zap_scan_ent/templates/plan.yml", "helper/pkcs7/**", "plugins/database/postgresql/scram/**", "tools/pipeline/internal/pkg/generate/fixtures/*", diff --git a/.github/actions/build-ui/action.yml b/.github/actions/build-ui/action.yml new file mode 100644 index 0000000000..973401a799 --- /dev/null +++ b/.github/actions/build-ui/action.yml @@ -0,0 +1,57 @@ +# Copyright IBM Corp. 2016, 2026 +# SPDX-License-Identifier: BUSL-1.1 + +--- +name: Build UI +description: | + Build the Vault Web UI assets and cache the results. + +inputs: + github-token: + description: An elevated Github token to access private dependencies if necessary. + default: "" + +outputs: + cache-key: + description: "The cache key for the built UI assets (format: ui-)" + value: ${{ steps.ui-hash.outputs.cache-key }} + cache-hit: + description: "Whether the UI was restored from cache (true) or built fresh (false)" + value: ${{ steps.cache-ui-assets.outputs.cache-hit }} + +runs: + using: composite + steps: + - name: Get UI hash + id: ui-hash + shell: bash + run: | + ui_hash=$(git ls-tree HEAD ui --object-only) + echo "cache-key=ui-${ui_hash}" | tee -a "$GITHUB_OUTPUT" + - name: Set up UI asset cache + id: cache-ui-assets + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + enableCrossOsArchive: true + lookup-only: true + path: http/web_ui + # Only restore the UI asset cache if we haven't modified anything in the ui directory. + # Never do a partial restore of the web_ui if we don't get a cache hit. + key: ${{ steps.ui-hash.outputs.cache-key }} + - name: Install PNPM + if: ${{ steps.cache-ui-assets.outputs.cache-hit != 'true' }} + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + run_install: false + package_json_file: './ui/package.json' + - name: Set up node and pnpm + if: ${{ steps.cache-ui-assets.outputs.cache-hit != 'true' }} + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version-file: ui/package.json + cache: pnpm + cache-dependency-path: ui/pnpm-lock.yaml + - name: Build UI + if: ${{ steps.cache-ui-assets.outputs.cache-hit != 'true' }} + shell: bash + run: make ci-build-ui diff --git a/.github/actions/build-vault/action.yml b/.github/actions/build-vault/action.yml index 4925939674..b72cd2fd97 100644 --- a/.github/actions/build-vault/action.yml +++ b/.github/actions/build-vault/action.yml @@ -117,7 +117,7 @@ runs: id: build-vault-select-license uses: hashicorp-forge/actions-pao-tool/select-license@6997f7457c338e008506005cc370e7b02f7fb421 # v1.0.3 with: - arch: ${{ matrix.goarch }} + arch: ${{ inputs.goarch }} - if: inputs.cgo-enabled == '0' name: ${{ steps.metadata.outputs.build-step-name }} env: diff --git a/.github/actions/run-enos-scenario/README.md b/.github/actions/run-enos-scenario/README.md new file mode 100644 index 0000000000..bdccb77631 --- /dev/null +++ b/.github/actions/run-enos-scenario/README.md @@ -0,0 +1,114 @@ +# run-enos-scenario + +Reusable composite action for running an Enos scenario with standardized retry, debug artifact upload, and destroy cleanup behavior. + +## Behavior + +The action performs this lifecycle: + +1. Optionally installs Terraform. +2. Optionally installs Enos. +3. Prepares a sanitized debug artifact name from the scenario filter. +4. Runs `enos scenario launch`. +5. Retries the launch command once if the first attempt fails. +6. Optionally prints `enos scenario exec --cmd show` and `plan` before retry and after a failed retry. +7. Uploads debug data on failure. +8. Always destroys the scenario and retries destroy once if needed. + +## Inputs + +| Name | Required | Default | Description | +| --- | --- | --- | --- | +| `scenario-filter` | yes | n/a | Scenario filter passed to Enos. | +| `working-directory` | no | `./enos` | Value passed to `--chdir`. | +| `timeout` | no | `45m0s` | Timeout for the first main command attempt. | +| `retry-timeout` | no | `30m0s` | Timeout for the retry attempt. | +| `destroy-timeout` | no | `10m0s` | Timeout for destroy. | +| `show-state-on-retry` | no | `false` | When `true`, prints scenario state and plan before retry and after failed retry. | +| `upload-debug-data` | no | `true` | When `true`, uploads debug data on failure. | +| `debug-data-retention-days` | no | `30` | Artifact retention days for debug data. | +| `debug-data-path` | no | empty | Explicit debug data directory. If empty, uses `ENOS_DEBUG_DATA_ROOT_DIR` or `/tmp/enos-debug-data`. | +| `setup-terraform` | no | `true` | When `true`, installs Terraform. | +| `setup-enos` | no | `true` | When `true`, installs Enos. | +| `launch-extra-args` | no | empty | Extra arguments added to the launch command before the scenario filter. | +| `retry-extra-args` | no | empty | Extra arguments added to the retry launch command before the scenario filter. | +| `destroy-extra-args` | no | empty | Extra arguments added to destroy before the scenario filter. | +| `pre-destroy-command` | no | empty | Optional enos scenario command to run before destroying (e.g., `exec`). | +| `pre-destroy-extra-args` | no | empty | Extra arguments added to the pre-destroy command before the scenario filter. | + +## Outputs + +| Name | Description | +| --- | --- | +| `launch-outcome` | Final main phase outcome: `success` or `failure`. | +| `launch-retry-attempted` | `true` when the main phase retry was attempted. | +| `destroy-outcome` | Final destroy outcome: `success` or `failure`. | +| `destroy-retry-attempted` | `true` when destroy retry was attempted. | +| `debug-data-artifact-name` | Sanitized debug artifact name derived from the scenario filter. | +| `debug-data-dir` | Debug data directory used by the action. | +| `error-message` | Failure message captured from the retry step, when present. | + +## Usage + +### Standard launch workflow + +```yaml +- name: Run Enos scenario + id: run-scenario + uses: ./.github/actions/run-enos-scenario + with: + scenario-filter: ${{ matrix.scenario.id.filter }} + timeout: 45m0s + retry-timeout: 30m0s + destroy-timeout: 10m0s + show-state-on-retry: 'true' +``` + +### With custom destroy arguments + +```yaml +- name: Run Enos scenario + id: run-scenario + uses: ./.github/actions/run-enos-scenario + with: + scenario-filter: ${{ inputs.scenario }} + timeout: 60m0s + retry-timeout: 60m0s + destroy-timeout: 60m0s + destroy-extra-args: --grpc-listen http://localhost +``` + +### Skip tool installation + +```yaml +- name: Run Enos scenario + uses: ./.github/actions/run-enos-scenario + with: + scenario-filter: ${{ steps.sample.outputs.filter }} + setup-terraform: 'false' + setup-enos: 'false' +``` + +### With pre-destroy command + +```yaml +- name: Run Enos scenario + uses: ./.github/actions/run-enos-scenario + with: + scenario-filter: ${{ inputs.scenario }} + timeout: 30m0s + pre-destroy-command: exec + pre-destroy-extra-args: --cmd 'output -raw scan_markdown' +``` + +### With pre-destroy command and output redirection + +```yaml +- name: Run Enos scenario + uses: ./.github/actions/run-enos-scenario + with: + scenario-filter: ${{ inputs.scenario }} + timeout: 30m0s + pre-destroy-command: exec + pre-destroy-extra-args: --cmd 'output -raw scan_markdown' | tee -a "$GITHUB_STEP_SUMMARY" +``` diff --git a/.github/actions/run-enos-scenario/action.yml b/.github/actions/run-enos-scenario/action.yml new file mode 100644 index 0000000000..51e131eabf --- /dev/null +++ b/.github/actions/run-enos-scenario/action.yml @@ -0,0 +1,266 @@ +# Copyright IBM Corp. 2016, 2026 +# SPDX-License-Identifier: BUSL-1.1 + +--- +name: Run and retry an Enos scenario +description: | + Run an Enos scenario with a standard launch retry flow, optional state/plan diagnostics, + automatic debug data artifact naming, and destroy cleanup with a destroy retry. + +inputs: + scenario-filter: + description: Enos scenario filter/identifier (for example "ui edition:ent backend:raft") + required: true + working-directory: + description: Working directory to pass to enos via --chdir + required: false + default: ./enos + timeout: + description: Timeout for the initial launch command + required: false + default: 45m0s + retry-timeout: + description: Timeout for the retry launch attempt + required: false + default: 30m0s + destroy-timeout: + description: Timeout for the destroy command + required: false + default: 10m0s + show-state-on-retry: + description: Whether to show scenario state and plan before and after a failed retry (true/false) + required: false + default: ${{ github.repository == 'hashicorp/vault-enterprise' && 'true' || 'false' }} + upload-debug-data: + description: Whether to upload Enos debug data on failure (true/false) + required: false + default: "true" + debug-data-retention-days: + description: Retention period in days for uploaded debug data artifacts + required: false + default: "30" + debug-data-path: + description: Debug data root directory to create and upload + required: false + default: "" + setup-terraform: + description: Whether to install Terraform in the action (true/false) + required: false + default: "true" + setup-enos: + description: Whether to install Enos in the action (true/false) + required: false + default: "true" + launch-extra-args: + description: Extra arguments to append to the launch command before the scenario filter + required: false + default: "" + retry-extra-args: + description: Extra arguments to append to the retry launch command before the scenario filter + required: false + default: "" + destroy-extra-args: + description: Extra arguments to append to the destroy command before the scenario filter + required: false + default: "" + pre-destroy-command: + description: Optional enos scenario command to run before destroying (e.g., "exec") + required: false + default: "" + pre-destroy-extra-args: + description: Extra arguments to append to the pre-destroy command before the scenario filter + required: false + default: "" + +outputs: + launch-outcome: + description: Final execution outcome for the main phase (success or failure) + value: ${{ steps.determine-launch-outcome.outputs.outcome }} + launch-retry-attempted: + description: Whether a retry was attempted for the main phase + value: ${{ steps.determine-launch-outcome.outputs.retry-attempted }} + destroy-outcome: + description: Final destroy outcome (success or failure) + value: ${{ steps.determine-destroy-outcome.outputs.outcome }} + destroy-retry-attempted: + description: Whether a destroy retry was attempted + value: ${{ steps.determine-destroy-outcome.outputs.retry-attempted }} + debug-data-artifact-name: + description: Debug data artifact name prepared by the action + value: ${{ steps.prepare-artifacts.outputs.debug-data-artifact-name }} + debug-data-dir: + description: Debug data directory prepared by the action + value: ${{ steps.prepare-artifacts.outputs.debug-data-dir }} + error-message: + description: Error message captured from a failed retry + value: ${{ steps.launch-retry.outputs.error-message }} + +runs: + using: composite + steps: + - if: inputs.setup-terraform == 'true' + name: Setup Terraform + uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0 + with: + terraform_wrapper: false + + - if: inputs.setup-enos == 'true' + name: Setup Enos + uses: hashicorp/action-setup-enos@6ec106c8f809fe645162d73bea565c65f3269907 # v1.52 + + - id: prepare-artifacts + name: Prepare artifacts and debug directory + shell: bash + run: | + artifact_name="enos-debug-data_$(echo "${{ inputs.scenario-filter }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g')" + debug_dir="${{ inputs.debug-data-path }}" + if [ -z "${debug_dir}" ]; then + debug_dir="${ENOS_DEBUG_DATA_ROOT_DIR:-/tmp/enos-debug-data}" + fi + mkdir -p "${debug_dir}" + { + echo "debug-data-artifact-name=${artifact_name}" + echo "debug-data-dir=${debug_dir}" + } | tee -a "$GITHUB_OUTPUT" + + - id: launch + name: Run Enos scenario + continue-on-error: true + shell: bash + run: | + enos scenario launch \ + --timeout ${{ inputs.timeout }} \ + ${{ inputs.launch-extra-args }} \ + --chdir ${{ inputs.working-directory }} \ + ${{ inputs.scenario-filter }} + + - if: steps.launch.outcome == 'failure' && inputs.show-state-on-retry == 'true' + name: Show scenario state before retry + shell: bash + run: | + echo "::group::Scenario state before retry" + enos scenario exec --cmd show \ + --chdir ${{ inputs.working-directory }} \ + ${{ inputs.scenario-filter }} || true + echo "::endgroup::" + echo "::group::Scenario plan before retry" + enos scenario exec --cmd plan \ + --chdir ${{ inputs.working-directory }} \ + ${{ inputs.scenario-filter }} || true + echo "::endgroup::" + + - if: steps.launch.outcome == 'failure' + id: launch-retry + name: Retry Enos scenario + shell: bash + run: | + if ! output=$(enos scenario launch \ + --timeout ${{ inputs.retry-timeout }} \ + ${{ inputs.retry-extra-args }} \ + --chdir ${{ inputs.working-directory }} \ + ${{ inputs.scenario-filter }} 2>&1); then + if [ "${{ inputs.show-state-on-retry }}" = "true" ]; then + echo "::group::Scenario state after retry" + enos scenario exec --cmd show \ + --chdir ${{ inputs.working-directory }} \ + ${{ inputs.scenario-filter }} || true + echo "::endgroup::" + echo "::group::Scenario plan after retry" + enos scenario exec --cmd plan \ + --chdir ${{ inputs.working-directory }} \ + ${{ inputs.scenario-filter }} || true + echo "::endgroup::" + fi + echo "::group::Scenario retry failure" + echo "$output" >&2 + { + echo "# Enos Scenario Failed!" + echo 'See the [workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for more information' + echo '```' + echo "$output" + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + echo "::endgroup::" + exit 1 + fi + + - id: determine-launch-outcome + name: Determine launch outcome + shell: bash + run: | + retry_attempted=false + if [ "${{ steps.launch.outcome }}" = "failure" ]; then + retry_attempted=true + fi + if [ "${{ steps.launch.outcome }}" = "success" ] || [ "${{ steps.launch-retry.outcome }}" = "success" ]; then + outcome=success + else + outcome=failure + fi + { + echo "outcome=${outcome}" + echo "retry-attempted=${retry_attempted}" + } | tee -a "$GITHUB_OUTPUT" + + - if: failure() && inputs.upload-debug-data == 'true' + name: Upload debug data + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: ${{ steps.prepare-artifacts.outputs.debug-data-artifact-name }} + path: ${{ steps.prepare-artifacts.outputs.debug-data-dir }} + retention-days: ${{ inputs.debug-data-retention-days }} + + - if: always() && inputs.pre-destroy-command != '' + id: pre-destroy + name: Run pre-destroy command + continue-on-error: true + shell: bash + run: | + enos scenario ${{ inputs.pre-destroy-command }} \ + --chdir ${{ inputs.working-directory }} \ + ${{ inputs.scenario-filter }} \ + ${{ inputs.pre-destroy-extra-args }} + + - if: always() + id: destroy + name: Destroy Enos scenario + continue-on-error: true + shell: bash + run: | + enos scenario destroy \ + --timeout ${{ inputs.destroy-timeout }} \ + ${{ inputs.destroy-extra-args }} \ + --chdir ${{ inputs.working-directory }} \ + ${{ inputs.scenario-filter }} + + - if: steps.destroy.outcome == 'failure' + id: destroy-retry + name: Retry Enos scenario destroy + continue-on-error: true + shell: bash + run: | + enos scenario destroy \ + --timeout ${{ inputs.destroy-timeout }} \ + ${{ inputs.destroy-extra-args }} \ + --chdir ${{ inputs.working-directory }} \ + ${{ inputs.scenario-filter }} + + - if: always() + id: determine-destroy-outcome + name: Determine destroy outcome + shell: bash + run: | + retry_attempted=false + if [ "${{ steps.destroy.outcome }}" = "failure" ]; then + retry_attempted=true + fi + if [ "${{ steps.destroy.outcome }}" = "success" ] || [ "${{ steps.destroy-retry.outcome }}" = "success" ]; then + outcome=success + else + outcome=failure + fi + { + echo "outcome=${outcome}" + echo "retry-attempted=${retry_attempted}" + } | tee -a "$GITHUB_OUTPUT" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1c65713f68..17e22c6eca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -293,40 +293,13 @@ jobs: needs: setup runs-on: ${{ fromJSON(needs.setup.outputs.compute-build-ui) }} outputs: - cache-key: ui-${{ steps.ui-hash.outputs.ui-hash }} + cache-key: ${{ steps.build-ui.outputs.cache-key }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ needs.setup.outputs.checkout-ref }} - - name: Get UI hash - id: ui-hash - run: echo "ui-hash=$(git ls-tree HEAD ui --object-only)" | tee -a "$GITHUB_OUTPUT" - - name: Set up UI asset cache - id: cache-ui-assets - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 - with: - enableCrossOsArchive: true - lookup-only: true - path: http/web_ui - # Only restore the UI asset cache if we haven't modified anything in the ui directory. - # Never do a partial restore of the web_ui if we don't get a cache hit. - key: ui-${{ steps.ui-hash.outputs.ui-hash }} - - if: steps.cache-ui-assets.outputs.cache-hit != 'true' - name: Install PNPM - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 - with: - run_install: false - package_json_file: './ui/package.json' - - if: steps.cache-ui-assets.outputs.cache-hit != 'true' - name: Set up node and pnpm - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 - with: - node-version-file: ui/package.json - cache: pnpm - cache-dependency-path: ui/pnpm-lock.yaml - - if: steps.cache-ui-assets.outputs.cache-hit != 'true' - name: Build UI - run: make ci-build-ui + - uses: ./.github/actions/build-ui + id: build-ui # Artifacts is where we'll build the various Vault binaries and package them into their respective # Zip bundles, RPM and Deb packages, and container images. After we've packaged them we upload diff --git a/.github/workflows/test-run-enos-scenario-containers.yml b/.github/workflows/test-run-enos-scenario-containers.yml index 553fde2257..5690a51754 100644 --- a/.github/workflows/test-run-enos-scenario-containers.yml +++ b/.github/workflows/test-run-enos-scenario-containers.yml @@ -82,14 +82,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0 - with: - # the Terraform wrapper will break Terraform execution in Enos because - # it changes the output to text when we expect it to be JSON. - terraform_wrapper: false - - uses: hashicorp/action-setup-enos@6ec106c8f809fe645162d73bea565c65f3269907 # v1.52 - with: - github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} - name: Download Docker Image id: download uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -101,40 +93,21 @@ jobs: run: | echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic || true - name: Run Enos scenario - id: run - # Continue once and retry to handle occasional blips when creating - # infrastructure. - continue-on-error: true + id: run-scenario env: ENOS_VAR_terraform_plugin_cache_dir: ../support/terraform-plugin-cache ENOS_VAR_vault_build_date: ${{ needs.metadata.outputs.build-date }} ENOS_VAR_vault_version: ${{ needs.metadata.outputs.vault-version }} ENOS_VAR_vault_revision: ${{ inputs.vault-revision }} ENOS_VAR_container_image_archive: ${{steps.download.outputs.download-path}}/${{ inputs.build-artifact-name }} - run: | - mkdir -p ./enos/support/terraform-plugin-cache - enos scenario run --timeout 10m0s --chdir ./enos/k8s ${{ matrix.scenario.id.filter }} - - name: Retry Enos scenario - id: run_retry - if: steps.run.outcome == 'failure' - env: - ENOS_VAR_terraform_plugin_cache_dir: ../support/terraform-plugin-cache - ENOS_VAR_vault_build_date: ${{ needs.metadata.outputs.build-date }} - ENOS_VAR_vault_version: ${{ needs.metadata.outputs.vault-version }} - ENOS_VAR_vault_revision: ${{ inputs.vault-revision }} - ENOS_VAR_container_image_archive: ${{steps.download.outputs.download-path}}/${{ inputs.build-artifact-name }} - run: | - enos scenario run --timeout 10m0s --chdir ./enos/k8s ${{ matrix.scenario.id.filter }} - - name: Destroy Enos scenario - if: ${{ always() }} - env: - ENOS_VAR_terraform_plugin_cache_dir: ../support/terraform-plugin-cache - ENOS_VAR_vault_build_date: ${{ needs.metadata.outputs.build-date }} - ENOS_VAR_vault_version: ${{ needs.metadata.outputs.vault-version }} - ENOS_VAR_vault_revision: ${{ inputs.vault-revision }} - ENOS_VAR_container_image_archive: ${{steps.download.outputs.download-path}}/${{ inputs.build-artifact-name }} - run: | - enos scenario destroy --timeout 10m0s --grpc-listen http://localhost --chdir ./enos/k8s ${{ matrix.scenario.id.filter }} + uses: ./.github/actions/run-enos-scenario + with: + scenario-filter: ${{ matrix.scenario.id.filter }} + working-directory: ./enos/k8s + timeout: 10m0s + retry-timeout: 10m0s + destroy-timeout: 10m0s + destroy-extra-args: --grpc-listen http://localhost - name: Cleanup Enos runtime directories if: ${{ always() }} run: | diff --git a/.github/workflows/test-run-enos-scenario-matrix.yml b/.github/workflows/test-run-enos-scenario-matrix.yml index 0af3726fae..16a7498c3a 100644 --- a/.github/workflows/test-run-enos-scenario-matrix.yml +++ b/.github/workflows/test-run-enos-scenario-matrix.yml @@ -43,7 +43,7 @@ on: jobs: metadata: - runs-on: ${{ fromJSON(inputs.runs-on) }} + runs-on: ${{ github.repository == 'hashicorp/vault' && '"ubuntu-latest"' || fromJSON('["self-hosted","ubuntu-latest-x64"]') }} permissions: id-token: write # vault-auth contents: read @@ -112,6 +112,7 @@ jobs: permissions: id-token: write # vault-auth contents: read + steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -140,6 +141,7 @@ jobs: kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE; kv/data/github/${{ github.repository }}/ibm-license license_1 | VAULT_LICENSE_IBM; kv/data/github/${{ github.repository }}/github-token token | ELEVATED_GITHUB_TOKEN; + - id: secrets run: | if [[ "${{ needs.metadata.outputs.is-ent-repo }}" != 'true' ]]; then @@ -175,6 +177,7 @@ jobs: echo 'vault-license-ibm=${{ steps.vault-secrets.outputs.VAULT_LICENSE_IBM }}' } | tee -a "$GITHUB_OUTPUT" fi + - id: env run: | # Configure input environment variables. @@ -204,19 +207,18 @@ jobs: echo 'ENOS_VAR_verify_ldap_secrets_engine=false' echo 'ENOS_VAR_verify_log_secrets=true' } | tee -a "$GITHUB_ENV" + - uses: ./.github/actions/set-up-go with: github-token: ${{ steps.secrets.outputs.github-token }} + - name: Install LDAP client tools run: | sudo apt-get update sudo apt-get install -y ldap-utils + - uses: ./.github/actions/install-tools - - uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0 - with: - # the Terraform wrapper will break Terraform execution in Enos because - # it changes the output to text when we expect it to be JSON. - terraform_wrapper: false + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: @@ -226,14 +228,13 @@ jobs: role-to-assume: ${{ steps.secrets.outputs.aws-role-arn }} role-skip-session-tagging: true role-duration-seconds: 3600 - - uses: hashicorp/action-setup-enos@6ec106c8f809fe645162d73bea565c65f3269907 # v1.52 - with: - github-token: ${{ steps.secrets.outputs.github-token }} + - uses: ./.github/actions/create-dynamic-config with: github-token: ${{ steps.secrets.outputs.github-token }} vault-version: ${{ inputs.vault-version }} vault-edition: ${{ inputs.vault-edition }} + - name: Prepare scenario dependencies id: prepare_scenario run: | @@ -247,52 +248,42 @@ jobs: echo "test_results_artifact_name=test-results_$(echo "${{ matrix.scenario.id.filter }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g')" echo "junit_results_artifact_name=junit-results_$(echo "${{ matrix.scenario.id.filter }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g')" echo "failure_summary_artifact_name=failure-summary-enos_$(echo "${{ matrix.scenario.id.filter }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g').md" - } >> "$GITHUB_OUTPUT" + } | tee -a "$GITHUB_OUTPUT" + - if: contains(inputs.sample-name, 'build') uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ inputs.build-artifact-name }} path: ./enos/support/downloads + - if: contains(inputs.sample-name, 'ent') name: Configure Vault licenses run: | echo "${{ steps.secrets.outputs.vault-license }}" > ./enos/support/vault.hclic || true echo "${{ steps.secrets.outputs.vault-license-ibm }}" > ./enos/support/ibm-pao.lic || true + - if: contains(matrix.scenario.id.filter, 'consul_edition:ent') name: Configure Consul license run: | echo "${{ steps.secrets.outputs.consul-license }}" > ./enos/support/consul.hclic || true + - name: Configure Vault Radar license run: | echo "${{ steps.secrets.outputs.radar-license }}" > ./enos/support/vault-radar.hclic || true - - id: launch - name: enos scenario launch ${{ matrix.scenario.id.filter }} - # Continue once and retry to handle occasional blips when creating infrastructure. - continue-on-error: true - run: enos scenario launch --timeout 45m0s --chdir ./enos ${{ matrix.scenario.id.filter }} - - if: steps.launch.outcome == 'failure' - id: launch_retry - name: Retry enos scenario launch ${{ matrix.scenario.id.filter }} - run: enos scenario launch --timeout 45m0s --chdir ./enos ${{ matrix.scenario.id.filter }} - - name: Upload Debug Data - if: failure() - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + + - id: run-scenario + name: Run Enos scenario ${{ matrix.scenario.id.filter }} + uses: ./.github/actions/run-enos-scenario with: - # The name of the artifact is the same as the matrix scenario name with the spaces replaced with underscores and colons replaced by equals. - name: ${{ steps.prepare_scenario.outputs.debug_data_artifact_name }} - path: ${{ env.ENOS_DEBUG_DATA_ROOT_DIR }} - retention-days: 30 - continue-on-error: true - - if: ${{ always() }} - id: destroy - name: enos scenario destroy ${{ matrix.scenario.id.filter }} - continue-on-error: true - run: enos scenario destroy --timeout 10m0s --chdir ./enos ${{ matrix.scenario.id.filter }} - - if: steps.destroy.outcome == 'failure' - id: destroy_retry - name: Retry enos scenario destroy ${{ matrix.scenario.id.filter }} - continue-on-error: true - run: enos scenario destroy --timeout 10m0s --chdir ./enos ${{ matrix.scenario.id.filter }} + scenario-filter: ${{ matrix.scenario.id.filter }} + working-directory: ./enos + timeout: 45m0s + retry-timeout: 30m0s + destroy-timeout: 10m0s + upload-debug-data: 'true' + debug-data-retention-days: '30' + debug-data-path: ${{ env.ENOS_DEBUG_DATA_ROOT_DIR }} + - name: Upload Test Results if: always() id: upload_test_results @@ -303,6 +294,7 @@ jobs: retention-days: 7 if-no-files-found: ignore continue-on-error: true + - name: Upload JUnit Test Results if: always() id: upload_junit_results @@ -313,6 +305,7 @@ jobs: retention-days: 7 if-no-files-found: ignore continue-on-error: true + - name: Check for test results if: always() id: check_test_results @@ -323,6 +316,7 @@ jobs: else echo "has_results=false" >> "$GITHUB_OUTPUT" fi + - name: Prepare Test Results Summary if: always() && steps.check_test_results.outputs.has_results == 'true' continue-on-error: true @@ -368,6 +362,7 @@ jobs: else echo "⚠️ No test results found in /tmp/vault_test_results_*.json" >> "$GITHUB_STEP_SUMMARY" fi + - name: Upload Failure Summary if: always() && steps.check_test_results.outputs.has_results == 'true' uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 @@ -376,6 +371,7 @@ jobs: path: ${{ steps.prepare_scenario.outputs.failure_summary_artifact_name }} if-no-files-found: ignore continue-on-error: true + - name: Clean up Enos runtime directories id: cleanup if: ${{ always() }} @@ -384,34 +380,39 @@ jobs: rm -rf /tmp/enos* rm -rf ./enos/support rm -rf ./enos/.enos + # Send slack notifications to #feed-vault-enos-failures any of our enos scenario commands fail. # There is an incoming webhook set up on the "Enos Vault Failure Bot" Slackbot: # https://api.slack.com/apps/A05E31CH1LG/incoming-webhooks + - if: ${{ always() && ! cancelled() }} name: Notify launch failed uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1 with: failure-message: "enos scenario launch ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`" - status: ${{ steps.launch.outcome }} + status: ${{ steps.run-scenario.outputs.launch-outcome == 'failure' && 'failure' || 'success' }} slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }} + - if: ${{ always() && ! cancelled() }} name: Notify retry launch failed uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1 with: failure-message: "retry enos scenario launch ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`" - status: ${{ steps.launch_retry.outcome }} + status: ${{ steps.run-scenario.outputs.launch-retry-attempted == 'true' && steps.run-scenario.outputs.launch-outcome == 'failure' && 'failure' || 'success' }} slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }} + - if: ${{ always() && ! cancelled() }} name: Notify destroy failed uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1 with: failure-message: "enos scenario destroy ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`" - status: ${{ steps.destroy.outcome }} + status: ${{ steps.run-scenario.outputs.destroy-outcome == 'failure' && 'failure' || 'success' }} slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }} + - if: ${{ always() && ! cancelled() }} name: Notify retry destroy failed uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1 with: failure-message: "retry enos scenario destroy ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`" - status: ${{ steps.destroy_retry.outcome }} + status: ${{ steps.run-scenario.outputs.destroy-retry-attempted == 'true' && steps.run-scenario.outputs.destroy-outcome == 'failure' && 'failure' || 'success' }} slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }} diff --git a/.github/workflows/test-run-enos-scenario.yml b/.github/workflows/test-run-enos-scenario.yml index 088154065b..4373f93fa8 100644 --- a/.github/workflows/test-run-enos-scenario.yml +++ b/.github/workflows/test-run-enos-scenario.yml @@ -79,11 +79,6 @@ jobs: uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: package_json_file: './ui/package.json' - - uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0 - with: - # the Terraform wrapper will break Terraform execution in Enos because - # it changes the output to text when we expect it to be JSON. - terraform_wrapper: false - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: @@ -93,9 +88,6 @@ jobs: role-to-assume: ${{ secrets.AWS_ROLE_ARN_CI }} role-skip-session-tagging: true role-duration-seconds: 3600 - - uses: hashicorp/action-setup-enos@6ec106c8f809fe645162d73bea565c65f3269907 # v1.52 - with: - github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} - name: Prepare scenario dependencies id: scenario-deps run: | @@ -108,8 +100,17 @@ jobs: id: license run: echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic - name: Run Enos scenario - id: run - run: enos scenario run --timeout 60m0s --chdir ./enos ${{ inputs.scenario }} + id: run-scenario + uses: ./.github/actions/run-enos-scenario + with: + scenario-filter: ${{ inputs.scenario }} + working-directory: ./enos + timeout: 60m0s + retry-timeout: 60m0s + destroy-timeout: 60m0s + destroy-extra-args: --grpc-listen http://localhost + debug-data-path: ${{ env.ENOS_DEBUG_DATA_ROOT_DIR }} + debug-data-retention-days: '1' - name: Collect logs when scenario fails id: collect_logs if: ${{ always() }} @@ -122,15 +123,6 @@ jobs: name: enos-scenario-logs path: ${{ steps.scenario-deps.outputs.logsdir }} retention-days: 1 - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - if: ${{ always() }} - with: - name: enos-debug-data-logs - path: ${{ env.ENOS_DEBUG_DATA_ROOT_DIR }} - retention-days: 1 - - name: Ensure scenario has been destroyed - if: ${{ always() }} - run: enos scenario destroy --timeout 60m0s --grpc-listen http://localhost --chdir ./enos ${{ inputs.scenario }} - name: Clean up Enos runtime directories if: ${{ always() }} run: | diff --git a/.gitignore b/.gitignore index c180ae09fb..674fd6691c 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,8 @@ Vagrantfile enos-local.vars.hcl enos/**/support enos/**/kubeconfig +enos/modules/**/plan.yml +enos/sarif**.json .terraform .terraform.lock.hcl .tfstate.* diff --git a/.release/pipeline.hcl b/.release/pipeline.hcl index 42ab4ba393..099b8d5b3e 100644 --- a/.release/pipeline.hcl +++ b/.release/pipeline.hcl @@ -119,6 +119,7 @@ changed_files { # These exist on CE branches to please Github Actions. joinpath(".github", "workflows", "build-artifacts-ent.yml"), joinpath(".github", "workflows", "backport-automation-ent.yml"), + joinpath(".github", "workflows", "test-run-enos-scenario-cloud.yml"), ] } @@ -127,7 +128,6 @@ changed_files { base_dir = [ "website", joinpath(".release", "docker"), - joinpath("enos", "modules"), joinpath("scripts", "docker"), ] } @@ -147,6 +147,26 @@ changed_files { ] } + // Ignore some enos modules related to HSM + ignore { + base_dir = [ + # The next matcher looks for HSM but we can ignore the softhsm modules + joinpath("enos", "modules", "softhsm_install"), + joinpath("enos", "modules", "softhsm_create_vault_keys"), + joinpath("enos", "modules", "softhsm_init"), + joinpath("enos", "modules", "softhsm_distribute_vault_keys"), + # Some filename have ent in them + joinpath("enos", "modules", "verify_secrets_engines"), + ] + } + + // Make sure our zap scanner is always ent only + match { + base_dir = [ + joinpath("enos", "modules", "zap_scan_ent") + ] + } + // Match whole directories that are enterprise only match { base_dir = [ @@ -287,4 +307,14 @@ changed_files { base_dir = ["ui"] } } + + // The "zap_scan" group matches the Zap scanner + group "zap_scan" { + match { + contains = [ + "security-scan-zap", + "zap_scan", + ] + } + } } diff --git a/enos/enos-descriptions.hcl b/enos/enos-descriptions.hcl index eefba5fd73..cf71dc765f 100644 --- a/enos/enos-descriptions.hcl +++ b/enos/enos-descriptions.hcl @@ -217,12 +217,6 @@ globals { target hosts. EOF - run_verify_blackbox_tests_remote = <<-EOF - Run blackbox verification tests directly on the Vault leader host. These tests execute - the Vault CLI binary on the target machine to validate version metadata and other - functionality that requires local binary access. - EOF - wait_for_cluster_to_have_leader = <<-EOF Wait for a leader election to occur before we proceed with any further quality verification. EOF diff --git a/enos/enos-providers.hcl b/enos/enos-providers.hcl index a9bddf33bb..1e6cab0d6f 100644 --- a/enos/enos-providers.hcl +++ b/enos/enos-providers.hcl @@ -30,3 +30,6 @@ provider "hcp" "default" { provider "docker" "default" { } + +provider "local" "default" { +} diff --git a/enos/enos-scenario-agent.hcl b/enos/enos-scenario-agent.hcl index e2bc2d6ab4..f0155e13af 100644 --- a/enos/enos-scenario-agent.hcl +++ b/enos/enos-scenario-agent.hcl @@ -178,11 +178,12 @@ scenario "agent" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - instance_count = 1 - vpc_id = step.create_vpc.id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + instance_count = 1 + root_volume_size = 64 + vpc_id = step.create_vpc.id } } @@ -501,37 +502,16 @@ scenario "agent" { ] variables { - leader_host = step.get_vault_cluster_ips.leader_host - leader_public_ip = step.get_vault_cluster_ips.leader_public_ip - vault_root_token = step.create_vault_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultServerVersion"] - vault_edition = matrix.edition - } - } - - step "run_verify_blackbox_tests_remote" { - description = global.description.run_verify_blackbox_tests_remote - module = module.vault_run_blackbox_test - depends_on = [step.run_verify_blackbox_tests] - - providers = { - enos = local.enos_provider[matrix.distro] - } - - verifies = [ - quality.vault_version_build_date, - quality.vault_version_edition, - quality.vault_version_release, - ] - - variables { - leader_host = step.get_vault_cluster_ips.leader_host - leader_public_ip = step.get_vault_cluster_ips.leader_public_ip - vault_root_token = step.create_vault_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultCLIVersionLocal"] - vault_edition = matrix.edition + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] } } diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index db695dc38e..50e65a3322 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -175,11 +175,12 @@ scenario "autopilot" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - instance_count = 1 - vpc_id = step.create_vpc.id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + instance_count = 1 + root_volume_size = 64 + vpc_id = step.create_vpc.id } } @@ -845,7 +846,6 @@ scenario "autopilot" { } step "run_verify_blackbox_tests" { - skip_step = true # Skip black box tests on autopilot for now description = global.description.run_verify_blackbox_tests module = module.vault_run_blackbox_test depends_on = [ @@ -869,12 +869,16 @@ scenario "autopilot" { ] variables { - leader_host = step.get_updated_vault_cluster_ips.leader_host - leader_public_ip = step.get_updated_vault_cluster_ips.leader_public_ip - vault_root_token = step.create_vault_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultServerVersion"] - vault_edition = matrix.edition + leader_host = step.get_updated_vault_cluster_ips.leader_host + leader_public_ip = step.get_updated_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = local.vault_install_dir } } diff --git a/enos/enos-scenario-dr-replication.hcl b/enos/enos-scenario-dr-replication.hcl index df020db62f..6c8456ce6a 100644 --- a/enos/enos-scenario-dr-replication.hcl +++ b/enos/enos-scenario-dr-replication.hcl @@ -218,11 +218,12 @@ scenario "dr_replication" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - instance_count = 1 - vpc_id = step.create_vpc.id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + instance_count = 1 + root_volume_size = 64 + vpc_id = step.create_vpc.id } } @@ -666,6 +667,9 @@ scenario "dr_replication" { } step "run_verify_blackbox_tests" { + // NOTE: This must run before we transition the cluster to a DR secondary. + // because afterwards the namespaces API won't be available and the bbsdk + // init will fail. description = global.description.run_verify_blackbox_tests module = module.vault_run_blackbox_test depends_on = [step.get_primary_cluster_ips] @@ -683,37 +687,16 @@ scenario "dr_replication" { ] variables { - leader_host = step.get_primary_cluster_ips.leader_host - leader_public_ip = step.get_primary_cluster_ips.leader_public_ip - vault_root_token = step.create_primary_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultServerVersion"] - vault_edition = matrix.edition - } - } - - step "run_verify_blackbox_tests_remote" { - description = global.description.run_verify_blackbox_tests_remote - module = module.vault_run_blackbox_test - depends_on = [step.run_verify_blackbox_tests] - - providers = { - enos = local.enos_provider[matrix.distro] - } - - verifies = [ - quality.vault_version_build_date, - quality.vault_version_edition, - quality.vault_version_release, - ] - - variables { - leader_host = step.get_primary_cluster_ips.leader_host - leader_public_ip = step.get_primary_cluster_ips.leader_public_ip - vault_root_token = step.create_primary_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultCLIVersionLocal"] - vault_edition = matrix.edition + leader_host = step.get_primary_cluster_ips.leader_host + leader_public_ip = step.get_primary_cluster_ips.leader_public_ip + vault_root_token = step.create_primary_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] } } @@ -804,6 +787,8 @@ scenario "dr_replication" { depends_on = [ step.get_primary_cluster_ips, step.get_secondary_cluster_ips, + step.run_verify_blackbox_tests, + step.verify_ui, step.verify_secrets_engines_on_primary, ] diff --git a/enos/enos-scenario-plugin.hcl b/enos/enos-scenario-plugin.hcl index 7ab614e6bf..aa89af41b9 100644 --- a/enos/enos-scenario-plugin.hcl +++ b/enos/enos-scenario-plugin.hcl @@ -178,11 +178,12 @@ scenario "plugin" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] - cluster_tag_key = "plugin-integration" - common_tags = global.tags - instance_count = 1 - vpc_id = step.create_vpc.id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] + cluster_tag_key = "plugin-integration" + common_tags = global.tags + instance_count = 1 + root_volume_size = 64 + vpc_id = step.create_vpc.id } } @@ -451,37 +452,17 @@ scenario "plugin" { ] variables { - leader_host = step.get_vault_cluster_ips.leader_host - leader_public_ip = step.get_vault_cluster_ips.leader_public_ip - vault_root_token = step.create_vault_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultServerVersion"] - vault_edition = matrix.edition - } - } + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] - step "run_verify_blackbox_tests_remote" { - description = global.description.run_verify_blackbox_tests_remote - module = module.vault_run_blackbox_test - depends_on = [step.run_verify_blackbox_tests] - - providers = { - enos = local.enos_provider[matrix.distro] - } - - verifies = [ - quality.vault_version_build_date, - quality.vault_version_edition, - quality.vault_version_release, - ] - - variables { - leader_host = step.get_vault_cluster_ips.leader_host - leader_public_ip = step.get_vault_cluster_ips.leader_public_ip - vault_root_token = step.create_vault_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultCLIVersionLocal"] - vault_edition = matrix.edition } } diff --git a/enos/enos-scenario-pr-replication.hcl b/enos/enos-scenario-pr-replication.hcl index 512b8cfd7f..7972e5f385 100644 --- a/enos/enos-scenario-pr-replication.hcl +++ b/enos/enos-scenario-pr-replication.hcl @@ -218,11 +218,12 @@ scenario "pr_replication" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - instance_count = 1 - vpc_id = step.create_vpc.id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + instance_count = 1 + root_volume_size = 64 + vpc_id = step.create_vpc.id } } @@ -705,37 +706,16 @@ scenario "pr_replication" { ] variables { - leader_host = step.get_primary_cluster_ips.leader_host - leader_public_ip = step.get_primary_cluster_ips.leader_public_ip - vault_root_token = step.create_primary_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultServerVersion"] - vault_edition = matrix.edition - } - } - - step "run_verify_blackbox_tests_remote" { - description = global.description.run_verify_blackbox_tests_remote - module = module.vault_run_blackbox_test - depends_on = [step.run_verify_blackbox_tests] - - providers = { - enos = local.enos_provider[matrix.distro] - } - - verifies = [ - quality.vault_version_build_date, - quality.vault_version_edition, - quality.vault_version_release, - ] - - variables { - leader_host = step.get_primary_cluster_ips.leader_host - leader_public_ip = step.get_primary_cluster_ips.leader_public_ip - vault_root_token = step.create_primary_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultCLIVersionLocal"] - vault_edition = matrix.edition + leader_host = step.get_primary_cluster_ips.leader_host + leader_public_ip = step.get_primary_cluster_ips.leader_public_ip + vault_root_token = step.create_primary_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] } } diff --git a/enos/enos-scenario-proxy.hcl b/enos/enos-scenario-proxy.hcl index 9d932bdb82..0d14d5028a 100644 --- a/enos/enos-scenario-proxy.hcl +++ b/enos/enos-scenario-proxy.hcl @@ -185,11 +185,12 @@ scenario "proxy" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - instance_count = 1 - vpc_id = step.create_vpc.id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + instance_count = 1 + root_volume_size = 64 + vpc_id = step.create_vpc.id } } @@ -477,37 +478,17 @@ scenario "proxy" { ] variables { - leader_host = step.get_vault_cluster_ips.leader_host - leader_public_ip = step.get_vault_cluster_ips.leader_public_ip - vault_root_token = step.create_vault_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultServerVersion"] - vault_edition = matrix.edition - } - } + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] - step "run_verify_blackbox_tests_remote" { - description = global.description.run_verify_blackbox_tests_remote - module = module.vault_run_blackbox_test - depends_on = [step.run_verify_blackbox_tests] - - providers = { - enos = local.enos_provider[matrix.distro] - } - - verifies = [ - quality.vault_version_build_date, - quality.vault_version_edition, - quality.vault_version_release, - ] - - variables { - leader_host = step.get_vault_cluster_ips.leader_host - leader_public_ip = step.get_vault_cluster_ips.leader_public_ip - vault_root_token = step.create_vault_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultCLIVersionLocal"] - vault_edition = matrix.edition } } diff --git a/enos/enos-scenario-seal-ha.hcl b/enos/enos-scenario-seal-ha.hcl index defe1e68f5..492f85a156 100644 --- a/enos/enos-scenario-seal-ha.hcl +++ b/enos/enos-scenario-seal-ha.hcl @@ -217,11 +217,12 @@ scenario "seal_ha" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - instance_count = 1 - vpc_id = step.create_vpc.id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + instance_count = 1 + root_volume_size = 64 + vpc_id = step.create_vpc.id } } @@ -778,37 +779,17 @@ scenario "seal_ha" { ] variables { - leader_host = step.get_vault_cluster_ips.leader_host - leader_public_ip = step.get_vault_cluster_ips.leader_public_ip - vault_root_token = step.create_vault_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultServerVersion"] - vault_edition = matrix.edition - } - } + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] - step "run_verify_blackbox_tests_remote" { - description = global.description.run_verify_blackbox_tests_remote - module = module.vault_run_blackbox_test - depends_on = [step.run_verify_blackbox_tests] - - providers = { - enos = local.enos_provider[matrix.distro] - } - - verifies = [ - quality.vault_version_build_date, - quality.vault_version_edition, - quality.vault_version_release, - ] - - variables { - leader_host = step.get_vault_cluster_ips.leader_host - leader_public_ip = step.get_vault_cluster_ips.leader_public_ip - vault_root_token = step.create_vault_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultCLIVersionLocal"] - vault_edition = matrix.edition } } diff --git a/enos/enos-scenario-smoke-sdk.hcl b/enos/enos-scenario-smoke-sdk.hcl index 276b687bf8..38b7224c0d 100644 --- a/enos/enos-scenario-smoke-sdk.hcl +++ b/enos/enos-scenario-smoke-sdk.hcl @@ -178,11 +178,12 @@ scenario "smoke_sdk" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - instance_count = 1 - vpc_id = step.create_vpc.id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + instance_count = 1 + root_volume_size = 64 + vpc_id = step.create_vpc.id } } @@ -412,7 +413,9 @@ scenario "smoke_sdk" { locals { // Default test packages for smoke_sdk scenario - default_test_packages = ["core", "secrets", "replication", "raft", "integration"] + // TODO: "integration" has been removed from the default packages pending + // a restructure to handle tests that mutate the cluster via sys endpoints. + default_test_packages = ["core", "secrets", "replication", "raft"] // Determine if filter contains test names (starts with "Test") or package names is_test_name_filter = length(var.blackbox_test_filter) > 0 && length([for t in var.blackbox_test_filter : t if can(regex("^Test", t))]) > 0 diff --git a/enos/enos-scenario-smoke.hcl b/enos/enos-scenario-smoke.hcl index 3d5c5f284e..7b48a449e8 100644 --- a/enos/enos-scenario-smoke.hcl +++ b/enos/enos-scenario-smoke.hcl @@ -175,11 +175,12 @@ scenario "smoke" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - instance_count = 1 - vpc_id = step.create_vpc.id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + instance_count = 1 + root_volume_size = 64 + vpc_id = step.create_vpc.id } } @@ -519,41 +520,20 @@ scenario "smoke" { ] variables { - leader_host = step.get_vault_cluster_ips.leader_host - leader_public_ip = step.get_vault_cluster_ips.leader_public_ip - vault_root_token = step.create_vault_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultServerVersion"] - vault_edition = matrix.edition + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } } - step "run_verify_blackbox_tests_remote" { - description = global.description.run_verify_blackbox_tests_remote - module = module.vault_run_blackbox_test - depends_on = [step.run_verify_blackbox_tests] - - providers = { - enos = local.enos_provider[matrix.distro] - } - - verifies = [ - quality.vault_version_build_date, - quality.vault_version_edition, - quality.vault_version_release, - ] - - variables { - leader_host = step.get_vault_cluster_ips.leader_host - leader_public_ip = step.get_vault_cluster_ips.leader_public_ip - vault_root_token = step.create_vault_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultCLIVersionLocal"] - vault_edition = matrix.edition - } - } - - step "verify_raft_auto_join_voter" { description = global.description.verify_raft_cluster_all_nodes_are_voters skip_step = matrix.backend != "raft" diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index b47dcbf992..5c75ff3bec 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -192,11 +192,12 @@ scenario "upgrade" { } variables { - ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] - cluster_tag_key = global.vault_tag_key - common_tags = global.tags - instance_count = 1 - vpc_id = step.create_vpc.id + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + instance_count = 1 + root_volume_size = 64 + vpc_id = step.create_vpc.id } } @@ -713,37 +714,17 @@ scenario "upgrade" { ] variables { - leader_host = step.get_vault_cluster_ips.leader_host - leader_public_ip = step.get_vault_cluster_ips.leader_public_ip - vault_root_token = step.create_vault_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultServerVersion"] - vault_edition = matrix.edition - } - } + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_package = "./vault/external_tests/blackbox/verify" + test_names = ["TestVaultServerVersion"] + vault_edition = matrix.edition + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_install_dir = global.vault_install_dir[matrix.artifact_type] - step "run_verify_blackbox_tests_remote" { - description = global.description.run_verify_blackbox_tests_remote - module = module.vault_run_blackbox_test - depends_on = [step.run_verify_blackbox_tests] - - providers = { - enos = local.enos_provider[matrix.distro] - } - - verifies = [ - quality.vault_version_build_date, - quality.vault_version_edition, - quality.vault_version_release, - ] - - variables { - leader_host = step.get_vault_cluster_ips.leader_host - leader_public_ip = step.get_vault_cluster_ips.leader_public_ip - vault_root_token = step.create_vault_cluster.root_token - test_package = "./vault/external_tests/blackbox/verify" - test_names = ["TestVaultCLIVersionLocal"] - vault_edition = matrix.edition } } diff --git a/enos/modules/build_local/scripts/build.sh b/enos/modules/build_local/scripts/build.sh index ae69ce3980..48bf05022d 100755 --- a/enos/modules/build_local/scripts/build.sh +++ b/enos/modules/build_local/scripts/build.sh @@ -22,7 +22,9 @@ root_dir="$(git rev-parse --show-toplevel)" pushd "$root_dir" > /dev/null if [ -n "$BUILD_UI" ] && [ "$BUILD_UI" = "true" ]; then - make ci-build-ui + if ! output=$(make ci-build-ui 2>&1); then + echo "Failed to build the UI assets. Make sure you have the required node version (defined in ui/package.json) install and have set up pnpm (npm i -g pnpm): ${output}" 1>&2 + fi fi make ci-build diff --git a/enos/modules/cloud_docker_vault_cluster/main.tf b/enos/modules/cloud_docker_vault_cluster/main.tf index 801c193c29..0500612b11 100644 --- a/enos/modules/cloud_docker_vault_cluster/main.tf +++ b/enos/modules/cloud_docker_vault_cluster/main.tf @@ -152,6 +152,8 @@ locals { tls_disable = true } + administrative_namespace_path = "admin" + storage "raft" { path = "/vault/data" node_id = "node%s" diff --git a/enos/modules/ec2_info/main.tf b/enos/modules/ec2_info/main.tf index f66425c142..576c7da26d 100644 --- a/enos/modules/ec2_info/main.tf +++ b/enos/modules/ec2_info/main.tf @@ -245,7 +245,7 @@ output "ami_ids" { } output "current_region" { - value = data.aws_region.current + value = data.aws_region.current.id } output "availability_zones" { diff --git a/enos/modules/vault_run_blackbox_test/main.tf b/enos/modules/vault_run_blackbox_test/main.tf index 52ff5668e1..468bda0b94 100644 --- a/enos/modules/vault_run_blackbox_test/main.tf +++ b/enos/modules/vault_run_blackbox_test/main.tf @@ -32,17 +32,26 @@ resource "random_string" "test_id" { } resource "enos_local_exec" "run_blackbox_test" { - scripts = [abspath("${path.module}/scripts/run-test.sh")] - environment = merge({ - VAULT_TOKEN = var.vault_root_token - VAULT_ADDR = var.vault_addr != null ? var.vault_addr : "http://${var.leader_public_ip}:8200" - VAULT_TEST_PACKAGE = var.test_package - VAULT_TEST_MATRIX = length(local.test_names) > 0 ? local_file.test_matrix.filename : "" - VAULT_EDITION = var.vault_edition - # PATH and Go-related environment variables are inherited from the calling process - }, var.vault_namespace != null ? { - VAULT_NAMESPACE = var.vault_namespace - } : {}, local.ldap_environment, local.postgres_environment + scripts = [abspath("${path.module}/scripts/run-test.sh")] + depends_on = [local_file.test_matrix] + + environment = merge( + { + VAULT_TOKEN = var.vault_root_token + VAULT_ADDR = var.vault_addr != null ? var.vault_addr : "http://${var.leader_public_ip}:8200" + VAULT_TEST_PACKAGE = var.test_package + VAULT_TEST_MATRIX = length(local.test_names) > 0 ? local_file.test_matrix.filename : "" + VAULT_EDITION = var.vault_edition + # PATH and Go-related environment variables are inherited from the calling process + }, + var.vault_namespace != null ? { VAULT_NAMESPACE = var.vault_namespace } : {}, + var.vault_product_version != null ? { VAULT_VERSION = var.vault_product_version } : {}, + var.vault_revision != null ? { VAULT_REVISION = var.vault_revision } : {}, + var.vault_build_date != null ? { VAULT_BUILD_DATE = var.vault_build_date } : {}, + var.vault_install_dir != null ? { VAULT_INSTALL_DIR = var.vault_install_dir } : {}, + local.ldap_environment, + local.postgres_environment, + local.mongodb_environment ) } @@ -60,6 +69,7 @@ locals { LDAP_URL_PUBLIC = "ldap://${local.ldap_config.host.public_ip}:${local.ldap_config.port}" LDAP_BIND_DN = "cn=admin,${local.domain_dn}" LDAP_BIND_PASS = local.ldap_config.admin_pw + LDAP_USERNAME = "enos" } : {} # Extract PostgreSQL configuration safely, defaulting to empty map if not available diff --git a/enos/modules/vault_run_blackbox_test/variables.tf b/enos/modules/vault_run_blackbox_test/variables.tf index 032d1ab391..62814e18b7 100644 --- a/enos/modules/vault_run_blackbox_test/variables.tf +++ b/enos/modules/vault_run_blackbox_test/variables.tf @@ -53,3 +53,27 @@ variable "vault_edition" { description = "The Vault edition (ce, ent, ent.hsm, ent.fips1402, ent.hsm.fips1402)" default = "ent" } + +variable "vault_product_version" { + type = string + description = "The Vault product version (e.g., 1.15.0)" + default = null +} + +variable "vault_revision" { + type = string + description = "The Vault git revision/commit SHA" + default = null +} + +variable "vault_build_date" { + type = string + description = "The Vault build date" + default = null +} + +variable "vault_install_dir" { + type = string + description = "The directory where Vault is installed" + default = null +} diff --git a/sdk/helper/testcluster/blackbox/assertions.go b/sdk/helper/testcluster/blackbox/assertions.go index 7d138fe276..cab4eb8e7f 100644 --- a/sdk/helper/testcluster/blackbox/assertions.go +++ b/sdk/helper/testcluster/blackbox/assertions.go @@ -74,6 +74,17 @@ func (ma *MapAssertion) HasKey(key string, expected any) *MapAssertion { return ma } +func (ma *MapAssertion) GetKey(key string) any { + ma.t.Helper() + + val, ok := ma.data[key] + if !ok { + ma.t.Fatalf("[%s] missing expected key: %q", ma.path, key) + } + + return val +} + func (ma *MapAssertion) HasKeyCustom(key string, f func(val any) bool) *MapAssertion { ma.t.Helper() diff --git a/sdk/helper/testcluster/blackbox/session.go b/sdk/helper/testcluster/blackbox/session.go index e135069017..21ee347e3d 100644 --- a/sdk/helper/testcluster/blackbox/session.go +++ b/sdk/helper/testcluster/blackbox/session.go @@ -88,11 +88,14 @@ func New(t *testing.T, opts ...SessionOpts) *Session { t.Cleanup(func() { if session.NoCleanup { - t.Logf("WARN: NoDebug has been set, not cleaning up namespace") + t.Logf("WARN: NoCleanup has been set, not cleaning up namespace") return } - _, err = privClient.Logical().Delete(nsURLPath) - require.NoError(t, err) + privClient.SetClientTimeout(time.Second) + session.Eventually(func() error { + _, err = privClient.Logical().Delete(nsURLPath) + return err + }) t.Logf("Cleaned up namespace %s", nsName) }) diff --git a/sdk/helper/testcluster/blackbox/session_autopilot.go b/sdk/helper/testcluster/blackbox/session_autopilot.go new file mode 100644 index 0000000000..16319f345c --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_autopilot.go @@ -0,0 +1,58 @@ +// Copyright IBM Corp. 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "errors" + "time" + + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +func (s *Session) AssertAutopilotHealthy() { + s.t.Helper() + + // query the autopilot state endpoint and verify that all nodes are healthy according to autopilot + healthy, err := s.autopilotStateHealthy() + require.NoError(s.t, err) + require.Truef(s.t, healthy, "expected autopilot state to be healthy") +} + +func (s *Session) autopilotState() (*api.Secret, error) { + // query the autopilot state endpoint and verify that all nodes are healthy according to autopilot + var state *api.Secret + return state, s.Req( + func(c *api.Client) error { + var err error + state, err = c.Logical().Read("sys/storage/raft/autopilot/state") + return err + }, + WithClientRootNamespace(), + WithClientTimeout(2*time.Second), + ) +} + +func (s *Session) autopilotStateHealthy() (bool, error) { + state, err := s.autopilotState() + if err != nil { + return false, err + } + + if state == nil { + return false, errors.New("no raft data response") + } + + health, ok := state.Data["healthy"] + if !ok { + return false, errors.New("raft data missing 'healthy' key") + } + + healthy, ok := health.(bool) + if !ok { + return false, errors.New("raft data 'healthy' key is unknown type") + } + + return healthy, nil +} diff --git a/sdk/helper/testcluster/blackbox/session_client.go b/sdk/helper/testcluster/blackbox/session_client.go new file mode 100644 index 0000000000..d2d180be27 --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_client.go @@ -0,0 +1,67 @@ +// Copyright IBM Corp. 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "os" + "time" + + "github.com/hashicorp/vault/api" +) + +type ClientOpt func(*api.Client) + +func WithClientRootNamespace() ClientOpt { + return func(c *api.Client) { + c.ClearNamespace() + } +} + +func WithClientParentNamespace() ClientOpt { + return func(c *api.Client) { + c.SetNamespace(getParentNamespace()) + } +} + +func WithClientTimeout(d time.Duration) ClientOpt { + return func(c *api.Client) { + c.SetClientTimeout(d) + } +} + +func (s *Session) Req(fn func(*api.Client) error, opts ...ClientOpt) error { + s.t.Helper() + + c, err := api.NewClient(s.Client.CloneConfig()) + if err != nil { + return err + } + + for _, opt := range opts { + opt(c) + } + + return fn(c) +} + +// GetParentNamespace returns the namespace from VAULT_NAMESPACE environment variable. +// The blackbox test framework auto-creates a unique child namespace for each test +// (e.g., "admin/bbsdk-xxxxx") for isolation. VAULT_NAMESPACE contains the base namespace +// (e.g., "admin"), which is the parent of the test's namespace. +// Example: VAULT_NAMESPACE="admin" → test runs in "admin/bbsdk-xxxxx" → returns "admin" +// Note: This doesn't traverse the namespace hierarchy - it simply returns VAULT_NAMESPACE, +// which happens to be the parent of the test namespace. +func (s *Session) GetParentNamespace() string { + return getParentNamespace() +} + +func getParentNamespace() string { + ns := os.Getenv("VAULT_NAMESPACE") + // If VAULT_NAMESPACE is not set, default to root namespace (empty string). + // This handles cases where tests run in non-namespaced environments. + if ns == "" { + return "" + } + return ns +} diff --git a/sdk/helper/testcluster/blackbox/session_cluster.go b/sdk/helper/testcluster/blackbox/session_cluster.go new file mode 100644 index 0000000000..79b6e803ff --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_cluster.go @@ -0,0 +1,89 @@ +// Copyright IBM Corp. 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "fmt" + "time" +) + +// AssertClusterHealthy verifies that the cluster is healthy, with fallback for managed environments +// like HCP where raft APIs may not be accessible. This is the recommended method for general +// cluster health checks in blackbox tests. Uses backoff helper for reasonable retry logic. +// +// Deprecated: Use s.EventuallyClusterHealthy() with an explicit timeout +// retry times. +func (s *Session) AssertClusterHealthy() { + s.t.Helper() + + s.EventuallyClusterHealthyUnsealed(1 * time.Minute) +} + +// EventuallyClusterHealthyUnsealed verifies that the cluster is healthy and +// unsealed. +func (s *Session) EventuallyClusterHealthyUnsealed(timeout time.Duration) { + s.t.Helper() + + hasActiveNode := func() error { + // First, wait until we have an active HA node + _, err := s.haActiveNode() + return err + } + + autopilotHealthyIfRaft := func() error { + // Now, make sure autopilot is healthy if we're using integrated storage + // and we have the necessary permissions. + if s.GetParentNamespace() != "" { + s.t.Log("Skipping autopilot health check because we've been configured with a parent namespace and autopilot needs root namespace access") + return nil + } + + storage, err := s.getConfigStorageType() + if err != nil { + return err + } + + if storage != "raft" { + return nil + } + + healthy, err := s.autopilotStateHealthy() + if err != nil { + return err + } + + if !healthy { + return fmt.Errorf("raft autopilot state is not healthy") + } + + return nil + } + + clusterUnsealed := func() error { + sealed, err := s.sealed() + if err != nil { + return err + } + + if sealed { + return fmt.Errorf("cluster is sealed") + } + + return nil + } + + // Go through our checks and wait for each. + started := time.Now() + for _, check := range []func() error{ + hasActiveNode, + autopilotHealthyIfRaft, + clusterUnsealed, + } { + if timeout < 0 { + s.t.Fatal("timed out waiting for cluster to be healthy") + } + s.EventuallyWithTimeout(check, timeout) + timeout -= time.Since(started) + } +} diff --git a/sdk/helper/testcluster/blackbox/session_config.go b/sdk/helper/testcluster/blackbox/session_config.go new file mode 100644 index 0000000000..177b4a977b --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_config.go @@ -0,0 +1,89 @@ +// Copyright IBM Corp. 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "fmt" + "time" + + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +func (s *Session) MustSanitizedConfig() *api.Secret { + s.t.Helper() + + var config *api.Secret + err := s.Req( + func(c *api.Client) error { + var err error + config, err = c.Logical().Read("sys/config/state/sanitized") + return err + }, + WithClientRootNamespace(), + WithClientTimeout(2*time.Second), + ) + + require.NoError(s.t, err) + require.NotNil(s.t, config) + + return config +} + +func (s *Session) MustGetConfigStorageType() string { + s.t.Helper() + + sanitizedConfig := s.MustSanitizedConfig() + // Verify we have at least one server configured + storageType := s.AssertSecret(sanitizedConfig). + Data(). + GetMap("storage"). + GetKey("type") + + storage, ok := storageType.(string) + if !ok { + s.t.Fatalf("cluster storage is unknown: %v", storageType) + } + + if storage == "" { + s.t.Fatal("cluster storage is empty") + } + + s.t.Logf("cluster is using storage type: %s", storage) + + return storage +} + +func (s *Session) getConfigStorageType() (string, error) { + s.t.Helper() + + var storageTypeStr string + getStorageType := func(c *api.Client) error { + secret, err := c.Logical().Read("sys/seal-status") + if err != nil { + return err + } + + if secret == nil || len(secret.Data) < 1 { + return fmt.Errorf("seal-status is empty") + } + + storage, ok := secret.Data["storage_type"] + if !ok { + return fmt.Errorf("seal-status does not include storage_type") + } + + storageTypeStr, ok = storage.(string) + if !ok { + return fmt.Errorf("malformed storage type") + } + + return nil + } + + return storageTypeStr, s.Req( + getStorageType, + WithClientTimeout(2*time.Second), + ) +} diff --git a/sdk/helper/testcluster/blackbox/session_ha.go b/sdk/helper/testcluster/blackbox/session_ha.go new file mode 100644 index 0000000000..906e81ab1e --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_ha.go @@ -0,0 +1,85 @@ +// Copyright IBM Corp. 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "errors" + "time" + + "github.com/hashicorp/vault/api" +) + +// getClusterNodeCount returns the number of nodes in the cluster +func (s *Session) getClusterNodeCount() (int, error) { + nodes, err := s.haNodes() + if err != nil { + return 0, err + } + + return len(nodes), nil +} + +// haNodes returns a slice of nodes from the ha-status endpoint. +func (s *Session) haNodes() ([]map[string]any, error) { + res := []map[string]any{} + + return res, s.Req( + func(c *api.Client) error { + status, err := c.Logical().Read("sys/ha-status") + if err != nil { + return err + } + if status == nil { + return errors.New("no ha-status returned") + } + + nodesAny, ok := status.Data["nodes"] + if !ok { + return errors.New("no HA nodes found in ha-status") + } + + nodes, ok := nodesAny.([]any) + if !ok { + return errors.New("invalid ha-status response body") + } + + for _, node := range nodes { + nv, ok := node.(map[string]any) + if !ok { + return errors.New("malformed node in ha-status response body") + } + res = append(res, nv) + } + + return nil + }, + WithClientTimeout(2*time.Second), + ) +} + +// haActiveNode returns the active node from ha-status. +func (s *Session) haActiveNode() (map[string]any, error) { + nodes, err := s.haNodes() + if err != nil { + return nil, err + } + + for _, node := range nodes { + active, ok := node["active_node"] + if !ok { + continue + } + + activeVal, ok := active.(bool) + if !ok { + continue + } + + if activeVal { + return node, nil + } + } + + return nil, errors.New("no active node in ha-status") +} diff --git a/sdk/helper/testcluster/blackbox/session_logical.go b/sdk/helper/testcluster/blackbox/session_logical.go index c3d8601b8e..b847f5bbc5 100644 --- a/sdk/helper/testcluster/blackbox/session_logical.go +++ b/sdk/helper/testcluster/blackbox/session_logical.go @@ -10,6 +10,12 @@ import ( "github.com/stretchr/testify/require" ) +func (s *Session) Read(path string) (*api.Secret, error) { + s.t.Helper() + + return s.Client.Logical().Read(path) +} + func (s *Session) MustWrite(path string, data map[string]any) *api.Secret { s.t.Helper() @@ -26,6 +32,14 @@ func (s *Session) MustRead(path string) *api.Secret { return secret } +func (s *Session) MustList(path string) *api.Secret { + s.t.Helper() + + secret, err := s.Client.Logical().List(path) + require.NoError(s.t, err) + return secret +} + // MustReadRequired is a stricter version of MustRead that fails if a 404/nil is returned func (s *Session) MustReadRequired(path string) *api.Secret { s.t.Helper() @@ -51,3 +65,10 @@ func (s *Session) MustReadKV2(mountPath, secretPath string) *api.Secret { fullPath := path.Join(mountPath, "data", secretPath) return s.MustRead(fullPath) } + +func (s *Session) MustDelete(path string) { + s.t.Helper() + + _, err := s.Client.Logical().Delete(path) + require.NoError(s.t, err) +} diff --git a/sdk/helper/testcluster/blackbox/session_raft.go b/sdk/helper/testcluster/blackbox/session_raft.go index f7890579e5..550e6a28da 100644 --- a/sdk/helper/testcluster/blackbox/session_raft.go +++ b/sdk/helper/testcluster/blackbox/session_raft.go @@ -4,11 +4,10 @@ package blackbox import ( - "errors" + "fmt" "time" "github.com/hashicorp/vault/api" - "github.com/hashicorp/vault/sdk/helper/backoff" "github.com/stretchr/testify/require" ) @@ -33,20 +32,78 @@ func (s *Session) AssertRaftStable(numNodes int, allowNonVoters bool) { } } -func (s *Session) AssertRaftHealthy() { +func (s *Session) raftConfig() (*api.Secret, error) { + // TODO + // query the autopilot state endpoint and verify that all nodes are healthy according to autopilot + var state *api.Secret + return state, s.Req( + func(c *api.Client) error { + var err error + state, err = c.Logical().Read("sys/storage/raft/autopilot/state") + return err + }, + WithClientRootNamespace(), + WithClientTimeout(2*time.Second), + ) +} + +// EventuallyRaftClusterHealthy verifies that the raft cluster eventually becomes +// healthy regardless of node count. +func (s *Session) EventuallyRaftClusterHealthy(timeout time.Duration) { s.t.Helper() - // query the autopilot state endpoint and verify that all nodes are healthy according to autopilot + s.EventuallyWithTimeout( + func() error { + healthy, err := s.autopilotStateHealthy() + if err != nil { + return err + } + + if !healthy { + return fmt.Errorf("expected raft state to be healthy: got %t", healthy) + } + + return nil + }, timeout, + ) + + // TODO: Perhaps move the server config and voter checks into the retry above. + + // Get raft configuration to ensure we have at least one node secret, err := s.WithRootNamespace(func() (*api.Secret, error) { - return s.Client.Logical().Read("sys/storage/raft/autopilot/state") + return s.Client.Logical().Read("sys/storage/raft/configuration") }) require.NoError(s.t, err) require.NotNil(s.t, secret) - _ = s.AssertSecret(secret). + // Verify we have at least one server configured + servers := s.AssertSecret(secret). Data(). - HasKey("healthy", true) + GetMap("config"). + GetSlice("servers") + + // Ensure we have at least 1 server + if len(servers.data) < 1 { + s.t.Fatal("Expected at least 1 raft server, got 0") + } + + // Verify that we have at least one voter in the cluster + hasVoter := false + for _, server := range servers.data { + if serverMap, ok := server.(map[string]any); ok { + if voter, exists := serverMap["voter"]; exists { + if voterBool, ok := voter.(bool); ok && voterBool { + hasVoter = true + break + } + } + } + } + + if !hasVoter { + s.t.Fatal("Expected at least one voter in the raft cluster") + } } // AssertRaftClusterHealthy verifies that the raft cluster is healthy regardless of node count @@ -56,7 +113,7 @@ func (s *Session) AssertRaftClusterHealthy() { s.t.Helper() // First verify autopilot reports the cluster as healthy - s.AssertRaftHealthy() + s.AssertAutopilotHealthy() // Get raft configuration to ensure we have at least one node secret, err := s.WithRootNamespace(func() (*api.Secret, error) { @@ -150,210 +207,12 @@ func (s *Session) MustStepDownLeader() { require.NoError(s.t, err) } -// GetClusterNodeCount returns the number of nodes in the raft cluster -func (s *Session) GetClusterNodeCount() int { +// MustGetClusterNodeCount returns the number of nodes in the cluster +func (s *Session) MustGetClusterNodeCount() int { s.t.Helper() - secret, err := s.WithRootNamespace(func() (*api.Secret, error) { - return s.Client.Logical().Read("sys/storage/raft/configuration") - }) - if err != nil { - s.t.Logf("Failed to read raft configuration: %v", err) - return 0 - } + count, err := s.getClusterNodeCount() + require.NoError(s.t, err) - if secret == nil { - s.t.Log("Raft configuration response was nil") - return 0 - } - - configData, ok := secret.Data["config"].(map[string]any) - if !ok { - s.t.Log("Could not parse raft config data") - return 0 - } - - serversData, ok := configData["servers"].([]any) - if !ok { - s.t.Log("Could not parse raft servers data") - return 0 - } - - return len(serversData) -} - -// WaitForNewLeader waits for a new leader to be elected that is different from initialLeader -// and for the cluster to become healthy. For single-node clusters, it just waits for the -// cluster to become healthy again after stepdown. Uses reasonable timeouts to detect race conditions early. -func (s *Session) WaitForNewLeader(initialLeader string, timeoutSeconds int) { - s.t.Helper() - - // Use reasonable timeout - if it takes more than a few seconds, there's likely a race condition - if timeoutSeconds > 10 { - s.t.Logf("Warning: timeout of %d seconds is quite high, consider investigating potential race conditions", timeoutSeconds) - } - - // Check cluster size to handle single-node case - nodeCount := s.GetClusterNodeCount() - if nodeCount <= 1 { - s.t.Logf("Single-node cluster detected, waiting for cluster to recover after stepdown...") - - // Use backoff helper for single-node recovery - b := backoff.NewBackoff(20, 100*time.Millisecond, 1*time.Second) // Max ~10 seconds with backoff - - err := b.Retry(func() error { - secret, err := s.WithRootNamespace(func() (*api.Secret, error) { - return s.Client.Logical().Read("sys/storage/raft/autopilot/state") - }) - if err != nil { - return err - } - if secret == nil { - return errors.New("no autopilot state returned") - } - - healthy, ok := secret.Data["healthy"].(bool) - if !ok { - return errors.New("autopilot healthy status not found") - } - if !healthy { - return errors.New("cluster not yet healthy") - } - - return nil - }) - if err != nil { - s.t.Fatalf("Single-node cluster failed to recover: %v", err) - } - - s.t.Log("Single-node cluster has recovered and is healthy") - return - } - - // Multi-node cluster logic - wait for actual leader change - s.t.Logf("Multi-node cluster detected, waiting for new leader election...") - - // Phase 1: Wait for new leader (should be fast) - leaderBackoff := backoff.NewBackoff(20, 100*time.Millisecond, 500*time.Millisecond) // Max ~5 seconds - var currentLeader string - - err := leaderBackoff.Retry(func() error { - secret, err := s.WithRootNamespace(func() (*api.Secret, error) { - return s.Client.Logical().Read("sys/leader") - }) - if err != nil { - return err - } - if secret == nil { - return errors.New("no leader data returned") - } - - leaderAddress, ok := secret.Data["leader_address"].(string) - if !ok || leaderAddress == "" { - return errors.New("no leader address found") - } - - if leaderAddress == initialLeader { - return errors.New("still waiting for new leader") - } - - currentLeader = leaderAddress - return nil - }) - if err != nil { - s.t.Fatalf("Failed to elect new leader: %v", err) - } - - s.t.Logf("New leader elected: %s (was: %s)", currentLeader, initialLeader) - - // Phase 2: Wait for cluster health (should also be fast) - healthBackoff := backoff.NewBackoff(20, 100*time.Millisecond, 500*time.Millisecond) // Max ~5 seconds - - err = healthBackoff.Retry(func() error { - secret, err := s.WithRootNamespace(func() (*api.Secret, error) { - return s.Client.Logical().Read("sys/storage/raft/autopilot/state") - }) - if err != nil { - return err - } - if secret == nil { - return errors.New("no autopilot state returned") - } - - healthy, ok := secret.Data["healthy"].(bool) - if !ok { - return errors.New("autopilot healthy status not found") - } - if !healthy { - return errors.New("cluster not yet healthy") - } - - return nil - }) - if err != nil { - s.t.Fatalf("Cluster failed to become healthy with new leader: %v", err) - } - - s.t.Logf("Cluster is now healthy with new leader: %s", currentLeader) -} - -// AssertClusterHealthy verifies that the cluster is healthy, with fallback for managed environments -// like HCP where raft APIs may not be accessible. This is the recommended method for general -// cluster health checks in blackbox tests. Uses backoff helper for reasonable retry logic. -func (s *Session) AssertClusterHealthy() { - s.t.Helper() - - // Use backoff helper for cluster readiness checks - b := backoff.NewBackoff(15, 200*time.Millisecond, 2*time.Second) // Max ~15 seconds with backoff - - err := b.Retry(func() error { - // Try raft-based health check first (works for self-managed clusters) - secret, err := s.WithRootNamespace(func() (*api.Secret, error) { - return s.Client.Logical().Read("sys/storage/raft/autopilot/state") - }) - - if err == nil && secret != nil { - // Check if autopilot reports healthy - if healthy, ok := secret.Data["healthy"].(bool); ok && healthy { - // Raft API is available and healthy, use full raft health check - s.AssertRaftClusterHealthy() - return nil - } else if ok && !healthy { - return errors.New("cluster not yet healthy according to autopilot") - } - } - - // Raft API not accessible or no healthy status - check basic connectivity - sealStatus, err := s.WithRootNamespace(func() (*api.Secret, error) { - return s.Client.Logical().Read("sys/seal-status") - }) - if err != nil { - return err - } - - if sealStatus == nil { - return errors.New("seal status response was nil") - } - - // Verify cluster is unsealed - sealed, ok := sealStatus.Data["sealed"].(bool) - if !ok { - return errors.New("could not determine seal status") - } - - if sealed { - return errors.New("cluster is sealed") - } - - // If we get here, cluster is unsealed and responsive - if secret != nil { - s.t.Log("Cluster health verified (self-managed environment)") - } else { - s.t.Log("Cluster health verified (managed environment - raft APIs not accessible)") - } - return nil - }) - if err != nil { - s.t.Fatalf("Cluster health check failed: %v", err) - } + return count } diff --git a/sdk/helper/testcluster/blackbox/session_replication.go b/sdk/helper/testcluster/blackbox/session_replication.go new file mode 100644 index 0000000000..f7ec38d3e2 --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_replication.go @@ -0,0 +1,63 @@ +// Copyright IBM Corp. 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "fmt" + + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +// GetDRReplicationMode gets the DR replication mode of the node +func (s *Session) GetDRReplicationMode() (string, error) { + secret, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/replication/dr/status") + }) + if err != nil { + return "", err + } + + if secret == nil { + return "", fmt.Errorf("empty status response") + } + + mode, ok := secret.Data["mode"] + if !ok { + return "", fmt.Errorf("no mode field in status response") + } + + return mode.(string), nil +} + +func (s *Session) AssertReplicationDisabled() { + s.assertReplicationStatus("ce", "disabled") +} + +func (s *Session) AssertDRReplicationStatus(expectedMode string) { + s.assertReplicationStatus("dr", expectedMode) +} + +func (s *Session) AssertPerformanceReplicationStatus(expectedMode string) { + s.assertReplicationStatus("performance", expectedMode) +} + +func (s *Session) assertReplicationStatus(which, expectedMode string) { + s.t.Helper() + + secret, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/replication/status") + }) + + require.NoError(s.t, err) + require.NotNil(s.t, secret) + + data := s.AssertSecret(secret).Data() + + if which == "ce" { + data.HasKey("mode", "disabled") + } else { + data.GetMap(which).HasKey("mode", expectedMode) + } +} diff --git a/sdk/helper/testcluster/blackbox/session_seal.go b/sdk/helper/testcluster/blackbox/session_seal.go new file mode 100644 index 0000000000..be609b1146 --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_seal.go @@ -0,0 +1,27 @@ +// Copyright IBM Corp. 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "errors" +) + +func (s *Session) sealed() (bool, error) { + sealStatus, err := s.Client.Logical().Read("sys/seal-status") + if err != nil { + return true, err + } + + if sealStatus == nil { + return true, errors.New("seal status response was nil") + } + + // Verify cluster is unsealed + sealed, ok := sealStatus.Data["sealed"].(bool) + if !ok { + return true, errors.New("could not determine seal status") + } + + return sealed, nil +} diff --git a/sdk/helper/testcluster/blackbox/session_status.go b/sdk/helper/testcluster/blackbox/session_status.go index 3b9f97ad63..4117205cf7 100644 --- a/sdk/helper/testcluster/blackbox/session_status.go +++ b/sdk/helper/testcluster/blackbox/session_status.go @@ -8,7 +8,6 @@ import ( "os/exec" "strings" - "github.com/hashicorp/vault/api" "github.com/stretchr/testify/require" ) @@ -158,34 +157,3 @@ func (s *Session) AssertServerVersion(version, buildDate string) { s.AssertVersion(version) s.AssertBuildDate(version, buildDate) } - -func (s *Session) AssertReplicationDisabled() { - s.assertReplicationStatus("ce", "disabled") -} - -func (s *Session) AssertDRReplicationStatus(expectedMode string) { - s.assertReplicationStatus("dr", expectedMode) -} - -func (s *Session) AssertPerformanceReplicationStatus(expectedMode string) { - s.assertReplicationStatus("performance", expectedMode) -} - -func (s *Session) assertReplicationStatus(which, expectedMode string) { - s.t.Helper() - - secret, err := s.WithRootNamespace(func() (*api.Secret, error) { - return s.Client.Logical().Read("sys/replication/status") - }) - - require.NoError(s.t, err) - require.NotNil(s.t, secret) - - data := s.AssertSecret(secret).Data() - - if which == "ce" { - data.HasKey("mode", "disabled") - } else { - data.GetMap(which).HasKey("mode", expectedMode) - } -} diff --git a/sdk/helper/testcluster/blackbox/session_util.go b/sdk/helper/testcluster/blackbox/session_util.go index 943dadb3b5..0fe65ddc0b 100644 --- a/sdk/helper/testcluster/blackbox/session_util.go +++ b/sdk/helper/testcluster/blackbox/session_util.go @@ -4,7 +4,6 @@ package blackbox import ( - "os" "time" "github.com/hashicorp/vault/api" @@ -63,20 +62,3 @@ func (s *Session) WithParentNamespace(fn func() (*api.Secret, error)) (*api.Secr return fn() } - -// GetParentNamespace returns the namespace from VAULT_NAMESPACE environment variable. -// The blackbox test framework auto-creates a unique child namespace for each test -// (e.g., "admin/bbsdk-xxxxx") for isolation. VAULT_NAMESPACE contains the base namespace -// (e.g., "admin"), which is the parent of the test's namespace. -// Example: VAULT_NAMESPACE="admin" → test runs in "admin/bbsdk-xxxxx" → returns "admin" -// Note: This doesn't traverse the namespace hierarchy - it simply returns VAULT_NAMESPACE, -// which happens to be the parent of the test namespace. -func (s *Session) GetParentNamespace() string { - ns := os.Getenv("VAULT_NAMESPACE") - // If VAULT_NAMESPACE is not set, default to root namespace (empty string). - // This handles cases where tests run in non-namespaced environments. - if ns == "" { - return "" - } - return ns -} diff --git a/vault/external_tests/blackbox/doc.go b/vault/external_tests/blackbox/doc.go new file mode 100644 index 0000000000..c348460fcb --- /dev/null +++ b/vault/external_tests/blackbox/doc.go @@ -0,0 +1,6 @@ +// Copyright IBM Corp. 2026 +// SPDX-License-Identifier: BUSL-1.1 + +// Package blackbox includes external integration tests for testing Vault at the +// API level. +package blackbox diff --git a/vault/external_tests/blackbox/integration/external_secrets_test.go b/vault/external_tests/blackbox/integration/external_secrets_test.go deleted file mode 100644 index 2e0fcab614..0000000000 --- a/vault/external_tests/blackbox/integration/external_secrets_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright IBM Corp. 2025, 2026 -// SPDX-License-Identifier: BUSL-1.1 - -package integration - -import ( - "testing" - - "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" -) - -// TestSecretsEngineExternalCreate tests creation/setup of secrets engines that require external infrastructure -// These tests are excluded from cloud environments (HCP/Docker) which don't have access to AWS, LDAP servers, etc. -func TestSecretsEngineExternalCreate(t *testing.T) { - v := blackbox.New(t) - - // Verify we have a healthy cluster first - v.AssertClusterHealthy() - - t.Run("AWSSecrets", func(t *testing.T) { - testAWSSecretsCreate(t, v) - }) - - t.Run("LDAPSecrets", func(t *testing.T) { - testLDAPSecretsCreate(t, v) - }) - - t.Run("KMIPSecrets", func(t *testing.T) { - testKMIPSecretsCreate(t, v) - }) -} - -// TestSecretsEngineExternalRead tests read operations for secrets engines that require external infrastructure -func TestSecretsEngineExternalRead(t *testing.T) { - v := blackbox.New(t) - - // Verify we have a healthy cluster first - v.AssertClusterHealthy() - - t.Run("AWSSecrets", func(t *testing.T) { - testAWSSecretsRead(t, v) - }) - - t.Run("LDAPSecrets", func(t *testing.T) { - testLDAPSecretsRead(t, v) - }) - - t.Run("KMIPSecrets", func(t *testing.T) { - testKMIPSecretsRead(t, v) - }) -} - -// TestSecretsEngineExternalDelete tests delete operations for secrets engines that require external infrastructure -func TestSecretsEngineExternalDelete(t *testing.T) { - v := blackbox.New(t) - - // Verify we have a healthy cluster first - v.AssertClusterHealthy() - - t.Run("LDAPSecrets", func(t *testing.T) { - testLDAPSecretsDelete(t, v) - }) -} diff --git a/vault/external_tests/blackbox/integration/smoke_test.go b/vault/external_tests/blackbox/integration/smoke_test.go index 10848c03a5..27bd8ae51f 100644 --- a/vault/external_tests/blackbox/integration/smoke_test.go +++ b/vault/external_tests/blackbox/integration/smoke_test.go @@ -5,43 +5,27 @@ package integration import ( "testing" + "time" "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" + "github.com/stretchr/testify/require" ) -// TestEnosSmoke performs comprehensive smoke testing for Enos scenarios, -// verifying cluster health, replication status, raft stability, and basic -// KV operations with authentication. This test validates core functionality. -func TestEnosSmoke(t *testing.T) { - v := blackbox.New(t) - - v.AssertUnsealedAny() - v.AssertDRReplicationStatus("primary") - v.AssertPerformanceReplicationStatus("disabled") - v.AssertRaftStable(3, false) - v.AssertRaftHealthy() - - // Setup using common utilities - bob := SetupStandardKVUserpass(v, "secret", "bob", "lol") - - // Write and verify standard test data - v.MustWriteKV2("secret", "app-config", StandardKVData) - - secret := bob.MustReadKV2("secret", "app-config") - AssertKVData(t, bob, secret, StandardKVData) -} - -// TestStepdownAndLeaderElection tests raft leadership changes by forcing the current -// leader to step down and verifying that a new leader is elected successfully, -// while ensuring the cluster remains healthy throughout the process. +// TestStepdownAndLeaderElection tests raft leadership changes by forcing the +// current leader to step down and verifying that a new leader is elected +// successfully, while ensuring the cluster remains healthy throughout the +// process. +// +// NOTE: We do not run this test in parallel to avoid making other tests flaky +// during leader elections. func TestStepdownAndLeaderElection(t *testing.T) { v := blackbox.New(t) - // Verify we have a healthy raft cluster first - v.AssertRaftClusterHealthy() + // Wait for a healthy cluster + v.EventuallyClusterHealthyUnsealed(15 * time.Second) // Check cluster size to determine expected behavior - nodeCount := v.GetClusterNodeCount() + nodeCount := v.MustGetClusterNodeCount() t.Logf("Cluster has %d nodes", nodeCount) // Get current leader before step down @@ -51,20 +35,17 @@ func TestStepdownAndLeaderElection(t *testing.T) { // Force leader to step down v.MustStepDownLeader() - // Wait for new leader election (with timeout) - // Use generous timeout to handle network latency and complex backend coordination - v.WaitForNewLeader(initialLeader, 300) + // Wait for a new leader to be active and the cluster to be healthy + v.EventuallyClusterHealthyUnsealed(1 * time.Minute) - // Verify cluster is still healthy after leader change/recovery - v.AssertRaftClusterHealthy() + // Get current leader before step down + newLeader := v.MustGetCurrentLeader() + t.Logf("New leader: %s", initialLeader) // For multi-node clusters, verify new leader is different from initial leader // For single-node clusters, just verify it's healthy again - newLeader := v.MustGetCurrentLeader() if nodeCount > 1 { - if newLeader == initialLeader { - t.Fatalf("Expected new leader to be different from initial leader %s, got %s", initialLeader, newLeader) - } + require.NotEqual(t, initialLeader, newLeader, "Expected new leader to be different from initial leader") t.Logf("Successfully elected new leader: %s (was: %s)", newLeader, initialLeader) } else { t.Logf("Single-node cluster successfully recovered with leader: %s", newLeader) diff --git a/vault/external_tests/blackbox/plugins/aws/helpers.go b/vault/external_tests/blackbox/plugins/aws/helpers.go new file mode 100644 index 0000000000..e9852ed715 --- /dev/null +++ b/vault/external_tests/blackbox/plugins/aws/helpers.go @@ -0,0 +1,290 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package aws + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/sts" +) + +var ErrPolicyNotFound = errors.New("policy not found") + +// getPolicyArnByName returns the ARN for a policy with the given name. +func getPolicyArnByName(ctx context.Context, iamClient *iam.Client, policyName string) (string, error) { + paginator := iam.NewListPoliciesPaginator(iamClient, &iam.ListPoliciesInput{Scope: "All"}) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return "", err + } + for _, p := range page.Policies { + if aws.ToString(p.PolicyName) == policyName { + return aws.ToString(p.Arn), nil + } + } + } + return "", ErrPolicyNotFound +} + +// getRoleArnByName returns the ARN for a role with the given name. +func getRoleArnByName(ctx context.Context, iamClient *iam.Client, roleName string) (string, error) { + paginator := iam.NewListRolesPaginator(iamClient, &iam.ListRolesInput{}) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return "", err + } + for _, r := range page.Roles { + if aws.ToString(r.RoleName) == roleName { + return aws.ToString(r.Arn), nil + } + } + } + return "", fmt.Errorf("role %s not found", roleName) +} + +// hasDemoUserPolicy determines whether or not the DemoUser policy has been +// assigned to the AWS credential idendity being used. +func hasDemoUserPolicy(ctx context.Context) (bool, error) { + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return false, err + } + iamClient := iam.NewFromConfig(cfg) + + // Lookup DemoUser policy ARN + _, err = getPolicyArnByName(ctx, iamClient, "DemoUser") + if err == nil { + return true, nil + } + + if errors.Is(err, ErrPolicyNotFound) { + return false, nil + } + + return false, err +} + +// createTestIAMUser creates a new IAM user with a unique name, attaches the DemoUser policy, and returns the user/access key info and AWS region. +func createTestIAMUser(t *testing.T) ( + userName string, + accessKeyID string, + secretAccessKey string, + demoUserPolicyArn string, + assumedRoleArn string, + awsRegion string, +) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + t.Fatalf("failed to load AWS config: %v", err) + } + awsRegion = cfg.Region + if awsRegion == "" { + t.Fatalf("AWS region is empty in config") + } + iamClient := iam.NewFromConfig(cfg) + stsClient := sts.NewFromConfig(cfg) + + // Get current AWS account identity (for unique name) + caller, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + t.Fatalf("failed to get caller identity: %v", err) + } + accountID := aws.ToString(caller.Account) + + // Generate a random hex suffix for uniqueness + const randomSuffixByteLength = 4 + suffix := make([]byte, randomSuffixByteLength) + if _, err := rand.Read(suffix); err != nil { + t.Fatalf("failed to generate random suffix: %v", err) + } + hexSuffix := hex.EncodeToString(suffix) + userName = fmt.Sprintf("demo-GitHubActions-%s-%s", accountID, hexSuffix) + + // Lookup DemoUser policy ARN + demoUserPolicyArn, err = getPolicyArnByName(ctx, iamClient, "DemoUser") + if err != nil { + t.Fatalf("DemoUser policy not found: %v", err) + } + + // Lookup vault-assumed-role-credentials-demo role ARN + assumedRoleArn, err = getRoleArnByName(ctx, iamClient, "vault-assumed-role-credentials-demo") + if err != nil { + t.Fatalf("vault-assumed-role-credentials-demo role not found: %v", err) + } + + // Create IAM user + _, err = iamClient.CreateUser(ctx, &iam.CreateUserInput{ + UserName: aws.String(userName), + PermissionsBoundary: aws.String(demoUserPolicyArn), + }) + if err != nil { + t.Fatalf("failed to create IAM user: %v", err) + } + + // Attach policy to user + _, err = iamClient.AttachUserPolicy(ctx, &iam.AttachUserPolicyInput{ + UserName: aws.String(userName), + PolicyArn: aws.String(demoUserPolicyArn), + }) + if err != nil { + t.Fatalf("failed to attach policy: %v", err) + } + + // Create access key + keyOut, err := iamClient.CreateAccessKey(ctx, &iam.CreateAccessKeyInput{ + UserName: aws.String(userName), + }) + if err != nil { + t.Fatalf("failed to create access key: %v", err) + } + accessKeyID = aws.ToString(keyOut.AccessKey.AccessKeyId) + secretAccessKey = aws.ToString(keyOut.AccessKey.SecretAccessKey) + + // IAM is eventually consistent; wait briefly before verifying the user is readable. + t.Logf("Verifying IAM user %s exists...", userName) + waitTime := 10 * time.Second + verifyDeadline := time.Now().Add(waitTime * 2) + var lastErr error + for time.Now().Before(verifyDeadline) { + time.Sleep(waitTime) + _, lastErr = iamClient.GetUser(ctx, &iam.GetUserInput{UserName: aws.String(userName)}) + if lastErr == nil { + break + } + t.Logf("IAM user %q not readable yet; retrying: %v", userName, lastErr) + } + if lastErr != nil { + t.Fatalf("failed to verify IAM user %q: %v", userName, lastErr) + } + + return userName, accessKeyID, secretAccessKey, demoUserPolicyArn, assumedRoleArn, awsRegion +} + +// getAwsUsernameTemplate returns the username template string for Vault AWS config. +func getAwsUsernameTemplate(awsUserName string) string { + const prefix = `{{ if (eq .Type "STS") }}{{ printf "` + const stsSuffix = `-%s-%s" (random 20) (unix_time) | truncate 32 }}{{ else }}{{ printf "` + const iamUserSuffix = `-%s-%s" (unix_time) (random 20) | truncate 60 }}{{ end }}` + return prefix + awsUserName + stsSuffix + awsUserName + iamUserSuffix +} + +// getAllowDescribeRegionsPolicy returns a policy document allowing ec2:DescribeRegions. +func getAllowDescribeRegionsPolicy() string { + return `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["ec2:DescribeRegions"], + "Resource": ["*"] + } + ] +}` +} + +// deleteIAMUserByAccessKey deletes the IAM user that owns the given access key. +func deleteIAMUserByAccessKey(t *testing.T, targetAccessKeyID string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + t.Fatalf("failed to load AWS config: %v", err) + } + iamClient := iam.NewFromConfig(cfg) + + paginator := iam.NewListUsersPaginator(iamClient, &iam.ListUsersInput{}) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + t.Fatalf("failed to list IAM users: %v", err) + } + for _, user := range page.Users { + userName := aws.ToString(user.UserName) + if !strings.Contains(userName, "demo-GitHubActions") { + continue + } + // List all access keys for this user + keyPaginator := iam.NewListAccessKeysPaginator(iamClient, &iam.ListAccessKeysInput{ + UserName: &userName, + }) + for keyPaginator.HasMorePages() { + keyPage, err := keyPaginator.NextPage(ctx) + if err != nil { + t.Logf("warning: failed to list access keys for user %q: %v", userName, err) + continue + } + for _, key := range keyPage.AccessKeyMetadata { + accessKeyId := aws.ToString(key.AccessKeyId) + if accessKeyId == targetAccessKeyID { + // Found the user with the target access key. Detach managed policies first, + // then delete all access keys, then delete the user. + policyPaginator := iam.NewListAttachedUserPoliciesPaginator(iamClient, &iam.ListAttachedUserPoliciesInput{ + UserName: &userName, + }) + for policyPaginator.HasMorePages() { + policyPage, err := policyPaginator.NextPage(ctx) + if err != nil { + t.Logf("warning: failed to list attached policies for user %q: %v", userName, err) + continue + } + for _, policy := range policyPage.AttachedPolicies { + policyArn := aws.ToString(policy.PolicyArn) + if _, err := iamClient.DetachUserPolicy(ctx, &iam.DetachUserPolicyInput{ + UserName: &userName, + PolicyArn: &policyArn, + }); err != nil { + t.Logf("warning: failed to detach policy %q from user %q: %v", policyArn, userName, err) + } + } + } + keyPaginator2 := iam.NewListAccessKeysPaginator(iamClient, &iam.ListAccessKeysInput{ + UserName: &userName, + }) + for keyPaginator2.HasMorePages() { + keyPage2, err := keyPaginator2.NextPage(ctx) + if err != nil { + t.Logf("warning: failed to list access keys for cleanup on user %q: %v", userName, err) + continue + } + for _, key2 := range keyPage2.AccessKeyMetadata { + accessKeyId2 := aws.ToString(key2.AccessKeyId) + if _, err := iamClient.DeleteAccessKey(ctx, &iam.DeleteAccessKeyInput{ + UserName: &userName, + AccessKeyId: &accessKeyId2, + }); err != nil { + t.Logf("warning: failed to delete access key %q for user %q: %v", accessKeyId2, userName, err) + } + } + } + // Delete the user + if _, err := iamClient.DeleteUser(ctx, &iam.DeleteUserInput{ + UserName: &userName, + }); err != nil { + t.Logf("warning: failed to delete user %q: %v", userName, err) + } + return + } + } + } + } + } +} diff --git a/vault/external_tests/blackbox/plugins/aws/secrets_aws_test.go b/vault/external_tests/blackbox/plugins/aws/secrets_aws_test.go new file mode 100644 index 0000000000..1ad9dd3845 --- /dev/null +++ b/vault/external_tests/blackbox/plugins/aws/secrets_aws_test.go @@ -0,0 +1,113 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package aws + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/hashicorp/vault/api" + "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" +) + +// TestAWS_GenerateNewUser verifies AWS secrets engine can generate IAM user credentials. +func TestAWS_GenerateNewUser(t *testing.T) { + t.Parallel() + v := blackbox.New(t) + + accessKey := os.Getenv("AWS_ACCESS_KEY_ID") + secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY") + if accessKey == "" || secretKey == "" { + t.Log("AWS credentials not available - skipping AWS secrets engine test") + t.Skip("AWS credentials not available - skipping AWS secrets engine test") + } + + hasDUP, err := hasDemoUserPolicy(t.Context()) + if err != nil { + t.Fatal(err) + } + + if !hasDUP { + // TODO: We can probably check for IAM permission instead of a policy. That + // would require rewriting the whole test though. + t.Skip("Skipping test as it requires a special DemoUser policy that is not assigned to the current AWS credentials") + } + + t.Logf("Creating test IAM user via helpers.go...") + userName, tempAccessKeyId, tempSecretAccessKey, demoUserPolicyArn, _, _ := createTestIAMUser(t) + t.Logf("Created test IAM user: %s", userName) + var newAccessKey string + t.Cleanup(func() { + if newAccessKey != "" { + t.Logf("Cleanup: deleting IAM user created by Vault with access key: %s", newAccessKey) + deleteIAMUserByAccessKey(t, newAccessKey) + } + + t.Logf("Cleanup: deleting IAM user by initial access key: %s", tempAccessKeyId) + deleteIAMUserByAccessKey(t, tempAccessKeyId) + }) + + path := fmt.Sprintf("aws-test-%d", time.Now().UnixNano()) + t.Logf("Enabling AWS secrets engine at path: %s", path) + v.MustEnableSecretsEngine(path, &api.MountInput{Type: "aws"}) + + t.Logf("Configuring AWS secrets engine with root credentials and username template for user: %s", userName) + v.MustWrite(fmt.Sprintf("%s/config/root", path), map[string]any{ + "access_key": tempAccessKeyId, + "secret_key": tempSecretAccessKey, + "region": "us-east-1", + "username_template": getAwsUsernameTemplate(userName), + }) + + roleName := "aws-enos-role" + t.Logf("Creating Vault AWS role: %s", roleName) + v.MustWrite(fmt.Sprintf("%s/roles/%s", path, roleName), map[string]any{ + "credential_type": "iam_user", + "permissions_boundary_arn": demoUserPolicyArn, + "policy_document": getAllowDescribeRegionsPolicy(), + }) + + t.Logf("Reading and verifying AWS role configuration for role: %s", roleName) + roleResp := v.MustRead(fmt.Sprintf("%s/roles/%s", path, roleName)) + if roleResp.Data == nil { + t.Fatal("Expected to read AWS role configuration") + } + + t.Logf("Listing AWS roles at path: %s/roles", path) + rolesList := v.MustList(fmt.Sprintf("%s/roles", path)) + if rolesList == nil || rolesList.Data == nil { + t.Fatal("No AWS roles created! (rolesList is nil or Data is nil)") + } + roleKeys, ok := rolesList.Data["keys"].([]interface{}) + if !ok || len(roleKeys) == 0 { + t.Fatal("No AWS roles created! (rolesList.Data['keys'] is empty or not a slice)") + } + t.Logf("Found AWS roles: %v", roleKeys) + + t.Logf("Reading root config to verify username template is set correctly") + rootUser := v.MustRead(fmt.Sprintf("%s/config/root", path)) + if rootUser == nil || rootUser.Data == nil { + t.Fatalf("Expected to read root config, got nil: %#v", rootUser) + } + if val, ok := rootUser.Data["username_template"]; !ok || val == nil { + t.Fatalf("username_template missing in root config: %#v", rootUser) + } + + t.Logf("Generating new credentials for IAM user using role: %s", roleName) + newUser := v.MustRead(fmt.Sprintf("%s/creds/%s", path, roleName)) + if newUser == nil || newUser.Data == nil { + t.Fatalf("Failed to generate new credentials for IAM user: %s", roleName) + } + if val, ok := newUser.Data["access_key"]; !ok || val == nil || val == tempAccessKeyId { + t.Fatalf("The new access key is empty or is matching the old one: %v", val) + } + + newAccessKey, ok = newUser.Data["access_key"].(string) + if !ok || newAccessKey == "" { + t.Fatalf("Could not extract access_key from new credentials: %v", newUser.Data["access_key"]) + } + t.Logf("Captured Vault-created access key for cleanup: %s", newAccessKey) +} diff --git a/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_connection_config_test.go b/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_connection_config_test.go index a90911d1ed..219b3789d4 100644 --- a/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_connection_config_test.go +++ b/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_connection_config_test.go @@ -36,7 +36,7 @@ func TestMongoDBConnectionConfigCRUDWorkflows(t *testing.T) { // MongoDB database connection succeeds at database/config/{name}. func testMongoDBConnectionConfigCreateBasic(t *testing.T, v *blackbox.Session) { requireVaultEnv(t) - cleanup, connURL := PrepareTestContainer(t) + cleanup, connURL, _, _ := PrepareTestContainer(t) defer cleanup() mount := fmt.Sprintf("database-%s", sanitize(t.Name())) diff --git a/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_delete_test.go b/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_delete_test.go index be6bb9d0cc..e26bd8876b 100644 --- a/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_delete_test.go +++ b/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_delete_test.go @@ -4,7 +4,6 @@ package mongodb import ( - "strings" "testing" "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" @@ -45,9 +44,9 @@ func TestMongoDBStaticRoleDeleteWorkflows(t *testing.T) { // testMongoDBStaticRoleDeleteExistingRole verifies deleting an existing // static role succeeds and removes it from the list. func testMongoDBStaticRoleDeleteExistingRole(t *testing.T, v *blackbox.Session) { - mount, connURL := setupMongoDBTest(t, v) + mount, _, dbName, client := setupMongoDBTest(t, v) - createMongoDBUser(t, connURL, deleteTestUsername, testInitialPassword) + createMongoDBUser(t, client, dbName, deleteTestUsername, testInitialPassword) v.MustWrite(mount+"/static-roles/"+deleteTestRoleName, map[string]any{ "db_name": testConnectionName, @@ -62,20 +61,24 @@ func testMongoDBStaticRoleDeleteExistingRole(t *testing.T, v *blackbox.Session) // Delete the role v.MustDelete(mount + "/static-roles/" + deleteTestRoleName) - // Verify role is gone - _, err := v.Read(mount + "/static-roles/" + deleteTestRoleName) - if err == nil { - t.Fatal("expected error when reading deleted role") + // Verify role is gone - Read returns nil secret for non-existent paths + secret, err := v.Read(mount + "/static-roles/" + deleteTestRoleName) + if err != nil { + // Error is acceptable + } else if secret != nil { + t.Fatal("expected nil secret when reading deleted role") } // Verify role is not in list list := v.MustList(mount + "/static-roles") - if list.Data["keys"] != nil { - keys := list.Data["keys"].([]interface{}) - for _, k := range keys { - if k.(string) == deleteTestRoleName { - t.Fatalf("deleted role %s should not appear in list", deleteTestRoleName) - } + if list == nil || list.Data == nil || list.Data["keys"] == nil { + return + } + + keys := list.Data["keys"].([]interface{}) + for _, k := range keys { + if k.(string) == deleteTestRoleName { + t.Fatalf("deleted role %s should not appear in list", deleteTestRoleName) } } } @@ -83,9 +86,9 @@ func testMongoDBStaticRoleDeleteExistingRole(t *testing.T, v *blackbox.Session) // testMongoDBStaticRoleDeletePreventsCredentialAccess verifies that after // deleting a static role, credentials can no longer be read. func testMongoDBStaticRoleDeletePreventsCredentialAccess(t *testing.T, v *blackbox.Session) { - mount, connURL := setupMongoDBTest(t, v) + mount, _, dbName, client := setupMongoDBTest(t, v) - createMongoDBUser(t, connURL, deleteTestUsername, testInitialPassword) + createMongoDBUser(t, client, dbName, deleteTestUsername, testInitialPassword) v.MustWrite(mount+"/static-roles/"+deleteTestRoleName, map[string]any{ "db_name": testConnectionName, @@ -95,24 +98,27 @@ func testMongoDBStaticRoleDeletePreventsCredentialAccess(t *testing.T, v *blackb // Verify credentials can be read before deletion creds := v.MustReadRequired(mount + "/static-creds/" + deleteTestRoleName) - v.AssertSecret(creds).Data().HasKey("username").HasKey("password") + v.AssertSecret(creds).Data().HasKeyExists("username") + v.AssertSecret(creds).Data().HasKeyExists("password") // Delete the role v.MustDelete(mount + "/static-roles/" + deleteTestRoleName) - // Verify credentials can no longer be read - _, err := v.Read(mount + "/static-creds/" + deleteTestRoleName) - if err == nil { - t.Fatal("expected error when reading credentials for deleted role") + // Verify credentials can no longer be read - Read returns nil secret for deleted roles + secret, err := v.Read(mount + "/static-creds/" + deleteTestRoleName) + if err != nil { + // Error is acceptable + } else if secret != nil { + t.Fatal("expected nil secret when reading credentials for deleted role") } } // testMongoDBStaticRoleDeleteIsIdempotent verifies that deleting a role // multiple times succeeds without error. func testMongoDBStaticRoleDeleteIsIdempotent(t *testing.T, v *blackbox.Session) { - mount, connURL := setupMongoDBTest(t, v) + mount, _, dbName, client := setupMongoDBTest(t, v) - createMongoDBUser(t, connURL, deleteTestUsername, testInitialPassword) + createMongoDBUser(t, client, dbName, deleteTestUsername, testInitialPassword) v.MustWrite(mount+"/static-roles/"+deleteTestRoleName, map[string]any{ "db_name": testConnectionName, @@ -123,10 +129,12 @@ func testMongoDBStaticRoleDeleteIsIdempotent(t *testing.T, v *blackbox.Session) // Delete the role first time v.MustDelete(mount + "/static-roles/" + deleteTestRoleName) - // Verify role is gone - _, err := v.Read(mount + "/static-roles/" + deleteTestRoleName) - if err == nil { - t.Fatal("expected error when reading deleted role") + // Verify role is gone - Read returns nil secret for non-existent paths + secret, err := v.Read(mount + "/static-roles/" + deleteTestRoleName) + if err != nil { + // Error is acceptable + } else if secret != nil { + t.Fatal("expected nil secret when reading deleted role") } // Delete the role again - should succeed (idempotent) @@ -139,18 +147,18 @@ func testMongoDBStaticRoleDeleteIsIdempotent(t *testing.T, v *blackbox.Session) // testMongoDBStaticRoleDeleteNonExistentRole verifies that attempting to // delete a non-existent role succeeds (idempotent behavior). func testMongoDBStaticRoleDeleteNonExistentRole(t *testing.T, v *blackbox.Session) { - mount, _ := setupMongoDBTest(t, v) + mount, _, _, _ := setupMongoDBTest(t, v) // Attempt to delete a role that was never created v.MustDelete(mount + "/static-roles/never-existed-role") - // Verify it's still not there - _, err := v.Read(mount + "/static-roles/never-existed-role") - if err == nil { - t.Fatal("expected error when reading non-existent role") + // Verify it's still not there - Read returns nil secret for non-existent paths + secret, err := v.Read(mount + "/static-roles/never-existed-role") + if err != nil { + // Error is acceptable + return } - - if !strings.Contains(err.Error(), "not found") && !strings.Contains(err.Error(), "no value") { - t.Logf("Note: error message may not clearly indicate 'not found': %v", err) + if secret != nil { + t.Fatal("expected nil secret when reading non-existent role") } } diff --git a/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_read_test.go b/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_read_test.go index 056a1631b5..2e59f9e244 100644 --- a/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_read_test.go +++ b/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_read_test.go @@ -45,9 +45,9 @@ func TestMongoDBStaticRoleReadWorkflows(t *testing.T) { // testMongoDBStaticRoleReadReturnsConfiguration verifies reading a static role // returns its configuration without sensitive data. func testMongoDBStaticRoleReadReturnsConfiguration(t *testing.T, v *blackbox.Session) { - mount, connURL := setupMongoDBTest(t, v) + mount, _, dbName, client := setupMongoDBTest(t, v) - createMongoDBUser(t, connURL, readTestUsername, testInitialPassword) + createMongoDBUser(t, client, dbName, readTestUsername, testInitialPassword) v.MustWrite(mount+"/static-roles/"+readTestRoleName, map[string]any{ "db_name": testConnectionName, @@ -68,26 +68,30 @@ func testMongoDBStaticRoleReadReturnsConfiguration(t *testing.T, v *blackbox.Ses } // Verify last_vault_rotation is present - v.AssertSecret(role).Data().HasKey("last_vault_rotation") + v.AssertSecret(role).Data().HasKeyExists("last_vault_rotation") } // testMongoDBStaticRoleReadNonExistentRole verifies reading a non-existent -// static role returns an appropriate error. +// static role returns nil secret. func testMongoDBStaticRoleReadNonExistentRole(t *testing.T, v *blackbox.Session) { - mount, _ := setupMongoDBTest(t, v) + mount, _, _, _ := setupMongoDBTest(t, v) - _, err := v.Read(mount + "/static-roles/nonexistent-role") - if err == nil { - t.Fatal("expected error when reading non-existent role") + secret, err := v.Read(mount + "/static-roles/nonexistent-role") + if err != nil { + // Error is acceptable + return + } + if secret != nil { + t.Fatal("expected nil secret when reading non-existent role") } } // testMongoDBStaticRoleReadAfterUpdate verifies reading a static role after // updating it returns the updated configuration. func testMongoDBStaticRoleReadAfterUpdate(t *testing.T, v *blackbox.Session) { - mount, connURL := setupMongoDBTest(t, v) + mount, _, dbName, client := setupMongoDBTest(t, v) - createMongoDBUser(t, connURL, readTestUsername, testInitialPassword) + createMongoDBUser(t, client, dbName, readTestUsername, testInitialPassword) v.MustWrite(mount+"/static-roles/"+readTestRoleName, map[string]any{ "db_name": testConnectionName, @@ -98,9 +102,12 @@ func testMongoDBStaticRoleReadAfterUpdate(t *testing.T, v *blackbox.Session) { role1 := v.MustReadRequired(mount + "/static-roles/" + readTestRoleName) v.AssertSecret(role1).Data().HasKey("rotation_period", float64(testRotationPeriod)) - // Update rotation period + // Update rotation period. Static role updates must continue to include the + // existing username to avoid being treated as a username mutation. newRotationPeriod := 7200 v.MustWrite(mount+"/static-roles/"+readTestRoleName, map[string]any{ + "db_name": testConnectionName, + "username": readTestUsername, "rotation_period": newRotationPeriod, }) @@ -114,13 +121,13 @@ func testMongoDBStaticRoleReadAfterUpdate(t *testing.T, v *blackbox.Session) { // testMongoDBStaticRoleListMultipleRoles verifies LIST /static-roles // returns all configured role names. func testMongoDBStaticRoleListMultipleRoles(t *testing.T, v *blackbox.Session) { - mount, connURL := setupMongoDBTest(t, v) + mount, _, dbName, client := setupMongoDBTest(t, v) // Create multiple static roles roles := []string{"read-role-1", "read-role-2", "read-role-3"} for i, roleName := range roles { username := fmt.Sprintf("listuser%d", i+1) - createMongoDBUser(t, connURL, username, testInitialPassword) + createMongoDBUser(t, client, dbName, username, testInitialPassword) v.MustWrite(mount+"/static-roles/"+roleName, map[string]any{ "db_name": testConnectionName, @@ -130,7 +137,7 @@ func testMongoDBStaticRoleListMultipleRoles(t *testing.T, v *blackbox.Session) { } list := v.MustList(mount + "/static-roles") - v.AssertSecret(list).Data().HasKey("keys") + v.AssertSecret(list).Data().HasKeyExists("keys") keys := list.Data["keys"].([]interface{}) if len(keys) != 3 { diff --git a/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_test.go b/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_test.go index 0a7519174c..0ccfa20b67 100644 --- a/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_test.go +++ b/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_test.go @@ -4,30 +4,23 @@ package mongodb import ( - "context" - "fmt" "strings" "testing" "time" - "github.com/hashicorp/vault/api" "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" ) const ( - testConnectionName = "my-mongodb-db" testStaticRoleName = "my-static-role" testUsername = "staticuser1" - testInitialPassword = "initialpass" - testRotationPeriod = 86400 // 24 hours in seconds mongoConnectTimeout = 10 * time.Second ) // TestMongoDBStaticRoleWorkflows runs all MongoDB static role workflow tests. func TestMongoDBStaticRoleWorkflows(t *testing.T) { + t.Parallel() + t.Run("CreateBasic", func(t *testing.T) { t.Parallel() v := blackbox.New(t) @@ -57,9 +50,9 @@ func TestMongoDBStaticRoleWorkflows(t *testing.T) { // testMongoDBStaticRoleCreateBasic verifies that creating a basic static role // succeeds with required fields. func testMongoDBStaticRoleCreateBasic(t *testing.T, v *blackbox.Session) { - mount, connURL := setupMongoDBTest(t, v) + mount, _, dbName, client := setupMongoDBTest(t, v) - createMongoDBUser(t, connURL, testUsername, testInitialPassword) + createMongoDBUser(t, client, dbName, testUsername, testInitialPassword) v.MustWrite(mount+"/static-roles/"+testStaticRoleName, map[string]any{ "db_name": testConnectionName, @@ -85,9 +78,9 @@ func testMongoDBStaticRoleCreateBasic(t *testing.T, v *blackbox.Session) { // testMongoDBStaticRoleManualRotation verifies manually rotating a static // role's credentials succeeds. func testMongoDBStaticRoleManualRotation(t *testing.T, v *blackbox.Session) { - mount, connURL := setupMongoDBTest(t, v) + mount, credVerifyURL, dbName, client := setupMongoDBTest(t, v) - createMongoDBUser(t, connURL, testUsername, testInitialPassword) + createMongoDBUser(t, client, dbName, testUsername, testInitialPassword) v.MustWrite(mount+"/static-roles/"+testStaticRoleName, map[string]any{ "db_name": testConnectionName, @@ -111,7 +104,7 @@ func testMongoDBStaticRoleManualRotation(t *testing.T, v *blackbox.Session) { t.Fatal("expected password to change after rotation") } - verifyMongoDBCredentials(t, connURL, testUsername, password2) + verifyMongoDBCredentials(t, credVerifyURL, testUsername, password2) } // TODO: testMongoDBStaticRoleAutomaticRotation - Test automatic rotation with short period @@ -121,9 +114,9 @@ func testMongoDBStaticRoleManualRotation(t *testing.T, v *blackbox.Session) { // testMongoDBStaticRoleReadCredentials verifies reading static credentials // returns the current password. func testMongoDBStaticRoleReadCredentials(t *testing.T, v *blackbox.Session) { - mount, connURL := setupMongoDBTest(t, v) + mount, credVerifyURL, dbName, client := setupMongoDBTest(t, v) - createMongoDBUser(t, connURL, testUsername, testInitialPassword) + createMongoDBUser(t, client, dbName, testUsername, testInitialPassword) v.MustWrite(mount+"/static-roles/"+testStaticRoleName, map[string]any{ "db_name": testConnectionName, @@ -142,13 +135,13 @@ func testMongoDBStaticRoleReadCredentials(t *testing.T, v *blackbox.Session) { t.Fatal("expected non-empty password") } - verifyMongoDBCredentials(t, connURL, testUsername, password) + verifyMongoDBCredentials(t, credVerifyURL, testUsername, password) } // testMongoDBStaticRoleRequiresUsername verifies creating a static role // without username fails. func testMongoDBStaticRoleRequiresUsername(t *testing.T, v *blackbox.Session) { - mount, _ := setupMongoDBTest(t, v) + mount, _, _, _ := setupMongoDBTest(t, v) _, err := v.Client.Logical().Write(mount+"/static-roles/"+testStaticRoleName, map[string]any{ "db_name": testConnectionName, @@ -163,193 +156,3 @@ func testMongoDBStaticRoleRequiresUsername(t *testing.T, v *blackbox.Session) { t.Fatalf("expected error message to mention 'username', got: %v", err) } } - -// setupMongoDBTest performs common test setup: creates container, enables mount, configures connection. -// Returns mount path and connection URL. -func setupMongoDBTest(t *testing.T, v *blackbox.Session) (string, string) { - t.Helper() - - requireVaultEnv(t) - cleanup, connURL := PrepareTestContainer(t) - t.Cleanup(cleanup) - - mount := fmt.Sprintf("database-%s", sanitize(t.Name())) - v.MustEnableSecretsEngine(mount, &api.MountInput{Type: "database"}) - - v.MustWrite( - mount+"/config/"+testConnectionName, - mongoConnectionConfigPayload(connURL, "*", false), - ) - - return mount, connURL -} - -// createMongoDBUser creates a MongoDB user for testing static roles. -func createMongoDBUser(t *testing.T, connURL, username, password string) { - t.Helper() - - ctx, cancel := context.WithTimeout(context.Background(), mongoConnectTimeout) - defer cancel() - - client, err := mongo.Connect(ctx, options.Client().ApplyURI(connURL)) - if err != nil { - t.Fatalf("failed to connect to MongoDB: %v", err) - } - defer client.Disconnect(ctx) - - db := client.Database("admin") - err = db.RunCommand(ctx, bson.D{ - {Key: "createUser", Value: username}, - {Key: "pwd", Value: password}, - {Key: "roles", Value: bson.A{ - bson.D{ - {Key: "role", Value: "readWrite"}, - {Key: "db", Value: "admin"}, - }, - }}, - }).Err() - if err != nil { - t.Fatalf("failed to create MongoDB user: %v", err) - } - - t.Logf("Created MongoDB user: %s", username) -} - -// verifyMongoDBCredentials verifies that the given credentials work for MongoDB. -func verifyMongoDBCredentials(t *testing.T, connURL, username, password string) { - t.Helper() - - ctx, cancel := context.WithTimeout(context.Background(), mongoConnectTimeout) - defer cancel() - - // Replace credentials in connection URL - u, err := parseMongoURL(connURL) - if err != nil { - t.Fatalf("failed to parse connection URL: %v", err) - } - - u.User = username - u.Password = password - testURL := buildMongoURL(u) - - client, err := mongo.Connect(ctx, options.Client().ApplyURI(testURL)) - if err != nil { - t.Fatalf("failed to connect with credentials: %v", err) - } - defer client.Disconnect(ctx) - - if err := client.Ping(ctx, nil); err != nil { - t.Fatalf("failed to ping with credentials: %v", err) - } - - t.Logf("Verified MongoDB credentials for user: %s", username) -} - -// mongoURL represents a parsed MongoDB connection URL. -type mongoURL struct { - Scheme string - User string - Password string - Host string - Database string - Options string -} - -// parseMongoURL parses a MongoDB connection URL into components. -func parseMongoURL(connURL string) (*mongoURL, error) { - // Simple parser for mongodb:// URLs - // Format: mongodb://user:pass@host/database?options - u := &mongoURL{Scheme: "mongodb"} - - // Remove scheme - rest := connURL - if len(rest) > 10 && rest[:10] == "mongodb://" { - rest = rest[10:] - } - - // Extract user:pass if present - atIdx := -1 - for i, c := range rest { - if c == '@' { - atIdx = i - break - } - } - - if atIdx > 0 { - userPass := rest[:atIdx] - rest = rest[atIdx+1:] - - colonIdx := -1 - for i, c := range userPass { - if c == ':' { - colonIdx = i - break - } - } - - if colonIdx > 0 { - u.User = userPass[:colonIdx] - u.Password = userPass[colonIdx+1:] - } - } - - // Extract host and database - slashIdx := -1 - for i, c := range rest { - if c == '/' { - slashIdx = i - break - } - } - - if slashIdx > 0 { - u.Host = rest[:slashIdx] - rest = rest[slashIdx+1:] - - // Extract database and options - qIdx := -1 - for i, c := range rest { - if c == '?' { - qIdx = i - break - } - } - - if qIdx > 0 { - u.Database = rest[:qIdx] - u.Options = rest[qIdx+1:] - } else { - u.Database = rest - } - } else { - u.Host = rest - } - - return u, nil -} - -// buildMongoURL builds a MongoDB connection URL from components. -func buildMongoURL(u *mongoURL) string { - url := u.Scheme + "://" - - if u.User != "" { - url += u.User - if u.Password != "" { - url += ":" + u.Password - } - url += "@" - } - - url += u.Host - - if u.Database != "" { - url += "/" + u.Database - } - - if u.Options != "" { - url += "?" + u.Options - } - - return url -} diff --git a/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_test_helper.go b/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_test_helper.go index 36d9660592..1aa1272879 100644 --- a/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_test_helper.go +++ b/vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_test_helper.go @@ -15,7 +15,10 @@ import ( "testing" "time" + "github.com/hashicorp/vault/api" "github.com/hashicorp/vault/sdk/helper/docker" + "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" + "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" ) @@ -25,6 +28,10 @@ const ( defaultMongoVersion = "7.0" defaultMongoUser = "admin" defaultMongoPass = "secret" + + testConnectionName = "my-mongodb-db" + testInitialPassword = "initialpass" + testRotationPeriod = 86400 // 24 hours in seconds ) // defaultRunOpts returns default Docker run options for MongoDB container @@ -86,12 +93,14 @@ func sanitize(name string) string { return out } -// PrepareTestContainer starts a MongoDB container for testing -// Returns cleanup function and connection URL -// If MONGO_URL environment variable is set, uses that instead of starting a container -func PrepareTestContainer(t *testing.T) (func(), string) { - _, cleanup, connURL, _ := prepareTestContainer(t, defaultRunOpts(t), defaultMongoPass, false, true) - return cleanup, connURL +// PrepareTestContainer starts a MongoDB container for testing. +// Returns cleanup function, Vault connection URL, test runner connection URL, +// and the generated per-test database name. +// If MONGO_URL environment variable is set, uses that instead of starting a container. +// In CI: vaultURL is private (same VPC), testRunnerURL is public (different VPC) +func PrepareTestContainer(t *testing.T) (func(), string, string, string) { + _, cleanup, vaultURL, testRunnerURL, dbName := prepareTestContainer(t, defaultRunOpts(t), defaultMongoPass, false, true) + return cleanup, vaultURL, testRunnerURL, dbName } // prepareTestContainer is the internal function that handles container setup @@ -102,7 +111,7 @@ func prepareTestContainer( password string, addSuffix bool, forceLocalAddr bool, -) (*docker.Runner, func(), string, string) { +) (*docker.Runner, func(), string, string, string) { requireVaultEnv(t) // Check for external MongoDB URL @@ -115,9 +124,20 @@ func prepareTestContainer( vaultMongoURL = envMongoURL } - // Create unique database for this test - dbName := fmt.Sprintf("test_%s_%d", sanitize(t.Name()), time.Now().Unix()) - testURL := replaceDatabase(vaultMongoURL, dbName) + // Create unique database for this test (max 63 chars for MongoDB) + sanitized := sanitize(t.Name()) + timestamp := time.Now().Unix() + // Format: test_{name}_{ts} - ensure total length <= 63 + // Reserve 5 for "test_", 10 for timestamp, 1 for underscore = 16 chars overhead + maxNameLen := 63 - 16 + if len(sanitized) > maxNameLen { + sanitized = sanitized[:maxNameLen] + } + dbName := fmt.Sprintf("test_%s_%d", sanitized, timestamp) + + // Vault uses private URL (same VPC), test runner uses public URL (different VPC) + vaultURL := replaceDatabase(vaultMongoURL, dbName) + testRunnerURL := replaceDatabase(envMongoURL, dbName) // Create the database (test runner uses public URL) if err := createDatabase(t, envMongoURL, dbName); err != nil { @@ -128,7 +148,7 @@ func prepareTestContainer( dropDatabase(t, envMongoURL, dbName) } - return nil, cleanup, testURL, "" + return nil, cleanup, vaultURL, testRunnerURL, dbName } // Start Docker container @@ -142,13 +162,12 @@ func prepareTestContainer( // Retry StartNewService with small delays to handle port mapping timing var svc *docker.Service - var containerID string for attempt := 0; attempt < 5; attempt++ { if attempt > 0 { time.Sleep(time.Duration(attempt) * 500 * time.Millisecond) } - svc, containerID, err = runner.StartNewService(context.Background(), addSuffix, forceLocalAddr, connectMongoDB(password)) + svc, _, err = runner.StartNewService(context.Background(), addSuffix, forceLocalAddr, connectMongoDB(password)) if err == nil { break } @@ -167,8 +186,16 @@ func prepareTestContainer( connURL := svc.Config.URL().String() - // Create unique database for this test - dbName := fmt.Sprintf("test_%s_%d", sanitize(t.Name()), time.Now().Unix()) + // Create unique database for this test (max 63 chars for MongoDB) + sanitized := sanitize(t.Name()) + timestamp := time.Now().Unix() + // Format: test_{name}_{ts} - ensure total length <= 63 + // Reserve 5 for "test_", 10 for timestamp, 1 for underscore = 16 chars overhead + maxNameLen := 63 - 16 + if len(sanitized) > maxNameLen { + sanitized = sanitized[:maxNameLen] + } + dbName := fmt.Sprintf("test_%s_%d", sanitized, timestamp) testURL := replaceDatabase(connURL, dbName) // Create the database @@ -182,7 +209,8 @@ func prepareTestContainer( svc.Cleanup() } - return runner, cleanup, testURL, containerID + // For Docker, both Vault and test runner use the same URL (localhost) + return runner, cleanup, testURL, testURL, dbName } // connectMongoDB returns a ServiceAdapter that connects to MongoDB @@ -232,21 +260,58 @@ func replaceDatabase(connURL, dbName string) string { return connURL } - u.Path = "/" + dbName + u.Path = dbName + + // Ensure authSource=admin so authentication works against admin database + // even when connecting to a different database + q := u.Query() + if q.Get("authSource") == "" { + q.Set("authSource", "admin") + u.RawQuery = q.Encode() + } + return u.String() } // createDatabase creates a new database in MongoDB // MongoDB creates databases lazily, so we create a collection to ensure it exists +// Uses 60-second timeout with retry logic for CI environments with network latency. func createDatabase(t *testing.T, connURL, dbName string) error { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - client, err := mongo.Connect(ctx, options.Client().ApplyURI(connURL)) + // Retry connection with backoff for CI network latency + var client *mongo.Client + var err error + deadline := time.Now().Add(60 * time.Second) + + for time.Now().Before(deadline) { + // Set shorter server selection timeout (10s) to allow multiple retries within 60s window + clientOpts := options.Client(). + ApplyURI(connURL). + SetServerSelectionTimeout(10 * time.Second). + SetConnectTimeout(10 * time.Second) + + client, err = mongo.Connect(ctx, clientOpts) + if err != nil { + time.Sleep(2 * time.Second) + continue + } + + // Verify connection with ping + if err = client.Ping(ctx, nil); err != nil { + client.Disconnect(ctx) + time.Sleep(2 * time.Second) + continue + } + + break + } + if err != nil { - return fmt.Errorf("failed to connect to MongoDB: %w", err) + return fmt.Errorf("failed to connect to MongoDB after retries: %w", err) } defer client.Disconnect(ctx) @@ -261,15 +326,43 @@ func createDatabase(t *testing.T, connURL, dbName string) error { } // dropDatabase drops a database from MongoDB +// Uses 60-second timeout with retry logic for CI environments with network latency. func dropDatabase(t *testing.T, connURL, dbName string) { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - client, err := mongo.Connect(ctx, options.Client().ApplyURI(connURL)) + // Retry connection with backoff for CI network latency + var client *mongo.Client + var err error + deadline := time.Now().Add(60 * time.Second) + + for time.Now().Before(deadline) { + // Set shorter server selection timeout (10s) to allow multiple retries within 60s window + clientOpts := options.Client(). + ApplyURI(connURL). + SetServerSelectionTimeout(10 * time.Second). + SetConnectTimeout(10 * time.Second) + + client, err = mongo.Connect(ctx, clientOpts) + if err != nil { + time.Sleep(2 * time.Second) + continue + } + + // Verify connection with ping + if err = client.Ping(ctx, nil); err != nil { + client.Disconnect(ctx) + time.Sleep(2 * time.Second) + continue + } + + break + } + if err != nil { - t.Logf("Warning: failed to connect for cleanup: %v", err) + t.Logf("Warning: failed to connect for cleanup after retries: %v", err) return } defer client.Disconnect(ctx) @@ -291,3 +384,137 @@ func mongoConnectionConfigPayload(connURL, allowedRoles string, verifyConnection "verify_connection": verifyConnection, } } + +// setupMongoDBTest performs common test setup: creates container, enables mount, configures connection. +// Returns mount path, connection URL for verifying credentials, the generated +// per-test database name, and a reusable MongoDB client for test operations. +// In CI: Vault uses private URL (same VPC), test runner uses public URL (different VPC) +func setupMongoDBTest(t *testing.T, v *blackbox.Session) (string, string, string, *mongo.Client) { + t.Helper() + + requireVaultEnv(t) + cleanup, vaultURL, testRunnerURL, dbName := PrepareTestContainer(t) + t.Cleanup(cleanup) + + mount := fmt.Sprintf("database-%s", sanitize(t.Name())) + v.MustEnableSecretsEngine(mount, &api.MountInput{Type: "database"}) + + // Vault uses private URL (same VPC as MongoDB in CI) + v.MustWrite( + mount+"/config/"+testConnectionName, + mongoConnectionConfigPayload(vaultURL, "*", false), + ) + + // Test runner uses public URL (different VPC in CI, needs public access) + client := getMongoClient(t, testRunnerURL) + t.Cleanup(func() { + if err := client.Disconnect(context.Background()); err != nil { + t.Logf("Warning: failed to disconnect MongoDB client: %v", err) + } + }) + + // Return testRunnerURL for credential verification (test runner needs public access) + return mount, testRunnerURL, dbName, client +} + +// getMongoClient creates a MongoDB client with optimized settings for CI environments. +// Uses connection pooling and shorter timeouts for faster failure detection. +func getMongoClient(t *testing.T, connURL string) *mongo.Client { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + clientOpts := options.Client(). + ApplyURI(connURL). + SetServerSelectionTimeout(10 * time.Second). + SetConnectTimeout(10 * time.Second). + SetMaxPoolSize(10). + SetMinPoolSize(2) + + client, err := mongo.Connect(ctx, clientOpts) + if err != nil { + t.Fatalf("failed to create MongoDB client: %v", err) + } + + // Verify connection + if err := client.Ping(ctx, nil); err != nil { + client.Disconnect(ctx) + t.Fatalf("failed to ping MongoDB: %v", err) + } + + return client +} + +// createMongoDBUser creates a MongoDB user for testing static roles. +// Uses the provided client connection to avoid connection overhead. +// The user is created in the database identified by dbName, while auth +// continues to use authSource=admin. +func createMongoDBUser(t *testing.T, client *mongo.Client, dbName, username, password string) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + db := client.Database(dbName) + + // First, try to drop the user if it exists (for test cleanup/retry scenarios). + _ = db.RunCommand(ctx, bson.D{{Key: "dropUser", Value: username}}).Err() + + err := db.RunCommand(ctx, bson.D{ + {Key: "createUser", Value: username}, + {Key: "pwd", Value: password}, + {Key: "roles", Value: bson.A{ + bson.D{ + {Key: "role", Value: "readWrite"}, + {Key: "db", Value: dbName}, + }, + }}, + }).Err() + if err != nil { + t.Fatalf("failed to create MongoDB user: %v", err) + } + + t.Logf("Created MongoDB user: %s in database %s", username, dbName) +} + +// verifyMongoDBCredentials verifies that the given credentials work for MongoDB. +// Creates a new client with the provided credentials to test authentication. +func verifyMongoDBCredentials(t *testing.T, connURL, username, password string) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Replace credentials in connection URL + u, err := url.Parse(connURL) + if err != nil { + t.Fatalf("failed to parse connection URL: %v", err) + } + u.User = url.UserPassword(username, password) + + // Remove the auth source so we verify with the default database + values := u.Query() + if values.Get("authSource") != "" { + values.Del("authSource") + u.RawQuery = values.Encode() + } + + // Create client with new credentials + clientOpts := options.Client(). + ApplyURI(u.String()). + SetServerSelectionTimeout(10 * time.Second). + SetConnectTimeout(10 * time.Second) + + client, err := mongo.Connect(ctx, clientOpts) + if err != nil { + t.Fatalf("failed to connect to mongodb: %v: url %s", err, u.String()) + } + defer client.Disconnect(ctx) + + if err := client.Ping(ctx, nil); err != nil { + t.Fatalf("failed to ping mongodb: %v: url %s", err, u.String()) + } + + t.Logf("Verified MongoDB credentials for user: %s", username) +} diff --git a/vault/external_tests/blackbox/plugins/database/postgres/secret_postgresql_connection_config_failover_test.go b/vault/external_tests/blackbox/plugins/database/postgres/secret_postgresql_connection_config_failover_test.go index a1a679c5a2..cdd2e84df6 100644 --- a/vault/external_tests/blackbox/plugins/database/postgres/secret_postgresql_connection_config_failover_test.go +++ b/vault/external_tests/blackbox/plugins/database/postgres/secret_postgresql_connection_config_failover_test.go @@ -74,10 +74,6 @@ func testPostgreSQLConnectionConfigCreateMultiHostPrimaryUnavailable(t *testing. writeAndAssertPostgresConfig(t, v, path, templatedConnectionURL(failoverConnURL)) } -func templatedConnectionURL(connURL string) string { - return strings.Replace(connURL, "postgres:secret@", "{{username}}:{{password}}@", 1) -} - func writeAndAssertPostgresConfig(t *testing.T, v *blackbox.Session, path, connURL string) { t.Helper() diff --git a/vault/external_tests/blackbox/plugins/database/postgres/secret_postgresql_connection_test_helper.go b/vault/external_tests/blackbox/plugins/database/postgres/secret_postgresql_connection_test_helper.go index 1ac3275ec8..c8f7fc4e8a 100644 --- a/vault/external_tests/blackbox/plugins/database/postgres/secret_postgresql_connection_test_helper.go +++ b/vault/external_tests/blackbox/plugins/database/postgres/secret_postgresql_connection_test_helper.go @@ -174,3 +174,7 @@ func connectPostgres(password string, useFallback bool) docker.ServiceAdapter { return docker.NewServiceURL(u), nil } } + +func templatedConnectionURL(connURL string) string { + return strings.Replace(connURL, "postgres:secret@", "{{username}}:{{password}}@", 1) +} diff --git a/vault/external_tests/blackbox/plugins/ldap/helpers.go b/vault/external_tests/blackbox/plugins/ldap/helpers.go index 18c94ba66c..a2ce8f313a 100644 --- a/vault/external_tests/blackbox/plugins/ldap/helpers.go +++ b/vault/external_tests/blackbox/plugins/ldap/helpers.go @@ -36,20 +36,19 @@ func setupLDAPSecretsEngine(t *testing.T, v *blackbox.Session, mount string) { v.MustEnableSecretsEngine(mount, &api.MountInput{Type: "ldap"}) // Configure using environment variables set by ldap.tf - ldapServer := os.Getenv("LDAP_SERVER") - ldapPort := os.Getenv("LDAP_PORT") + ldapURLPrivate := os.Getenv("LDAP_URL_PRIVATE") ldapBindDN := os.Getenv("LDAP_BIND_DN") ldapBindPass := os.Getenv("LDAP_BIND_PASS") ldapUsername := os.Getenv("LDAP_USERNAME") // "enos" - if ldapServer == "" || ldapPort == "" || ldapBindDN == "" || ldapBindPass == "" { + if ldapURLPrivate == "" || ldapBindDN == "" || ldapBindPass == "" || ldapUsername == "" { t.Fatal("Required LDAP environment variables not set") } v.MustWrite(mount+"/config", map[string]any{ "binddn": ldapBindDN, "bindpass": ldapBindPass, - "url": "ldap://" + ldapServer + ":" + ldapPort, + "url": ldapURLPrivate, "userdn": "ou=users,dc=" + ldapUsername + ",dc=com", }) } @@ -110,14 +109,14 @@ func WriteLibrarySetWithRetry(t *testing.T, v *blackbox.Session, path string, da // checkLDAPUserExists verifies if a user exists in the LDAP directory func checkLDAPUserExists(t *testing.T, username string) bool { - ldapServer := os.Getenv("LDAP_SERVER") - ldapPort := os.Getenv("LDAP_PORT") + ldapURLPrivate := os.Getenv("LDAP_URL_PRIVATE") ldapUsername := os.Getenv("LDAP_USERNAME") ldapAdminPw := os.Getenv("LDAP_ADMIN_PW") - cmd := exec.Command("ldapsearch", + cmd := exec.Command( + "ldapsearch", "-x", - "-H", "ldap://"+ldapServer+":"+ldapPort, + "-H", ldapURLPrivate, "-b", "ou=users,dc="+ldapUsername+",dc=com", "-D", "cn=admin,dc="+ldapUsername+",dc=com", "-w", ldapAdminPw, @@ -142,7 +141,7 @@ func getLDIFPath(filename string) string { // skipIfLDAPNotAvailable skips the test if LDAP configuration is not available func skipIfLDAPNotAvailable(t *testing.T) { - if os.Getenv("LDAP_SERVER") == "" { + if os.Getenv("LDAP_URL_PRIVATE") == "" { t.Skip("LDAP configuration not available - skipping LDAP test") } } @@ -171,7 +170,8 @@ func waitForLDAP(t *testing.T, config *LDAPDomainConfig, timeout time.Duration) for time.Now().Before(deadline) { attempt++ - cmd := exec.Command("ldapsearch", + cmd := exec.Command( + "ldapsearch", "-x", "-H", config.SetupURL, // Use public IP for connectivity checks from GitHub runner "-D", config.BindDN, @@ -217,18 +217,17 @@ func PrepareTestLDAPDomain( t.Helper() // Get LDAP connection info from environment (set by Enos) - // LDAP_SERVER: private IP for Vault operations (runs on Vault leader) - // LDAP_SERVER_PUBLIC: public IP for setup operations (runs from GitHub runner) - ldapServer := os.Getenv("LDAP_SERVER") - ldapServerPublic := os.Getenv("LDAP_SERVER_PUBLIC") - ldapPort := os.Getenv("LDAP_PORT") + // LDAP_URL_PRIVATE: ldap url to ldap internal listerning Vault operations (runs on Vault leader) + // LDAP_SERVER_PUBLIC: public URL ldap external listener for setup operations (runs from GitHub runner) + ldapURLPrivate := os.Getenv("LDAP_URL_PRIVATE") + ldapURLPublic := os.Getenv("LDAP_URL_PUBLIC") ldapBindDN := os.Getenv("LDAP_BIND_DN") ldapBindPass := os.Getenv("LDAP_BIND_PASS") - t.Logf("LDAP Environment: SERVER=%s SERVER_PUBLIC=%s PORT=%s BIND_DN=%s BIND_PASS=%s", - ldapServer, ldapServerPublic, ldapPort, ldapBindDN, maskPassword(ldapBindPass)) + t.Logf("LDAP Environment: LDAP_URL_PRIVATE=%s LDAP_URL_PUBLIC=%s LDAP_BIND_DN=%s LDAP_BIND_PASS=%s", + ldapURLPrivate, ldapURLPublic, ldapBindDN, maskPassword(ldapBindPass)) - if ldapServer == "" || ldapServerPublic == "" || ldapPort == "" || ldapBindDN == "" || ldapBindPass == "" { + if ldapURLPrivate == "" || ldapURLPublic == "" || ldapBindDN == "" || ldapBindPass == "" { err := fmt.Errorf("LDAP environment variables not set") if requireIsolation { return nil, nil, fmt.Errorf("%w - required in CI", err) @@ -242,7 +241,7 @@ func PrepareTestLDAPDomain( baseDN := "dc=enos,dc=com" // Use existing base domain config = &LDAPDomainConfig{ - URL: fmt.Sprintf("ldap://%s:%s", ldapServer, ldapPort), // Private IP for Vault + URL: ldapURLPrivate, BindDN: ldapBindDN, BindPass: ldapBindPass, BaseDN: baseDN, @@ -251,7 +250,7 @@ func PrepareTestLDAPDomain( } // Store public IP for setup operations (ldapadd commands from GitHub runner) - config.SetupURL = fmt.Sprintf("ldap://%s:%s", ldapServerPublic, ldapPort) + config.SetupURL = ldapURLPublic // Create domain structure if err := createDomain(t, session, config); err != nil { @@ -307,7 +306,8 @@ ou: %s // Use Eventually to retry ldapadd until LDAP server is ready session.Eventually(func() error { - cmd := exec.Command("ldapadd", + cmd := exec.Command( + "ldapadd", "-x", "-H", config.SetupURL, // Use public IP for setup operations from GitHub runner "-D", config.BindDN, @@ -335,7 +335,8 @@ func deleteDomain(t *testing.T, config *LDAPDomainConfig) { // Delete both user and group OUs for _, dn := range []string{config.UserDN, config.GroupDN} { - cmd := exec.Command("ldapdelete", + cmd := exec.Command( + "ldapdelete", "-x", "-r", // recursive "-H", config.SetupURL, // Use public IP for cleanup operations from GitHub runner @@ -383,7 +384,8 @@ userPassword: %s t.Logf(" ldapsearch -x -H %s -b \"dc=enos,dc=com\" -D \"%s\" -w \"%s\" -s base", config.SetupURL, userDN, password) - cmd := exec.Command("ldapadd", + cmd := exec.Command( + "ldapadd", "-x", "-H", config.SetupURL, // Use public IP for setup operations from GitHub runner "-D", config.BindDN, @@ -423,7 +425,8 @@ cn: %s %s `, groupDN, groupName, strings.Join(memberDNs, "\n")) - cmd := exec.Command("ldapadd", + cmd := exec.Command( + "ldapadd", "-x", "-H", config.URL, "-D", config.BindDN, @@ -456,7 +459,8 @@ add: member member: %s `, groupDN, userDN) - cmd := exec.Command("ldapmodify", + cmd := exec.Command( + "ldapmodify", "-x", "-H", config.URL, "-D", config.BindDN, @@ -489,7 +493,8 @@ delete: member member: %s `, groupDN, userDN) - cmd := exec.Command("ldapmodify", + cmd := exec.Command( + "ldapmodify", "-x", "-H", config.URL, "-D", config.BindDN, @@ -512,7 +517,8 @@ member: %s func CheckLDAPUserExistsInDomain(t *testing.T, config *LDAPDomainConfig, username string) bool { t.Helper() - cmd := exec.Command("ldapsearch", + cmd := exec.Command( + "ldapsearch", "-x", "-H", config.URL, "-b", config.UserDN, diff --git a/vault/external_tests/blackbox/plugins/ldap/secrets_ldap_root_credential_rollback_test.go b/vault/external_tests/blackbox/plugins/ldap/secrets_ldap_root_credential_rollback_test.go index 8bd3a36930..0d7acc82e0 100644 --- a/vault/external_tests/blackbox/plugins/ldap/secrets_ldap_root_credential_rollback_test.go +++ b/vault/external_tests/blackbox/plugins/ldap/secrets_ldap_root_credential_rollback_test.go @@ -13,6 +13,28 @@ import ( "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" ) +// TestLDAPRootCredentialRollbackWorkflows runs all rollback workflow tests +// This is the main test function that gets triggered by enos-scenario-plugin.hcl +func TestLDAPRootCredentialRollbackWorkflows(t *testing.T) { + t.Run("RollbackSuccess", func(t *testing.T) { + t.Parallel() + v := blackbox.New(t) + testLDAPRootCredentialRollbackSuccess(t, v) + }) + + t.Run("RollbackFailure", func(t *testing.T) { + t.Parallel() + v := blackbox.New(t) + testLDAPRootCredentialRollbackFailure(t, v) + }) + + t.Run("AutomaticRollback", func(t *testing.T) { + t.Parallel() + v := blackbox.New(t) + testLDAPRootCredentialAutomaticRollbackOnFailure(t, v) + }) +} + // testLDAPRootCredentialRollbackSuccess tests successful rollback when rotation fails // Converts: secrets-rollback-invalid-config.sh // Scenario: Rotation fails with invalid LDAP endpoint, old password is preserved @@ -21,7 +43,7 @@ func testLDAPRootCredentialRollbackSuccess(t *testing.T, v *blackbox.Session) { cleanup, ldapConfig, err := PrepareTestLDAPDomain(t, v, isCI()) if err != nil { if isCI() { - t.Fatalf("Failed to create LDAP domain in CI: %v", err) + t.Fatalf("LDAP domain creation failed in CI: %v", err) } t.Skipf("LDAP domain creation not available: %v", err) } @@ -102,7 +124,7 @@ func testLDAPRootCredentialRollbackFailure(t *testing.T, v *blackbox.Session) { cleanup, ldapConfig, err := PrepareTestLDAPDomain(t, v, isCI()) if err != nil { if isCI() { - t.Fatalf("Failed to create LDAP domain in CI: %v", err) + t.Fatalf("LDAP domain creation failed in CI: %v", err) } t.Skipf("LDAP domain creation not available: %v", err) } @@ -180,7 +202,7 @@ func testLDAPRootCredentialAutomaticRollbackOnFailure(t *testing.T, v *blackbox. cleanup, ldapConfig, err := PrepareTestLDAPDomain(t, v, isCI()) if err != nil { if isCI() { - t.Fatalf("Failed to create LDAP domain in CI: %v", err) + t.Fatalf("LDAP domain creation failed in CI: %v", err) } t.Skipf("LDAP domain creation not available: %v", err) } @@ -276,7 +298,8 @@ func verifyLDAPAuth(t *testing.T, config *LDAPDomainConfig, bindDN, password str // Use SetupURL (public IP) for authentication checks from GitHub runner // config.URL contains private IP which is only accessible from Vault cluster - cmd := exec.Command("ldapwhoami", + cmd := exec.Command( + "ldapwhoami", "-x", "-H", config.SetupURL, "-D", bindDN, @@ -291,25 +314,3 @@ func verifyLDAPAuth(t *testing.T, config *LDAPDomainConfig, bindDN, password str return true } - -// TestLDAPRootCredentialRollbackWorkflows runs all rollback workflow tests -// This is the main test function that gets triggered by enos-scenario-plugin.hcl -func TestLDAPRootCredentialRollbackWorkflows(t *testing.T) { - t.Run("RollbackSuccess", func(t *testing.T) { - t.Parallel() - v := blackbox.New(t) - testLDAPRootCredentialRollbackSuccess(t, v) - }) - - t.Run("RollbackFailure", func(t *testing.T) { - t.Parallel() - v := blackbox.New(t) - testLDAPRootCredentialRollbackFailure(t, v) - }) - - t.Run("AutomaticRollback", func(t *testing.T) { - t.Parallel() - v := blackbox.New(t) - testLDAPRootCredentialAutomaticRollbackOnFailure(t, v) - }) -} diff --git a/vault/external_tests/blackbox/plugins/ldap/secrets_ldap_test.go b/vault/external_tests/blackbox/plugins/ldap/secrets_ldap_test.go index 83622b993f..f571d2b6f7 100644 --- a/vault/external_tests/blackbox/plugins/ldap/secrets_ldap_test.go +++ b/vault/external_tests/blackbox/plugins/ldap/secrets_ldap_test.go @@ -15,7 +15,7 @@ func testLDAPSecretsCreate(t *testing.T, v *blackbox.Session) { cleanup, ldapConfig, err := PrepareTestLDAPDomain(t, v, isCI()) if err != nil { if isCI() { - t.Fatalf("Failed to create LDAP domain in CI: %v", err) + t.Fatalf("LDAP domain creation failed in CI: %v", err) } t.Skipf("LDAP domain creation not available: %v", err) } @@ -51,7 +51,7 @@ func testLDAPSecretsRead(t *testing.T, v *blackbox.Session) { cleanup, ldapConfig, err := PrepareTestLDAPDomain(t, v, isCI()) if err != nil { if isCI() { - t.Fatalf("Failed to create LDAP domain in CI: %v", err) + t.Fatalf("LDAP domain creation failed in CI: %v", err) } t.Skipf("LDAP domain creation not available: %v", err) } @@ -105,7 +105,7 @@ func testLDAPSecretsDelete(t *testing.T, v *blackbox.Session) { cleanup, ldapConfig, err := PrepareTestLDAPDomain(t, v, isCI()) if err != nil { if isCI() { - t.Fatalf("Failed to create LDAP domain in CI: %v", err) + t.Fatalf("LDAP domain creation failed in CI: %v", err) } t.Skipf("LDAP domain creation not available: %v", err) } diff --git a/vault/external_tests/blackbox/raft/voters_test.go b/vault/external_tests/blackbox/raft/voters_test.go index 21280fe3b7..91e07a97e8 100644 --- a/vault/external_tests/blackbox/raft/voters_test.go +++ b/vault/external_tests/blackbox/raft/voters_test.go @@ -5,6 +5,7 @@ package raft import ( "testing" + "time" "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" ) @@ -13,8 +14,16 @@ import ( func TestRaftVoters(t *testing.T) { v := blackbox.New(t) - // Verify we have a healthy cluster regardless of node count - v.AssertClusterHealthy() + // Wait for a healthy cluster + v.EventuallyClusterHealthyUnsealed(15 * time.Second) + + storage := v.MustGetConfigStorageType() + if storage != "raft" { + t.Log("skipping as cluster is not using integrated storage") + return + } + + v.EventuallyRaftClusterHealthy(5 * time.Second) t.Log("Successfully verified raft cluster is healthy with at least one voter") } diff --git a/vault/external_tests/blackbox/secrets/secrets_pki_test.go b/vault/external_tests/blackbox/secrets/secrets_pki_test.go index eb5418242e..0e0ff4d21e 100644 --- a/vault/external_tests/blackbox/secrets/secrets_pki_test.go +++ b/vault/external_tests/blackbox/secrets/secrets_pki_test.go @@ -16,11 +16,11 @@ func testPKISecretsCreate(t *testing.T, v *blackbox.Session) { v.MustEnableSecretsEngine("pki-create", &api.MountInput{Type: "pki"}) // Configure max TTL for the mount - err := v.Client.Sys().TuneMount("pki-create", api.MountConfigInput{ - MaxLeaseTTL: "87600h", + err := v.Client.Sys().TuneMountAllowNilWithContext(t.Context(), "pki-create", api.TuneMountConfigInput{ + MaxLeaseTTL: new(string("87600h")), }) if err != nil { - t.Fatalf("Failed to tune PKI mount: %v", err) + t.Fatalf("Failed to tune PKI mount: %v: %v", err, v.Client.Address()) } // Generate root CA diff --git a/vault/external_tests/blackbox/system/billing_test.go b/vault/external_tests/blackbox/system/billing_test.go index 17f6e15805..47cfb701e3 100644 --- a/vault/external_tests/blackbox/system/billing_test.go +++ b/vault/external_tests/blackbox/system/billing_test.go @@ -20,33 +20,30 @@ import ( func TestBillingOverviewNamespaceRestrictions(t *testing.T) { v := blackbox.New(t) - // Verify cluster stability first - v.AssertClusterHealthy() - // Check if we're in HVD (has base namespace from VAULT_NAMESPACE) baseNS := v.GetParentNamespace() if baseNS == "" { t.Skip("Skipping namespace restriction tests - no base namespace configured (not in HVD)") } - testCases := []struct { - name string + // Verify cluster stability first + v.AssertClusterHealthy() + + testCases := map[string]struct { namespaceSwitcher func(func() (*api.Secret, error)) (*api.Secret, error) expectedError string }{ - { - name: "base_namespace_supported", + "base_namespace_supported": { namespaceSwitcher: v.WithParentNamespace, }, - { - name: "root_namespace_permission_denied", + "root_namespace_supported": { namespaceSwitcher: v.WithRootNamespace, expectedError: "permission denied", }, } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { var rawResp *api.Response var err error _, err = tc.namespaceSwitcher(func() (*api.Secret, error) { diff --git a/vault/external_tests/blackbox/verify/version_verification_test.go b/vault/external_tests/blackbox/verify/version_verification_test.go index 61e0ee0dd6..e5557b66fe 100644 --- a/vault/external_tests/blackbox/verify/version_verification_test.go +++ b/vault/external_tests/blackbox/verify/version_verification_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" + "github.com/stretchr/testify/require" ) // TestVaultServerVersion verifies the Vault server version via sys/version-history API @@ -26,6 +27,14 @@ func TestVaultServerVersion(t *testing.T) { } v := blackbox.New(t) - v.AssertVersion(version) - v.AssertBuildDate(version, buildDate) + replicationMode, err := v.GetDRReplicationMode() + require.NoError(t, err) + t.Logf("Found DR Replication mode: %s", replicationMode) + switch replicationMode { + case "primary", "secondary": + t.Skip("Skipping server version check on DR cluster as path is not available in that mode") + default: + v.AssertVersion(version) + v.AssertBuildDate(version, buildDate) + } }