upstream/vault
1
0
Fork 0
mirror of https://github.com/hashicorp/vault.git synced 2026-05-28 04:10:44 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

[VAULT-44431] enos: merge changes for enterprise zap scenario into ce/main (#14849)

Browse source 
Backport community files that changed as part the enterprise only zap scenarios. This mostly includes fixes to scenario execution,  retries, and blackbox SDK tests that were broken.

Signed-off-by: Ryan Cragun <me@ryan.ec>
Co-authored-by: Ryan Cragun <me@ryan.ec>
This commit is contained in:
Vault Automation 2026-05-18 10:54:15 -06:00 committed by GitHub
parent a7c8fece0e
commit 3dae110c82
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
60 changed files with 2190 additions and 1173 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.copywrite.hcl
View file

@ -2,8 +2,9 @@ schema_version = 1
project {
license = "BUSL-1.1"
copyright_year = 2026
copyright_year = 2016
copyright_holder = "IBM Corp."
ignore_year1 = true
# (OPTIONAL) A list of globs that should not have copyright/license headers.
# Supports doublestar glob patterns for more flexibility in defining which
@ -14,6 +15,7 @@ project {
"enos/.terraform/**",
"enos/k8s/.enos/**",
"enos/modules/k8s_deploy_vault/raft-config.hcl",
"enos/modules/zap_scan_ent/templates/plan.yml",
"helper/pkcs7/**",
"plugins/database/postgresql/scram/**",
"tools/pipeline/internal/pkg/generate/fixtures/*",

57
.github/actions/build-ui/action.yml vendored Normal file
View file

@ -0,0 +1,57 @@
# Copyright IBM Corp. 2016, 2026
# SPDX-License-Identifier: BUSL-1.1
---
name: Build UI
description: |
Build the Vault Web UI assets and cache the results.
inputs:
github-token:
description: An elevated Github token to access private dependencies if necessary.
default: ""
outputs:
cache-key:
description: "The cache key for the built UI assets (format: ui-<git-hash>)"
value: ${{ steps.ui-hash.outputs.cache-key }}
cache-hit:
description: "Whether the UI was restored from cache (true) or built fresh (false)"
value: ${{ steps.cache-ui-assets.outputs.cache-hit }}
runs:
using: composite
steps:
- name: Get UI hash
id: ui-hash
shell: bash
run: |
ui_hash=$(git ls-tree HEAD ui --object-only)
echo "cache-key=ui-${ui_hash}" | tee -a "$GITHUB_OUTPUT"
- name: Set up UI asset cache
id: cache-ui-assets
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
enableCrossOsArchive: true
lookup-only: true
path: http/web_ui
# Only restore the UI asset cache if we haven't modified anything in the ui directory.
# Never do a partial restore of the web_ui if we don't get a cache hit.
key: ${{ steps.ui-hash.outputs.cache-key }}
- name: Install PNPM
if: ${{ steps.cache-ui-assets.outputs.cache-hit != 'true' }}
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
with:
run_install: false
package_json_file: './ui/package.json'
- name: Set up node and pnpm
if: ${{ steps.cache-ui-assets.outputs.cache-hit != 'true' }}
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version-file: ui/package.json
cache: pnpm
cache-dependency-path: ui/pnpm-lock.yaml
- name: Build UI
if: ${{ steps.cache-ui-assets.outputs.cache-hit != 'true' }}
shell: bash
run: make ci-build-ui

2
.github/actions/build-vault/action.yml vendored
View file

@ -117,7 +117,7 @@ runs:
id: build-vault-select-license
uses: hashicorp-forge/actions-pao-tool/select-license@6997f7457c338e008506005cc370e7b02f7fb421 # v1.0.3
with:
arch: ${{ matrix.goarch }}
arch: ${{ inputs.goarch }}
- if: inputs.cgo-enabled == '0'
name: ${{ steps.metadata.outputs.build-step-name }}
env:

114
.github/actions/run-enos-scenario/README.md vendored Normal file
View file

@ -0,0 +1,114 @@
# run-enos-scenario
Reusable composite action for running an Enos scenario with standardized retry, debug artifact upload, and destroy cleanup behavior.
## Behavior
The action performs this lifecycle:
1. Optionally installs Terraform.
2. Optionally installs Enos.
3. Prepares a sanitized debug artifact name from the scenario filter.
4. Runs `enos scenario launch`.
5. Retries the launch command once if the first attempt fails.
6. Optionally prints `enos scenario exec --cmd show` and `plan` before retry and after a failed retry.
7. Uploads debug data on failure.
8. Always destroys the scenario and retries destroy once if needed.
## Inputs
| Name | Required | Default | Description |
| --- | --- | --- | --- |
| `scenario-filter` | yes | n/a | Scenario filter passed to Enos. |
| `working-directory` | no | `./enos` | Value passed to `--chdir`. |
| `timeout` | no | `45m0s` | Timeout for the first main command attempt. |
| `retry-timeout` | no | `30m0s` | Timeout for the retry attempt. |
| `destroy-timeout` | no | `10m0s` | Timeout for destroy. |
| `show-state-on-retry` | no | `false` | When `true`, prints scenario state and plan before retry and after failed retry. |
| `upload-debug-data` | no | `true` | When `true`, uploads debug data on failure. |
| `debug-data-retention-days` | no | `30` | Artifact retention days for debug data. |
| `debug-data-path` | no | empty | Explicit debug data directory. If empty, uses `ENOS_DEBUG_DATA_ROOT_DIR` or `/tmp/enos-debug-data`. |
| `setup-terraform` | no | `true` | When `true`, installs Terraform. |
| `setup-enos` | no | `true` | When `true`, installs Enos. |
| `launch-extra-args` | no | empty | Extra arguments added to the launch command before the scenario filter. |
| `retry-extra-args` | no | empty | Extra arguments added to the retry launch command before the scenario filter. |
| `destroy-extra-args` | no | empty | Extra arguments added to destroy before the scenario filter. |
| `pre-destroy-command` | no | empty | Optional enos scenario command to run before destroying (e.g., `exec`). |
| `pre-destroy-extra-args` | no | empty | Extra arguments added to the pre-destroy command before the scenario filter. |
## Outputs
| Name | Description |
| --- | --- |
| `launch-outcome` | Final main phase outcome: `success` or `failure`. |
| `launch-retry-attempted` | `true` when the main phase retry was attempted. |
| `destroy-outcome` | Final destroy outcome: `success` or `failure`. |
| `destroy-retry-attempted` | `true` when destroy retry was attempted. |
| `debug-data-artifact-name` | Sanitized debug artifact name derived from the scenario filter. |
| `debug-data-dir` | Debug data directory used by the action. |
| `error-message` | Failure message captured from the retry step, when present. |
## Usage
### Standard launch workflow
```yaml
- name: Run Enos scenario
id: run-scenario
uses: ./.github/actions/run-enos-scenario
with:
scenario-filter: ${{ matrix.scenario.id.filter }}
timeout: 45m0s
retry-timeout: 30m0s
destroy-timeout: 10m0s
show-state-on-retry: 'true'
```
### With custom destroy arguments
```yaml
- name: Run Enos scenario
id: run-scenario
uses: ./.github/actions/run-enos-scenario
with:
scenario-filter: ${{ inputs.scenario }}
timeout: 60m0s
retry-timeout: 60m0s
destroy-timeout: 60m0s
destroy-extra-args: --grpc-listen http://localhost
```
### Skip tool installation
```yaml
- name: Run Enos scenario
uses: ./.github/actions/run-enos-scenario
with:
scenario-filter: ${{ steps.sample.outputs.filter }}
setup-terraform: 'false'
setup-enos: 'false'
```
### With pre-destroy command
```yaml
- name: Run Enos scenario
uses: ./.github/actions/run-enos-scenario
with:
scenario-filter: ${{ inputs.scenario }}
timeout: 30m0s
pre-destroy-command: exec
pre-destroy-extra-args: --cmd 'output -raw scan_markdown'
```
### With pre-destroy command and output redirection
```yaml
- name: Run Enos scenario
uses: ./.github/actions/run-enos-scenario
with:
scenario-filter: ${{ inputs.scenario }}
timeout: 30m0s
pre-destroy-command: exec
pre-destroy-extra-args: --cmd 'output -raw scan_markdown' | tee -a "$GITHUB_STEP_SUMMARY"
```

266
.github/actions/run-enos-scenario/action.yml vendored Normal file
View file

@ -0,0 +1,266 @@
# Copyright IBM Corp. 2016, 2026
# SPDX-License-Identifier: BUSL-1.1
---
name: Run and retry an Enos scenario
description: |
Run an Enos scenario with a standard launch retry flow, optional state/plan diagnostics,
automatic debug data artifact naming, and destroy cleanup with a destroy retry.
inputs:
scenario-filter:
description: Enos scenario filter/identifier (for example "ui edition:ent backend:raft")
required: true
working-directory:
description: Working directory to pass to enos via --chdir
required: false
default: ./enos
timeout:
description: Timeout for the initial launch command
required: false
default: 45m0s
retry-timeout:
description: Timeout for the retry launch attempt
required: false
default: 30m0s
destroy-timeout:
description: Timeout for the destroy command
required: false
default: 10m0s
show-state-on-retry:
description: Whether to show scenario state and plan before and after a failed retry (true/false)
required: false
default: ${{ github.repository == 'hashicorp/vault-enterprise' && 'true' || 'false' }}
upload-debug-data:
description: Whether to upload Enos debug data on failure (true/false)
required: false
default: "true"
debug-data-retention-days:
description: Retention period in days for uploaded debug data artifacts
required: false
default: "30"
debug-data-path:
description: Debug data root directory to create and upload
required: false
default: ""
setup-terraform:
description: Whether to install Terraform in the action (true/false)
required: false
default: "true"
setup-enos:
description: Whether to install Enos in the action (true/false)
required: false
default: "true"
launch-extra-args:
description: Extra arguments to append to the launch command before the scenario filter
required: false
default: ""
retry-extra-args:
description: Extra arguments to append to the retry launch command before the scenario filter
required: false
default: ""
destroy-extra-args:
description: Extra arguments to append to the destroy command before the scenario filter
required: false
default: ""
pre-destroy-command:
description: Optional enos scenario command to run before destroying (e.g., "exec")
required: false
default: ""
pre-destroy-extra-args:
description: Extra arguments to append to the pre-destroy command before the scenario filter
required: false
default: ""
outputs:
launch-outcome:
description: Final execution outcome for the main phase (success or failure)
value: ${{ steps.determine-launch-outcome.outputs.outcome }}
launch-retry-attempted:
description: Whether a retry was attempted for the main phase
value: ${{ steps.determine-launch-outcome.outputs.retry-attempted }}
destroy-outcome:
description: Final destroy outcome (success or failure)
value: ${{ steps.determine-destroy-outcome.outputs.outcome }}
destroy-retry-attempted:
description: Whether a destroy retry was attempted
value: ${{ steps.determine-destroy-outcome.outputs.retry-attempted }}
debug-data-artifact-name:
description: Debug data artifact name prepared by the action
value: ${{ steps.prepare-artifacts.outputs.debug-data-artifact-name }}
debug-data-dir:
description: Debug data directory prepared by the action
value: ${{ steps.prepare-artifacts.outputs.debug-data-dir }}
error-message:
description: Error message captured from a failed retry
value: ${{ steps.launch-retry.outputs.error-message }}
runs:
using: composite
steps:
- if: inputs.setup-terraform == 'true'
name: Setup Terraform
uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0
with:
terraform_wrapper: false
- if: inputs.setup-enos == 'true'
name: Setup Enos
uses: hashicorp/action-setup-enos@6ec106c8f809fe645162d73bea565c65f3269907 # v1.52
- id: prepare-artifacts
name: Prepare artifacts and debug directory
shell: bash
run: |
artifact_name="enos-debug-data_$(echo "${{ inputs.scenario-filter }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g')"
debug_dir="${{ inputs.debug-data-path }}"
if [ -z "${debug_dir}" ]; then
debug_dir="${ENOS_DEBUG_DATA_ROOT_DIR:-/tmp/enos-debug-data}"
fi
mkdir -p "${debug_dir}"
{
echo "debug-data-artifact-name=${artifact_name}"
echo "debug-data-dir=${debug_dir}"
} | tee -a "$GITHUB_OUTPUT"
- id: launch
name: Run Enos scenario
continue-on-error: true
shell: bash
run: |
enos scenario launch \
--timeout ${{ inputs.timeout }} \
${{ inputs.launch-extra-args }} \
--chdir ${{ inputs.working-directory }} \
${{ inputs.scenario-filter }}
- if: steps.launch.outcome == 'failure' && inputs.show-state-on-retry == 'true'
name: Show scenario state before retry
shell: bash
run: |
echo "::group::Scenario state before retry"
enos scenario exec --cmd show \
--chdir ${{ inputs.working-directory }} \
${{ inputs.scenario-filter }} || true
echo "::endgroup::"
echo "::group::Scenario plan before retry"
enos scenario exec --cmd plan \
--chdir ${{ inputs.working-directory }} \
${{ inputs.scenario-filter }} || true
echo "::endgroup::"
- if: steps.launch.outcome == 'failure'
id: launch-retry
name: Retry Enos scenario
shell: bash
run: |
if ! output=$(enos scenario launch \
--timeout ${{ inputs.retry-timeout }} \
${{ inputs.retry-extra-args }} \
--chdir ${{ inputs.working-directory }} \
${{ inputs.scenario-filter }} 2>&1); then
if [ "${{ inputs.show-state-on-retry }}" = "true" ]; then
echo "::group::Scenario state after retry"
enos scenario exec --cmd show \
--chdir ${{ inputs.working-directory }} \
${{ inputs.scenario-filter }} || true
echo "::endgroup::"
echo "::group::Scenario plan after retry"
enos scenario exec --cmd plan \
--chdir ${{ inputs.working-directory }} \
${{ inputs.scenario-filter }} || true
echo "::endgroup::"
fi
echo "::group::Scenario retry failure"
echo "$output" >&2
{
echo "# Enos Scenario Failed!"
echo 'See the [workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for more information'
echo '```'
echo "$output"
echo '```'
} | tee -a "$GITHUB_STEP_SUMMARY"
echo "::endgroup::"
exit 1
fi
- id: determine-launch-outcome
name: Determine launch outcome
shell: bash
run: |
retry_attempted=false
if [ "${{ steps.launch.outcome }}" = "failure" ]; then
retry_attempted=true
fi
if [ "${{ steps.launch.outcome }}" = "success" ] || [ "${{ steps.launch-retry.outcome }}" = "success" ]; then
outcome=success
else
outcome=failure
fi
{
echo "outcome=${outcome}"
echo "retry-attempted=${retry_attempted}"
} | tee -a "$GITHUB_OUTPUT"
- if: failure() && inputs.upload-debug-data == 'true'
name: Upload debug data
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: ${{ steps.prepare-artifacts.outputs.debug-data-artifact-name }}
path: ${{ steps.prepare-artifacts.outputs.debug-data-dir }}
retention-days: ${{ inputs.debug-data-retention-days }}
- if: always() && inputs.pre-destroy-command != ''
id: pre-destroy
name: Run pre-destroy command
continue-on-error: true
shell: bash
run: |
enos scenario ${{ inputs.pre-destroy-command }} \
--chdir ${{ inputs.working-directory }} \
${{ inputs.scenario-filter }} \
${{ inputs.pre-destroy-extra-args }}
- if: always()
id: destroy
name: Destroy Enos scenario
continue-on-error: true
shell: bash
run: |
enos scenario destroy \
--timeout ${{ inputs.destroy-timeout }} \
${{ inputs.destroy-extra-args }} \
--chdir ${{ inputs.working-directory }} \
${{ inputs.scenario-filter }}
- if: steps.destroy.outcome == 'failure'
id: destroy-retry
name: Retry Enos scenario destroy
continue-on-error: true
shell: bash
run: |
enos scenario destroy \
--timeout ${{ inputs.destroy-timeout }} \
${{ inputs.destroy-extra-args }} \
--chdir ${{ inputs.working-directory }} \
${{ inputs.scenario-filter }}
- if: always()
id: determine-destroy-outcome
name: Determine destroy outcome
shell: bash
run: |
retry_attempted=false
if [ "${{ steps.destroy.outcome }}" = "failure" ]; then
retry_attempted=true
fi
if [ "${{ steps.destroy.outcome }}" = "success" ] || [ "${{ steps.destroy-retry.outcome }}" = "success" ]; then
outcome=success
else
outcome=failure
fi
{
echo "outcome=${outcome}"
echo "retry-attempted=${retry_attempted}"
} | tee -a "$GITHUB_OUTPUT"

33
.github/workflows/build.yml vendored
View file

@ -293,40 +293,13 @@ jobs:
needs: setup
runs-on: ${{ fromJSON(needs.setup.outputs.compute-build-ui) }}
outputs:
cache-key: ui-${{ steps.ui-hash.outputs.ui-hash }}
cache-key: ${{ steps.build-ui.outputs.cache-key }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ needs.setup.outputs.checkout-ref }}
- name: Get UI hash
id: ui-hash
run: echo "ui-hash=$(git ls-tree HEAD ui --object-only)" | tee -a "$GITHUB_OUTPUT"
- name: Set up UI asset cache
id: cache-ui-assets
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
with:
enableCrossOsArchive: true
lookup-only: true
path: http/web_ui
# Only restore the UI asset cache if we haven't modified anything in the ui directory.
# Never do a partial restore of the web_ui if we don't get a cache hit.
key: ui-${{ steps.ui-hash.outputs.ui-hash }}
- if: steps.cache-ui-assets.outputs.cache-hit != 'true'
name: Install PNPM
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
with:
run_install: false
package_json_file: './ui/package.json'
- if: steps.cache-ui-assets.outputs.cache-hit != 'true'
name: Set up node and pnpm
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version-file: ui/package.json
cache: pnpm
cache-dependency-path: ui/pnpm-lock.yaml
- if: steps.cache-ui-assets.outputs.cache-hit != 'true'
name: Build UI
run: make ci-build-ui
- uses: ./.github/actions/build-ui
id: build-ui
# Artifacts is where we'll build the various Vault binaries and package them into their respective
# Zip bundles, RPM and Deb packages, and container images. After we've packaged them we upload

45
.github/workflows/test-run-enos-scenario-containers.yml vendored
View file

@ -82,14 +82,6 @@ jobs:
GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0
with:
# the Terraform wrapper will break Terraform execution in Enos because
# it changes the output to text when we expect it to be JSON.
terraform_wrapper: false
- uses: hashicorp/action-setup-enos@6ec106c8f809fe645162d73bea565c65f3269907 # v1.52
with:
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
- name: Download Docker Image
id: download
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
@ -101,40 +93,21 @@ jobs:
run: |
echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic || true
- name: Run Enos scenario
id: run
# Continue once and retry to handle occasional blips when creating
# infrastructure.
continue-on-error: true
id: run-scenario
env:
ENOS_VAR_terraform_plugin_cache_dir: ../support/terraform-plugin-cache
ENOS_VAR_vault_build_date: ${{ needs.metadata.outputs.build-date }}
ENOS_VAR_vault_version: ${{ needs.metadata.outputs.vault-version }}
ENOS_VAR_vault_revision: ${{ inputs.vault-revision }}
ENOS_VAR_container_image_archive: ${{steps.download.outputs.download-path}}/${{ inputs.build-artifact-name }}
run: |
mkdir -p ./enos/support/terraform-plugin-cache
enos scenario run --timeout 10m0s --chdir ./enos/k8s ${{ matrix.scenario.id.filter }}
- name: Retry Enos scenario
id: run_retry
if: steps.run.outcome == 'failure'
env:
ENOS_VAR_terraform_plugin_cache_dir: ../support/terraform-plugin-cache
ENOS_VAR_vault_build_date: ${{ needs.metadata.outputs.build-date }}
ENOS_VAR_vault_version: ${{ needs.metadata.outputs.vault-version }}
ENOS_VAR_vault_revision: ${{ inputs.vault-revision }}
ENOS_VAR_container_image_archive: ${{steps.download.outputs.download-path}}/${{ inputs.build-artifact-name }}
run: |
enos scenario run --timeout 10m0s --chdir ./enos/k8s ${{ matrix.scenario.id.filter }}
- name: Destroy Enos scenario
if: ${{ always() }}
env:
ENOS_VAR_terraform_plugin_cache_dir: ../support/terraform-plugin-cache
ENOS_VAR_vault_build_date: ${{ needs.metadata.outputs.build-date }}
ENOS_VAR_vault_version: ${{ needs.metadata.outputs.vault-version }}
ENOS_VAR_vault_revision: ${{ inputs.vault-revision }}
ENOS_VAR_container_image_archive: ${{steps.download.outputs.download-path}}/${{ inputs.build-artifact-name }}
run: |
enos scenario destroy --timeout 10m0s --grpc-listen http://localhost --chdir ./enos/k8s ${{ matrix.scenario.id.filter }}
uses: ./.github/actions/run-enos-scenario
with:
scenario-filter: ${{ matrix.scenario.id.filter }}
working-directory: ./enos/k8s
timeout: 10m0s
retry-timeout: 10m0s
destroy-timeout: 10m0s
destroy-extra-args: --grpc-listen http://localhost
- name: Cleanup Enos runtime directories
if: ${{ always() }}
run: |

83
.github/workflows/test-run-enos-scenario-matrix.yml vendored
View file

@ -43,7 +43,7 @@ on:
jobs:
metadata:
runs-on: ${{ fromJSON(inputs.runs-on) }}
runs-on: ${{ github.repository == 'hashicorp/vault' && '"ubuntu-latest"' || fromJSON('["self-hosted","ubuntu-latest-x64"]') }}
permissions:
id-token: write # vault-auth
contents: read
@ -112,6 +112,7 @@ jobs:
permissions:
id-token: write # vault-auth
contents: read
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
@ -140,6 +141,7 @@ jobs:
kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE;
kv/data/github/${{ github.repository }}/ibm-license license_1 | VAULT_LICENSE_IBM;
kv/data/github/${{ github.repository }}/github-token token | ELEVATED_GITHUB_TOKEN;
- id: secrets
run: |
if [[ "${{ needs.metadata.outputs.is-ent-repo }}" != 'true' ]]; then
@ -175,6 +177,7 @@ jobs:
echo 'vault-license-ibm=${{ steps.vault-secrets.outputs.VAULT_LICENSE_IBM }}'
} | tee -a "$GITHUB_OUTPUT"
fi
- id: env
run: |
# Configure input environment variables.
@ -204,19 +207,18 @@ jobs:
echo 'ENOS_VAR_verify_ldap_secrets_engine=false'
echo 'ENOS_VAR_verify_log_secrets=true'
} | tee -a "$GITHUB_ENV"
- uses: ./.github/actions/set-up-go
with:
github-token: ${{ steps.secrets.outputs.github-token }}
- name: Install LDAP client tools
run: |
sudo apt-get update
sudo apt-get install -y ldap-utils
- uses: ./.github/actions/install-tools
- uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0
with:
# the Terraform wrapper will break Terraform execution in Enos because
# it changes the output to text when we expect it to be JSON.
terraform_wrapper: false
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
with:
@ -226,14 +228,13 @@ jobs:
role-to-assume: ${{ steps.secrets.outputs.aws-role-arn }}
role-skip-session-tagging: true
role-duration-seconds: 3600
- uses: hashicorp/action-setup-enos@6ec106c8f809fe645162d73bea565c65f3269907 # v1.52
with:
github-token: ${{ steps.secrets.outputs.github-token }}
- uses: ./.github/actions/create-dynamic-config
with:
github-token: ${{ steps.secrets.outputs.github-token }}
vault-version: ${{ inputs.vault-version }}
vault-edition: ${{ inputs.vault-edition }}
- name: Prepare scenario dependencies
id: prepare_scenario
run: |
@ -247,52 +248,42 @@ jobs:
echo "test_results_artifact_name=test-results_$(echo "${{ matrix.scenario.id.filter }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g')"
echo "junit_results_artifact_name=junit-results_$(echo "${{ matrix.scenario.id.filter }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g')"
echo "failure_summary_artifact_name=failure-summary-enos_$(echo "${{ matrix.scenario.id.filter }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g').md"
} >> "$GITHUB_OUTPUT"
} | tee -a "$GITHUB_OUTPUT"
- if: contains(inputs.sample-name, 'build')
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: ${{ inputs.build-artifact-name }}
path: ./enos/support/downloads
- if: contains(inputs.sample-name, 'ent')
name: Configure Vault licenses
run: |
echo "${{ steps.secrets.outputs.vault-license }}" > ./enos/support/vault.hclic || true
echo "${{ steps.secrets.outputs.vault-license-ibm }}" > ./enos/support/ibm-pao.lic || true
- if: contains(matrix.scenario.id.filter, 'consul_edition:ent')
name: Configure Consul license
run: |
echo "${{ steps.secrets.outputs.consul-license }}" > ./enos/support/consul.hclic || true
- name: Configure Vault Radar license
run: |
echo "${{ steps.secrets.outputs.radar-license }}" > ./enos/support/vault-radar.hclic || true
- id: launch
name: enos scenario launch ${{ matrix.scenario.id.filter }}
# Continue once and retry to handle occasional blips when creating infrastructure.
continue-on-error: true
run: enos scenario launch --timeout 45m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
- if: steps.launch.outcome == 'failure'
id: launch_retry
name: Retry enos scenario launch ${{ matrix.scenario.id.filter }}
run: enos scenario launch --timeout 45m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
- name: Upload Debug Data
if: failure()
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
- id: run-scenario
name: Run Enos scenario ${{ matrix.scenario.id.filter }}
uses: ./.github/actions/run-enos-scenario
with:
# The name of the artifact is the same as the matrix scenario name with the spaces replaced with underscores and colons replaced by equals.
name: ${{ steps.prepare_scenario.outputs.debug_data_artifact_name }}
path: ${{ env.ENOS_DEBUG_DATA_ROOT_DIR }}
retention-days: 30
continue-on-error: true
- if: ${{ always() }}
id: destroy
name: enos scenario destroy ${{ matrix.scenario.id.filter }}
continue-on-error: true
run: enos scenario destroy --timeout 10m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
- if: steps.destroy.outcome == 'failure'
id: destroy_retry
name: Retry enos scenario destroy ${{ matrix.scenario.id.filter }}
continue-on-error: true
run: enos scenario destroy --timeout 10m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
scenario-filter: ${{ matrix.scenario.id.filter }}
working-directory: ./enos
timeout: 45m0s
retry-timeout: 30m0s
destroy-timeout: 10m0s
upload-debug-data: 'true'
debug-data-retention-days: '30'
debug-data-path: ${{ env.ENOS_DEBUG_DATA_ROOT_DIR }}
- name: Upload Test Results
if: always()
id: upload_test_results
@ -303,6 +294,7 @@ jobs:
retention-days: 7
if-no-files-found: ignore
continue-on-error: true
- name: Upload JUnit Test Results
if: always()
id: upload_junit_results
@ -313,6 +305,7 @@ jobs:
retention-days: 7
if-no-files-found: ignore
continue-on-error: true
- name: Check for test results
if: always()
id: check_test_results
@ -323,6 +316,7 @@ jobs:
else
echo "has_results=false" >> "$GITHUB_OUTPUT"
fi
- name: Prepare Test Results Summary
if: always() && steps.check_test_results.outputs.has_results == 'true'
continue-on-error: true
@ -368,6 +362,7 @@ jobs:
else
echo "⚠️ No test results found in /tmp/vault_test_results_*.json" >> "$GITHUB_STEP_SUMMARY"
fi
- name: Upload Failure Summary
if: always() && steps.check_test_results.outputs.has_results == 'true'
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
@ -376,6 +371,7 @@ jobs:
path: ${{ steps.prepare_scenario.outputs.failure_summary_artifact_name }}
if-no-files-found: ignore
continue-on-error: true
- name: Clean up Enos runtime directories
id: cleanup
if: ${{ always() }}
@ -384,34 +380,39 @@ jobs:
rm -rf /tmp/enos*
rm -rf ./enos/support
rm -rf ./enos/.enos
# Send slack notifications to #feed-vault-enos-failures any of our enos scenario commands fail.
# There is an incoming webhook set up on the "Enos Vault Failure Bot" Slackbot:
# https://api.slack.com/apps/A05E31CH1LG/incoming-webhooks
- if: ${{ always() && ! cancelled() }}
name: Notify launch failed
uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1
with:
failure-message: "enos scenario launch ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
status: ${{ steps.launch.outcome }}
status: ${{ steps.run-scenario.outputs.launch-outcome == 'failure' && 'failure' || 'success' }}
slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}
- if: ${{ always() && ! cancelled() }}
name: Notify retry launch failed
uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1
with:
failure-message: "retry enos scenario launch ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
status: ${{ steps.launch_retry.outcome }}
status: ${{ steps.run-scenario.outputs.launch-retry-attempted == 'true' && steps.run-scenario.outputs.launch-outcome == 'failure' && 'failure' || 'success' }}
slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}
- if: ${{ always() && ! cancelled() }}
name: Notify destroy failed
uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1
with:
failure-message: "enos scenario destroy ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
status: ${{ steps.destroy.outcome }}
status: ${{ steps.run-scenario.outputs.destroy-outcome == 'failure' && 'failure' || 'success' }}
slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}
- if: ${{ always() && ! cancelled() }}
name: Notify retry destroy failed
uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1
with:
failure-message: "retry enos scenario destroy ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
status: ${{ steps.destroy_retry.outcome }}
status: ${{ steps.run-scenario.outputs.destroy-retry-attempted == 'true' && steps.run-scenario.outputs.destroy-outcome == 'failure' && 'failure' || 'success' }}
slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}

30
.github/workflows/test-run-enos-scenario.yml vendored
View file

@ -79,11 +79,6 @@ jobs:
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
with:
package_json_file: './ui/package.json'
- uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0
with:
# the Terraform wrapper will break Terraform execution in Enos because
# it changes the output to text when we expect it to be JSON.
terraform_wrapper: false
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
with:
@ -93,9 +88,6 @@ jobs:
role-to-assume: ${{ secrets.AWS_ROLE_ARN_CI }}
role-skip-session-tagging: true
role-duration-seconds: 3600
- uses: hashicorp/action-setup-enos@6ec106c8f809fe645162d73bea565c65f3269907 # v1.52
with:
github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
- name: Prepare scenario dependencies
id: scenario-deps
run: |
@ -108,8 +100,17 @@ jobs:
id: license
run: echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic
- name: Run Enos scenario
id: run
run: enos scenario run --timeout 60m0s --chdir ./enos ${{ inputs.scenario }}
id: run-scenario
uses: ./.github/actions/run-enos-scenario
with:
scenario-filter: ${{ inputs.scenario }}
working-directory: ./enos
timeout: 60m0s
retry-timeout: 60m0s
destroy-timeout: 60m0s
destroy-extra-args: --grpc-listen http://localhost
debug-data-path: ${{ env.ENOS_DEBUG_DATA_ROOT_DIR }}
debug-data-retention-days: '1'
- name: Collect logs when scenario fails
id: collect_logs
if: ${{ always() }}
@ -122,15 +123,6 @@ jobs:
name: enos-scenario-logs
path: ${{ steps.scenario-deps.outputs.logsdir }}
retention-days: 1
- uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
if: ${{ always() }}
with:
name: enos-debug-data-logs
path: ${{ env.ENOS_DEBUG_DATA_ROOT_DIR }}
retention-days: 1
- name: Ensure scenario has been destroyed
if: ${{ always() }}
run: enos scenario destroy --timeout 60m0s --grpc-listen http://localhost --chdir ./enos ${{ inputs.scenario }}
- name: Clean up Enos runtime directories
if: ${{ always() }}
run: |

2
.gitignore vendored
View file

@ -66,6 +66,8 @@ Vagrantfile
enos-local.vars.hcl
enos/**/support
enos/**/kubeconfig
enos/modules/**/plan.yml
enos/sarif**.json
.terraform
.terraform.lock.hcl
.tfstate.*

32
.release/pipeline.hcl
View file

@ -119,6 +119,7 @@ changed_files {
# These exist on CE branches to please Github Actions.
joinpath(".github", "workflows", "build-artifacts-ent.yml"),
joinpath(".github", "workflows", "backport-automation-ent.yml"),
joinpath(".github", "workflows", "test-run-enos-scenario-cloud.yml"),
]
}
@ -127,7 +128,6 @@ changed_files {
base_dir = [
"website",
joinpath(".release", "docker"),
joinpath("enos", "modules"),
joinpath("scripts", "docker"),
]
}
@ -147,6 +147,26 @@ changed_files {
]
}
// Ignore some enos modules related to HSM
ignore {
base_dir = [
# The next matcher looks for HSM but we can ignore the softhsm modules
joinpath("enos", "modules", "softhsm_install"),
joinpath("enos", "modules", "softhsm_create_vault_keys"),
joinpath("enos", "modules", "softhsm_init"),
joinpath("enos", "modules", "softhsm_distribute_vault_keys"),
# Some filename have ent in them
joinpath("enos", "modules", "verify_secrets_engines"),
]
}
// Make sure our zap scanner is always ent only
match {
base_dir = [
joinpath("enos", "modules", "zap_scan_ent")
]
}
// Match whole directories that are enterprise only
match {
base_dir = [
@ -287,4 +307,14 @@ changed_files {
base_dir = ["ui"]
}
}
// The "zap_scan" group matches the Zap scanner
group "zap_scan" {
match {
contains = [
"security-scan-zap",
"zap_scan",
]
}
}
}

6
enos/enos-descriptions.hcl
View file

@ -217,12 +217,6 @@ globals {
target hosts.
EOF
run_verify_blackbox_tests_remote = <<-EOF
Run blackbox verification tests directly on the Vault leader host. These tests execute
the Vault CLI binary on the target machine to validate version metadata and other
functionality that requires local binary access.
EOF
wait_for_cluster_to_have_leader = <<-EOF
Wait for a leader election to occur before we proceed with any further quality verification.
EOF

3
enos/enos-providers.hcl
View file

@ -30,3 +30,6 @@ provider "hcp" "default" {
provider "docker" "default" {
}
provider "local" "default" {
}

52
enos/enos-scenario-agent.hcl
View file

@ -178,11 +178,12 @@ scenario "agent" {
}
variables {
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
vpc_id = step.create_vpc.id
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
root_volume_size = 64
vpc_id = step.create_vpc.id
}
}
@ -501,37 +502,16 @@ scenario "agent" {
]
variables {
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
}
}
step "run_verify_blackbox_tests_remote" {
description = global.description.run_verify_blackbox_tests_remote
module = module.vault_run_blackbox_test
depends_on = [step.run_verify_blackbox_tests]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_version_build_date,
quality.vault_version_edition,
quality.vault_version_release,
]
variables {
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultCLIVersionLocal"]
vault_edition = matrix.edition
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
}
}

28
enos/enos-scenario-autopilot.hcl
View file

@ -175,11 +175,12 @@ scenario "autopilot" {
}
variables {
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
vpc_id = step.create_vpc.id
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
root_volume_size = 64
vpc_id = step.create_vpc.id
}
}
@ -845,7 +846,6 @@ scenario "autopilot" {
}
step "run_verify_blackbox_tests" {
skip_step = true # Skip black box tests on autopilot for now
description = global.description.run_verify_blackbox_tests
module = module.vault_run_blackbox_test
depends_on = [
@ -869,12 +869,16 @@ scenario "autopilot" {
]
variables {
leader_host = step.get_updated_vault_cluster_ips.leader_host
leader_public_ip = step.get_updated_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
leader_host = step.get_updated_vault_cluster_ips.leader_host
leader_public_ip = step.get_updated_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_install_dir = local.vault_install_dir
}
}

57
enos/enos-scenario-dr-replication.hcl
View file

@ -218,11 +218,12 @@ scenario "dr_replication" {
}
variables {
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
vpc_id = step.create_vpc.id
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
root_volume_size = 64
vpc_id = step.create_vpc.id
}
}
@ -666,6 +667,9 @@ scenario "dr_replication" {
}
step "run_verify_blackbox_tests" {
// NOTE: This must run before we transition the cluster to a DR secondary.
// because afterwards the namespaces API won't be available and the bbsdk
// init will fail.
description = global.description.run_verify_blackbox_tests
module = module.vault_run_blackbox_test
depends_on = [step.get_primary_cluster_ips]
@ -683,37 +687,16 @@ scenario "dr_replication" {
]
variables {
leader_host = step.get_primary_cluster_ips.leader_host
leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
vault_root_token = step.create_primary_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
}
}
step "run_verify_blackbox_tests_remote" {
description = global.description.run_verify_blackbox_tests_remote
module = module.vault_run_blackbox_test
depends_on = [step.run_verify_blackbox_tests]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_version_build_date,
quality.vault_version_edition,
quality.vault_version_release,
]
variables {
leader_host = step.get_primary_cluster_ips.leader_host
leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
vault_root_token = step.create_primary_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultCLIVersionLocal"]
vault_edition = matrix.edition
leader_host = step.get_primary_cluster_ips.leader_host
leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
vault_root_token = step.create_primary_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
}
}
@ -804,6 +787,8 @@ scenario "dr_replication" {
depends_on = [
step.get_primary_cluster_ips,
step.get_secondary_cluster_ips,
step.run_verify_blackbox_tests,
step.verify_ui,
step.verify_secrets_engines_on_primary,
]

51
enos/enos-scenario-plugin.hcl
View file

@ -178,11 +178,12 @@ scenario "plugin" {
}
variables {
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = "plugin-integration"
common_tags = global.tags
instance_count = 1
vpc_id = step.create_vpc.id
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = "plugin-integration"
common_tags = global.tags
instance_count = 1
root_volume_size = 64
vpc_id = step.create_vpc.id
}
}
@ -451,37 +452,17 @@ scenario "plugin" {
]
variables {
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
}
}
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
step "run_verify_blackbox_tests_remote" {
description = global.description.run_verify_blackbox_tests_remote
module = module.vault_run_blackbox_test
depends_on = [step.run_verify_blackbox_tests]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_version_build_date,
quality.vault_version_edition,
quality.vault_version_release,
]
variables {
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultCLIVersionLocal"]
vault_edition = matrix.edition
}
}

52
enos/enos-scenario-pr-replication.hcl
View file

@ -218,11 +218,12 @@ scenario "pr_replication" {
}
variables {
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
vpc_id = step.create_vpc.id
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
root_volume_size = 64
vpc_id = step.create_vpc.id
}
}
@ -705,37 +706,16 @@ scenario "pr_replication" {
]
variables {
leader_host = step.get_primary_cluster_ips.leader_host
leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
vault_root_token = step.create_primary_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
}
}
step "run_verify_blackbox_tests_remote" {
description = global.description.run_verify_blackbox_tests_remote
module = module.vault_run_blackbox_test
depends_on = [step.run_verify_blackbox_tests]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_version_build_date,
quality.vault_version_edition,
quality.vault_version_release,
]
variables {
leader_host = step.get_primary_cluster_ips.leader_host
leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
vault_root_token = step.create_primary_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultCLIVersionLocal"]
vault_edition = matrix.edition
leader_host = step.get_primary_cluster_ips.leader_host
leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
vault_root_token = step.create_primary_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
}
}

51
enos/enos-scenario-proxy.hcl
View file

@ -185,11 +185,12 @@ scenario "proxy" {
}
variables {
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
vpc_id = step.create_vpc.id
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
root_volume_size = 64
vpc_id = step.create_vpc.id
}
}
@ -477,37 +478,17 @@ scenario "proxy" {
]
variables {
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
}
}
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
step "run_verify_blackbox_tests_remote" {
description = global.description.run_verify_blackbox_tests_remote
module = module.vault_run_blackbox_test
depends_on = [step.run_verify_blackbox_tests]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_version_build_date,
quality.vault_version_edition,
quality.vault_version_release,
]
variables {
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultCLIVersionLocal"]
vault_edition = matrix.edition
}
}

51
enos/enos-scenario-seal-ha.hcl
View file

@ -217,11 +217,12 @@ scenario "seal_ha" {
}
variables {
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
vpc_id = step.create_vpc.id
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
root_volume_size = 64
vpc_id = step.create_vpc.id
}
}
@ -778,37 +779,17 @@ scenario "seal_ha" {
]
variables {
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
}
}
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
step "run_verify_blackbox_tests_remote" {
description = global.description.run_verify_blackbox_tests_remote
module = module.vault_run_blackbox_test
depends_on = [step.run_verify_blackbox_tests]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_version_build_date,
quality.vault_version_edition,
quality.vault_version_release,
]
variables {
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultCLIVersionLocal"]
vault_edition = matrix.edition
}
}

15
enos/enos-scenario-smoke-sdk.hcl
View file

@ -178,11 +178,12 @@ scenario "smoke_sdk" {
}
variables {
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
vpc_id = step.create_vpc.id
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
root_volume_size = 64
vpc_id = step.create_vpc.id
}
}
@ -412,7 +413,9 @@ scenario "smoke_sdk" {
locals {
// Default test packages for smoke_sdk scenario
default_test_packages = ["core", "secrets", "replication", "raft", "integration"]
// TODO: "integration" has been removed from the default packages pending
// a restructure to handle tests that mutate the cluster via sys endpoints.
default_test_packages = ["core", "secrets", "replication", "raft"]
// Determine if filter contains test names (starts with "Test") or package names
is_test_name_filter = length(var.blackbox_test_filter) > 0 && length([for t in var.blackbox_test_filter : t if can(regex("^Test", t))]) > 0

54
enos/enos-scenario-smoke.hcl
View file

@ -175,11 +175,12 @@ scenario "smoke" {
}
variables {
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
vpc_id = step.create_vpc.id
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
root_volume_size = 64
vpc_id = step.create_vpc.id
}
}
@ -519,41 +520,20 @@ scenario "smoke" {
]
variables {
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
}
}
step "run_verify_blackbox_tests_remote" {
description = global.description.run_verify_blackbox_tests_remote
module = module.vault_run_blackbox_test
depends_on = [step.run_verify_blackbox_tests]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_version_build_date,
quality.vault_version_edition,
quality.vault_version_release,
]
variables {
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultCLIVersionLocal"]
vault_edition = matrix.edition
}
}
step "verify_raft_auto_join_voter" {
description = global.description.verify_raft_cluster_all_nodes_are_voters
skip_step = matrix.backend != "raft"

51
enos/enos-scenario-upgrade.hcl
View file

@ -192,11 +192,12 @@ scenario "upgrade" {
}
variables {
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
vpc_id = step.create_vpc.id
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
instance_count = 1
root_volume_size = 64
vpc_id = step.create_vpc.id
}
}
@ -713,37 +714,17 @@ scenario "upgrade" {
]
variables {
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
}
}
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultServerVersion"]
vault_edition = matrix.edition
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_install_dir = global.vault_install_dir[matrix.artifact_type]
step "run_verify_blackbox_tests_remote" {
description = global.description.run_verify_blackbox_tests_remote
module = module.vault_run_blackbox_test
depends_on = [step.run_verify_blackbox_tests]
providers = {
enos = local.enos_provider[matrix.distro]
}
verifies = [
quality.vault_version_build_date,
quality.vault_version_edition,
quality.vault_version_release,
]
variables {
leader_host = step.get_vault_cluster_ips.leader_host
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
vault_root_token = step.create_vault_cluster.root_token
test_package = "./vault/external_tests/blackbox/verify"
test_names = ["TestVaultCLIVersionLocal"]
vault_edition = matrix.edition
}
}

4
enos/modules/build_local/scripts/build.sh
View file

@ -22,7 +22,9 @@ root_dir="$(git rev-parse --show-toplevel)"
pushd "$root_dir" > /dev/null
if [ -n "$BUILD_UI" ] && [ "$BUILD_UI" = "true" ]; then
make ci-build-ui
if ! output=$(make ci-build-ui 2>&1); then
echo "Failed to build the UI assets. Make sure you have the required node version (defined in ui/package.json) install and have set up pnpm (npm i -g pnpm): ${output}" 1>&2
fi
fi
make ci-build

2
enos/modules/cloud_docker_vault_cluster/main.tf
View file

@ -152,6 +152,8 @@ locals {
tls_disable = true
}
administrative_namespace_path = "admin"
storage "raft" {
path = "/vault/data"
node_id = "node%s"

2
enos/modules/ec2_info/main.tf
View file

@ -245,7 +245,7 @@ output "ami_ids" {
}
output "current_region" {
value = data.aws_region.current
value = data.aws_region.current.id
}
output "availability_zones" {

32
enos/modules/vault_run_blackbox_test/main.tf
View file

@ -32,17 +32,26 @@ resource "random_string" "test_id" {
}
resource "enos_local_exec" "run_blackbox_test" {
scripts = [abspath("${path.module}/scripts/run-test.sh")]
environment = merge({
VAULT_TOKEN = var.vault_root_token
VAULT_ADDR = var.vault_addr != null ? var.vault_addr : "http://${var.leader_public_ip}:8200"
VAULT_TEST_PACKAGE = var.test_package
VAULT_TEST_MATRIX = length(local.test_names) > 0 ? local_file.test_matrix.filename : ""
VAULT_EDITION = var.vault_edition
# PATH and Go-related environment variables are inherited from the calling process
}, var.vault_namespace != null ? {
VAULT_NAMESPACE = var.vault_namespace
} : {}, local.ldap_environment, local.postgres_environment
scripts = [abspath("${path.module}/scripts/run-test.sh")]
depends_on = [local_file.test_matrix]
environment = merge(
{
VAULT_TOKEN = var.vault_root_token
VAULT_ADDR = var.vault_addr != null ? var.vault_addr : "http://${var.leader_public_ip}:8200"
VAULT_TEST_PACKAGE = var.test_package
VAULT_TEST_MATRIX = length(local.test_names) > 0 ? local_file.test_matrix.filename : ""
VAULT_EDITION = var.vault_edition
# PATH and Go-related environment variables are inherited from the calling process
},
var.vault_namespace != null ? { VAULT_NAMESPACE = var.vault_namespace } : {},
var.vault_product_version != null ? { VAULT_VERSION = var.vault_product_version } : {},
var.vault_revision != null ? { VAULT_REVISION = var.vault_revision } : {},
var.vault_build_date != null ? { VAULT_BUILD_DATE = var.vault_build_date } : {},
var.vault_install_dir != null ? { VAULT_INSTALL_DIR = var.vault_install_dir } : {},
local.ldap_environment,
local.postgres_environment,
local.mongodb_environment
)
}
@ -60,6 +69,7 @@ locals {
LDAP_URL_PUBLIC = "ldap://${local.ldap_config.host.public_ip}:${local.ldap_config.port}"
LDAP_BIND_DN = "cn=admin,${local.domain_dn}"
LDAP_BIND_PASS = local.ldap_config.admin_pw
LDAP_USERNAME = "enos"
} : {}
# Extract PostgreSQL configuration safely, defaulting to empty map if not available

24
enos/modules/vault_run_blackbox_test/variables.tf
View file

@ -53,3 +53,27 @@ variable "vault_edition" {
description = "The Vault edition (ce, ent, ent.hsm, ent.fips1402, ent.hsm.fips1402)"
default = "ent"
}
variable "vault_product_version" {
type = string
description = "The Vault product version (e.g., 1.15.0)"
default = null
}
variable "vault_revision" {
type = string
description = "The Vault git revision/commit SHA"
default = null
}
variable "vault_build_date" {
type = string
description = "The Vault build date"
default = null
}
variable "vault_install_dir" {
type = string
description = "The directory where Vault is installed"
default = null
}

11
sdk/helper/testcluster/blackbox/assertions.go
View file

@ -74,6 +74,17 @@ func (ma *MapAssertion) HasKey(key string, expected any) *MapAssertion {
return ma
}
func (ma *MapAssertion) GetKey(key string) any {
ma.t.Helper()
val, ok := ma.data[key]
if !ok {
ma.t.Fatalf("[%s] missing expected key: %q", ma.path, key)
}
return val
}
func (ma *MapAssertion) HasKeyCustom(key string, f func(val any) bool) *MapAssertion {
ma.t.Helper()

9
sdk/helper/testcluster/blackbox/session.go
View file

@ -88,11 +88,14 @@ func New(t *testing.T, opts ...SessionOpts) *Session {
t.Cleanup(func() {
if session.NoCleanup {
t.Logf("WARN: NoDebug has been set, not cleaning up namespace")
t.Logf("WARN: NoCleanup has been set, not cleaning up namespace")
return
}
_, err = privClient.Logical().Delete(nsURLPath)
require.NoError(t, err)
privClient.SetClientTimeout(time.Second)
session.Eventually(func() error {
_, err = privClient.Logical().Delete(nsURLPath)
return err
})
t.Logf("Cleaned up namespace %s", nsName)
})

58
sdk/helper/testcluster/blackbox/session_autopilot.go Normal file
View file

@ -0,0 +1,58 @@
// Copyright IBM Corp. 2026
// SPDX-License-Identifier: BUSL-1.1
package blackbox
import (
"errors"
"time"
"github.com/hashicorp/vault/api"
"github.com/stretchr/testify/require"
)
func (s *Session) AssertAutopilotHealthy() {
s.t.Helper()
// query the autopilot state endpoint and verify that all nodes are healthy according to autopilot
healthy, err := s.autopilotStateHealthy()
require.NoError(s.t, err)
require.Truef(s.t, healthy, "expected autopilot state to be healthy")
}
func (s *Session) autopilotState() (*api.Secret, error) {
// query the autopilot state endpoint and verify that all nodes are healthy according to autopilot
var state *api.Secret
return state, s.Req(
func(c *api.Client) error {
var err error
state, err = c.Logical().Read("sys/storage/raft/autopilot/state")
return err
},
WithClientRootNamespace(),
WithClientTimeout(2*time.Second),
)
}
func (s *Session) autopilotStateHealthy() (bool, error) {
state, err := s.autopilotState()
if err != nil {
return false, err
}
if state == nil {
return false, errors.New("no raft data response")
}
health, ok := state.Data["healthy"]
if !ok {
return false, errors.New("raft data missing 'healthy' key")
}
healthy, ok := health.(bool)
if !ok {
return false, errors.New("raft data 'healthy' key is unknown type")
}
return healthy, nil
}

67
sdk/helper/testcluster/blackbox/session_client.go Normal file
View file

@ -0,0 +1,67 @@
// Copyright IBM Corp. 2026
// SPDX-License-Identifier: BUSL-1.1
package blackbox
import (
"os"
"time"
"github.com/hashicorp/vault/api"
)
type ClientOpt func(*api.Client)
func WithClientRootNamespace() ClientOpt {
return func(c *api.Client) {
c.ClearNamespace()
}
}
func WithClientParentNamespace() ClientOpt {
return func(c *api.Client) {
c.SetNamespace(getParentNamespace())
}
}
func WithClientTimeout(d time.Duration) ClientOpt {
return func(c *api.Client) {
c.SetClientTimeout(d)
}
}
func (s *Session) Req(fn func(*api.Client) error, opts ...ClientOpt) error {
s.t.Helper()
c, err := api.NewClient(s.Client.CloneConfig())
if err != nil {
return err
}
for _, opt := range opts {
opt(c)
}
return fn(c)
}
// GetParentNamespace returns the namespace from VAULT_NAMESPACE environment variable.
// The blackbox test framework auto-creates a unique child namespace for each test
// (e.g., "admin/bbsdk-xxxxx") for isolation. VAULT_NAMESPACE contains the base namespace
// (e.g., "admin"), which is the parent of the test's namespace.
// Example: VAULT_NAMESPACE="admin" → test runs in "admin/bbsdk-xxxxx" → returns "admin"
// Note: This doesn't traverse the namespace hierarchy - it simply returns VAULT_NAMESPACE,
// which happens to be the parent of the test namespace.
func (s *Session) GetParentNamespace() string {
return getParentNamespace()
}
func getParentNamespace() string {
ns := os.Getenv("VAULT_NAMESPACE")
// If VAULT_NAMESPACE is not set, default to root namespace (empty string).
// This handles cases where tests run in non-namespaced environments.
if ns == "" {
return ""
}
return ns
}

89
sdk/helper/testcluster/blackbox/session_cluster.go Normal file
View file

@ -0,0 +1,89 @@
// Copyright IBM Corp. 2026
// SPDX-License-Identifier: BUSL-1.1
package blackbox
import (
"fmt"
"time"
)
// AssertClusterHealthy verifies that the cluster is healthy, with fallback for managed environments
// like HCP where raft APIs may not be accessible. This is the recommended method for general
// cluster health checks in blackbox tests. Uses backoff helper for reasonable retry logic.
//
// Deprecated: Use s.EventuallyClusterHealthy() with an explicit timeout
// retry times.
func (s *Session) AssertClusterHealthy() {
s.t.Helper()
s.EventuallyClusterHealthyUnsealed(1 * time.Minute)
}
// EventuallyClusterHealthyUnsealed verifies that the cluster is healthy and
// unsealed.
func (s *Session) EventuallyClusterHealthyUnsealed(timeout time.Duration) {
s.t.Helper()
hasActiveNode := func() error {
// First, wait until we have an active HA node
_, err := s.haActiveNode()
return err
}
autopilotHealthyIfRaft := func() error {
// Now, make sure autopilot is healthy if we're using integrated storage
// and we have the necessary permissions.
if s.GetParentNamespace() != "" {
s.t.Log("Skipping autopilot health check because we've been configured with a parent namespace and autopilot needs root namespace access")
return nil
}
storage, err := s.getConfigStorageType()
if err != nil {
return err
}
if storage != "raft" {
return nil
}
healthy, err := s.autopilotStateHealthy()
if err != nil {
return err
}
if !healthy {
return fmt.Errorf("raft autopilot state is not healthy")
}
return nil
}
clusterUnsealed := func() error {
sealed, err := s.sealed()
if err != nil {
return err
}
if sealed {
return fmt.Errorf("cluster is sealed")
}
return nil
}
// Go through our checks and wait for each.
started := time.Now()
for _, check := range []func() error{
hasActiveNode,
autopilotHealthyIfRaft,
clusterUnsealed,
} {
if timeout < 0 {
s.t.Fatal("timed out waiting for cluster to be healthy")
}
s.EventuallyWithTimeout(check, timeout)
timeout -= time.Since(started)
}
}

89
sdk/helper/testcluster/blackbox/session_config.go Normal file
View file

@ -0,0 +1,89 @@
// Copyright IBM Corp. 2026
// SPDX-License-Identifier: BUSL-1.1
package blackbox
import (
"fmt"
"time"
"github.com/hashicorp/vault/api"
"github.com/stretchr/testify/require"
)
func (s *Session) MustSanitizedConfig() *api.Secret {
s.t.Helper()
var config *api.Secret
err := s.Req(
func(c *api.Client) error {
var err error
config, err = c.Logical().Read("sys/config/state/sanitized")
return err
},
WithClientRootNamespace(),
WithClientTimeout(2*time.Second),
)
require.NoError(s.t, err)
require.NotNil(s.t, config)
return config
}
func (s *Session) MustGetConfigStorageType() string {
s.t.Helper()
sanitizedConfig := s.MustSanitizedConfig()
// Verify we have at least one server configured
storageType := s.AssertSecret(sanitizedConfig).
Data().
GetMap("storage").
GetKey("type")
storage, ok := storageType.(string)
if !ok {
s.t.Fatalf("cluster storage is unknown: %v", storageType)
}
if storage == "" {
s.t.Fatal("cluster storage is empty")
}
s.t.Logf("cluster is using storage type: %s", storage)
return storage
}
func (s *Session) getConfigStorageType() (string, error) {
s.t.Helper()
var storageTypeStr string
getStorageType := func(c *api.Client) error {
secret, err := c.Logical().Read("sys/seal-status")
if err != nil {
return err
}
if secret == nil || len(secret.Data) < 1 {
return fmt.Errorf("seal-status is empty")
}
storage, ok := secret.Data["storage_type"]
if !ok {
return fmt.Errorf("seal-status does not include storage_type")
}
storageTypeStr, ok = storage.(string)
if !ok {
return fmt.Errorf("malformed storage type")
}
return nil
}
return storageTypeStr, s.Req(
getStorageType,
WithClientTimeout(2*time.Second),
)
}

85
sdk/helper/testcluster/blackbox/session_ha.go Normal file
View file

@ -0,0 +1,85 @@
// Copyright IBM Corp. 2026
// SPDX-License-Identifier: BUSL-1.1
package blackbox
import (
"errors"
"time"
"github.com/hashicorp/vault/api"
)
// getClusterNodeCount returns the number of nodes in the cluster
func (s *Session) getClusterNodeCount() (int, error) {
nodes, err := s.haNodes()
if err != nil {
return 0, err
}
return len(nodes), nil
}
// haNodes returns a slice of nodes from the ha-status endpoint.
func (s *Session) haNodes() ([]map[string]any, error) {
res := []map[string]any{}
return res, s.Req(
func(c *api.Client) error {
status, err := c.Logical().Read("sys/ha-status")
if err != nil {
return err
}
if status == nil {
return errors.New("no ha-status returned")
}
nodesAny, ok := status.Data["nodes"]
if !ok {
return errors.New("no HA nodes found in ha-status")
}
nodes, ok := nodesAny.([]any)
if !ok {
return errors.New("invalid ha-status response body")
}
for _, node := range nodes {
nv, ok := node.(map[string]any)
if !ok {
return errors.New("malformed node in ha-status response body")
}
res = append(res, nv)
}
return nil
},
WithClientTimeout(2*time.Second),
)
}
// haActiveNode returns the active node from ha-status.
func (s *Session) haActiveNode() (map[string]any, error) {
nodes, err := s.haNodes()
if err != nil {
return nil, err
}
for _, node := range nodes {
active, ok := node["active_node"]
if !ok {
continue
}
activeVal, ok := active.(bool)
if !ok {
continue
}
if activeVal {
return node, nil
}
}
return nil, errors.New("no active node in ha-status")
}

21
sdk/helper/testcluster/blackbox/session_logical.go
View file

@ -10,6 +10,12 @@ import (
"github.com/stretchr/testify/require"
)
func (s *Session) Read(path string) (*api.Secret, error) {
s.t.Helper()
return s.Client.Logical().Read(path)
}
func (s *Session) MustWrite(path string, data map[string]any) *api.Secret {
s.t.Helper()
@ -26,6 +32,14 @@ func (s *Session) MustRead(path string) *api.Secret {
return secret
}
func (s *Session) MustList(path string) *api.Secret {
s.t.Helper()
secret, err := s.Client.Logical().List(path)
require.NoError(s.t, err)
return secret
}
// MustReadRequired is a stricter version of MustRead that fails if a 404/nil is returned
func (s *Session) MustReadRequired(path string) *api.Secret {
s.t.Helper()
@ -51,3 +65,10 @@ func (s *Session) MustReadKV2(mountPath, secretPath string) *api.Secret {
fullPath := path.Join(mountPath, "data", secretPath)
return s.MustRead(fullPath)
}
func (s *Session) MustDelete(path string) {
s.t.Helper()
_, err := s.Client.Logical().Delete(path)
require.NoError(s.t, err)
}

281
sdk/helper/testcluster/blackbox/session_raft.go
View file

@ -4,11 +4,10 @@
package blackbox
import (
"errors"
"fmt"
"time"
"github.com/hashicorp/vault/api"
"github.com/hashicorp/vault/sdk/helper/backoff"
"github.com/stretchr/testify/require"
)
@ -33,20 +32,78 @@ func (s *Session) AssertRaftStable(numNodes int, allowNonVoters bool) {
}
}
func (s *Session) AssertRaftHealthy() {
func (s *Session) raftConfig() (*api.Secret, error) {
// TODO
// query the autopilot state endpoint and verify that all nodes are healthy according to autopilot
var state *api.Secret
return state, s.Req(
func(c *api.Client) error {
var err error
state, err = c.Logical().Read("sys/storage/raft/autopilot/state")
return err
},
WithClientRootNamespace(),
WithClientTimeout(2*time.Second),
)
}
// EventuallyRaftClusterHealthy verifies that the raft cluster eventually becomes
// healthy regardless of node count.
func (s *Session) EventuallyRaftClusterHealthy(timeout time.Duration) {
s.t.Helper()
// query the autopilot state endpoint and verify that all nodes are healthy according to autopilot
s.EventuallyWithTimeout(
func() error {
healthy, err := s.autopilotStateHealthy()
if err != nil {
return err
}
if !healthy {
return fmt.Errorf("expected raft state to be healthy: got %t", healthy)
}
return nil
}, timeout,
)
// TODO: Perhaps move the server config and voter checks into the retry above.
// Get raft configuration to ensure we have at least one node
secret, err := s.WithRootNamespace(func() (*api.Secret, error) {
return s.Client.Logical().Read("sys/storage/raft/autopilot/state")
return s.Client.Logical().Read("sys/storage/raft/configuration")
})
require.NoError(s.t, err)
require.NotNil(s.t, secret)
_ = s.AssertSecret(secret).
// Verify we have at least one server configured
servers := s.AssertSecret(secret).
Data().
HasKey("healthy", true)
GetMap("config").
GetSlice("servers")
// Ensure we have at least 1 server
if len(servers.data) < 1 {
s.t.Fatal("Expected at least 1 raft server, got 0")
}
// Verify that we have at least one voter in the cluster
hasVoter := false
for _, server := range servers.data {
if serverMap, ok := server.(map[string]any); ok {
if voter, exists := serverMap["voter"]; exists {
if voterBool, ok := voter.(bool); ok && voterBool {
hasVoter = true
break
}
}
}
}
if !hasVoter {
s.t.Fatal("Expected at least one voter in the raft cluster")
}
}
// AssertRaftClusterHealthy verifies that the raft cluster is healthy regardless of node count
@ -56,7 +113,7 @@ func (s *Session) AssertRaftClusterHealthy() {
s.t.Helper()
// First verify autopilot reports the cluster as healthy
s.AssertRaftHealthy()
s.AssertAutopilotHealthy()
// Get raft configuration to ensure we have at least one node
secret, err := s.WithRootNamespace(func() (*api.Secret, error) {
@ -150,210 +207,12 @@ func (s *Session) MustStepDownLeader() {
require.NoError(s.t, err)
}
// GetClusterNodeCount returns the number of nodes in the raft cluster
func (s *Session) GetClusterNodeCount() int {
// MustGetClusterNodeCount returns the number of nodes in the cluster
func (s *Session) MustGetClusterNodeCount() int {
s.t.Helper()
secret, err := s.WithRootNamespace(func() (*api.Secret, error) {
return s.Client.Logical().Read("sys/storage/raft/configuration")
})
if err != nil {
s.t.Logf("Failed to read raft configuration: %v", err)
return 0
}
count, err := s.getClusterNodeCount()
require.NoError(s.t, err)
if secret == nil {
s.t.Log("Raft configuration response was nil")
return 0
}
configData, ok := secret.Data["config"].(map[string]any)
if !ok {
s.t.Log("Could not parse raft config data")
return 0
}
serversData, ok := configData["servers"].([]any)
if !ok {
s.t.Log("Could not parse raft servers data")
return 0
}
return len(serversData)
}
// WaitForNewLeader waits for a new leader to be elected that is different from initialLeader
// and for the cluster to become healthy. For single-node clusters, it just waits for the
// cluster to become healthy again after stepdown. Uses reasonable timeouts to detect race conditions early.
func (s *Session) WaitForNewLeader(initialLeader string, timeoutSeconds int) {
s.t.Helper()
// Use reasonable timeout - if it takes more than a few seconds, there's likely a race condition
if timeoutSeconds > 10 {
s.t.Logf("Warning: timeout of %d seconds is quite high, consider investigating potential race conditions", timeoutSeconds)
}
// Check cluster size to handle single-node case
nodeCount := s.GetClusterNodeCount()
if nodeCount <= 1 {
s.t.Logf("Single-node cluster detected, waiting for cluster to recover after stepdown...")
// Use backoff helper for single-node recovery
b := backoff.NewBackoff(20, 100*time.Millisecond, 1*time.Second) // Max ~10 seconds with backoff
err := b.Retry(func() error {
secret, err := s.WithRootNamespace(func() (*api.Secret, error) {
return s.Client.Logical().Read("sys/storage/raft/autopilot/state")
})
if err != nil {
return err
}
if secret == nil {
return errors.New("no autopilot state returned")
}
healthy, ok := secret.Data["healthy"].(bool)
if !ok {
return errors.New("autopilot healthy status not found")
}
if !healthy {
return errors.New("cluster not yet healthy")
}
return nil
})
if err != nil {
s.t.Fatalf("Single-node cluster failed to recover: %v", err)
}
s.t.Log("Single-node cluster has recovered and is healthy")
return
}
// Multi-node cluster logic - wait for actual leader change
s.t.Logf("Multi-node cluster detected, waiting for new leader election...")
// Phase 1: Wait for new leader (should be fast)
leaderBackoff := backoff.NewBackoff(20, 100*time.Millisecond, 500*time.Millisecond) // Max ~5 seconds
var currentLeader string
err := leaderBackoff.Retry(func() error {
secret, err := s.WithRootNamespace(func() (*api.Secret, error) {
return s.Client.Logical().Read("sys/leader")
})
if err != nil {
return err
}
if secret == nil {
return errors.New("no leader data returned")
}
leaderAddress, ok := secret.Data["leader_address"].(string)
if !ok || leaderAddress == "" {
return errors.New("no leader address found")
}
if leaderAddress == initialLeader {
return errors.New("still waiting for new leader")
}
currentLeader = leaderAddress
return nil
})
if err != nil {
s.t.Fatalf("Failed to elect new leader: %v", err)
}
s.t.Logf("New leader elected: %s (was: %s)", currentLeader, initialLeader)
// Phase 2: Wait for cluster health (should also be fast)
healthBackoff := backoff.NewBackoff(20, 100*time.Millisecond, 500*time.Millisecond) // Max ~5 seconds
err = healthBackoff.Retry(func() error {
secret, err := s.WithRootNamespace(func() (*api.Secret, error) {
return s.Client.Logical().Read("sys/storage/raft/autopilot/state")
})
if err != nil {
return err
}
if secret == nil {
return errors.New("no autopilot state returned")
}
healthy, ok := secret.Data["healthy"].(bool)
if !ok {
return errors.New("autopilot healthy status not found")
}
if !healthy {
return errors.New("cluster not yet healthy")
}
return nil
})
if err != nil {
s.t.Fatalf("Cluster failed to become healthy with new leader: %v", err)
}
s.t.Logf("Cluster is now healthy with new leader: %s", currentLeader)
}
// AssertClusterHealthy verifies that the cluster is healthy, with fallback for managed environments
// like HCP where raft APIs may not be accessible. This is the recommended method for general
// cluster health checks in blackbox tests. Uses backoff helper for reasonable retry logic.
func (s *Session) AssertClusterHealthy() {
s.t.Helper()
// Use backoff helper for cluster readiness checks
b := backoff.NewBackoff(15, 200*time.Millisecond, 2*time.Second) // Max ~15 seconds with backoff
err := b.Retry(func() error {
// Try raft-based health check first (works for self-managed clusters)
secret, err := s.WithRootNamespace(func() (*api.Secret, error) {
return s.Client.Logical().Read("sys/storage/raft/autopilot/state")
})
if err == nil && secret != nil {
// Check if autopilot reports healthy
if healthy, ok := secret.Data["healthy"].(bool); ok && healthy {
// Raft API is available and healthy, use full raft health check
s.AssertRaftClusterHealthy()
return nil
} else if ok && !healthy {
return errors.New("cluster not yet healthy according to autopilot")
}
}
// Raft API not accessible or no healthy status - check basic connectivity
sealStatus, err := s.WithRootNamespace(func() (*api.Secret, error) {
return s.Client.Logical().Read("sys/seal-status")
})
if err != nil {
return err
}
if sealStatus == nil {
return errors.New("seal status response was nil")
}
// Verify cluster is unsealed
sealed, ok := sealStatus.Data["sealed"].(bool)
if !ok {
return errors.New("could not determine seal status")
}
if sealed {
return errors.New("cluster is sealed")
}
// If we get here, cluster is unsealed and responsive
if secret != nil {
s.t.Log("Cluster health verified (self-managed environment)")
} else {
s.t.Log("Cluster health verified (managed environment - raft APIs not accessible)")
}
return nil
})
if err != nil {
s.t.Fatalf("Cluster health check failed: %v", err)
}
return count
}

63
sdk/helper/testcluster/blackbox/session_replication.go Normal file
View file

@ -0,0 +1,63 @@
// Copyright IBM Corp. 2026
// SPDX-License-Identifier: BUSL-1.1
package blackbox
import (
"fmt"
"github.com/hashicorp/vault/api"
"github.com/stretchr/testify/require"
)
// GetDRReplicationMode gets the DR replication mode of the node
func (s *Session) GetDRReplicationMode() (string, error) {
secret, err := s.WithRootNamespace(func() (*api.Secret, error) {
return s.Client.Logical().Read("sys/replication/dr/status")
})
if err != nil {
return "", err
}
if secret == nil {
return "", fmt.Errorf("empty status response")
}
mode, ok := secret.Data["mode"]
if !ok {
return "", fmt.Errorf("no mode field in status response")
}
return mode.(string), nil
}
func (s *Session) AssertReplicationDisabled() {
s.assertReplicationStatus("ce", "disabled")
}
func (s *Session) AssertDRReplicationStatus(expectedMode string) {
s.assertReplicationStatus("dr", expectedMode)
}
func (s *Session) AssertPerformanceReplicationStatus(expectedMode string) {
s.assertReplicationStatus("performance", expectedMode)
}
func (s *Session) assertReplicationStatus(which, expectedMode string) {
s.t.Helper()
secret, err := s.WithRootNamespace(func() (*api.Secret, error) {
return s.Client.Logical().Read("sys/replication/status")
})
require.NoError(s.t, err)
require.NotNil(s.t, secret)
data := s.AssertSecret(secret).Data()
if which == "ce" {
data.HasKey("mode", "disabled")
} else {
data.GetMap(which).HasKey("mode", expectedMode)
}
}

27
sdk/helper/testcluster/blackbox/session_seal.go Normal file
View file

@ -0,0 +1,27 @@
// Copyright IBM Corp. 2026
// SPDX-License-Identifier: BUSL-1.1
package blackbox
import (
"errors"
)
func (s *Session) sealed() (bool, error) {
sealStatus, err := s.Client.Logical().Read("sys/seal-status")
if err != nil {
return true, err
}
if sealStatus == nil {
return true, errors.New("seal status response was nil")
}
// Verify cluster is unsealed
sealed, ok := sealStatus.Data["sealed"].(bool)
if !ok {
return true, errors.New("could not determine seal status")
}
return sealed, nil
}

32
sdk/helper/testcluster/blackbox/session_status.go
View file

@ -8,7 +8,6 @@ import (
"os/exec"
"strings"
"github.com/hashicorp/vault/api"
"github.com/stretchr/testify/require"
)
@ -158,34 +157,3 @@ func (s *Session) AssertServerVersion(version, buildDate string) {
s.AssertVersion(version)
s.AssertBuildDate(version, buildDate)
}
func (s *Session) AssertReplicationDisabled() {
s.assertReplicationStatus("ce", "disabled")
}
func (s *Session) AssertDRReplicationStatus(expectedMode string) {
s.assertReplicationStatus("dr", expectedMode)
}
func (s *Session) AssertPerformanceReplicationStatus(expectedMode string) {
s.assertReplicationStatus("performance", expectedMode)
}
func (s *Session) assertReplicationStatus(which, expectedMode string) {
s.t.Helper()
secret, err := s.WithRootNamespace(func() (*api.Secret, error) {
return s.Client.Logical().Read("sys/replication/status")
})
require.NoError(s.t, err)
require.NotNil(s.t, secret)
data := s.AssertSecret(secret).Data()
if which == "ce" {
data.HasKey("mode", "disabled")
} else {
data.GetMap(which).HasKey("mode", expectedMode)
}
}

18
sdk/helper/testcluster/blackbox/session_util.go
View file

@ -4,7 +4,6 @@
package blackbox
import (
"os"
"time"
"github.com/hashicorp/vault/api"
@ -63,20 +62,3 @@ func (s *Session) WithParentNamespace(fn func() (*api.Secret, error)) (*api.Secr
return fn()
}
// GetParentNamespace returns the namespace from VAULT_NAMESPACE environment variable.
// The blackbox test framework auto-creates a unique child namespace for each test
// (e.g., "admin/bbsdk-xxxxx") for isolation. VAULT_NAMESPACE contains the base namespace
// (e.g., "admin"), which is the parent of the test's namespace.
// Example: VAULT_NAMESPACE="admin" → test runs in "admin/bbsdk-xxxxx" → returns "admin"
// Note: This doesn't traverse the namespace hierarchy - it simply returns VAULT_NAMESPACE,
// which happens to be the parent of the test namespace.
func (s *Session) GetParentNamespace() string {
ns := os.Getenv("VAULT_NAMESPACE")
// If VAULT_NAMESPACE is not set, default to root namespace (empty string).
// This handles cases where tests run in non-namespaced environments.
if ns == "" {
return ""
}
return ns
}

6
vault/external_tests/blackbox/doc.go Normal file
View file

@ -0,0 +1,6 @@
// Copyright IBM Corp. 2026
// SPDX-License-Identifier: BUSL-1.1
// Package blackbox includes external integration tests for testing Vault at the
// API level.
package blackbox

63
vault/external_tests/blackbox/integration/external_secrets_test.go
View file

@ -1,63 +0,0 @@
// Copyright IBM Corp. 2025, 2026
// SPDX-License-Identifier: BUSL-1.1
package integration
import (
"testing"
"github.com/hashicorp/vault/sdk/helper/testcluster/blackbox"
)
// TestSecretsEngineExternalCreate tests creation/setup of secrets engines that require external infrastructure
// These tests are excluded from cloud environments (HCP/Docker) which don't have access to AWS, LDAP servers, etc.
func TestSecretsEngineExternalCreate(t *testing.T) {
v := blackbox.New(t)
// Verify we have a healthy cluster first
v.AssertClusterHealthy()
t.Run("AWSSecrets", func(t *testing.T) {
testAWSSecretsCreate(t, v)
})
t.Run("LDAPSecrets", func(t *testing.T) {
testLDAPSecretsCreate(t, v)
})
t.Run("KMIPSecrets", func(t *testing.T) {
testKMIPSecretsCreate(t, v)
})
}
// TestSecretsEngineExternalRead tests read operations for secrets engines that require external infrastructure
func TestSecretsEngineExternalRead(t *testing.T) {
v := blackbox.New(t)
// Verify we have a healthy cluster first
v.AssertClusterHealthy()
t.Run("AWSSecrets", func(t *testing.T) {
testAWSSecretsRead(t, v)
})
t.Run("LDAPSecrets", func(t *testing.T) {
testLDAPSecretsRead(t, v)
})
t.Run("KMIPSecrets", func(t *testing.T) {
testKMIPSecretsRead(t, v)
})
}
// TestSecretsEngineExternalDelete tests delete operations for secrets engines that require external infrastructure
func TestSecretsEngineExternalDelete(t *testing.T) {
v := blackbox.New(t)
// Verify we have a healthy cluster first
v.AssertClusterHealthy()
t.Run("LDAPSecrets", func(t *testing.T) {
testLDAPSecretsDelete(t, v)
})
}

55
vault/external_tests/blackbox/integration/smoke_test.go
View file

@ -5,43 +5,27 @@ package integration
import (
"testing"
"time"
"github.com/hashicorp/vault/sdk/helper/testcluster/blackbox"
"github.com/stretchr/testify/require"
)
// TestEnosSmoke performs comprehensive smoke testing for Enos scenarios,
// verifying cluster health, replication status, raft stability, and basic
// KV operations with authentication. This test validates core functionality.
func TestEnosSmoke(t *testing.T) {
v := blackbox.New(t)
v.AssertUnsealedAny()
v.AssertDRReplicationStatus("primary")
v.AssertPerformanceReplicationStatus("disabled")
v.AssertRaftStable(3, false)
v.AssertRaftHealthy()
// Setup using common utilities
bob := SetupStandardKVUserpass(v, "secret", "bob", "lol")
// Write and verify standard test data
v.MustWriteKV2("secret", "app-config", StandardKVData)
secret := bob.MustReadKV2("secret", "app-config")
AssertKVData(t, bob, secret, StandardKVData)
}
// TestStepdownAndLeaderElection tests raft leadership changes by forcing the current
// leader to step down and verifying that a new leader is elected successfully,
// while ensuring the cluster remains healthy throughout the process.
// TestStepdownAndLeaderElection tests raft leadership changes by forcing the
// current leader to step down and verifying that a new leader is elected
// successfully, while ensuring the cluster remains healthy throughout the
// process.
//
// NOTE: We do not run this test in parallel to avoid making other tests flaky
// during leader elections.
func TestStepdownAndLeaderElection(t *testing.T) {
v := blackbox.New(t)
// Verify we have a healthy raft cluster first
v.AssertRaftClusterHealthy()
// Wait for a healthy cluster
v.EventuallyClusterHealthyUnsealed(15 * time.Second)
// Check cluster size to determine expected behavior
nodeCount := v.GetClusterNodeCount()
nodeCount := v.MustGetClusterNodeCount()
t.Logf("Cluster has %d nodes", nodeCount)
// Get current leader before step down
@ -51,20 +35,17 @@ func TestStepdownAndLeaderElection(t *testing.T) {
// Force leader to step down
v.MustStepDownLeader()
// Wait for new leader election (with timeout)
// Use generous timeout to handle network latency and complex backend coordination
v.WaitForNewLeader(initialLeader, 300)
// Wait for a new leader to be active and the cluster to be healthy
v.EventuallyClusterHealthyUnsealed(1 * time.Minute)
// Verify cluster is still healthy after leader change/recovery
v.AssertRaftClusterHealthy()
// Get current leader before step down
newLeader := v.MustGetCurrentLeader()
t.Logf("New leader: %s", initialLeader)
// For multi-node clusters, verify new leader is different from initial leader
// For single-node clusters, just verify it's healthy again
newLeader := v.MustGetCurrentLeader()
if nodeCount > 1 {
if newLeader == initialLeader {
t.Fatalf("Expected new leader to be different from initial leader %s, got %s", initialLeader, newLeader)
}
require.NotEqual(t, initialLeader, newLeader, "Expected new leader to be different from initial leader")
t.Logf("Successfully elected new leader: %s (was: %s)", newLeader, initialLeader)
} else {
t.Logf("Single-node cluster successfully recovered with leader: %s", newLeader)

290
vault/external_tests/blackbox/plugins/aws/helpers.go Normal file
View file

@ -0,0 +1,290 @@
// Copyright IBM Corp. 2025, 2026
// SPDX-License-Identifier: BUSL-1.1
package aws
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/iam"
"github.com/aws/aws-sdk-go-v2/service/sts"
)
var ErrPolicyNotFound = errors.New("policy not found")
// getPolicyArnByName returns the ARN for a policy with the given name.
func getPolicyArnByName(ctx context.Context, iamClient *iam.Client, policyName string) (string, error) {
paginator := iam.NewListPoliciesPaginator(iamClient, &iam.ListPoliciesInput{Scope: "All"})
for paginator.HasMorePages() {
page, err := paginator.NextPage(ctx)
if err != nil {
return "", err
}
for _, p := range page.Policies {
if aws.ToString(p.PolicyName) == policyName {
return aws.ToString(p.Arn), nil
}
}
}
return "", ErrPolicyNotFound
}
// getRoleArnByName returns the ARN for a role with the given name.
func getRoleArnByName(ctx context.Context, iamClient *iam.Client, roleName string) (string, error) {
paginator := iam.NewListRolesPaginator(iamClient, &iam.ListRolesInput{})
for paginator.HasMorePages() {
page, err := paginator.NextPage(ctx)
if err != nil {
return "", err
}
for _, r := range page.Roles {
if aws.ToString(r.RoleName) == roleName {
return aws.ToString(r.Arn), nil
}
}
}
return "", fmt.Errorf("role %s not found", roleName)
}
// hasDemoUserPolicy determines whether or not the DemoUser policy has been
// assigned to the AWS credential idendity being used.
func hasDemoUserPolicy(ctx context.Context) (bool, error) {
cfg, err := config.LoadDefaultConfig(ctx)
if err != nil {
return false, err
}
iamClient := iam.NewFromConfig(cfg)
// Lookup DemoUser policy ARN
_, err = getPolicyArnByName(ctx, iamClient, "DemoUser")
if err == nil {
return true, nil
}
if errors.Is(err, ErrPolicyNotFound) {
return false, nil
}
return false, err
}
// createTestIAMUser creates a new IAM user with a unique name, attaches the DemoUser policy, and returns the user/access key info and AWS region.
func createTestIAMUser(t *testing.T) (
userName string,
accessKeyID string,
secretAccessKey string,
demoUserPolicyArn string,
assumedRoleArn string,
awsRegion string,
) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
cfg, err := config.LoadDefaultConfig(ctx)
if err != nil {
t.Fatalf("failed to load AWS config: %v", err)
}
awsRegion = cfg.Region
if awsRegion == "" {
t.Fatalf("AWS region is empty in config")
}
iamClient := iam.NewFromConfig(cfg)
stsClient := sts.NewFromConfig(cfg)
// Get current AWS account identity (for unique name)
caller, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{})
if err != nil {
t.Fatalf("failed to get caller identity: %v", err)
}
accountID := aws.ToString(caller.Account)
// Generate a random hex suffix for uniqueness
const randomSuffixByteLength = 4
suffix := make([]byte, randomSuffixByteLength)
if _, err := rand.Read(suffix); err != nil {
t.Fatalf("failed to generate random suffix: %v", err)
}
hexSuffix := hex.EncodeToString(suffix)
userName = fmt.Sprintf("demo-GitHubActions-%s-%s", accountID, hexSuffix)
// Lookup DemoUser policy ARN
demoUserPolicyArn, err = getPolicyArnByName(ctx, iamClient, "DemoUser")
if err != nil {
t.Fatalf("DemoUser policy not found: %v", err)
}
// Lookup vault-assumed-role-credentials-demo role ARN
assumedRoleArn, err = getRoleArnByName(ctx, iamClient, "vault-assumed-role-credentials-demo")
if err != nil {
t.Fatalf("vault-assumed-role-credentials-demo role not found: %v", err)
}
// Create IAM user
_, err = iamClient.CreateUser(ctx, &iam.CreateUserInput{
UserName: aws.String(userName),
PermissionsBoundary: aws.String(demoUserPolicyArn),
})
if err != nil {
t.Fatalf("failed to create IAM user: %v", err)
}
// Attach policy to user
_, err = iamClient.AttachUserPolicy(ctx, &iam.AttachUserPolicyInput{
UserName: aws.String(userName),
PolicyArn: aws.String(demoUserPolicyArn),
})
if err != nil {
t.Fatalf("failed to attach policy: %v", err)
}
// Create access key
keyOut, err := iamClient.CreateAccessKey(ctx, &iam.CreateAccessKeyInput{
UserName: aws.String(userName),
})
if err != nil {
t.Fatalf("failed to create access key: %v", err)
}
accessKeyID = aws.ToString(keyOut.AccessKey.AccessKeyId)
secretAccessKey = aws.ToString(keyOut.AccessKey.SecretAccessKey)
// IAM is eventually consistent; wait briefly before verifying the user is readable.
t.Logf("Verifying IAM user %s exists...", userName)
waitTime := 10 * time.Second
verifyDeadline := time.Now().Add(waitTime * 2)
var lastErr error
for time.Now().Before(verifyDeadline) {
time.Sleep(waitTime)
_, lastErr = iamClient.GetUser(ctx, &iam.GetUserInput{UserName: aws.String(userName)})
if lastErr == nil {
break
}
t.Logf("IAM user %q not readable yet; retrying: %v", userName, lastErr)
}
if lastErr != nil {
t.Fatalf("failed to verify IAM user %q: %v", userName, lastErr)
}
return userName, accessKeyID, secretAccessKey, demoUserPolicyArn, assumedRoleArn, awsRegion
}
// getAwsUsernameTemplate returns the username template string for Vault AWS config.
func getAwsUsernameTemplate(awsUserName string) string {
const prefix = `{{ if (eq .Type "STS") }}{{ printf "`
const stsSuffix = `-%s-%s" (random 20) (unix_time) | truncate 32 }}{{ else }}{{ printf "`
const iamUserSuffix = `-%s-%s" (unix_time) (random 20) | truncate 60 }}{{ end }}`
return prefix + awsUserName + stsSuffix + awsUserName + iamUserSuffix
}
// getAllowDescribeRegionsPolicy returns a policy document allowing ec2:DescribeRegions.
func getAllowDescribeRegionsPolicy() string {
return `{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["ec2:DescribeRegions"],
"Resource": ["*"]
}
]
}`
}
// deleteIAMUserByAccessKey deletes the IAM user that owns the given access key.
func deleteIAMUserByAccessKey(t *testing.T, targetAccessKeyID string) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
cfg, err := config.LoadDefaultConfig(ctx)
if err != nil {
t.Fatalf("failed to load AWS config: %v", err)
}
iamClient := iam.NewFromConfig(cfg)
paginator := iam.NewListUsersPaginator(iamClient, &iam.ListUsersInput{})
for paginator.HasMorePages() {
page, err := paginator.NextPage(ctx)
if err != nil {
t.Fatalf("failed to list IAM users: %v", err)
}
for _, user := range page.Users {
userName := aws.ToString(user.UserName)
if !strings.Contains(userName, "demo-GitHubActions") {
continue
}
// List all access keys for this user
keyPaginator := iam.NewListAccessKeysPaginator(iamClient, &iam.ListAccessKeysInput{
UserName: &userName,
})
for keyPaginator.HasMorePages() {
keyPage, err := keyPaginator.NextPage(ctx)
if err != nil {
t.Logf("warning: failed to list access keys for user %q: %v", userName, err)
continue
}
for _, key := range keyPage.AccessKeyMetadata {
accessKeyId := aws.ToString(key.AccessKeyId)
if accessKeyId == targetAccessKeyID {
// Found the user with the target access key. Detach managed policies first,
// then delete all access keys, then delete the user.
policyPaginator := iam.NewListAttachedUserPoliciesPaginator(iamClient, &iam.ListAttachedUserPoliciesInput{
UserName: &userName,
})
for policyPaginator.HasMorePages() {
policyPage, err := policyPaginator.NextPage(ctx)
if err != nil {
t.Logf("warning: failed to list attached policies for user %q: %v", userName, err)
continue
}
for _, policy := range policyPage.AttachedPolicies {
policyArn := aws.ToString(policy.PolicyArn)
if _, err := iamClient.DetachUserPolicy(ctx, &iam.DetachUserPolicyInput{
UserName: &userName,
PolicyArn: &policyArn,
}); err != nil {
t.Logf("warning: failed to detach policy %q from user %q: %v", policyArn, userName, err)
}
}
}
keyPaginator2 := iam.NewListAccessKeysPaginator(iamClient, &iam.ListAccessKeysInput{
UserName: &userName,
})
for keyPaginator2.HasMorePages() {
keyPage2, err := keyPaginator2.NextPage(ctx)
if err != nil {
t.Logf("warning: failed to list access keys for cleanup on user %q: %v", userName, err)
continue
}
for _, key2 := range keyPage2.AccessKeyMetadata {
accessKeyId2 := aws.ToString(key2.AccessKeyId)
if _, err := iamClient.DeleteAccessKey(ctx, &iam.DeleteAccessKeyInput{
UserName: &userName,
AccessKeyId: &accessKeyId2,
}); err != nil {
t.Logf("warning: failed to delete access key %q for user %q: %v", accessKeyId2, userName, err)
}
}
}
// Delete the user
if _, err := iamClient.DeleteUser(ctx, &iam.DeleteUserInput{
UserName: &userName,
}); err != nil {
t.Logf("warning: failed to delete user %q: %v", userName, err)
}
return
}
}
}
}
}
}

113
vault/external_tests/blackbox/plugins/aws/secrets_aws_test.go Normal file
View file

@ -0,0 +1,113 @@
// Copyright IBM Corp. 2025, 2026
// SPDX-License-Identifier: BUSL-1.1
package aws
import (
"fmt"
"os"
"testing"
"time"
"github.com/hashicorp/vault/api"
"github.com/hashicorp/vault/sdk/helper/testcluster/blackbox"
)
// TestAWS_GenerateNewUser verifies AWS secrets engine can generate IAM user credentials.
func TestAWS_GenerateNewUser(t *testing.T) {
t.Parallel()
v := blackbox.New(t)
accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
if accessKey == "" || secretKey == "" {
t.Log("AWS credentials not available - skipping AWS secrets engine test")
t.Skip("AWS credentials not available - skipping AWS secrets engine test")
}
hasDUP, err := hasDemoUserPolicy(t.Context())
if err != nil {
t.Fatal(err)
}
if !hasDUP {
// TODO: We can probably check for IAM permission instead of a policy. That
// would require rewriting the whole test though.
t.Skip("Skipping test as it requires a special DemoUser policy that is not assigned to the current AWS credentials")
}
t.Logf("Creating test IAM user via helpers.go...")
userName, tempAccessKeyId, tempSecretAccessKey, demoUserPolicyArn, _, _ := createTestIAMUser(t)
t.Logf("Created test IAM user: %s", userName)
var newAccessKey string
t.Cleanup(func() {
if newAccessKey != "" {
t.Logf("Cleanup: deleting IAM user created by Vault with access key: %s", newAccessKey)
deleteIAMUserByAccessKey(t, newAccessKey)
}
t.Logf("Cleanup: deleting IAM user by initial access key: %s", tempAccessKeyId)
deleteIAMUserByAccessKey(t, tempAccessKeyId)
})
path := fmt.Sprintf("aws-test-%d", time.Now().UnixNano())
t.Logf("Enabling AWS secrets engine at path: %s", path)
v.MustEnableSecretsEngine(path, &api.MountInput{Type: "aws"})
t.Logf("Configuring AWS secrets engine with root credentials and username template for user: %s", userName)
v.MustWrite(fmt.Sprintf("%s/config/root", path), map[string]any{
"access_key": tempAccessKeyId,
"secret_key": tempSecretAccessKey,
"region": "us-east-1",
"username_template": getAwsUsernameTemplate(userName),
})
roleName := "aws-enos-role"
t.Logf("Creating Vault AWS role: %s", roleName)
v.MustWrite(fmt.Sprintf("%s/roles/%s", path, roleName), map[string]any{
"credential_type": "iam_user",
"permissions_boundary_arn": demoUserPolicyArn,
"policy_document": getAllowDescribeRegionsPolicy(),
})
t.Logf("Reading and verifying AWS role configuration for role: %s", roleName)
roleResp := v.MustRead(fmt.Sprintf("%s/roles/%s", path, roleName))
if roleResp.Data == nil {
t.Fatal("Expected to read AWS role configuration")
}
t.Logf("Listing AWS roles at path: %s/roles", path)
rolesList := v.MustList(fmt.Sprintf("%s/roles", path))
if rolesList == nil || rolesList.Data == nil {
t.Fatal("No AWS roles created! (rolesList is nil or Data is nil)")
}
roleKeys, ok := rolesList.Data["keys"].([]interface{})
if !ok || len(roleKeys) == 0 {
t.Fatal("No AWS roles created! (rolesList.Data['keys'] is empty or not a slice)")
}
t.Logf("Found AWS roles: %v", roleKeys)
t.Logf("Reading root config to verify username template is set correctly")
rootUser := v.MustRead(fmt.Sprintf("%s/config/root", path))
if rootUser == nil || rootUser.Data == nil {
t.Fatalf("Expected to read root config, got nil: %#v", rootUser)
}
if val, ok := rootUser.Data["username_template"]; !ok || val == nil {
t.Fatalf("username_template missing in root config: %#v", rootUser)
}
t.Logf("Generating new credentials for IAM user using role: %s", roleName)
newUser := v.MustRead(fmt.Sprintf("%s/creds/%s", path, roleName))
if newUser == nil || newUser.Data == nil {
t.Fatalf("Failed to generate new credentials for IAM user: %s", roleName)
}
if val, ok := newUser.Data["access_key"]; !ok || val == nil || val == tempAccessKeyId {
t.Fatalf("The new access key is empty or is matching the old one: %v", val)
}
newAccessKey, ok = newUser.Data["access_key"].(string)
if !ok || newAccessKey == "" {
t.Fatalf("Could not extract access_key from new credentials: %v", newUser.Data["access_key"])
}
t.Logf("Captured Vault-created access key for cleanup: %s", newAccessKey)
}

2
vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_connection_config_test.go
View file

@ -36,7 +36,7 @@ func TestMongoDBConnectionConfigCRUDWorkflows(t *testing.T) {
// MongoDB database connection succeeds at database/config/{name}.
func testMongoDBConnectionConfigCreateBasic(t *testing.T, v *blackbox.Session) {
requireVaultEnv(t)
cleanup, connURL := PrepareTestContainer(t)
cleanup, connURL, _, _ := PrepareTestContainer(t)
defer cleanup()
mount := fmt.Sprintf("database-%s", sanitize(t.Name()))

76
vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_delete_test.go
View file

@ -4,7 +4,6 @@
package mongodb
import (
"strings"
"testing"
"github.com/hashicorp/vault/sdk/helper/testcluster/blackbox"
@ -45,9 +44,9 @@ func TestMongoDBStaticRoleDeleteWorkflows(t *testing.T) {
// testMongoDBStaticRoleDeleteExistingRole verifies deleting an existing
// static role succeeds and removes it from the list.
func testMongoDBStaticRoleDeleteExistingRole(t *testing.T, v *blackbox.Session) {
mount, connURL := setupMongoDBTest(t, v)
mount, _, dbName, client := setupMongoDBTest(t, v)
createMongoDBUser(t, connURL, deleteTestUsername, testInitialPassword)
createMongoDBUser(t, client, dbName, deleteTestUsername, testInitialPassword)
v.MustWrite(mount+"/static-roles/"+deleteTestRoleName, map[string]any{
"db_name": testConnectionName,
@ -62,20 +61,24 @@ func testMongoDBStaticRoleDeleteExistingRole(t *testing.T, v *blackbox.Session)
// Delete the role
v.MustDelete(mount + "/static-roles/" + deleteTestRoleName)
// Verify role is gone
_, err := v.Read(mount + "/static-roles/" + deleteTestRoleName)
if err == nil {
t.Fatal("expected error when reading deleted role")
// Verify role is gone - Read returns nil secret for non-existent paths
secret, err := v.Read(mount + "/static-roles/" + deleteTestRoleName)
if err != nil {
// Error is acceptable
} else if secret != nil {
t.Fatal("expected nil secret when reading deleted role")
}
// Verify role is not in list
list := v.MustList(mount + "/static-roles")
if list.Data["keys"] != nil {
keys := list.Data["keys"].([]interface{})
for _, k := range keys {
if k.(string) == deleteTestRoleName {
t.Fatalf("deleted role %s should not appear in list", deleteTestRoleName)
}
if list == nil || list.Data == nil || list.Data["keys"] == nil {
return
}
keys := list.Data["keys"].([]interface{})
for _, k := range keys {
if k.(string) == deleteTestRoleName {
t.Fatalf("deleted role %s should not appear in list", deleteTestRoleName)
}
}
}
@ -83,9 +86,9 @@ func testMongoDBStaticRoleDeleteExistingRole(t *testing.T, v *blackbox.Session)
// testMongoDBStaticRoleDeletePreventsCredentialAccess verifies that after
// deleting a static role, credentials can no longer be read.
func testMongoDBStaticRoleDeletePreventsCredentialAccess(t *testing.T, v *blackbox.Session) {
mount, connURL := setupMongoDBTest(t, v)
mount, _, dbName, client := setupMongoDBTest(t, v)
createMongoDBUser(t, connURL, deleteTestUsername, testInitialPassword)
createMongoDBUser(t, client, dbName, deleteTestUsername, testInitialPassword)
v.MustWrite(mount+"/static-roles/"+deleteTestRoleName, map[string]any{
"db_name": testConnectionName,
@ -95,24 +98,27 @@ func testMongoDBStaticRoleDeletePreventsCredentialAccess(t *testing.T, v *blackb
// Verify credentials can be read before deletion
creds := v.MustReadRequired(mount + "/static-creds/" + deleteTestRoleName)
v.AssertSecret(creds).Data().HasKey("username").HasKey("password")
v.AssertSecret(creds).Data().HasKeyExists("username")
v.AssertSecret(creds).Data().HasKeyExists("password")
// Delete the role
v.MustDelete(mount + "/static-roles/" + deleteTestRoleName)
// Verify credentials can no longer be read
_, err := v.Read(mount + "/static-creds/" + deleteTestRoleName)
if err == nil {
t.Fatal("expected error when reading credentials for deleted role")
// Verify credentials can no longer be read - Read returns nil secret for deleted roles
secret, err := v.Read(mount + "/static-creds/" + deleteTestRoleName)
if err != nil {
// Error is acceptable
} else if secret != nil {
t.Fatal("expected nil secret when reading credentials for deleted role")
}
}
// testMongoDBStaticRoleDeleteIsIdempotent verifies that deleting a role
// multiple times succeeds without error.
func testMongoDBStaticRoleDeleteIsIdempotent(t *testing.T, v *blackbox.Session) {
mount, connURL := setupMongoDBTest(t, v)
mount, _, dbName, client := setupMongoDBTest(t, v)
createMongoDBUser(t, connURL, deleteTestUsername, testInitialPassword)
createMongoDBUser(t, client, dbName, deleteTestUsername, testInitialPassword)
v.MustWrite(mount+"/static-roles/"+deleteTestRoleName, map[string]any{
"db_name": testConnectionName,
@ -123,10 +129,12 @@ func testMongoDBStaticRoleDeleteIsIdempotent(t *testing.T, v *blackbox.Session)
// Delete the role first time
v.MustDelete(mount + "/static-roles/" + deleteTestRoleName)
// Verify role is gone
_, err := v.Read(mount + "/static-roles/" + deleteTestRoleName)
if err == nil {
t.Fatal("expected error when reading deleted role")
// Verify role is gone - Read returns nil secret for non-existent paths
secret, err := v.Read(mount + "/static-roles/" + deleteTestRoleName)
if err != nil {
// Error is acceptable
} else if secret != nil {
t.Fatal("expected nil secret when reading deleted role")
}
// Delete the role again - should succeed (idempotent)
@ -139,18 +147,18 @@ func testMongoDBStaticRoleDeleteIsIdempotent(t *testing.T, v *blackbox.Session)
// testMongoDBStaticRoleDeleteNonExistentRole verifies that attempting to
// delete a non-existent role succeeds (idempotent behavior).
func testMongoDBStaticRoleDeleteNonExistentRole(t *testing.T, v *blackbox.Session) {
mount, _ := setupMongoDBTest(t, v)
mount, _, _, _ := setupMongoDBTest(t, v)
// Attempt to delete a role that was never created
v.MustDelete(mount + "/static-roles/never-existed-role")
// Verify it's still not there
_, err := v.Read(mount + "/static-roles/never-existed-role")
if err == nil {
t.Fatal("expected error when reading non-existent role")
// Verify it's still not there - Read returns nil secret for non-existent paths
secret, err := v.Read(mount + "/static-roles/never-existed-role")
if err != nil {
// Error is acceptable
return
}
if !strings.Contains(err.Error(), "not found") && !strings.Contains(err.Error(), "no value") {
t.Logf("Note: error message may not clearly indicate 'not found': %v", err)
if secret != nil {
t.Fatal("expected nil secret when reading non-existent role")
}
}

35
vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_read_test.go
View file

@ -45,9 +45,9 @@ func TestMongoDBStaticRoleReadWorkflows(t *testing.T) {
// testMongoDBStaticRoleReadReturnsConfiguration verifies reading a static role
// returns its configuration without sensitive data.
func testMongoDBStaticRoleReadReturnsConfiguration(t *testing.T, v *blackbox.Session) {
mount, connURL := setupMongoDBTest(t, v)
mount, _, dbName, client := setupMongoDBTest(t, v)
createMongoDBUser(t, connURL, readTestUsername, testInitialPassword)
createMongoDBUser(t, client, dbName, readTestUsername, testInitialPassword)
v.MustWrite(mount+"/static-roles/"+readTestRoleName, map[string]any{
"db_name": testConnectionName,
@ -68,26 +68,30 @@ func testMongoDBStaticRoleReadReturnsConfiguration(t *testing.T, v *blackbox.Ses
}
// Verify last_vault_rotation is present
v.AssertSecret(role).Data().HasKey("last_vault_rotation")
v.AssertSecret(role).Data().HasKeyExists("last_vault_rotation")
}
// testMongoDBStaticRoleReadNonExistentRole verifies reading a non-existent
// static role returns an appropriate error.
// static role returns nil secret.
func testMongoDBStaticRoleReadNonExistentRole(t *testing.T, v *blackbox.Session) {
mount, _ := setupMongoDBTest(t, v)
mount, _, _, _ := setupMongoDBTest(t, v)
_, err := v.Read(mount + "/static-roles/nonexistent-role")
if err == nil {
t.Fatal("expected error when reading non-existent role")
secret, err := v.Read(mount + "/static-roles/nonexistent-role")
if err != nil {
// Error is acceptable
return
}
if secret != nil {
t.Fatal("expected nil secret when reading non-existent role")
}
}
// testMongoDBStaticRoleReadAfterUpdate verifies reading a static role after
// updating it returns the updated configuration.
func testMongoDBStaticRoleReadAfterUpdate(t *testing.T, v *blackbox.Session) {
mount, connURL := setupMongoDBTest(t, v)
mount, _, dbName, client := setupMongoDBTest(t, v)
createMongoDBUser(t, connURL, readTestUsername, testInitialPassword)
createMongoDBUser(t, client, dbName, readTestUsername, testInitialPassword)
v.MustWrite(mount+"/static-roles/"+readTestRoleName, map[string]any{
"db_name": testConnectionName,
@ -98,9 +102,12 @@ func testMongoDBStaticRoleReadAfterUpdate(t *testing.T, v *blackbox.Session) {
role1 := v.MustReadRequired(mount + "/static-roles/" + readTestRoleName)
v.AssertSecret(role1).Data().HasKey("rotation_period", float64(testRotationPeriod))
// Update rotation period
// Update rotation period. Static role updates must continue to include the
// existing username to avoid being treated as a username mutation.
newRotationPeriod := 7200
v.MustWrite(mount+"/static-roles/"+readTestRoleName, map[string]any{
"db_name": testConnectionName,
"username": readTestUsername,
"rotation_period": newRotationPeriod,
})
@ -114,13 +121,13 @@ func testMongoDBStaticRoleReadAfterUpdate(t *testing.T, v *blackbox.Session) {
// testMongoDBStaticRoleListMultipleRoles verifies LIST /static-roles
// returns all configured role names.
func testMongoDBStaticRoleListMultipleRoles(t *testing.T, v *blackbox.Session) {
mount, connURL := setupMongoDBTest(t, v)
mount, _, dbName, client := setupMongoDBTest(t, v)
// Create multiple static roles
roles := []string{"read-role-1", "read-role-2", "read-role-3"}
for i, roleName := range roles {
username := fmt.Sprintf("listuser%d", i+1)
createMongoDBUser(t, connURL, username, testInitialPassword)
createMongoDBUser(t, client, dbName, username, testInitialPassword)
v.MustWrite(mount+"/static-roles/"+roleName, map[string]any{
"db_name": testConnectionName,
@ -130,7 +137,7 @@ func testMongoDBStaticRoleListMultipleRoles(t *testing.T, v *blackbox.Session) {
}
list := v.MustList(mount + "/static-roles")
v.AssertSecret(list).Data().HasKey("keys")
v.AssertSecret(list).Data().HasKeyExists("keys")
keys := list.Data["keys"].([]interface{})
if len(keys) != 3 {

219
vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_static_roles_test.go
View file

@ -4,30 +4,23 @@
package mongodb
import (
"context"
"fmt"
"strings"
"testing"
"time"
"github.com/hashicorp/vault/api"
"github.com/hashicorp/vault/sdk/helper/testcluster/blackbox"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
const (
testConnectionName = "my-mongodb-db"
testStaticRoleName = "my-static-role"
testUsername = "staticuser1"
testInitialPassword = "initialpass"
testRotationPeriod = 86400 // 24 hours in seconds
mongoConnectTimeout = 10 * time.Second
)
// TestMongoDBStaticRoleWorkflows runs all MongoDB static role workflow tests.
func TestMongoDBStaticRoleWorkflows(t *testing.T) {
t.Parallel()
t.Run("CreateBasic", func(t *testing.T) {
t.Parallel()
v := blackbox.New(t)
@ -57,9 +50,9 @@ func TestMongoDBStaticRoleWorkflows(t *testing.T) {
// testMongoDBStaticRoleCreateBasic verifies that creating a basic static role
// succeeds with required fields.
func testMongoDBStaticRoleCreateBasic(t *testing.T, v *blackbox.Session) {
mount, connURL := setupMongoDBTest(t, v)
mount, _, dbName, client := setupMongoDBTest(t, v)
createMongoDBUser(t, connURL, testUsername, testInitialPassword)
createMongoDBUser(t, client, dbName, testUsername, testInitialPassword)
v.MustWrite(mount+"/static-roles/"+testStaticRoleName, map[string]any{
"db_name": testConnectionName,
@ -85,9 +78,9 @@ func testMongoDBStaticRoleCreateBasic(t *testing.T, v *blackbox.Session) {
// testMongoDBStaticRoleManualRotation verifies manually rotating a static
// role's credentials succeeds.
func testMongoDBStaticRoleManualRotation(t *testing.T, v *blackbox.Session) {
mount, connURL := setupMongoDBTest(t, v)
mount, credVerifyURL, dbName, client := setupMongoDBTest(t, v)
createMongoDBUser(t, connURL, testUsername, testInitialPassword)
createMongoDBUser(t, client, dbName, testUsername, testInitialPassword)
v.MustWrite(mount+"/static-roles/"+testStaticRoleName, map[string]any{
"db_name": testConnectionName,
@ -111,7 +104,7 @@ func testMongoDBStaticRoleManualRotation(t *testing.T, v *blackbox.Session) {
t.Fatal("expected password to change after rotation")
}
verifyMongoDBCredentials(t, connURL, testUsername, password2)
verifyMongoDBCredentials(t, credVerifyURL, testUsername, password2)
}
// TODO: testMongoDBStaticRoleAutomaticRotation - Test automatic rotation with short period
@ -121,9 +114,9 @@ func testMongoDBStaticRoleManualRotation(t *testing.T, v *blackbox.Session) {
// testMongoDBStaticRoleReadCredentials verifies reading static credentials
// returns the current password.
func testMongoDBStaticRoleReadCredentials(t *testing.T, v *blackbox.Session) {
mount, connURL := setupMongoDBTest(t, v)
mount, credVerifyURL, dbName, client := setupMongoDBTest(t, v)
createMongoDBUser(t, connURL, testUsername, testInitialPassword)
createMongoDBUser(t, client, dbName, testUsername, testInitialPassword)
v.MustWrite(mount+"/static-roles/"+testStaticRoleName, map[string]any{
"db_name": testConnectionName,
@ -142,13 +135,13 @@ func testMongoDBStaticRoleReadCredentials(t *testing.T, v *blackbox.Session) {
t.Fatal("expected non-empty password")
}
verifyMongoDBCredentials(t, connURL, testUsername, password)
verifyMongoDBCredentials(t, credVerifyURL, testUsername, password)
}
// testMongoDBStaticRoleRequiresUsername verifies creating a static role
// without username fails.
func testMongoDBStaticRoleRequiresUsername(t *testing.T, v *blackbox.Session) {
mount, _ := setupMongoDBTest(t, v)
mount, _, _, _ := setupMongoDBTest(t, v)
_, err := v.Client.Logical().Write(mount+"/static-roles/"+testStaticRoleName, map[string]any{
"db_name": testConnectionName,
@ -163,193 +156,3 @@ func testMongoDBStaticRoleRequiresUsername(t *testing.T, v *blackbox.Session) {
t.Fatalf("expected error message to mention 'username', got: %v", err)
}
}
// setupMongoDBTest performs common test setup: creates container, enables mount, configures connection.
// Returns mount path and connection URL.
func setupMongoDBTest(t *testing.T, v *blackbox.Session) (string, string) {
t.Helper()
requireVaultEnv(t)
cleanup, connURL := PrepareTestContainer(t)
t.Cleanup(cleanup)
mount := fmt.Sprintf("database-%s", sanitize(t.Name()))
v.MustEnableSecretsEngine(mount, &api.MountInput{Type: "database"})
v.MustWrite(
mount+"/config/"+testConnectionName,
mongoConnectionConfigPayload(connURL, "*", false),
)
return mount, connURL
}
// createMongoDBUser creates a MongoDB user for testing static roles.
func createMongoDBUser(t *testing.T, connURL, username, password string) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), mongoConnectTimeout)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(connURL))
if err != nil {
t.Fatalf("failed to connect to MongoDB: %v", err)
}
defer client.Disconnect(ctx)
db := client.Database("admin")
err = db.RunCommand(ctx, bson.D{
{Key: "createUser", Value: username},
{Key: "pwd", Value: password},
{Key: "roles", Value: bson.A{
bson.D{
{Key: "role", Value: "readWrite"},
{Key: "db", Value: "admin"},
},
}},
}).Err()
if err != nil {
t.Fatalf("failed to create MongoDB user: %v", err)
}
t.Logf("Created MongoDB user: %s", username)
}
// verifyMongoDBCredentials verifies that the given credentials work for MongoDB.
func verifyMongoDBCredentials(t *testing.T, connURL, username, password string) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), mongoConnectTimeout)
defer cancel()
// Replace credentials in connection URL
u, err := parseMongoURL(connURL)
if err != nil {
t.Fatalf("failed to parse connection URL: %v", err)
}
u.User = username
u.Password = password
testURL := buildMongoURL(u)
client, err := mongo.Connect(ctx, options.Client().ApplyURI(testURL))
if err != nil {
t.Fatalf("failed to connect with credentials: %v", err)
}
defer client.Disconnect(ctx)
if err := client.Ping(ctx, nil); err != nil {
t.Fatalf("failed to ping with credentials: %v", err)
}
t.Logf("Verified MongoDB credentials for user: %s", username)
}
// mongoURL represents a parsed MongoDB connection URL.
type mongoURL struct {
Scheme string
User string
Password string
Host string
Database string
Options string
}
// parseMongoURL parses a MongoDB connection URL into components.
func parseMongoURL(connURL string) (*mongoURL, error) {
// Simple parser for mongodb:// URLs
// Format: mongodb://user:pass@host/database?options
u := &mongoURL{Scheme: "mongodb"}
// Remove scheme
rest := connURL
if len(rest) > 10 && rest[:10] == "mongodb://" {
rest = rest[10:]
}
// Extract user:pass if present
atIdx := -1
for i, c := range rest {
if c == '@' {
atIdx = i
break
}
}
if atIdx > 0 {
userPass := rest[:atIdx]
rest = rest[atIdx+1:]
colonIdx := -1
for i, c := range userPass {
if c == ':' {
colonIdx = i
break
}
}
if colonIdx > 0 {
u.User = userPass[:colonIdx]
u.Password = userPass[colonIdx+1:]
}
}
// Extract host and database
slashIdx := -1
for i, c := range rest {
if c == '/' {
slashIdx = i
break
}
}
if slashIdx > 0 {
u.Host = rest[:slashIdx]
rest = rest[slashIdx+1:]
// Extract database and options
qIdx := -1
for i, c := range rest {
if c == '?' {
qIdx = i
break
}
}
if qIdx > 0 {
u.Database = rest[:qIdx]
u.Options = rest[qIdx+1:]
} else {
u.Database = rest
}
} else {
u.Host = rest
}
return u, nil
}
// buildMongoURL builds a MongoDB connection URL from components.
func buildMongoURL(u *mongoURL) string {
url := u.Scheme + "://"
if u.User != "" {
url += u.User
if u.Password != "" {
url += ":" + u.Password
}
url += "@"
}
url += u.Host
if u.Database != "" {
url += "/" + u.Database
}
if u.Options != "" {
url += "?" + u.Options
}
return url
}

273
vault/external_tests/blackbox/plugins/database/mongodb/secret_mongodb_test_helper.go
View file

@ -15,7 +15,10 @@ import (
"testing"
"time"
"github.com/hashicorp/vault/api"
"github.com/hashicorp/vault/sdk/helper/docker"
"github.com/hashicorp/vault/sdk/helper/testcluster/blackbox"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
@ -25,6 +28,10 @@ const (
defaultMongoVersion = "7.0"
defaultMongoUser = "admin"
defaultMongoPass = "secret"
testConnectionName = "my-mongodb-db"
testInitialPassword = "initialpass"
testRotationPeriod = 86400 // 24 hours in seconds
)
// defaultRunOpts returns default Docker run options for MongoDB container
@ -86,12 +93,14 @@ func sanitize(name string) string {
return out
}
// PrepareTestContainer starts a MongoDB container for testing
// Returns cleanup function and connection URL
// If MONGO_URL environment variable is set, uses that instead of starting a container
func PrepareTestContainer(t *testing.T) (func(), string) {
_, cleanup, connURL, _ := prepareTestContainer(t, defaultRunOpts(t), defaultMongoPass, false, true)
return cleanup, connURL
// PrepareTestContainer starts a MongoDB container for testing.
// Returns cleanup function, Vault connection URL, test runner connection URL,
// and the generated per-test database name.
// If MONGO_URL environment variable is set, uses that instead of starting a container.
// In CI: vaultURL is private (same VPC), testRunnerURL is public (different VPC)
func PrepareTestContainer(t *testing.T) (func(), string, string, string) {
_, cleanup, vaultURL, testRunnerURL, dbName := prepareTestContainer(t, defaultRunOpts(t), defaultMongoPass, false, true)
return cleanup, vaultURL, testRunnerURL, dbName
}
// prepareTestContainer is the internal function that handles container setup
@ -102,7 +111,7 @@ func prepareTestContainer(
password string,
addSuffix bool,
forceLocalAddr bool,
) (*docker.Runner, func(), string, string) {
) (*docker.Runner, func(), string, string, string) {
requireVaultEnv(t)
// Check for external MongoDB URL
@ -115,9 +124,20 @@ func prepareTestContainer(
vaultMongoURL = envMongoURL
}
// Create unique database for this test
dbName := fmt.Sprintf("test_%s_%d", sanitize(t.Name()), time.Now().Unix())
testURL := replaceDatabase(vaultMongoURL, dbName)
// Create unique database for this test (max 63 chars for MongoDB)
sanitized := sanitize(t.Name())
timestamp := time.Now().Unix()
// Format: test_{name}_{ts} - ensure total length <= 63
// Reserve 5 for "test_", 10 for timestamp, 1 for underscore = 16 chars overhead
maxNameLen := 63 - 16
if len(sanitized) > maxNameLen {
sanitized = sanitized[:maxNameLen]
}
dbName := fmt.Sprintf("test_%s_%d", sanitized, timestamp)
// Vault uses private URL (same VPC), test runner uses public URL (different VPC)
vaultURL := replaceDatabase(vaultMongoURL, dbName)
testRunnerURL := replaceDatabase(envMongoURL, dbName)
// Create the database (test runner uses public URL)
if err := createDatabase(t, envMongoURL, dbName); err != nil {
@ -128,7 +148,7 @@ func prepareTestContainer(
dropDatabase(t, envMongoURL, dbName)
}
return nil, cleanup, testURL, ""
return nil, cleanup, vaultURL, testRunnerURL, dbName
}
// Start Docker container
@ -142,13 +162,12 @@ func prepareTestContainer(
// Retry StartNewService with small delays to handle port mapping timing
var svc *docker.Service
var containerID string
for attempt := 0; attempt < 5; attempt++ {
if attempt > 0 {
time.Sleep(time.Duration(attempt) * 500 * time.Millisecond)
}
svc, containerID, err = runner.StartNewService(context.Background(), addSuffix, forceLocalAddr, connectMongoDB(password))
svc, _, err = runner.StartNewService(context.Background(), addSuffix, forceLocalAddr, connectMongoDB(password))
if err == nil {
break
}
@ -167,8 +186,16 @@ func prepareTestContainer(
connURL := svc.Config.URL().String()
// Create unique database for this test
dbName := fmt.Sprintf("test_%s_%d", sanitize(t.Name()), time.Now().Unix())
// Create unique database for this test (max 63 chars for MongoDB)
sanitized := sanitize(t.Name())
timestamp := time.Now().Unix()
// Format: test_{name}_{ts} - ensure total length <= 63
// Reserve 5 for "test_", 10 for timestamp, 1 for underscore = 16 chars overhead
maxNameLen := 63 - 16
if len(sanitized) > maxNameLen {
sanitized = sanitized[:maxNameLen]
}
dbName := fmt.Sprintf("test_%s_%d", sanitized, timestamp)
testURL := replaceDatabase(connURL, dbName)
// Create the database
@ -182,7 +209,8 @@ func prepareTestContainer(
svc.Cleanup()
}
return runner, cleanup, testURL, containerID
// For Docker, both Vault and test runner use the same URL (localhost)
return runner, cleanup, testURL, testURL, dbName
}
// connectMongoDB returns a ServiceAdapter that connects to MongoDB
@ -232,21 +260,58 @@ func replaceDatabase(connURL, dbName string) string {
return connURL
}
u.Path = "/" + dbName
u.Path = dbName
// Ensure authSource=admin so authentication works against admin database
// even when connecting to a different database
q := u.Query()
if q.Get("authSource") == "" {
q.Set("authSource", "admin")
u.RawQuery = q.Encode()
}
return u.String()
}
// createDatabase creates a new database in MongoDB
// MongoDB creates databases lazily, so we create a collection to ensure it exists
// Uses 60-second timeout with retry logic for CI environments with network latency.
func createDatabase(t *testing.T, connURL, dbName string) error {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(connURL))
// Retry connection with backoff for CI network latency
var client *mongo.Client
var err error
deadline := time.Now().Add(60 * time.Second)
for time.Now().Before(deadline) {
// Set shorter server selection timeout (10s) to allow multiple retries within 60s window
clientOpts := options.Client().
ApplyURI(connURL).
SetServerSelectionTimeout(10 * time.Second).
SetConnectTimeout(10 * time.Second)
client, err = mongo.Connect(ctx, clientOpts)
if err != nil {
time.Sleep(2 * time.Second)
continue
}
// Verify connection with ping
if err = client.Ping(ctx, nil); err != nil {
client.Disconnect(ctx)
time.Sleep(2 * time.Second)
continue
}
break
}
if err != nil {
return fmt.Errorf("failed to connect to MongoDB: %w", err)
return fmt.Errorf("failed to connect to MongoDB after retries: %w", err)
}
defer client.Disconnect(ctx)
@ -261,15 +326,43 @@ func createDatabase(t *testing.T, connURL, dbName string) error {
}
// dropDatabase drops a database from MongoDB
// Uses 60-second timeout with retry logic for CI environments with network latency.
func dropDatabase(t *testing.T, connURL, dbName string) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(connURL))
// Retry connection with backoff for CI network latency
var client *mongo.Client
var err error
deadline := time.Now().Add(60 * time.Second)
for time.Now().Before(deadline) {
// Set shorter server selection timeout (10s) to allow multiple retries within 60s window
clientOpts := options.Client().
ApplyURI(connURL).
SetServerSelectionTimeout(10 * time.Second).
SetConnectTimeout(10 * time.Second)
client, err = mongo.Connect(ctx, clientOpts)
if err != nil {
time.Sleep(2 * time.Second)
continue
}
// Verify connection with ping
if err = client.Ping(ctx, nil); err != nil {
client.Disconnect(ctx)
time.Sleep(2 * time.Second)
continue
}
break
}
if err != nil {
t.Logf("Warning: failed to connect for cleanup: %v", err)
t.Logf("Warning: failed to connect for cleanup after retries: %v", err)
return
}
defer client.Disconnect(ctx)
@ -291,3 +384,137 @@ func mongoConnectionConfigPayload(connURL, allowedRoles string, verifyConnection
"verify_connection": verifyConnection,
}
}
// setupMongoDBTest performs common test setup: creates container, enables mount, configures connection.
// Returns mount path, connection URL for verifying credentials, the generated
// per-test database name, and a reusable MongoDB client for test operations.
// In CI: Vault uses private URL (same VPC), test runner uses public URL (different VPC)
func setupMongoDBTest(t *testing.T, v *blackbox.Session) (string, string, string, *mongo.Client) {
t.Helper()
requireVaultEnv(t)
cleanup, vaultURL, testRunnerURL, dbName := PrepareTestContainer(t)
t.Cleanup(cleanup)
mount := fmt.Sprintf("database-%s", sanitize(t.Name()))
v.MustEnableSecretsEngine(mount, &api.MountInput{Type: "database"})
// Vault uses private URL (same VPC as MongoDB in CI)
v.MustWrite(
mount+"/config/"+testConnectionName,
mongoConnectionConfigPayload(vaultURL, "*", false),
)
// Test runner uses public URL (different VPC in CI, needs public access)
client := getMongoClient(t, testRunnerURL)
t.Cleanup(func() {
if err := client.Disconnect(context.Background()); err != nil {
t.Logf("Warning: failed to disconnect MongoDB client: %v", err)
}
})
// Return testRunnerURL for credential verification (test runner needs public access)
return mount, testRunnerURL, dbName, client
}
// getMongoClient creates a MongoDB client with optimized settings for CI environments.
// Uses connection pooling and shorter timeouts for faster failure detection.
func getMongoClient(t *testing.T, connURL string) *mongo.Client {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
clientOpts := options.Client().
ApplyURI(connURL).
SetServerSelectionTimeout(10 * time.Second).
SetConnectTimeout(10 * time.Second).
SetMaxPoolSize(10).
SetMinPoolSize(2)
client, err := mongo.Connect(ctx, clientOpts)
if err != nil {
t.Fatalf("failed to create MongoDB client: %v", err)
}
// Verify connection
if err := client.Ping(ctx, nil); err != nil {
client.Disconnect(ctx)
t.Fatalf("failed to ping MongoDB: %v", err)
}
return client
}
// createMongoDBUser creates a MongoDB user for testing static roles.
// Uses the provided client connection to avoid connection overhead.
// The user is created in the database identified by dbName, while auth
// continues to use authSource=admin.
func createMongoDBUser(t *testing.T, client *mongo.Client, dbName, username, password string) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
db := client.Database(dbName)
// First, try to drop the user if it exists (for test cleanup/retry scenarios).
_ = db.RunCommand(ctx, bson.D{{Key: "dropUser", Value: username}}).Err()
err := db.RunCommand(ctx, bson.D{
{Key: "createUser", Value: username},
{Key: "pwd", Value: password},
{Key: "roles", Value: bson.A{
bson.D{
{Key: "role", Value: "readWrite"},
{Key: "db", Value: dbName},
},
}},
}).Err()
if err != nil {
t.Fatalf("failed to create MongoDB user: %v", err)
}
t.Logf("Created MongoDB user: %s in database %s", username, dbName)
}
// verifyMongoDBCredentials verifies that the given credentials work for MongoDB.
// Creates a new client with the provided credentials to test authentication.
func verifyMongoDBCredentials(t *testing.T, connURL, username, password string) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
// Replace credentials in connection URL
u, err := url.Parse(connURL)
if err != nil {
t.Fatalf("failed to parse connection URL: %v", err)
}
u.User = url.UserPassword(username, password)
// Remove the auth source so we verify with the default database
values := u.Query()
if values.Get("authSource") != "" {
values.Del("authSource")
u.RawQuery = values.Encode()
}
// Create client with new credentials
clientOpts := options.Client().
ApplyURI(u.String()).
SetServerSelectionTimeout(10 * time.Second).
SetConnectTimeout(10 * time.Second)
client, err := mongo.Connect(ctx, clientOpts)
if err != nil {
t.Fatalf("failed to connect to mongodb: %v: url %s", err, u.String())
}
defer client.Disconnect(ctx)
if err := client.Ping(ctx, nil); err != nil {
t.Fatalf("failed to ping mongodb: %v: url %s", err, u.String())
}
t.Logf("Verified MongoDB credentials for user: %s", username)
}

4
vault/external_tests/blackbox/plugins/database/postgres/secret_postgresql_connection_config_failover_test.go
View file

@ -74,10 +74,6 @@ func testPostgreSQLConnectionConfigCreateMultiHostPrimaryUnavailable(t *testing.
writeAndAssertPostgresConfig(t, v, path, templatedConnectionURL(failoverConnURL))
}
func templatedConnectionURL(connURL string) string {
return strings.Replace(connURL, "postgres:secret@", "{{username}}:{{password}}@", 1)
}
func writeAndAssertPostgresConfig(t *testing.T, v *blackbox.Session, path, connURL string) {
t.Helper()

4
vault/external_tests/blackbox/plugins/database/postgres/secret_postgresql_connection_test_helper.go
View file

@ -174,3 +174,7 @@ func connectPostgres(password string, useFallback bool) docker.ServiceAdapter {
return docker.NewServiceURL(u), nil
}
}
func templatedConnectionURL(connURL string) string {
return strings.Replace(connURL, "postgres:secret@", "{{username}}:{{password}}@", 1)
}

60
vault/external_tests/blackbox/plugins/ldap/helpers.go
View file

@ -36,20 +36,19 @@ func setupLDAPSecretsEngine(t *testing.T, v *blackbox.Session, mount string) {
v.MustEnableSecretsEngine(mount, &api.MountInput{Type: "ldap"})
// Configure using environment variables set by ldap.tf
ldapServer := os.Getenv("LDAP_SERVER")
ldapPort := os.Getenv("LDAP_PORT")
ldapURLPrivate := os.Getenv("LDAP_URL_PRIVATE")
ldapBindDN := os.Getenv("LDAP_BIND_DN")
ldapBindPass := os.Getenv("LDAP_BIND_PASS")
ldapUsername := os.Getenv("LDAP_USERNAME") // "enos"
if ldapServer == "" || ldapPort == "" || ldapBindDN == "" || ldapBindPass == "" {
if ldapURLPrivate == "" || ldapBindDN == "" || ldapBindPass == "" || ldapUsername == "" {
t.Fatal("Required LDAP environment variables not set")
}
v.MustWrite(mount+"/config", map[string]any{
"binddn": ldapBindDN,
"bindpass": ldapBindPass,
"url": "ldap://" + ldapServer + ":" + ldapPort,
"url": ldapURLPrivate,
"userdn": "ou=users,dc=" + ldapUsername + ",dc=com",
})
}
@ -110,14 +109,14 @@ func WriteLibrarySetWithRetry(t *testing.T, v *blackbox.Session, path string, da
// checkLDAPUserExists verifies if a user exists in the LDAP directory
func checkLDAPUserExists(t *testing.T, username string) bool {
ldapServer := os.Getenv("LDAP_SERVER")
ldapPort := os.Getenv("LDAP_PORT")
ldapURLPrivate := os.Getenv("LDAP_URL_PRIVATE")
ldapUsername := os.Getenv("LDAP_USERNAME")
ldapAdminPw := os.Getenv("LDAP_ADMIN_PW")
cmd := exec.Command("ldapsearch",
cmd := exec.Command(
"ldapsearch",
"-x",
"-H", "ldap://"+ldapServer+":"+ldapPort,
"-H", ldapURLPrivate,
"-b", "ou=users,dc="+ldapUsername+",dc=com",
"-D", "cn=admin,dc="+ldapUsername+",dc=com",
"-w", ldapAdminPw,
@ -142,7 +141,7 @@ func getLDIFPath(filename string) string {
// skipIfLDAPNotAvailable skips the test if LDAP configuration is not available
func skipIfLDAPNotAvailable(t *testing.T) {
if os.Getenv("LDAP_SERVER") == "" {
if os.Getenv("LDAP_URL_PRIVATE") == "" {
t.Skip("LDAP configuration not available - skipping LDAP test")
}
}
@ -171,7 +170,8 @@ func waitForLDAP(t *testing.T, config *LDAPDomainConfig, timeout time.Duration)
for time.Now().Before(deadline) {
attempt++
cmd := exec.Command("ldapsearch",
cmd := exec.Command(
"ldapsearch",
"-x",
"-H", config.SetupURL, // Use public IP for connectivity checks from GitHub runner
"-D", config.BindDN,
@ -217,18 +217,17 @@ func PrepareTestLDAPDomain(
t.Helper()
// Get LDAP connection info from environment (set by Enos)
// LDAP_SERVER: private IP for Vault operations (runs on Vault leader)
// LDAP_SERVER_PUBLIC: public IP for setup operations (runs from GitHub runner)
ldapServer := os.Getenv("LDAP_SERVER")
ldapServerPublic := os.Getenv("LDAP_SERVER_PUBLIC")
ldapPort := os.Getenv("LDAP_PORT")
// LDAP_URL_PRIVATE: ldap url to ldap internal listerning Vault operations (runs on Vault leader)
// LDAP_SERVER_PUBLIC: public URL ldap external listener for setup operations (runs from GitHub runner)
ldapURLPrivate := os.Getenv("LDAP_URL_PRIVATE")
ldapURLPublic := os.Getenv("LDAP_URL_PUBLIC")
ldapBindDN := os.Getenv("LDAP_BIND_DN")
ldapBindPass := os.Getenv("LDAP_BIND_PASS")
t.Logf("LDAP Environment: SERVER=%s SERVER_PUBLIC=%s PORT=%s BIND_DN=%s BIND_PASS=%s",
ldapServer, ldapServerPublic, ldapPort, ldapBindDN, maskPassword(ldapBindPass))
t.Logf("LDAP Environment: LDAP_URL_PRIVATE=%s LDAP_URL_PUBLIC=%s LDAP_BIND_DN=%s LDAP_BIND_PASS=%s",
ldapURLPrivate, ldapURLPublic, ldapBindDN, maskPassword(ldapBindPass))
if ldapServer == "" || ldapServerPublic == "" || ldapPort == "" || ldapBindDN == "" || ldapBindPass == "" {
if ldapURLPrivate == "" || ldapURLPublic == "" || ldapBindDN == "" || ldapBindPass == "" {
err := fmt.Errorf("LDAP environment variables not set")
if requireIsolation {
return nil, nil, fmt.Errorf("%w - required in CI", err)
@ -242,7 +241,7 @@ func PrepareTestLDAPDomain(
baseDN := "dc=enos,dc=com" // Use existing base domain
config = &LDAPDomainConfig{
URL: fmt.Sprintf("ldap://%s:%s", ldapServer, ldapPort), // Private IP for Vault
URL: ldapURLPrivate,
BindDN: ldapBindDN,
BindPass: ldapBindPass,
BaseDN: baseDN,
@ -251,7 +250,7 @@ func PrepareTestLDAPDomain(
}
// Store public IP for setup operations (ldapadd commands from GitHub runner)
config.SetupURL = fmt.Sprintf("ldap://%s:%s", ldapServerPublic, ldapPort)
config.SetupURL = ldapURLPublic
// Create domain structure
if err := createDomain(t, session, config); err != nil {
@ -307,7 +306,8 @@ ou: %s
// Use Eventually to retry ldapadd until LDAP server is ready
session.Eventually(func() error {
cmd := exec.Command("ldapadd",
cmd := exec.Command(
"ldapadd",
"-x",
"-H", config.SetupURL, // Use public IP for setup operations from GitHub runner
"-D", config.BindDN,
@ -335,7 +335,8 @@ func deleteDomain(t *testing.T, config *LDAPDomainConfig) {
// Delete both user and group OUs
for _, dn := range []string{config.UserDN, config.GroupDN} {
cmd := exec.Command("ldapdelete",
cmd := exec.Command(
"ldapdelete",
"-x",
"-r", // recursive
"-H", config.SetupURL, // Use public IP for cleanup operations from GitHub runner
@ -383,7 +384,8 @@ userPassword: %s
t.Logf(" ldapsearch -x -H %s -b \"dc=enos,dc=com\" -D \"%s\" -w \"%s\" -s base",
config.SetupURL, userDN, password)
cmd := exec.Command("ldapadd",
cmd := exec.Command(
"ldapadd",
"-x",
"-H", config.SetupURL, // Use public IP for setup operations from GitHub runner
"-D", config.BindDN,
@ -423,7 +425,8 @@ cn: %s
%s
`, groupDN, groupName, strings.Join(memberDNs, "\n"))
cmd := exec.Command("ldapadd",
cmd := exec.Command(
"ldapadd",
"-x",
"-H", config.URL,
"-D", config.BindDN,
@ -456,7 +459,8 @@ add: member
member: %s
`, groupDN, userDN)
cmd := exec.Command("ldapmodify",
cmd := exec.Command(
"ldapmodify",
"-x",
"-H", config.URL,
"-D", config.BindDN,
@ -489,7 +493,8 @@ delete: member
member: %s
`, groupDN, userDN)
cmd := exec.Command("ldapmodify",
cmd := exec.Command(
"ldapmodify",
"-x",
"-H", config.URL,
"-D", config.BindDN,
@ -512,7 +517,8 @@ member: %s
func CheckLDAPUserExistsInDomain(t *testing.T, config *LDAPDomainConfig, username string) bool {
t.Helper()
cmd := exec.Command("ldapsearch",
cmd := exec.Command(
"ldapsearch",
"-x",
"-H", config.URL,
"-b", config.UserDN,

53
vault/external_tests/blackbox/plugins/ldap/secrets_ldap_root_credential_rollback_test.go
View file

@ -13,6 +13,28 @@ import (
"github.com/hashicorp/vault/sdk/helper/testcluster/blackbox"
)
// TestLDAPRootCredentialRollbackWorkflows runs all rollback workflow tests
// This is the main test function that gets triggered by enos-scenario-plugin.hcl
func TestLDAPRootCredentialRollbackWorkflows(t *testing.T) {
t.Run("RollbackSuccess", func(t *testing.T) {
t.Parallel()
v := blackbox.New(t)
testLDAPRootCredentialRollbackSuccess(t, v)
})
t.Run("RollbackFailure", func(t *testing.T) {
t.Parallel()
v := blackbox.New(t)
testLDAPRootCredentialRollbackFailure(t, v)
})
t.Run("AutomaticRollback", func(t *testing.T) {
t.Parallel()
v := blackbox.New(t)
testLDAPRootCredentialAutomaticRollbackOnFailure(t, v)
})
}
// testLDAPRootCredentialRollbackSuccess tests successful rollback when rotation fails
// Converts: secrets-rollback-invalid-config.sh
// Scenario: Rotation fails with invalid LDAP endpoint, old password is preserved
@ -21,7 +43,7 @@ func testLDAPRootCredentialRollbackSuccess(t *testing.T, v *blackbox.Session) {
cleanup, ldapConfig, err := PrepareTestLDAPDomain(t, v, isCI())
if err != nil {
if isCI() {
t.Fatalf("Failed to create LDAP domain in CI: %v", err)
t.Fatalf("LDAP domain creation failed in CI: %v", err)
}
t.Skipf("LDAP domain creation not available: %v", err)
}
@ -102,7 +124,7 @@ func testLDAPRootCredentialRollbackFailure(t *testing.T, v *blackbox.Session) {
cleanup, ldapConfig, err := PrepareTestLDAPDomain(t, v, isCI())
if err != nil {
if isCI() {
t.Fatalf("Failed to create LDAP domain in CI: %v", err)
t.Fatalf("LDAP domain creation failed in CI: %v", err)
}
t.Skipf("LDAP domain creation not available: %v", err)
}
@ -180,7 +202,7 @@ func testLDAPRootCredentialAutomaticRollbackOnFailure(t *testing.T, v *blackbox.
cleanup, ldapConfig, err := PrepareTestLDAPDomain(t, v, isCI())
if err != nil {
if isCI() {
t.Fatalf("Failed to create LDAP domain in CI: %v", err)
t.Fatalf("LDAP domain creation failed in CI: %v", err)
}
t.Skipf("LDAP domain creation not available: %v", err)
}
@ -276,7 +298,8 @@ func verifyLDAPAuth(t *testing.T, config *LDAPDomainConfig, bindDN, password str
// Use SetupURL (public IP) for authentication checks from GitHub runner
// config.URL contains private IP which is only accessible from Vault cluster
cmd := exec.Command("ldapwhoami",
cmd := exec.Command(
"ldapwhoami",
"-x",
"-H", config.SetupURL,
"-D", bindDN,
@ -291,25 +314,3 @@ func verifyLDAPAuth(t *testing.T, config *LDAPDomainConfig, bindDN, password str
return true
}
// TestLDAPRootCredentialRollbackWorkflows runs all rollback workflow tests
// This is the main test function that gets triggered by enos-scenario-plugin.hcl
func TestLDAPRootCredentialRollbackWorkflows(t *testing.T) {
t.Run("RollbackSuccess", func(t *testing.T) {
t.Parallel()
v := blackbox.New(t)
testLDAPRootCredentialRollbackSuccess(t, v)
})
t.Run("RollbackFailure", func(t *testing.T) {
t.Parallel()
v := blackbox.New(t)
testLDAPRootCredentialRollbackFailure(t, v)
})
t.Run("AutomaticRollback", func(t *testing.T) {
t.Parallel()
v := blackbox.New(t)
testLDAPRootCredentialAutomaticRollbackOnFailure(t, v)
})
}

6
vault/external_tests/blackbox/plugins/ldap/secrets_ldap_test.go
View file

@ -15,7 +15,7 @@ func testLDAPSecretsCreate(t *testing.T, v *blackbox.Session) {
cleanup, ldapConfig, err := PrepareTestLDAPDomain(t, v, isCI())
if err != nil {
if isCI() {
t.Fatalf("Failed to create LDAP domain in CI: %v", err)
t.Fatalf("LDAP domain creation failed in CI: %v", err)
}
t.Skipf("LDAP domain creation not available: %v", err)
}
@ -51,7 +51,7 @@ func testLDAPSecretsRead(t *testing.T, v *blackbox.Session) {
cleanup, ldapConfig, err := PrepareTestLDAPDomain(t, v, isCI())
if err != nil {
if isCI() {
t.Fatalf("Failed to create LDAP domain in CI: %v", err)
t.Fatalf("LDAP domain creation failed in CI: %v", err)
}
t.Skipf("LDAP domain creation not available: %v", err)
}
@ -105,7 +105,7 @@ func testLDAPSecretsDelete(t *testing.T, v *blackbox.Session) {
cleanup, ldapConfig, err := PrepareTestLDAPDomain(t, v, isCI())
if err != nil {
if isCI() {
t.Fatalf("Failed to create LDAP domain in CI: %v", err)
t.Fatalf("LDAP domain creation failed in CI: %v", err)
}
t.Skipf("LDAP domain creation not available: %v", err)
}

13
vault/external_tests/blackbox/raft/voters_test.go
View file

@ -5,6 +5,7 @@ package raft
import (
"testing"
"time"
"github.com/hashicorp/vault/sdk/helper/testcluster/blackbox"
)
@ -13,8 +14,16 @@ import (
func TestRaftVoters(t *testing.T) {
v := blackbox.New(t)
// Verify we have a healthy cluster regardless of node count
v.AssertClusterHealthy()
// Wait for a healthy cluster
v.EventuallyClusterHealthyUnsealed(15 * time.Second)
storage := v.MustGetConfigStorageType()
if storage != "raft" {
t.Log("skipping as cluster is not using integrated storage")
return
}
v.EventuallyRaftClusterHealthy(5 * time.Second)
t.Log("Successfully verified raft cluster is healthy with at least one voter")
}

6
vault/external_tests/blackbox/secrets/secrets_pki_test.go
View file

@ -16,11 +16,11 @@ func testPKISecretsCreate(t *testing.T, v *blackbox.Session) {
v.MustEnableSecretsEngine("pki-create", &api.MountInput{Type: "pki"})
// Configure max TTL for the mount
err := v.Client.Sys().TuneMount("pki-create", api.MountConfigInput{
MaxLeaseTTL: "87600h",
err := v.Client.Sys().TuneMountAllowNilWithContext(t.Context(), "pki-create", api.TuneMountConfigInput{
MaxLeaseTTL: new(string("87600h")),
})
if err != nil {
t.Fatalf("Failed to tune PKI mount: %v", err)
t.Fatalf("Failed to tune PKI mount: %v: %v", err, v.Client.Address())
}
// Generate root CA

19
vault/external_tests/blackbox/system/billing_test.go
View file

@ -20,33 +20,30 @@ import (
func TestBillingOverviewNamespaceRestrictions(t *testing.T) {
v := blackbox.New(t)
// Verify cluster stability first
v.AssertClusterHealthy()
// Check if we're in HVD (has base namespace from VAULT_NAMESPACE)
baseNS := v.GetParentNamespace()
if baseNS == "" {
t.Skip("Skipping namespace restriction tests - no base namespace configured (not in HVD)")
}
testCases := []struct {
name string
// Verify cluster stability first
v.AssertClusterHealthy()
testCases := map[string]struct {
namespaceSwitcher func(func() (*api.Secret, error)) (*api.Secret, error)
expectedError string
}{
{
name: "base_namespace_supported",
"base_namespace_supported": {
namespaceSwitcher: v.WithParentNamespace,
},
{
name: "root_namespace_permission_denied",
"root_namespace_supported": {
namespaceSwitcher: v.WithRootNamespace,
expectedError: "permission denied",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
var rawResp *api.Response
var err error
_, err = tc.namespaceSwitcher(func() (*api.Secret, error) {

13
vault/external_tests/blackbox/verify/version_verification_test.go
View file

@ -8,6 +8,7 @@ import (
"testing"
"github.com/hashicorp/vault/sdk/helper/testcluster/blackbox"
"github.com/stretchr/testify/require"
)
// TestVaultServerVersion verifies the Vault server version via sys/version-history API
@ -26,6 +27,14 @@ func TestVaultServerVersion(t *testing.T) {
}
v := blackbox.New(t)
v.AssertVersion(version)
v.AssertBuildDate(version, buildDate)
replicationMode, err := v.GetDRReplicationMode()
require.NoError(t, err)
t.Logf("Found DR Replication mode: %s", replicationMode)
switch replicationMode {
case "primary", "secondary":
t.Skip("Skipping server version check on DR cluster as path is not available in that mode")
default:
v.AssertVersion(version)
v.AssertBuildDate(version, buildDate)
}
}