From bb106f1bef05c3515da39d88fb07a5f60e53a6c4 Mon Sep 17 00:00:00 2001 From: Vault Automation Date: Fri, 6 Feb 2026 16:36:29 -0500 Subject: [PATCH] Add blackbox testing SDK (#11210) (#12245) * rough draft * add some stuff for dynamic secrets * add some more helpers and sample tests * new helpers, new tests, refactoring * Add Basic Smoke SDK Scenario (#11678) * Add simple test for stepdown election * Add a smoke_sdk scenario * add script to run tests locally * fix up a few things * VAULT-39746 - Add Tests to Smoke SDK and Cloud Scenarios (#11795) * Add some go verification steps in enos sdk test run script * formatting * Add a smoke_sdk scenario userpass secret engine create test (#11808) * Add a smoke_sdk scenario userpass secret engine create test * Add the some additional tests * Add Smoke tests to Cloud Scenario (#11876) * Add a smoke_sdk scenario userpass secret engine create test * Add the some additional tests * Add smoke testing to cloud * Add test results to output and test filtering * comment * fix test * fix the smoke scenario * Address some various feedback * missed cleanup * remove node count dependency in the tests * Fix test perms * Adjust the testing and clean them up a bit * formatting * fmt * fmt2 * more fmt * formatting * tryagain * remove the docker/hcp divide * use the SHA as ID * adjust perms * Add transit test * skip blackbox testing in test-go * copywrite * Apply suggestion from @brewgator * Add godoc * grep cleanup --------- Co-authored-by: Josh Black Co-authored-by: Luis (LT) Carbonell --- .github/workflows/test-go.yml | 2 +- enos/enos-modules.hcl | 4 + enos/enos-scenario-smoke-sdk.hcl | 505 ++++++++++++++++++ enos/enos-scenario-smoke.hcl | 2 +- enos/modules/docker_namespace_token/main.tf | 34 +- enos/modules/hcp/create_vault_cluster/main.tf | 17 +- enos/modules/vault_run_blackbox_test/main.tf | 57 ++ .../vault_run_blackbox_test/outputs.tf | 21 + .../scripts/run-test.sh | 153 ++++++ .../vault_run_blackbox_test/variables.tf | 43 ++ sdk/helper/testcluster/blackbox/assertions.go | 291 ++++++++++ sdk/helper/testcluster/blackbox/session.go | 91 ++++ .../testcluster/blackbox/session_auth.go | 115 ++++ .../testcluster/blackbox/session_dynamic.go | 140 +++++ .../testcluster/blackbox/session_logical.go | 53 ++ .../testcluster/blackbox/session_ops.go | 60 +++ .../testcluster/blackbox/session_pki.go | 39 ++ .../testcluster/blackbox/session_plugin.go | 64 +++ .../testcluster/blackbox/session_raft.go | 399 ++++++++++++++ .../testcluster/blackbox/session_status.go | 130 +++++ .../testcluster/blackbox/session_sys.go | 37 ++ .../testcluster/blackbox/session_token.go | 94 ++++ .../testcluster/blackbox/session_util.go | 43 ++ .../blackbox/auth_engines_test.go | 218 ++++++++ vault/external_tests/blackbox/dynamic_test.go | 49 ++ vault/external_tests/blackbox/kvv2_test.go | 79 +++ vault/external_tests/blackbox/pki_test.go | 49 ++ .../blackbox/secrets_engines_test.go | 289 ++++++++++ vault/external_tests/blackbox/smoke_test.go | 71 +++ vault/external_tests/blackbox/system_test.go | 105 ++++ vault/external_tests/blackbox/test_utils.go | 137 +++++ vault/external_tests/blackbox/token_test.go | 37 ++ 32 files changed, 3402 insertions(+), 26 deletions(-) create mode 100644 enos/enos-scenario-smoke-sdk.hcl create mode 100644 enos/modules/vault_run_blackbox_test/main.tf create mode 100644 enos/modules/vault_run_blackbox_test/outputs.tf create mode 100755 enos/modules/vault_run_blackbox_test/scripts/run-test.sh create mode 100644 enos/modules/vault_run_blackbox_test/variables.tf create mode 100644 sdk/helper/testcluster/blackbox/assertions.go create mode 100644 sdk/helper/testcluster/blackbox/session.go create mode 100644 sdk/helper/testcluster/blackbox/session_auth.go create mode 100644 sdk/helper/testcluster/blackbox/session_dynamic.go create mode 100644 sdk/helper/testcluster/blackbox/session_logical.go create mode 100644 sdk/helper/testcluster/blackbox/session_ops.go create mode 100644 sdk/helper/testcluster/blackbox/session_pki.go create mode 100644 sdk/helper/testcluster/blackbox/session_plugin.go create mode 100644 sdk/helper/testcluster/blackbox/session_raft.go create mode 100644 sdk/helper/testcluster/blackbox/session_status.go create mode 100644 sdk/helper/testcluster/blackbox/session_sys.go create mode 100644 sdk/helper/testcluster/blackbox/session_token.go create mode 100644 sdk/helper/testcluster/blackbox/session_util.go create mode 100644 vault/external_tests/blackbox/auth_engines_test.go create mode 100644 vault/external_tests/blackbox/dynamic_test.go create mode 100644 vault/external_tests/blackbox/kvv2_test.go create mode 100644 vault/external_tests/blackbox/pki_test.go create mode 100644 vault/external_tests/blackbox/secrets_engines_test.go create mode 100644 vault/external_tests/blackbox/smoke_test.go create mode 100644 vault/external_tests/blackbox/system_test.go create mode 100644 vault/external_tests/blackbox/test_utils.go create mode 100644 vault/external_tests/blackbox/token_test.go diff --git a/.github/workflows/test-go.yml b/.github/workflows/test-go.yml index 4b51d22a10..1191aca642 100644 --- a/.github/workflows/test-go.yml +++ b/.github/workflows/test-go.yml @@ -168,7 +168,7 @@ jobs: run: | # testonly tests need additional build tag though let's exclude them anyway for clarity ( - make all-packages | grep -v "_binary" | grep -v "vault/integ" | grep -v "testonly" | gotestsum tool ci-matrix --debug \ + make all-packages | grep -v "_binary\|testonly\|vault/integ\|vault/external_tests/blackbox" | gotestsum tool ci-matrix --debug \ --partitions "${{ inputs.total-runners }}" \ --timing-files '${{ steps.local-metadata.outputs.go-test-dir }}/*.json' > matrix.json ) diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index a941ac6762..443c78140e 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -304,6 +304,10 @@ module "vault_test_ui" { ui_run_tests = var.ui_run_tests } +module "vault_run_blackbox_test" { + source = "./modules/vault_run_blackbox_test" +} + module "vault_unseal_replication_followers" { source = "./modules/vault_unseal_replication_followers" diff --git a/enos/enos-scenario-smoke-sdk.hcl b/enos/enos-scenario-smoke-sdk.hcl new file mode 100644 index 0000000000..9962956a47 --- /dev/null +++ b/enos/enos-scenario-smoke-sdk.hcl @@ -0,0 +1,505 @@ +// Copyright IBM Corp. 2016, 2025 +// SPDX-License-Identifier: BUSL-1.1 + +scenario "smoke_sdk" { + description = <<-EOF + The smoke_sdk scenario deploys a Vault cluster and runs a comprehensive suite of blackbox SDK tests + instead of enos-based verification. This scenario validates core Vault functionality including leader election, + secrets engines, UI assets, replication status, and backend-specific features using the blackbox SDK. + + # How to run this scenario + + For general instructions on running a scenario, refer to the Enos docs: https://eng-handbook.hashicorp.services/internal-tools/enos/running-a-scenario/ + For troubleshooting tips and common errors, see https://eng-handbook.hashicorp.services/internal-tools/enos/troubleshooting/. + + Variables required for all scenario variants: + - aws_ssh_private_key_path (more info about AWS SSH keypairs: https://eng-handbook.hashicorp.services/internal-tools/enos/getting-started/#set-your-aws-key-pair-name-and-private-key) + - aws_ssh_keypair_name + - vault_build_date* + - vault_product_version + - vault_revision* + + * If you don't already know what build date and revision you should be using, see + https://eng-handbook.hashicorp.services/internal-tools/enos/troubleshooting/#execution-error-expected-vs-got-for-vault-versioneditionrevisionbuild-date. + + Variables required for some scenario variants: + - artifactory_token (if using `artifact_source:artifactory` in your filter) + - aws_region (if different from the default value in enos-variables.hcl) + - consul_license_path (if using an ENT edition of Consul) + - distro_version_ (if different from the default version for your target + distro. See supported distros and default versions in the distro_version_ + definitions in enos-variables.hcl) + - vault_artifact_path (the path to where you have a Vault artifact already downloaded, + if using `artifact_source:crt` in your filter) + - vault_license_path (if using an ENT edition of Vault) + EOF + + matrix { + arch = global.archs + artifact_source = global.artifact_sources + artifact_type = global.artifact_types + backend = global.backends + config_mode = global.config_modes + consul_edition = global.consul_editions + consul_version = global.consul_versions + distro = global.distros + edition = global.editions + ip_version = global.ip_versions + seal = global.seals + + // Our local builder always creates bundles + exclude { + artifact_source = ["local"] + artifact_type = ["package"] + } + + // PKCS#11 can only be used on ent.hsm and ent.hsm.fips1403. + exclude { + seal = ["pkcs11"] + edition = [for e in matrix.edition : e if !strcontains(e, "hsm")] + } + + // softhsm packages not available for leap/sles. + exclude { + seal = ["pkcs11"] + distro = ["leap", "sles"] + } + + // Testing in IPV6 mode is currently implemented for integrated Raft storage only + exclude { + ip_version = ["6"] + backend = ["consul"] + } + } + + terraform_cli = terraform_cli.default + terraform = terraform.default + providers = [ + provider.aws.default, + provider.enos.ec2_user, + provider.enos.ubuntu + ] + + locals { + artifact_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null + enos_provider = { + amzn = provider.enos.ec2_user + leap = provider.enos.ec2_user + rhel = provider.enos.ec2_user + sles = provider.enos.ec2_user + ubuntu = provider.enos.ubuntu + } + manage_service = matrix.artifact_type == "bundle" + } + + step "build_vault" { + description = global.description.build_vault + module = "build_${matrix.artifact_source}" + + variables { + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition] + artifact_path = local.artifact_path + goarch = matrix.arch + goos = "linux" + artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null + artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null + artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null + arch = matrix.artifact_source == "artifactory" ? matrix.arch : null + product_version = var.vault_product_version + artifact_type = matrix.artifact_type + distro = matrix.artifact_source == "artifactory" ? matrix.distro : null + edition = matrix.artifact_source == "artifactory" ? matrix.edition : null + revision = var.vault_revision + } + } + + step "ec2_info" { + description = global.description.ec2_info + module = module.ec2_info + } + + step "create_vpc" { + description = global.description.create_vpc + module = module.create_vpc + + variables { + common_tags = global.tags + ip_version = matrix.ip_version + } + } + + step "read_backend_license" { + description = global.description.read_backend_license + module = module.read_license + skip_step = matrix.backend == "raft" || matrix.consul_edition == "ce" + + variables { + file_name = global.backend_license_path + } + } + + step "read_vault_license" { + description = global.description.read_vault_license + skip_step = matrix.edition == "ce" + module = module.read_license + + variables { + file_name = global.vault_license_path + } + } + + step "create_seal_key" { + description = global.description.create_seal_key + module = "seal_${matrix.seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + common_tags = global.tags + } + } + + step "create_external_integration_target" { + description = global.description.create_external_integration_target + module = module.target_ec2_instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider["ubuntu"] + } + + variables { + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["24.04"] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + instance_count = 1 + vpc_id = step.create_vpc.id + } + } + + step "create_vault_cluster_targets" { + description = global.description.create_vault_cluster_targets + module = module.target_ec2_instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id + } + } + + step "create_vault_cluster_backend_targets" { + description = global.description.create_vault_cluster_targets + module = matrix.backend == "consul" ? module.target_ec2_instances : module.target_ec2_shim + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"][global.distro_version["ubuntu"]] + cluster_tag_key = global.backend_tag_key + common_tags = global.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id + } + } + + step "set_up_external_integration_target" { + description = global.description.set_up_external_integration_target + module = module.set_up_external_integration_target + depends_on = [ + step.create_external_integration_target + ] + + providers = { + enos = local.enos_provider["ubuntu"] + } + + variables { + hosts = step.create_external_integration_target.hosts + ip_version = matrix.ip_version + packages = concat(global.packages, global.distro_packages["ubuntu"]["24.04"], ["podman", "podman-docker"]) + ports = global.integration_host_ports + } + } + + step "create_backend_cluster" { + description = global.description.create_backend_cluster + module = "backend_${matrix.backend}" + depends_on = [ + step.create_vault_cluster_backend_targets + ] + + providers = { + enos = provider.enos.ubuntu + } + + verifies = [ + // verified in modules + quality.consul_autojoin_aws, + quality.consul_config_file, + quality.consul_ha_leader_election, + quality.consul_service_start_server, + // verified in enos_consul_start resource + quality.consul_api_agent_host_read, + quality.consul_api_health_node_read, + quality.consul_api_operator_raft_config_read, + quality.consul_cli_validate, + quality.consul_health_state_passing_read_nodes_minimum, + quality.consul_operator_raft_configuration_read_voters_minimum, + quality.consul_service_systemd_notified, + quality.consul_service_systemd_unit, + ] + + variables { + cluster_name = step.create_vault_cluster_backend_targets.cluster_name + cluster_tag_key = global.backend_tag_key + hosts = step.create_vault_cluster_backend_targets.hosts + license = (matrix.backend == "consul" && matrix.consul_edition == "ent") ? step.read_backend_license.license : null + release = { + edition = matrix.consul_edition + version = matrix.consul_version + } + } + } + + step "create_vault_cluster" { + description = global.description.create_vault_cluster + module = module.vault_cluster + depends_on = [ + step.create_backend_cluster, + step.build_vault, + step.create_vault_cluster_targets, + step.set_up_external_integration_target + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + // verified in modules + quality.consul_service_start_client, + quality.vault_artifact_bundle, + quality.vault_artifact_deb, + quality.vault_artifact_rpm, + quality.vault_audit_log, + quality.vault_audit_socket, + quality.vault_audit_syslog, + quality.vault_autojoin_aws, + quality.vault_config_env_variables, + quality.vault_config_file, + quality.vault_config_log_level, + quality.vault_init, + quality.vault_license_required_ent, + quality.vault_listener_ipv4, + quality.vault_listener_ipv6, + quality.vault_service_start, + quality.vault_storage_backend_consul, + quality.vault_storage_backend_raft, + // verified in enos_vault_start resource + quality.vault_api_sys_config_read, + quality.vault_api_sys_ha_status_read, + quality.vault_api_sys_health_read, + quality.vault_api_sys_host_info_read, + quality.vault_api_sys_replication_status_read, + quality.vault_api_sys_seal_status_api_read_matches_sys_health, + quality.vault_api_sys_storage_raft_autopilot_configuration_read, + quality.vault_api_sys_storage_raft_autopilot_state_read, + quality.vault_api_sys_storage_raft_configuration_read, + quality.vault_cli_status_exit_code, + quality.vault_service_systemd_notified, + quality.vault_service_systemd_unit, + ] + + variables { + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + backend_cluster_name = step.create_vault_cluster_backend_targets.cluster_name + backend_cluster_tag_key = global.backend_tag_key + cluster_name = step.create_vault_cluster_targets.cluster_name + config_mode = matrix.config_mode + consul_license = (matrix.backend == "consul" && matrix.consul_edition == "ent") ? step.read_backend_license.license : null + consul_release = matrix.backend == "consul" ? { + edition = matrix.consul_edition + version = matrix.consul_version + } : null + enable_audit_devices = var.vault_enable_audit_devices + hosts = step.create_vault_cluster_targets.hosts + install_dir = global.vault_install_dir[matrix.artifact_type] + ip_version = matrix.ip_version + license = matrix.edition != "ce" ? step.read_vault_license.license : null + local_artifact_path = local.artifact_path + manage_service = local.manage_service + packages = concat(global.packages, global.distro_packages[matrix.distro][global.distro_version[matrix.distro]]) + seal_attributes = step.create_seal_key.attributes + seal_type = matrix.seal + storage_backend = matrix.backend + } + } + + step "get_local_metadata" { + description = global.description.get_local_metadata + skip_step = matrix.artifact_source != "local" + module = module.get_local_metadata + } + + // Wait for our cluster to elect a leader + step "wait_for_leader" { + description = global.description.wait_for_cluster_to_have_leader + module = module.vault_wait_for_leader + depends_on = [step.create_vault_cluster] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_leader_read, + quality.vault_unseal_ha_leader_election, + ] + + variables { + timeout = 120 // seconds + ip_version = matrix.ip_version + hosts = step.create_vault_cluster_targets.hosts + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_vault_cluster.root_token + } + } + + step "get_vault_cluster_ips" { + description = global.description.get_vault_cluster_ip_addresses + module = module.vault_get_cluster_ips + depends_on = [step.wait_for_leader] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_ha_status_read, + quality.vault_api_sys_leader_read, + quality.vault_cli_operator_members, + ] + + variables { + hosts = step.create_vault_cluster_targets.hosts + ip_version = matrix.ip_version + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_vault_cluster.root_token + } + } + + // Define smoke test suite + locals { + smoke_tests = [ + "TestStepdownAndLeaderElection", + "TestSecretsEngineCreate", + "TestUnsealedStatus", + "TestVaultVersion", + "TestSecretsEngineRead", + "TestReplicationStatus", + "TestUIAssets", + "TestSecretsEngineDelete" + ] + + // Add backend-specific tests + smoke_tests_with_backend = concat( + local.smoke_tests, + matrix.backend == "raft" ? [ + "TestRaftVoters", + "TestNodeRemovalAndRejoin" + ] : [] + ) + } + + // Run all blackbox SDK smoke tests + step "run_blackbox_tests" { + description = "Run blackbox SDK smoke tests: ${join(", ", local.smoke_tests_with_backend)}" + module = module.vault_run_blackbox_test + depends_on = [step.get_vault_cluster_ips] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + leader_host = step.get_vault_cluster_ips.leader_host + leader_public_ip = step.get_vault_cluster_ips.leader_public_ip + vault_root_token = step.create_vault_cluster.root_token + test_names = local.smoke_tests_with_backend + test_package = "./vault/external_tests/blackbox" + } + } + + + output "cluster_name" { + description = "The Vault cluster name" + value = step.create_vault_cluster.cluster_name + } + + output "hosts" { + description = "The Vault cluster target hosts" + value = step.create_vault_cluster.hosts + } + + output "private_ips" { + description = "The Vault cluster private IPs" + value = step.create_vault_cluster.private_ips + } + + output "public_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.public_ips + } + + output "root_token" { + description = "The Vault cluster root token" + value = step.create_vault_cluster.root_token + } + + output "recovery_key_shares" { + description = "The Vault cluster recovery key shares" + value = step.create_vault_cluster.recovery_key_shares + } + + output "recovery_keys_b64" { + description = "The Vault cluster recovery keys b64" + value = step.create_vault_cluster.recovery_keys_b64 + } + + output "recovery_keys_hex" { + description = "The Vault cluster recovery keys hex" + value = step.create_vault_cluster.recovery_keys_hex + } + + output "seal_key_attributes" { + description = "The Vault cluster seal attributes" + value = step.create_seal_key.attributes + } + + output "unseal_keys_b64" { + description = "The Vault cluster unseal keys" + value = step.create_vault_cluster.unseal_keys_b64 + } + + output "unseal_keys_hex" { + description = "The Vault cluster unseal keys hex" + value = step.create_vault_cluster.unseal_keys_hex + } +} diff --git a/enos/enos-scenario-smoke.hcl b/enos/enos-scenario-smoke.hcl index c6ffff5226..0097acbdfd 100644 --- a/enos/enos-scenario-smoke.hcl +++ b/enos/enos-scenario-smoke.hcl @@ -556,7 +556,7 @@ scenario "smoke" { description = <<-EOF Remove a follower and ensure that it's marked as removed and can be added back once its data has been deleted EOF - module = semverconstraint(var.vault_product_version, ">=1.19.0-0") && matrix.backend == "raft" ? "vault_raft_remove_node_and_verify" : "vault_verify_removed_node_shim" + module = matrix.backend == "raft" ? "vault_raft_remove_node_and_verify" : "vault_verify_removed_node_shim" depends_on = [ step.create_vault_cluster, step.get_vault_cluster_ips, diff --git a/enos/modules/docker_namespace_token/main.tf b/enos/modules/docker_namespace_token/main.tf index f7ab739311..69ad34d8d9 100644 --- a/enos/modules/docker_namespace_token/main.tf +++ b/enos/modules/docker_namespace_token/main.tf @@ -42,10 +42,8 @@ variable "namespace_name" { default = "admin" } -# Create namespace using the root token (only when all required vars are present) +# Create namespace using the root token resource "enos_local_exec" "docker_create_namespace" { - count = var.vault_address != null && var.vault_root_token != null && var.container_name != null ? 1 : 0 - inline = [ <<-EOT docker exec -e VAULT_ADDR=${var.vault_address} -e VAULT_TOKEN=${var.vault_root_token} \ @@ -54,21 +52,21 @@ resource "enos_local_exec" "docker_create_namespace" { ] } -# Create policy in the namespace +# Create policy at root level for blackbox testing (matches HVD admin namespace permissions) resource "enos_local_exec" "docker_create_policy" { - count = var.vault_address != null && var.vault_root_token != null && var.container_name != null ? 1 : 0 - inline = [ <<-EOT # Write policy to a temp file in the container docker exec ${var.container_name} sh -c 'cat > /tmp/${var.namespace_name}-policy.hcl << EOF +# HVD admin namespace compatible policy - restricted permissions to match cloud environment path "*" { - capabilities = ["create", "read", "update", "delete", "list", "sudo"] + capabilities = ["sudo","read","create","update","delete","list","patch","subscribe"] + subscribe_event_types = ["*"] } EOF' - # Apply the policy in the namespace - docker exec -e VAULT_ADDR=${var.vault_address} -e VAULT_TOKEN=${var.vault_root_token} -e VAULT_NAMESPACE=${var.namespace_name} \ + # Apply the policy at root level (not in a namespace) + docker exec -e VAULT_ADDR=${var.vault_address} -e VAULT_TOKEN=${var.vault_root_token} \ ${var.container_name} vault policy write ${var.namespace_name}-policy /tmp/${var.namespace_name}-policy.hcl EOT ] @@ -76,18 +74,16 @@ EOF' depends_on = [enos_local_exec.docker_create_namespace] } -# Create token in the namespace +# Create token at root level with the policy that allows namespace operations resource "enos_local_exec" "docker_create_token" { - count = var.vault_address != null && var.vault_root_token != null && var.container_name != null ? 1 : 0 - inline = [ <<-EOT - docker exec -e VAULT_ADDR=${var.vault_address} -e VAULT_TOKEN=${var.vault_root_token} -e VAULT_NAMESPACE=${var.namespace_name} \ + docker exec -e VAULT_ADDR=${var.vault_address} -e VAULT_TOKEN=${var.vault_root_token} \ ${var.container_name} vault token create \ -policy=${var.namespace_name}-policy \ -ttl=24h \ -renewable=true \ - -metadata="purpose=${var.namespace_name}-token" \ + -metadata="purpose=${var.namespace_name}-hvd-compatible-token" \ -metadata="created_by=docker_namespace_token_module" \ -format=json | jq -r '.auth.client_token' EOT @@ -97,8 +93,8 @@ resource "enos_local_exec" "docker_create_token" { } locals { - # For Docker: use the created namespace token, for HCP: use root token (fallback) - namespace_token = length(enos_local_exec.docker_create_token) > 0 ? trimspace(enos_local_exec.docker_create_token[0].stdout) : var.vault_root_token + # Use the created namespace token + namespace_token = trimspace(enos_local_exec.docker_create_token.stdout) } output "created_at" { @@ -115,11 +111,11 @@ output "token" { } output "namespace" { - value = length(enos_local_exec.docker_create_token) > 0 ? var.namespace_name : "root" + value = var.namespace_name description = "The namespace where the token is valid" } output "policy" { - value = length(enos_local_exec.docker_create_token) > 0 ? "${var.namespace_name}-policy" : "root" - description = "The policy assigned to the token" + value = "${var.namespace_name}-policy" + description = "The HVD-compatible policy assigned to the token (matches cloud environment permissions)" } diff --git a/enos/modules/hcp/create_vault_cluster/main.tf b/enos/modules/hcp/create_vault_cluster/main.tf index 1f628215ec..583668bf94 100644 --- a/enos/modules/hcp/create_vault_cluster/main.tf +++ b/enos/modules/hcp/create_vault_cluster/main.tf @@ -92,6 +92,11 @@ variable "local_build_path" { data "enos_environment" "localhost" {} +# Get current git SHA for unique cluster naming +resource "enos_local_exec" "git_sha" { + inline = ["git rev-parse --short HEAD"] +} + resource "random_string" "id" { length = 4 lower = true @@ -101,11 +106,13 @@ resource "random_string" "id" { } locals { - // Generate a unique identifier for our scenario. If we've been given a - // min_vault_version we'll use that as it will likely be the version and - // a SHA of a custom image. Make sure it doesn't have special characters. - // Otherwise, just use a random string. - id = var.min_vault_version != null ? try(replace(var.min_vault_version, "/[^0-9A-Za-z]/", "-"), random_string.id.result) : random_string.id.result + // Generate a unique identifier for our scenario using git SHA for uniqueness. + // If min_vault_version contains a SHA (indicating a custom build), use that SHA. + // Otherwise, use the current git commit SHA to ensure uniqueness. + has_custom_sha = var.min_vault_version != null ? can(regex("\\+[a-z]+.*-[0-9a-f]{7,}", var.min_vault_version)) : false + custom_sha = local.has_custom_sha ? regex("([0-9a-f]{7,})", var.min_vault_version)[0] : "" + git_sha = trimspace(enos_local_exec.git_sha.stdout) + id = local.has_custom_sha ? "custom-${local.custom_sha}" : "git-${local.git_sha}" } resource "hcp_hvn" "default" { diff --git a/enos/modules/vault_run_blackbox_test/main.tf b/enos/modules/vault_run_blackbox_test/main.tf new file mode 100644 index 0000000000..05dc3ed757 --- /dev/null +++ b/enos/modules/vault_run_blackbox_test/main.tf @@ -0,0 +1,57 @@ +# Copyright IBM Corp. 2016, 2025 +# SPDX-License-Identifier: BUSL-1.1 + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + } +} + +# Generate matrix.json for gotestsum from the test list +resource "local_file" "test_matrix" { + filename = "/tmp/vault_test_matrix_${random_string.test_id.result}.json" + content = jsonencode({ + include = length(var.test_names) > 0 ? [ + for test in var.test_names : { + test = test + } + ] : [] + }) +} + +resource "random_string" "test_id" { + length = 8 + special = false + upper = false +} + +resource "enos_local_exec" "run_blackbox_test" { + scripts = [abspath("${path.module}/scripts/run-test.sh")] + environment = merge({ + VAULT_TOKEN = var.vault_root_token + VAULT_ADDR = var.vault_addr != null ? var.vault_addr : "http://${var.leader_public_ip}:8200" + VAULT_TEST_PACKAGE = var.test_package + VAULT_TEST_MATRIX = length(var.test_names) > 0 ? local_file.test_matrix.filename : "" + }, var.vault_namespace != null ? { + VAULT_NAMESPACE = var.vault_namespace + } : {}) + depends_on = [local_file.test_matrix] +} + +# Extract information from the script output +locals { + json_file_path = try( + regex("JSON_RESULTS_FILE=(.+)", enos_local_exec.run_blackbox_test.stdout)[0], + "" + ) + test_status = try( + regex("TEST_STATUS=(.+)", enos_local_exec.run_blackbox_test.stdout)[0], + "UNKNOWN" + ) + test_exit_code = try( + tonumber(regex("TEST_EXIT_CODE=(.+)", enos_local_exec.run_blackbox_test.stdout)[0]), + null + ) +} diff --git a/enos/modules/vault_run_blackbox_test/outputs.tf b/enos/modules/vault_run_blackbox_test/outputs.tf new file mode 100644 index 0000000000..870103e9ab --- /dev/null +++ b/enos/modules/vault_run_blackbox_test/outputs.tf @@ -0,0 +1,21 @@ +# Copyright IBM Corp. 2016, 2025 +# SPDX-License-Identifier: BUSL-1.1 + +output "test_result" { + description = "The result of the blackbox test execution (human-readable)" + value = enos_local_exec.run_blackbox_test.stdout +} + + +output "test_results_summary" { + description = "Summary of test results for dashboards" + value = { + status = local.test_status + passed = local.test_status == "PASSED" + exit_code = local.test_exit_code + timestamp = timestamp() + json_file = local.json_file_path + test_filter = length(var.test_names) > 0 ? join(", ", var.test_names) : "all tests" + test_package = var.test_package + } +} diff --git a/enos/modules/vault_run_blackbox_test/scripts/run-test.sh b/enos/modules/vault_run_blackbox_test/scripts/run-test.sh new file mode 100755 index 0000000000..aa5d157e55 --- /dev/null +++ b/enos/modules/vault_run_blackbox_test/scripts/run-test.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Copyright IBM Corp. 2016, 2025 +# SPDX-License-Identifier: BUSL-1.1 + +set -euo pipefail + +fail() { + echo "$1" 1>&2 + exit 1 +} + +# Check required environment variables +[[ -z "${VAULT_TOKEN}" ]] && fail "VAULT_TOKEN env variable has not been set" +[[ -z "${VAULT_ADDR}" ]] && fail "VAULT_ADDR env variable has not been set" +[[ -z "${VAULT_TEST_PACKAGE}" ]] && fail "VAULT_TEST_PACKAGE env variable has not been set" + +# Check required dependencies +echo "Checking required dependencies..." + +# Check if Go is installed +if ! command -v go &> /dev/null; then + fail "Go is not installed or not in PATH. Please install Go to run tests." +fi + +# Check if gotestsum is installed (required) +if ! command -v gotestsum &> /dev/null; then + fail "gotestsum is not installed or not in PATH. Please install gotestsum: go install gotest.tools/gotestsum@latest" +fi + +# Check if jq is available (needed for parsing test matrix) +if ! command -v jq &> /dev/null; then + fail "jq is not installed or not in PATH. jq is required to parse test matrix files." +fi + +# Check if git is available (needed for git rev-parse) +if ! command -v git &> /dev/null; then + fail "Git is not installed or not in PATH. Git is required to determine the repository root." +fi + +# Verify we're in a git repository and get the root directory +if ! root_dir="$(git rev-parse --show-toplevel 2> /dev/null)"; then + fail "Not in a git repository. Tests must be run from within the Vault repository." +fi + +echo "All required dependencies are available." +pushd "$root_dir" > /dev/null + +# Create unique output files for test results +timestamp="$(date +%s)_$$" +json_output="/tmp/vault_test_results_${timestamp}.json" +junit_output="/tmp/vault_test_results_${timestamp}.xml" + +echo "Test results will be written to: $json_output" + +# Run tests using gotestsum with JSON output and JUnit reporting +echo "Using gotestsum for enhanced test output and JUnit reporting" +echo "JUnit results will be written to: $junit_output" + +echo "Running tests..." +echo "Vault environment variables:" +env | grep VAULT | sed 's/VAULT_TOKEN=.*/VAULT_TOKEN=***REDACTED***/' + +# Build gotestsum command based on whether we have specific tests +set -x # Show commands being executed +set +e # Temporarily disable exit on error +if [ -n "$VAULT_TEST_MATRIX" ] && [ -f "$VAULT_TEST_MATRIX" ]; then + echo "Using test matrix from: $VAULT_TEST_MATRIX" + # Extract test names from matrix and create regex pattern + test_pattern=$(jq -r '.include[].test' "$VAULT_TEST_MATRIX" | paste -sd '|' -) + echo "Running specific tests: $test_pattern" + gotestsum --junitfile="$junit_output" --format=standard-verbose --jsonfile="$json_output" -- -count=1 -run="$test_pattern" "$VAULT_TEST_PACKAGE" +else + echo "Running all tests in package" + gotestsum --junitfile="$junit_output" --format=standard-verbose --jsonfile="$json_output" -- -count=1 "$VAULT_TEST_PACKAGE" +fi +test_exit_code=$? +set -e # Re-enable exit on error +set +x # Turn off command tracing + +echo "Test execution completed with exit code: $test_exit_code" + +# Check if JSON file was created successfully +if [ -f "$json_output" ] && [ -s "$json_output" ]; then + echo "JSON file created successfully: $(wc -l < "$json_output") lines" + echo "JSON_RESULTS_FILE=$json_output" + + # Check if JUnit file was created (only when using gotestsum) + if [ -f "$junit_output" ] && [ -s "$junit_output" ]; then + echo "JUnit file created successfully: $(wc -l < "$junit_output") lines" + echo "JUNIT_RESULTS_FILE=$junit_output" + else + echo "JUNIT_RESULTS_FILE=" + fi +else + echo "WARNING: Test results file not created or empty" >&2 + echo "TEST_STATUS=ERROR" + echo "TEST_EXIT_CODE=$test_exit_code" + echo "JSON_RESULTS_FILE=" + echo "JUNIT_RESULTS_FILE=" + # Don't exit here - continue to show what we can +fi + +# Also output human-readable results to stdout +echo "=== TEST EXECUTION SUMMARY ===" +if [ $test_exit_code -eq 0 ]; then + echo "✅ Tests PASSED" +else + echo "❌ Tests FAILED (exit code: $test_exit_code)" +fi + +# Parse JSON results and create a summary +echo "=== DETAILED RESULTS ===" +if [ -f "$json_output" ] && [ -s "$json_output" ]; then + if command -v jq &> /dev/null; then + # Use jq if available for better parsing + echo "Test Results Summary (JSON):" + set +e # Temporarily disable exit on error for jq parsing + if jq -e . "$json_output" > /dev/null 2>&1; then + # JSON is valid, proceed with parsing + jq -r 'select(.Action == "pass" or .Action == "fail") | "\(.Time) \(.Action | ascii_upcase) \(.Test // "PACKAGE")"' "$json_output" 2> /dev/null || echo "Failed to parse test results" + else + echo "Invalid JSON in test results file, showing raw output:" + cat "$json_output" 2> /dev/null || echo "Failed to read JSON file" + fi + set -e # Re-enable exit on error + else + # Fallback: show raw JSON output without jq + echo "Test Results (JSON output - install jq for better formatting):" + set +e # Temporarily disable exit on error + if grep -q '"Action"' "$json_output" 2> /dev/null; then + cat "$json_output" 2> /dev/null || echo "Failed to read JSON file" + else + echo "No structured test results found, showing raw output:" + cat "$json_output" 2> /dev/null || echo "Failed to read JSON file" + fi + set -e # Re-enable exit on error + fi +else + echo "No JSON file to parse" +fi + +# Output the JSON file path so Terraform can capture it (if not already output above) +if [ -f "$json_output" ] && [ -s "$json_output" ]; then + echo "JSON_RESULTS_FILE=$json_output" +fi + +popd > /dev/null + +# Always output exit code for Terraform to capture, but exit 0 so script doesn't fail +echo "Final test exit code: $test_exit_code" + +# Exit with the actual test exit code so Terraform fails on test failures +exit $test_exit_code diff --git a/enos/modules/vault_run_blackbox_test/variables.tf b/enos/modules/vault_run_blackbox_test/variables.tf new file mode 100644 index 0000000000..499b44473b --- /dev/null +++ b/enos/modules/vault_run_blackbox_test/variables.tf @@ -0,0 +1,43 @@ +# Copyright IBM Corp. 2016, 2025 +# SPDX-License-Identifier: BUSL-1.1 + +variable "leader_host" { + type = object({ + private_ip = string + public_ip = string + }) + description = "The vault cluster host that is the leader" +} + +variable "leader_public_ip" { + type = string + description = "The public IP of the Vault leader" +} + +variable "vault_root_token" { + type = string + description = "The vault root token" +} + +variable "test_names" { + type = list(string) + description = "List of specific tests to run (e.g., ['TestStepdownAndLeaderElection', 'TestUnsealedStatus']). Empty list runs all tests." + default = [] +} + +variable "test_package" { + type = string + description = "The Go package path for the tests (e.g., ./vault/external_tests/blackbox)" +} + +variable "vault_addr" { + type = string + description = "The full Vault address (for cloud environments). If provided, takes precedence over leader_public_ip." + default = null +} + +variable "vault_namespace" { + type = string + description = "The Vault namespace to operate in (for HCP environments). Optional." + default = null +} diff --git a/sdk/helper/testcluster/blackbox/assertions.go b/sdk/helper/testcluster/blackbox/assertions.go new file mode 100644 index 0000000000..7d138fe276 --- /dev/null +++ b/sdk/helper/testcluster/blackbox/assertions.go @@ -0,0 +1,291 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "encoding/json" + "fmt" + "reflect" + "testing" + + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +type SecretAssertion struct { + t *testing.T + secret *api.Secret +} + +type MapAssertion struct { + t *testing.T + data map[string]any + path string +} + +type SliceAssertion struct { + t *testing.T + data []any + path string +} + +func (s *Session) AssertSecret(secret *api.Secret) *SecretAssertion { + s.t.Helper() + require.NotNil(s.t, secret) + return &SecretAssertion{t: s.t, secret: secret} +} + +func (sa *SecretAssertion) Data() *MapAssertion { + sa.t.Helper() + require.NotNil(sa.t, sa.secret.Data) + return &MapAssertion{t: sa.t, data: sa.secret.Data, path: "Data"} +} + +func (sa *SecretAssertion) KV2() *MapAssertion { + sa.t.Helper() + + require.NotNil(sa.t, sa.secret.Data) + inner, ok := sa.secret.Data["data"] + if !ok { + sa.t.Fatal("data not found in secret") + } + + innerMap, ok := inner.(map[string]any) + if !ok { + sa.t.Fatalf("expected 'data' to be a map, got %T", inner) + } + + return &MapAssertion{t: sa.t, data: innerMap, path: "Data.data"} +} + +func (ma *MapAssertion) HasKey(key string, expected any) *MapAssertion { + ma.t.Helper() + + val, ok := ma.data[key] + if !ok { + ma.t.Fatalf("[%s] missing expected key: %q", ma.path, key) + } + + if !smartCompare(val, expected) { + ma.t.Fatalf("[%s] key %q:\n\texpected: %v\n\tgot: %v", ma.path, key, expected, val) + } + + return ma +} + +func (ma *MapAssertion) HasKeyCustom(key string, f func(val any) bool) *MapAssertion { + ma.t.Helper() + + val, ok := ma.data[key] + if !ok { + ma.t.Fatalf("[%s] missing expected key: %q", ma.path, key) + } + + okAgain := f(val) + if !okAgain { + ma.t.Fatalf("[%s] key %q failed custom check", ma.path, key) + } + + return ma +} + +func (ma *MapAssertion) HasKeyExists(key string) *MapAssertion { + ma.t.Helper() + + if _, ok := ma.data[key]; !ok { + ma.t.Fatalf("[%s] missing expected key: %q", ma.path, key) + } + + return ma +} + +func (ma *MapAssertion) GetMap(key string) *MapAssertion { + ma.t.Helper() + + val, ok := ma.data[key] + if !ok { + ma.t.Fatalf("[%s] missing expected key: %q", ma.path, key) + } + + nestedMap, ok := val.(map[string]any) + if !ok { + ma.t.Fatalf("[%s] key %q is not a map, it is %T", ma.path, key, val) + } + + return &MapAssertion{ + t: ma.t, + data: nestedMap, + path: ma.path + "." + key, + } +} + +func (ma *MapAssertion) GetSlice(key string) *SliceAssertion { + ma.t.Helper() + + val, ok := ma.data[key] + if !ok { + ma.t.Fatalf("[%s] missing expected key: %q", ma.path, key) + } + + slice, ok := val.([]any) + if !ok { + ma.t.Fatalf("[%s] key %q is not a slice, it is %T", ma.path, key, val) + } + + return &SliceAssertion{ + t: ma.t, + data: slice, + path: ma.path, + } +} + +func (sa *SliceAssertion) Length(expected int) *SliceAssertion { + sa.t.Helper() + + if len(sa.data) != expected { + sa.t.Fatalf("[%s] expected slice length %d, got %d", sa.path, expected, len(sa.data)) + } + + return sa +} + +func (sa *SliceAssertion) FindMap(key string, expectedValue any) *MapAssertion { + sa.t.Helper() + + for i, item := range sa.data { + // we expect the slice to contain maps + m, ok := item.(map[string]any) + if !ok { + continue + } + + // check if this map has the key/value we're looking for + if val, exists := m[key]; exists { + if smartCompare(val, expectedValue) { + return &MapAssertion{ + t: sa.t, + data: m, + path: fmt.Sprintf("%s[%d]", sa.path, i), + } + } + } + } + + sa.t.Fatalf("[%s] could not find element with %q == %v", sa.path, key, expectedValue) + return nil +} + +func (sa *SliceAssertion) AllHaveKey(key string, expectedValue any) *SliceAssertion { + sa.t.Helper() + + for i, item := range sa.data { + m, ok := item.(map[string]any) + if !ok { + sa.t.Fatalf("[%s[%d]] expected element to be a map, got %T", sa.path, i, item) + } + + val, exists := m[key] + if !exists { + sa.t.Fatalf("[%s[%d]] missing expected key: %q", sa.path, i, key) + } + + if !smartCompare(val, expectedValue) { + sa.t.Fatalf("[%s[%d]] key %q mismatch:\n\texpected: %v\n\tgot: %v", sa.path, i, key, expectedValue, val) + } + } + + return sa +} + +// AllHaveKeyCustom asserts that every element in the slice is a map +// containing the key, and that the provided function returns true for the value. +func (sa *SliceAssertion) AllHaveKeyCustom(key string, check func(val any) bool) *SliceAssertion { + sa.t.Helper() + + for i, item := range sa.data { + m, ok := item.(map[string]any) + if !ok { + sa.t.Fatalf("[%s[%d]] expected element to be a map, got %T", sa.path, i, item) + } + + val, exists := m[key] + if !exists { + sa.t.Fatalf("[%s[%d]] missing expected key: %q", sa.path, i, key) + } + + if !check(val) { + sa.t.Fatalf("[%s[%d]] key %q failed custom check. Value was: %v", sa.path, i, key, val) + } + } + + return sa +} + +// NoneHaveKeyVal asserts that NO element in the slice contains the specific key/value pair. +// It succeeds if the key is missing, or if the key is present but has a different value. +func (sa *SliceAssertion) NoneHaveKeyVal(key string, restrictedValue any) *SliceAssertion { + sa.t.Helper() + + for i, item := range sa.data { + m, ok := item.(map[string]any) + if !ok { + sa.t.Fatalf("[%s[%d]] expected element to be a map, got %T", sa.path, i, item) + } + + if val, exists := m[key]; exists { + if smartCompare(val, restrictedValue) { + sa.t.Fatalf("[%s[%d]] found restricted key/value pair: %q: %v", sa.path, i, key, val) + } + } + } + + return sa +} + +// smartCompare is designed to get around the weird stuff that happens when Vault's API sometimes +// returns json.Number, sometimes strings with numbers, sometimes actual numbers. It's a mess. +func smartCompare(actual, expected any) bool { + // if they match exactly (type and value), we are done. + if reflect.DeepEqual(actual, expected) { + return true + } + + // if actual is NOT a json.Number, and step 1 failed, they aren't equal. + jNum, isJSON := actual.(json.Number) + if !isJSON { + return false + } + + switch v := expected.(type) { + case int: + // user expects an int (e.g., HasKey("count", 5)) + // json.Number stores as string. we convert to int64, then cast to int. + i64, err := jNum.Int64() + if err != nil { + return false // not a valid integer + } + return int(i64) == v + + case int64: + i64, err := jNum.Int64() + if err != nil { + return false + } + return i64 == v + + case float64: + // user expects float (e.g., HasKey("ttl", 1.5)) + f64, err := jNum.Float64() + if err != nil { + return false + } + return f64 == v + + case string: + // user expects string (e.g. huge ID), just compare string to string + return jNum.String() == v + } + + return false +} diff --git a/sdk/helper/testcluster/blackbox/session.go b/sdk/helper/testcluster/blackbox/session.go new file mode 100644 index 0000000000..c827ab7631 --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session.go @@ -0,0 +1,91 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "os" + "path" + "testing" + + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +// Session holds the test context and Vault client +type Session struct { + t *testing.T + Client *api.Client + Namespace string +} + +func New(t *testing.T) *Session { + t.Helper() + + addr := os.Getenv("VAULT_ADDR") + token := os.Getenv("VAULT_TOKEN") + + // detect the parent namespace, e.g. "admin" in HVD + parentNS := os.Getenv("VAULT_NAMESPACE") + + if addr == "" || token == "" { + t.Fatal("VAULT_ADDR and VAULT_TOKEN are required") + } + + config := api.DefaultConfig() + config.Address = addr + + privClient, err := api.NewClient(config) + require.NoError(t, err) + privClient.SetToken(token) + + nsName := fmt.Sprintf("bbsdk-%s", randomString(8)) + nsURLPath := fmt.Sprintf("sys/namespaces/%s", nsName) + + _, err = privClient.Logical().Write(nsURLPath, nil) + require.NoError(t, err) + + t.Cleanup(func() { + _, err = privClient.Logical().Delete(nsURLPath) + require.NoError(t, err) + t.Logf("Cleaned up namespace %s", nsName) + }) + + // session client should get the full namespace of parent + test + fullNSPath := nsName + if parentNS != "" { + fullNSPath = path.Join(parentNS, nsName) + } + + sessionConfig := privClient.CloneConfig() + sessionClient, err := api.NewClient(sessionConfig) + require.NoError(t, err) + sessionClient.SetToken(token) + sessionClient.SetNamespace(fullNSPath) + + session := &Session{ + t: t, + Client: sessionClient, + Namespace: nsName, + } + + // make sure the namespace has been created + session.Eventually(func() error { + // this runs inside the new namespace, so if it succeeds, we're good + _, err := sessionClient.Auth().Token().LookupSelf() + return err + }) + + return session +} + +func randomString(n int) string { + bytes := make([]byte, n) + if _, err := rand.Read(bytes); err != nil { + panic(err) + } + return hex.EncodeToString(bytes) +} diff --git a/sdk/helper/testcluster/blackbox/session_auth.go b/sdk/helper/testcluster/blackbox/session_auth.go new file mode 100644 index 0000000000..f087eb4106 --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_auth.go @@ -0,0 +1,115 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "fmt" + + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +// Login authenticates against the current namespace and returns a new session object +// acting as that user. +func (s *Session) Login(path string, data map[string]any) *Session { + s.t.Helper() + + newClient := s.newClient() + secret, err := newClient.Logical().Write(path, data) + require.NoError(s.t, err) + + if secret == nil || secret.Auth == nil { + s.t.Fatal("failed to login") + } + + newClient.SetToken(secret.Auth.ClientToken) + + return &Session{ + t: s.t, + Client: newClient, + Namespace: s.Namespace, + } +} + +func (s *Session) NewClientFromToken(token string) *Session { + s.t.Helper() + + newClient := s.newClient() + newClient.SetToken(token) + + return &Session{ + t: s.t, + Client: newClient, + Namespace: s.Namespace, + } +} + +func (s *Session) LoginUserpass(username, password string) *Session { + s.t.Helper() + + path := fmt.Sprintf("auth/userpass/login/%s", username) + payload := map[string]any{ + "password": password, + } + + return s.Login(path, payload) +} + +// TryLoginUserpass attempts to login with userpass but returns error instead of failing test +// This is useful for testing in environments where auth may not be available (e.g., managed HCP) +func (s *Session) TryLoginUserpass(username, password string) (*Session, error) { + s.t.Helper() + + path := fmt.Sprintf("auth/userpass/login/%s", username) + payload := map[string]any{ + "password": password, + } + + secret, err := s.Client.Logical().Write(path, payload) + if err != nil { + return nil, err + } + + clientToken, ok := secret.Auth.ClientToken, secret.Auth != nil + if !ok { + return nil, fmt.Errorf("login response missing client token") + } + + newClient, err := s.Client.Clone() + if err != nil { + return nil, err + } + + newClient.SetToken(clientToken) + return &Session{ + t: s.t, + Client: newClient, + Namespace: s.Namespace, + }, nil +} + +func (s *Session) AssertWriteFails(path string, data map[string]any) { + s.t.Helper() + + _, err := s.Client.Logical().Write(path, data) + require.NotNil(s.t, err) +} + +func (s *Session) AssertReadFails(path string) { + s.t.Helper() + + _, err := s.Client.Logical().Read(path) + require.NotNil(s.t, err) +} + +func (s *Session) newClient() *api.Client { + s.t.Helper() + + parentConfig := s.Client.CloneConfig() + newClient, err := api.NewClient(parentConfig) + require.NoError(s.t, err) + newClient.SetNamespace(s.Namespace) + + return newClient +} diff --git a/sdk/helper/testcluster/blackbox/session_dynamic.go b/sdk/helper/testcluster/blackbox/session_dynamic.go new file mode 100644 index 0000000000..d2af876ab2 --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_dynamic.go @@ -0,0 +1,140 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "database/sql" + "fmt" + "os" + + "github.com/hashicorp/vault/api" + _ "github.com/jackc/pgx/v4/stdlib" + "github.com/stretchr/testify/require" +) + +type DynamicSecret struct { + Secret *api.Secret + LeaseID string + Username string + Password string +} + +func (s *Session) MustGenerateCreds(path string) *DynamicSecret { + s.t.Helper() + + secret := s.MustReadRequired(path) + + ds := &DynamicSecret{ + Secret: secret, + LeaseID: secret.LeaseID, + } + + // usually the creds are in the 'data' map + if val, ok := secret.Data["username"]; ok { + if username, ok := val.(string); ok { + ds.Username = username + } else { + s.t.Fatalf("username field is not a string, got type %T with value %v", val, val) + } + } + if val, ok := secret.Data["password"]; ok { + if password, ok := val.(string); ok { + ds.Password = password + } else { + s.t.Fatalf("password field is not a string, got type %T with value %v", val, val) + } + } + + if ds.Username == "" || ds.Password == "" { + s.t.Fatal("expected username and password to be populated") + } + + return ds +} + +func (s *Session) MustRevokeLease(leaseID string) { + s.t.Helper() + + err := s.Client.Sys().Revoke(leaseID) + require.NoError(s.t, err) +} + +func (s *Session) AssertLeaseExists(leaseID string) { + s.t.Helper() + + _, err := s.Client.Sys().Lookup(leaseID) + require.NoError(s.t, err) +} + +func (s *Session) AssertLeaseRevoked(leaseID string) { + s.t.Helper() + + // when a lease is revoked, Lookup returns an error, so we expect one here + _, err := s.Client.Sys().Lookup(leaseID) + require.Error(s.t, err) +} + +func (s *Session) MustConfigureDBConnection(mountPath, name, plugin, connectionURL string, extraConfig map[string]any) { + s.t.Helper() + + path := fmt.Sprintf("%s/config/%s", mountPath, name) + payload := map[string]any{ + "plugin_name": plugin, + "connection_url": connectionURL, + "allowed_roles": "*", + } + + // merge any extras + for k, v := range extraConfig { + payload[k] = v + } + + s.MustWrite(path, payload) +} + +func (s *Session) MustCreateDBRole(mountPath, roleName, dbName, creationSQL string) { + s.t.Helper() + + path := fmt.Sprintf("%s/roles/%s", mountPath, roleName) + payload := map[string]any{ + "db_name": dbName, + "creation_statements": creationSQL, + "default_ttl": "1h", + "max_ttl": "24h", + } + + s.MustWrite(path, payload) +} + +// MustCheckCreds verifies database credentials work (or don't work) against PostgreSQL. +// Uses POSTGRES_HOST, POSTGRES_PORT, and POSTGRES_DB environment variables, +// defaulting to localhost:5432/vault if not set. +func (s *Session) MustCheckCreds(username, password string, shouldBeValid bool) { + s.t.Helper() + + host := os.Getenv("POSTGRES_HOST") + if host == "" { + host = "localhost" + } + port := os.Getenv("POSTGRES_PORT") + if port == "" { + port = "5432" + } + dbName := os.Getenv("POSTGRES_DB") + if dbName == "" { + dbName = "vault" + } + + connStr := fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=disable", username, password, host, port, dbName) + db, err := sql.Open("pgx", connStr) + require.NoError(s.t, err) + defer func() { _ = db.Close() }() + + err = db.Ping() + if shouldBeValid { + require.NoError(s.t, err) + } else { + require.Error(s.t, err) + } +} diff --git a/sdk/helper/testcluster/blackbox/session_logical.go b/sdk/helper/testcluster/blackbox/session_logical.go new file mode 100644 index 0000000000..c3d8601b8e --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_logical.go @@ -0,0 +1,53 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "path" + + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +func (s *Session) MustWrite(path string, data map[string]any) *api.Secret { + s.t.Helper() + + secret, err := s.Client.Logical().Write(path, data) + require.NoError(s.t, err) + return secret +} + +func (s *Session) MustRead(path string) *api.Secret { + s.t.Helper() + + secret, err := s.Client.Logical().Read(path) + require.NoError(s.t, err) + return secret +} + +// MustReadRequired is a stricter version of MustRead that fails if a 404/nil is returned +func (s *Session) MustReadRequired(path string) *api.Secret { + s.t.Helper() + + secret := s.MustRead(path) + require.NotNil(s.t, secret) + return secret +} + +func (s *Session) MustWriteKV2(mountPath, secretPath string, data map[string]any) { + s.t.Helper() + + fullPath := path.Join(mountPath, "data", secretPath) + payload := map[string]any{ + "data": data, + } + s.MustWrite(fullPath, payload) +} + +func (s *Session) MustReadKV2(mountPath, secretPath string) *api.Secret { + s.t.Helper() + + fullPath := path.Join(mountPath, "data", secretPath) + return s.MustRead(fullPath) +} diff --git a/sdk/helper/testcluster/blackbox/session_ops.go b/sdk/helper/testcluster/blackbox/session_ops.go new file mode 100644 index 0000000000..a6fc756b23 --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_ops.go @@ -0,0 +1,60 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "fmt" + "io" + "net/http" + "os" + "strings" + + "github.com/stretchr/testify/require" +) + +// AssertUIAvailable performs a raw HTTP GET request to the Vault UI +// to ensure it returns a 200 OK and serves HTML. +func (s *Session) AssertUIAvailable() { + s.t.Helper() + + // client.Address() returns the API address (e.g. http://127.0.0.1:8200) + // The UI is usually at /ui/ + uiURL := fmt.Sprintf("%s/ui/", s.Client.Address()) + + resp, err := http.Get(uiURL) + require.NoError(s.t, err) + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + }() + + require.Equal(s.t, http.StatusOK, resp.StatusCode, "UI endpoint returned non-200 status") + + // Optional: Check Content-Type + ct := resp.Header.Get("Content-Type") + if !strings.Contains(ct, "text/html") { + s.t.Fatalf("Expected text/html content type for UI, got %s", ct) + } +} + +// AssertFileContainsSecret scans a file on the local disk (audit log) +// and ensures the provided secret string is NOT present. +func (s *Session) AssertFileDoesNotContainSecret(filePath, secretValue string) { + s.t.Helper() + + if secretValue == "" { + return + } + + content, err := os.ReadFile(filePath) + if os.IsNotExist(err) { + s.t.Fatalf("Audit log file not found: %s", filePath) + } + require.NoError(s.t, err) + + fileBody := string(content) + if strings.Contains(fileBody, secretValue) { + s.t.Fatalf("Security Violation: Found secret value in file %s", filePath) + } +} diff --git a/sdk/helper/testcluster/blackbox/session_pki.go b/sdk/helper/testcluster/blackbox/session_pki.go new file mode 100644 index 0000000000..26153839b4 --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_pki.go @@ -0,0 +1,39 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "path" + + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +// MustSetupPKIRoot bootstraps a PKI engine as a Root CA in one shot. +// It returns the role name you can use to issue certs immediately. +func (s *Session) MustSetupPKIRoot(mountPath string) string { + s.t.Helper() + + s.MustEnableSecretsEngine(mountPath, &api.MountInput{Type: "pki"}) + + // Root CA generation often fails if MaxTTL < requested TTL + err := s.Client.Sys().TuneMount(mountPath, api.MountConfigInput{ + MaxLeaseTTL: "87600h", + }) + require.NoError(s.t, err) + + s.MustWrite(path.Join(mountPath, "root/generate/internal"), map[string]any{ + "common_name": "vault-test-root", + "ttl": "8760h", + }) + + roleName := "server-cert" + s.MustWrite(path.Join(mountPath, "roles", roleName), map[string]any{ + "allowed_domains": "example.com", + "allow_subdomains": true, + "max_ttl": "72h", + }) + + return roleName +} diff --git a/sdk/helper/testcluster/blackbox/session_plugin.go b/sdk/helper/testcluster/blackbox/session_plugin.go new file mode 100644 index 0000000000..35aa2d7cb0 --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_plugin.go @@ -0,0 +1,64 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +func (s *Session) MustRegisterPlugin(pluginName, binaryPath, pluginType string) { + s.t.Helper() + + f, err := os.Open(binaryPath) + require.NoError(s.t, err) + defer func() { _ = f.Close() }() + + hasher := sha256.New() + _, err = io.Copy(hasher, f) + require.NoError(s.t, err) + + shaSum := hex.EncodeToString(hasher.Sum(nil)) + + payload := map[string]any{ + "sha256": shaSum, + "command": filepath.Base(binaryPath), + "type": pluginType, + } + + s.MustWrite(filepath.Join("sys/plugins/catalog", pluginType, pluginName), payload) +} + +func (s *Session) MustEnablePlugin(path, pluginName, pluginType string) { + s.t.Helper() + + switch pluginType { + case "auth": + s.MustEnableAuth(path, &api.EnableAuthOptions{Type: pluginName}) + case "secret": + s.MustEnableSecretsEngine(path, &api.MountInput{Type: pluginName}) + default: + s.t.Fatalf("unknown plugin type: %s", pluginType) + } +} + +func (s *Session) AssertPluginRegistered(pluginName string) { + s.t.Helper() + + secret := s.MustRead(filepath.Join("sys/plugins/catalog", pluginName)) + require.NotNil(s.t, secret) +} + +func (s *Session) AssertPluginConfigured(path string) { + s.t.Helper() + + configPath := filepath.Join(path, "config") + s.MustRead(configPath) +} diff --git a/sdk/helper/testcluster/blackbox/session_raft.go b/sdk/helper/testcluster/blackbox/session_raft.go new file mode 100644 index 0000000000..32cc416125 --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_raft.go @@ -0,0 +1,399 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "time" + + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +func (s *Session) AssertRaftStable(numNodes int, allowNonVoters bool) { + s.t.Helper() + + secret, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/storage/raft/configuration") + }) + + require.NoError(s.t, err) + require.NotNil(s.t, secret) + + assertions := s.AssertSecret(secret). + Data(). + GetMap("config"). + GetSlice("servers"). + Length(numNodes) + + if !allowNonVoters { + assertions.AllHaveKey("voter", true) + } +} + +func (s *Session) AssertRaftHealthy() { + s.t.Helper() + + // query the autopilot state endpoint and verify that all nodes are healthy according to autopilot + secret, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/storage/raft/autopilot/state") + }) + + require.NoError(s.t, err) + require.NotNil(s.t, secret) + + _ = s.AssertSecret(secret). + Data(). + HasKey("healthy", true) +} + +// AssertRaftClusterHealthy verifies that the raft cluster is healthy regardless of node count +// This is a more flexible alternative to AssertRaftStable for cases where you don't know +// or don't care about the exact cluster size, just that it's working properly. +func (s *Session) AssertRaftClusterHealthy() { + s.t.Helper() + + // First verify autopilot reports the cluster as healthy + s.AssertRaftHealthy() + + // Get raft configuration to ensure we have at least one node + secret, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/storage/raft/configuration") + }) + + require.NoError(s.t, err) + require.NotNil(s.t, secret) + + // Verify we have at least one server configured + servers := s.AssertSecret(secret). + Data(). + GetMap("config"). + GetSlice("servers") + + // Ensure we have at least 1 server + if len(servers.data) < 1 { + s.t.Fatal("Expected at least 1 raft server, got 0") + } + + // Verify that we have at least one voter in the cluster + hasVoter := false + for _, server := range servers.data { + if serverMap, ok := server.(map[string]any); ok { + if voter, exists := serverMap["voter"]; exists { + if voterBool, ok := voter.(bool); ok && voterBool { + hasVoter = true + break + } + } + } + } + + if !hasVoter { + s.t.Fatal("Expected at least one voter in the raft cluster") + } +} + +func (s *Session) MustRaftRemovePeer(nodeID string) { + s.t.Helper() + + _, err := s.Client.Logical().Write("sys/storage/raft/remove-peer", map[string]any{ + "server_id": nodeID, + }) + require.NoError(s.t, err) +} + +func (s *Session) AssertRaftPeerRemoved(nodeID string) { + s.t.Helper() + + secret, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/storage/raft/configuration") + }) + + require.NoError(s.t, err) + require.NotNil(s.t, secret) + + _ = s.AssertSecret(secret). + Data(). + GetMap("config"). + GetSlice("servers"). + NoneHaveKeyVal("node_id", nodeID) +} + +// MustGetCurrentLeader returns the current leader's node ID +func (s *Session) MustGetCurrentLeader() string { + s.t.Helper() + + secret, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/leader") + }) + + require.NoError(s.t, err) + require.NotNil(s.t, secret) + + leaderAddress, ok := secret.Data["leader_address"].(string) + require.True(s.t, ok, "leader_address not found or not a string") + require.NotEmpty(s.t, leaderAddress, "leader_address is empty") + + return leaderAddress +} + +// MustStepDownLeader forces the current leader to step down +func (s *Session) MustStepDownLeader() { + s.t.Helper() + + _, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Write("sys/step-down", nil) + }) + + require.NoError(s.t, err) +} + +// GetClusterNodeCount returns the number of nodes in the raft cluster +func (s *Session) GetClusterNodeCount() int { + s.t.Helper() + + secret, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/storage/raft/configuration") + }) + if err != nil { + s.t.Logf("Failed to read raft configuration: %v", err) + return 0 + } + + if secret == nil { + s.t.Log("Raft configuration response was nil") + return 0 + } + + configData, ok := secret.Data["config"].(map[string]any) + if !ok { + s.t.Log("Could not parse raft config data") + return 0 + } + + serversData, ok := configData["servers"].([]any) + if !ok { + s.t.Log("Could not parse raft servers data") + return 0 + } + + return len(serversData) +} + +// WaitForNewLeader waits for a new leader to be elected that is different from initialLeader +// and for the cluster to become healthy. For single-node clusters, it just waits for the +// cluster to become healthy again after stepdown. +func (s *Session) WaitForNewLeader(initialLeader string, timeoutSeconds int) { + s.t.Helper() + + // Check cluster size to handle single-node case + nodeCount := s.GetClusterNodeCount() + if nodeCount <= 1 { + s.t.Logf("Single-node cluster detected, waiting for cluster to recover after stepdown...") + + // For single-node clusters, just wait for the same leader to come back and be healthy + timeout := time.After(time.Duration(timeoutSeconds) * time.Second) + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-timeout: + s.t.Fatalf("Timeout waiting for single-node cluster to recover after %d seconds", timeoutSeconds) + case <-ticker.C: + // Check if cluster is healthy again + secret, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/storage/raft/autopilot/state") + }) + if err != nil { + s.t.Logf("Error reading autopilot state: %v, retrying...", err) + continue + } + + if secret == nil { + s.t.Logf("No autopilot state returned, retrying...") + continue + } + + healthy, ok := secret.Data["healthy"].(bool) + if !ok { + s.t.Logf("Autopilot healthy status not found, retrying...") + continue + } + + if healthy { + s.t.Log("Single-node cluster has recovered and is healthy") + return + } else { + s.t.Logf("Single-node cluster not yet healthy, waiting...") + } + } + } + } + + // Multi-node cluster logic - wait for actual leader change + timeout := time.After(time.Duration(timeoutSeconds) * time.Second) + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + newLeaderFound := false + var currentLeader string + + for { + select { + case <-timeout: + if newLeaderFound { + s.t.Fatalf("Timeout waiting for cluster to become healthy after %d seconds (new leader: %s)", timeoutSeconds, currentLeader) + } else { + s.t.Fatalf("Timeout waiting for new leader election after %d seconds", timeoutSeconds) + } + case <-ticker.C: + // First, check if a new leader has been elected + if !newLeaderFound { + secret, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/leader") + }) + if err != nil { + s.t.Logf("Error reading leader status: %v, retrying...", err) + continue + } + + if secret == nil { + s.t.Logf("No leader data returned, retrying...") + continue + } + + leaderAddress, ok := secret.Data["leader_address"].(string) + if !ok || leaderAddress == "" { + s.t.Logf("No leader address found, retrying...") + continue + } + + if leaderAddress != initialLeader { + s.t.Logf("New leader elected: %s (was: %s)", leaderAddress, initialLeader) + currentLeader = leaderAddress + newLeaderFound = true + } else { + s.t.Logf("Still waiting for new leader, current: %s", leaderAddress) + continue + } + } + + // Once we have a new leader, wait for cluster to be healthy + if newLeaderFound { + secret, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/storage/raft/autopilot/state") + }) + if err != nil { + s.t.Logf("Error reading autopilot state: %v, retrying...", err) + continue + } + + if secret == nil { + s.t.Logf("No autopilot state returned, retrying...") + continue + } + + healthy, ok := secret.Data["healthy"].(bool) + if !ok { + s.t.Logf("Autopilot healthy status not found, retrying...") + continue + } + + if healthy { + s.t.Logf("Cluster is now healthy with new leader: %s", currentLeader) + return + } else { + s.t.Logf("Cluster not yet healthy, waiting...") + } + } + } + } +} + +// AssertClusterHealthy verifies that the cluster is healthy, with fallback for managed environments +// like HCP where raft APIs may not be accessible. This is the recommended method for general +// cluster health checks in blackbox tests. It includes retry logic for Docker environments +// where the cluster may not be immediately ready. +func (s *Session) AssertClusterHealthy() { + s.t.Helper() + + // For Docker environments, wait for the cluster to be ready with retry logic + maxRetries := 30 + retryDelay := 2 * time.Second + + for attempt := 1; attempt <= maxRetries; attempt++ { + // Try raft-based health check first (works for self-managed clusters) + secret, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/storage/raft/autopilot/state") + }) + + if err == nil && secret != nil { + // Check if autopilot reports healthy + if healthy, ok := secret.Data["healthy"].(bool); ok && healthy { + // Raft API is available and healthy, use full raft health check + s.AssertRaftClusterHealthy() + return + } else if ok && !healthy { + // Raft API available but not healthy yet, retry if we have attempts left + if attempt < maxRetries { + s.t.Logf("Cluster not yet healthy (attempt %d/%d), waiting %v...", attempt, maxRetries, retryDelay) + time.Sleep(retryDelay) + continue + } else { + s.t.Fatalf("Cluster failed to become healthy after %d attempts", maxRetries) + } + } + } + + // Raft API not accessible or no healthy status - check basic connectivity + sealStatus, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/seal-status") + }) + if err != nil { + if attempt < maxRetries { + s.t.Logf("Failed to read seal status (attempt %d/%d): %v, retrying in %v...", attempt, maxRetries, err, retryDelay) + time.Sleep(retryDelay) + continue + } + require.NoError(s.t, err, "Failed to read seal status - cluster may be unreachable") + } + + if sealStatus == nil { + if attempt < maxRetries { + s.t.Logf("Seal status response was nil (attempt %d/%d), retrying in %v...", attempt, maxRetries, retryDelay) + time.Sleep(retryDelay) + continue + } + require.NotNil(s.t, sealStatus, "Seal status response was nil") + } + + // Verify cluster is unsealed + sealed, ok := sealStatus.Data["sealed"].(bool) + if !ok { + if attempt < maxRetries { + s.t.Logf("Could not determine seal status (attempt %d/%d), retrying in %v...", attempt, maxRetries, retryDelay) + time.Sleep(retryDelay) + continue + } + require.True(s.t, ok, "Could not determine seal status") + } + + if sealed { + if attempt < maxRetries { + s.t.Logf("Cluster is sealed (attempt %d/%d), retrying in %v...", attempt, maxRetries, retryDelay) + time.Sleep(retryDelay) + continue + } + require.False(s.t, sealed, "Cluster is sealed") + } + + // If we get here, cluster is unsealed and responsive + if secret != nil { + s.t.Log("Cluster health verified (self-managed environment)") + } else { + s.t.Log("Cluster health verified (managed environment - raft APIs not accessible)") + } + return + } +} diff --git a/sdk/helper/testcluster/blackbox/session_status.go b/sdk/helper/testcluster/blackbox/session_status.go new file mode 100644 index 0000000000..e158866653 --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_status.go @@ -0,0 +1,130 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "fmt" + "os/exec" + "strings" + + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +func (s *Session) AssertUnsealed(expectedType string) { + s.t.Helper() + + status, err := s.Client.Sys().SealStatus() + require.NoError(s.t, err) + + if status.Sealed { + s.t.Fatal("Vault is sealed") + } + + if expectedType != "" { + require.Equal(s.t, expectedType, status.Type, "unexpected seal type") + } +} + +// AssertUnsealedAny verifies that the cluster is unsealed regardless of seal type. +// This is useful for environments where the seal type may vary (e.g., HCP uses awskms, Docker uses shamir). +func (s *Session) AssertUnsealedAny() { + s.t.Helper() + + status, err := s.Client.Sys().SealStatus() + require.NoError(s.t, err) + + if status.Sealed { + s.t.Fatal("Vault is sealed") + } + + s.t.Logf("Vault is unsealed (seal type: %s)", status.Type) +} + +func (s *Session) AssertCLIVersion(version, sha, buildDate, edition string) { + s.t.Helper() + + // make sure the binary exists first + _, err := exec.LookPath("vault") + require.NoError(s.t, err) + + cmd := exec.Command("vault", "version") + out, err := cmd.CombinedOutput() + require.NoError(s.t, err) + + output := string(out) + + expectedVersion := fmt.Sprintf("Vault v%s ('%s'), built %s", version, sha, buildDate) + + switch edition { + case "ce", "ent": + case "ent.hsm", "ent.fips1403", "ent.hsm.fips1403": + expectedVersion += " (cgo)" + default: + s.t.Fatalf("unknown Vault edition: %s", edition) + } + + if !strings.Contains(output, expectedVersion) { + s.t.Fatalf("CLI version mismatch. expected %s. got %s", expectedVersion, output) + } +} + +func (s *Session) AssertServerVersion(version string) { + s.t.Helper() + + // strip off any version metadata + b, _, _ := strings.Cut(version, "+") + expectedVersion, _, _ := strings.Cut(b, "-") + + secret, err := s.Client.Logical().List("sys/version-history") + require.NoError(s.t, err) + + keysRaw, ok := secret.Data["keys"].([]any) + if !ok { + s.t.Fatal("sys/version-history missing 'keys'") + } + + found := false + for _, k := range keysRaw { + if kStr, ok := k.(string); ok && kStr == expectedVersion { + found = true + break + } + } + + if !found { + s.t.Fatalf("expected to find %s in version history but didn't", expectedVersion) + } +} + +func (s *Session) AssertReplicationDisabled() { + s.assertReplicationStatus("ce", "disabled") +} + +func (s *Session) AssertDRReplicationStatus(expectedMode string) { + s.assertReplicationStatus("dr", expectedMode) +} + +func (s *Session) AssertPerformanceReplicationStatus(expectedMode string) { + s.assertReplicationStatus("performance", expectedMode) +} + +func (s *Session) assertReplicationStatus(which, expectedMode string) { + s.t.Helper() + + secret, err := s.WithRootNamespace(func() (*api.Secret, error) { + return s.Client.Logical().Read("sys/replication/status") + }) + + require.NoError(s.t, err) + require.NotNil(s.t, secret) + + data := s.AssertSecret(secret).Data() + + if which == "ce" { + data.HasKey("mode", "disabled") + } else { + data.GetMap(which).HasKey("mode", expectedMode) + } +} diff --git a/sdk/helper/testcluster/blackbox/session_sys.go b/sdk/helper/testcluster/blackbox/session_sys.go new file mode 100644 index 0000000000..9baba9ee11 --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_sys.go @@ -0,0 +1,37 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +func (s *Session) MustEnableSecretsEngine(path string, input *api.MountInput) { + s.t.Helper() + + err := s.Client.Sys().Mount(path, input) + require.NoError(s.t, err) +} + +func (s *Session) MustDisableSecretsEngine(path string) { + s.t.Helper() + + err := s.Client.Sys().Unmount(path) + require.NoError(s.t, err) +} + +func (s *Session) MustEnableAuth(path string, options *api.EnableAuthOptions) { + s.t.Helper() + + err := s.Client.Sys().EnableAuthWithOptions(path, options) + require.NoError(s.t, err) +} + +func (s *Session) MustWritePolicy(name, rules string) { + s.t.Helper() + + err := s.Client.Sys().PutPolicy(name, rules) + require.NoError(s.t, err) +} diff --git a/sdk/helper/testcluster/blackbox/session_token.go b/sdk/helper/testcluster/blackbox/session_token.go new file mode 100644 index 0000000000..def7992a0f --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_token.go @@ -0,0 +1,94 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +type TokenOptions struct { + Policies []string + TTL string + Renewable bool + NoParent bool + DisplayName string +} + +// MustCreateToken generates a new token with specific properties. +func (s *Session) MustCreateToken(opts TokenOptions) string { + s.t.Helper() + + payload := map[string]any{ + "policies": opts.Policies, + "ttl": opts.TTL, + "renewable": opts.Renewable, + "no_parent": opts.NoParent, + "display_name": opts.DisplayName, + } + + // Use auth/token/create for child tokens, or auth/token/create-orphan + path := "auth/token/create" + if opts.NoParent { + path = "auth/token/create-orphan" + } + + secret := s.MustWrite(path, payload) + if secret.Auth == nil { + s.t.Fatal("Token creation response missing Auth data") + } + + return secret.Auth.ClientToken +} + +// AssertTokenIsValid checks that a token works and (optionally) has specific policies. +func (s *Session) AssertTokenIsValid(token string, expectedPolicies ...string) { + s.t.Helper() + + if token == "" { + s.t.Fatal("token is empty") + } + + clonedConfig := s.Client.CloneConfig() + tempClient, err := api.NewClient(clonedConfig) + require.NoError(s.t, err) + + tempClient.SetToken(token) + tempClient.SetNamespace(s.Namespace) + + secret, err := tempClient.Auth().Token().LookupSelf() + require.NoError(s.t, err) + + if len(expectedPolicies) == 0 { + return + } + + rawPolicies, ok := secret.Data["policies"].([]any) + if !ok { + s.t.Fatalf("token does not contain any policies") + } + + actualPolicies := make(map[string]struct{}) + for _, p := range rawPolicies { + if val, ok := p.(string); ok { + actualPolicies[val] = struct{}{} + } + } + + var missing []string + for _, expected := range expectedPolicies { + if _, ok := actualPolicies[expected]; !ok { + missing = append(missing, expected) + } + } + + if len(missing) > 0 { + allActual := make([]string, 0, len(actualPolicies)) + for k := range actualPolicies { + allActual = append(allActual, k) + } + + s.t.Fatalf("token policy mismatch.\n\tmissing: %v\n\tactual: %v", missing, allActual) + } +} diff --git a/sdk/helper/testcluster/blackbox/session_util.go b/sdk/helper/testcluster/blackbox/session_util.go new file mode 100644 index 0000000000..c079a52b3e --- /dev/null +++ b/sdk/helper/testcluster/blackbox/session_util.go @@ -0,0 +1,43 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "time" + + "github.com/hashicorp/vault/api" +) + +// Eventually retries the function 'fn' until it returns nil or timeout occurs. +func (s *Session) Eventually(fn func() error) { + s.t.Helper() + + timeout := time.After(5 * time.Second) + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + + var lastErr error + + for { + select { + case <-timeout: + s.t.Fatalf("Eventually failed after 5s. Last error: %v", lastErr) + case <-ticker.C: + lastErr = fn() + if lastErr == nil { + return + } + } + } +} + +func (s *Session) WithRootNamespace(fn func() (*api.Secret, error)) (*api.Secret, error) { + s.t.Helper() + + oldNamespace := s.Client.Namespace() + defer s.Client.SetNamespace(oldNamespace) + s.Client.ClearNamespace() + + return fn() +} diff --git a/vault/external_tests/blackbox/auth_engines_test.go b/vault/external_tests/blackbox/auth_engines_test.go new file mode 100644 index 0000000000..7cc91b5b8a --- /dev/null +++ b/vault/external_tests/blackbox/auth_engines_test.go @@ -0,0 +1,218 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "testing" + + "github.com/hashicorp/vault/api" + "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" +) + +// TestAuthEngineCreate tests creation/setup of various auth engines +func TestAuthEngineCreate(t *testing.T) { + v := blackbox.New(t) + + // Verify we have a healthy cluster first + v.AssertClusterHealthy() + + // Test userpass auth engine + t.Run("UserpassAuth", func(t *testing.T) { + testUserpassAuthCreate(t, v) + }) + + // Stub out remaining auth engine creation tests + t.Run("LDAPAuth", func(t *testing.T) { + t.Skip("LDAP auth engine create test - implementation pending") + }) + + t.Run("OIDCAuth", func(t *testing.T) { + t.Skip("OIDC auth engine create test - implementation pending") + }) + + t.Run("AWSAuth", func(t *testing.T) { + t.Skip("AWS auth engine create test - implementation pending") + }) + + t.Run("KubernetesAuth", func(t *testing.T) { + t.Skip("Kubernetes auth engine create test - implementation pending") + }) + + t.Run("AppRoleAuth", func(t *testing.T) { + t.Skip("AppRole auth engine create test - implementation pending") + }) + + t.Run("CertAuth", func(t *testing.T) { + t.Skip("Cert auth engine create test - implementation pending") + }) +} + +// TestAuthEngineRead tests read operations for various auth engines +func TestAuthEngineRead(t *testing.T) { + v := blackbox.New(t) + + // Verify we have a healthy cluster first + v.AssertClusterHealthy() + + // Test userpass auth engine read operations + t.Run("UserpassAuth", func(t *testing.T) { + testUserpassAuthRead(t, v) + }) + + // Stub out remaining auth engine read tests + t.Run("LDAPAuth", func(t *testing.T) { + t.Skip("LDAP auth engine read test - implementation pending") + }) + + t.Run("OIDCAuth", func(t *testing.T) { + t.Skip("OIDC auth engine read test - implementation pending") + }) + + t.Run("AWSAuth", func(t *testing.T) { + t.Skip("AWS auth engine read test - implementation pending") + }) + + t.Run("KubernetesAuth", func(t *testing.T) { + t.Skip("Kubernetes auth engine read test - implementation pending") + }) + + t.Run("AppRoleAuth", func(t *testing.T) { + t.Skip("AppRole auth engine read test - implementation pending") + }) + + t.Run("CertAuth", func(t *testing.T) { + t.Skip("Cert auth engine read test - implementation pending") + }) +} + +// TestAuthEngineDelete tests delete operations for various auth engines +func TestAuthEngineDelete(t *testing.T) { + v := blackbox.New(t) + + // Verify we have a healthy cluster first + v.AssertClusterHealthy() + + // Test userpass auth engine delete operations + t.Run("UserpassAuth", func(t *testing.T) { + testUserpassAuthDelete(t, v) + }) + + // Stub out remaining auth engine delete tests + t.Run("LDAPAuth", func(t *testing.T) { + t.Skip("LDAP auth engine delete test - implementation pending") + }) + + t.Run("OIDCAuth", func(t *testing.T) { + t.Skip("OIDC auth engine delete test - implementation pending") + }) + + t.Run("AWSAuth", func(t *testing.T) { + t.Skip("AWS auth engine delete test - implementation pending") + }) + + t.Run("KubernetesAuth", func(t *testing.T) { + t.Skip("Kubernetes auth engine delete test - implementation pending") + }) + + t.Run("AppRoleAuth", func(t *testing.T) { + t.Skip("AppRole auth engine delete test - implementation pending") + }) + + t.Run("CertAuth", func(t *testing.T) { + t.Skip("Cert auth engine delete test - implementation pending") + }) +} + +// Userpass Auth Engine Test Implementation Functions + +func testUserpassAuthCreate(t *testing.T, v *blackbox.Session) { + // Create a policy for our test user + userPolicy := ` + path "*" { + capabilities = ["read", "list"] + } + ` + + // Use common utility to setup userpass auth + userClient := SetupUserpassAuth(v, "testuser", "passtestuser1", "reguser", userPolicy) + + // Verify the auth method was enabled by reading auth mounts + authMounts := v.MustRead("sys/auth") + if authMounts.Data == nil { + t.Fatal("Could not read auth mounts") + } + + // Verify userpass auth method is enabled + if userpassAuth, ok := authMounts.Data["userpass/"]; !ok { + t.Fatal("userpass auth method not found in sys/auth") + } else { + userpassMap := userpassAuth.(map[string]any) + if userpassMap["type"] != "userpass" { + t.Fatalf("Expected userpass auth method type to be 'userpass', got: %v", userpassMap["type"]) + } + } + + // Test that the user session was created successfully + if userClient != nil { + // Login successful, verify we can read basic info + tokenInfo := userClient.MustRead("auth/token/lookup-self") + if tokenInfo.Data == nil { + t.Fatal("Expected user to be able to read own token info after login") + } + t.Log("Userpass login test successful") + } else { + t.Log("Userpass login not available (likely managed environment)") + } + + t.Log("Successfully created userpass auth with user: testuser") +} + +func testUserpassAuthRead(t *testing.T, v *blackbox.Session) { + // Use common utility to setup userpass auth with default policy + userClient := SetupUserpassAuth(v, "readuser", "readpass123", "default", "") + + // Read the user configuration + userConfig := v.MustRead("auth/userpass/users/readuser") + if userConfig.Data == nil { + t.Fatal("Expected to read user configuration") + } + + // Test that the user session was created successfully + if userClient != nil { + // Login successful, verify we can read basic info + tokenInfo := userClient.MustRead("auth/token/lookup-self") + if tokenInfo.Data == nil { + t.Fatal("Expected user to be able to read own token info after login") + } + t.Log("Userpass login test successful") + } else { + t.Log("Userpass login not available (likely managed environment)") + } + + t.Log("Successfully read userpass auth config for user: readuser") +} + +func testUserpassAuthDelete(t *testing.T, v *blackbox.Session) { + // Enable userpass auth method with unique mount for delete test + v.MustEnableAuth("userpass-delete", &api.EnableAuthOptions{Type: "userpass"}) + + // Create a user to delete + userName := "deleteuser" + userPassword := "deletepass123" + v.MustWrite("auth/userpass-delete/users/"+userName, map[string]any{ + "password": userPassword, + "policies": "default", + }) + + // Verify the user exists + userConfig := v.MustRead("auth/userpass-delete/users/" + userName) + if userConfig.Data == nil { + t.Fatal("Expected user to exist before deletion") + } + + // Delete the user + v.MustWrite("auth/userpass-delete/users/"+userName, nil) + + t.Logf("Successfully deleted userpass auth user: %s", userName) +} diff --git a/vault/external_tests/blackbox/dynamic_test.go b/vault/external_tests/blackbox/dynamic_test.go new file mode 100644 index 0000000000..bae377488c --- /dev/null +++ b/vault/external_tests/blackbox/dynamic_test.go @@ -0,0 +1,49 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "fmt" + "os" + "testing" + + "github.com/hashicorp/vault/api" + "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" +) + +// TestPostgresDynamicSecrets verifies the database dynamic secrets engine functionality +// by configuring a PostgreSQL connection, creating a role, generating credentials, +// and testing the full lifecycle including credential revocation. +func TestPostgresDynamicSecrets(t *testing.T) { + v := blackbox.New(t) + + user := os.Getenv("POSTGRES_USER") + pass := os.Getenv("POSTGRES_PASSWORD") + db := os.Getenv("POSTGRES_DB") + connURL := fmt.Sprintf("postgres://{{username}}:{{password}}@localhost:5432/%s?sslmode=disable", db) + + v.MustEnableSecretsEngine("database", &api.MountInput{Type: "database"}) + v.MustConfigureDBConnection( + "database", + "my-postgres", + "postgresql-database-plugin", + connURL, + map[string]any{ + "username": user, + "password": pass, + }, + ) + + creationSQL := `CREATE ROLE "{{name}}" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';` + v.MustCreateDBRole("database", "readonly-role", "my-postgres", creationSQL) + + creds := v.MustGenerateCreds("database/creds/readonly-role") + t.Logf("generated DB user/pass: %s / %s", creds.Username, creds.Password) + + v.AssertLeaseExists(creds.LeaseID) + v.MustCheckCreds(creds.Username, creds.Password, true) + v.MustRevokeLease(creds.LeaseID) + v.AssertLeaseRevoked(creds.LeaseID) + v.MustCheckCreds(creds.Username, creds.Password, false) +} diff --git a/vault/external_tests/blackbox/kvv2_test.go b/vault/external_tests/blackbox/kvv2_test.go new file mode 100644 index 0000000000..906f585d3a --- /dev/null +++ b/vault/external_tests/blackbox/kvv2_test.go @@ -0,0 +1,79 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "testing" + + "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" +) + +// TestKVv2_SoftDeleteAndRestore_Workflow tests the complete workflow of KV v2 soft delete and restore +func TestKVv2_SoftDeleteAndRestore_Workflow(t *testing.T) { + v := blackbox.New(t) + + // Setup KV engine and authenticated user using common utilities + bob := SetupStandardKVUserpass(v, "secret", "bob", "lol") + + // Write initial data using standard test data + testData := map[string]any{ + "api_key": "A1B2-C3D4", + "is_active": true, + "retry_count": 3, + } + v.MustWriteKV2("secret", "app-config", testData) + + // Verify data can be read + secret := bob.MustReadKV2("secret", "app-config") + AssertKVData(t, bob, secret, testData) + + // Perform soft delete + bob.MustWrite("secret/delete/app-config", map[string]any{ + "versions": []int{1}, + }) + + // Verify data is deleted + deletedSecret := bob.MustReadRequired("secret/data/app-config") + if deletedSecret.Data["data"] != nil { + t.Fatal("Expected secret data to be nil after soft delete, but got data") + } + + // Restore the data + bob.MustWrite("secret/undelete/app-config", map[string]any{ + "versions": []int{1}, + }) + + // Verify data is restored + restoredSecret := bob.MustReadRequired("secret/data/app-config") + bob.AssertSecret(restoredSecret). + KV2(). + HasKey("api_key", "A1B2-C3D4") +} + +// TestKVv2_BasicOperations tests basic KV v2 create, read, update operations +func TestKVv2_BasicOperations(t *testing.T) { + v := blackbox.New(t) + + // Setup using common utilities + user := SetupStandardKVUserpass(v, "kv-basic", "testuser", "testpass") + + // Test create + user.MustWriteKV2("kv-basic", "test/data", StandardKVData) + + // Test read + secret := user.MustReadKV2("kv-basic", "test/data") + AssertKVData(t, user, secret, StandardKVData) + + // Test update + updatedData := map[string]any{ + "api_key": "updated-key-456", + "is_active": false, + "retry_count": 5, + } + user.MustWriteKV2("kv-basic", "test/data", updatedData) + + // Verify update + updatedSecret := user.MustReadKV2("kv-basic", "test/data") + AssertKVData(t, user, updatedSecret, updatedData) +} diff --git a/vault/external_tests/blackbox/pki_test.go b/vault/external_tests/blackbox/pki_test.go new file mode 100644 index 0000000000..d3bdf795e6 --- /dev/null +++ b/vault/external_tests/blackbox/pki_test.go @@ -0,0 +1,49 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "fmt" + "testing" + + "github.com/hashicorp/vault/api" + "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" +) + +// TestPKI_IssueCertificate verifies PKI engine functionality by setting up a root CA, +// issuing a certificate with specific parameters, and validating the certificate +// response contains the expected fields and values. +func TestPKI_IssueCertificate(t *testing.T) { + v := blackbox.New(t) + + roleName := v.MustSetupPKIRoot("pki") + + // issue a cert + issuePath := fmt.Sprintf("pki/issue/%s", roleName) + payload := map[string]any{ + "common_name": "api.example.com", + "ttl": "24h", + } + + var secret *api.Secret + v.Eventually(func() error { + var err error + secret, err = v.Client.Logical().Write(issuePath, payload) + return err + }) + + if secret == nil { + t.Fatal("Expected certificate secret, got nil") + } + + assertions := v.AssertSecret(secret) + assertions.Data(). + HasKeyExists("certificate"). + HasKeyExists("issuing_ca"). + HasKeyExists("private_key"). + HasKeyCustom("serial_number", func(val any) bool { + s, ok := val.(string) + return ok && len(s) > 0 + }) +} diff --git a/vault/external_tests/blackbox/secrets_engines_test.go b/vault/external_tests/blackbox/secrets_engines_test.go new file mode 100644 index 0000000000..72f1d7edd3 --- /dev/null +++ b/vault/external_tests/blackbox/secrets_engines_test.go @@ -0,0 +1,289 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "testing" + + "github.com/hashicorp/vault/api" + "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" +) + +// TestSecretsEngineCreate tests creation/setup of various secrets engines +func TestSecretsEngineCreate(t *testing.T) { + v := blackbox.New(t) + + // Verify we have a healthy cluster first + v.AssertClusterHealthy() + + // KV secrets engine tests are now in kvv2_test.go - just test basic enablement here + t.Run("KVSecrets", func(t *testing.T) { + SetupKVEngine(v, "kv-create") + // Write and read test data to verify engine works + v.MustWriteKV2("kv-create", "test/path", StandardKVData) + secret := v.MustReadKV2("kv-create", "test/path") + AssertKVData(t, v, secret, StandardKVData) + t.Log("Successfully created and tested KV secrets engine") + }) + + // Stub out remaining secret engine creation tests + t.Run("PKISecrets", func(t *testing.T) { + t.Skip("PKI secrets engine create test - implementation pending") + }) + + t.Run("SSHSecrets", func(t *testing.T) { + t.Skip("SSH secrets engine create test - implementation pending") + }) + + t.Run("IdentitySecrets", func(t *testing.T) { + t.Skip("Identity secrets engine create test - implementation pending") + }) + + t.Run("AWSSecrets", func(t *testing.T) { + t.Skip("AWS secrets engine create test - implementation pending") + }) + + t.Run("LDAPSecrets", func(t *testing.T) { + t.Skip("LDAP secrets engine create test - implementation pending") + }) + + t.Run("KMIPSecrets", func(t *testing.T) { + t.Skip("KMIP secrets engine create test - implementation pending") + }) + + t.Run("DatabaseSecrets", func(t *testing.T) { + t.Skip("Database secrets engine create test - implementation pending") + }) + + t.Run("TransitSecrets", func(t *testing.T) { + testTransitSecretsCreate(t, v) + }) +} + +// TestSecretsEngineRead tests read operations for various secrets engines +func TestSecretsEngineRead(t *testing.T) { + v := blackbox.New(t) + + // Verify we have a healthy cluster first + v.AssertClusterHealthy() + + // KV read tests are in kvv2_test.go - test basic read functionality here + t.Run("KVSecrets", func(t *testing.T) { + SetupKVEngine(v, "kv-read") + v.MustWriteKV2("kv-read", "read/test", AltKVData) + secret := v.MustReadKV2("kv-read", "read/test") + AssertKVData(t, v, secret, AltKVData) + t.Log("Successfully read KV secrets engine data") + }) + + // Stub out remaining secret engine read tests + t.Run("PKISecrets", func(t *testing.T) { + t.Skip("PKI secrets engine read test - implementation pending") + }) + + t.Run("SSHSecrets", func(t *testing.T) { + t.Skip("SSH secrets engine read test - implementation pending") + }) + + t.Run("IdentitySecrets", func(t *testing.T) { + t.Skip("Identity secrets engine read test - implementation pending") + }) + + t.Run("AWSSecrets", func(t *testing.T) { + t.Skip("AWS secrets engine read test - implementation pending") + }) + + t.Run("LDAPSecrets", func(t *testing.T) { + t.Skip("LDAP secrets engine read test - implementation pending") + }) + + t.Run("KMIPSecrets", func(t *testing.T) { + t.Skip("KMIP secrets engine read test - implementation pending") + }) + + t.Run("DatabaseSecrets", func(t *testing.T) { + t.Skip("Database secrets engine read test - implementation pending") + }) + + t.Run("TransitSecrets", func(t *testing.T) { + t.Skip("Transit secrets engine read test - implementation pending") + }) +} + +// TestSecretsEngineDelete tests delete operations for various secrets engines +func TestSecretsEngineDelete(t *testing.T) { + v := blackbox.New(t) + + // Verify we have a healthy cluster first + v.AssertClusterHealthy() + + // KV delete tests are in kvv2_test.go - test basic delete functionality here + t.Run("KVSecrets", func(t *testing.T) { + SetupKVEngine(v, "kv-delete") + + // Write test data + v.MustWriteKV2("kv-delete", "delete/test", StandardKVData) + + // Verify it exists + secret := v.MustReadKV2("kv-delete", "delete/test") + AssertKVData(t, v, secret, StandardKVData) + + // Delete using KV v2 delete endpoint + v.MustWrite("kv-delete/delete/delete/test", map[string]any{ + "versions": []int{1}, + }) + + t.Log("Successfully deleted KV secrets engine data") + }) + + // Stub out remaining secret engine delete tests + t.Run("PKISecrets", func(t *testing.T) { + t.Skip("PKI secrets engine delete test - implementation pending") + }) + + t.Run("SSHSecrets", func(t *testing.T) { + t.Skip("SSH secrets engine delete test - implementation pending") + }) + + t.Run("IdentitySecrets", func(t *testing.T) { + t.Skip("Identity secrets engine delete test - implementation pending") + }) + + t.Run("AWSSecrets", func(t *testing.T) { + t.Skip("AWS secrets engine delete test - implementation pending") + }) + + t.Run("LDAPSecrets", func(t *testing.T) { + t.Skip("LDAP secrets engine delete test - implementation pending") + }) + + t.Run("KMIPSecrets", func(t *testing.T) { + t.Skip("KMIP secrets engine delete test - implementation pending") + }) + + t.Run("DatabaseSecrets", func(t *testing.T) { + t.Skip("Database secrets engine delete test - implementation pending") + }) + + t.Run("TransitSecrets", func(t *testing.T) { + testTransitSecretsDelete(t, v) + }) +} + +// Transit Secrets Engine Test Implementation Functions + +func testTransitSecretsCreate(t *testing.T, v *blackbox.Session) { + // Enable transit secrets engine + v.MustEnableSecretsEngine("transit", &api.MountInput{Type: "transit"}) + + // Create an encryption key + keyName := "test-key" + v.MustWrite("transit/keys/"+keyName, map[string]any{ + "type": "aes256-gcm96", + }) + + // Verify the key was created by reading it + keyInfo := v.MustRead("transit/keys/" + keyName) + if keyInfo.Data == nil { + t.Fatal("Expected to read key configuration") + } + + // Verify key type + if keyType, ok := keyInfo.Data["type"]; !ok || keyType != "aes256-gcm96" { + t.Fatalf("Expected key type 'aes256-gcm96', got: %v", keyInfo.Data["type"]) + } + + // Test encryption + plaintext := "dGhlIHF1aWNrIGJyb3duIGZveA==" // base64 encoded "the quick brown fox" + encryptResp := v.MustWrite("transit/encrypt/"+keyName, map[string]any{ + "plaintext": plaintext, + }) + + if encryptResp.Data == nil || encryptResp.Data["ciphertext"] == nil { + t.Fatal("Expected ciphertext in encryption response") + } + + ciphertext := encryptResp.Data["ciphertext"].(string) + t.Logf("Encrypted ciphertext: %s", ciphertext[:20]+"...") + + // Test decryption + decryptResp := v.MustWrite("transit/decrypt/"+keyName, map[string]any{ + "ciphertext": ciphertext, + }) + + if decryptResp.Data == nil || decryptResp.Data["plaintext"] == nil { + t.Fatal("Expected plaintext in decryption response") + } + + decryptedText := decryptResp.Data["plaintext"].(string) + if decryptedText != plaintext { + t.Fatalf("Decrypted text doesn't match original. Expected: %s, Got: %s", plaintext, decryptedText) + } + + t.Log("Successfully created transit secrets engine and tested encryption/decryption") +} + +func testTransitSecretsRead(t *testing.T, v *blackbox.Session) { + // Enable transit secrets engine with unique mount + v.MustEnableSecretsEngine("transit-read", &api.MountInput{Type: "transit"}) + + // Create an encryption key + keyName := "read-test-key" + v.MustWrite("transit-read/keys/"+keyName, map[string]any{ + "type": "aes256-gcm96", + "exportable": false, + }) + + // Read the key configuration + keyInfo := v.MustRead("transit-read/keys/" + keyName) + if keyInfo.Data == nil { + t.Fatal("Expected to read key configuration") + } + + // Verify key properties + assertions := v.AssertSecret(keyInfo) + assertions.Data(). + HasKey("type", "aes256-gcm96"). + HasKey("exportable", false). + HasKeyExists("keys"). + HasKeyExists("latest_version") + + t.Log("Successfully read transit secrets engine key configuration") +} + +func testTransitSecretsDelete(t *testing.T, v *blackbox.Session) { + // Enable transit secrets engine with unique mount + v.MustEnableSecretsEngine("transit-delete", &api.MountInput{Type: "transit"}) + + // Create an encryption key + keyName := "delete-test-key" + v.MustWrite("transit-delete/keys/"+keyName, map[string]any{ + "type": "aes256-gcm96", + }) + + // Verify the key exists + keyInfo := v.MustRead("transit-delete/keys/" + keyName) + if keyInfo.Data == nil { + t.Fatal("Expected key to exist before deletion") + } + + // Configure the key to allow deletion (transit keys require this) + v.MustWrite("transit-delete/keys/"+keyName+"/config", map[string]any{ + "deletion_allowed": true, + }) + + // Delete the key + _, err := v.Client.Logical().Delete("transit-delete/keys/" + keyName) + if err != nil { + t.Fatalf("Failed to delete transit key: %v", err) + } + + // Verify the key is deleted by attempting to read it + readSecret, err := v.Client.Logical().Read("transit-delete/keys/" + keyName) + if err == nil && readSecret != nil { + t.Fatal("Expected key to be deleted, but it still exists") + } + + t.Logf("Successfully deleted transit key: %s", keyName) +} diff --git a/vault/external_tests/blackbox/smoke_test.go b/vault/external_tests/blackbox/smoke_test.go new file mode 100644 index 0000000000..461cab6d32 --- /dev/null +++ b/vault/external_tests/blackbox/smoke_test.go @@ -0,0 +1,71 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "testing" + + "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" +) + +// TestEnosSmoke performs comprehensive smoke testing for Enos scenarios, +// verifying cluster health, replication status, raft stability, and basic +// KV operations with authentication. This test validates core functionality. +func TestEnosSmoke(t *testing.T) { + v := blackbox.New(t) + + v.AssertUnsealedAny() + v.AssertDRReplicationStatus("primary") + v.AssertPerformanceReplicationStatus("disabled") + v.AssertRaftStable(3, false) + v.AssertRaftHealthy() + + // Setup using common utilities + bob := SetupStandardKVUserpass(v, "secret", "bob", "lol") + + // Write and verify standard test data + v.MustWriteKV2("secret", "app-config", StandardKVData) + + secret := bob.MustReadKV2("secret", "app-config") + AssertKVData(t, bob, secret, StandardKVData) +} + +// TestStepdownAndLeaderElection tests raft leadership changes by forcing the current +// leader to step down and verifying that a new leader is elected successfully, +// while ensuring the cluster remains healthy throughout the process. +func TestStepdownAndLeaderElection(t *testing.T) { + v := blackbox.New(t) + + // Verify we have a healthy raft cluster first + v.AssertRaftClusterHealthy() + + // Check cluster size to determine expected behavior + nodeCount := v.GetClusterNodeCount() + t.Logf("Cluster has %d nodes", nodeCount) + + // Get current leader before step down + initialLeader := v.MustGetCurrentLeader() + t.Logf("Initial leader: %s", initialLeader) + + // Force leader to step down + v.MustStepDownLeader() + + // Wait for new leader election (with timeout) + v.WaitForNewLeader(initialLeader, 120) + + // Verify cluster is still healthy after leader change/recovery + v.AssertRaftClusterHealthy() + + // For multi-node clusters, verify new leader is different from initial leader + // For single-node clusters, just verify it's healthy again + newLeader := v.MustGetCurrentLeader() + if nodeCount > 1 { + if newLeader == initialLeader { + t.Fatalf("Expected new leader to be different from initial leader %s, got %s", initialLeader, newLeader) + } + t.Logf("Successfully elected new leader: %s (was: %s)", newLeader, initialLeader) + } else { + t.Logf("Single-node cluster successfully recovered with leader: %s", newLeader) + } +} diff --git a/vault/external_tests/blackbox/system_test.go b/vault/external_tests/blackbox/system_test.go new file mode 100644 index 0000000000..70433696c7 --- /dev/null +++ b/vault/external_tests/blackbox/system_test.go @@ -0,0 +1,105 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "testing" + + "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" +) + +// TestUnsealedStatus verifies that the Vault cluster is unsealed and healthy +func TestUnsealedStatus(t *testing.T) { + v := blackbox.New(t) + + // Verify the cluster is unsealed + v.AssertUnsealedAny() + + t.Log("Successfully verified Vault cluster is unsealed") +} + +// TestVaultVersion verifies Vault version endpoint accessibility and response +func TestVaultVersion(t *testing.T) { + v := blackbox.New(t) + + // Read the sys/seal-status endpoint which should contain version info + sealStatus := v.MustRead("sys/seal-status") + if sealStatus.Data["version"] == nil { + t.Fatal("Could not retrieve version from sys/seal-status") + } + + t.Logf("Vault version: %v", sealStatus.Data["version"]) +} + +// TestRaftVoters verifies that all nodes in the raft cluster are voters +func TestRaftVoters(t *testing.T) { + v := blackbox.New(t) + + // Verify we have a healthy cluster regardless of node count + v.AssertClusterHealthy() + + t.Log("Successfully verified raft cluster is healthy with at least one voter") +} + +// TestReplicationStatus verifies replication status for both DR and performance replication +func TestReplicationStatus(t *testing.T) { + v := blackbox.New(t) + + // Read replication status with proper nil checks + drStatus := v.MustRead("sys/replication/dr/status") + if drStatus == nil || drStatus.Data == nil { + t.Log("DR replication not available or not configured - skipping DR replication check") + } else { + if drMode, ok := drStatus.Data["mode"]; ok { + t.Logf("DR replication mode: %v", drMode) + } else { + t.Log("DR replication mode not available") + } + } + + prStatus := v.MustRead("sys/replication/performance/status") + if prStatus == nil || prStatus.Data == nil { + t.Log("Performance replication not available or not configured - skipping performance replication check") + } else { + if prMode, ok := prStatus.Data["mode"]; ok { + t.Logf("Performance replication mode: %v", prMode) + } else { + t.Log("Performance replication mode not available") + } + } + + t.Log("Successfully verified replication status endpoints are accessible") +} + +// TestUIAssets verifies that the Vault UI is accessible +func TestUIAssets(t *testing.T) { + v := blackbox.New(t) + + // This is a stub - in a real implementation, you would verify UI assets are accessible + // For now, just verify the UI endpoint is available by checking sys/internal/ui/mounts + uiMounts := v.MustRead("sys/internal/ui/mounts") + if uiMounts == nil || uiMounts.Data == nil { + t.Fatal("Could not access UI mounts endpoint") + } + + t.Log("Successfully verified UI assets are accessible") +} + +// TestLogSecrets is a stub for log secrets verification +func TestLogSecrets(t *testing.T) { + // This is a stub for log secrets verification + // In a real implementation, you would check audit logs for proper secret handling + t.Skip("Log secrets verification - implementation pending") +} + +// TestNodeRemovalAndRejoin tests raft node removal and rejoin capabilities +func TestNodeRemovalAndRejoin(t *testing.T) { + v := blackbox.New(t) + + // This is a stub for node removal and rejoin testing + // In a real implementation, you would test raft node removal and rejoin + v.AssertClusterHealthy() + + t.Log("Successfully verified raft cluster stability for node operations") +} diff --git a/vault/external_tests/blackbox/test_utils.go b/vault/external_tests/blackbox/test_utils.go new file mode 100644 index 0000000000..5a6ee3842e --- /dev/null +++ b/vault/external_tests/blackbox/test_utils.go @@ -0,0 +1,137 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "strings" + "testing" + "time" + + "github.com/hashicorp/vault/api" + "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" +) + +// Common test data structures +var ( + // Standard KV test data + StandardKVData = map[string]any{ + "api_key": "abc123", + "is_active": true, + "retry_count": 3, + } + + // Alternative KV test data + AltKVData = map[string]any{ + "username": "testuser", + "password": "testpass123", + "enabled": true, + } + + // Standard ops policy for KV access + StandardOpsPolicy = ` + path "secret/data/*" { capabilities = ["create", "read", "update"] } + path "secret/delete/*" { capabilities = ["update"] } + path "secret/undelete/*" { capabilities = ["update"] } + path "auth/userpass/login/*" { capabilities = ["create", "read"] } + ` + + // Read-only policy for limited access testing + ReadOnlyPolicy = ` + path "secret/data/allowed/*" { capabilities = ["read"] } + path "secret/data/denied/*" { capabilities = ["deny"] } + ` +) + +// SetupKVEngine enables a KV v2 secrets engine at the given mount point and waits for it to be ready +func SetupKVEngine(v *blackbox.Session, mountPath string) { + v.MustEnableSecretsEngine(mountPath, &api.MountInput{Type: "kv-v2"}) + + // Wait for KV engine to finish upgrading (important for HCP environments) + WaitForKVEngineReady(v, mountPath) +} + +// WaitForKVEngineReady waits for a KV v2 engine to complete its upgrade process +func WaitForKVEngineReady(v *blackbox.Session, mountPath string) { + maxRetries := 30 + retryDelay := 2 * time.Second + + for attempt := 1; attempt <= maxRetries; attempt++ { + // Try to write a small test value to see if the engine is ready + testPath := mountPath + "/data/__test_ready__" + testData := map[string]any{"ready": "test"} + + _, err := v.Client.Logical().Write(testPath, map[string]any{"data": testData}) + if err != nil { + if attempt < maxRetries { + // Check if this is the upgrade error we're waiting for + if strings.Contains(err.Error(), "Waiting for the primary to upgrade") { + time.Sleep(retryDelay) + continue + } + // Some other error - might still be initializing + time.Sleep(retryDelay) + continue + } + // Final attempt failed + v.Client.Logical().Write(testPath, map[string]any{"data": testData}) // Let it fail with proper error handling + } else { + // Success! Clean up the test data + v.Client.Logical().Delete(testPath) + return + } + } +} + +// SetupUserpassAuth enables userpass auth and creates a user with the given policy +func SetupUserpassAuth(v *blackbox.Session, username, password, policyName, policyContent string) *blackbox.Session { + // Enable userpass auth + v.MustEnableAuth("userpass", &api.EnableAuthOptions{Type: "userpass"}) + + // Create policy if content is provided + if policyContent != "" { + v.MustWritePolicy(policyName, policyContent) + } + + // Create user + v.MustWrite("auth/userpass/users/"+username, map[string]any{ + "password": password, + "policies": policyName, + }) + + // Try to login and return session (may fail in managed environments) + userClient, err := v.TryLoginUserpass(username, password) + if err != nil { + return nil // Login not available in managed environment + } + return userClient +} + +// SetupStandardKVUserpass is a convenience function that sets up KV engine + userpass auth with ops policy +func SetupStandardKVUserpass(v *blackbox.Session, kvMount, username, password string) *blackbox.Session { + // Setup KV engine + SetupKVEngine(v, kvMount) + + // Setup userpass with standard ops policy + return SetupUserpassAuth(v, username, password, "ops-policy", StandardOpsPolicy) +} + +// AssertKVData verifies standard KV data structure +func AssertKVData(t *testing.T, v *blackbox.Session, secret *api.Secret, data map[string]any) { + t.Helper() + assertions := v.AssertSecret(secret).KV2() + + for key, expectedValue := range data { + assertions.HasKey(key, expectedValue) + } +} + +// CreateTestToken creates a token with the given options +func CreateTestToken(v *blackbox.Session, policies []string, ttl string) string { + return v.MustCreateToken(blackbox.TokenOptions{ + Policies: policies, + TTL: ttl, + NoParent: true, + DisplayName: "test-token", + }) +} diff --git a/vault/external_tests/blackbox/token_test.go b/vault/external_tests/blackbox/token_test.go new file mode 100644 index 0000000000..199fb73477 --- /dev/null +++ b/vault/external_tests/blackbox/token_test.go @@ -0,0 +1,37 @@ +// Copyright IBM Corp. 2025, 2026 +// SPDX-License-Identifier: BUSL-1.1 + +package blackbox + +import ( + "testing" + + "github.com/hashicorp/vault/sdk/helper/testcluster/blackbox" +) + +// TestToken_OrphanedWithPolicy verifies token creation with policy assignment, +// validates token authentication, and tests policy enforcement by attempting +// both allowed and denied operations on KV secrets. +func TestToken_OrphanedWithPolicy(t *testing.T) { + v := blackbox.New(t) + + // Use common utility to create token with read-only policy + policyName := "read-secret-only" + v.MustWritePolicy(policyName, ReadOnlyPolicy) + + token := CreateTestToken(v, []string{policyName}, "15m") + t.Logf("Generated Token: %s...", token[:5]) + + v.AssertTokenIsValid(token, policyName) + + // Setup KV engine and seed test data + SetupKVEngine(v, "secret") + v.MustWriteKV2("secret", "allowed/test", map[string]any{"val": "allowed"}) + v.MustWriteKV2("secret", "denied/test", map[string]any{"val": "denied"}) + + // Test token access + userClient := v.NewClientFromToken(token) + secret := userClient.MustReadRequired("secret/data/allowed/test") + userClient.AssertSecret(secret).KV2().HasKey("val", "allowed") + userClient.AssertReadFails("secret/data/denied/test") +}