diff --git a/api/sys_seal.go b/api/sys_seal.go index 62002496c3..d5548aef77 100644 --- a/api/sys_seal.go +++ b/api/sys_seal.go @@ -96,24 +96,25 @@ func sealStatusRequestWithContext(ctx context.Context, c *Sys, r *Request) (*Sea } type SealStatusResponse struct { - Type string `json:"type"` - Initialized bool `json:"initialized"` - Sealed bool `json:"sealed"` - T int `json:"t"` - N int `json:"n"` - Progress int `json:"progress"` - Nonce string `json:"nonce"` - Version string `json:"version"` - BuildDate string `json:"build_date"` - Migration bool `json:"migration"` - ClusterName string `json:"cluster_name,omitempty"` - ClusterID string `json:"cluster_id,omitempty"` - RecoverySeal bool `json:"recovery_seal"` - RecoverySealType string `json:"recovery_seal_type,omitempty"` - StorageType string `json:"storage_type,omitempty"` - HCPLinkStatus string `json:"hcp_link_status,omitempty"` - HCPLinkResourceID string `json:"hcp_link_resource_ID,omitempty"` - Warnings []string `json:"warnings,omitempty"` + Type string `json:"type"` + Initialized bool `json:"initialized"` + Sealed bool `json:"sealed"` + T int `json:"t"` + N int `json:"n"` + Progress int `json:"progress"` + Nonce string `json:"nonce"` + Version string `json:"version"` + BuildDate string `json:"build_date"` + Migration bool `json:"migration"` + ClusterName string `json:"cluster_name,omitempty"` + ClusterID string `json:"cluster_id,omitempty"` + RecoverySeal bool `json:"recovery_seal"` + RecoverySealType string `json:"recovery_seal_type,omitempty"` + StorageType string `json:"storage_type,omitempty"` + HCPLinkStatus string `json:"hcp_link_status,omitempty"` + HCPLinkResourceID string `json:"hcp_link_resource_ID,omitempty"` + RemovedFromCluster *bool `json:"removed_from_cluster,omitempty"` + Warnings []string `json:"warnings,omitempty"` } type UnsealOpts struct { diff --git a/changelog/28938.txt b/changelog/28938.txt new file mode 100644 index 0000000000..2fe42304f0 --- /dev/null +++ b/changelog/28938.txt @@ -0,0 +1,3 @@ +```release-note:improvement +core: Add `removed_from_cluster` field to sys/seal-status and vault status output to indicate whether the node has been removed from the HA cluster. +``` diff --git a/command/command_test.go b/command/command_test.go index def68c4fbc..ed3d31545e 100644 --- a/command/command_test.go +++ b/command/command_test.go @@ -22,6 +22,8 @@ import ( "github.com/hashicorp/vault/builtin/logical/ssh" "github.com/hashicorp/vault/builtin/logical/transit" "github.com/hashicorp/vault/helper/builtinplugins" + "github.com/hashicorp/vault/helper/testhelpers" + "github.com/hashicorp/vault/helper/testhelpers/teststorage" vaulthttp "github.com/hashicorp/vault/http" "github.com/hashicorp/vault/sdk/logical" "github.com/hashicorp/vault/sdk/physical/inmem" @@ -161,6 +163,23 @@ func testVaultServerUnsealWithKVVersionWithSeal(tb testing.TB, kvVersion string, }) } +func testVaultRaftCluster(tb testing.TB) *vault.TestCluster { + conf := &vault.CoreConfig{ + CredentialBackends: defaultVaultCredentialBackends, + AuditBackends: defaultVaultAuditBackends, + LogicalBackends: defaultVaultLogicalBackends, + BuiltinRegistry: builtinplugins.Registry, + } + opts := &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + NumCores: 3, + } + teststorage.RaftBackendSetup(conf, opts) + cluster := vault.NewTestCluster(tb, conf, opts) + testhelpers.WaitForActiveNodeAndStandbys(tb, cluster) + return cluster +} + // testVaultServerUnseal creates a test vault cluster and returns a configured // API client, list of unseal keys (as strings), and a closer function // configured with the given plugin directory. diff --git a/command/format.go b/command/format.go index 548a9a089c..b83fad6366 100644 --- a/command/format.go +++ b/command/format.go @@ -357,6 +357,10 @@ func (t TableFormatter) OutputSealStatusStruct(ui cli.Ui, secret *api.Secret, da out = append(out, fmt.Sprintf("Cluster ID | %s", status.ClusterID)) } + if status.RemovedFromCluster != nil { + out = append(out, fmt.Sprintf("Removed From Cluster | %t", *status.RemovedFromCluster)) + } + // Output if HCP link is configured if status.HCPLinkStatus != "" { out = append(out, fmt.Sprintf("HCP Link Status | %s", status.HCPLinkStatus)) diff --git a/command/status_test.go b/command/status_test.go index 47a2803d66..1eff8ff8b8 100644 --- a/command/status_test.go +++ b/command/status_test.go @@ -4,10 +4,15 @@ package command import ( + "errors" + "fmt" "strings" "testing" + "time" "github.com/hashicorp/cli" + "github.com/hashicorp/vault/helper/testhelpers" + "github.com/stretchr/testify/require" ) func testStatusCommand(tb testing.TB) (*cli.MockUi, *StatusCommand) { @@ -21,6 +26,41 @@ func testStatusCommand(tb testing.TB) (*cli.MockUi, *StatusCommand) { } } +// TestStatusCommand_RaftCluster creates a raft cluster and verifies that a +// follower has "Removed From Cluster" returned as false in the status command. +// The test then removes that follower, and checks that "Removed From Cluster" +// is now true +func TestStatusCommand_RaftCluster(t *testing.T) { + t.Parallel() + cluster := testVaultRaftCluster(t) + defer cluster.Cleanup() + + toRemove := cluster.Cores[1] + expectRemovedFromCluster := func(expectCode int, removed bool) { + ui, cmd := testStatusCommand(t) + cmd.client = toRemove.Client + code := cmd.Run(nil) + require.Equal(t, expectCode, code) + combined := ui.OutputWriter.String() + ui.ErrorWriter.String() + require.Regexp(t, fmt.Sprintf(".*Removed From Cluster\\s+%t.*", removed), combined) + } + + expectRemovedFromCluster(0, false) + + _, err := cluster.Cores[0].Client.Logical().Write("sys/storage/raft/remove-peer", + map[string]interface{}{ + "server_id": toRemove.NodeID, + }) + require.NoError(t, err) + testhelpers.RetryUntil(t, 10*time.Second, func() error { + if !toRemove.Sealed() { + return errors.New("core not sealed") + } + return nil + }) + expectRemovedFromCluster(2, true) +} + func TestStatusCommand_Run(t *testing.T) { t.Parallel() diff --git a/vault/logical_system.go b/vault/logical_system.go index 4b3b18b2ec..3e328fd1dc 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -5503,24 +5503,25 @@ func (b *SystemBackend) pathInternalOpenAPI(ctx context.Context, req *logical.Re } type SealStatusResponse struct { - Type string `json:"type"` - Initialized bool `json:"initialized"` - Sealed bool `json:"sealed"` - T int `json:"t"` - N int `json:"n"` - Progress int `json:"progress"` - Nonce string `json:"nonce"` - Version string `json:"version"` - BuildDate string `json:"build_date"` - Migration bool `json:"migration"` - ClusterName string `json:"cluster_name,omitempty"` - ClusterID string `json:"cluster_id,omitempty"` - RecoverySeal bool `json:"recovery_seal"` - StorageType string `json:"storage_type,omitempty"` - HCPLinkStatus string `json:"hcp_link_status,omitempty"` - HCPLinkResourceID string `json:"hcp_link_resource_ID,omitempty"` - Warnings []string `json:"warnings,omitempty"` - RecoverySealType string `json:"recovery_seal_type,omitempty"` + Type string `json:"type"` + Initialized bool `json:"initialized"` + Sealed bool `json:"sealed"` + T int `json:"t"` + N int `json:"n"` + Progress int `json:"progress"` + Nonce string `json:"nonce"` + Version string `json:"version"` + BuildDate string `json:"build_date"` + Migration bool `json:"migration"` + ClusterName string `json:"cluster_name,omitempty"` + ClusterID string `json:"cluster_id,omitempty"` + RecoverySeal bool `json:"recovery_seal"` + StorageType string `json:"storage_type,omitempty"` + HCPLinkStatus string `json:"hcp_link_status,omitempty"` + HCPLinkResourceID string `json:"hcp_link_resource_ID,omitempty"` + Warnings []string `json:"warnings,omitempty"` + RecoverySealType string `json:"recovery_seal_type,omitempty"` + RemovedFromCluster *bool `json:"removed_from_cluster,omitempty"` } type SealBackendStatus struct { @@ -5557,16 +5558,22 @@ func (core *Core) GetSealStatus(ctx context.Context, lock bool) (*SealStatusResp hcpLinkStatus, resourceIDonHCP := core.GetHCPLinkStatus() redactVersion, _, redactClusterName, _ := logical.CtxRedactionSettingsValue(ctx) + var removed *bool + isRemoved, shouldInclude := core.IsRemovedFromCluster() + if shouldInclude { + removed = &isRemoved + } if sealConfig == nil { s := &SealStatusResponse{ - Type: core.SealAccess().BarrierSealConfigType().String(), - Initialized: initialized, - Sealed: true, - RecoverySeal: core.SealAccess().RecoveryKeySupported(), - StorageType: core.StorageType(), - Version: version.GetVersion().VersionNumber(), - BuildDate: version.BuildDate, + Type: core.SealAccess().BarrierSealConfigType().String(), + Initialized: initialized, + Sealed: true, + RecoverySeal: core.SealAccess().RecoveryKeySupported(), + StorageType: core.StorageType(), + Version: version.GetVersion().VersionNumber(), + BuildDate: version.BuildDate, + RemovedFromCluster: removed, } if redactVersion { @@ -5608,21 +5615,22 @@ func (core *Core) GetSealStatus(ctx context.Context, lock bool) (*SealStatusResp progress, nonce := core.SecretProgress(lock) s := &SealStatusResponse{ - Type: sealType, - Initialized: initialized, - Sealed: sealed, - T: sealConfig.SecretThreshold, - N: sealConfig.SecretShares, - Progress: progress, - Nonce: nonce, - Version: version.GetVersion().VersionNumber(), - BuildDate: version.BuildDate, - Migration: core.IsInSealMigrationMode(lock) && !core.IsSealMigrated(lock), - ClusterName: clusterName, - ClusterID: clusterID, - RecoverySeal: core.SealAccess().RecoveryKeySupported(), - RecoverySealType: recoverySealType, - StorageType: core.StorageType(), + Type: sealType, + Initialized: initialized, + Sealed: sealed, + T: sealConfig.SecretThreshold, + N: sealConfig.SecretShares, + Progress: progress, + Nonce: nonce, + Version: version.GetVersion().VersionNumber(), + BuildDate: version.BuildDate, + Migration: core.IsInSealMigrationMode(lock) && !core.IsSealMigrated(lock), + RemovedFromCluster: removed, + ClusterName: clusterName, + ClusterID: clusterID, + RecoverySeal: core.SealAccess().RecoveryKeySupported(), + RecoverySealType: recoverySealType, + StorageType: core.StorageType(), } if resourceIDonHCP != "" { diff --git a/vault/logical_system_test.go b/vault/logical_system_test.go index 47c71c1fb5..66dfbbe34f 100644 --- a/vault/logical_system_test.go +++ b/vault/logical_system_test.go @@ -7349,3 +7349,48 @@ func TestFuzz_sanitizePath(t *testing.T) { require.True(t, valid(path, newPath), `"%s" not sanitized correctly, got "%s"`, path, newPath) } } + +// TestSealStatus_Removed checks if the seal-status endpoint returns the +// correct value for RemovedFromCluster when provided with different backends +func TestSealStatus_Removed(t *testing.T) { + removedCore, err := TestCoreWithMockRemovableNodeHABackend(t, true) + require.NoError(t, err) + notRemovedCore, err := TestCoreWithMockRemovableNodeHABackend(t, false) + require.NoError(t, err) + testCases := []struct { + name string + core *Core + wantField bool + wantTrue bool + }{ + { + name: "removed", + core: removedCore, + wantField: true, + wantTrue: true, + }, + { + name: "not removed", + core: notRemovedCore, + wantField: true, + wantTrue: false, + }, + { + name: "different backend", + core: TestCore(t), + wantField: false, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + status, err := tc.core.GetSealStatus(context.Background(), true) + require.NoError(t, err) + if tc.wantField { + require.NotNil(t, status.RemovedFromCluster) + require.Equal(t, tc.wantTrue, *status.RemovedFromCluster) + } else { + require.Nil(t, status.RemovedFromCluster) + } + }) + } +} diff --git a/website/content/api-docs/system/seal-status.mdx b/website/content/api-docs/system/seal-status.mdx index feb69e68e3..2db53955a9 100644 --- a/website/content/api-docs/system/seal-status.mdx +++ b/website/content/api-docs/system/seal-status.mdx @@ -30,18 +30,19 @@ The "t" parameter is the threshold, and "n" is the number of shares. ```json { - "type": "shamir", + "build_date": "2024-11-15T14:17:42Z", "initialized": true, - "sealed": true, - "t": 3, - "n": 5, - "progress": 2, - "nonce": "", - "version": "1.11.0", - "build_date": "2022-05-03T08:34:11Z", "migration": false, + "n": 3, + "nonce": "", + "progress": 1, "recovery_seal": false, - "storage_type": "file" + "removed_from_cluster": false, + "sealed": true, + "storage_type": "raft", + "t": 2, + "type": "shamir", + "version": "1.19.0-beta1" } ``` @@ -49,19 +50,20 @@ Sample response when Vault is unsealed. ```json { - "type": "shamir", + "build_date": "2024-11-14T18:11:15Z", + "cluster_id": "ebdd80fb-0c7f-bce9-f9b9-a0fa86aa3249", + "cluster_name": "vault-cluster-f090409a", "initialized": true, - "sealed": false, - "t": 3, - "n": 5, - "progress": 0, - "nonce": "", - "version": "1.11.0", - "build_date": "2022-05-03T08:34:11Z", "migration": false, - "cluster_name": "vault-cluster-336172e1", - "cluster_id": "f94053ad-d80e-4270-2006-2efd67d0910a", + "n": 3, + "nonce": "", + "progress": 0, "recovery_seal": false, - "storage_type": "file" + "removed_from_cluster": false, + "sealed": false, + "storage_type": "raft", + "t": 2, + "type": "shamir", + "version": "1.19.0-beta1" } ``` diff --git a/website/content/docs/commands/status.mdx b/website/content/docs/commands/status.mdx index ab7d415301..4034def61a 100644 --- a/website/content/docs/commands/status.mdx +++ b/website/content/docs/commands/status.mdx @@ -54,7 +54,7 @@ By default, the output is displayed in "table" format. #### Output fields -1. The field for total shares is displayed as `"n"` instead of `n` in yaml outputs. +1. The field for total shares is displayed as `"n"` instead of `n` in yaml outputs. 2. The following fields in "table" format are displayed only when relevant: - "Unseal Progress" and "Unseal Nonce" are displayed when vault is sealed. - "HCP Link Status" and "HCP Link Resource ID" are displayed when HCP link is configured. @@ -67,3 +67,4 @@ By default, the output is displayed in "table" format. - "HA Mode". - "Active Since" is displayed if the node is active and has a valid active time. - "Performance Standby" Node and "Performance Standby Last Remote WAL" are displayed for performance standby nodes. +- The "Removed From Cluster" field is only displayed when the storage or HA backend is raft.