From f3f30022d046a458c75fb78c7906bdbda3142fc2 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Wed, 2 Mar 2016 20:26:38 -0500 Subject: [PATCH] Add forced revocation. In some situations, it can be impossible to revoke leases (for instance, if someone has gone and manually removed users created by Vault). This can not only cause Vault to cycle trying to revoke them, but it also prevents mounts from being unmounted, leaving them in a tainted state where the only operations allowed are to revoke (or rollback), which will never successfully complete. This adds a new endpoint that works similarly to `revoke-prefix` but ignores errors coming from a backend upon revocation (it does not ignore errors coming from within the expiration manager, such as errors accessing the data store). This can be used to force Vault to abandon leases. Like `revoke-prefix`, this is a very sensitive operation and requires `sudo`. It is implemented as a separate endpoint, rather than an argument to `revoke-prefix`, to ensure that control can be delegated appropriately, as even most administrators should not normally have this privilege. Fixes #1135 --- api/sys_lease.go | 9 ++++++ command/revoke.go | 30 ++++++++++++++------ vault/expiration.go | 63 +++++++++++++++++++++++++++-------------- vault/logical_system.go | 55 ++++++++++++++++++++++++++++++++++- 4 files changed, 127 insertions(+), 30 deletions(-) diff --git a/api/sys_lease.go b/api/sys_lease.go index 4c29fdcaf6..e103990d4f 100644 --- a/api/sys_lease.go +++ b/api/sys_lease.go @@ -34,3 +34,12 @@ func (c *Sys) RevokePrefix(id string) error { } return err } + +func (c *Sys) RevokeForce(id string) error { + r := c.c.NewRequest("PUT", "/v1/sys/revoke-force/"+id) + resp, err := c.c.RawRequest(r) + if err == nil { + defer resp.Body.Close() + } + return err +} diff --git a/command/revoke.go b/command/revoke.go index 149cbad84e..6cd7296797 100644 --- a/command/revoke.go +++ b/command/revoke.go @@ -11,9 +11,10 @@ type RevokeCommand struct { } func (c *RevokeCommand) Run(args []string) int { - var prefix bool + var prefix, force bool flags := c.Meta.FlagSet("revoke", FlagSetDefault) flags.BoolVar(&prefix, "prefix", false, "") + flags.BoolVar(&force, "force", false, "") flags.Usage = func() { c.Ui.Error(c.Help()) } if err := flags.Parse(args); err != nil { return 1 @@ -35,9 +36,16 @@ func (c *RevokeCommand) Run(args []string) int { return 2 } - if prefix { + switch { + case force && !prefix: + c.Ui.Error(fmt.Sprintf( + "-force requires -prefix")) + return 1 + case force && prefix: + err = client.Sys().RevokeForce(leaseId) + case prefix: err = client.Sys().RevokePrefix(leaseId) - } else { + default: err = client.Sys().Revoke(leaseId) } if err != nil { @@ -60,12 +68,16 @@ Usage: vault revoke [options] id Revoke a secret by its lease ID. - This command revokes a secret by its lease ID that was returned - with it. Once the key is revoked, it is no longer valid. + This command revokes a secret by its lease ID that was returned with it. Once + the key is revoked, it is no longer valid. - With the -prefix flag, the revoke is done by prefix: any secret prefixed - with the given partial ID is revoked. Lease IDs are structured in such - a way to make revocation of prefixes useful. + With the -prefix flag, the revoke is done by prefix: any secret prefixed with + the given partial ID is revoked. Lease IDs are structured in such a way to + make revocation of prefixes useful. + + With the -force flag, the lease is removed from Vault even if the revocation + fails. This is meant for certain recovery scenarios and should not be used + lightly. This option requires -prefix. General Options: @@ -76,6 +88,8 @@ Revoke Options: -prefix=true Revoke all secrets with the matching prefix. This defaults to false: an exact revocation. + -force=true Delete the lease even if the actual revocation + operation fails. ` return strings.TrimSpace(helpText) } diff --git a/vault/expiration.go b/vault/expiration.go index 3e06d2591f..8863810142 100644 --- a/vault/expiration.go +++ b/vault/expiration.go @@ -173,6 +173,14 @@ func (m *ExpirationManager) Stop() error { // Revoke is used to revoke a secret named by the given LeaseID func (m *ExpirationManager) Revoke(leaseID string) error { defer metrics.MeasureSince([]string{"expire", "revoke"}, time.Now()) + + return m.revokeCommon(leaseID, false) +} + +// revokeCommon does the heavy lifting. If force is true, we ignore a problem +// during revocation and still remove entries/index/lease timers +func (m *ExpirationManager) revokeCommon(leaseID string, force bool) error { + defer metrics.MeasureSince([]string{"expire", "revoke-common"}, time.Now()) // Load the entry le, err := m.loadEntry(leaseID) if err != nil { @@ -185,7 +193,7 @@ func (m *ExpirationManager) Revoke(leaseID string) error { } // Revoke the entry - if err := m.revokeEntry(le); err != nil { + if err := m.revokeEntry(le); err != nil && !force { return err } @@ -209,32 +217,21 @@ func (m *ExpirationManager) Revoke(leaseID string) error { return nil } +// RevokeForce works similarly to RevokePrefix but continues in the case of a +// revocation error; this is mostly meant for recovery operations +func (m *ExpirationManager) RevokeForce(prefix string) error { + defer metrics.MeasureSince([]string{"expire", "revoke-force"}, time.Now()) + + return m.revokePrefixCommon(prefix, true) +} + // RevokePrefix is used to revoke all secrets with a given prefix. // The prefix maps to that of the mount table to make this simpler // to reason about. func (m *ExpirationManager) RevokePrefix(prefix string) error { defer metrics.MeasureSince([]string{"expire", "revoke-prefix"}, time.Now()) - // Ensure there is a trailing slash - if !strings.HasSuffix(prefix, "/") { - prefix = prefix + "/" - } - // Accumulate existing leases - sub := m.idView.SubView(prefix) - existing, err := CollectKeys(sub) - if err != nil { - return fmt.Errorf("failed to scan for leases: %v", err) - } - - // Revoke all the keys - for idx, suffix := range existing { - leaseID := prefix + suffix - if err := m.Revoke(leaseID); err != nil { - return fmt.Errorf("failed to revoke '%s' (%d / %d): %v", - leaseID, idx+1, len(existing), err) - } - } - return nil + return m.revokePrefixCommon(prefix, false) } // RevokeByToken is used to revoke all the secrets issued with @@ -257,6 +254,30 @@ func (m *ExpirationManager) RevokeByToken(token string) error { return nil } +func (m *ExpirationManager) revokePrefixCommon(prefix string, force bool) error { + // Ensure there is a trailing slash + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + + // Accumulate existing leases + sub := m.idView.SubView(prefix) + existing, err := CollectKeys(sub) + if err != nil { + return fmt.Errorf("failed to scan for leases: %v", err) + } + + // Revoke all the keys + for idx, suffix := range existing { + leaseID := prefix + suffix + if err := m.revokeCommon(leaseID, force); err != nil { + return fmt.Errorf("failed to revoke '%s' (%d / %d): %v", + leaseID, idx+1, len(existing), err) + } + } + return nil +} + // Renew is used to renew a secret using the given leaseID // and a renew interval. The increment may be ignored. func (m *ExpirationManager) Renew(leaseID string, increment time.Duration) (*logical.Response, error) { diff --git a/vault/logical_system.go b/vault/logical_system.go index e2eac88e39..293d69c0de 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -185,6 +185,24 @@ func NewSystemBackend(core *Core, config *logical.BackendConfig) logical.Backend HelpDescription: strings.TrimSpace(sysHelp["revoke"][1]), }, + &framework.Path{ + Pattern: "revoke-force/(?P.+)", + + Fields: map[string]*framework.FieldSchema{ + "prefix": &framework.FieldSchema{ + Type: framework.TypeString, + Description: strings.TrimSpace(sysHelp["revoke-force-path"][0]), + }, + }, + + Callbacks: map[logical.Operation]framework.OperationFunc{ + logical.UpdateOperation: b.handleRevokeForce, + }, + + HelpSynopsis: strings.TrimSpace(sysHelp["revoke-force"][0]), + HelpDescription: strings.TrimSpace(sysHelp["revoke-force"][1]), + }, + &framework.Path{ Pattern: "revoke-prefix/(?P.+)", @@ -733,11 +751,29 @@ func (b *SystemBackend) handleRevoke( // handleRevokePrefix is used to revoke a prefix with many LeaseIDs func (b *SystemBackend) handleRevokePrefix( req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + return b.handleRevokePrefixCommon(req, data, false) +} + +// handleRevokeForce is used to revoke a prefix with many LeaseIDs, ignoring errors +func (b *SystemBackend) handleRevokeForce( + req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + return b.handleRevokePrefixCommon(req, data, true) +} + +// handleRevokePrefixCommon is used to revoke a prefix with many LeaseIDs +func (b *SystemBackend) handleRevokePrefixCommon( + req *logical.Request, data *framework.FieldData, force bool) (*logical.Response, error) { // Get all the options prefix := data.Get("prefix").(string) // Invoke the expiration manager directly - if err := b.Core.expiration.RevokePrefix(prefix); err != nil { + var err error + if force { + err = b.Core.expiration.RevokeForce(prefix) + } else { + err = b.Core.expiration.RevokePrefix(prefix) + } + if err != nil { b.Backend.Logger().Printf("[ERR] sys: revoke prefix '%s' failed: %v", prefix, err) return handleError(err) } @@ -1211,6 +1247,23 @@ all matching leases. "", }, + "revoke-force": { + "Revoke all secrets generated in a given prefix, ignoring errors.", + ` +See the path help for 'revoke-prefix'; this behaves the same, except that it +ignores errors encountered during revocation. This can be used in certain +recovery situations; for instance, when you want to unmount a backend, but it +is impossible to fix revocation errors and these errors prevent the unmount +from proceeding. This is a DANGEROUS operation as it removes Vault's oversight +of external secrets. Access to this prefix should be tightly controlled. + `, + }, + + "revoke-force-path": { + `The path to revoke keys under. Example: "prod/aws/ops"`, + "", + }, + "auth-table": { "List the currently enabled credential backends.", `