Add forced revocation.

In some situations, it can be impossible to revoke leases (for instance,
if someone has gone and manually removed users created by Vault). This
can not only cause Vault to cycle trying to revoke them, but it also
prevents mounts from being unmounted, leaving them in a tainted state
where the only operations allowed are to revoke (or rollback), which
will never successfully complete.

This adds a new endpoint that works similarly to `revoke-prefix` but
ignores errors coming from a backend upon revocation (it does not ignore
errors coming from within the expiration manager, such as errors
accessing the data store). This can be used to force Vault to abandon
leases.

Like `revoke-prefix`, this is a very sensitive operation and requires
`sudo`. It is implemented as a separate endpoint, rather than an
argument to `revoke-prefix`, to ensure that control can be delegated
appropriately, as even most administrators should not normally have
this privilege.

Fixes #1135

api/sys_lease.go

@@ -34,3 +34,12 @@ func (c *Sys) RevokePrefix(id string) error {
 	}
 	return err
 }
+
+func (c *Sys) RevokeForce(id string) error {
+	r := c.c.NewRequest("PUT", "/v1/sys/revoke-force/"+id)
+	resp, err := c.c.RawRequest(r)
+	if err == nil {
+		defer resp.Body.Close()
+	}
+	return err
+}

command/revoke.go

@@ -11,9 +11,10 @@ type RevokeCommand struct {
 }
 
 func (c *RevokeCommand) Run(args []string) int {
-	var prefix bool
+	var prefix, force bool
 	flags := c.Meta.FlagSet("revoke", FlagSetDefault)
 	flags.BoolVar(&prefix, "prefix", false, "")
+	flags.BoolVar(&force, "force", false, "")
 	flags.Usage = func() { c.Ui.Error(c.Help()) }
 	if err := flags.Parse(args); err != nil {
 		return 1
@@ -35,9 +36,16 @@ func (c *RevokeCommand) Run(args []string) int {
 		return 2
 	}
 
-	if prefix {
+	switch {
+	case force && !prefix:
+		c.Ui.Error(fmt.Sprintf(
+			"-force requires -prefix"))
+		return 1
+	case force && prefix:
+		err = client.Sys().RevokeForce(leaseId)
+	case prefix:
 		err = client.Sys().RevokePrefix(leaseId)
-	} else {
+	default:
 		err = client.Sys().Revoke(leaseId)
 	}
 	if err != nil {
@@ -60,12 +68,16 @@ Usage: vault revoke [options] id
 
   Revoke a secret by its lease ID.
 
-  This command revokes a secret by its lease ID that was returned
-  with it. Once the key is revoked, it is no longer valid.
+  This command revokes a secret by its lease ID that was returned with it. Once
+  the key is revoked, it is no longer valid.
 
-  With the -prefix flag, the revoke is done by prefix: any secret prefixed
-  with the given partial ID is revoked. Lease IDs are structured in such
-  a way to make revocation of prefixes useful.
+  With the -prefix flag, the revoke is done by prefix: any secret prefixed with
+  the given partial ID is revoked. Lease IDs are structured in such a way to
+  make revocation of prefixes useful.
+
+  With the -force flag, the lease is removed from Vault even if the revocation
+  fails. This is meant for certain recovery scenarios and should not be used
+  lightly. This option requires -prefix.
 
 General Options:
 
@@ -76,6 +88,8 @@ Revoke Options:
   -prefix=true            Revoke all secrets with the matching prefix. This
                           defaults to false: an exact revocation.
 
+  -force=true             Delete the lease even if the actual revocation
+                          operation fails.
 `
 	return strings.TrimSpace(helpText)
 }

vault/expiration.go

@@ -173,6 +173,14 @@ func (m *ExpirationManager) Stop() error {
 // Revoke is used to revoke a secret named by the given LeaseID
 func (m *ExpirationManager) Revoke(leaseID string) error {
 	defer metrics.MeasureSince([]string{"expire", "revoke"}, time.Now())
+
+	return m.revokeCommon(leaseID, false)
+}
+
+// revokeCommon does the heavy lifting. If force is true, we ignore a problem
+// during revocation and still remove entries/index/lease timers
+func (m *ExpirationManager) revokeCommon(leaseID string, force bool) error {
+	defer metrics.MeasureSince([]string{"expire", "revoke-common"}, time.Now())
 	// Load the entry
 	le, err := m.loadEntry(leaseID)
 	if err != nil {
@@ -185,7 +193,7 @@ func (m *ExpirationManager) Revoke(leaseID string) error {
 	}
 
 	// Revoke the entry
-	if err := m.revokeEntry(le); err != nil {
+	if err := m.revokeEntry(le); err != nil && !force {
 		return err
 	}
 
@@ -209,32 +217,21 @@ func (m *ExpirationManager) Revoke(leaseID string) error {
 	return nil
 }
 
+// RevokeForce works similarly to RevokePrefix but continues in the case of a
+// revocation error; this is mostly meant for recovery operations
+func (m *ExpirationManager) RevokeForce(prefix string) error {
+	defer metrics.MeasureSince([]string{"expire", "revoke-force"}, time.Now())
+
+	return m.revokePrefixCommon(prefix, true)
+}
+
 // RevokePrefix is used to revoke all secrets with a given prefix.
 // The prefix maps to that of the mount table to make this simpler
 // to reason about.
 func (m *ExpirationManager) RevokePrefix(prefix string) error {
 	defer metrics.MeasureSince([]string{"expire", "revoke-prefix"}, time.Now())
-	// Ensure there is a trailing slash
-	if !strings.HasSuffix(prefix, "/") {
-		prefix = prefix + "/"
-	}
 
-	// Accumulate existing leases
-	sub := m.idView.SubView(prefix)
-	existing, err := CollectKeys(sub)
-	if err != nil {
-		return fmt.Errorf("failed to scan for leases: %v", err)
-	}
-
-	// Revoke all the keys
-	for idx, suffix := range existing {
-		leaseID := prefix + suffix
-		if err := m.Revoke(leaseID); err != nil {
-			return fmt.Errorf("failed to revoke '%s' (%d / %d): %v",
-				leaseID, idx+1, len(existing), err)
-		}
-	}
-	return nil
+	return m.revokePrefixCommon(prefix, false)
 }
 
 // RevokeByToken is used to revoke all the secrets issued with
@@ -257,6 +254,30 @@ func (m *ExpirationManager) RevokeByToken(token string) error {
 	return nil
 }
 
+func (m *ExpirationManager) revokePrefixCommon(prefix string, force bool) error {
+	// Ensure there is a trailing slash
+	if !strings.HasSuffix(prefix, "/") {
+		prefix = prefix + "/"
+	}
+
+	// Accumulate existing leases
+	sub := m.idView.SubView(prefix)
+	existing, err := CollectKeys(sub)
+	if err != nil {
+		return fmt.Errorf("failed to scan for leases: %v", err)
+	}
+
+	// Revoke all the keys
+	for idx, suffix := range existing {
+		leaseID := prefix + suffix
+		if err := m.revokeCommon(leaseID, force); err != nil {
+			return fmt.Errorf("failed to revoke '%s' (%d / %d): %v",
+				leaseID, idx+1, len(existing), err)
+		}
+	}
+	return nil
+}
+
 // Renew is used to renew a secret using the given leaseID
 // and a renew interval. The increment may be ignored.
 func (m *ExpirationManager) Renew(leaseID string, increment time.Duration) (*logical.Response, error) {

vault/logical_system.go

@@ -185,6 +185,24 @@ func NewSystemBackend(core *Core, config *logical.BackendConfig) logical.Backend
 				HelpDescription: strings.TrimSpace(sysHelp["revoke"][1]),
 			},
 
+			&framework.Path{
+				Pattern: "revoke-force/(?P<prefix>.+)",
+
+				Fields: map[string]*framework.FieldSchema{
+					"prefix": &framework.FieldSchema{
+						Type:        framework.TypeString,
+						Description: strings.TrimSpace(sysHelp["revoke-force-path"][0]),
+					},
+				},
+
+				Callbacks: map[logical.Operation]framework.OperationFunc{
+					logical.UpdateOperation: b.handleRevokeForce,
+				},
+
+				HelpSynopsis:    strings.TrimSpace(sysHelp["revoke-force"][0]),
+				HelpDescription: strings.TrimSpace(sysHelp["revoke-force"][1]),
+			},
+
 			&framework.Path{
 				Pattern: "revoke-prefix/(?P<prefix>.+)",
 
@@ -733,11 +751,29 @@ func (b *SystemBackend) handleRevoke(
 // handleRevokePrefix is used to revoke a prefix with many LeaseIDs
 func (b *SystemBackend) handleRevokePrefix(
 	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+	return b.handleRevokePrefixCommon(req, data, false)
+}
+
+// handleRevokeForce is used to revoke a prefix with many LeaseIDs, ignoring errors
+func (b *SystemBackend) handleRevokeForce(
+	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+	return b.handleRevokePrefixCommon(req, data, true)
+}
+
+// handleRevokePrefixCommon is used to revoke a prefix with many LeaseIDs
+func (b *SystemBackend) handleRevokePrefixCommon(
+	req *logical.Request, data *framework.FieldData, force bool) (*logical.Response, error) {
 	// Get all the options
 	prefix := data.Get("prefix").(string)
 
 	// Invoke the expiration manager directly
-	if err := b.Core.expiration.RevokePrefix(prefix); err != nil {
+	var err error
+	if force {
+		err = b.Core.expiration.RevokeForce(prefix)
+	} else {
+		err = b.Core.expiration.RevokePrefix(prefix)
+	}
+	if err != nil {
 		b.Backend.Logger().Printf("[ERR] sys: revoke prefix '%s' failed: %v", prefix, err)
 		return handleError(err)
 	}
@@ -1211,6 +1247,23 @@ all matching leases.
 		"",
 	},
 
+	"revoke-force": {
+		"Revoke all secrets generated in a given prefix, ignoring errors.",
+		`
+See the path help for 'revoke-prefix'; this behaves the same, except that it
+ignores errors encountered during revocation. This can be used in certain
+recovery situations; for instance, when you want to unmount a backend, but it
+is impossible to fix revocation errors and these errors prevent the unmount
+from proceeding. This is a DANGEROUS operation as it removes Vault's oversight
+of external secrets. Access to this prefix should be tightly controlled.
+		`,
+	},
+
+	"revoke-force-path": {
+		`The path to revoke keys under. Example: "prod/aws/ops"`,
+		"",
+	},
+
 	"auth-table": {
 		"List the currently enabled credential backends.",
 		`