Merge branch 'master' of github.com:hashicorp/vault

api/secret.go

@@ -7,11 +7,10 @@ import (
 
 // Secret is the structure returned for every secret within Vault.
 type Secret struct {
-	VaultId          string `json:"vault_id"`
-	Renewable        bool
-	LeaseDuration    int                    `json:"lease_duration"`
-	LeaseDurationMax int                    `json:"lease_duration_max"`
-	Data             map[string]interface{} `json:"data"`
+	VaultId       string `json:"vault_id"`
+	Renewable     bool
+	LeaseDuration int                    `json:"lease_duration"`
+	Data          map[string]interface{} `json:"data"`
 }
 
 // ParseSecret is used to parse a secret value from JSON from an io.Reader.

api/secret_test.go

@@ -12,8 +12,9 @@ func TestParseSecret(t *testing.T) {
 	"vault_id": "foo",
 	"renewable": true,
 	"lease_duration": 10,
-	"lease_duration_max": 100,
-	"key": "value"
+	"data": {
+		"key": "value"
+	}
 }`)
 
 	secret, err := ParseSecret(strings.NewReader(raw))
@@ -22,15 +23,14 @@ func TestParseSecret(t *testing.T) {
 	}
 
 	expected := &Secret{
-		VaultId:          "foo",
-		Renewable:        true,
-		LeaseDuration:    10,
-		LeaseDurationMax: 100,
+		VaultId:       "foo",
+		Renewable:     true,
+		LeaseDuration: 10,
 		Data: map[string]interface{}{
 			"key": "value",
 		},
 	}
 	if !reflect.DeepEqual(secret, expected) {
-		t.Fatalf("bad: %#v", secret)
+		t.Fatalf("bad: %#v %#v", secret, expected)
 	}
 }

http/logical.go

@@ -64,7 +64,6 @@ func handleLogical(core *vault.Core) http.Handler {
 				logicalResp.VaultId = resp.Lease.VaultID
 				logicalResp.Renewable = resp.Lease.Renewable
 				logicalResp.LeaseDuration = int(resp.Lease.Duration.Seconds())
-				logicalResp.LeaseDurationMax = int(resp.Lease.MaxDuration.Seconds())
 			}
 
 			httpResp = logicalResp
@@ -76,9 +75,8 @@ func handleLogical(core *vault.Core) http.Handler {
 }
 
 type LogicalResponse struct {
-	VaultId          string                 `json:"vault_id"`
-	Renewable        bool                   `json:"renewable"`
-	LeaseDuration    int                    `json:"lease_duration"`
-	LeaseDurationMax int                    `json:"lease_duration_max"`
-	Data             map[string]interface{} `json:"data"`
+	VaultId       string                 `json:"vault_id"`
+	Renewable     bool                   `json:"renewable"`
+	LeaseDuration int                    `json:"lease_duration"`
+	Data          map[string]interface{} `json:"data"`
 }

http/logical_test.go

@@ -25,10 +25,9 @@ func TestLogical(t *testing.T) {
 
 	var actual map[string]interface{}
 	expected := map[string]interface{}{
-		"vault_id":           "",
-		"renewable":          false,
-		"lease_duration":     float64(0),
-		"lease_duration_max": float64(0),
+		"vault_id":       "",
+		"renewable":      false,
+		"lease_duration": float64(0),
 		"data": map[string]interface{}{
 			"data": "bar",
 		},

logical/request.go

@@ -49,6 +49,7 @@ const (
 	DeleteOperation           = "delete"
 	ListOperation             = "list"
 	RevokeOperation           = "revoke"
+	RenewOperation            = "renew"
 	HelpOperation             = "help"
 )
 

logical/response.go

@@ -21,12 +21,10 @@ type Response struct {
 
 // Lease is used to provide more information about the lease
 type Lease struct {
-	VaultID      string        // VaultID is the unique identifier used for renewal and revocation
-	Renewable    bool          // Is the VaultID renewable
-	Revokable    bool          // Is the secret revokable. Must support 'Revoke' operation.
-	Duration     time.Duration // Current lease duration
-	MaxDuration  time.Duration // Maximum lease duration
-	MaxIncrement time.Duration // Maximum increment to lease duration
+	VaultID     string        // VaultID is the unique identifier used for renewal and revocation
+	Renewable   bool          // Is the VaultID renewable
+	Duration    time.Duration // Current lease duration
+	GracePeriod time.Duration // Lease revocation grace period (Duration+GracePeriod=RevokePeriod)
 }
 
 // Validate is used to sanity check a lease
@@ -34,14 +32,8 @@ func (l *Lease) Validate() error {
 	if l.Duration <= 0 {
 		return fmt.Errorf("lease duration must be greater than zero")
 	}
-	if l.MaxDuration <= 0 {
-		return fmt.Errorf("maximum lease duration must be greater than zero")
-	}
-	if l.Duration > l.MaxDuration {
-		return fmt.Errorf("lease duration cannot be greater than maximum lease duration")
-	}
-	if l.MaxIncrement < 0 {
-		return fmt.Errorf("maximum lease increment cannot be negative")
+	if l.GracePeriod < 0 {
+		return fmt.Errorf("grace period cannot be less than zero")
 	}
 	return nil
 }

vault/barrier_view.go

@@ -1,6 +1,7 @@
 package vault
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/hashicorp/vault/logical"
@@ -80,3 +81,30 @@ func (v *BarrierView) expandKey(suffix string) string {
 func (v *BarrierView) truncateKey(full string) string {
 	return strings.TrimPrefix(full, v.prefix)
 }
+
+// ScanView is used to scan all the keys in a view iteratively
+func ScanView(view *BarrierView, cb func(path string)) error {
+	frontier := []string{""}
+	for len(frontier) > 0 {
+		n := len(frontier)
+		current := frontier[n-1]
+		frontier = frontier[:n-1]
+
+		// List the contents
+		contents, err := view.List(current)
+		if err != nil {
+			return fmt.Errorf("list failed at path '%s': %v", current, err)
+		}
+
+		// Handle the contents in the directory
+		for _, c := range contents {
+			fullPath := current + c
+			if strings.HasSuffix(c, "/") {
+				frontier = append(frontier, fullPath)
+			} else {
+				cb(fullPath)
+			}
+		}
+	}
+	return nil
+}

vault/barrier_view_test.go

@@ -1,6 +1,8 @@
 package vault
 
 import (
+	"reflect"
+	"sort"
 	"testing"
 
 	"github.com/hashicorp/vault/logical"
@@ -143,3 +145,41 @@ func TestBarrierView_SubView(t *testing.T) {
 		t.Fatalf("nested foo/bar/test should be gone")
 	}
 }
+
+func TestBarrierView_Scan(t *testing.T) {
+	_, barrier, _ := mockBarrier(t)
+	view := NewBarrierView(barrier, "view/")
+
+	expect := []string{}
+	ent := []*logical.StorageEntry{
+		&logical.StorageEntry{Key: "foo", Value: []byte("test")},
+		&logical.StorageEntry{Key: "zip", Value: []byte("test")},
+		&logical.StorageEntry{Key: "foo/bar", Value: []byte("test")},
+		&logical.StorageEntry{Key: "foo/zap", Value: []byte("test")},
+		&logical.StorageEntry{Key: "foo/bar/baz", Value: []byte("test")},
+		&logical.StorageEntry{Key: "foo/bar/zoo", Value: []byte("test")},
+	}
+
+	for _, e := range ent {
+		expect = append(expect, e.Key)
+		if err := view.Put(e); err != nil {
+			t.Fatalf("err: %v", err)
+		}
+	}
+
+	var out []string
+	cb := func(path string) {
+		out = append(out, path)
+	}
+
+	// Collect the keys
+	if err := ScanView(view, cb); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	sort.Strings(out)
+	sort.Strings(expect)
+	if !reflect.DeepEqual(out, expect) {
+		t.Fatalf("out: %v expect: %v", out, expect)
+	}
+}

vault/expiration.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"path"
+	"strings"
 	"sync"
 	"time"
 
@@ -16,6 +17,15 @@ const (
 	// expirationSubPath is the sub-path used for the expiration manager
 	// view. This is nested under the system view.
 	expirationSubPath = "expire/"
+
+	// maxRevokeAttempts limits how many revoke attempts are made
+	maxRevokeAttempts = 6
+
+	// revokeRetryBase is a baseline retry time
+	revokeRetryBase = 10 * time.Second
+
+	// minRevokeDelay is used to prevent an instant revoke on restore
+	minRevokeDelay = 5 * time.Second
 )
 
 // ExpirationManager is used by the Core to manage leases. Secrets
@@ -28,7 +38,7 @@ type ExpirationManager struct {
 	logger *log.Logger
 
 	pending     map[string]*time.Timer
-	pendingLock sync.RWMutex
+	pendingLock sync.Mutex
 }
 
 // NewExpirationManager creates a new ExpirationManager that is backed
@@ -76,7 +86,45 @@ func (c *Core) stopExpiration() error {
 // Restore is used to recover the lease states when starting.
 // This is used after starting the vault.
 func (m *ExpirationManager) Restore() error {
-	// TODO: Restore...
+	m.pendingLock.Lock()
+	defer m.pendingLock.Unlock()
+
+	// Accumulate existing leases
+	var existing []string
+	cb := func(path string) {
+		existing = append(existing, path)
+	}
+
+	// Scan for all the leases
+	if err := ScanView(m.view, cb); err != nil {
+		return fmt.Errorf("failed to scan for leases: %v", err)
+	}
+
+	// Restore each key
+	for _, vaultID := range existing {
+		// Load the entry
+		le, err := m.loadEntry(vaultID)
+		if err != nil {
+			return err
+		}
+
+		// If there is no entry, nothing to restore
+		if le == nil {
+			continue
+		}
+
+		// Determine the remaining time to expiration
+		expires := time.Now().UTC().Sub(le.ExpireTime)
+		if expires <= 0 {
+			expires = minRevokeDelay
+		}
+
+		// Setup revocation timer
+		m.pending[le.VaultID] = time.AfterFunc(expires, func() {
+			m.expireID(le.VaultID)
+		})
+	}
+	m.logger.Printf("[INFO] expire: restored %d leases", len(m.pending))
 	return nil
 }
 
@@ -95,6 +143,34 @@ func (m *ExpirationManager) Stop() error {
 
 // Revoke is used to revoke a secret named by the given vaultID
 func (m *ExpirationManager) Revoke(vaultID string) error {
+	// Load the entry
+	le, err := m.loadEntry(vaultID)
+	if err != nil {
+		return err
+	}
+
+	// If there is no entry, nothing to revoke
+	if le == nil {
+		return nil
+	}
+
+	// Revoke the entry
+	if err := m.revokeEntry(le); err != nil {
+		return err
+	}
+
+	// Delete the entry
+	if err := m.deleteEntry(vaultID); err != nil {
+		return err
+	}
+
+	// Clear the expiration handler
+	m.pendingLock.Lock()
+	if timer, ok := m.pending[vaultID]; ok {
+		timer.Stop()
+		delete(m.pending, vaultID)
+	}
+	m.pendingLock.Unlock()
 	return nil
 }
 
@@ -102,13 +178,84 @@ func (m *ExpirationManager) Revoke(vaultID string) error {
 // The prefix maps to that of the mount table to make this simpler
 // to reason about.
 func (m *ExpirationManager) RevokePrefix(prefix string) error {
+	// Accumulate existing leases
+	var existing []string
+	cb := func(path string) {
+		existing = append(existing, path)
+	}
+
+	// Ensure there is a trailing slash
+	if !strings.HasSuffix(prefix, "/") {
+		prefix = prefix + "/"
+	}
+
+	// Scan for all the leases in the prefix
+	if err := ScanView(m.view.SubView(prefix), cb); err != nil {
+		return fmt.Errorf("failed to scan for leases: %v", err)
+	}
+
+	// Revoke all the keys
+	for idx, vaultID := range existing {
+		if err := m.Revoke(vaultID); err != nil {
+			return fmt.Errorf("failed to revoke '%s' (%d / %d): %v",
+				vaultID, idx+1, len(existing), err)
+		}
+	}
 	return nil
 }
 
 // Renew is used to renew a secret using the given vaultID
 // and a renew interval. The increment may be ignored.
-func (m *ExpirationManager) Renew(vaultID string, increment time.Duration) (*logical.Lease, error) {
-	return nil, nil
+func (m *ExpirationManager) Renew(vaultID string, increment time.Duration) (*logical.Response, error) {
+	// Load the entry
+	le, err := m.loadEntry(vaultID)
+	if err != nil {
+		return nil, err
+	}
+
+	// If there is no entry, cannot review
+	if le == nil {
+		return nil, fmt.Errorf("lease not found")
+	}
+
+	// Determine if the lease is expired
+	if le.ExpireTime.Before(time.Now().UTC()) {
+		return nil, fmt.Errorf("lease expired")
+	}
+
+	// Attempt to renew the entry
+	resp, err := m.renewEntry(le)
+	if err != nil {
+		return nil, err
+	}
+
+	// Fast-path if there is no lease
+	if resp == nil || resp.Lease == nil || !resp.IsSecret {
+		return resp, nil
+	}
+
+	// Validate the lease
+	if err := resp.Lease.Validate(); err != nil {
+		return nil, err
+	}
+
+	// Update the lease entry
+	le.Data = resp.Data
+	le.Lease = resp.Lease
+	le.ExpireTime = time.Now().UTC().Add(resp.Lease.Duration)
+	if err := m.persistEntry(le); err != nil {
+		return nil, err
+	}
+
+	// Update the expiration time
+	m.pendingLock.Lock()
+	if timer, ok := m.pending[vaultID]; ok {
+		timer.Reset(resp.Lease.Duration)
+	}
+	m.pendingLock.Unlock()
+
+	// Return the response
+	return resp, nil
 }
 
 // Register is used to take a request and response with an associated
@@ -133,12 +280,12 @@ func (m *ExpirationManager) Register(req *logical.Request, resp *logical.Respons
 	// Create a lease entry
 	now := time.Now().UTC()
 	le := leaseEntry{
-		VaultID:   path.Join(req.Path, generateUUID()),
-		Path:      req.Path,
-		Data:      resp.Data,
-		Lease:     resp.Lease,
-		IssueTime: now,
-		RenewTime: now,
+		VaultID:    path.Join(req.Path, generateUUID()),
+		Path:       req.Path,
+		Data:       resp.Data,
+		Lease:      resp.Lease,
+		IssueTime:  now,
+		ExpireTime: now.Add(resp.Lease.Duration),
 	}
 
 	// Encode the entry
@@ -148,10 +295,9 @@ func (m *ExpirationManager) Register(req *logical.Request, resp *logical.Respons
 
 	// Setup revocation timer
 	m.pendingLock.Lock()
-	timer := time.AfterFunc(resp.Lease.Duration, func() {
+	m.pending[le.VaultID] = time.AfterFunc(resp.Lease.Duration, func() {
 		m.expireID(le.VaultID)
 	})
-	m.pending[le.VaultID] = timer
 	m.pendingLock.Unlock()
 
 	// Done
@@ -165,22 +311,16 @@ func (m *ExpirationManager) expireID(vaultID string) {
 	delete(m.pending, vaultID)
 	m.pendingLock.Unlock()
 
-	// Load the entry
-	le, err := m.loadEntry(vaultID)
-	if err != nil {
-		m.logger.Printf("[ERR] expire: failed to read entry '%s': %v", vaultID, err)
+	for attempt := uint(0); attempt < maxRevokeAttempts; attempt++ {
+		err := m.Revoke(vaultID)
+		if err == nil {
+			m.logger.Printf("[INFO] expire: revoked '%s'", vaultID)
+			return
+		}
+		m.logger.Printf("[ERR] expire: failed to revoke '%s': %v", vaultID, err)
+		time.Sleep((1 << attempt) * revokeRetryBase)
 	}
-
-	// Revoke the entry
-	if err := m.revokeEntry(le); err != nil {
-		m.logger.Printf("[ERR] expire: failed to revoke entry '%s': %v", vaultID, err)
-	}
-
-	// Delete the entry
-	if err := m.deleteEntry(vaultID); err != nil {
-		m.logger.Printf("[ERR] expire: failed to delete entry '%s': %v", vaultID, err)
-	}
-	m.logger.Printf("[INFO] expire: revoked '%s'", vaultID)
+	m.logger.Printf("[ERR] expire: maximum revoke attempts for '%s' reached", vaultID)
 }
 
 // revokeEntry is used to attempt revocation of an internal entry
@@ -191,7 +331,24 @@ func (m *ExpirationManager) revokeEntry(le *leaseEntry) error {
 		Data:      le.Data,
 	}
 	_, err := m.router.Route(req)
-	return err
+	if err != nil {
+		return fmt.Errorf("failed to revoke entry: %v", err)
+	}
+	return nil
+}
+
+// renewEntry is used to attempt renew of an internal entry
+func (m *ExpirationManager) renewEntry(le *leaseEntry) (*logical.Response, error) {
+	req := &logical.Request{
+		Operation: logical.RenewOperation,
+		Path:      le.Path,
+		Data:      le.Data,
+	}
+	resp, err := m.router.Route(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to renew entry: %v", err)
+	}
+	return resp, nil
 }
 
 // loadEntry is used to read a lease entry
@@ -240,13 +397,12 @@ func (m *ExpirationManager) deleteEntry(vaultID string) error {
 // leaseEntry is used to structure the values the expiration
 // manager stores. This is used to handle renew and revocation.
 type leaseEntry struct {
-	VaultID        string                 `json:"vault_id"`
-	Path           string                 `json:"path"`
-	Data           map[string]interface{} `json:"data"`
-	Lease          *logical.Lease         `json:"lease"`
-	IssueTime      time.Time              `json:"issue_time"`
-	RenewTime      time.Time              `json:"renew_time"`
-	RevokeAttempts int                    `json:"renew_attempts"`
+	VaultID    string                 `json:"vault_id"`
+	Path       string                 `json:"path"`
+	Data       map[string]interface{} `json:"data"`
+	Lease      *logical.Lease         `json:"lease"`
+	IssueTime  time.Time              `json:"issue_time"`
+	ExpireTime time.Time              `json:"expire_time"`
 }
 
 // encode is used to JSON encode the lease entry

vault/expiration_test.go

@@ -62,8 +62,7 @@ func TestExpiration_Register(t *testing.T) {
 	resp := &logical.Response{
 		IsSecret: true,
 		Lease: &logical.Lease{
-			Duration:    time.Hour,
-			MaxDuration: time.Hour,
+			Duration: time.Hour,
 		},
 		Data: map[string]interface{}{
 			"access_key": "xyz",
@@ -93,12 +92,10 @@ func TestLeaseEntry(t *testing.T) {
 			"testing": true,
 		},
 		Lease: &logical.Lease{
-			Renewable:   true,
-			Duration:    time.Minute,
-			MaxDuration: time.Hour,
+			Duration: time.Minute,
 		},
-		IssueTime: time.Now(),
-		RenewTime: time.Now(),
+		IssueTime:  time.Now(),
+		ExpireTime: time.Now(),
 	}
 
 	enc, err := le.encode()

vault/logical_passthrough.go

@@ -70,11 +70,8 @@ func (b *PassthroughBackend) handleRead(
 		leaseDuration, err := time.ParseDuration(leaseVal)
 		if err == nil {
 			lease = &logical.Lease{
-				Renewable:    false,
-				Revokable:    false,
-				Duration:     leaseDuration,
-				MaxDuration:  leaseDuration,
-				MaxIncrement: 0,
+				Renewable: false,
+				Duration:  leaseDuration,
 			}
 		}
 	}

vault/logical_passthrough_test.go

@@ -60,11 +60,8 @@ func TestPassthroughBackend_Read(t *testing.T) {
 	expected := &logical.Response{
 		IsSecret: true,
 		Lease: &logical.Lease{
-			Renewable:    false,
-			Revokable:    false,
-			Duration:     time.Hour,
-			MaxDuration:  time.Hour,
-			MaxIncrement: 0,
+			Renewable: false,
+			Duration:  time.Hour,
 		},
 		Data: map[string]interface{}{
 			"raw":   "test",