1package vault 2 3import ( 4 "context" 5 "encoding/json" 6 "errors" 7 "fmt" 8 "os" 9 "path" 10 "strings" 11 "sync" 12 "sync/atomic" 13 "time" 14 15 metrics "github.com/armon/go-metrics" 16 "github.com/hashicorp/errwrap" 17 log "github.com/hashicorp/go-hclog" 18 multierror "github.com/hashicorp/go-multierror" 19 "github.com/hashicorp/vault/helper/namespace" 20 "github.com/hashicorp/vault/sdk/framework" 21 "github.com/hashicorp/vault/sdk/helper/base62" 22 "github.com/hashicorp/vault/sdk/helper/consts" 23 "github.com/hashicorp/vault/sdk/helper/jsonutil" 24 "github.com/hashicorp/vault/sdk/helper/locksutil" 25 "github.com/hashicorp/vault/sdk/logical" 26 uberAtomic "go.uber.org/atomic" 27) 28 29const ( 30 // expirationSubPath is the sub-path used for the expiration manager 31 // view. This is nested under the system view. 32 expirationSubPath = "expire/" 33 34 // leaseViewPrefix is the prefix used for the ID based lookup of leases. 35 leaseViewPrefix = "id/" 36 37 // tokenViewPrefix is the prefix used for the token based lookup of leases. 38 tokenViewPrefix = "token/" 39 40 // maxRevokeAttempts limits how many revoke attempts are made 41 maxRevokeAttempts = 6 42 43 // revokeRetryBase is a baseline retry time 44 revokeRetryBase = 10 * time.Second 45 46 // maxLeaseDuration is the default maximum lease duration 47 maxLeaseTTL = 32 * 24 * time.Hour 48 49 // defaultLeaseDuration is the default lease duration used when no lease is specified 50 defaultLeaseTTL = maxLeaseTTL 51 52 // maxLeaseThreshold is the maximum lease count before generating log warning 53 maxLeaseThreshold = 256000 54) 55 56type pendingInfo struct { 57 exportLeaseTimes *leaseEntry 58 timer *time.Timer 59} 60 61// ExpirationManager is used by the Core to manage leases. Secrets 62// can provide a lease, meaning that they can be renewed or revoked. 63// If a secret is not renewed in timely manner, it may be expired, and 64// the ExpirationManager will handle doing automatic revocation. 65type ExpirationManager struct { 66 core *Core 67 router *Router 68 idView *BarrierView 69 tokenView *BarrierView 70 tokenStore *TokenStore 71 logger log.Logger 72 73 pending map[string]pendingInfo 74 pendingLock sync.RWMutex 75 76 tidyLock *int32 77 78 restoreMode *int32 79 restoreModeLock sync.RWMutex 80 restoreRequestLock sync.RWMutex 81 restoreLocks []*locksutil.LockEntry 82 restoreLoaded sync.Map 83 quitCh chan struct{} 84 85 coreStateLock *sync.RWMutex 86 quitContext context.Context 87 leaseCheckCounter *uint32 88 89 logLeaseExpirations bool 90 expireFunc ExpireLeaseStrategy 91 92 // testRegisterAuthFailure, if set to true, triggers an explicit failure on 93 // RegisterAuth to simulate a partial failure during a token creation 94 // request. This value should only be set by tests. 95 testRegisterAuthFailure uberAtomic.Bool 96} 97 98type ExpireLeaseStrategy func(context.Context, *ExpirationManager, *leaseEntry) 99 100// revokeIDFunc is invoked when a given ID is expired 101func expireLeaseStrategyRevoke(ctx context.Context, m *ExpirationManager, le *leaseEntry) { 102 for attempt := uint(0); attempt < maxRevokeAttempts; attempt++ { 103 revokeCtx, cancel := context.WithTimeout(ctx, DefaultMaxRequestDuration) 104 revokeCtx = namespace.ContextWithNamespace(revokeCtx, le.namespace) 105 106 go func() { 107 select { 108 case <-ctx.Done(): 109 case <-m.quitCh: 110 cancel() 111 case <-revokeCtx.Done(): 112 } 113 }() 114 115 select { 116 case <-m.quitCh: 117 m.logger.Error("shutting down, not attempting further revocation of lease", "lease_id", le.LeaseID) 118 cancel() 119 return 120 case <-m.quitContext.Done(): 121 m.logger.Error("core context canceled, not attempting further revocation of lease", "lease_id", le.LeaseID) 122 cancel() 123 return 124 default: 125 } 126 127 m.coreStateLock.RLock() 128 err := m.Revoke(revokeCtx, le.LeaseID) 129 m.coreStateLock.RUnlock() 130 cancel() 131 if err == nil { 132 return 133 } 134 135 m.logger.Error("failed to revoke lease", "lease_id", le.LeaseID, "error", err) 136 time.Sleep((1 << attempt) * revokeRetryBase) 137 } 138 m.logger.Error("maximum revoke attempts reached", "lease_id", le.LeaseID) 139} 140 141// NewExpirationManager creates a new ExpirationManager that is backed 142// using a given view, and uses the provided router for revocation. 143func NewExpirationManager(c *Core, view *BarrierView, e ExpireLeaseStrategy, logger log.Logger) *ExpirationManager { 144 exp := &ExpirationManager{ 145 core: c, 146 router: c.router, 147 idView: view.SubView(leaseViewPrefix), 148 tokenView: view.SubView(tokenViewPrefix), 149 tokenStore: c.tokenStore, 150 logger: logger, 151 pending: make(map[string]pendingInfo), 152 tidyLock: new(int32), 153 154 // new instances of the expiration manager will go immediately into 155 // restore mode 156 restoreMode: new(int32), 157 restoreLocks: locksutil.CreateLocks(), 158 quitCh: make(chan struct{}), 159 160 coreStateLock: &c.stateLock, 161 quitContext: c.activeContext, 162 leaseCheckCounter: new(uint32), 163 164 logLeaseExpirations: os.Getenv("VAULT_SKIP_LOGGING_LEASE_EXPIRATIONS") == "", 165 expireFunc: e, 166 } 167 *exp.restoreMode = 1 168 169 if exp.logger == nil { 170 opts := log.LoggerOptions{Name: "expiration_manager"} 171 exp.logger = log.New(&opts) 172 } 173 174 return exp 175} 176 177// setupExpiration is invoked after we've loaded the mount table to 178// initialize the expiration manager 179func (c *Core) setupExpiration(e ExpireLeaseStrategy) error { 180 c.metricsMutex.Lock() 181 defer c.metricsMutex.Unlock() 182 // Create a sub-view 183 view := c.systemBarrierView.SubView(expirationSubPath) 184 185 // Create the manager 186 expLogger := c.baseLogger.Named("expiration") 187 c.AddLogger(expLogger) 188 mgr := NewExpirationManager(c, view, e, expLogger) 189 c.expiration = mgr 190 191 // Link the token store to this 192 c.tokenStore.SetExpirationManager(mgr) 193 194 // Restore the existing state 195 c.logger.Info("restoring leases") 196 errorFunc := func() { 197 c.logger.Error("shutting down") 198 if err := c.Shutdown(); err != nil { 199 c.logger.Error("error shutting down core", "error", err) 200 } 201 } 202 go c.expiration.Restore(errorFunc) 203 204 return nil 205} 206 207// stopExpiration is used to stop the expiration manager before 208// sealing the Vault. 209func (c *Core) stopExpiration() error { 210 if c.expiration != nil { 211 if err := c.expiration.Stop(); err != nil { 212 return err 213 } 214 c.metricsMutex.Lock() 215 defer c.metricsMutex.Unlock() 216 c.expiration = nil 217 } 218 return nil 219} 220 221// lockLease takes out a lock for a given lease ID 222func (m *ExpirationManager) lockLease(leaseID string) { 223 locksutil.LockForKey(m.restoreLocks, leaseID).Lock() 224} 225 226// unlockLease unlocks a given lease ID 227func (m *ExpirationManager) unlockLease(leaseID string) { 228 locksutil.LockForKey(m.restoreLocks, leaseID).Unlock() 229} 230 231// inRestoreMode returns if we are currently in restore mode 232func (m *ExpirationManager) inRestoreMode() bool { 233 return atomic.LoadInt32(m.restoreMode) == 1 234} 235 236func (m *ExpirationManager) invalidate(key string) { 237 238 switch { 239 case strings.HasPrefix(key, leaseViewPrefix): 240 // Clear from the pending expiration 241 leaseID := strings.TrimPrefix(key, leaseViewPrefix) 242 m.pendingLock.Lock() 243 if pending, ok := m.pending[leaseID]; ok { 244 pending.timer.Stop() 245 delete(m.pending, leaseID) 246 } 247 m.pendingLock.Unlock() 248 } 249} 250 251// Tidy cleans up the dangling storage entries for leases. It scans the storage 252// view to find all the available leases, checks if the token embedded in it is 253// either empty or invalid and in both the cases, it revokes them. It also uses 254// a token cache to avoid multiple lookups of the same token ID. It is normally 255// not required to use the API that invokes this. This is only intended to 256// clean up the corrupt storage due to bugs. 257func (m *ExpirationManager) Tidy(ctx context.Context) error { 258 if m.inRestoreMode() { 259 return errors.New("cannot run tidy while restoring leases") 260 } 261 262 var tidyErrors *multierror.Error 263 264 logger := m.logger.Named("tidy") 265 m.core.AddLogger(logger) 266 267 if !atomic.CompareAndSwapInt32(m.tidyLock, 0, 1) { 268 logger.Warn("tidy operation on leases is already in progress") 269 return nil 270 } 271 272 defer atomic.CompareAndSwapInt32(m.tidyLock, 1, 0) 273 274 logger.Info("beginning tidy operation on leases") 275 defer logger.Info("finished tidy operation on leases") 276 277 // Create a cache to keep track of looked up tokens 278 tokenCache := make(map[string]bool) 279 var countLease, revokedCount, deletedCountInvalidToken, deletedCountEmptyToken int64 280 281 tidyFunc := func(leaseID string) { 282 countLease++ 283 if countLease%500 == 0 { 284 logger.Info("tidying leases", "progress", countLease) 285 } 286 287 le, err := m.loadEntry(ctx, leaseID) 288 if err != nil { 289 tidyErrors = multierror.Append(tidyErrors, errwrap.Wrapf(fmt.Sprintf("failed to load the lease ID %q: {{err}}", leaseID), err)) 290 return 291 } 292 293 if le == nil { 294 tidyErrors = multierror.Append(tidyErrors, errwrap.Wrapf(fmt.Sprintf("nil entry for lease ID %q: {{err}}", leaseID), err)) 295 return 296 } 297 298 var isValid, ok bool 299 revokeLease := false 300 if le.ClientToken == "" { 301 logger.Debug("revoking lease which has an empty token", "lease_id", leaseID) 302 revokeLease = true 303 deletedCountEmptyToken++ 304 goto REVOKE_CHECK 305 } 306 307 isValid, ok = tokenCache[le.ClientToken] 308 if !ok { 309 lock := locksutil.LockForKey(m.tokenStore.tokenLocks, le.ClientToken) 310 lock.RLock() 311 te, err := m.tokenStore.lookupInternal(ctx, le.ClientToken, false, true) 312 lock.RUnlock() 313 314 if err != nil { 315 tidyErrors = multierror.Append(tidyErrors, errwrap.Wrapf("failed to lookup token: {{err}}", err)) 316 return 317 } 318 319 if te == nil { 320 logger.Debug("revoking lease which holds an invalid token", "lease_id", leaseID) 321 revokeLease = true 322 deletedCountInvalidToken++ 323 tokenCache[le.ClientToken] = false 324 } else { 325 tokenCache[le.ClientToken] = true 326 } 327 goto REVOKE_CHECK 328 } else { 329 if isValid { 330 return 331 } 332 333 logger.Debug("revoking lease which contains an invalid token", "lease_id", leaseID) 334 revokeLease = true 335 deletedCountInvalidToken++ 336 goto REVOKE_CHECK 337 } 338 339 REVOKE_CHECK: 340 if revokeLease { 341 // Force the revocation and skip going through the token store 342 // again 343 err = m.revokeCommon(ctx, leaseID, true, true) 344 if err != nil { 345 tidyErrors = multierror.Append(tidyErrors, errwrap.Wrapf(fmt.Sprintf("failed to revoke an invalid lease with ID %q: {{err}}", leaseID), err)) 346 return 347 } 348 revokedCount++ 349 } 350 } 351 352 ns, err := namespace.FromContext(ctx) 353 if err != nil { 354 return err 355 } 356 leaseView := m.leaseView(ns) 357 if err := logical.ScanView(m.quitContext, leaseView, tidyFunc); err != nil { 358 return err 359 } 360 361 logger.Info("number of leases scanned", "count", countLease) 362 logger.Info("number of leases which had empty tokens", "count", deletedCountEmptyToken) 363 logger.Info("number of leases which had invalid tokens", "count", deletedCountInvalidToken) 364 logger.Info("number of leases successfully revoked", "count", revokedCount) 365 366 return tidyErrors.ErrorOrNil() 367} 368 369// Restore is used to recover the lease states when starting. 370// This is used after starting the vault. 371func (m *ExpirationManager) Restore(errorFunc func()) (retErr error) { 372 defer func() { 373 // Turn off restore mode. We can do this safely without the lock because 374 // if restore mode finished successfully, restore mode was already 375 // disabled with the lock. In an error state, this will allow the 376 // Stop() function to shut everything down. 377 atomic.StoreInt32(m.restoreMode, 0) 378 379 switch { 380 case retErr == nil: 381 case strings.Contains(retErr.Error(), context.Canceled.Error()): 382 // Don't run error func because we lost leadership 383 m.logger.Warn("context canceled while restoring leases, stopping lease loading") 384 retErr = nil 385 case errwrap.Contains(retErr, ErrBarrierSealed.Error()): 386 // Don't run error func because we're likely already shutting down 387 m.logger.Warn("barrier sealed while restoring leases, stopping lease loading") 388 retErr = nil 389 default: 390 m.logger.Error("error restoring leases", "error", retErr) 391 if errorFunc != nil { 392 errorFunc() 393 } 394 } 395 }() 396 397 // Accumulate existing leases 398 m.logger.Debug("collecting leases") 399 existing, leaseCount, err := m.collectLeases() 400 if err != nil { 401 return err 402 } 403 m.logger.Debug("leases collected", "num_existing", leaseCount) 404 405 // Make the channels used for the worker pool 406 type lease struct { 407 namespace *namespace.Namespace 408 id string 409 } 410 broker := make(chan *lease) 411 quit := make(chan bool) 412 // Buffer these channels to prevent deadlocks 413 errs := make(chan error, len(existing)) 414 result := make(chan struct{}, len(existing)) 415 416 // Use a wait group 417 wg := &sync.WaitGroup{} 418 419 // Create 64 workers to distribute work to 420 for i := 0; i < consts.ExpirationRestoreWorkerCount; i++ { 421 wg.Add(1) 422 go func() { 423 defer wg.Done() 424 425 for { 426 select { 427 case lease, ok := <-broker: 428 // broker has been closed, we are done 429 if !ok { 430 return 431 } 432 433 ctx := namespace.ContextWithNamespace(m.quitContext, lease.namespace) 434 err := m.processRestore(ctx, lease.id) 435 if err != nil { 436 errs <- err 437 continue 438 } 439 440 // Send message that lease is done 441 result <- struct{}{} 442 443 // quit early 444 case <-quit: 445 return 446 447 case <-m.quitCh: 448 return 449 } 450 } 451 }() 452 } 453 454 // Distribute the collected keys to the workers in a go routine 455 wg.Add(1) 456 go func() { 457 defer wg.Done() 458 i := 0 459 for ns := range existing { 460 for _, leaseID := range existing[ns] { 461 i++ 462 if i%500 == 0 { 463 m.logger.Debug("leases loading", "progress", i) 464 } 465 466 select { 467 case <-quit: 468 return 469 470 case <-m.quitCh: 471 return 472 473 default: 474 broker <- &lease{ 475 namespace: ns, 476 id: leaseID, 477 } 478 } 479 } 480 } 481 482 // Close the broker, causing worker routines to exit 483 close(broker) 484 }() 485 486 // Ensure all keys on the chan are processed 487 for i := 0; i < leaseCount; i++ { 488 select { 489 case err := <-errs: 490 // Close all go routines 491 close(quit) 492 return err 493 494 case <-m.quitCh: 495 close(quit) 496 return nil 497 498 case <-result: 499 } 500 } 501 502 // Let all go routines finish 503 wg.Wait() 504 505 m.restoreModeLock.Lock() 506 atomic.StoreInt32(m.restoreMode, 0) 507 m.restoreLoaded.Range(func(k, v interface{}) bool { 508 m.restoreLoaded.Delete(k) 509 return true 510 }) 511 m.restoreLocks = nil 512 m.restoreModeLock.Unlock() 513 514 m.logger.Info("lease restore complete") 515 return nil 516} 517 518// processRestore takes a lease and restores it in the expiration manager if it has 519// not already been seen 520func (m *ExpirationManager) processRestore(ctx context.Context, leaseID string) error { 521 m.restoreRequestLock.RLock() 522 defer m.restoreRequestLock.RUnlock() 523 524 // Check if the lease has been seen 525 if _, ok := m.restoreLoaded.Load(leaseID); ok { 526 return nil 527 } 528 529 m.lockLease(leaseID) 530 defer m.unlockLease(leaseID) 531 532 // Check again with the lease locked 533 if _, ok := m.restoreLoaded.Load(leaseID); ok { 534 return nil 535 } 536 537 // Load lease and restore expiration timer 538 _, err := m.loadEntryInternal(ctx, leaseID, true, false) 539 if err != nil { 540 return err 541 } 542 return nil 543} 544 545// Stop is used to prevent further automatic revocations. 546// This must be called before sealing the view. 547func (m *ExpirationManager) Stop() error { 548 // Stop all the pending expiration timers 549 m.logger.Debug("stop triggered") 550 defer m.logger.Debug("finished stopping") 551 552 // Do this before stopping pending timers to avoid potential races with 553 // expiring timers 554 close(m.quitCh) 555 556 m.pendingLock.Lock() 557 for _, pending := range m.pending { 558 pending.timer.Stop() 559 } 560 m.pending = make(map[string]pendingInfo) 561 m.pendingLock.Unlock() 562 563 if m.inRestoreMode() { 564 for { 565 if !m.inRestoreMode() { 566 break 567 } 568 time.Sleep(10 * time.Millisecond) 569 } 570 } 571 572 return nil 573} 574 575// Revoke is used to revoke a secret named by the given LeaseID 576func (m *ExpirationManager) Revoke(ctx context.Context, leaseID string) error { 577 defer metrics.MeasureSince([]string{"expire", "revoke"}, time.Now()) 578 579 return m.revokeCommon(ctx, leaseID, false, false) 580} 581 582// LazyRevoke is used to queue revocation for a secret named by the given 583// LeaseID. If the lease was not found it returns nil; if the lease was found 584// it triggers a return of a 202. 585func (m *ExpirationManager) LazyRevoke(ctx context.Context, leaseID string) error { 586 defer metrics.MeasureSince([]string{"expire", "lazy-revoke"}, time.Now()) 587 588 // Load the entry 589 le, err := m.loadEntry(ctx, leaseID) 590 if err != nil { 591 return err 592 } 593 594 // If there is no entry, nothing to revoke 595 if le == nil { 596 return nil 597 } 598 599 le.ExpireTime = time.Now() 600 { 601 m.pendingLock.Lock() 602 if err := m.persistEntry(ctx, le); err != nil { 603 m.pendingLock.Unlock() 604 return err 605 } 606 607 m.updatePendingInternal(le, 0) 608 m.pendingLock.Unlock() 609 } 610 611 return nil 612} 613 614// revokeCommon does the heavy lifting. If force is true, we ignore a problem 615// during revocation and still remove entries/index/lease timers 616func (m *ExpirationManager) revokeCommon(ctx context.Context, leaseID string, force, skipToken bool) error { 617 defer metrics.MeasureSince([]string{"expire", "revoke-common"}, time.Now()) 618 619 // Load the entry 620 le, err := m.loadEntry(ctx, leaseID) 621 if err != nil { 622 return err 623 } 624 625 // If there is no entry, nothing to revoke 626 if le == nil { 627 return nil 628 } 629 630 // Revoke the entry 631 if !skipToken || le.Auth == nil { 632 if err := m.revokeEntry(ctx, le); err != nil { 633 if !force { 634 return err 635 } 636 637 if m.logger.IsWarn() { 638 m.logger.Warn("revocation from the backend failed, but in force mode so ignoring", "error", err) 639 } 640 } 641 } 642 643 // Delete the entry 644 if err := m.deleteEntry(ctx, le); err != nil { 645 return err 646 } 647 648 // Delete the secondary index, but only if it's a leased secret (not auth) 649 if le.Secret != nil { 650 if err := m.removeIndexByToken(ctx, le); err != nil { 651 return err 652 } 653 } 654 655 // Clear the expiration handler 656 m.pendingLock.Lock() 657 if pending, ok := m.pending[leaseID]; ok { 658 pending.timer.Stop() 659 delete(m.pending, leaseID) 660 } 661 m.pendingLock.Unlock() 662 663 if m.logger.IsInfo() && !skipToken && m.logLeaseExpirations { 664 m.logger.Info("revoked lease", "lease_id", leaseID) 665 } 666 667 return nil 668} 669 670// RevokeForce works similarly to RevokePrefix but continues in the case of a 671// revocation error; this is mostly meant for recovery operations 672func (m *ExpirationManager) RevokeForce(ctx context.Context, prefix string) error { 673 defer metrics.MeasureSince([]string{"expire", "revoke-force"}, time.Now()) 674 675 return m.revokePrefixCommon(ctx, prefix, true, true) 676} 677 678// RevokePrefix is used to revoke all secrets with a given prefix. 679// The prefix maps to that of the mount table to make this simpler 680// to reason about. 681func (m *ExpirationManager) RevokePrefix(ctx context.Context, prefix string, sync bool) error { 682 defer metrics.MeasureSince([]string{"expire", "revoke-prefix"}, time.Now()) 683 684 return m.revokePrefixCommon(ctx, prefix, false, sync) 685} 686 687// RevokeByToken is used to revoke all the secrets issued with a given token. 688// This is done by using the secondary index. It also removes the lease entry 689// for the token itself. As a result it should *ONLY* ever be called from the 690// token store's revokeSalted function. 691func (m *ExpirationManager) RevokeByToken(ctx context.Context, te *logical.TokenEntry) error { 692 defer metrics.MeasureSince([]string{"expire", "revoke-by-token"}, time.Now()) 693 tokenNS, err := NamespaceByID(ctx, te.NamespaceID, m.core) 694 if err != nil { 695 return err 696 } 697 if tokenNS == nil { 698 return namespace.ErrNoNamespace 699 } 700 701 tokenCtx := namespace.ContextWithNamespace(ctx, tokenNS) 702 // Lookup the leases 703 existing, err := m.lookupLeasesByToken(tokenCtx, te) 704 if err != nil { 705 return errwrap.Wrapf("failed to scan for leases: {{err}}", err) 706 } 707 708 // Revoke all the keys 709 for _, leaseID := range existing { 710 // Load the entry 711 le, err := m.loadEntry(ctx, leaseID) 712 if err != nil { 713 return err 714 } 715 716 // If there's a lease, set expiration to now, persist, and call 717 // updatePending to hand off revocation to the expiration manager's pending 718 // timer map 719 if le != nil { 720 le.ExpireTime = time.Now() 721 722 { 723 m.pendingLock.Lock() 724 if err := m.persistEntry(ctx, le); err != nil { 725 m.pendingLock.Unlock() 726 return err 727 } 728 729 m.updatePendingInternal(le, 0) 730 m.pendingLock.Unlock() 731 } 732 } 733 } 734 735 // te.Path should never be empty, but we check just in case 736 if te.Path != "" { 737 saltCtx := namespace.ContextWithNamespace(ctx, tokenNS) 738 saltedID, err := m.tokenStore.SaltID(saltCtx, te.ID) 739 if err != nil { 740 return err 741 } 742 tokenLeaseID := path.Join(te.Path, saltedID) 743 744 if tokenNS.ID != namespace.RootNamespaceID { 745 tokenLeaseID = fmt.Sprintf("%s.%s", tokenLeaseID, tokenNS.ID) 746 } 747 748 // We want to skip the revokeEntry call as that will call back into 749 // revocation logic in the token store, which is what is running this 750 // function in the first place -- it'd be a deadlock loop. Since the only 751 // place that this function is called is revokeSalted in the token store, 752 // we're already revoking the token, so we just want to clean up the lease. 753 // This avoids spurious revocations later in the log when the timer runs 754 // out, and eases up resource usage. 755 return m.revokeCommon(ctx, tokenLeaseID, false, true) 756 } 757 758 return nil 759} 760 761func (m *ExpirationManager) revokePrefixCommon(ctx context.Context, prefix string, force, sync bool) error { 762 if m.inRestoreMode() { 763 m.restoreRequestLock.Lock() 764 defer m.restoreRequestLock.Unlock() 765 } 766 767 // Ensure there is a trailing slash; or, if there is no slash, see if there 768 // is a matching specific ID 769 if !strings.HasSuffix(prefix, "/") { 770 le, err := m.loadEntry(ctx, prefix) 771 if err == nil && le != nil { 772 if sync { 773 if err := m.revokeCommon(ctx, prefix, force, false); err != nil { 774 return errwrap.Wrapf(fmt.Sprintf("failed to revoke %q: {{err}}", prefix), err) 775 } 776 return nil 777 } 778 return m.LazyRevoke(ctx, prefix) 779 } 780 prefix = prefix + "/" 781 } 782 783 // Accumulate existing leases 784 ns, err := namespace.FromContext(ctx) 785 if err != nil { 786 return err 787 } 788 view := m.leaseView(ns) 789 sub := view.SubView(prefix) 790 existing, err := logical.CollectKeys(ctx, sub) 791 if err != nil { 792 return errwrap.Wrapf("failed to scan for leases: {{err}}", err) 793 } 794 795 // Revoke all the keys 796 for idx, suffix := range existing { 797 leaseID := prefix + suffix 798 switch { 799 case sync: 800 if err := m.revokeCommon(ctx, leaseID, force, false); err != nil { 801 return errwrap.Wrapf(fmt.Sprintf("failed to revoke %q (%d / %d): {{err}}", leaseID, idx+1, len(existing)), err) 802 } 803 default: 804 if err := m.LazyRevoke(ctx, leaseID); err != nil { 805 return errwrap.Wrapf(fmt.Sprintf("failed to revoke %q (%d / %d): {{err}}", leaseID, idx+1, len(existing)), err) 806 } 807 } 808 } 809 810 return nil 811} 812 813// Renew is used to renew a secret using the given leaseID 814// and a renew interval. The increment may be ignored. 815func (m *ExpirationManager) Renew(ctx context.Context, leaseID string, increment time.Duration) (*logical.Response, error) { 816 defer metrics.MeasureSince([]string{"expire", "renew"}, time.Now()) 817 818 // Load the entry 819 le, err := m.loadEntry(ctx, leaseID) 820 if err != nil { 821 return nil, err 822 } 823 824 // Check if the lease is renewable 825 if _, err := le.renewable(); err != nil { 826 return nil, err 827 } 828 829 if le.Secret == nil { 830 if le.Auth != nil { 831 return logical.ErrorResponse("tokens cannot be renewed through this endpoint"), nil 832 } 833 return logical.ErrorResponse("lease does not correspond to a secret"), nil 834 } 835 836 ns, err := namespace.FromContext(ctx) 837 if err != nil { 838 return nil, err 839 } 840 if ns.ID != le.namespace.ID { 841 return nil, errors.New("cannot renew a lease across namespaces") 842 } 843 844 sysViewCtx := namespace.ContextWithNamespace(ctx, le.namespace) 845 sysView := m.router.MatchingSystemView(sysViewCtx, le.Path) 846 if sysView == nil { 847 return nil, fmt.Errorf("unable to retrieve system view from router") 848 } 849 850 // Attempt to renew the entry 851 resp, err := m.renewEntry(ctx, le, increment) 852 if err != nil { 853 return nil, err 854 } 855 if resp == nil { 856 return nil, nil 857 } 858 if resp.IsError() { 859 return &logical.Response{ 860 Data: resp.Data, 861 }, nil 862 } 863 if resp.Secret == nil { 864 return nil, nil 865 } 866 867 ttl, warnings, err := framework.CalculateTTL(sysView, increment, resp.Secret.TTL, 0, resp.Secret.MaxTTL, 0, le.IssueTime) 868 if err != nil { 869 return nil, err 870 } 871 for _, warning := range warnings { 872 resp.AddWarning(warning) 873 } 874 resp.Secret.TTL = ttl 875 876 // Attach the LeaseID 877 resp.Secret.LeaseID = leaseID 878 879 // Update the lease entry 880 le.Data = resp.Data 881 le.Secret = resp.Secret 882 le.ExpireTime = resp.Secret.ExpirationTime() 883 le.LastRenewalTime = time.Now() 884 885 // If the token it's associated with is a batch token, constrain lease 886 // times 887 if le.ClientTokenType == logical.TokenTypeBatch { 888 te, err := m.tokenStore.Lookup(ctx, le.ClientToken) 889 if err != nil { 890 return nil, err 891 } 892 if te == nil { 893 return nil, errors.New("cannot renew lease, no valid associated token") 894 } 895 tokenLeaseTimes, err := m.FetchLeaseTimesByToken(ctx, te) 896 if err != nil { 897 return nil, err 898 } 899 if le.ExpireTime.After(tokenLeaseTimes.ExpireTime) { 900 resp.Secret.TTL = tokenLeaseTimes.ExpireTime.Sub(le.LastRenewalTime) 901 le.ExpireTime = tokenLeaseTimes.ExpireTime 902 } 903 } 904 905 { 906 m.pendingLock.Lock() 907 if err := m.persistEntry(ctx, le); err != nil { 908 m.pendingLock.Unlock() 909 return nil, err 910 } 911 912 // Update the expiration time 913 m.updatePendingInternal(le, resp.Secret.LeaseTotal()) 914 m.pendingLock.Unlock() 915 } 916 917 // Return the response 918 return resp, nil 919} 920 921// RenewToken is used to renew a token which does not need to 922// invoke a logical backend. 923func (m *ExpirationManager) RenewToken(ctx context.Context, req *logical.Request, te *logical.TokenEntry, 924 increment time.Duration) (*logical.Response, error) { 925 defer metrics.MeasureSince([]string{"expire", "renew-token"}, time.Now()) 926 927 tokenNS, err := NamespaceByID(ctx, te.NamespaceID, m.core) 928 if err != nil { 929 return nil, err 930 } 931 if tokenNS == nil { 932 return nil, namespace.ErrNoNamespace 933 } 934 935 ns, err := namespace.FromContext(ctx) 936 if err != nil { 937 return nil, err 938 } 939 if ns.ID != tokenNS.ID { 940 return nil, errors.New("cannot renew a token across namespaces") 941 } 942 943 // Compute the Lease ID 944 saltedID, err := m.tokenStore.SaltID(ctx, te.ID) 945 if err != nil { 946 return nil, err 947 } 948 949 leaseID := path.Join(te.Path, saltedID) 950 951 if ns.ID != namespace.RootNamespaceID { 952 leaseID = fmt.Sprintf("%s.%s", leaseID, ns.ID) 953 } 954 955 // Load the entry 956 le, err := m.loadEntry(ctx, leaseID) 957 if err != nil { 958 return nil, err 959 } 960 if le == nil { 961 return logical.ErrorResponse("invalid lease ID"), logical.ErrInvalidRequest 962 } 963 964 // Check if the lease is renewable. Note that this also checks for a nil 965 // lease and errors in that case as well. 966 if _, err := le.renewable(); err != nil { 967 return logical.ErrorResponse(err.Error()), logical.ErrInvalidRequest 968 } 969 970 // Attempt to renew the auth entry 971 resp, err := m.renewAuthEntry(ctx, req, le, increment) 972 if err != nil { 973 return nil, err 974 } 975 if resp == nil { 976 return nil, nil 977 } 978 if resp.IsError() { 979 return &logical.Response{ 980 Data: resp.Data, 981 }, nil 982 } 983 if resp.Auth == nil { 984 return nil, nil 985 } 986 987 sysViewCtx := namespace.ContextWithNamespace(ctx, le.namespace) 988 sysView := m.router.MatchingSystemView(sysViewCtx, le.Path) 989 if sysView == nil { 990 return nil, fmt.Errorf("unable to retrieve system view from router") 991 } 992 993 ttl, warnings, err := framework.CalculateTTL(sysView, increment, resp.Auth.TTL, resp.Auth.Period, resp.Auth.MaxTTL, resp.Auth.ExplicitMaxTTL, le.IssueTime) 994 if err != nil { 995 return nil, err 996 } 997 retResp := &logical.Response{} 998 for _, warning := range warnings { 999 retResp.AddWarning(warning) 1000 } 1001 resp.Auth.TTL = ttl 1002 1003 // Attach the ClientToken 1004 resp.Auth.ClientToken = te.ID 1005 1006 // Refresh groups 1007 if resp.Auth.EntityID != "" && 1008 resp.Auth.GroupAliases != nil && 1009 m.core.identityStore != nil { 1010 validAliases, err := m.core.identityStore.refreshExternalGroupMembershipsByEntityID(ctx, resp.Auth.EntityID, resp.Auth.GroupAliases) 1011 if err != nil { 1012 return nil, err 1013 } 1014 resp.Auth.GroupAliases = validAliases 1015 } 1016 1017 // Update the lease entry 1018 le.Auth = resp.Auth 1019 le.ExpireTime = resp.Auth.ExpirationTime() 1020 le.LastRenewalTime = time.Now() 1021 1022 { 1023 m.pendingLock.Lock() 1024 if err := m.persistEntry(ctx, le); err != nil { 1025 m.pendingLock.Unlock() 1026 return nil, err 1027 } 1028 1029 // Update the expiration time 1030 m.updatePendingInternal(le, resp.Auth.LeaseTotal()) 1031 m.pendingLock.Unlock() 1032 } 1033 1034 retResp.Auth = resp.Auth 1035 return retResp, nil 1036} 1037 1038// Register is used to take a request and response with an associated 1039// lease. The secret gets assigned a LeaseID and the management of 1040// of lease is assumed by the expiration manager. 1041func (m *ExpirationManager) Register(ctx context.Context, req *logical.Request, resp *logical.Response) (id string, retErr error) { 1042 defer metrics.MeasureSince([]string{"expire", "register"}, time.Now()) 1043 1044 te := req.TokenEntry() 1045 if te == nil { 1046 return "", fmt.Errorf("cannot register a lease with an empty client token") 1047 } 1048 1049 // Ignore if there is no leased secret 1050 if resp == nil || resp.Secret == nil { 1051 return "", nil 1052 } 1053 1054 // Validate the secret 1055 if err := resp.Secret.Validate(); err != nil { 1056 return "", err 1057 } 1058 1059 // Create a lease entry 1060 leaseRand, err := base62.Random(TokenLength) 1061 if err != nil { 1062 return "", err 1063 } 1064 1065 ns, err := namespace.FromContext(ctx) 1066 if err != nil { 1067 return "", err 1068 } 1069 1070 leaseID := path.Join(req.Path, leaseRand) 1071 1072 if ns.ID != namespace.RootNamespaceID { 1073 leaseID = fmt.Sprintf("%s.%s", leaseID, ns.ID) 1074 } 1075 1076 le := &leaseEntry{ 1077 LeaseID: leaseID, 1078 ClientToken: req.ClientToken, 1079 ClientTokenType: te.Type, 1080 Path: req.Path, 1081 Data: resp.Data, 1082 Secret: resp.Secret, 1083 IssueTime: time.Now(), 1084 ExpireTime: resp.Secret.ExpirationTime(), 1085 namespace: ns, 1086 Version: 1, 1087 } 1088 1089 defer func() { 1090 // If there is an error we want to rollback as much as possible (note 1091 // that errors here are ignored to do as much cleanup as we can). We 1092 // want to revoke a generated secret (since an error means we may not 1093 // be successfully tracking it), remove indexes, and delete the entry. 1094 if retErr != nil { 1095 revokeCtx := namespace.ContextWithNamespace(m.quitContext, ns) 1096 revResp, err := m.router.Route(revokeCtx, logical.RevokeRequest(req.Path, resp.Secret, resp.Data)) 1097 if err != nil { 1098 retErr = multierror.Append(retErr, errwrap.Wrapf("an additional internal error was encountered revoking the newly-generated secret: {{err}}", err)) 1099 } else if revResp != nil && revResp.IsError() { 1100 retErr = multierror.Append(retErr, errwrap.Wrapf("an additional error was encountered revoking the newly-generated secret: {{err}}", revResp.Error())) 1101 } 1102 1103 if err := m.deleteEntry(ctx, le); err != nil { 1104 retErr = multierror.Append(retErr, errwrap.Wrapf("an additional error was encountered deleting any lease associated with the newly-generated secret: {{err}}", err)) 1105 } 1106 1107 if err := m.removeIndexByToken(ctx, le); err != nil { 1108 retErr = multierror.Append(retErr, errwrap.Wrapf("an additional error was encountered removing lease indexes associated with the newly-generated secret: {{err}}", err)) 1109 } 1110 } 1111 }() 1112 1113 // If the token is a batch token, we want to constrain the maximum lifetime 1114 // by the token's lifetime 1115 if te.Type == logical.TokenTypeBatch { 1116 tokenLeaseTimes, err := m.FetchLeaseTimesByToken(ctx, te) 1117 if err != nil { 1118 return "", err 1119 } 1120 if le.ExpireTime.After(tokenLeaseTimes.ExpireTime) { 1121 le.ExpireTime = tokenLeaseTimes.ExpireTime 1122 } 1123 } 1124 1125 // Encode the entry 1126 if err := m.persistEntry(ctx, le); err != nil { 1127 return "", err 1128 } 1129 1130 // Maintain secondary index by token, except for orphan batch tokens 1131 switch { 1132 case te.Type != logical.TokenTypeBatch: 1133 if err := m.createIndexByToken(ctx, le, le.ClientToken); err != nil { 1134 return "", err 1135 } 1136 case te.Parent != "": 1137 // If it's a non-orphan batch token, assign the secondary index to its 1138 // parent 1139 if err := m.createIndexByToken(ctx, le, te.Parent); err != nil { 1140 return "", err 1141 } 1142 } 1143 1144 // Setup revocation timer if there is a lease 1145 m.updatePending(le, resp.Secret.LeaseTotal()) 1146 1147 // Done 1148 return le.LeaseID, nil 1149} 1150 1151// RegisterAuth is used to take an Auth response with an associated lease. 1152// The token does not get a LeaseID, but the lease management is handled by 1153// the expiration manager. 1154func (m *ExpirationManager) RegisterAuth(ctx context.Context, te *logical.TokenEntry, auth *logical.Auth) error { 1155 defer metrics.MeasureSince([]string{"expire", "register-auth"}, time.Now()) 1156 1157 // Triggers failure of RegisterAuth. This should only be set and triggered 1158 // by tests to simulate partial failure during a token creation request. 1159 if m.testRegisterAuthFailure.Load() { 1160 return fmt.Errorf("failing explicitly on RegisterAuth") 1161 } 1162 1163 authExpirationTime := auth.ExpirationTime() 1164 1165 if te.TTL == 0 && authExpirationTime.IsZero() && (len(te.Policies) != 1 || te.Policies[0] != "root") { 1166 return errors.New("refusing to register a lease for a non-root token with no TTL") 1167 } 1168 1169 if te.Type == logical.TokenTypeBatch { 1170 return errors.New("cannot register a lease for a batch token") 1171 } 1172 1173 if auth.ClientToken == "" { 1174 return errors.New("cannot register an auth lease with an empty token") 1175 } 1176 1177 if strings.Contains(te.Path, "..") { 1178 return consts.ErrPathContainsParentReferences 1179 } 1180 1181 tokenNS, err := NamespaceByID(ctx, te.NamespaceID, m.core) 1182 if err != nil { 1183 return err 1184 } 1185 if tokenNS == nil { 1186 return namespace.ErrNoNamespace 1187 } 1188 1189 saltCtx := namespace.ContextWithNamespace(ctx, tokenNS) 1190 saltedID, err := m.tokenStore.SaltID(saltCtx, auth.ClientToken) 1191 if err != nil { 1192 return err 1193 } 1194 1195 leaseID := path.Join(te.Path, saltedID) 1196 if tokenNS.ID != namespace.RootNamespaceID { 1197 leaseID = fmt.Sprintf("%s.%s", leaseID, tokenNS.ID) 1198 } 1199 1200 // Create a lease entry 1201 le := leaseEntry{ 1202 LeaseID: leaseID, 1203 ClientToken: auth.ClientToken, 1204 Auth: auth, 1205 Path: te.Path, 1206 IssueTime: time.Now(), 1207 ExpireTime: authExpirationTime, 1208 namespace: tokenNS, 1209 Version: 1, 1210 } 1211 1212 // Encode the entry 1213 if err := m.persistEntry(ctx, &le); err != nil { 1214 return err 1215 } 1216 1217 // Setup revocation timer 1218 m.updatePending(&le, auth.LeaseTotal()) 1219 1220 return nil 1221} 1222 1223// FetchLeaseTimesByToken is a helper function to use token values to compute 1224// the leaseID, rather than pushing that logic back into the token store. 1225// As a special case, for a batch token it simply returns the information 1226// encoded on it. 1227func (m *ExpirationManager) FetchLeaseTimesByToken(ctx context.Context, te *logical.TokenEntry) (*leaseEntry, error) { 1228 defer metrics.MeasureSince([]string{"expire", "fetch-lease-times-by-token"}, time.Now()) 1229 1230 if te == nil { 1231 return nil, errors.New("cannot fetch lease times for nil token") 1232 } 1233 1234 if te.Type == logical.TokenTypeBatch { 1235 issueTime := time.Unix(te.CreationTime, 0) 1236 return &leaseEntry{ 1237 IssueTime: issueTime, 1238 ExpireTime: issueTime.Add(te.TTL), 1239 ClientTokenType: logical.TokenTypeBatch, 1240 }, nil 1241 } 1242 1243 tokenNS, err := NamespaceByID(ctx, te.NamespaceID, m.core) 1244 if err != nil { 1245 return nil, err 1246 } 1247 if tokenNS == nil { 1248 return nil, namespace.ErrNoNamespace 1249 } 1250 1251 saltCtx := namespace.ContextWithNamespace(ctx, tokenNS) 1252 saltedID, err := m.tokenStore.SaltID(saltCtx, te.ID) 1253 if err != nil { 1254 return nil, err 1255 } 1256 1257 leaseID := path.Join(te.Path, saltedID) 1258 1259 if tokenNS.ID != namespace.RootNamespaceID { 1260 leaseID = fmt.Sprintf("%s.%s", leaseID, tokenNS.ID) 1261 } 1262 1263 return m.FetchLeaseTimes(ctx, leaseID) 1264} 1265 1266// FetchLeaseTimes is used to fetch the issue time, expiration time, and last 1267// renewed time of a lease entry. It returns a leaseEntry itself, but with only 1268// those values copied over. 1269func (m *ExpirationManager) FetchLeaseTimes(ctx context.Context, leaseID string) (*leaseEntry, error) { 1270 defer metrics.MeasureSince([]string{"expire", "fetch-lease-times"}, time.Now()) 1271 1272 m.pendingLock.RLock() 1273 val := m.pending[leaseID] 1274 m.pendingLock.RUnlock() 1275 1276 if val.exportLeaseTimes != nil { 1277 return val.exportLeaseTimes, nil 1278 } 1279 1280 // Load the entry 1281 le, err := m.loadEntryInternal(ctx, leaseID, true, false) 1282 if err != nil { 1283 return nil, err 1284 } 1285 if le == nil { 1286 return nil, nil 1287 } 1288 1289 return m.leaseTimesForExport(le), nil 1290} 1291 1292// Returns lease times for outside callers based on the full leaseEntry passed in 1293func (m *ExpirationManager) leaseTimesForExport(le *leaseEntry) *leaseEntry { 1294 ret := &leaseEntry{ 1295 IssueTime: le.IssueTime, 1296 ExpireTime: le.ExpireTime, 1297 LastRenewalTime: le.LastRenewalTime, 1298 } 1299 if le.Secret != nil { 1300 ret.Secret = &logical.Secret{} 1301 ret.Secret.Renewable = le.Secret.Renewable 1302 ret.Secret.TTL = le.Secret.TTL 1303 } 1304 if le.Auth != nil { 1305 ret.Auth = &logical.Auth{} 1306 ret.Auth.Renewable = le.Auth.Renewable 1307 ret.Auth.TTL = le.Auth.TTL 1308 } 1309 1310 return ret 1311} 1312 1313// updatePending is used to update a pending invocation for a lease 1314func (m *ExpirationManager) updatePending(le *leaseEntry, leaseTotal time.Duration) { 1315 m.pendingLock.Lock() 1316 defer m.pendingLock.Unlock() 1317 1318 m.updatePendingInternal(le, leaseTotal) 1319} 1320 1321// updatePendingInternal is the locked version of updatePending; do not call 1322// this without a write lock on m.pending 1323func (m *ExpirationManager) updatePendingInternal(le *leaseEntry, leaseTotal time.Duration) { 1324 // Check for an existing timer 1325 pending, ok := m.pending[le.LeaseID] 1326 1327 // If there is no expiry time, don't do anything 1328 if le.ExpireTime.IsZero() { 1329 // if the timer happened to exist, stop the time and delete it from the 1330 // pending timers. 1331 if ok { 1332 pending.timer.Stop() 1333 delete(m.pending, le.LeaseID) 1334 } 1335 return 1336 } 1337 1338 // Create entry if it does not exist or reset if it does 1339 if ok { 1340 pending.timer.Reset(leaseTotal) 1341 } else { 1342 timer := time.AfterFunc(leaseTotal, func() { 1343 m.expireFunc(m.quitContext, m, le) 1344 }) 1345 pending = pendingInfo{ 1346 timer: timer, 1347 } 1348 } 1349 1350 // Extend the timer by the lease total 1351 pending.exportLeaseTimes = m.leaseTimesForExport(le) 1352 1353 m.pending[le.LeaseID] = pending 1354} 1355 1356// revokeEntry is used to attempt revocation of an internal entry 1357func (m *ExpirationManager) revokeEntry(ctx context.Context, le *leaseEntry) error { 1358 // Revocation of login tokens is special since we can by-pass the 1359 // backend and directly interact with the token store 1360 if le.Auth != nil { 1361 if le.ClientTokenType == logical.TokenTypeBatch { 1362 return errors.New("batch tokens cannot be revoked") 1363 } 1364 1365 if err := m.tokenStore.revokeTree(ctx, le); err != nil { 1366 return errwrap.Wrapf("failed to revoke token: {{err}}", err) 1367 } 1368 1369 return nil 1370 } 1371 1372 if le.Secret != nil { 1373 // not sure if this is really valid to have a leaseEntry with a nil Secret 1374 // (if there's a nil Secret, what are you really leasing?), but the tests 1375 // create one, and good to be defensive 1376 le.Secret.IssueTime = le.IssueTime 1377 } 1378 1379 // Make sure we're operating in the right namespace 1380 nsCtx := namespace.ContextWithNamespace(ctx, le.namespace) 1381 1382 // Handle standard revocation via backends 1383 resp, err := m.router.Route(nsCtx, logical.RevokeRequest(le.Path, le.Secret, le.Data)) 1384 if err != nil || (resp != nil && resp.IsError()) { 1385 return errwrap.Wrapf(fmt.Sprintf("failed to revoke entry: resp: %#v err: {{err}}", resp), err) 1386 } 1387 return nil 1388} 1389 1390// renewEntry is used to attempt renew of an internal entry 1391func (m *ExpirationManager) renewEntry(ctx context.Context, le *leaseEntry, increment time.Duration) (*logical.Response, error) { 1392 secret := *le.Secret 1393 secret.IssueTime = le.IssueTime 1394 secret.Increment = increment 1395 secret.LeaseID = "" 1396 1397 // Make sure we're operating in the right namespace 1398 nsCtx := namespace.ContextWithNamespace(ctx, le.namespace) 1399 1400 req := logical.RenewRequest(le.Path, &secret, le.Data) 1401 resp, err := m.router.Route(nsCtx, req) 1402 if err != nil || (resp != nil && resp.IsError()) { 1403 return nil, errwrap.Wrapf(fmt.Sprintf("failed to renew entry: resp: %#v err: {{err}}", resp), err) 1404 } 1405 return resp, nil 1406} 1407 1408// renewAuthEntry is used to attempt renew of an auth entry. Only the token 1409// store should get the actual token ID intact. 1410func (m *ExpirationManager) renewAuthEntry(ctx context.Context, req *logical.Request, le *leaseEntry, increment time.Duration) (*logical.Response, error) { 1411 if le.ClientTokenType == logical.TokenTypeBatch { 1412 return logical.ErrorResponse("batch tokens cannot be renewed"), nil 1413 } 1414 1415 auth := *le.Auth 1416 auth.IssueTime = le.IssueTime 1417 auth.Increment = increment 1418 if strings.HasPrefix(le.Path, "auth/token/") { 1419 auth.ClientToken = le.ClientToken 1420 } else { 1421 auth.ClientToken = "" 1422 } 1423 1424 // Make sure we're operating in the right namespace 1425 nsCtx := namespace.ContextWithNamespace(ctx, le.namespace) 1426 1427 authReq := logical.RenewAuthRequest(le.Path, &auth, nil) 1428 authReq.Connection = req.Connection 1429 resp, err := m.router.Route(nsCtx, authReq) 1430 if err != nil { 1431 return nil, errwrap.Wrapf("failed to renew entry: {{err}}", err) 1432 } 1433 return resp, nil 1434} 1435 1436// loadEntry is used to read a lease entry 1437func (m *ExpirationManager) loadEntry(ctx context.Context, leaseID string) (*leaseEntry, error) { 1438 // Take out the lease locks after we ensure we are in restore mode 1439 restoreMode := m.inRestoreMode() 1440 if restoreMode { 1441 m.restoreModeLock.RLock() 1442 defer m.restoreModeLock.RUnlock() 1443 1444 restoreMode = m.inRestoreMode() 1445 if restoreMode { 1446 m.lockLease(leaseID) 1447 defer m.unlockLease(leaseID) 1448 } 1449 } 1450 1451 _, nsID := namespace.SplitIDFromString(leaseID) 1452 if nsID != "" { 1453 leaseNS, err := NamespaceByID(ctx, nsID, m.core) 1454 if err != nil { 1455 return nil, err 1456 } 1457 if leaseNS != nil { 1458 ctx = namespace.ContextWithNamespace(ctx, leaseNS) 1459 } 1460 } else { 1461 ctx = namespace.ContextWithNamespace(ctx, namespace.RootNamespace) 1462 } 1463 return m.loadEntryInternal(ctx, leaseID, restoreMode, true) 1464} 1465 1466// loadEntryInternal is used when you need to load an entry but also need to 1467// control the lifecycle of the restoreLock 1468func (m *ExpirationManager) loadEntryInternal(ctx context.Context, leaseID string, restoreMode bool, checkRestored bool) (*leaseEntry, error) { 1469 ns, err := namespace.FromContext(ctx) 1470 if err != nil { 1471 return nil, err 1472 } 1473 1474 view := m.leaseView(ns) 1475 out, err := view.Get(ctx, leaseID) 1476 if err != nil { 1477 return nil, errwrap.Wrapf(fmt.Sprintf("failed to read lease entry %s: {{err}}", leaseID), err) 1478 } 1479 if out == nil { 1480 return nil, nil 1481 } 1482 le, err := decodeLeaseEntry(out.Value) 1483 if err != nil { 1484 return nil, errwrap.Wrapf(fmt.Sprintf("failed to decode lease entry %s: {{err}}", leaseID), err) 1485 } 1486 le.namespace = ns 1487 1488 if restoreMode { 1489 if checkRestored { 1490 // If we have already loaded this lease, we don't need to update on 1491 // load. In the case of renewal and revocation, updatePending will be 1492 // done after making the appropriate modifications to the lease. 1493 if _, ok := m.restoreLoaded.Load(leaseID); ok { 1494 return le, nil 1495 } 1496 } 1497 1498 // Update the cache of restored leases, either synchronously or through 1499 // the lazy loaded restore process 1500 m.restoreLoaded.Store(le.LeaseID, struct{}{}) 1501 1502 // Setup revocation timer 1503 m.updatePending(le, le.ExpireTime.Sub(time.Now())) 1504 } 1505 return le, nil 1506} 1507 1508// persistEntry is used to persist a lease entry 1509func (m *ExpirationManager) persistEntry(ctx context.Context, le *leaseEntry) error { 1510 // Encode the entry 1511 buf, err := le.encode() 1512 if err != nil { 1513 return errwrap.Wrapf("failed to encode lease entry: {{err}}", err) 1514 } 1515 1516 // Write out to the view 1517 ent := logical.StorageEntry{ 1518 Key: le.LeaseID, 1519 Value: buf, 1520 } 1521 if le.Auth != nil && len(le.Auth.Policies) == 1 && le.Auth.Policies[0] == "root" { 1522 ent.SealWrap = true 1523 } 1524 1525 view := m.leaseView(le.namespace) 1526 if err := view.Put(ctx, &ent); err != nil { 1527 return errwrap.Wrapf("failed to persist lease entry: {{err}}", err) 1528 } 1529 return nil 1530} 1531 1532// deleteEntry is used to delete a lease entry 1533func (m *ExpirationManager) deleteEntry(ctx context.Context, le *leaseEntry) error { 1534 view := m.leaseView(le.namespace) 1535 if err := view.Delete(ctx, le.LeaseID); err != nil { 1536 return errwrap.Wrapf("failed to delete lease entry: {{err}}", err) 1537 } 1538 return nil 1539} 1540 1541// createIndexByToken creates a secondary index from the token to a lease entry 1542func (m *ExpirationManager) createIndexByToken(ctx context.Context, le *leaseEntry, token string) error { 1543 tokenNS := namespace.RootNamespace 1544 saltCtx := namespace.ContextWithNamespace(ctx, namespace.RootNamespace) 1545 _, nsID := namespace.SplitIDFromString(token) 1546 if nsID != "" { 1547 var err error 1548 tokenNS, err = NamespaceByID(ctx, nsID, m.core) 1549 if err != nil { 1550 return err 1551 } 1552 if tokenNS != nil { 1553 saltCtx = namespace.ContextWithNamespace(ctx, tokenNS) 1554 } 1555 } 1556 1557 saltedID, err := m.tokenStore.SaltID(saltCtx, token) 1558 if err != nil { 1559 return err 1560 } 1561 1562 leaseSaltedID, err := m.tokenStore.SaltID(saltCtx, le.LeaseID) 1563 if err != nil { 1564 return err 1565 } 1566 1567 ent := logical.StorageEntry{ 1568 Key: saltedID + "/" + leaseSaltedID, 1569 Value: []byte(le.LeaseID), 1570 } 1571 tokenView := m.tokenIndexView(tokenNS) 1572 if err := tokenView.Put(ctx, &ent); err != nil { 1573 return errwrap.Wrapf("failed to persist lease index entry: {{err}}", err) 1574 } 1575 return nil 1576} 1577 1578// indexByToken looks up the secondary index from the token to a lease entry 1579func (m *ExpirationManager) indexByToken(ctx context.Context, le *leaseEntry) (*logical.StorageEntry, error) { 1580 tokenNS := namespace.RootNamespace 1581 saltCtx := namespace.ContextWithNamespace(ctx, tokenNS) 1582 _, nsID := namespace.SplitIDFromString(le.ClientToken) 1583 if nsID != "" { 1584 var err error 1585 tokenNS, err = NamespaceByID(ctx, nsID, m.core) 1586 if err != nil { 1587 return nil, err 1588 } 1589 if tokenNS != nil { 1590 saltCtx = namespace.ContextWithNamespace(ctx, tokenNS) 1591 } 1592 } 1593 1594 saltedID, err := m.tokenStore.SaltID(saltCtx, le.ClientToken) 1595 if err != nil { 1596 return nil, err 1597 } 1598 1599 leaseSaltedID, err := m.tokenStore.SaltID(saltCtx, le.LeaseID) 1600 if err != nil { 1601 return nil, err 1602 } 1603 1604 key := saltedID + "/" + leaseSaltedID 1605 tokenView := m.tokenIndexView(tokenNS) 1606 entry, err := tokenView.Get(ctx, key) 1607 if err != nil { 1608 return nil, fmt.Errorf("failed to look up secondary index entry") 1609 } 1610 return entry, nil 1611} 1612 1613// removeIndexByToken removes the secondary index from the token to a lease entry 1614func (m *ExpirationManager) removeIndexByToken(ctx context.Context, le *leaseEntry) error { 1615 tokenNS := namespace.RootNamespace 1616 saltCtx := namespace.ContextWithNamespace(ctx, namespace.RootNamespace) 1617 _, nsID := namespace.SplitIDFromString(le.ClientToken) 1618 if nsID != "" { 1619 var err error 1620 tokenNS, err = NamespaceByID(ctx, nsID, m.core) 1621 if err != nil { 1622 return err 1623 } 1624 if tokenNS != nil { 1625 saltCtx = namespace.ContextWithNamespace(ctx, tokenNS) 1626 } 1627 1628 // Downgrade logic for old-style (V0) namespace leases that had its 1629 // secondary index live in the root namespace. This reverts to the old 1630 // behavior of looking for the secondary index on these leases in the 1631 // root namespace to be cleaned up properly. We set it here because the 1632 // old behavior used the namespace's token store salt for its saltCtx. 1633 if le.Version < 1 { 1634 tokenNS = namespace.RootNamespace 1635 } 1636 } 1637 1638 saltedID, err := m.tokenStore.SaltID(saltCtx, le.ClientToken) 1639 if err != nil { 1640 return err 1641 } 1642 1643 leaseSaltedID, err := m.tokenStore.SaltID(saltCtx, le.LeaseID) 1644 if err != nil { 1645 return err 1646 } 1647 1648 key := saltedID + "/" + leaseSaltedID 1649 tokenView := m.tokenIndexView(tokenNS) 1650 if err := tokenView.Delete(ctx, key); err != nil { 1651 return errwrap.Wrapf("failed to delete lease index entry: {{err}}", err) 1652 } 1653 return nil 1654} 1655 1656// CreateOrFetchRevocationLeaseByToken is used to create or fetch the matching 1657// leaseID for a particular token. The lease is set to expire immediately after 1658// it's created. 1659func (m *ExpirationManager) CreateOrFetchRevocationLeaseByToken(ctx context.Context, te *logical.TokenEntry) (string, error) { 1660 // Fetch the saltedID of the token and construct the leaseID 1661 tokenNS, err := NamespaceByID(ctx, te.NamespaceID, m.core) 1662 if err != nil { 1663 return "", err 1664 } 1665 if tokenNS == nil { 1666 return "", namespace.ErrNoNamespace 1667 } 1668 1669 saltCtx := namespace.ContextWithNamespace(ctx, tokenNS) 1670 saltedID, err := m.tokenStore.SaltID(saltCtx, te.ID) 1671 if err != nil { 1672 return "", err 1673 } 1674 leaseID := path.Join(te.Path, saltedID) 1675 1676 if tokenNS.ID != namespace.RootNamespaceID { 1677 leaseID = fmt.Sprintf("%s.%s", leaseID, tokenNS.ID) 1678 } 1679 1680 // Load the entry 1681 le, err := m.loadEntry(ctx, leaseID) 1682 if err != nil { 1683 return "", err 1684 } 1685 1686 // If there's no associated leaseEntry for the token, we create one 1687 if le == nil { 1688 auth := &logical.Auth{ 1689 ClientToken: te.ID, 1690 LeaseOptions: logical.LeaseOptions{ 1691 TTL: time.Nanosecond, 1692 }, 1693 } 1694 1695 if strings.Contains(te.Path, "..") { 1696 return "", consts.ErrPathContainsParentReferences 1697 } 1698 1699 // Create a lease entry 1700 now := time.Now() 1701 le = &leaseEntry{ 1702 LeaseID: leaseID, 1703 ClientToken: auth.ClientToken, 1704 Auth: auth, 1705 Path: te.Path, 1706 IssueTime: now, 1707 ExpireTime: now.Add(time.Nanosecond), 1708 namespace: tokenNS, 1709 Version: 1, 1710 } 1711 1712 // Encode the entry 1713 if err := m.persistEntry(ctx, le); err != nil { 1714 return "", err 1715 } 1716 } 1717 1718 return le.LeaseID, nil 1719} 1720 1721// lookupLeasesByToken is used to lookup all the leaseID's via the tokenID 1722func (m *ExpirationManager) lookupLeasesByToken(ctx context.Context, te *logical.TokenEntry) ([]string, error) { 1723 tokenNS, err := NamespaceByID(ctx, te.NamespaceID, m.core) 1724 if err != nil { 1725 return nil, err 1726 } 1727 if tokenNS == nil { 1728 return nil, namespace.ErrNoNamespace 1729 } 1730 1731 saltCtx := namespace.ContextWithNamespace(ctx, tokenNS) 1732 saltedID, err := m.tokenStore.SaltID(saltCtx, te.ID) 1733 if err != nil { 1734 return nil, err 1735 } 1736 1737 tokenView := m.tokenIndexView(tokenNS) 1738 1739 // Scan via the index for sub-leases 1740 prefix := saltedID + "/" 1741 subKeys, err := tokenView.List(ctx, prefix) 1742 if err != nil { 1743 return nil, errwrap.Wrapf("failed to list leases: {{err}}", err) 1744 } 1745 1746 // Read each index entry 1747 leaseIDs := make([]string, 0, len(subKeys)) 1748 for _, sub := range subKeys { 1749 out, err := tokenView.Get(ctx, prefix+sub) 1750 if err != nil { 1751 return nil, errwrap.Wrapf("failed to read lease index: {{err}}", err) 1752 } 1753 if out == nil { 1754 continue 1755 } 1756 leaseIDs = append(leaseIDs, string(out.Value)) 1757 } 1758 1759 // Downgrade logic for old-style (V0) leases entries created by a namespace 1760 // token that lived in the root namespace. 1761 if tokenNS.ID != namespace.RootNamespaceID { 1762 tokenView := m.tokenIndexView(namespace.RootNamespace) 1763 1764 // Scan via the index for sub-leases on the root namespace 1765 prefix := saltedID + "/" 1766 subKeys, err := tokenView.List(ctx, prefix) 1767 if err != nil { 1768 return nil, errwrap.Wrapf("failed to list leases on root namespace: {{err}}", err) 1769 } 1770 1771 for _, sub := range subKeys { 1772 out, err := tokenView.Get(ctx, prefix+sub) 1773 if err != nil { 1774 return nil, errwrap.Wrapf("failed to read lease index on root namespace: {{err}}", err) 1775 } 1776 if out == nil { 1777 continue 1778 } 1779 leaseIDs = append(leaseIDs, string(out.Value)) 1780 } 1781 } 1782 1783 return leaseIDs, nil 1784} 1785 1786// emitMetrics is invoked periodically to emit statistics 1787func (m *ExpirationManager) emitMetrics() { 1788 m.pendingLock.RLock() 1789 num := len(m.pending) 1790 m.pendingLock.RUnlock() 1791 metrics.SetGauge([]string{"expire", "num_leases"}, float32(num)) 1792 // Check if lease count is greater than the threshold 1793 if num > maxLeaseThreshold { 1794 if atomic.LoadUint32(m.leaseCheckCounter) > 59 { 1795 m.logger.Warn("lease count exceeds warning lease threshold") 1796 atomic.StoreUint32(m.leaseCheckCounter, 0) 1797 } else { 1798 atomic.AddUint32(m.leaseCheckCounter, 1) 1799 } 1800 } 1801} 1802 1803// leaseEntry is used to structure the values the expiration 1804// manager stores. This is used to handle renew and revocation. 1805type leaseEntry struct { 1806 LeaseID string `json:"lease_id"` 1807 ClientToken string `json:"client_token"` 1808 ClientTokenType logical.TokenType `json:"token_type"` 1809 Path string `json:"path"` 1810 Data map[string]interface{} `json:"data"` 1811 Secret *logical.Secret `json:"secret"` 1812 Auth *logical.Auth `json:"auth"` 1813 IssueTime time.Time `json:"issue_time"` 1814 ExpireTime time.Time `json:"expire_time"` 1815 LastRenewalTime time.Time `json:"last_renewal_time"` 1816 1817 // Version is used to track new different versions of leases. V0 (or 1818 // zero-value) had non-root namespaced secondary indexes live in the root 1819 // namespace, and V1 has secondary indexes live in the matching namespace. 1820 Version int `json:"version"` 1821 1822 namespace *namespace.Namespace 1823} 1824 1825// encode is used to JSON encode the lease entry 1826func (le *leaseEntry) encode() ([]byte, error) { 1827 return json.Marshal(le) 1828} 1829 1830func (le *leaseEntry) renewable() (bool, error) { 1831 switch { 1832 // If there is no entry, cannot review to renew 1833 case le == nil: 1834 return false, fmt.Errorf("lease not found") 1835 1836 case le.ExpireTime.IsZero(): 1837 return false, fmt.Errorf("lease is not renewable") 1838 1839 case le.ClientTokenType == logical.TokenTypeBatch: 1840 return false, nil 1841 1842 // Determine if the lease is expired 1843 case le.ExpireTime.Before(time.Now()): 1844 return false, fmt.Errorf("lease expired") 1845 1846 // Determine if the lease is renewable 1847 case le.Secret != nil && !le.Secret.Renewable: 1848 return false, fmt.Errorf("lease is not renewable") 1849 1850 case le.Auth != nil && !le.Auth.Renewable: 1851 return false, fmt.Errorf("lease is not renewable") 1852 } 1853 1854 return true, nil 1855} 1856 1857func (le *leaseEntry) ttl() int64 { 1858 return int64(le.ExpireTime.Sub(time.Now().Round(time.Second)).Seconds()) 1859} 1860 1861// decodeLeaseEntry is used to reverse encode and return a new entry 1862func decodeLeaseEntry(buf []byte) (*leaseEntry, error) { 1863 out := new(leaseEntry) 1864 return out, jsonutil.DecodeJSON(buf, out) 1865} 1866