1package vault 2 3import ( 4 "context" 5 "encoding/json" 6 "errors" 7 "fmt" 8 "math/rand" 9 "os" 10 "path" 11 "sort" 12 "strconv" 13 "strings" 14 "sync" 15 "sync/atomic" 16 "time" 17 18 metrics "github.com/armon/go-metrics" 19 "github.com/hashicorp/errwrap" 20 log "github.com/hashicorp/go-hclog" 21 multierror "github.com/hashicorp/go-multierror" 22 "github.com/hashicorp/vault/helper/fairshare" 23 "github.com/hashicorp/vault/helper/metricsutil" 24 "github.com/hashicorp/vault/helper/namespace" 25 "github.com/hashicorp/vault/sdk/framework" 26 "github.com/hashicorp/vault/sdk/helper/base62" 27 "github.com/hashicorp/vault/sdk/helper/consts" 28 "github.com/hashicorp/vault/sdk/helper/jsonutil" 29 "github.com/hashicorp/vault/sdk/helper/locksutil" 30 "github.com/hashicorp/vault/sdk/logical" 31 "github.com/hashicorp/vault/sdk/physical" 32 "github.com/hashicorp/vault/vault/quotas" 33 uberAtomic "go.uber.org/atomic" 34) 35 36const ( 37 // expirationSubPath is the sub-path used for the expiration manager 38 // view. This is nested under the system view. 39 expirationSubPath = "expire/" 40 41 // leaseViewPrefix is the prefix used for the ID based lookup of leases. 42 leaseViewPrefix = "id/" 43 44 // tokenViewPrefix is the prefix used for the token based lookup of leases. 45 tokenViewPrefix = "token/" 46 47 // maxRevokeAttempts limits how many revoke attempts are made 48 maxRevokeAttempts = 6 49 50 // revokeRetryBase is a baseline retry time 51 revokeRetryBase = 10 * time.Second 52 53 // maxLeaseDuration is the default maximum lease duration 54 maxLeaseTTL = 32 * 24 * time.Hour 55 56 // defaultLeaseDuration is the default lease duration used when no lease is specified 57 defaultLeaseTTL = maxLeaseTTL 58 59 // maxLeaseThreshold is the maximum lease count before generating log warning 60 maxLeaseThreshold = 256000 61 62 // numExpirationWorkersDefault is the maximum amount of workers working on lease expiration 63 numExpirationWorkersDefault = 200 64 65 // number of workers to use for general purpose testing 66 numExpirationWorkersTest = 10 67 68 fairshareWorkersOverrideVar = "VAULT_LEASE_REVOCATION_WORKERS" 69 70 // limit irrevocable error messages to 240 characters to be respectful of 71 // storage/memory 72 maxIrrevocableErrorLength = 240 73 74 genericIrrevocableErrorMessage = "unknown" 75 76 outOfRetriesMessage = "out of retries" 77 78 // maximum number of irrevocable leases we return to the irrevocable lease 79 // list API **without** the `force` flag set 80 MaxIrrevocableLeasesToReturn = 10000 81 82 MaxIrrevocableLeasesWarning = "Command halted because many irrevocable leases were found. To emit the entire list, re-run the command with force set true." 83) 84 85type pendingInfo struct { 86 // A subset of the lease entry, cached in memory 87 cachedLeaseInfo *leaseEntry 88 timer *time.Timer 89 revokesAttempted uint8 90} 91 92// ExpirationManager is used by the Core to manage leases. Secrets 93// can provide a lease, meaning that they can be renewed or revoked. 94// If a secret is not renewed in timely manner, it may be expired, and 95// the ExpirationManager will handle doing automatic revocation. 96type ExpirationManager struct { 97 core *Core 98 router *Router 99 idView *BarrierView 100 tokenView *BarrierView 101 tokenStore *TokenStore 102 logger log.Logger 103 104 // Although the data structure itself is atomic, 105 // pendingLock should be held to ensure lease modifications 106 // are atomic (with respect to storage, expiration time, 107 // and particularly the lease count.) 108 // The nonexpiring map holds entries for root tokens with 109 // TTL zero, which we want to count but have no timer associated. 110 pending sync.Map 111 nonexpiring sync.Map 112 leaseCount int 113 pendingLock sync.RWMutex 114 115 // A sync.Lock for every active leaseID 116 lockPerLease sync.Map 117 // Track expired leases that have been determined to be irrevocable (without 118 // manual intervention). We retain a subset of the lease info in memory 119 irrevocable sync.Map 120 121 // Track count for metrics reporting 122 // This value is protected by pendingLock 123 irrevocableLeaseCount int 124 125 // The uniquePolicies map holds policy sets, so they can 126 // be deduplicated. It is periodically emptied to prevent 127 // unbounded growth. 128 uniquePolicies map[string][]string 129 emptyUniquePolicies *time.Ticker 130 131 tidyLock *int32 132 133 restoreMode *int32 134 restoreModeLock sync.RWMutex 135 restoreRequestLock sync.RWMutex 136 restoreLocks []*locksutil.LockEntry 137 restoreLoaded sync.Map 138 quitCh chan struct{} 139 140 // do not hold coreStateLock in any API handler code - it is already held 141 coreStateLock *DeadlockRWMutex 142 quitContext context.Context 143 leaseCheckCounter *uint32 144 145 logLeaseExpirations bool 146 expireFunc ExpireLeaseStrategy 147 148 revokePermitPool *physical.PermitPool 149 150 // testRegisterAuthFailure, if set to true, triggers an explicit failure on 151 // RegisterAuth to simulate a partial failure during a token creation 152 // request. This value should only be set by tests. 153 testRegisterAuthFailure uberAtomic.Bool 154 155 jobManager *fairshare.JobManager 156} 157 158type ExpireLeaseStrategy func(context.Context, *ExpirationManager, string, *namespace.Namespace) 159 160// revocationJob should only be created through newRevocationJob() 161type revocationJob struct { 162 leaseID string 163 ns *namespace.Namespace 164 m *ExpirationManager 165 nsCtx context.Context 166 startTime time.Time 167} 168 169func newRevocationJob(nsCtx context.Context, leaseID string, ns *namespace.Namespace, m *ExpirationManager) (*revocationJob, error) { 170 if leaseID == "" { 171 return nil, fmt.Errorf("cannot have empty lease id") 172 } 173 if m == nil { 174 return nil, fmt.Errorf("cannot have nil expiration manager") 175 } 176 if nsCtx == nil { 177 return nil, fmt.Errorf("cannot have nil namespace context.Context") 178 } 179 180 return &revocationJob{ 181 leaseID: leaseID, 182 ns: ns, 183 m: m, 184 nsCtx: nsCtx, 185 startTime: time.Now(), 186 }, nil 187} 188 189// errIsUnrecoverable returns true if the logical error is unlikely to resolve 190// automatically or with additional retries 191func errIsUnrecoverable(err error) bool { 192 switch { 193 case errors.Is(err, logical.ErrUnrecoverable), 194 errors.Is(err, logical.ErrUnsupportedOperation), 195 errors.Is(err, logical.ErrUnsupportedPath), 196 errors.Is(err, logical.ErrInvalidRequest): 197 return true 198 } 199 200 return false 201} 202 203func (r *revocationJob) Execute() error { 204 r.m.core.metricSink.IncrCounterWithLabels([]string{"expire", "lease_expiration"}, 1, []metrics.Label{metricsutil.NamespaceLabel(r.ns)}) 205 r.m.core.metricSink.MeasureSinceWithLabels([]string{"expire", "lease_expiration", "time_in_queue"}, r.startTime, []metrics.Label{metricsutil.NamespaceLabel(r.ns)}) 206 207 // don't start the timer until the revocation is being executed 208 revokeCtx, cancel := context.WithTimeout(r.nsCtx, DefaultMaxRequestDuration) 209 defer cancel() 210 211 go func() { 212 select { 213 case <-r.m.quitCh: 214 cancel() 215 case <-revokeCtx.Done(): 216 } 217 }() 218 219 select { 220 case <-r.m.quitCh: 221 r.m.logger.Error("shutting down, not attempting further revocation of lease", "lease_id", r.leaseID) 222 return nil 223 case <-r.m.quitContext.Done(): 224 r.m.logger.Error("core context canceled, not attempting further revocation of lease", "lease_id", r.leaseID) 225 return nil 226 default: 227 } 228 229 r.m.coreStateLock.RLock() 230 err := r.m.Revoke(revokeCtx, r.leaseID) 231 r.m.coreStateLock.RUnlock() 232 233 return err 234} 235 236func (r *revocationJob) OnFailure(err error) { 237 r.m.core.metricSink.IncrCounterWithLabels([]string{"expire", "lease_expiration", "error"}, 1, []metrics.Label{metricsutil.NamespaceLabel(r.ns)}) 238 r.m.logger.Error("failed to revoke lease", "lease_id", r.leaseID, "error", err) 239 240 r.m.pendingLock.Lock() 241 defer r.m.pendingLock.Unlock() 242 pendingRaw, ok := r.m.pending.Load(r.leaseID) 243 if !ok { 244 r.m.logger.Warn("failed to find lease in pending map for revocation retry", "lease_id", r.leaseID) 245 return 246 } 247 248 pending := pendingRaw.(pendingInfo) 249 pending.revokesAttempted++ 250 if pending.revokesAttempted >= maxRevokeAttempts || errIsUnrecoverable(err) { 251 r.m.logger.Trace("marking lease as irrevocable", "lease_id", r.leaseID, "error", err) 252 if pending.revokesAttempted >= maxRevokeAttempts { 253 r.m.logger.Trace("lease has consumed all retry attempts", "lease_id", r.leaseID) 254 err = fmt.Errorf("%v: %w", outOfRetriesMessage, err) 255 } 256 257 le, loadErr := r.m.loadEntry(r.nsCtx, r.leaseID) 258 if loadErr != nil { 259 r.m.logger.Warn("failed to mark lease as irrevocable - failed to load", "lease_id", r.leaseID, "err", loadErr) 260 return 261 } 262 if le == nil { 263 r.m.logger.Warn("failed to mark lease as irrevocable - nil lease", "lease_id", r.leaseID) 264 return 265 } 266 267 r.m.markLeaseIrrevocable(r.nsCtx, le, err) 268 return 269 } 270 271 pending.timer.Reset(revokeExponentialBackoff(pending.revokesAttempted)) 272 r.m.pending.Store(r.leaseID, pending) 273} 274 275func expireLeaseStrategyFairsharing(ctx context.Context, m *ExpirationManager, leaseID string, ns *namespace.Namespace) { 276 nsCtx := namespace.ContextWithNamespace(ctx, ns) 277 278 mountAccessor := m.getLeaseMountAccessorLocked(ctx, leaseID) 279 280 job, err := newRevocationJob(nsCtx, leaseID, ns, m) 281 if err != nil { 282 m.logger.Warn("error creating revocation job", "error", err) 283 return 284 } 285 286 m.jobManager.AddJob(job, mountAccessor) 287} 288 289func revokeExponentialBackoff(attempt uint8) time.Duration { 290 exp := (1 << attempt) * revokeRetryBase 291 randomDelta := 0.5 * float64(exp) 292 293 // Allow backoff time to be a random value between exp +/- (0.5*exp) 294 backoffTime := (float64(exp) - randomDelta) + (rand.Float64() * (2 * randomDelta)) 295 return time.Duration(backoffTime) 296} 297 298// revokeIDFunc is invoked when a given ID is expired 299func expireLeaseStrategyRevoke(ctx context.Context, m *ExpirationManager, leaseID string, ns *namespace.Namespace) { 300 for attempt := uint(0); attempt < maxRevokeAttempts; attempt++ { 301 releasePermit := func() {} 302 if m.revokePermitPool != nil { 303 m.logger.Trace("expiring lease; waiting for permit pool") 304 m.revokePermitPool.Acquire() 305 releasePermit = m.revokePermitPool.Release 306 m.logger.Trace("expiring lease; got permit pool") 307 } 308 309 metrics.IncrCounterWithLabels([]string{"expire", "lease_expiration"}, 1, []metrics.Label{{"namespace", ns.ID}}) 310 311 revokeCtx, cancel := context.WithTimeout(ctx, DefaultMaxRequestDuration) 312 revokeCtx = namespace.ContextWithNamespace(revokeCtx, ns) 313 314 go func() { 315 select { 316 case <-ctx.Done(): 317 case <-m.quitCh: 318 cancel() 319 case <-revokeCtx.Done(): 320 } 321 }() 322 323 select { 324 case <-m.quitCh: 325 m.logger.Error("shutting down, not attempting further revocation of lease", "lease_id", leaseID) 326 releasePermit() 327 cancel() 328 return 329 case <-m.quitContext.Done(): 330 m.logger.Error("core context canceled, not attempting further revocation of lease", "lease_id", leaseID) 331 releasePermit() 332 cancel() 333 return 334 default: 335 } 336 337 m.coreStateLock.RLock() 338 err := m.Revoke(revokeCtx, leaseID) 339 m.coreStateLock.RUnlock() 340 releasePermit() 341 cancel() 342 if err == nil { 343 return 344 } 345 346 metrics.IncrCounterWithLabels([]string{"expire", "lease_expiration", "error"}, 1, []metrics.Label{{"namespace", ns.ID}}) 347 348 m.logger.Error("failed to revoke lease", "lease_id", leaseID, "error", err) 349 time.Sleep((1 << attempt) * revokeRetryBase) 350 } 351 m.logger.Error("maximum revoke attempts reached", "lease_id", leaseID) 352} 353 354func getNumExpirationWorkers(c *Core, l log.Logger) int { 355 numWorkers := c.numExpirationWorkers 356 357 workerOverride := os.Getenv(fairshareWorkersOverrideVar) 358 if workerOverride != "" { 359 i, err := strconv.Atoi(workerOverride) 360 if err != nil { 361 l.Warn("vault lease revocation workers override must be an integer", "value", workerOverride) 362 } else if i < 1 || i > 10000 { 363 l.Warn("vault lease revocation workers override out of range", "value", i) 364 } else { 365 numWorkers = i 366 } 367 } 368 369 return numWorkers 370} 371 372// NewExpirationManager creates a new ExpirationManager that is backed 373// using a given view, and uses the provided router for revocation. 374func NewExpirationManager(c *Core, view *BarrierView, e ExpireLeaseStrategy, logger log.Logger) *ExpirationManager { 375 var permitPool *physical.PermitPool 376 if os.Getenv("VAULT_16_REVOKE_PERMITPOOL") != "" { 377 permitPoolSize := 50 378 permitPoolSizeRaw, err := strconv.Atoi(os.Getenv("VAULT_16_REVOKE_PERMITPOOL")) 379 if err == nil && permitPoolSizeRaw > 0 { 380 permitPoolSize = permitPoolSizeRaw 381 } 382 383 permitPool = physical.NewPermitPool(permitPoolSize) 384 385 } 386 387 jobManager := fairshare.NewJobManager("expire", getNumExpirationWorkers(c, logger), logger.Named("job-manager"), c.metricSink) 388 jobManager.Start() 389 390 exp := &ExpirationManager{ 391 core: c, 392 router: c.router, 393 idView: view.SubView(leaseViewPrefix), 394 tokenView: view.SubView(tokenViewPrefix), 395 tokenStore: c.tokenStore, 396 logger: logger, 397 pending: sync.Map{}, 398 nonexpiring: sync.Map{}, 399 leaseCount: 0, 400 tidyLock: new(int32), 401 402 lockPerLease: sync.Map{}, 403 404 uniquePolicies: make(map[string][]string), 405 emptyUniquePolicies: time.NewTicker(7 * 24 * time.Hour), 406 407 // new instances of the expiration manager will go immediately into 408 // restore mode 409 restoreMode: new(int32), 410 restoreLocks: locksutil.CreateLocks(), 411 quitCh: make(chan struct{}), 412 413 coreStateLock: &c.stateLock, 414 quitContext: c.activeContext, 415 leaseCheckCounter: new(uint32), 416 417 logLeaseExpirations: os.Getenv("VAULT_SKIP_LOGGING_LEASE_EXPIRATIONS") == "", 418 expireFunc: e, 419 revokePermitPool: permitPool, 420 421 jobManager: jobManager, 422 } 423 *exp.restoreMode = 1 424 425 if exp.logger == nil { 426 opts := log.LoggerOptions{Name: "expiration_manager"} 427 exp.logger = log.New(&opts) 428 } 429 430 go exp.uniquePoliciesGc() 431 432 return exp 433} 434 435// setupExpiration is invoked after we've loaded the mount table to 436// initialize the expiration manager 437func (c *Core) setupExpiration(e ExpireLeaseStrategy) error { 438 c.metricsMutex.Lock() 439 defer c.metricsMutex.Unlock() 440 // Create a sub-view 441 view := c.systemBarrierView.SubView(expirationSubPath) 442 443 // Create the manager 444 expLogger := c.baseLogger.Named("expiration") 445 c.AddLogger(expLogger) 446 mgr := NewExpirationManager(c, view, e, expLogger) 447 c.expiration = mgr 448 449 // Link the token store to this 450 c.tokenStore.SetExpirationManager(mgr) 451 452 // Restore the existing state 453 c.logger.Info("restoring leases") 454 errorFunc := func() { 455 c.logger.Error("shutting down") 456 if err := c.Shutdown(); err != nil { 457 c.logger.Error("error shutting down core", "error", err) 458 } 459 } 460 go c.expiration.Restore(errorFunc) 461 462 quit := c.expiration.quitCh 463 go func() { 464 t := time.NewTimer(24 * time.Hour) 465 for { 466 select { 467 case <-quit: 468 return 469 case <-t.C: 470 c.expiration.attemptIrrevocableLeasesRevoke() 471 t.Reset(24 * time.Hour) 472 } 473 } 474 }() 475 476 return nil 477} 478 479// stopExpiration is used to stop the expiration manager before 480// sealing the Vault. 481func (c *Core) stopExpiration() error { 482 if c.expiration != nil { 483 if err := c.expiration.Stop(); err != nil { 484 return err 485 } 486 c.metricsMutex.Lock() 487 defer c.metricsMutex.Unlock() 488 c.expiration = nil 489 } 490 return nil 491} 492 493// lockLease takes out a lock for a given lease ID 494func (m *ExpirationManager) lockLease(leaseID string) { 495 locksutil.LockForKey(m.restoreLocks, leaseID).Lock() 496} 497 498// unlockLease unlocks a given lease ID 499func (m *ExpirationManager) unlockLease(leaseID string) { 500 locksutil.LockForKey(m.restoreLocks, leaseID).Unlock() 501} 502 503// inRestoreMode returns if we are currently in restore mode 504func (m *ExpirationManager) inRestoreMode() bool { 505 return atomic.LoadInt32(m.restoreMode) == 1 506} 507 508func (m *ExpirationManager) invalidate(key string) { 509 switch { 510 case strings.HasPrefix(key, leaseViewPrefix): 511 leaseID := strings.TrimPrefix(key, leaseViewPrefix) 512 ctx := m.quitContext 513 _, nsID := namespace.SplitIDFromString(leaseID) 514 leaseNS := namespace.RootNamespace 515 var err error 516 if nsID != "" { 517 leaseNS, err = NamespaceByID(ctx, nsID, m.core) 518 if err != nil { 519 m.logger.Error("failed to invalidate lease entry", "error", err) 520 return 521 } 522 } 523 524 le, err := m.loadEntryInternal(namespace.ContextWithNamespace(ctx, leaseNS), leaseID, false, false) 525 if err != nil { 526 m.logger.Error("failed to invalidate lease entry", "error", err) 527 return 528 } 529 530 m.pendingLock.Lock() 531 defer m.pendingLock.Unlock() 532 info, ok := m.pending.Load(leaseID) 533 switch { 534 case ok: 535 switch { 536 case le == nil: 537 // Handle lease deletion 538 pending := info.(pendingInfo) 539 pending.timer.Stop() 540 m.pending.Delete(leaseID) 541 m.leaseCount-- 542 543 if err := m.core.quotasHandleLeases(ctx, quotas.LeaseActionDeleted, []string{leaseID}); err != nil { 544 m.logger.Error("failed to update quota on lease invalidation", "error", err) 545 return 546 } 547 default: 548 // Update the lease in memory 549 m.updatePendingInternal(le) 550 } 551 default: 552 if le == nil { 553 // There is no entry in the pending map and the invalidation 554 // resulted in a nil entry. Therefore we should clean up the 555 // other maps, and update metrics/quotas if appropriate. 556 m.nonexpiring.Delete(leaseID) 557 558 if _, ok := m.irrevocable.Load(leaseID); ok { 559 m.irrevocable.Delete(leaseID) 560 m.irrevocableLeaseCount-- 561 562 m.leaseCount-- 563 if err := m.core.quotasHandleLeases(ctx, quotas.LeaseActionDeleted, []string{leaseID}); err != nil { 564 m.logger.Error("failed to update quota on lease invalidation", "error", err) 565 return 566 } 567 } 568 return 569 } 570 // Handle lease update (if irrevocable) or creation (if pending) 571 m.updatePendingInternal(le) 572 } 573 } 574} 575 576// Tidy cleans up the dangling storage entries for leases. It scans the storage 577// view to find all the available leases, checks if the token embedded in it is 578// either empty or invalid and in both the cases, it revokes them. It also uses 579// a token cache to avoid multiple lookups of the same token ID. It is normally 580// not required to use the API that invokes this. This is only intended to 581// clean up the corrupt storage due to bugs. 582func (m *ExpirationManager) Tidy(ctx context.Context) error { 583 if m.inRestoreMode() { 584 return errors.New("cannot run tidy while restoring leases") 585 } 586 587 var tidyErrors *multierror.Error 588 589 logger := m.logger.Named("tidy") 590 m.core.AddLogger(logger) 591 592 if !atomic.CompareAndSwapInt32(m.tidyLock, 0, 1) { 593 logger.Warn("tidy operation on leases is already in progress") 594 return nil 595 } 596 597 defer atomic.CompareAndSwapInt32(m.tidyLock, 1, 0) 598 599 logger.Info("beginning tidy operation on leases") 600 defer logger.Info("finished tidy operation on leases") 601 602 // Create a cache to keep track of looked up tokens 603 tokenCache := make(map[string]bool) 604 var countLease, revokedCount, deletedCountInvalidToken, deletedCountEmptyToken int64 605 606 tidyFunc := func(leaseID string) { 607 countLease++ 608 if countLease%500 == 0 { 609 logger.Info("tidying leases", "progress", countLease) 610 } 611 612 le, err := m.loadEntry(ctx, leaseID) 613 if err != nil { 614 tidyErrors = multierror.Append(tidyErrors, fmt.Errorf("failed to load the lease ID %q: %w", leaseID, err)) 615 return 616 } 617 618 if le == nil { 619 tidyErrors = multierror.Append(tidyErrors, fmt.Errorf("nil entry for lease ID %q: %w", leaseID, err)) 620 return 621 } 622 623 var isValid, ok bool 624 revokeLease := false 625 if le.ClientToken == "" { 626 logger.Debug("revoking lease which has an empty token", "lease_id", leaseID) 627 revokeLease = true 628 deletedCountEmptyToken++ 629 goto REVOKE_CHECK 630 } 631 632 isValid, ok = tokenCache[le.ClientToken] 633 if !ok { 634 lock := locksutil.LockForKey(m.tokenStore.tokenLocks, le.ClientToken) 635 lock.RLock() 636 te, err := m.tokenStore.lookupInternal(ctx, le.ClientToken, false, true) 637 lock.RUnlock() 638 639 if err != nil { 640 tidyErrors = multierror.Append(tidyErrors, fmt.Errorf("failed to lookup token: %w", err)) 641 return 642 } 643 644 if te == nil { 645 logger.Debug("revoking lease which holds an invalid token", "lease_id", leaseID) 646 revokeLease = true 647 deletedCountInvalidToken++ 648 tokenCache[le.ClientToken] = false 649 } else { 650 tokenCache[le.ClientToken] = true 651 } 652 653 goto REVOKE_CHECK 654 } else { 655 if isValid { 656 return 657 } 658 659 logger.Debug("revoking lease which contains an invalid token", "lease_id", leaseID) 660 revokeLease = true 661 deletedCountInvalidToken++ 662 goto REVOKE_CHECK 663 } 664 665 REVOKE_CHECK: 666 if revokeLease { 667 // Force the revocation and skip going through the token store 668 // again 669 670 leaseLock := m.lockForLeaseID(leaseID) 671 leaseLock.Lock() 672 err = m.revokeCommon(ctx, leaseID, true, true) 673 leaseLock.Unlock() 674 if err != nil { 675 tidyErrors = multierror.Append(tidyErrors, fmt.Errorf("failed to revoke an invalid lease with ID %q: %w", leaseID, err)) 676 return 677 } 678 revokedCount++ 679 } 680 } 681 682 ns, err := namespace.FromContext(ctx) 683 if err != nil { 684 return err 685 } 686 leaseView := m.leaseView(ns) 687 if err := logical.ScanView(m.quitContext, leaseView, tidyFunc); err != nil { 688 return err 689 } 690 691 logger.Info("number of leases scanned", "count", countLease) 692 logger.Info("number of leases which had empty tokens", "count", deletedCountEmptyToken) 693 logger.Info("number of leases which had invalid tokens", "count", deletedCountInvalidToken) 694 logger.Info("number of leases successfully revoked", "count", revokedCount) 695 696 return tidyErrors.ErrorOrNil() 697} 698 699// Restore is used to recover the lease states when starting. 700// This is used after starting the vault. 701func (m *ExpirationManager) Restore(errorFunc func()) (retErr error) { 702 defer func() { 703 // Turn off restore mode. We can do this safely without the lock because 704 // if restore mode finished successfully, restore mode was already 705 // disabled with the lock. In an error state, this will allow the 706 // Stop() function to shut everything down. 707 atomic.StoreInt32(m.restoreMode, 0) 708 709 switch { 710 case retErr == nil: 711 case strings.Contains(retErr.Error(), context.Canceled.Error()): 712 // Don't run error func because we lost leadership 713 m.logger.Warn("context canceled while restoring leases, stopping lease loading") 714 retErr = nil 715 case errwrap.Contains(retErr, ErrBarrierSealed.Error()): 716 // Don't run error func because we're likely already shutting down 717 m.logger.Warn("barrier sealed while restoring leases, stopping lease loading") 718 retErr = nil 719 default: 720 m.logger.Error("error restoring leases", "error", retErr) 721 if errorFunc != nil { 722 errorFunc() 723 } 724 } 725 }() 726 727 // Accumulate existing leases 728 m.logger.Debug("collecting leases") 729 existing, leaseCount, err := m.collectLeases() 730 if err != nil { 731 return err 732 } 733 m.logger.Debug("leases collected", "num_existing", leaseCount) 734 735 // Make the channels used for the worker pool 736 type lease struct { 737 namespace *namespace.Namespace 738 id string 739 } 740 broker := make(chan *lease) 741 quit := make(chan bool) 742 // Buffer these channels to prevent deadlocks 743 errs := make(chan error, len(existing)) 744 result := make(chan struct{}, len(existing)) 745 746 // Use a wait group 747 wg := &sync.WaitGroup{} 748 749 // Create 64 workers to distribute work to 750 for i := 0; i < consts.ExpirationRestoreWorkerCount; i++ { 751 wg.Add(1) 752 go func() { 753 defer wg.Done() 754 755 for { 756 select { 757 case lease, ok := <-broker: 758 // broker has been closed, we are done 759 if !ok { 760 return 761 } 762 763 ctx := namespace.ContextWithNamespace(m.quitContext, lease.namespace) 764 err := m.processRestore(ctx, lease.id) 765 if err != nil { 766 errs <- err 767 continue 768 } 769 770 // Send message that lease is done 771 result <- struct{}{} 772 773 // quit early 774 case <-quit: 775 return 776 777 case <-m.quitCh: 778 return 779 } 780 } 781 }() 782 } 783 784 // Distribute the collected keys to the workers in a go routine 785 wg.Add(1) 786 go func() { 787 defer wg.Done() 788 i := 0 789 for ns := range existing { 790 for _, leaseID := range existing[ns] { 791 i++ 792 if i%500 == 0 { 793 m.logger.Debug("leases loading", "progress", i) 794 } 795 796 select { 797 case <-quit: 798 return 799 800 case <-m.quitCh: 801 return 802 803 default: 804 broker <- &lease{ 805 namespace: ns, 806 id: leaseID, 807 } 808 } 809 } 810 } 811 812 // Close the broker, causing worker routines to exit 813 close(broker) 814 }() 815 816 // Ensure all keys on the chan are processed 817 for i := 0; i < leaseCount; i++ { 818 select { 819 case err := <-errs: 820 // Close all go routines 821 close(quit) 822 return err 823 824 case <-m.quitCh: 825 close(quit) 826 return nil 827 828 case <-result: 829 } 830 } 831 832 // Let all go routines finish 833 wg.Wait() 834 835 m.restoreModeLock.Lock() 836 atomic.StoreInt32(m.restoreMode, 0) 837 m.restoreLoaded.Range(func(k, v interface{}) bool { 838 m.restoreLoaded.Delete(k) 839 return true 840 }) 841 m.restoreLocks = nil 842 m.restoreModeLock.Unlock() 843 844 m.logger.Info("lease restore complete") 845 return nil 846} 847 848// processRestore takes a lease and restores it in the expiration manager if it has 849// not already been seen 850func (m *ExpirationManager) processRestore(ctx context.Context, leaseID string) error { 851 m.restoreRequestLock.RLock() 852 defer m.restoreRequestLock.RUnlock() 853 854 // Check if the lease has been seen 855 if _, ok := m.restoreLoaded.Load(leaseID); ok { 856 return nil 857 } 858 859 m.lockLease(leaseID) 860 defer m.unlockLease(leaseID) 861 862 // Check again with the lease locked 863 if _, ok := m.restoreLoaded.Load(leaseID); ok { 864 return nil 865 } 866 867 // Load lease and restore expiration timer 868 _, err := m.loadEntryInternal(ctx, leaseID, true, false) 869 if err != nil { 870 return err 871 } 872 873 return nil 874} 875 876// Stop is used to prevent further automatic revocations. 877// This must be called before sealing the view. 878func (m *ExpirationManager) Stop() error { 879 // Stop all the pending expiration timers 880 m.logger.Debug("stop triggered") 881 defer m.logger.Debug("finished stopping") 882 883 m.jobManager.Stop() 884 885 // Do this before stopping pending timers to avoid potential races with 886 // expiring timers 887 close(m.quitCh) 888 889 m.pendingLock.Lock() 890 // Replacing the entire map would cause a race with 891 // a simultaneous WalkTokens, which doesn't hold pendingLock. 892 m.pending.Range(func(key, value interface{}) bool { 893 info := value.(pendingInfo) 894 info.timer.Stop() 895 m.pending.Delete(key) 896 return true 897 }) 898 m.leaseCount = 0 899 m.nonexpiring.Range(func(key, value interface{}) bool { 900 m.nonexpiring.Delete(key) 901 return true 902 }) 903 m.uniquePolicies = make(map[string][]string) 904 m.irrevocable.Range(func(key, _ interface{}) bool { 905 m.irrevocable.Delete(key) 906 return true 907 }) 908 m.irrevocableLeaseCount = 0 909 m.pendingLock.Unlock() 910 911 if m.inRestoreMode() { 912 for { 913 if !m.inRestoreMode() { 914 break 915 } 916 time.Sleep(10 * time.Millisecond) 917 } 918 } 919 920 m.emptyUniquePolicies.Stop() 921 922 return nil 923} 924 925// Revoke is used to revoke a secret named by the given LeaseID 926func (m *ExpirationManager) Revoke(ctx context.Context, leaseID string) error { 927 defer metrics.MeasureSince([]string{"expire", "revoke"}, time.Now()) 928 929 return m.revokeCommon(ctx, leaseID, false, false) 930} 931 932// LazyRevoke is used to queue revocation for a secret named by the given 933// LeaseID. If the lease was not found it returns nil; if the lease was found 934// it triggers a return of a 202. 935func (m *ExpirationManager) LazyRevoke(ctx context.Context, leaseID string) error { 936 defer metrics.MeasureSince([]string{"expire", "lazy-revoke"}, time.Now()) 937 return m.lazyRevokeInternal(ctx, leaseID) 938} 939 940// Mark a lease as expiring immediately 941func (m *ExpirationManager) lazyRevokeInternal(ctx context.Context, leaseID string) error { 942 leaseLock := m.lockForLeaseID(leaseID) 943 leaseLock.Lock() 944 defer leaseLock.Unlock() 945 946 // Load the entry 947 le, err := m.loadEntry(ctx, leaseID) 948 if err != nil { 949 return err 950 } 951 952 // If there is no entry, nothing to revoke 953 if le == nil { 954 return nil 955 } 956 957 le.ExpireTime = time.Now() 958 if err := m.persistEntry(ctx, le); err != nil { 959 return err 960 } 961 m.updatePending(le) 962 963 return nil 964} 965 966// should be run on a schedule. something like once a day, maybe once a week 967func (m *ExpirationManager) attemptIrrevocableLeasesRevoke() { 968 m.irrevocable.Range(func(k, v interface{}) bool { 969 leaseID := k.(string) 970 le := v.(*leaseEntry) 971 972 if le.ExpireTime.Add(time.Hour).Before(time.Now()) { 973 // if we get an error (or no namespace) note it, but continue attempting 974 // to revoke other leases 975 leaseNS, err := m.getNamespaceFromLeaseID(m.core.activeContext, leaseID) 976 if err != nil { 977 m.logger.Debug("could not get lease namespace from ID", "error", err) 978 return true 979 } 980 if leaseNS == nil { 981 m.logger.Debug("could not get lease namespace from ID: nil namespace") 982 return true 983 } 984 985 ctxWithNS := namespace.ContextWithNamespace(m.core.activeContext, leaseNS) 986 ctxWithNSAndTimeout, _ := context.WithTimeout(ctxWithNS, time.Minute) 987 if err := m.revokeCommon(ctxWithNSAndTimeout, leaseID, false, false); err != nil { 988 // on failure, force some delay to mitigate resource spike while 989 // this is running. if revocations succeed, we are okay with 990 // the higher resource consumption. 991 time.Sleep(10 * time.Millisecond) 992 } 993 } 994 995 return true 996 }) 997} 998 999// revokeCommon does the heavy lifting. If force is true, we ignore a problem 1000// during revocation and still remove entries/index/lease timers 1001func (m *ExpirationManager) revokeCommon(ctx context.Context, leaseID string, force, skipToken bool) error { 1002 defer metrics.MeasureSince([]string{"expire", "revoke-common"}, time.Now()) 1003 1004 if !skipToken { 1005 // Acquire lock for this lease 1006 // If skipToken is true, then we're either being (1) called via RevokeByToken, so 1007 // probably the lock is already held, and if we re-acquire we get deadlock, or 1008 // (2) called by tidy, in which case the lock is held by the tidy thread. 1009 leaseLock := m.lockForLeaseID(leaseID) 1010 leaseLock.Lock() 1011 defer leaseLock.Unlock() 1012 } 1013 1014 // Load the entry 1015 le, err := m.loadEntry(ctx, leaseID) 1016 if err != nil { 1017 return err 1018 } 1019 1020 // If there is no entry, nothing to revoke 1021 if le == nil { 1022 return nil 1023 } 1024 1025 // Revoke the entry 1026 if !skipToken || le.Auth == nil { 1027 if err := m.revokeEntry(ctx, le); err != nil { 1028 if !force { 1029 return err 1030 } 1031 1032 if m.logger.IsWarn() { 1033 m.logger.Warn("revocation from the backend failed, but in force mode so ignoring", "error", err) 1034 } 1035 } 1036 } 1037 1038 // Delete the entry 1039 if err := m.deleteEntry(ctx, le); err != nil { 1040 return err 1041 } 1042 1043 // Lease has been removed, also remove the in-memory lock. 1044 m.deleteLockForLease(leaseID) 1045 1046 // Delete the secondary index, but only if it's a leased secret (not auth) 1047 if le.Secret != nil { 1048 var indexToken string 1049 // Maintain secondary index by token, except for orphan batch tokens 1050 switch le.ClientTokenType { 1051 case logical.TokenTypeBatch: 1052 te, err := m.tokenStore.lookupBatchTokenInternal(ctx, le.ClientToken) 1053 if err != nil { 1054 return err 1055 } 1056 // If it's a non-orphan batch token, assign the secondary index to its 1057 // parent 1058 indexToken = te.Parent 1059 default: 1060 indexToken = le.ClientToken 1061 } 1062 if indexToken != "" { 1063 if err := m.removeIndexByToken(ctx, le, indexToken); err != nil { 1064 return err 1065 } 1066 } 1067 } 1068 1069 // Clear the expiration handler 1070 m.pendingLock.Lock() 1071 m.removeFromPending(ctx, leaseID, true) 1072 m.nonexpiring.Delete(leaseID) 1073 1074 if _, ok := m.irrevocable.Load(le.LeaseID); ok { 1075 m.irrevocable.Delete(leaseID) 1076 m.irrevocableLeaseCount-- 1077 } 1078 m.pendingLock.Unlock() 1079 1080 if m.logger.IsInfo() && !skipToken && m.logLeaseExpirations { 1081 m.logger.Info("revoked lease", "lease_id", leaseID) 1082 } 1083 if m.logger.IsWarn() && !skipToken && le.isIncorrectlyNonExpiring() { 1084 var accessor string 1085 if le.Auth != nil { 1086 accessor = le.Auth.Accessor 1087 } 1088 m.logger.Warn("finished revoking incorrectly non-expiring lease", "leaseID", le.LeaseID, "accessor", accessor) 1089 } 1090 return nil 1091} 1092 1093// RevokeForce works similarly to RevokePrefix but continues in the case of a 1094// revocation error; this is mostly meant for recovery operations 1095func (m *ExpirationManager) RevokeForce(ctx context.Context, prefix string) error { 1096 defer metrics.MeasureSince([]string{"expire", "revoke-force"}, time.Now()) 1097 1098 return m.revokePrefixCommon(ctx, prefix, true, true) 1099} 1100 1101// RevokePrefix is used to revoke all secrets with a given prefix. 1102// The prefix maps to that of the mount table to make this simpler 1103// to reason about. 1104func (m *ExpirationManager) RevokePrefix(ctx context.Context, prefix string, sync bool) error { 1105 defer metrics.MeasureSince([]string{"expire", "revoke-prefix"}, time.Now()) 1106 1107 return m.revokePrefixCommon(ctx, prefix, false, sync) 1108} 1109 1110// RevokeByToken is used to revoke all the secrets issued with a given token. 1111// This is done by using the secondary index. It also removes the lease entry 1112// for the token itself. As a result it should *ONLY* ever be called from the 1113// token store's revokeInternal function. 1114// (NB: it's called by token tidy as well.) 1115func (m *ExpirationManager) RevokeByToken(ctx context.Context, te *logical.TokenEntry) error { 1116 defer metrics.MeasureSince([]string{"expire", "revoke-by-token"}, time.Now()) 1117 tokenNS, err := NamespaceByID(ctx, te.NamespaceID, m.core) 1118 if err != nil { 1119 return err 1120 } 1121 if tokenNS == nil { 1122 return namespace.ErrNoNamespace 1123 } 1124 1125 tokenCtx := namespace.ContextWithNamespace(ctx, tokenNS) 1126 // Lookup the leases 1127 existing, err := m.lookupLeasesByToken(tokenCtx, te) 1128 if err != nil { 1129 return fmt.Errorf("failed to scan for leases: %w", err) 1130 } 1131 1132 // Revoke all the keys by marking them expired 1133 for _, leaseID := range existing { 1134 err := m.lazyRevokeInternal(ctx, leaseID) 1135 if err != nil { 1136 return err 1137 } 1138 } 1139 1140 // te.Path should never be empty, but we check just in case 1141 if te.Path != "" { 1142 saltCtx := namespace.ContextWithNamespace(ctx, tokenNS) 1143 saltedID, err := m.tokenStore.SaltID(saltCtx, te.ID) 1144 if err != nil { 1145 return err 1146 } 1147 tokenLeaseID := path.Join(te.Path, saltedID) 1148 1149 if tokenNS.ID != namespace.RootNamespaceID { 1150 tokenLeaseID = fmt.Sprintf("%s.%s", tokenLeaseID, tokenNS.ID) 1151 } 1152 1153 // We want to skip the revokeEntry call as that will call back into 1154 // revocation logic in the token store, which is what is running this 1155 // function in the first place -- it'd be a deadlock loop. Since the only 1156 // place that this function is called is revokeSalted in the token store, 1157 // we're already revoking the token, so we just want to clean up the lease. 1158 // This avoids spurious revocations later in the log when the timer runs 1159 // out, and eases up resource usage. 1160 return m.revokeCommon(ctx, tokenLeaseID, false, true) 1161 } 1162 1163 return nil 1164} 1165 1166func (m *ExpirationManager) revokePrefixCommon(ctx context.Context, prefix string, force, sync bool) error { 1167 if m.inRestoreMode() { 1168 m.restoreRequestLock.Lock() 1169 defer m.restoreRequestLock.Unlock() 1170 } 1171 1172 // Ensure there is a trailing slash; or, if there is no slash, see if there 1173 // is a matching specific ID 1174 if !strings.HasSuffix(prefix, "/") { 1175 le, err := m.loadEntry(ctx, prefix) 1176 if err == nil && le != nil { 1177 if sync { 1178 if err := m.revokeCommon(ctx, prefix, force, false); err != nil { 1179 return fmt.Errorf("failed to revoke %q: %w", prefix, err) 1180 } 1181 return nil 1182 } 1183 return m.LazyRevoke(ctx, prefix) 1184 } 1185 prefix = prefix + "/" 1186 } 1187 1188 // Accumulate existing leases 1189 ns, err := namespace.FromContext(ctx) 1190 if err != nil { 1191 return err 1192 } 1193 view := m.leaseView(ns) 1194 sub := view.SubView(prefix) 1195 existing, err := logical.CollectKeys(ctx, sub) 1196 if err != nil { 1197 return fmt.Errorf("failed to scan for leases: %w", err) 1198 } 1199 1200 // Revoke all the keys 1201 for idx, suffix := range existing { 1202 leaseID := prefix + suffix 1203 // No need to acquire per-lease lock here, one of these two will do it. 1204 switch { 1205 case sync: 1206 if err := m.revokeCommon(ctx, leaseID, force, false); err != nil { 1207 return fmt.Errorf("failed to revoke %q (%d / %d): %w", leaseID, idx+1, len(existing), err) 1208 } 1209 default: 1210 if err := m.LazyRevoke(ctx, leaseID); err != nil { 1211 return fmt.Errorf("failed to revoke %q (%d / %d): %w", leaseID, idx+1, len(existing), err) 1212 } 1213 } 1214 } 1215 1216 return nil 1217} 1218 1219// Renew is used to renew a secret using the given leaseID 1220// and a renew interval. The increment may be ignored. 1221func (m *ExpirationManager) Renew(ctx context.Context, leaseID string, increment time.Duration) (*logical.Response, error) { 1222 defer metrics.MeasureSince([]string{"expire", "renew"}, time.Now()) 1223 1224 // Acquire lock for this lease 1225 leaseLock := m.lockForLeaseID(leaseID) 1226 leaseLock.Lock() 1227 defer leaseLock.Unlock() 1228 1229 // Load the entry 1230 le, err := m.loadEntry(ctx, leaseID) 1231 if err != nil { 1232 return nil, err 1233 } 1234 1235 // Check if the lease is renewable 1236 if _, err := le.renewable(); err != nil { 1237 return nil, err 1238 } 1239 1240 if le.Secret == nil { 1241 if le.Auth != nil { 1242 return logical.ErrorResponse("tokens cannot be renewed through this endpoint"), nil 1243 } 1244 return logical.ErrorResponse("lease does not correspond to a secret"), nil 1245 } 1246 1247 ns, err := namespace.FromContext(ctx) 1248 if err != nil { 1249 return nil, err 1250 } 1251 if ns.ID != le.namespace.ID { 1252 return nil, errors.New("cannot renew a lease across namespaces") 1253 } 1254 1255 sysViewCtx := namespace.ContextWithNamespace(ctx, le.namespace) 1256 sysView := m.router.MatchingSystemView(sysViewCtx, le.Path) 1257 if sysView == nil { 1258 return nil, fmt.Errorf("unable to retrieve system view from router") 1259 } 1260 1261 // Attempt to renew the entry 1262 resp, err := m.renewEntry(ctx, le, increment) 1263 if err != nil { 1264 return nil, err 1265 } 1266 if resp == nil { 1267 return nil, nil 1268 } 1269 if resp.IsError() { 1270 return &logical.Response{ 1271 Data: resp.Data, 1272 }, nil 1273 } 1274 if resp.Secret == nil { 1275 return nil, nil 1276 } 1277 1278 ttl, warnings, err := framework.CalculateTTL(sysView, increment, resp.Secret.TTL, 0, resp.Secret.MaxTTL, 0, le.IssueTime) 1279 if err != nil { 1280 return nil, err 1281 } 1282 for _, warning := range warnings { 1283 resp.AddWarning(warning) 1284 } 1285 resp.Secret.TTL = ttl 1286 1287 // Attach the LeaseID 1288 resp.Secret.LeaseID = leaseID 1289 1290 // Update the lease entry 1291 le.Data = resp.Data 1292 le.Secret = resp.Secret 1293 le.ExpireTime = resp.Secret.ExpirationTime() 1294 le.LastRenewalTime = time.Now() 1295 1296 // If the token it's associated with is a batch token, constrain lease 1297 // times 1298 if le.ClientTokenType == logical.TokenTypeBatch { 1299 te, err := m.tokenStore.Lookup(ctx, le.ClientToken) 1300 if err != nil { 1301 return nil, err 1302 } 1303 if te == nil { 1304 return nil, errors.New("cannot renew lease, no valid associated token") 1305 } 1306 tokenLeaseTimes, err := m.FetchLeaseTimesByToken(ctx, te) 1307 if err != nil { 1308 return nil, err 1309 } 1310 1311 if tokenLeaseTimes == nil { 1312 return nil, errors.New("failed to load batch token expiration time") 1313 } 1314 1315 if le.ExpireTime.After(tokenLeaseTimes.ExpireTime) { 1316 resp.Secret.TTL = tokenLeaseTimes.ExpireTime.Sub(le.LastRenewalTime) 1317 le.ExpireTime = tokenLeaseTimes.ExpireTime 1318 } 1319 } 1320 1321 if err := m.persistEntry(ctx, le); err != nil { 1322 return nil, err 1323 } 1324 1325 // Update the expiration time 1326 m.updatePending(le) 1327 1328 // Return the response 1329 return resp, nil 1330} 1331 1332// RenewToken is used to renew a token which does not need to 1333// invoke a logical backend. 1334func (m *ExpirationManager) RenewToken(ctx context.Context, req *logical.Request, te *logical.TokenEntry, 1335 increment time.Duration) (*logical.Response, error) { 1336 defer metrics.MeasureSince([]string{"expire", "renew-token"}, time.Now()) 1337 1338 tokenNS, err := NamespaceByID(ctx, te.NamespaceID, m.core) 1339 if err != nil { 1340 return nil, err 1341 } 1342 if tokenNS == nil { 1343 return nil, namespace.ErrNoNamespace 1344 } 1345 1346 ns, err := namespace.FromContext(ctx) 1347 if err != nil { 1348 return nil, err 1349 } 1350 if ns.ID != tokenNS.ID { 1351 return nil, errors.New("cannot renew a token across namespaces") 1352 } 1353 1354 // Compute the Lease ID 1355 saltedID, err := m.tokenStore.SaltID(ctx, te.ID) 1356 if err != nil { 1357 return nil, err 1358 } 1359 1360 leaseID := path.Join(te.Path, saltedID) 1361 1362 if ns.ID != namespace.RootNamespaceID { 1363 leaseID = fmt.Sprintf("%s.%s", leaseID, ns.ID) 1364 } 1365 1366 // Acquire lock for this lease 1367 leaseLock := m.lockForLeaseID(leaseID) 1368 leaseLock.Lock() 1369 defer leaseLock.Unlock() 1370 1371 // Load the entry 1372 le, err := m.loadEntry(ctx, leaseID) 1373 if err != nil { 1374 return nil, err 1375 } 1376 if le == nil { 1377 return logical.ErrorResponse("invalid lease ID"), logical.ErrInvalidRequest 1378 } 1379 1380 // Check if the lease is renewable. Note that this also checks for a nil 1381 // lease and errors in that case as well. 1382 if _, err := le.renewable(); err != nil { 1383 return logical.ErrorResponse(err.Error()), logical.ErrInvalidRequest 1384 } 1385 1386 // Attempt to renew the auth entry 1387 resp, err := m.renewAuthEntry(ctx, req, le, increment) 1388 if err != nil { 1389 return nil, err 1390 } 1391 if resp == nil { 1392 return nil, nil 1393 } 1394 if resp.IsError() { 1395 return &logical.Response{ 1396 Data: resp.Data, 1397 }, nil 1398 } 1399 if resp.Auth == nil { 1400 return nil, nil 1401 } 1402 1403 sysViewCtx := namespace.ContextWithNamespace(ctx, le.namespace) 1404 sysView := m.router.MatchingSystemView(sysViewCtx, le.Path) 1405 if sysView == nil { 1406 return nil, fmt.Errorf("unable to retrieve system view from router") 1407 } 1408 1409 ttl, warnings, err := framework.CalculateTTL(sysView, increment, resp.Auth.TTL, resp.Auth.Period, resp.Auth.MaxTTL, resp.Auth.ExplicitMaxTTL, le.IssueTime) 1410 if err != nil { 1411 return nil, err 1412 } 1413 retResp := &logical.Response{} 1414 for _, warning := range warnings { 1415 retResp.AddWarning(warning) 1416 } 1417 resp.Auth.TTL = ttl 1418 1419 // Attach the ClientToken 1420 resp.Auth.ClientToken = te.ID 1421 1422 // Refresh groups 1423 if resp.Auth.EntityID != "" && m.core.identityStore != nil { 1424 mountAccessor := "" 1425 if resp.Auth.Alias != nil { 1426 mountAccessor = resp.Auth.Alias.MountAccessor 1427 } 1428 validAliases, err := m.core.identityStore.refreshExternalGroupMembershipsByEntityID(ctx, resp.Auth.EntityID, resp.Auth.GroupAliases, mountAccessor) 1429 if err != nil { 1430 return nil, err 1431 } 1432 resp.Auth.GroupAliases = validAliases 1433 } 1434 1435 // Update the lease entry 1436 le.Auth = resp.Auth 1437 le.ExpireTime = resp.Auth.ExpirationTime() 1438 le.LastRenewalTime = time.Now() 1439 1440 if err := m.persistEntry(ctx, le); err != nil { 1441 return nil, err 1442 } 1443 m.updatePending(le) 1444 1445 retResp.Auth = resp.Auth 1446 return retResp, nil 1447} 1448 1449// Register is used to take a request and response with an associated 1450// lease. The secret gets assigned a LeaseID and the management of 1451// of lease is assumed by the expiration manager. 1452func (m *ExpirationManager) Register(ctx context.Context, req *logical.Request, resp *logical.Response) (id string, retErr error) { 1453 defer metrics.MeasureSince([]string{"expire", "register"}, time.Now()) 1454 1455 te := req.TokenEntry() 1456 if te == nil { 1457 return "", fmt.Errorf("cannot register a lease with an empty client token") 1458 } 1459 1460 // Ignore if there is no leased secret 1461 if resp == nil || resp.Secret == nil { 1462 return "", nil 1463 } 1464 1465 // Validate the secret 1466 if err := resp.Secret.Validate(); err != nil { 1467 return "", err 1468 } 1469 1470 // Create a lease entry 1471 leaseRand, err := base62.Random(TokenLength) 1472 if err != nil { 1473 return "", err 1474 } 1475 1476 ns, err := namespace.FromContext(ctx) 1477 if err != nil { 1478 return "", err 1479 } 1480 1481 leaseID := path.Join(req.Path, leaseRand) 1482 1483 if ns.ID != namespace.RootNamespaceID { 1484 leaseID = fmt.Sprintf("%s.%s", leaseID, ns.ID) 1485 } 1486 1487 le := &leaseEntry{ 1488 LeaseID: leaseID, 1489 ClientToken: req.ClientToken, 1490 ClientTokenType: te.Type, 1491 Path: req.Path, 1492 Data: resp.Data, 1493 Secret: resp.Secret, 1494 IssueTime: time.Now(), 1495 ExpireTime: resp.Secret.ExpirationTime(), 1496 namespace: ns, 1497 Version: 1, 1498 } 1499 1500 var indexToken string 1501 // Maintain secondary index by token, except for orphan batch tokens 1502 switch { 1503 case te.Type != logical.TokenTypeBatch: 1504 indexToken = le.ClientToken 1505 case te.Parent != "": 1506 // If it's a non-orphan batch token, assign the secondary index to its 1507 // parent 1508 indexToken = te.Parent 1509 } 1510 1511 defer func() { 1512 // If there is an error we want to rollback as much as possible (note 1513 // that errors here are ignored to do as much cleanup as we can). We 1514 // want to revoke a generated secret (since an error means we may not 1515 // be successfully tracking it), remove indexes, and delete the entry. 1516 if retErr != nil { 1517 revokeCtx := namespace.ContextWithNamespace(m.quitContext, ns) 1518 revResp, err := m.router.Route(revokeCtx, logical.RevokeRequest(req.Path, resp.Secret, resp.Data)) 1519 if err != nil { 1520 retErr = multierror.Append(retErr, fmt.Errorf("an additional internal error was encountered revoking the newly-generated secret: %w", err)) 1521 } else if revResp != nil && revResp.IsError() { 1522 retErr = multierror.Append(retErr, fmt.Errorf("an additional error was encountered revoking the newly-generated secret: %w", revResp.Error())) 1523 } 1524 1525 if err := m.deleteEntry(ctx, le); err != nil { 1526 retErr = multierror.Append(retErr, fmt.Errorf("an additional error was encountered deleting any lease associated with the newly-generated secret: %w", err)) 1527 } 1528 1529 if err := m.removeIndexByToken(ctx, le, indexToken); err != nil { 1530 retErr = multierror.Append(retErr, fmt.Errorf("an additional error was encountered removing lease indexes associated with the newly-generated secret: %w", err)) 1531 } 1532 1533 m.deleteLockForLease(leaseID) 1534 } 1535 }() 1536 1537 // If the token is a batch token, we want to constrain the maximum lifetime 1538 // by the token's lifetime 1539 if te.Type == logical.TokenTypeBatch { 1540 tokenLeaseTimes, err := m.FetchLeaseTimesByToken(ctx, te) 1541 if err != nil { 1542 return "", err 1543 } 1544 if tokenLeaseTimes == nil { 1545 return "", errors.New("failed to load batch token expiration time") 1546 } 1547 if le.ExpireTime.After(tokenLeaseTimes.ExpireTime) { 1548 le.ExpireTime = tokenLeaseTimes.ExpireTime 1549 } 1550 } 1551 1552 // Acquire the lock here so persistEntry and updatePending are atomic, 1553 // although it is *very unlikely* that anybody could grab the lease ID 1554 // before this function returns. (They could find it in an index, or 1555 // find it in a list.) 1556 leaseLock := m.lockForLeaseID(leaseID) 1557 leaseLock.Lock() 1558 defer leaseLock.Unlock() 1559 1560 // Encode the entry 1561 if err := m.persistEntry(ctx, le); err != nil { 1562 return "", err 1563 } 1564 1565 if indexToken != "" { 1566 if err := m.createIndexByToken(ctx, le, indexToken); err != nil { 1567 return "", err 1568 } 1569 } 1570 1571 // Setup revocation timer if there is a lease 1572 m.updatePending(le) 1573 1574 // We round here because the clock will have already started 1575 // ticking, so we'll end up always returning 299 instead of 300 or 1576 // 26399 instead of 26400, say, even if it's just a few 1577 // microseconds. This provides a nicer UX. 1578 resp.Secret.TTL = le.ExpireTime.Sub(time.Now()).Round(time.Second) 1579 1580 // Done 1581 return le.LeaseID, nil 1582} 1583 1584// RegisterAuth is used to take an Auth response with an associated lease. 1585// The token does not get a LeaseID, but the lease management is handled by 1586// the expiration manager. 1587func (m *ExpirationManager) RegisterAuth(ctx context.Context, te *logical.TokenEntry, auth *logical.Auth) error { 1588 defer metrics.MeasureSince([]string{"expire", "register-auth"}, time.Now()) 1589 1590 // Triggers failure of RegisterAuth. This should only be set and triggered 1591 // by tests to simulate partial failure during a token creation request. 1592 if m.testRegisterAuthFailure.Load() { 1593 return fmt.Errorf("failing explicitly on RegisterAuth") 1594 } 1595 1596 authExpirationTime := auth.ExpirationTime() 1597 1598 if te.TTL == 0 && authExpirationTime.IsZero() && (len(te.Policies) != 1 || te.Policies[0] != "root") { 1599 return errors.New("refusing to register a lease for a non-root token with no TTL") 1600 } 1601 1602 if te.Type == logical.TokenTypeBatch { 1603 return errors.New("cannot register a lease for a batch token") 1604 } 1605 1606 if auth.ClientToken == "" { 1607 return errors.New("cannot register an auth lease with an empty token") 1608 } 1609 1610 if strings.Contains(te.Path, "..") { 1611 return consts.ErrPathContainsParentReferences 1612 } 1613 1614 tokenNS, err := NamespaceByID(ctx, te.NamespaceID, m.core) 1615 if err != nil { 1616 return err 1617 } 1618 if tokenNS == nil { 1619 return namespace.ErrNoNamespace 1620 } 1621 1622 saltCtx := namespace.ContextWithNamespace(ctx, tokenNS) 1623 saltedID, err := m.tokenStore.SaltID(saltCtx, auth.ClientToken) 1624 if err != nil { 1625 return err 1626 } 1627 1628 leaseID := path.Join(te.Path, saltedID) 1629 if tokenNS.ID != namespace.RootNamespaceID { 1630 leaseID = fmt.Sprintf("%s.%s", leaseID, tokenNS.ID) 1631 } 1632 1633 // Create a lease entry 1634 le := leaseEntry{ 1635 LeaseID: leaseID, 1636 ClientToken: auth.ClientToken, 1637 Auth: auth, 1638 Path: te.Path, 1639 IssueTime: time.Now(), 1640 ExpireTime: authExpirationTime, 1641 namespace: tokenNS, 1642 Version: 1, 1643 } 1644 1645 leaseLock := m.lockForLeaseID(leaseID) 1646 leaseLock.Lock() 1647 defer leaseLock.Unlock() 1648 1649 // Encode the entry 1650 if err := m.persistEntry(ctx, &le); err != nil { 1651 return err 1652 } 1653 1654 // Setup revocation timer 1655 m.updatePending(&le) 1656 1657 return nil 1658} 1659 1660// FetchLeaseTimesByToken is a helper function to use token values to compute 1661// the leaseID, rather than pushing that logic back into the token store. 1662// As a special case, for a batch token it simply returns the information 1663// encoded on it. 1664func (m *ExpirationManager) FetchLeaseTimesByToken(ctx context.Context, te *logical.TokenEntry) (*leaseEntry, error) { 1665 defer metrics.MeasureSince([]string{"expire", "fetch-lease-times-by-token"}, time.Now()) 1666 1667 if te == nil { 1668 return nil, errors.New("cannot fetch lease times for nil token") 1669 } 1670 1671 if te.Type == logical.TokenTypeBatch { 1672 issueTime := time.Unix(te.CreationTime, 0) 1673 return &leaseEntry{ 1674 IssueTime: issueTime, 1675 ExpireTime: issueTime.Add(te.TTL), 1676 ClientTokenType: logical.TokenTypeBatch, 1677 }, nil 1678 } 1679 1680 tokenNS, err := NamespaceByID(ctx, te.NamespaceID, m.core) 1681 if err != nil { 1682 return nil, err 1683 } 1684 if tokenNS == nil { 1685 return nil, namespace.ErrNoNamespace 1686 } 1687 1688 saltCtx := namespace.ContextWithNamespace(ctx, tokenNS) 1689 saltedID, err := m.tokenStore.SaltID(saltCtx, te.ID) 1690 if err != nil { 1691 return nil, err 1692 } 1693 1694 leaseID := path.Join(te.Path, saltedID) 1695 1696 if tokenNS.ID != namespace.RootNamespaceID { 1697 leaseID = fmt.Sprintf("%s.%s", leaseID, tokenNS.ID) 1698 } 1699 1700 return m.FetchLeaseTimes(ctx, leaseID) 1701} 1702 1703// FetchLeaseTimes is used to fetch the issue time, expiration time, and last 1704// renewed time of a lease entry. It returns a leaseEntry itself, but with only 1705// those values copied over. 1706func (m *ExpirationManager) FetchLeaseTimes(ctx context.Context, leaseID string) (*leaseEntry, error) { 1707 defer metrics.MeasureSince([]string{"expire", "fetch-lease-times"}, time.Now()) 1708 1709 info, ok := m.pending.Load(leaseID) 1710 if ok && info.(pendingInfo).cachedLeaseInfo != nil { 1711 return m.leaseTimesForExport(info.(pendingInfo).cachedLeaseInfo), nil 1712 } 1713 1714 info, ok = m.irrevocable.Load(leaseID) 1715 if ok && info.(*leaseEntry) != nil { 1716 return m.leaseTimesForExport(info.(*leaseEntry)), nil 1717 } 1718 1719 // Load the entry 1720 le, err := m.loadEntryInternal(ctx, leaseID, true, false) 1721 if err != nil { 1722 return nil, err 1723 } 1724 if le == nil { 1725 return nil, nil 1726 } 1727 1728 return m.leaseTimesForExport(le), nil 1729} 1730 1731// Returns lease times for outside callers based on the full leaseEntry passed in 1732func (m *ExpirationManager) leaseTimesForExport(le *leaseEntry) *leaseEntry { 1733 ret := &leaseEntry{ 1734 IssueTime: le.IssueTime, 1735 ExpireTime: le.ExpireTime, 1736 LastRenewalTime: le.LastRenewalTime, 1737 } 1738 if le.Secret != nil { 1739 ret.Secret = &logical.Secret{} 1740 ret.Secret.Renewable = le.Secret.Renewable 1741 ret.Secret.TTL = le.Secret.TTL 1742 } 1743 if le.Auth != nil { 1744 ret.Auth = &logical.Auth{} 1745 ret.Auth.Renewable = le.Auth.Renewable 1746 ret.Auth.TTL = le.Auth.TTL 1747 } 1748 1749 return ret 1750} 1751 1752// Restricts lease entry stored in pendingInfo to a low-cost subset of the 1753// information. 1754func (m *ExpirationManager) inMemoryLeaseInfo(le *leaseEntry) *leaseEntry { 1755 ret := m.leaseTimesForExport(le) 1756 // Need to index: 1757 // namespace -- derived from lease ID 1758 // policies -- stored in Auth object 1759 // auth method -- derived from lease.Path 1760 if le.Auth != nil { 1761 // Ensure that list of policies is not copied more than 1762 // once. This method is called with pendingLock held. 1763 1764 // We could use hashstructure here to generate a key, but that 1765 // seems like it would be substantially slower? 1766 key := strings.Join(le.Auth.Policies, "\n") 1767 uniq, ok := m.uniquePolicies[key] 1768 if ok { 1769 ret.Auth.Policies = uniq 1770 } else { 1771 m.uniquePolicies[key] = le.Auth.Policies 1772 ret.Auth.Policies = le.Auth.Policies 1773 } 1774 ret.Path = le.Path 1775 } 1776 if le.isIrrevocable() { 1777 ret.RevokeErr = le.RevokeErr 1778 } 1779 return ret 1780} 1781 1782func (m *ExpirationManager) uniquePoliciesGc() { 1783 for { 1784 <-m.emptyUniquePolicies.C 1785 1786 // If the maximum lease is a month, and we blow away the unique 1787 // policy cache every week, the pessimal case is 4x larger space 1788 // utilization than keeping the cache indefinitely. 1789 m.pendingLock.Lock() 1790 m.uniquePolicies = make(map[string][]string) 1791 m.pendingLock.Unlock() 1792 } 1793} 1794 1795// Placing a lock in pendingMap means that we need to work very hard on reload 1796// to only create one lock. Instead, we'll create locks on-demand in an atomic fashion. 1797// 1798// Acquiring a lock from a leaseEntry is a bad idea because it could change 1799// between loading and acquiring the lock. So we only provide an ID-based map, and the 1800// locking discipline should be: 1801// 1. Lock lease 1802// 2. Load, or attempt to load, leaseEntry 1803// 3. Modify leaseEntry and pendingMap (atomic wrt operations on this lease) 1804// 4. Unlock lease 1805// 1806// The lock must be removed from the map when the lease is deleted, or is 1807// found to not exist in storage. loadEntry does this whenever it returns 1808// nil, but we should also do it in revokeCommon(). 1809func (m *ExpirationManager) lockForLeaseID(id string) *sync.Mutex { 1810 mutex := &sync.Mutex{} 1811 lock, _ := m.lockPerLease.LoadOrStore(id, mutex) 1812 return lock.(*sync.Mutex) 1813} 1814 1815func (m *ExpirationManager) deleteLockForLease(id string) { 1816 m.lockPerLease.Delete(id) 1817} 1818 1819// updatePending is used to update a pending invocation for a lease 1820func (m *ExpirationManager) updatePending(le *leaseEntry) { 1821 m.pendingLock.Lock() 1822 defer m.pendingLock.Unlock() 1823 1824 m.updatePendingInternal(le) 1825} 1826 1827// updatePendingInternal is the locked version of updatePending; do not call 1828// this without a write lock on m.pending 1829func (m *ExpirationManager) updatePendingInternal(le *leaseEntry) { 1830 // Check for an existing timer 1831 info, leaseInPending := m.pending.Load(le.LeaseID) 1832 1833 var pending pendingInfo 1834 1835 if le.ExpireTime.IsZero() && le.nonexpiringToken() { 1836 // Store this in the nonexpiring map instead of pending. 1837 // There does not appear to be any cases where a token that had 1838 // a nonzero can be can be assigned a zero TTL, but that can be 1839 // handled by the next check 1840 pending.cachedLeaseInfo = m.inMemoryLeaseInfo(le) 1841 m.nonexpiring.Store(le.LeaseID, pending) 1842 1843 // if the timer happened to exist, stop the time and delete it from the 1844 // pending timers. 1845 if leaseInPending { 1846 info.(pendingInfo).timer.Stop() 1847 m.pending.Delete(le.LeaseID) 1848 m.leaseCount-- 1849 if err := m.core.quotasHandleLeases(m.quitContext, quotas.LeaseActionDeleted, []string{le.LeaseID}); err != nil { 1850 m.logger.Error("failed to update quota on lease deletion", "error", err) 1851 return 1852 } 1853 } 1854 return 1855 } 1856 1857 leaseTotal := le.ExpireTime.Sub(time.Now()) 1858 leaseCreated := false 1859 1860 if le.isIrrevocable() { 1861 // It's possible this function is being called to update the in-memory state 1862 // for a lease from pending to irrevocable (we don't support the opposite). 1863 // If this is the case, we need to know if the lease was previously counted 1864 // so that we can maintain correct metric and quota lease counts. 1865 _, leaseInIrrevocable := m.irrevocable.Load(le.LeaseID) 1866 if !(leaseInPending || leaseInIrrevocable) { 1867 leaseCreated = true 1868 } 1869 1870 m.removeFromPending(m.quitContext, le.LeaseID, false) 1871 m.irrevocable.Store(le.LeaseID, m.inMemoryLeaseInfo(le)) 1872 1873 // Increment count if the lease was not present in the irrevocable map 1874 // prior to being added to it above 1875 if !leaseInIrrevocable { 1876 m.irrevocableLeaseCount++ 1877 } 1878 } else { 1879 // Create entry if it does not exist or reset if it does 1880 if leaseInPending { 1881 pending = info.(pendingInfo) 1882 pending.timer.Reset(leaseTotal) 1883 // No change to lease count in this case 1884 } else { 1885 leaseID, namespace := le.LeaseID, le.namespace 1886 // Extend the timer by the lease total 1887 timer := time.AfterFunc(leaseTotal, func() { 1888 m.expireFunc(m.quitContext, m, leaseID, namespace) 1889 }) 1890 pending = pendingInfo{ 1891 timer: timer, 1892 } 1893 1894 leaseCreated = true 1895 } 1896 1897 pending.cachedLeaseInfo = m.inMemoryLeaseInfo(le) 1898 m.pending.Store(le.LeaseID, pending) 1899 } 1900 1901 if leaseCreated { 1902 m.leaseCount++ 1903 if err := m.core.quotasHandleLeases(m.quitContext, quotas.LeaseActionCreated, []string{le.LeaseID}); err != nil { 1904 m.logger.Error("failed to update quota on lease creation", "error", err) 1905 return 1906 } 1907 } 1908} 1909 1910// revokeEntry is used to attempt revocation of an internal entry 1911func (m *ExpirationManager) revokeEntry(ctx context.Context, le *leaseEntry) error { 1912 // Revocation of login tokens is special since we can by-pass the 1913 // backend and directly interact with the token store 1914 if le.Auth != nil { 1915 if le.ClientTokenType == logical.TokenTypeBatch { 1916 return errors.New("batch tokens cannot be revoked") 1917 } 1918 1919 if err := m.tokenStore.revokeTree(ctx, le); err != nil { 1920 return fmt.Errorf("failed to revoke token: %w", err) 1921 } 1922 1923 return nil 1924 } 1925 1926 if le.Secret != nil { 1927 // not sure if this is really valid to have a leaseEntry with a nil Secret 1928 // (if there's a nil Secret, what are you really leasing?), but the tests 1929 // create one, and good to be defensive 1930 le.Secret.IssueTime = le.IssueTime 1931 } 1932 1933 // Make sure we're operating in the right namespace 1934 nsCtx := namespace.ContextWithNamespace(ctx, le.namespace) 1935 1936 // Handle standard revocation via backends 1937 resp, err := m.router.Route(nsCtx, logical.RevokeRequest(le.Path, le.Secret, le.Data)) 1938 if err != nil || (resp != nil && resp.IsError()) { 1939 return fmt.Errorf("failed to revoke entry: resp: %#v err: %w", resp, err) 1940 } 1941 return nil 1942} 1943 1944// renewEntry is used to attempt renew of an internal entry 1945func (m *ExpirationManager) renewEntry(ctx context.Context, le *leaseEntry, increment time.Duration) (*logical.Response, error) { 1946 secret := *le.Secret 1947 secret.IssueTime = le.IssueTime 1948 secret.Increment = increment 1949 secret.LeaseID = "" 1950 1951 // Make sure we're operating in the right namespace 1952 nsCtx := namespace.ContextWithNamespace(ctx, le.namespace) 1953 1954 req := logical.RenewRequest(le.Path, &secret, le.Data) 1955 resp, err := m.router.Route(nsCtx, req) 1956 if err != nil || (resp != nil && resp.IsError()) { 1957 return nil, fmt.Errorf("failed to renew entry: resp: %#v err: %w", resp, err) 1958 } 1959 return resp, nil 1960} 1961 1962// renewAuthEntry is used to attempt renew of an auth entry. Only the token 1963// store should get the actual token ID intact. 1964func (m *ExpirationManager) renewAuthEntry(ctx context.Context, req *logical.Request, le *leaseEntry, increment time.Duration) (*logical.Response, error) { 1965 if le.ClientTokenType == logical.TokenTypeBatch { 1966 return logical.ErrorResponse("batch tokens cannot be renewed"), nil 1967 } 1968 1969 auth := *le.Auth 1970 auth.IssueTime = le.IssueTime 1971 auth.Increment = increment 1972 if strings.HasPrefix(le.Path, "auth/token/") { 1973 auth.ClientToken = le.ClientToken 1974 } else { 1975 auth.ClientToken = "" 1976 } 1977 1978 // Make sure we're operating in the right namespace 1979 nsCtx := namespace.ContextWithNamespace(ctx, le.namespace) 1980 1981 authReq := logical.RenewAuthRequest(le.Path, &auth, nil) 1982 authReq.Connection = req.Connection 1983 resp, err := m.router.Route(nsCtx, authReq) 1984 if err != nil { 1985 return nil, fmt.Errorf("failed to renew entry: %w", err) 1986 } 1987 return resp, nil 1988} 1989 1990// loadEntry is used to read a lease entry 1991func (m *ExpirationManager) loadEntry(ctx context.Context, leaseID string) (*leaseEntry, error) { 1992 // Take out the lease locks after we ensure we are in restore mode 1993 restoreMode := m.inRestoreMode() 1994 if restoreMode { 1995 m.restoreModeLock.RLock() 1996 defer m.restoreModeLock.RUnlock() 1997 1998 restoreMode = m.inRestoreMode() 1999 if restoreMode { 2000 m.lockLease(leaseID) 2001 defer m.unlockLease(leaseID) 2002 } 2003 } 2004 2005 _, nsID := namespace.SplitIDFromString(leaseID) 2006 if nsID != "" { 2007 leaseNS, err := NamespaceByID(ctx, nsID, m.core) 2008 if err != nil { 2009 return nil, err 2010 } 2011 if leaseNS != nil { 2012 ctx = namespace.ContextWithNamespace(ctx, leaseNS) 2013 } 2014 } else { 2015 ctx = namespace.ContextWithNamespace(ctx, namespace.RootNamespace) 2016 } 2017 2018 // If a lease entry is nil, proactively delete the lease lock, in case we 2019 // created one erroneously. 2020 // If there was an error, we don't know whether the lease entry exists or not. 2021 leaseEntry, err := m.loadEntryInternal(ctx, leaseID, restoreMode, true) 2022 if err == nil && leaseEntry == nil { 2023 m.deleteLockForLease(leaseID) 2024 } 2025 return leaseEntry, err 2026 2027} 2028 2029// loadEntryInternal is used when you need to load an entry but also need to 2030// control the lifecycle of the restoreLock 2031func (m *ExpirationManager) loadEntryInternal(ctx context.Context, leaseID string, restoreMode bool, checkRestored bool) (*leaseEntry, error) { 2032 ns, err := namespace.FromContext(ctx) 2033 if err != nil { 2034 return nil, err 2035 } 2036 2037 view := m.leaseView(ns) 2038 out, err := view.Get(ctx, leaseID) 2039 if err != nil { 2040 return nil, fmt.Errorf("failed to read lease entry %s: %w", leaseID, err) 2041 } 2042 if out == nil { 2043 return nil, nil 2044 } 2045 le, err := decodeLeaseEntry(out.Value) 2046 if err != nil { 2047 return nil, fmt.Errorf("failed to decode lease entry %s: %w", leaseID, err) 2048 } 2049 le.namespace = ns 2050 2051 if restoreMode { 2052 if checkRestored { 2053 // If we have already loaded this lease, we don't need to update on 2054 // load. In the case of renewal and revocation, updatePending will be 2055 // done after making the appropriate modifications to the lease. 2056 if _, ok := m.restoreLoaded.Load(leaseID); ok { 2057 return le, nil 2058 } 2059 } 2060 2061 // Update the cache of restored leases, either synchronously or through 2062 // the lazy loaded restore process 2063 m.restoreLoaded.Store(le.LeaseID, struct{}{}) 2064 2065 // Setup revocation timer 2066 m.updatePending(le) 2067 } 2068 return le, nil 2069} 2070 2071// persistEntry is used to persist a lease entry 2072func (m *ExpirationManager) persistEntry(ctx context.Context, le *leaseEntry) error { 2073 // Encode the entry 2074 buf, err := le.encode() 2075 if err != nil { 2076 return fmt.Errorf("failed to encode lease entry: %w", err) 2077 } 2078 2079 // Write out to the view 2080 ent := logical.StorageEntry{ 2081 Key: le.LeaseID, 2082 Value: buf, 2083 } 2084 if le.Auth != nil && len(le.Auth.Policies) == 1 && le.Auth.Policies[0] == "root" { 2085 ent.SealWrap = true 2086 } 2087 2088 view := m.leaseView(le.namespace) 2089 if err := view.Put(ctx, &ent); err != nil { 2090 return fmt.Errorf("failed to persist lease entry: %w", err) 2091 } 2092 return nil 2093} 2094 2095// deleteEntry is used to delete a lease entry 2096func (m *ExpirationManager) deleteEntry(ctx context.Context, le *leaseEntry) error { 2097 view := m.leaseView(le.namespace) 2098 if err := view.Delete(ctx, le.LeaseID); err != nil { 2099 return fmt.Errorf("failed to delete lease entry: %w", err) 2100 } 2101 return nil 2102} 2103 2104// createIndexByToken creates a secondary index from the token to a lease entry 2105func (m *ExpirationManager) createIndexByToken(ctx context.Context, le *leaseEntry, token string) error { 2106 tokenNS := namespace.RootNamespace 2107 saltCtx := namespace.ContextWithNamespace(ctx, namespace.RootNamespace) 2108 _, nsID := namespace.SplitIDFromString(token) 2109 if nsID != "" { 2110 var err error 2111 tokenNS, err = NamespaceByID(ctx, nsID, m.core) 2112 if err != nil { 2113 return err 2114 } 2115 if tokenNS != nil { 2116 saltCtx = namespace.ContextWithNamespace(ctx, tokenNS) 2117 } 2118 } 2119 2120 saltedID, err := m.tokenStore.SaltID(saltCtx, token) 2121 if err != nil { 2122 return err 2123 } 2124 2125 leaseSaltedID, err := m.tokenStore.SaltID(saltCtx, le.LeaseID) 2126 if err != nil { 2127 return err 2128 } 2129 2130 ent := logical.StorageEntry{ 2131 Key: saltedID + "/" + leaseSaltedID, 2132 Value: []byte(le.LeaseID), 2133 } 2134 tokenView := m.tokenIndexView(tokenNS) 2135 if err := tokenView.Put(ctx, &ent); err != nil { 2136 return fmt.Errorf("failed to persist lease index entry: %w", err) 2137 } 2138 return nil 2139} 2140 2141// indexByToken looks up the secondary index from the token to a lease entry 2142func (m *ExpirationManager) indexByToken(ctx context.Context, le *leaseEntry) (*logical.StorageEntry, error) { 2143 tokenNS := namespace.RootNamespace 2144 saltCtx := namespace.ContextWithNamespace(ctx, tokenNS) 2145 _, nsID := namespace.SplitIDFromString(le.ClientToken) 2146 if nsID != "" { 2147 var err error 2148 tokenNS, err = NamespaceByID(ctx, nsID, m.core) 2149 if err != nil { 2150 return nil, err 2151 } 2152 if tokenNS != nil { 2153 saltCtx = namespace.ContextWithNamespace(ctx, tokenNS) 2154 } 2155 } 2156 2157 saltedID, err := m.tokenStore.SaltID(saltCtx, le.ClientToken) 2158 if err != nil { 2159 return nil, err 2160 } 2161 2162 leaseSaltedID, err := m.tokenStore.SaltID(saltCtx, le.LeaseID) 2163 if err != nil { 2164 return nil, err 2165 } 2166 2167 key := saltedID + "/" + leaseSaltedID 2168 tokenView := m.tokenIndexView(tokenNS) 2169 entry, err := tokenView.Get(ctx, key) 2170 if err != nil { 2171 return nil, fmt.Errorf("failed to look up secondary index entry") 2172 } 2173 return entry, nil 2174} 2175 2176// removeIndexByToken removes the secondary index from the token to a lease entry 2177func (m *ExpirationManager) removeIndexByToken(ctx context.Context, le *leaseEntry, token string) error { 2178 tokenNS := namespace.RootNamespace 2179 saltCtx := namespace.ContextWithNamespace(ctx, namespace.RootNamespace) 2180 _, nsID := namespace.SplitIDFromString(token) 2181 if nsID != "" { 2182 var err error 2183 tokenNS, err = NamespaceByID(ctx, nsID, m.core) 2184 if err != nil { 2185 return err 2186 } 2187 if tokenNS != nil { 2188 saltCtx = namespace.ContextWithNamespace(ctx, tokenNS) 2189 } 2190 2191 // Downgrade logic for old-style (V0) namespace leases that had its 2192 // secondary index live in the root namespace. This reverts to the old 2193 // behavior of looking for the secondary index on these leases in the 2194 // root namespace to be cleaned up properly. We set it here because the 2195 // old behavior used the namespace's token store salt for its saltCtx. 2196 if le.Version < 1 { 2197 tokenNS = namespace.RootNamespace 2198 } 2199 } 2200 2201 saltedID, err := m.tokenStore.SaltID(saltCtx, token) 2202 if err != nil { 2203 return err 2204 } 2205 2206 leaseSaltedID, err := m.tokenStore.SaltID(saltCtx, le.LeaseID) 2207 if err != nil { 2208 return err 2209 } 2210 2211 key := saltedID + "/" + leaseSaltedID 2212 tokenView := m.tokenIndexView(tokenNS) 2213 if err := tokenView.Delete(ctx, key); err != nil { 2214 return fmt.Errorf("failed to delete lease index entry: %w", err) 2215 } 2216 return nil 2217} 2218 2219// CreateOrFetchRevocationLeaseByToken is used to create or fetch the matching 2220// leaseID for a particular token. The lease is set to expire immediately after 2221// it's created. 2222func (m *ExpirationManager) CreateOrFetchRevocationLeaseByToken(ctx context.Context, te *logical.TokenEntry) (string, error) { 2223 // Fetch the saltedID of the token and construct the leaseID 2224 tokenNS, err := NamespaceByID(ctx, te.NamespaceID, m.core) 2225 if err != nil { 2226 return "", err 2227 } 2228 if tokenNS == nil { 2229 return "", namespace.ErrNoNamespace 2230 } 2231 2232 saltCtx := namespace.ContextWithNamespace(ctx, tokenNS) 2233 saltedID, err := m.tokenStore.SaltID(saltCtx, te.ID) 2234 if err != nil { 2235 return "", err 2236 } 2237 leaseID := path.Join(te.Path, saltedID) 2238 2239 if tokenNS.ID != namespace.RootNamespaceID { 2240 leaseID = fmt.Sprintf("%s.%s", leaseID, tokenNS.ID) 2241 } 2242 2243 // Load the entry 2244 le, err := m.loadEntry(ctx, leaseID) 2245 if err != nil { 2246 return "", err 2247 } 2248 2249 // If there's no associated leaseEntry for the token, we create one 2250 if le == nil { 2251 2252 // Acquire the lock here so persistEntry and updatePending are atomic, 2253 // although it is *very unlikely* that anybody could grab the lease ID 2254 // before this function returns. (They could find it in an index, or 2255 // find it in a list.) 2256 leaseLock := m.lockForLeaseID(leaseID) 2257 leaseLock.Lock() 2258 defer leaseLock.Unlock() 2259 2260 auth := &logical.Auth{ 2261 ClientToken: te.ID, 2262 LeaseOptions: logical.LeaseOptions{ 2263 TTL: time.Nanosecond, 2264 }, 2265 } 2266 2267 if strings.Contains(te.Path, "..") { 2268 return "", consts.ErrPathContainsParentReferences 2269 } 2270 2271 // Create a lease entry 2272 now := time.Now() 2273 le = &leaseEntry{ 2274 LeaseID: leaseID, 2275 ClientToken: auth.ClientToken, 2276 Auth: auth, 2277 Path: te.Path, 2278 IssueTime: now, 2279 ExpireTime: now.Add(time.Nanosecond), 2280 namespace: tokenNS, 2281 Version: 1, 2282 } 2283 2284 // Encode the entry 2285 if err := m.persistEntry(ctx, le); err != nil { 2286 m.deleteLockForLease(leaseID) 2287 return "", err 2288 } 2289 } 2290 2291 return le.LeaseID, nil 2292} 2293 2294// lookupLeasesByToken is used to lookup all the leaseID's via the tokenID 2295func (m *ExpirationManager) lookupLeasesByToken(ctx context.Context, te *logical.TokenEntry) ([]string, error) { 2296 tokenNS, err := NamespaceByID(ctx, te.NamespaceID, m.core) 2297 if err != nil { 2298 return nil, err 2299 } 2300 if tokenNS == nil { 2301 return nil, namespace.ErrNoNamespace 2302 } 2303 2304 saltCtx := namespace.ContextWithNamespace(ctx, tokenNS) 2305 saltedID, err := m.tokenStore.SaltID(saltCtx, te.ID) 2306 if err != nil { 2307 return nil, err 2308 } 2309 2310 tokenView := m.tokenIndexView(tokenNS) 2311 2312 // Scan via the index for sub-leases 2313 prefix := saltedID + "/" 2314 subKeys, err := tokenView.List(ctx, prefix) 2315 if err != nil { 2316 return nil, fmt.Errorf("failed to list leases: %w", err) 2317 } 2318 2319 // Read each index entry 2320 leaseIDs := make([]string, 0, len(subKeys)) 2321 for _, sub := range subKeys { 2322 out, err := tokenView.Get(ctx, prefix+sub) 2323 if err != nil { 2324 return nil, fmt.Errorf("failed to read lease index: %w", err) 2325 } 2326 if out == nil { 2327 continue 2328 } 2329 leaseIDs = append(leaseIDs, string(out.Value)) 2330 } 2331 2332 // Downgrade logic for old-style (V0) leases entries created by a namespace 2333 // token that lived in the root namespace. 2334 if tokenNS.ID != namespace.RootNamespaceID { 2335 tokenView := m.tokenIndexView(namespace.RootNamespace) 2336 2337 // Scan via the index for sub-leases on the root namespace 2338 prefix := saltedID + "/" 2339 subKeys, err := tokenView.List(ctx, prefix) 2340 if err != nil { 2341 return nil, fmt.Errorf("failed to list leases on root namespace: %w", err) 2342 } 2343 2344 for _, sub := range subKeys { 2345 out, err := tokenView.Get(ctx, prefix+sub) 2346 if err != nil { 2347 return nil, fmt.Errorf("failed to read lease index on root namespace: %w", err) 2348 } 2349 if out == nil { 2350 continue 2351 } 2352 leaseIDs = append(leaseIDs, string(out.Value)) 2353 } 2354 } 2355 2356 return leaseIDs, nil 2357} 2358 2359// emitMetrics is invoked periodically to emit statistics 2360func (m *ExpirationManager) emitMetrics() { 2361 // All updates of these values are with the pendingLock held. 2362 m.pendingLock.RLock() 2363 allLeases := m.leaseCount 2364 irrevocableLeases := m.irrevocableLeaseCount 2365 m.pendingLock.RUnlock() 2366 2367 metrics.SetGauge([]string{"expire", "num_leases"}, float32(allLeases)) 2368 2369 metrics.SetGauge([]string{"expire", "num_irrevocable_leases"}, float32(irrevocableLeases)) 2370 // Check if lease count is greater than the threshold 2371 if allLeases > maxLeaseThreshold { 2372 if atomic.LoadUint32(m.leaseCheckCounter) > 59 { 2373 m.logger.Warn("lease count exceeds warning lease threshold", "have", allLeases, "threshold", maxLeaseThreshold) 2374 atomic.StoreUint32(m.leaseCheckCounter, 0) 2375 } else { 2376 atomic.AddUint32(m.leaseCheckCounter, 1) 2377 } 2378 } 2379} 2380 2381func (m *ExpirationManager) leaseAggregationMetrics(ctx context.Context, consts metricsutil.TelemetryConstConfig) ([]metricsutil.GaugeLabelValues, error) { 2382 expiryTimes := make(map[metricsutil.LeaseExpiryLabel]int) 2383 leaseEpsilon := consts.LeaseMetricsEpsilon 2384 nsLabel := consts.LeaseMetricsNameSpaceLabels 2385 2386 rollingWindow := time.Now().Add(time.Duration(consts.NumLeaseMetricsTimeBuckets) * leaseEpsilon) 2387 2388 err := m.walkLeases(func(entryID string, expireTime time.Time) bool { 2389 select { 2390 // Abort and return empty collection if it's taking too much time, nonblocking check. 2391 case <-ctx.Done(): 2392 return false 2393 default: 2394 if entryID == "" { 2395 return true 2396 } 2397 _, nsID := namespace.SplitIDFromString(entryID) 2398 if nsID == "" { 2399 nsID = "root" // this is what metricsutil.NamespaceLabel does 2400 } 2401 label := metricsutil.ExpiryBucket(expireTime, leaseEpsilon, rollingWindow, nsID, nsLabel) 2402 if label != nil { 2403 expiryTimes[*label] += 1 2404 } 2405 return true 2406 } 2407 }) 2408 if err != nil { 2409 return []metricsutil.GaugeLabelValues{}, suppressRestoreModeError(err) 2410 } 2411 2412 // If collection was cancelled, return an empty array. 2413 select { 2414 case <-ctx.Done(): 2415 return []metricsutil.GaugeLabelValues{}, nil 2416 default: 2417 break 2418 } 2419 2420 flattenedResults := make([]metricsutil.GaugeLabelValues, 0, len(expiryTimes)) 2421 2422 for bucket, count := range expiryTimes { 2423 if nsLabel { 2424 flattenedResults = append(flattenedResults, 2425 metricsutil.GaugeLabelValues{ 2426 Labels: []metrics.Label{{"expiring", bucket.LabelName}, {"namespace", bucket.LabelNS}}, 2427 Value: float32(count), 2428 }) 2429 } else { 2430 flattenedResults = append(flattenedResults, 2431 metricsutil.GaugeLabelValues{ 2432 Labels: []metrics.Label{{"expiring", bucket.LabelName}}, 2433 Value: float32(count), 2434 }) 2435 } 2436 } 2437 return flattenedResults, nil 2438} 2439 2440// Callback function type to walk tokens referenced in the expiration 2441// manager. Don't want to use leaseEntry here because it's an unexported 2442// type (though most likely we would only call this from within the "vault" core package.) 2443type ExpirationWalkFunction = func(leaseID string, auth *logical.Auth, path string) bool 2444 2445var ErrInRestoreMode = errors.New("expiration manager in restore mode") 2446 2447// WalkTokens extracts the Auth structure from leases corresponding to tokens. 2448// Returning false from the walk function terminates the iteration. 2449func (m *ExpirationManager) WalkTokens(walkFn ExpirationWalkFunction) error { 2450 if m.inRestoreMode() { 2451 return ErrInRestoreMode 2452 } 2453 2454 callback := func(key, value interface{}) bool { 2455 p := value.(pendingInfo) 2456 if p.cachedLeaseInfo == nil { 2457 return true 2458 } 2459 lease := p.cachedLeaseInfo 2460 if lease.Auth != nil { 2461 return walkFn(key.(string), lease.Auth, lease.Path) 2462 } 2463 return true 2464 } 2465 2466 m.pending.Range(callback) 2467 m.nonexpiring.Range(callback) 2468 2469 return nil 2470} 2471 2472// leaseWalkFunction can only be used by the core package. 2473type leaseWalkFunction = func(leaseID string, expireTime time.Time) bool 2474 2475func (m *ExpirationManager) walkLeases(walkFn leaseWalkFunction) error { 2476 if m.inRestoreMode() { 2477 return ErrInRestoreMode 2478 } 2479 2480 callback := func(key, value interface{}) bool { 2481 p := value.(pendingInfo) 2482 if p.cachedLeaseInfo == nil { 2483 return true 2484 } 2485 lease := p.cachedLeaseInfo 2486 expireTime := lease.ExpireTime 2487 return walkFn(key.(string), expireTime) 2488 } 2489 2490 m.pending.Range(callback) 2491 m.nonexpiring.Range(callback) 2492 2493 return nil 2494} 2495 2496// must be called with m.pendingLock held 2497// set decrementCounters true to decrement the lease count metric and quota 2498func (m *ExpirationManager) removeFromPending(ctx context.Context, leaseID string, decrementCounters bool) { 2499 if info, ok := m.pending.Load(leaseID); ok { 2500 pending := info.(pendingInfo) 2501 pending.timer.Stop() 2502 m.pending.Delete(leaseID) 2503 if decrementCounters { 2504 m.leaseCount-- 2505 // Log but do not fail; unit tests (and maybe Tidy on production systems) 2506 if err := m.core.quotasHandleLeases(ctx, quotas.LeaseActionDeleted, []string{leaseID}); err != nil { 2507 m.logger.Error("failed to update quota on revocation", "error", err) 2508 } 2509 } 2510 } 2511} 2512 2513// Marks a pending lease as irrevocable. Because the lease is being moved from 2514// pending to irrevocable, no total lease count metrics/quotas updates are needed. 2515// However, irrevocable lease count will need to be incremented 2516// note: must be called with pending lock held 2517func (m *ExpirationManager) markLeaseIrrevocable(ctx context.Context, le *leaseEntry, err error) { 2518 if le == nil { 2519 m.logger.Warn("attempted to mark nil lease as irrevocable") 2520 return 2521 } 2522 if le.isIrrevocable() { 2523 m.logger.Info("attempted to re-mark lease as irrevocable", "original_error", le.RevokeErr, "new_error", err.Error()) 2524 return 2525 } 2526 2527 var errStr string 2528 if err != nil { 2529 errStr = err.Error() 2530 } 2531 if len(errStr) == 0 { 2532 errStr = genericIrrevocableErrorMessage 2533 } 2534 if len(errStr) > maxIrrevocableErrorLength { 2535 errStr = errStr[:maxIrrevocableErrorLength] 2536 } 2537 2538 le.RevokeErr = errStr 2539 m.persistEntry(ctx, le) 2540 2541 m.irrevocable.Store(le.LeaseID, m.inMemoryLeaseInfo(le)) 2542 m.irrevocableLeaseCount++ 2543 m.removeFromPending(ctx, le.LeaseID, false) 2544 m.nonexpiring.Delete(le.LeaseID) 2545} 2546 2547func (m *ExpirationManager) getNamespaceFromLeaseID(ctx context.Context, leaseID string) (*namespace.Namespace, error) { 2548 _, nsID := namespace.SplitIDFromString(leaseID) 2549 2550 // avoid re-declaring leaseNS and err with scope inside the if 2551 leaseNS := namespace.RootNamespace 2552 var err error 2553 if nsID != "" { 2554 leaseNS, err = NamespaceByID(ctx, nsID, m.core) 2555 if err != nil { 2556 return nil, err 2557 } 2558 } 2559 2560 if leaseNS == nil { 2561 return nil, namespace.ErrNoNamespace 2562 } 2563 2564 return leaseNS, nil 2565} 2566 2567func (m *ExpirationManager) getLeaseMountAccessorLocked(ctx context.Context, leaseID string) string { 2568 m.coreStateLock.RLock() 2569 defer m.coreStateLock.RUnlock() 2570 return m.getLeaseMountAccessor(ctx, leaseID) 2571} 2572 2573// note: this function must be called with m.coreStateLock held for read 2574func (m *ExpirationManager) getLeaseMountAccessor(ctx context.Context, leaseID string) string { 2575 mount := m.core.router.MatchingMountEntry(ctx, leaseID) 2576 2577 var mountAccessor string 2578 if mount == nil { 2579 mountAccessor = "mount-accessor-not-found" 2580 } else { 2581 mountAccessor = mount.Accessor 2582 } 2583 2584 return mountAccessor 2585} 2586 2587func (m *ExpirationManager) getIrrevocableLeaseCounts(ctx context.Context, includeChildNamespaces bool) (map[string]interface{}, error) { 2588 requestNS, err := namespace.FromContext(ctx) 2589 if err != nil { 2590 m.logger.Error("could not get namespace from context", "error", err) 2591 return nil, err 2592 } 2593 2594 numMatchingLeasesPerMount := make(map[string]int) 2595 numMatchingLeases := 0 2596 m.irrevocable.Range(func(k, v interface{}) bool { 2597 leaseID := k.(string) 2598 leaseNS, err := m.getNamespaceFromLeaseID(ctx, leaseID) 2599 if err != nil { 2600 // We should probably note that an error occured, but continue counting 2601 m.logger.Warn("could not get lease namespace from ID", "error", err) 2602 return true 2603 } 2604 2605 leaseMatches := (leaseNS == requestNS) || (includeChildNamespaces && leaseNS.HasParent(requestNS)) 2606 if !leaseMatches { 2607 // the lease doesn't meet our criteria, so keep looking 2608 return true 2609 } 2610 2611 mountAccessor := m.getLeaseMountAccessor(ctx, leaseID) 2612 2613 if _, ok := numMatchingLeasesPerMount[mountAccessor]; !ok { 2614 numMatchingLeasesPerMount[mountAccessor] = 0 2615 } 2616 2617 numMatchingLeases++ 2618 numMatchingLeasesPerMount[mountAccessor]++ 2619 2620 return true 2621 }) 2622 2623 resp := make(map[string]interface{}) 2624 resp["lease_count"] = numMatchingLeases 2625 resp["counts"] = numMatchingLeasesPerMount 2626 2627 return resp, nil 2628} 2629 2630type leaseResponse struct { 2631 LeaseID string `json:"lease_id"` 2632 MountID string `json:"mount_id"` 2633 ErrMsg string `json:"error"` 2634 expireTime time.Time 2635} 2636 2637// returns a warning string, if applicable 2638// limit specifies how many results to return, and must be >0 2639// includeAll specifies if all results should be returned, regardless of limit 2640func (m *ExpirationManager) listIrrevocableLeases(ctx context.Context, includeChildNamespaces, returnAll bool, limit int) (map[string]interface{}, string, error) { 2641 requestNS, err := namespace.FromContext(ctx) 2642 if err != nil { 2643 m.logger.Error("could not get namespace from context", "error", err) 2644 return nil, "", err 2645 } 2646 2647 // map of mount point : lease info 2648 matchingLeases := make([]*leaseResponse, 0) 2649 numMatchingLeases := 0 2650 var warning string 2651 m.irrevocable.Range(func(k, v interface{}) bool { 2652 leaseID := k.(string) 2653 leaseInfo := v.(*leaseEntry) 2654 2655 leaseNS, err := m.getNamespaceFromLeaseID(ctx, leaseID) 2656 if err != nil { 2657 // We probably want to track that an error occured, but continue counting 2658 m.logger.Warn("could not get lease namespace from ID", "error", err) 2659 return true 2660 } 2661 2662 leaseMatches := (leaseNS == requestNS) || (includeChildNamespaces && leaseNS.HasParent(requestNS)) 2663 if !leaseMatches { 2664 // the lease doesn't meet our criteria, so keep looking 2665 return true 2666 } 2667 2668 if !returnAll && (numMatchingLeases >= limit) { 2669 m.logger.Warn("hit max irrevocable leases without force flag set") 2670 warning = MaxIrrevocableLeasesWarning 2671 return false 2672 } 2673 2674 mountAccessor := m.getLeaseMountAccessor(ctx, leaseID) 2675 2676 numMatchingLeases++ 2677 matchingLeases = append(matchingLeases, &leaseResponse{ 2678 LeaseID: leaseID, 2679 MountID: mountAccessor, 2680 ErrMsg: leaseInfo.RevokeErr, 2681 expireTime: leaseInfo.ExpireTime, 2682 }) 2683 2684 return true 2685 }) 2686 2687 // sort the results for consistent API response. we primarily sort on 2688 // increasing expire time, and break ties with increasing lease id 2689 sort.Slice(matchingLeases, func(i, j int) bool { 2690 if !matchingLeases[i].expireTime.Equal(matchingLeases[j].expireTime) { 2691 return matchingLeases[i].expireTime.Before(matchingLeases[j].expireTime) 2692 } 2693 2694 return matchingLeases[i].LeaseID < matchingLeases[j].LeaseID 2695 }) 2696 2697 resp := make(map[string]interface{}) 2698 resp["lease_count"] = numMatchingLeases 2699 resp["leases"] = matchingLeases 2700 2701 return resp, warning, nil 2702} 2703 2704// leaseEntry is used to structure the values the expiration 2705// manager stores. This is used to handle renew and revocation. 2706type leaseEntry struct { 2707 LeaseID string `json:"lease_id"` 2708 ClientToken string `json:"client_token"` 2709 ClientTokenType logical.TokenType `json:"token_type"` 2710 Path string `json:"path"` 2711 Data map[string]interface{} `json:"data"` 2712 Secret *logical.Secret `json:"secret"` 2713 Auth *logical.Auth `json:"auth"` 2714 IssueTime time.Time `json:"issue_time"` 2715 ExpireTime time.Time `json:"expire_time"` 2716 LastRenewalTime time.Time `json:"last_renewal_time"` 2717 2718 // Version is used to track new different versions of leases. V0 (or 2719 // zero-value) had non-root namespaced secondary indexes live in the root 2720 // namespace, and V1 has secondary indexes live in the matching namespace. 2721 Version int `json:"version"` 2722 2723 namespace *namespace.Namespace 2724 2725 // RevokeErr tracks if a lease has failed revocation in a way that is 2726 // unlikely to be automatically resolved. The first time this happens, 2727 // RevokeErr will be set, thus marking this leaseEntry as irrevocable. From 2728 // there, it must be manually removed (force revoked). 2729 RevokeErr string `json:"revokeErr"` 2730} 2731 2732// encode is used to JSON encode the lease entry 2733func (le *leaseEntry) encode() ([]byte, error) { 2734 return json.Marshal(le) 2735} 2736 2737func (le *leaseEntry) renewable() (bool, error) { 2738 switch { 2739 // If there is no entry, cannot review to renew 2740 case le == nil: 2741 return false, fmt.Errorf("lease not found") 2742 2743 case le.isIrrevocable(): 2744 return false, fmt.Errorf("lease is expired and has failed previous revocation attempts") 2745 2746 case le.ExpireTime.IsZero(): 2747 return false, fmt.Errorf("lease is not renewable") 2748 2749 case le.ClientTokenType == logical.TokenTypeBatch: 2750 return false, nil 2751 2752 // Determine if the lease is expired 2753 case le.ExpireTime.Before(time.Now()): 2754 return false, fmt.Errorf("lease expired") 2755 2756 // Determine if the lease is renewable 2757 case le.Secret != nil && !le.Secret.Renewable: 2758 return false, fmt.Errorf("lease is not renewable") 2759 2760 case le.Auth != nil && !le.Auth.Renewable: 2761 return false, fmt.Errorf("lease is not renewable") 2762 } 2763 2764 return true, nil 2765} 2766 2767func (le *leaseEntry) ttl() int64 { 2768 return int64(le.ExpireTime.Sub(time.Now().Round(time.Second)).Seconds()) 2769} 2770 2771func (le *leaseEntry) nonexpiringToken() bool { 2772 if le.Auth == nil { 2773 return false 2774 } 2775 // Note that at this time the only non-expiring tokens are root tokens, this test is more involved as it is trying 2776 // to catch tokens created by the VAULT-1949 non-expiring tokens bug and ensure they become expiring. 2777 return !le.Auth.LeaseEnabled() && len(le.Auth.Policies) == 1 && le.Auth.Policies[0] == "root" && le.namespace != nil && 2778 le.namespace.ID == namespace.RootNamespaceID 2779} 2780 2781// TODO maybe lock RevokeErr once this goes in: https://github.com/hashicorp/vault/pull/11122 2782func (le *leaseEntry) isIrrevocable() bool { 2783 return le.RevokeErr != "" 2784} 2785 2786func (le *leaseEntry) isIncorrectlyNonExpiring() bool { 2787 return le.ExpireTime.IsZero() && !le.nonexpiringToken() 2788} 2789 2790// decodeLeaseEntry is used to reverse encode and return a new entry 2791func decodeLeaseEntry(buf []byte) (*leaseEntry, error) { 2792 out := new(leaseEntry) 2793 return out, jsonutil.DecodeJSON(buf, out) 2794} 2795