1package database 2 3import ( 4 "context" 5 "errors" 6 "fmt" 7 "time" 8 9 "github.com/hashicorp/errwrap" 10 "github.com/hashicorp/go-multierror" 11 "github.com/hashicorp/vault/sdk/database/dbplugin" 12 "github.com/hashicorp/vault/sdk/framework" 13 "github.com/hashicorp/vault/sdk/helper/consts" 14 "github.com/hashicorp/vault/sdk/helper/locksutil" 15 "github.com/hashicorp/vault/sdk/helper/strutil" 16 "github.com/hashicorp/vault/sdk/logical" 17 "github.com/hashicorp/vault/sdk/queue" 18) 19 20const ( 21 // Interval to check the queue for items needing rotation 22 queueTickSeconds = 5 23 queueTickInterval = queueTickSeconds * time.Second 24 25 // WAL storage key used for static account rotations 26 staticWALKey = "staticRotationKey" 27) 28 29// populateQueue loads the priority queue with existing static accounts. This 30// occurs at initialization, after any WAL entries of failed or interrupted 31// rotations have been processed. It lists the roles from storage and searches 32// for any that have an associated static account, then adds them to the 33// priority queue for rotations. 34func (b *databaseBackend) populateQueue(ctx context.Context, s logical.Storage) { 35 log := b.Logger() 36 log.Info("populating role rotation queue") 37 38 // Build map of role name / wal entries 39 walMap, err := b.loadStaticWALs(ctx, s) 40 if err != nil { 41 log.Warn("unable to load rotation WALs", "error", err) 42 } 43 44 roles, err := s.List(ctx, databaseStaticRolePath) 45 if err != nil { 46 log.Warn("unable to list role for enqueueing", "error", err) 47 return 48 } 49 50 for _, roleName := range roles { 51 select { 52 case <-ctx.Done(): 53 log.Info("rotation queue restore cancelled") 54 return 55 default: 56 } 57 58 role, err := b.StaticRole(ctx, s, roleName) 59 if err != nil { 60 log.Warn("unable to read static role", "error", err, "role", roleName) 61 continue 62 } 63 64 item := queue.Item{ 65 Key: roleName, 66 Priority: role.StaticAccount.LastVaultRotation.Add(role.StaticAccount.RotationPeriod).Unix(), 67 } 68 69 // Check if role name is in map 70 walEntry := walMap[roleName] 71 if walEntry != nil { 72 // Check walEntry last vault time 73 if !walEntry.LastVaultRotation.IsZero() && walEntry.LastVaultRotation.Before(role.StaticAccount.LastVaultRotation) { 74 // WAL's last vault rotation record is older than the role's data, so 75 // delete and move on 76 if err := framework.DeleteWAL(ctx, s, walEntry.walID); err != nil { 77 log.Warn("unable to delete WAL", "error", err, "WAL ID", walEntry.walID) 78 } 79 } else { 80 log.Info("adjusting priority for Role") 81 item.Value = walEntry.walID 82 item.Priority = time.Now().Unix() 83 } 84 } 85 86 if err := b.pushItem(&item); err != nil { 87 log.Warn("unable to enqueue item", "error", err, "role", roleName) 88 } 89 } 90} 91 92// runTicker kicks off a periodic ticker that invoke the automatic credential 93// rotation method at a determined interval. The default interval is 5 seconds. 94func (b *databaseBackend) runTicker(ctx context.Context, s logical.Storage) { 95 b.logger.Info("starting periodic ticker") 96 tick := time.NewTicker(queueTickInterval) 97 defer tick.Stop() 98 for { 99 select { 100 case <-tick.C: 101 b.rotateCredentials(ctx, s) 102 103 case <-ctx.Done(): 104 b.logger.Info("stopping periodic ticker") 105 return 106 } 107 } 108} 109 110// setCredentialsWAL is used to store information in a WAL that can retry a 111// credential setting or rotation in the event of partial failure. 112type setCredentialsWAL struct { 113 NewPassword string `json:"new_password"` 114 OldPassword string `json:"old_password"` 115 RoleName string `json:"role_name"` 116 Username string `json:"username"` 117 118 LastVaultRotation time.Time `json:"last_vault_rotation"` 119 120 walID string 121} 122 123// rotateCredentials sets a new password for a static account. This method is 124// invoked in the runTicker method, which is in it's own go-routine, and invoked 125// periodically (approximately every 5 seconds). 126// 127// This method loops through the priority queue, popping the highest priority 128// item until it encounters the first item that does not yet need rotation, 129// based on the current time. 130func (b *databaseBackend) rotateCredentials(ctx context.Context, s logical.Storage) { 131 for b.rotateCredential(ctx, s) { 132 } 133} 134 135func (b *databaseBackend) rotateCredential(ctx context.Context, s logical.Storage) bool { 136 // Quit rotating credentials if shutdown has started 137 select { 138 case <-ctx.Done(): 139 return false 140 default: 141 } 142 item, err := b.popFromRotationQueue() 143 if err != nil { 144 if err != queue.ErrEmpty { 145 b.logger.Error("error popping item from queue", "err", err) 146 } 147 return false 148 } 149 150 // Guard against possible nil item 151 if item == nil { 152 return false 153 } 154 155 // Grab the exclusive lock for this Role, to make sure we don't incur and 156 // writes during the rotation process 157 lock := locksutil.LockForKey(b.roleLocks, item.Key) 158 lock.Lock() 159 defer lock.Unlock() 160 161 // Validate the role still exists 162 role, err := b.StaticRole(ctx, s, item.Key) 163 if err != nil { 164 b.logger.Error("unable to load role", "role", item.Key, "error", err) 165 item.Priority = time.Now().Add(10 * time.Second).Unix() 166 if err := b.pushItem(item); err != nil { 167 b.logger.Error("unable to push item on to queue", "error", err) 168 } 169 return true 170 } 171 if role == nil { 172 b.logger.Warn("role not found", "role", item.Key, "error", err) 173 return true 174 } 175 176 // If "now" is less than the Item priority, then this item does not need to 177 // be rotated 178 if time.Now().Unix() < item.Priority { 179 if err := b.pushItem(item); err != nil { 180 b.logger.Error("unable to push item on to queue", "error", err) 181 } 182 // Break out of the for loop 183 return false 184 } 185 186 input := &setStaticAccountInput{ 187 RoleName: item.Key, 188 Role: role, 189 } 190 191 // If there is a WAL entry related to this Role, the corresponding WAL ID 192 // should be stored in the Item's Value field. 193 if walID, ok := item.Value.(string); ok { 194 walEntry, err := b.findStaticWAL(ctx, s, walID) 195 if err != nil { 196 b.logger.Error("error finding static WAL", "error", err) 197 item.Priority = time.Now().Add(10 * time.Second).Unix() 198 if err := b.pushItem(item); err != nil { 199 b.logger.Error("unable to push item on to queue", "error", err) 200 } 201 } 202 if walEntry != nil && walEntry.NewPassword != "" { 203 input.Password = walEntry.NewPassword 204 input.WALID = walID 205 } 206 } 207 208 resp, err := b.setStaticAccount(ctx, s, input) 209 if err != nil { 210 b.logger.Error("unable to rotate credentials in periodic function", "error", err) 211 // Increment the priority enough so that the next call to this method 212 // likely will not attempt to rotate it, as a back-off of sorts 213 item.Priority = time.Now().Add(10 * time.Second).Unix() 214 215 // Preserve the WALID if it was returned 216 if resp != nil && resp.WALID != "" { 217 item.Value = resp.WALID 218 } 219 220 if err := b.pushItem(item); err != nil { 221 b.logger.Error("unable to push item on to queue", "error", err) 222 } 223 // Go to next item 224 return true 225 } 226 227 lvr := resp.RotationTime 228 if lvr.IsZero() { 229 lvr = time.Now() 230 } 231 232 // Update priority and push updated Item to the queue 233 nextRotation := lvr.Add(role.StaticAccount.RotationPeriod) 234 item.Priority = nextRotation.Unix() 235 if err := b.pushItem(item); err != nil { 236 b.logger.Warn("unable to push item on to queue", "error", err) 237 } 238 return true 239} 240 241// findStaticWAL loads a WAL entry by ID. If found, only return the WAL if it 242// is of type staticWALKey, otherwise return nil 243func (b *databaseBackend) findStaticWAL(ctx context.Context, s logical.Storage, id string) (*setCredentialsWAL, error) { 244 wal, err := framework.GetWAL(ctx, s, id) 245 if err != nil { 246 return nil, err 247 } 248 249 if wal == nil || wal.Kind != staticWALKey { 250 return nil, nil 251 } 252 253 data := wal.Data.(map[string]interface{}) 254 walEntry := setCredentialsWAL{ 255 walID: id, 256 NewPassword: data["new_password"].(string), 257 OldPassword: data["old_password"].(string), 258 RoleName: data["role_name"].(string), 259 Username: data["username"].(string), 260 } 261 lvr, err := time.Parse(time.RFC3339, data["last_vault_rotation"].(string)) 262 if err != nil { 263 return nil, err 264 } 265 walEntry.LastVaultRotation = lvr 266 267 return &walEntry, nil 268} 269 270type setStaticAccountInput struct { 271 RoleName string 272 Role *roleEntry 273 Password string 274 CreateUser bool 275 WALID string 276} 277 278type setStaticAccountOutput struct { 279 RotationTime time.Time 280 Password string 281 // Optional return field, in the event WAL was created and not destroyed 282 // during the operation 283 WALID string 284} 285 286// setStaticAccount sets the password for a static account associated with a 287// Role. This method does many things: 288// - verifies role exists and is in the allowed roles list 289// - loads an existing WAL entry if WALID input is given, otherwise creates a 290// new WAL entry 291// - gets a database connection 292// - accepts an input password, otherwise generates a new one via gRPC to the 293// database plugin 294// - sets new password for the static account 295// - uses WAL for ensuring passwords are not lost if storage to Vault fails 296// 297// This method does not perform any operations on the priority queue. Those 298// tasks must be handled outside of this method. 299func (b *databaseBackend) setStaticAccount(ctx context.Context, s logical.Storage, input *setStaticAccountInput) (*setStaticAccountOutput, error) { 300 var merr error 301 if input == nil || input.Role == nil || input.RoleName == "" { 302 return nil, errors.New("input was empty when attempting to set credentials for static account") 303 } 304 // Re-use WAL ID if present, otherwise PUT a new WAL 305 output := &setStaticAccountOutput{WALID: input.WALID} 306 307 dbConfig, err := b.DatabaseConfig(ctx, s, input.Role.DBName) 308 if err != nil { 309 return output, err 310 } 311 312 // If role name isn't in the database's allowed roles, send back a 313 // permission denied. 314 if !strutil.StrListContains(dbConfig.AllowedRoles, "*") && !strutil.StrListContainsGlob(dbConfig.AllowedRoles, input.RoleName) { 315 return output, fmt.Errorf("%q is not an allowed role", input.RoleName) 316 } 317 318 // Get the Database object 319 db, err := b.GetConnection(ctx, s, input.Role.DBName) 320 if err != nil { 321 return output, err 322 } 323 324 db.RLock() 325 defer db.RUnlock() 326 327 // Use password from input if available. This happens if we're restoring from 328 // a WAL item or processing the rotation queue with an item that has a WAL 329 // associated with it 330 newPassword := input.Password 331 if newPassword == "" { 332 // Generate a new password 333 newPassword, err = db.GenerateCredentials(ctx) 334 if err != nil { 335 return output, err 336 } 337 } 338 output.Password = newPassword 339 340 config := dbplugin.StaticUserConfig{ 341 Username: input.Role.StaticAccount.Username, 342 Password: newPassword, 343 } 344 345 if output.WALID == "" { 346 output.WALID, err = framework.PutWAL(ctx, s, staticWALKey, &setCredentialsWAL{ 347 RoleName: input.RoleName, 348 Username: config.Username, 349 NewPassword: config.Password, 350 OldPassword: input.Role.StaticAccount.Password, 351 LastVaultRotation: input.Role.StaticAccount.LastVaultRotation, 352 }) 353 if err != nil { 354 return output, errwrap.Wrapf("error writing WAL entry: {{err}}", err) 355 } 356 } 357 358 _, password, err := db.SetCredentials(ctx, input.Role.Statements, config) 359 if err != nil { 360 b.CloseIfShutdown(db, err) 361 return output, errwrap.Wrapf("error setting credentials: {{err}}", err) 362 } 363 364 if newPassword != password { 365 return output, errors.New("mismatch passwords returned") 366 } 367 368 // Store updated role information 369 // lvr is the known LastVaultRotation 370 lvr := time.Now() 371 input.Role.StaticAccount.LastVaultRotation = lvr 372 input.Role.StaticAccount.Password = password 373 output.RotationTime = lvr 374 375 entry, err := logical.StorageEntryJSON(databaseStaticRolePath+input.RoleName, input.Role) 376 if err != nil { 377 return output, err 378 } 379 if err := s.Put(ctx, entry); err != nil { 380 return output, err 381 } 382 383 // Cleanup WAL after successfully rotating and pushing new item on to queue 384 if err := framework.DeleteWAL(ctx, s, output.WALID); err != nil { 385 merr = multierror.Append(merr, err) 386 return output, merr 387 } 388 389 // The WAL has been deleted, return new setStaticAccountOutput without it 390 return &setStaticAccountOutput{RotationTime: lvr}, merr 391} 392 393// initQueue preforms the necessary checks and initializations needed to preform 394// automatic credential rotation for roles associated with static accounts. This 395// method verifies if a queue is needed (primary server or local mount), and if 396// so initializes the queue and launches a go-routine to periodically invoke a 397// method to preform the rotations. 398// 399// initQueue is invoked by the Factory method in a go-routine. The Factory does 400// not wait for success or failure of it's tasks before continuing. This is to 401// avoid blocking the mount process while loading and evaluating existing roles, 402// etc. 403func (b *databaseBackend) initQueue(ctx context.Context, conf *logical.BackendConfig) { 404 // Verify this mount is on the primary server, or is a local mount. If not, do 405 // not create a queue or launch a ticker. Both processing the WAL list and 406 // populating the queue are done sequentially and before launching a 407 // go-routine to run the periodic ticker. 408 replicationState := conf.System.ReplicationState() 409 if (conf.System.LocalMount() || !replicationState.HasState(consts.ReplicationPerformanceSecondary)) && 410 !replicationState.HasState(consts.ReplicationDRSecondary) && 411 !replicationState.HasState(consts.ReplicationPerformanceStandby) { 412 b.Logger().Info("initializing database rotation queue") 413 414 // Poll for a PutWAL call that does not return a "read-only storage" error. 415 // This ensures the startup phases of loading WAL entries from any possible 416 // failed rotations can complete without error when deleting from storage. 417 READONLY_LOOP: 418 for { 419 select { 420 case <-ctx.Done(): 421 b.Logger().Info("queue initialization canceled") 422 return 423 default: 424 } 425 426 walID, err := framework.PutWAL(ctx, conf.StorageView, staticWALKey, &setCredentialsWAL{RoleName: "vault-readonlytest"}) 427 if walID != "" { 428 defer framework.DeleteWAL(ctx, conf.StorageView, walID) 429 } 430 switch { 431 case err == nil: 432 break READONLY_LOOP 433 case err.Error() == logical.ErrSetupReadOnly.Error(): 434 time.Sleep(10 * time.Millisecond) 435 default: 436 b.Logger().Error("deleting nil key resulted in error", "error", err) 437 return 438 } 439 } 440 441 // Load roles and populate queue with static accounts 442 b.populateQueue(ctx, conf.StorageView) 443 444 // Launch ticker 445 go b.runTicker(ctx, conf.StorageView) 446 } 447} 448 449// loadStaticWALs reads WAL entries and returns a map of roles and their 450// setCredentialsWAL, if found. 451func (b *databaseBackend) loadStaticWALs(ctx context.Context, s logical.Storage) (map[string]*setCredentialsWAL, error) { 452 keys, err := framework.ListWAL(ctx, s) 453 if err != nil { 454 return nil, err 455 } 456 if len(keys) == 0 { 457 b.Logger().Debug("no WAL entries found") 458 return nil, nil 459 } 460 461 walMap := make(map[string]*setCredentialsWAL) 462 // Loop through WAL keys and process any rotation ones 463 for _, walID := range keys { 464 walEntry, err := b.findStaticWAL(ctx, s, walID) 465 if err != nil { 466 b.Logger().Error("error loading static WAL", "id", walID, "error", err) 467 continue 468 } 469 if walEntry == nil { 470 continue 471 } 472 473 // Verify the static role still exists 474 roleName := walEntry.RoleName 475 role, err := b.StaticRole(ctx, s, roleName) 476 if err != nil { 477 b.Logger().Warn("unable to read static role", "error", err, "role", roleName) 478 continue 479 } 480 if role == nil || role.StaticAccount == nil { 481 if err := framework.DeleteWAL(ctx, s, walEntry.walID); err != nil { 482 b.Logger().Warn("unable to delete WAL", "error", err, "WAL ID", walEntry.walID) 483 } 484 continue 485 } 486 487 walEntry.walID = walID 488 walMap[walEntry.RoleName] = walEntry 489 } 490 return walMap, nil 491} 492 493// pushItem wraps the internal queue's Push call, to make sure a queue is 494// actually available. This is needed because both runTicker and initQueue 495// operate in go-routines, and could be accessing the queue concurrently 496func (b *databaseBackend) pushItem(item *queue.Item) error { 497 b.RLock() 498 unlockFunc := b.RUnlock 499 defer func() { unlockFunc() }() 500 501 if b.credRotationQueue != nil { 502 return b.credRotationQueue.Push(item) 503 } 504 505 b.Logger().Warn("no queue found during push item") 506 return nil 507} 508 509// popFromRotationQueue wraps the internal queue's Pop call, to make sure a queue is 510// actually available. This is needed because both runTicker and initQueue 511// operate in go-routines, and could be accessing the queue concurrently 512func (b *databaseBackend) popFromRotationQueue() (*queue.Item, error) { 513 b.RLock() 514 defer b.RUnlock() 515 if b.credRotationQueue != nil { 516 return b.credRotationQueue.Pop() 517 } 518 return nil, queue.ErrEmpty 519} 520 521// popFromRotationQueueByKey wraps the internal queue's PopByKey call, to make sure a queue is 522// actually available. This is needed because both runTicker and initQueue 523// operate in go-routines, and could be accessing the queue concurrently 524func (b *databaseBackend) popFromRotationQueueByKey(name string) (*queue.Item, error) { 525 b.RLock() 526 defer b.RUnlock() 527 if b.credRotationQueue != nil { 528 return b.credRotationQueue.PopByKey(name) 529 } 530 return nil, queue.ErrEmpty 531} 532