1package database 2 3import ( 4 "context" 5 "errors" 6 "fmt" 7 "time" 8 9 "github.com/hashicorp/errwrap" 10 "github.com/hashicorp/go-multierror" 11 "github.com/hashicorp/vault/sdk/database/dbplugin" 12 "github.com/hashicorp/vault/sdk/framework" 13 "github.com/hashicorp/vault/sdk/helper/consts" 14 "github.com/hashicorp/vault/sdk/helper/locksutil" 15 "github.com/hashicorp/vault/sdk/helper/strutil" 16 "github.com/hashicorp/vault/sdk/logical" 17 "github.com/hashicorp/vault/sdk/queue" 18) 19 20const ( 21 // Interval to check the queue for items needing rotation 22 queueTickSeconds = 5 23 queueTickInterval = queueTickSeconds * time.Second 24 25 // WAL storage key used for static account rotations 26 staticWALKey = "staticRotationKey" 27) 28 29// populateQueue loads the priority queue with existing static accounts. This 30// occurs at initialization, after any WAL entries of failed or interrupted 31// rotations have been processed. It lists the roles from storage and searches 32// for any that have an associated static account, then adds them to the 33// priority queue for rotations. 34func (b *databaseBackend) populateQueue(ctx context.Context, s logical.Storage) { 35 log := b.Logger() 36 log.Info("populating role rotation queue") 37 38 // Build map of role name / wal entries 39 walMap, err := b.loadStaticWALs(ctx, s) 40 if err != nil { 41 log.Warn("unable to load rotation WALs", "error", err) 42 } 43 44 roles, err := s.List(ctx, databaseStaticRolePath) 45 if err != nil { 46 log.Warn("unable to list role for enqueueing", "error", err) 47 return 48 } 49 50 for _, roleName := range roles { 51 select { 52 case <-ctx.Done(): 53 log.Info("rotation queue restore cancelled") 54 return 55 default: 56 } 57 58 role, err := b.StaticRole(ctx, s, roleName) 59 if err != nil { 60 log.Warn("unable to read static role", "error", err, "role", roleName) 61 continue 62 } 63 64 item := queue.Item{ 65 Key: roleName, 66 Priority: role.StaticAccount.LastVaultRotation.Add(role.StaticAccount.RotationPeriod).Unix(), 67 } 68 69 // Check if role name is in map 70 walEntry := walMap[roleName] 71 if walEntry != nil { 72 // Check walEntry last vault time 73 if !walEntry.LastVaultRotation.IsZero() && walEntry.LastVaultRotation.Before(role.StaticAccount.LastVaultRotation) { 74 // WAL's last vault rotation record is older than the role's data, so 75 // delete and move on 76 if err := framework.DeleteWAL(ctx, s, walEntry.walID); err != nil { 77 log.Warn("unable to delete WAL", "error", err, "WAL ID", walEntry.walID) 78 } 79 } else { 80 log.Info("adjusting priority for Role") 81 item.Value = walEntry.walID 82 item.Priority = time.Now().Unix() 83 } 84 } 85 86 if err := b.pushItem(&item); err != nil { 87 log.Warn("unable to enqueue item", "error", err, "role", roleName) 88 } 89 } 90} 91 92// runTicker kicks off a periodic ticker that invoke the automatic credential 93// rotation method at a determined interval. The default interval is 5 seconds. 94func (b *databaseBackend) runTicker(ctx context.Context, s logical.Storage) { 95 b.logger.Info("starting periodic ticker") 96 tick := time.NewTicker(queueTickInterval) 97 defer tick.Stop() 98 for { 99 select { 100 case <-tick.C: 101 b.rotateCredentials(ctx, s) 102 103 case <-ctx.Done(): 104 b.logger.Info("stopping periodic ticker") 105 return 106 } 107 } 108} 109 110// setCredentialsWAL is used to store information in a WAL that can retry a 111// credential setting or rotation in the event of partial failure. 112type setCredentialsWAL struct { 113 NewPassword string `json:"new_password"` 114 OldPassword string `json:"old_password"` 115 RoleName string `json:"role_name"` 116 Username string `json:"username"` 117 118 LastVaultRotation time.Time `json:"last_vault_rotation"` 119 120 walID string 121} 122 123// rotateCredentials sets a new password for a static account. This method is 124// invoked in the runTicker method, which is in it's own go-routine, and invoked 125// periodically (approximately every 5 seconds). 126// 127// This method loops through the priority queue, popping the highest priority 128// item until it encounters the first item that does not yet need rotation, 129// based on the current time. 130func (b *databaseBackend) rotateCredentials(ctx context.Context, s logical.Storage) error { 131 for { 132 // Quit rotating credentials if shutdown has started 133 select { 134 case <-ctx.Done(): 135 return nil 136 default: 137 } 138 item, err := b.popFromRotationQueue() 139 if err != nil { 140 if err == queue.ErrEmpty { 141 return nil 142 } 143 return err 144 } 145 146 // Guard against possible nil item 147 if item == nil { 148 return nil 149 } 150 151 // Grab the exclusive lock for this Role, to make sure we don't incur and 152 // writes during the rotation process 153 lock := locksutil.LockForKey(b.roleLocks, item.Key) 154 lock.Lock() 155 defer lock.Unlock() 156 157 // Validate the role still exists 158 role, err := b.StaticRole(ctx, s, item.Key) 159 if err != nil { 160 b.logger.Error("unable to load role", "role", item.Key, "error", err) 161 item.Priority = time.Now().Add(10 * time.Second).Unix() 162 if err := b.pushItem(item); err != nil { 163 b.logger.Error("unable to push item on to queue", "error", err) 164 } 165 continue 166 } 167 if role == nil { 168 b.logger.Warn("role not found", "role", item.Key, "error", err) 169 continue 170 } 171 172 // If "now" is less than the Item priority, then this item does not need to 173 // be rotated 174 if time.Now().Unix() < item.Priority { 175 if err := b.pushItem(item); err != nil { 176 b.logger.Error("unable to push item on to queue", "error", err) 177 } 178 // Break out of the for loop 179 break 180 } 181 182 input := &setStaticAccountInput{ 183 RoleName: item.Key, 184 Role: role, 185 } 186 187 // If there is a WAL entry related to this Role, the corresponding WAL ID 188 // should be stored in the Item's Value field. 189 if walID, ok := item.Value.(string); ok { 190 walEntry, err := b.findStaticWAL(ctx, s, walID) 191 if err != nil { 192 b.logger.Error("error finding static WAL", "error", err) 193 item.Priority = time.Now().Add(10 * time.Second).Unix() 194 if err := b.pushItem(item); err != nil { 195 b.logger.Error("unable to push item on to queue", "error", err) 196 } 197 } 198 if walEntry != nil && walEntry.NewPassword != "" { 199 input.Password = walEntry.NewPassword 200 input.WALID = walID 201 } 202 } 203 204 resp, err := b.setStaticAccount(ctx, s, input) 205 if err != nil { 206 b.logger.Error("unable to rotate credentials in periodic function", "error", err) 207 // Increment the priority enough so that the next call to this method 208 // likely will not attempt to rotate it, as a back-off of sorts 209 item.Priority = time.Now().Add(10 * time.Second).Unix() 210 211 // Preserve the WALID if it was returned 212 if resp != nil && resp.WALID != "" { 213 item.Value = resp.WALID 214 } 215 216 if err := b.pushItem(item); err != nil { 217 b.logger.Error("unable to push item on to queue", "error", err) 218 } 219 // Go to next item 220 continue 221 } 222 223 lvr := resp.RotationTime 224 if lvr.IsZero() { 225 lvr = time.Now() 226 } 227 228 // Update priority and push updated Item to the queue 229 nextRotation := lvr.Add(role.StaticAccount.RotationPeriod) 230 item.Priority = nextRotation.Unix() 231 if err := b.pushItem(item); err != nil { 232 b.logger.Warn("unable to push item on to queue", "error", err) 233 } 234 } 235 return nil 236} 237 238// findStaticWAL loads a WAL entry by ID. If found, only return the WAL if it 239// is of type staticWALKey, otherwise return nil 240func (b *databaseBackend) findStaticWAL(ctx context.Context, s logical.Storage, id string) (*setCredentialsWAL, error) { 241 wal, err := framework.GetWAL(ctx, s, id) 242 if err != nil { 243 return nil, err 244 } 245 246 if wal == nil || wal.Kind != staticWALKey { 247 return nil, nil 248 } 249 250 data := wal.Data.(map[string]interface{}) 251 walEntry := setCredentialsWAL{ 252 walID: id, 253 NewPassword: data["new_password"].(string), 254 OldPassword: data["old_password"].(string), 255 RoleName: data["role_name"].(string), 256 Username: data["username"].(string), 257 } 258 lvr, err := time.Parse(time.RFC3339, data["last_vault_rotation"].(string)) 259 if err != nil { 260 return nil, err 261 } 262 walEntry.LastVaultRotation = lvr 263 264 return &walEntry, nil 265} 266 267type setStaticAccountInput struct { 268 RoleName string 269 Role *roleEntry 270 Password string 271 CreateUser bool 272 WALID string 273} 274 275type setStaticAccountOutput struct { 276 RotationTime time.Time 277 Password string 278 // Optional return field, in the event WAL was created and not destroyed 279 // during the operation 280 WALID string 281} 282 283// setStaticAccount sets the password for a static account associated with a 284// Role. This method does many things: 285// - verifies role exists and is in the allowed roles list 286// - loads an existing WAL entry if WALID input is given, otherwise creates a 287// new WAL entry 288// - gets a database connection 289// - accepts an input password, otherwise generates a new one via gRPC to the 290// database plugin 291// - sets new password for the static account 292// - uses WAL for ensuring passwords are not lost if storage to Vault fails 293// 294// This method does not perform any operations on the priority queue. Those 295// tasks must be handled outside of this method. 296func (b *databaseBackend) setStaticAccount(ctx context.Context, s logical.Storage, input *setStaticAccountInput) (*setStaticAccountOutput, error) { 297 var merr error 298 if input == nil || input.Role == nil || input.RoleName == "" { 299 return nil, errors.New("input was empty when attempting to set credentials for static account") 300 } 301 // Re-use WAL ID if present, otherwise PUT a new WAL 302 output := &setStaticAccountOutput{WALID: input.WALID} 303 304 dbConfig, err := b.DatabaseConfig(ctx, s, input.Role.DBName) 305 if err != nil { 306 return output, err 307 } 308 309 // If role name isn't in the database's allowed roles, send back a 310 // permission denied. 311 if !strutil.StrListContains(dbConfig.AllowedRoles, "*") && !strutil.StrListContainsGlob(dbConfig.AllowedRoles, input.RoleName) { 312 return output, fmt.Errorf("%q is not an allowed role", input.RoleName) 313 } 314 315 // Get the Database object 316 db, err := b.GetConnection(ctx, s, input.Role.DBName) 317 if err != nil { 318 return output, err 319 } 320 321 db.RLock() 322 defer db.RUnlock() 323 324 // Use password from input if available. This happens if we're restoring from 325 // a WAL item or processing the rotation queue with an item that has a WAL 326 // associated with it 327 newPassword := input.Password 328 if newPassword == "" { 329 // Generate a new password 330 newPassword, err = db.GenerateCredentials(ctx) 331 if err != nil { 332 return output, err 333 } 334 } 335 output.Password = newPassword 336 337 config := dbplugin.StaticUserConfig{ 338 Username: input.Role.StaticAccount.Username, 339 Password: newPassword, 340 } 341 342 if output.WALID == "" { 343 output.WALID, err = framework.PutWAL(ctx, s, staticWALKey, &setCredentialsWAL{ 344 RoleName: input.RoleName, 345 Username: config.Username, 346 NewPassword: config.Password, 347 OldPassword: input.Role.StaticAccount.Password, 348 LastVaultRotation: input.Role.StaticAccount.LastVaultRotation, 349 }) 350 if err != nil { 351 return output, errwrap.Wrapf("error writing WAL entry: {{err}}", err) 352 } 353 } 354 355 _, password, err := db.SetCredentials(ctx, input.Role.Statements, config) 356 if err != nil { 357 b.CloseIfShutdown(db, err) 358 return output, errwrap.Wrapf("error setting credentials: {{err}}", err) 359 } 360 361 if newPassword != password { 362 return output, errors.New("mismatch passwords returned") 363 } 364 365 // Store updated role information 366 // lvr is the known LastVaultRotation 367 lvr := time.Now() 368 input.Role.StaticAccount.LastVaultRotation = lvr 369 input.Role.StaticAccount.Password = password 370 output.RotationTime = lvr 371 372 entry, err := logical.StorageEntryJSON(databaseStaticRolePath+input.RoleName, input.Role) 373 if err != nil { 374 return output, err 375 } 376 if err := s.Put(ctx, entry); err != nil { 377 return output, err 378 } 379 380 // Cleanup WAL after successfully rotating and pushing new item on to queue 381 if err := framework.DeleteWAL(ctx, s, output.WALID); err != nil { 382 merr = multierror.Append(merr, err) 383 return output, merr 384 } 385 386 // The WAL has been deleted, return new setStaticAccountOutput without it 387 return &setStaticAccountOutput{RotationTime: lvr}, merr 388} 389 390// initQueue preforms the necessary checks and initializations needed to preform 391// automatic credential rotation for roles associated with static accounts. This 392// method verifies if a queue is needed (primary server or local mount), and if 393// so initializes the queue and launches a go-routine to periodically invoke a 394// method to preform the rotations. 395// 396// initQueue is invoked by the Factory method in a go-routine. The Factory does 397// not wait for success or failure of it's tasks before continuing. This is to 398// avoid blocking the mount process while loading and evaluating existing roles, 399// etc. 400func (b *databaseBackend) initQueue(ctx context.Context, conf *logical.BackendConfig) { 401 // Verify this mount is on the primary server, or is a local mount. If not, do 402 // not create a queue or launch a ticker. Both processing the WAL list and 403 // populating the queue are done sequentially and before launching a 404 // go-routine to run the periodic ticker. 405 replicationState := conf.System.ReplicationState() 406 if (conf.System.LocalMount() || !replicationState.HasState(consts.ReplicationPerformanceSecondary)) && 407 !replicationState.HasState(consts.ReplicationDRSecondary) && 408 !replicationState.HasState(consts.ReplicationPerformanceStandby) { 409 b.Logger().Info("initializing database rotation queue") 410 411 // Poll for a PutWAL call that does not return a "read-only storage" error. 412 // This ensures the startup phases of loading WAL entries from any possible 413 // failed rotations can complete without error when deleting from storage. 414 READONLY_LOOP: 415 for { 416 select { 417 case <-ctx.Done(): 418 b.Logger().Info("queue initialization canceled") 419 return 420 default: 421 } 422 423 walID, err := framework.PutWAL(ctx, conf.StorageView, staticWALKey, &setCredentialsWAL{RoleName: "vault-readonlytest"}) 424 if walID != "" { 425 defer framework.DeleteWAL(ctx, conf.StorageView, walID) 426 } 427 switch { 428 case err == nil: 429 break READONLY_LOOP 430 case err.Error() == logical.ErrSetupReadOnly.Error(): 431 time.Sleep(10 * time.Millisecond) 432 default: 433 b.Logger().Error("deleting nil key resulted in error", "error", err) 434 return 435 } 436 } 437 438 // Load roles and populate queue with static accounts 439 b.populateQueue(ctx, conf.StorageView) 440 441 // Launch ticker 442 go b.runTicker(ctx, conf.StorageView) 443 } 444} 445 446// loadStaticWALs reads WAL entries and returns a map of roles and their 447// setCredentialsWAL, if found. 448func (b *databaseBackend) loadStaticWALs(ctx context.Context, s logical.Storage) (map[string]*setCredentialsWAL, error) { 449 keys, err := framework.ListWAL(ctx, s) 450 if err != nil { 451 return nil, err 452 } 453 if len(keys) == 0 { 454 b.Logger().Debug("no WAL entries found") 455 return nil, nil 456 } 457 458 walMap := make(map[string]*setCredentialsWAL) 459 // Loop through WAL keys and process any rotation ones 460 for _, walID := range keys { 461 walEntry, err := b.findStaticWAL(ctx, s, walID) 462 if err != nil { 463 b.Logger().Error("error loading static WAL", "id", walID, "error", err) 464 continue 465 } 466 if walEntry == nil { 467 continue 468 } 469 470 // Verify the static role still exists 471 roleName := walEntry.RoleName 472 role, err := b.StaticRole(ctx, s, roleName) 473 if err != nil { 474 b.Logger().Warn("unable to read static role", "error", err, "role", roleName) 475 continue 476 } 477 if role == nil || role.StaticAccount == nil { 478 if err := framework.DeleteWAL(ctx, s, walEntry.walID); err != nil { 479 b.Logger().Warn("unable to delete WAL", "error", err, "WAL ID", walEntry.walID) 480 } 481 continue 482 } 483 484 walEntry.walID = walID 485 walMap[walEntry.RoleName] = walEntry 486 } 487 return walMap, nil 488} 489 490// pushItem wraps the internal queue's Push call, to make sure a queue is 491// actually available. This is needed because both runTicker and initQueue 492// operate in go-routines, and could be accessing the queue concurrently 493func (b *databaseBackend) pushItem(item *queue.Item) error { 494 b.RLock() 495 unlockFunc := b.RUnlock 496 defer func() { unlockFunc() }() 497 498 if b.credRotationQueue != nil { 499 return b.credRotationQueue.Push(item) 500 } 501 502 b.Logger().Warn("no queue found during push item") 503 return nil 504} 505 506// popFromRotationQueue wraps the internal queue's Pop call, to make sure a queue is 507// actually available. This is needed because both runTicker and initQueue 508// operate in go-routines, and could be accessing the queue concurrently 509func (b *databaseBackend) popFromRotationQueue() (*queue.Item, error) { 510 b.RLock() 511 defer b.RUnlock() 512 if b.credRotationQueue != nil { 513 return b.credRotationQueue.Pop() 514 } 515 return nil, queue.ErrEmpty 516} 517 518// popFromRotationQueueByKey wraps the internal queue's PopByKey call, to make sure a queue is 519// actually available. This is needed because both runTicker and initQueue 520// operate in go-routines, and could be accessing the queue concurrently 521func (b *databaseBackend) popFromRotationQueueByKey(name string) (*queue.Item, error) { 522 b.RLock() 523 defer b.RUnlock() 524 if b.credRotationQueue != nil { 525 return b.credRotationQueue.PopByKey(name) 526 } 527 return nil, queue.ErrEmpty 528} 529