1package vault 2 3import ( 4 "context" 5 "crypto/subtle" 6 "encoding/base64" 7 "errors" 8 "strings" 9 10 proto "github.com/golang/protobuf/proto" 11 uuid "github.com/hashicorp/go-uuid" 12 "github.com/hashicorp/vault/helper/namespace" 13 "github.com/hashicorp/vault/physical/raft" 14 "github.com/hashicorp/vault/sdk/framework" 15 "github.com/hashicorp/vault/sdk/logical" 16 "github.com/hashicorp/vault/sdk/physical" 17 "github.com/hashicorp/vault/vault/seal" 18) 19 20// raftStoragePaths returns paths for use when raft is the storage mechanism. 21func (b *SystemBackend) raftStoragePaths() []*framework.Path { 22 return []*framework.Path{ 23 { 24 Pattern: "storage/raft/bootstrap/answer", 25 26 Fields: map[string]*framework.FieldSchema{ 27 "server_id": { 28 Type: framework.TypeString, 29 }, 30 "answer": { 31 Type: framework.TypeString, 32 }, 33 "cluster_addr": { 34 Type: framework.TypeString, 35 }, 36 }, 37 38 Operations: map[logical.Operation]framework.OperationHandler{ 39 logical.UpdateOperation: &framework.PathOperation{ 40 Callback: b.handleRaftBootstrapAnswerWrite(), 41 Summary: "Accepts an answer from the peer to be joined to the fact cluster.", 42 }, 43 }, 44 45 HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-bootstrap-answer"][0]), 46 HelpDescription: strings.TrimSpace(sysRaftHelp["raft-bootstrap-answer"][1]), 47 }, 48 { 49 Pattern: "storage/raft/bootstrap/challenge", 50 51 Fields: map[string]*framework.FieldSchema{ 52 "server_id": { 53 Type: framework.TypeString, 54 }, 55 }, 56 57 Operations: map[logical.Operation]framework.OperationHandler{ 58 logical.UpdateOperation: &framework.PathOperation{ 59 Callback: b.handleRaftBootstrapChallengeWrite(), 60 Summary: "Creates a challenge for the new peer to be joined to the raft cluster.", 61 }, 62 }, 63 64 HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-bootstrap-challenge"][0]), 65 HelpDescription: strings.TrimSpace(sysRaftHelp["raft-bootstrap-challenge"][1]), 66 }, 67 { 68 Pattern: "storage/raft/remove-peer", 69 70 Fields: map[string]*framework.FieldSchema{ 71 "server_id": { 72 Type: framework.TypeString, 73 }, 74 }, 75 76 Operations: map[logical.Operation]framework.OperationHandler{ 77 logical.UpdateOperation: &framework.PathOperation{ 78 Callback: b.handleRaftRemovePeerUpdate(), 79 Summary: "Remove a peer from the raft cluster.", 80 }, 81 }, 82 83 HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-remove-peer"][0]), 84 HelpDescription: strings.TrimSpace(sysRaftHelp["raft-remove-peer"][1]), 85 }, 86 { 87 Pattern: "storage/raft/configuration", 88 89 Operations: map[logical.Operation]framework.OperationHandler{ 90 logical.ReadOperation: &framework.PathOperation{ 91 Callback: b.handleRaftConfigurationGet(), 92 Summary: "Returns the configuration of the raft cluster.", 93 }, 94 }, 95 96 HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-remove-peer"][0]), 97 HelpDescription: strings.TrimSpace(sysRaftHelp["raft-remove-peer"][1]), 98 }, 99 { 100 Pattern: "storage/raft/snapshot", 101 Operations: map[logical.Operation]framework.OperationHandler{ 102 logical.ReadOperation: &framework.PathOperation{ 103 Callback: b.handleStorageRaftSnapshotRead(), 104 Summary: "Retruns a snapshot of the current state of vault.", 105 }, 106 logical.UpdateOperation: &framework.PathOperation{ 107 Callback: b.handleStorageRaftSnapshotWrite(false), 108 Summary: "Installs the provided snapshot, returning the cluster to the state defined in it.", 109 }, 110 }, 111 112 HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-remove-peer"][0]), 113 HelpDescription: strings.TrimSpace(sysRaftHelp["raft-remove-peer"][1]), 114 }, 115 { 116 Pattern: "storage/raft/snapshot-force", 117 Operations: map[logical.Operation]framework.OperationHandler{ 118 logical.UpdateOperation: &framework.PathOperation{ 119 Callback: b.handleStorageRaftSnapshotWrite(true), 120 Summary: "Installs the provided snapshot, returning the cluster to the state defined in it. This bypasses checks ensuring the current Autounseal or Shamir keys are consistent with the snapshot data.", 121 }, 122 }, 123 124 HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-remove-peer"][0]), 125 HelpDescription: strings.TrimSpace(sysRaftHelp["raft-remove-peer"][1]), 126 }, 127 } 128} 129 130func (b *SystemBackend) handleRaftConfigurationGet() framework.OperationFunc { 131 return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { 132 133 raftStorage, ok := b.Core.underlyingPhysical.(*raft.RaftBackend) 134 if !ok { 135 return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest 136 } 137 138 config, err := raftStorage.GetConfiguration(ctx) 139 if err != nil { 140 return nil, err 141 } 142 143 return &logical.Response{ 144 Data: map[string]interface{}{ 145 "config": config, 146 }, 147 }, nil 148 } 149} 150 151func (b *SystemBackend) handleRaftRemovePeerUpdate() framework.OperationFunc { 152 return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { 153 serverID := d.Get("server_id").(string) 154 if len(serverID) == 0 { 155 return logical.ErrorResponse("no server id provided"), logical.ErrInvalidRequest 156 } 157 158 raftStorage, ok := b.Core.underlyingPhysical.(*raft.RaftBackend) 159 if !ok { 160 return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest 161 } 162 163 if err := raftStorage.RemovePeer(ctx, serverID); err != nil { 164 return nil, err 165 } 166 if b.Core.raftFollowerStates != nil { 167 b.Core.raftFollowerStates.delete(serverID) 168 } 169 170 return nil, nil 171 } 172} 173 174func (b *SystemBackend) handleRaftBootstrapChallengeWrite() framework.OperationFunc { 175 return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { 176 _, ok := b.Core.underlyingPhysical.(*raft.RaftBackend) 177 if !ok { 178 return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest 179 } 180 181 serverID := d.Get("server_id").(string) 182 if len(serverID) == 0 { 183 return logical.ErrorResponse("no server id provided"), logical.ErrInvalidRequest 184 } 185 186 uuid, err := uuid.GenerateRandomBytes(16) 187 if err != nil { 188 return nil, err 189 } 190 191 sealAccess := b.Core.seal.GetAccess() 192 eBlob, err := sealAccess.Encrypt(ctx, uuid) 193 if err != nil { 194 return nil, err 195 } 196 protoBlob, err := proto.Marshal(eBlob) 197 if err != nil { 198 return nil, err 199 } 200 201 b.Core.pendingRaftPeers[serverID] = uuid 202 sealConfig, err := b.Core.seal.BarrierConfig(ctx) 203 if err != nil { 204 return nil, err 205 } 206 207 return &logical.Response{ 208 Data: map[string]interface{}{ 209 "challenge": base64.StdEncoding.EncodeToString(protoBlob), 210 "seal_config": sealConfig, 211 }, 212 }, nil 213 } 214} 215 216func (b *SystemBackend) handleRaftBootstrapAnswerWrite() framework.OperationFunc { 217 return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { 218 raftStorage, ok := b.Core.underlyingPhysical.(*raft.RaftBackend) 219 if !ok { 220 return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest 221 } 222 223 serverID := d.Get("server_id").(string) 224 if len(serverID) == 0 { 225 return logical.ErrorResponse("no server_id provided"), logical.ErrInvalidRequest 226 } 227 answerRaw := d.Get("answer").(string) 228 if len(answerRaw) == 0 { 229 return logical.ErrorResponse("no answer provided"), logical.ErrInvalidRequest 230 } 231 clusterAddr := d.Get("cluster_addr").(string) 232 if len(clusterAddr) == 0 { 233 return logical.ErrorResponse("no cluster_addr provided"), logical.ErrInvalidRequest 234 } 235 236 answer, err := base64.StdEncoding.DecodeString(answerRaw) 237 if err != nil { 238 return logical.ErrorResponse("could not base64 decode answer"), logical.ErrInvalidRequest 239 } 240 241 expectedAnswer, ok := b.Core.pendingRaftPeers[serverID] 242 if !ok { 243 return logical.ErrorResponse("no expected answer for the server id provided"), logical.ErrInvalidRequest 244 } 245 246 delete(b.Core.pendingRaftPeers, serverID) 247 248 if subtle.ConstantTimeCompare(answer, expectedAnswer) == 0 { 249 return logical.ErrorResponse("invalid answer given"), logical.ErrInvalidRequest 250 } 251 252 tlsKeyringEntry, err := b.Core.barrier.Get(ctx, raftTLSStoragePath) 253 if err != nil { 254 return nil, err 255 } 256 if tlsKeyringEntry == nil { 257 return nil, errors.New("could not find raft TLS configuration") 258 } 259 var keyring raft.RaftTLSKeyring 260 if err := tlsKeyringEntry.DecodeJSON(&keyring); err != nil { 261 return nil, errors.New("could not decode raft TLS configuration") 262 } 263 264 if err := raftStorage.AddPeer(ctx, serverID, clusterAddr); err != nil { 265 return nil, err 266 } 267 if b.Core.raftFollowerStates != nil { 268 b.Core.raftFollowerStates.update(serverID, 0) 269 } 270 271 peers, err := raftStorage.Peers(ctx) 272 if err != nil { 273 return nil, err 274 } 275 276 return &logical.Response{ 277 Data: map[string]interface{}{ 278 "peers": peers, 279 "tls_keyring": &keyring, 280 }, 281 }, nil 282 } 283} 284 285func (b *SystemBackend) handleStorageRaftSnapshotRead() framework.OperationFunc { 286 return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { 287 raftStorage, ok := b.Core.underlyingPhysical.(*raft.RaftBackend) 288 if !ok { 289 return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest 290 } 291 if req.ResponseWriter == nil { 292 return nil, errors.New("no writer for request") 293 } 294 295 err := raftStorage.Snapshot(req.ResponseWriter, b.Core.seal.GetAccess()) 296 if err != nil { 297 return nil, err 298 } 299 300 return nil, nil 301 } 302} 303 304func (b *SystemBackend) handleStorageRaftSnapshotWrite(force bool) framework.OperationFunc { 305 return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { 306 raftStorage, ok := b.Core.underlyingPhysical.(*raft.RaftBackend) 307 if !ok { 308 return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest 309 } 310 if req.RequestReader == nil { 311 return nil, errors.New("no reader for request") 312 } 313 314 access := b.Core.seal.GetAccess() 315 if force { 316 access = nil 317 } 318 319 // We want to buffer the http request reader into a temp file here so we 320 // don't have to hold the full snapshot in memory. We also want to do 321 // the restore in two parts so we can restore the snapshot while the 322 // stateLock is write locked. 323 snapFile, cleanup, metadata, err := raftStorage.WriteSnapshotToTemp(req.RequestReader, access) 324 switch { 325 case err == nil: 326 case strings.Contains(err.Error(), "failed to open the sealed hashes"): 327 switch b.Core.seal.BarrierType() { 328 case seal.Shamir: 329 return logical.ErrorResponse("could not verify hash file, possibly the snapshot is using a different set of unseal keys; use the snapshot-force API to bypass this check"), logical.ErrInvalidRequest 330 default: 331 return logical.ErrorResponse("could not verify hash file, possibly the snapshot is using a different autoseal key; use the snapshot-force API to bypass this check"), logical.ErrInvalidRequest 332 } 333 case err != nil: 334 b.Core.logger.Error("raft snapshot restore: failed to write snapshot", "error", err) 335 return nil, err 336 } 337 338 // We want to do this in a go routine so we can upgrade the lock and 339 // allow the client to disconnect. 340 go func() (retErr error) { 341 // Cleanup the temp file 342 defer cleanup() 343 344 // Grab statelock 345 if stopped := grabLockOrStop(b.Core.stateLock.Lock, b.Core.stateLock.Unlock, b.Core.standbyStopCh); stopped { 346 b.Core.logger.Error("not applying snapshot; shutting down") 347 return 348 } 349 defer b.Core.stateLock.Unlock() 350 351 // If we failed to restore the snapshot we should seal this node as 352 // it's in an unknown state 353 defer func() { 354 if retErr != nil { 355 if err := b.Core.sealInternalWithOptions(false, false, true); err != nil { 356 b.Core.logger.Error("failed to seal node", "error", err) 357 } 358 } 359 }() 360 361 ctx, ctxCancel := context.WithCancel(namespace.RootContext(nil)) 362 363 // We are calling the callback function synchronously here while we 364 // have the lock. So set it to nil and restore the callback when we 365 // finish. 366 raftStorage.SetRestoreCallback(nil) 367 defer raftStorage.SetRestoreCallback(b.Core.raftSnapshotRestoreCallback(true, true)) 368 369 // Do a preSeal to clear vault's in-memory caches and shut down any 370 // systems that might be holding the encryption access. 371 b.Core.logger.Info("shutting down prior to restoring snapshot") 372 if err := b.Core.preSeal(); err != nil { 373 b.Core.logger.Error("raft snapshot restore failed preSeal", "error", err) 374 return err 375 } 376 377 b.Core.logger.Info("applying snapshot") 378 if err := raftStorage.RestoreSnapshot(ctx, metadata, snapFile); err != nil { 379 b.Core.logger.Error("error while restoring raft snapshot", "error", err) 380 return err 381 } 382 383 // Run invalidation logic synchronously here 384 callback := b.Core.raftSnapshotRestoreCallback(false, false) 385 if err := callback(ctx); err != nil { 386 return err 387 } 388 389 { 390 // If the snapshot was taken while another node was leader we 391 // need to reset the leader information to this node. 392 if err := b.Core.underlyingPhysical.Put(ctx, &physical.Entry{ 393 Key: CoreLockPath, 394 Value: []byte(b.Core.leaderUUID), 395 }); err != nil { 396 b.Core.logger.Error("cluster setup failed", "error", err) 397 return err 398 } 399 // re-advertise our cluster information 400 if err := b.Core.advertiseLeader(ctx, b.Core.leaderUUID, nil); err != nil { 401 b.Core.logger.Error("cluster setup failed", "error", err) 402 return err 403 } 404 } 405 if err := b.Core.postUnseal(ctx, ctxCancel, standardUnsealStrategy{}); err != nil { 406 b.Core.logger.Error("raft snapshot restore failed postUnseal", "error", err) 407 return err 408 } 409 410 return nil 411 412 }() 413 414 return nil, nil 415 } 416} 417 418var sysRaftHelp = map[string][2]string{ 419 "raft-bootstrap-challenge": { 420 "Creates a challenge for the new peer to be joined to the raft cluster.", 421 "", 422 }, 423 "raft-bootstrap-answer": { 424 "Accepts an answer from the peer to be joined to the fact cluster.", 425 "", 426 }, 427 "raft-remove-peer": { 428 "Removes a peer from the raft cluster.", 429 "", 430 }, 431} 432