1package kubernetes 2 3import ( 4 "bytes" 5 "compress/gzip" 6 "crypto/md5" 7 "encoding/base64" 8 "encoding/json" 9 "errors" 10 "fmt" 11 "strings" 12 13 "github.com/hashicorp/terraform/internal/states/remote" 14 "github.com/hashicorp/terraform/internal/states/statemgr" 15 k8serrors "k8s.io/apimachinery/pkg/api/errors" 16 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 17 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 18 "k8s.io/apimachinery/pkg/util/validation" 19 "k8s.io/client-go/dynamic" 20 "k8s.io/utils/pointer" 21 22 coordinationv1 "k8s.io/api/coordination/v1" 23 coordinationclientv1 "k8s.io/client-go/kubernetes/typed/coordination/v1" 24) 25 26const ( 27 tfstateKey = "tfstate" 28 tfstateSecretSuffixKey = "tfstateSecretSuffix" 29 tfstateWorkspaceKey = "tfstateWorkspace" 30 tfstateLockInfoAnnotation = "app.terraform.io/lock-info" 31 managedByKey = "app.kubernetes.io/managed-by" 32) 33 34type RemoteClient struct { 35 kubernetesSecretClient dynamic.ResourceInterface 36 kubernetesLeaseClient coordinationclientv1.LeaseInterface 37 namespace string 38 labels map[string]string 39 nameSuffix string 40 workspace string 41} 42 43func (c *RemoteClient) Get() (payload *remote.Payload, err error) { 44 secretName, err := c.createSecretName() 45 if err != nil { 46 return nil, err 47 } 48 secret, err := c.kubernetesSecretClient.Get(secretName, metav1.GetOptions{}) 49 if err != nil { 50 if k8serrors.IsNotFound(err) { 51 return nil, nil 52 } 53 return nil, err 54 } 55 56 secretData := getSecretData(secret) 57 stateRaw, ok := secretData[tfstateKey] 58 if !ok { 59 // The secret exists but there is no state in it 60 return nil, nil 61 } 62 63 stateRawString := stateRaw.(string) 64 65 state, err := uncompressState(stateRawString) 66 if err != nil { 67 return nil, err 68 } 69 70 md5 := md5.Sum(state) 71 72 p := &remote.Payload{ 73 Data: state, 74 MD5: md5[:], 75 } 76 return p, nil 77} 78 79func (c *RemoteClient) Put(data []byte) error { 80 secretName, err := c.createSecretName() 81 if err != nil { 82 return err 83 } 84 85 payload, err := compressState(data) 86 if err != nil { 87 return err 88 } 89 90 secret, err := c.getSecret(secretName) 91 if err != nil { 92 if !k8serrors.IsNotFound(err) { 93 return err 94 } 95 96 secret = &unstructured.Unstructured{ 97 Object: map[string]interface{}{ 98 "metadata": metav1.ObjectMeta{ 99 Name: secretName, 100 Namespace: c.namespace, 101 Labels: c.getLabels(), 102 Annotations: map[string]string{"encoding": "gzip"}, 103 }, 104 }, 105 } 106 107 secret, err = c.kubernetesSecretClient.Create(secret, metav1.CreateOptions{}) 108 if err != nil { 109 return err 110 } 111 } 112 113 setState(secret, payload) 114 _, err = c.kubernetesSecretClient.Update(secret, metav1.UpdateOptions{}) 115 return err 116} 117 118// Delete the state secret 119func (c *RemoteClient) Delete() error { 120 secretName, err := c.createSecretName() 121 if err != nil { 122 return err 123 } 124 125 err = c.deleteSecret(secretName) 126 if err != nil { 127 if !k8serrors.IsNotFound(err) { 128 return err 129 } 130 } 131 132 leaseName, err := c.createLeaseName() 133 if err != nil { 134 return err 135 } 136 137 err = c.deleteLease(leaseName) 138 if err != nil { 139 if !k8serrors.IsNotFound(err) { 140 return err 141 } 142 } 143 return nil 144} 145 146func (c *RemoteClient) Lock(info *statemgr.LockInfo) (string, error) { 147 leaseName, err := c.createLeaseName() 148 if err != nil { 149 return "", err 150 } 151 152 lease, err := c.getLease(leaseName) 153 if err != nil { 154 if !k8serrors.IsNotFound(err) { 155 return "", err 156 } 157 158 labels := c.getLabels() 159 lease = &coordinationv1.Lease{ 160 ObjectMeta: metav1.ObjectMeta{ 161 Name: leaseName, 162 Labels: labels, 163 Annotations: map[string]string{ 164 tfstateLockInfoAnnotation: string(info.Marshal()), 165 }, 166 }, 167 Spec: coordinationv1.LeaseSpec{ 168 HolderIdentity: pointer.StringPtr(info.ID), 169 }, 170 } 171 172 _, err = c.kubernetesLeaseClient.Create(lease) 173 if err != nil { 174 return "", err 175 } else { 176 return info.ID, nil 177 } 178 } 179 180 if lease.Spec.HolderIdentity != nil { 181 if *lease.Spec.HolderIdentity == info.ID { 182 return info.ID, nil 183 } 184 185 currentLockInfo, err := c.getLockInfo(lease) 186 if err != nil { 187 return "", err 188 } 189 190 lockErr := &statemgr.LockError{ 191 Info: currentLockInfo, 192 Err: errors.New("the state is already locked by another terraform client"), 193 } 194 return "", lockErr 195 } 196 197 lease.Spec.HolderIdentity = pointer.StringPtr(info.ID) 198 setLockInfo(lease, info.Marshal()) 199 _, err = c.kubernetesLeaseClient.Update(lease) 200 if err != nil { 201 return "", err 202 } 203 204 return info.ID, err 205} 206 207func (c *RemoteClient) Unlock(id string) error { 208 leaseName, err := c.createLeaseName() 209 if err != nil { 210 return err 211 } 212 213 lease, err := c.getLease(leaseName) 214 if err != nil { 215 return err 216 } 217 218 if lease.Spec.HolderIdentity == nil { 219 return fmt.Errorf("state is already unlocked") 220 } 221 222 lockInfo, err := c.getLockInfo(lease) 223 if err != nil { 224 return err 225 } 226 227 lockErr := &statemgr.LockError{Info: lockInfo} 228 if *lease.Spec.HolderIdentity != id { 229 lockErr.Err = fmt.Errorf("lock id %q does not match existing lock", id) 230 return lockErr 231 } 232 233 lease.Spec.HolderIdentity = nil 234 removeLockInfo(lease) 235 236 _, err = c.kubernetesLeaseClient.Update(lease) 237 if err != nil { 238 lockErr.Err = err 239 return lockErr 240 } 241 242 return nil 243} 244 245func (c *RemoteClient) getLockInfo(lease *coordinationv1.Lease) (*statemgr.LockInfo, error) { 246 lockData, ok := getLockInfo(lease) 247 if len(lockData) == 0 || !ok { 248 return nil, nil 249 } 250 251 lockInfo := &statemgr.LockInfo{} 252 err := json.Unmarshal(lockData, lockInfo) 253 if err != nil { 254 return nil, err 255 } 256 257 return lockInfo, nil 258} 259 260func (c *RemoteClient) getLabels() map[string]string { 261 l := map[string]string{ 262 tfstateKey: "true", 263 tfstateSecretSuffixKey: c.nameSuffix, 264 tfstateWorkspaceKey: c.workspace, 265 managedByKey: "terraform", 266 } 267 268 if len(c.labels) != 0 { 269 for k, v := range c.labels { 270 l[k] = v 271 } 272 } 273 274 return l 275} 276 277func (c *RemoteClient) getSecret(name string) (*unstructured.Unstructured, error) { 278 return c.kubernetesSecretClient.Get(name, metav1.GetOptions{}) 279} 280 281func (c *RemoteClient) getLease(name string) (*coordinationv1.Lease, error) { 282 return c.kubernetesLeaseClient.Get(name, metav1.GetOptions{}) 283} 284 285func (c *RemoteClient) deleteSecret(name string) error { 286 secret, err := c.getSecret(name) 287 if err != nil { 288 return err 289 } 290 291 labels := secret.GetLabels() 292 v, ok := labels[tfstateKey] 293 if !ok || v != "true" { 294 return fmt.Errorf("Secret does does not have %q label", tfstateKey) 295 } 296 297 delProp := metav1.DeletePropagationBackground 298 delOps := &metav1.DeleteOptions{PropagationPolicy: &delProp} 299 return c.kubernetesSecretClient.Delete(name, delOps) 300} 301 302func (c *RemoteClient) deleteLease(name string) error { 303 secret, err := c.getLease(name) 304 if err != nil { 305 return err 306 } 307 308 labels := secret.GetLabels() 309 v, ok := labels[tfstateKey] 310 if !ok || v != "true" { 311 return fmt.Errorf("Lease does does not have %q label", tfstateKey) 312 } 313 314 delProp := metav1.DeletePropagationBackground 315 delOps := &metav1.DeleteOptions{PropagationPolicy: &delProp} 316 return c.kubernetesLeaseClient.Delete(name, delOps) 317} 318 319func (c *RemoteClient) createSecretName() (string, error) { 320 secretName := strings.Join([]string{tfstateKey, c.workspace, c.nameSuffix}, "-") 321 322 errs := validation.IsDNS1123Subdomain(secretName) 323 if len(errs) > 0 { 324 k8sInfo := ` 325This is a requirement for Kubernetes secret names. 326The workspace name and key must adhere to Kubernetes naming conventions.` 327 msg := fmt.Sprintf("the secret name %v is invalid, ", secretName) 328 return "", errors.New(msg + strings.Join(errs, ",") + k8sInfo) 329 } 330 331 return secretName, nil 332} 333 334func (c *RemoteClient) createLeaseName() (string, error) { 335 n, err := c.createSecretName() 336 if err != nil { 337 return "", err 338 } 339 return "lock-" + n, nil 340} 341 342func compressState(data []byte) ([]byte, error) { 343 b := new(bytes.Buffer) 344 gz := gzip.NewWriter(b) 345 if _, err := gz.Write(data); err != nil { 346 return nil, err 347 } 348 if err := gz.Close(); err != nil { 349 return nil, err 350 } 351 return b.Bytes(), nil 352} 353 354func uncompressState(data string) ([]byte, error) { 355 decode, err := base64.StdEncoding.DecodeString(data) 356 if err != nil { 357 return nil, err 358 } 359 360 b := new(bytes.Buffer) 361 gz, err := gzip.NewReader(bytes.NewReader(decode)) 362 if err != nil { 363 return nil, err 364 } 365 b.ReadFrom(gz) 366 if err := gz.Close(); err != nil { 367 return nil, err 368 } 369 return b.Bytes(), nil 370} 371 372func getSecretData(secret *unstructured.Unstructured) map[string]interface{} { 373 if m, ok := secret.Object["data"].(map[string]interface{}); ok { 374 return m 375 } 376 return map[string]interface{}{} 377} 378 379func getLockInfo(lease *coordinationv1.Lease) ([]byte, bool) { 380 info, ok := lease.ObjectMeta.GetAnnotations()[tfstateLockInfoAnnotation] 381 if !ok { 382 return nil, false 383 } 384 return []byte(info), true 385} 386 387func setLockInfo(lease *coordinationv1.Lease, l []byte) { 388 annotations := lease.ObjectMeta.GetAnnotations() 389 if annotations != nil { 390 annotations[tfstateLockInfoAnnotation] = string(l) 391 } else { 392 annotations = map[string]string{ 393 tfstateLockInfoAnnotation: string(l), 394 } 395 } 396 lease.ObjectMeta.SetAnnotations(annotations) 397} 398 399func removeLockInfo(lease *coordinationv1.Lease) { 400 annotations := lease.ObjectMeta.GetAnnotations() 401 delete(annotations, tfstateLockInfoAnnotation) 402 lease.ObjectMeta.SetAnnotations(annotations) 403} 404 405func setState(secret *unstructured.Unstructured, t []byte) { 406 secretData := getSecretData(secret) 407 secretData[tfstateKey] = t 408 secret.Object["data"] = secretData 409} 410