1package api 2 3import ( 4 "fmt" 5 "net/url" 6 "sort" 7 "time" 8) 9 10// CSIVolumes is used to access Container Storage Interface (CSI) endpoints. 11type CSIVolumes struct { 12 client *Client 13} 14 15// CSIVolumes returns a handle on the CSIVolumes endpoint. 16func (c *Client) CSIVolumes() *CSIVolumes { 17 return &CSIVolumes{client: c} 18} 19 20// List returns all CSI volumes. 21func (v *CSIVolumes) List(q *QueryOptions) ([]*CSIVolumeListStub, *QueryMeta, error) { 22 var resp []*CSIVolumeListStub 23 qm, err := v.client.query("/v1/volumes?type=csi", &resp, q) 24 if err != nil { 25 return nil, nil, err 26 } 27 sort.Sort(CSIVolumeIndexSort(resp)) 28 return resp, qm, nil 29} 30 31// ListExternal returns all CSI volumes, as understood by the external storage 32// provider. These volumes may or may not be currently registered with Nomad. 33// The response is paginated by the plugin and accepts the 34// QueryOptions.PerPage and QueryOptions.NextToken fields. 35func (v *CSIVolumes) ListExternal(pluginID string, q *QueryOptions) (*CSIVolumeListExternalResponse, *QueryMeta, error) { 36 var resp *CSIVolumeListExternalResponse 37 38 qp := url.Values{} 39 qp.Set("plugin_id", pluginID) 40 if q.NextToken != "" { 41 qp.Set("next_token", q.NextToken) 42 } 43 if q.PerPage != 0 { 44 qp.Set("per_page", fmt.Sprint(q.PerPage)) 45 } 46 47 qm, err := v.client.query("/v1/volumes/external?"+qp.Encode(), &resp, q) 48 if err != nil { 49 return nil, nil, err 50 } 51 52 sort.Sort(CSIVolumeExternalStubSort(resp.Volumes)) 53 return resp, qm, nil 54} 55 56// PluginList returns all CSI volumes for the specified plugin id 57func (v *CSIVolumes) PluginList(pluginID string) ([]*CSIVolumeListStub, *QueryMeta, error) { 58 return v.List(&QueryOptions{Prefix: pluginID}) 59} 60 61// Info is used to retrieve a single CSIVolume 62func (v *CSIVolumes) Info(id string, q *QueryOptions) (*CSIVolume, *QueryMeta, error) { 63 var resp CSIVolume 64 qm, err := v.client.query("/v1/volume/csi/"+id, &resp, q) 65 if err != nil { 66 return nil, nil, err 67 } 68 69 return &resp, qm, nil 70} 71 72// Register registers a single CSIVolume with Nomad. The volume must already 73// exist in the external storage provider. 74func (v *CSIVolumes) Register(vol *CSIVolume, w *WriteOptions) (*WriteMeta, error) { 75 req := CSIVolumeRegisterRequest{ 76 Volumes: []*CSIVolume{vol}, 77 } 78 meta, err := v.client.write("/v1/volume/csi/"+vol.ID, req, nil, w) 79 return meta, err 80} 81 82// Deregister deregisters a single CSIVolume from Nomad. The volume will not be deleted from the external storage provider. 83func (v *CSIVolumes) Deregister(id string, force bool, w *WriteOptions) error { 84 _, err := v.client.delete(fmt.Sprintf("/v1/volume/csi/%v?force=%t", url.PathEscape(id), force), nil, w) 85 return err 86} 87 88// Create creates a single CSIVolume in an external storage provider and 89// registers it with Nomad. You do not need to call Register if this call is 90// successful. 91func (v *CSIVolumes) Create(vol *CSIVolume, w *WriteOptions) ([]*CSIVolume, *WriteMeta, error) { 92 req := CSIVolumeCreateRequest{ 93 Volumes: []*CSIVolume{vol}, 94 } 95 96 resp := &CSIVolumeCreateResponse{} 97 meta, err := v.client.write(fmt.Sprintf("/v1/volume/csi/%v/create", vol.ID), req, resp, w) 98 return resp.Volumes, meta, err 99} 100 101// Delete deletes a CSI volume from an external storage provider. The ID 102// passed as an argument here is for the storage provider's ID, so a volume 103// that's already been deregistered can be deleted. 104func (v *CSIVolumes) Delete(externalVolID string, w *WriteOptions) error { 105 _, err := v.client.delete(fmt.Sprintf("/v1/volume/csi/%v/delete", url.PathEscape(externalVolID)), nil, w) 106 return err 107} 108 109// Detach causes Nomad to attempt to detach a CSI volume from a client 110// node. This is used in the case that the node is temporarily lost and the 111// allocations are unable to drop their claims automatically. 112func (v *CSIVolumes) Detach(volID, nodeID string, w *WriteOptions) error { 113 _, err := v.client.delete(fmt.Sprintf("/v1/volume/csi/%v/detach?node=%v", url.PathEscape(volID), nodeID), nil, w) 114 return err 115} 116 117// CreateSnapshot snapshots an external storage volume. 118func (v *CSIVolumes) CreateSnapshot(snap *CSISnapshot, w *WriteOptions) (*CSISnapshotCreateResponse, *WriteMeta, error) { 119 req := &CSISnapshotCreateRequest{ 120 Snapshots: []*CSISnapshot{snap}, 121 } 122 resp := &CSISnapshotCreateResponse{} 123 meta, err := v.client.write("/v1/volumes/snapshot", req, resp, w) 124 return resp, meta, err 125} 126 127// DeleteSnapshot deletes an external storage volume snapshot. 128func (v *CSIVolumes) DeleteSnapshot(snap *CSISnapshot, w *WriteOptions) error { 129 qp := url.Values{} 130 qp.Set("snapshot_id", snap.ID) 131 qp.Set("plugin_id", snap.PluginID) 132 for k, v := range snap.Secrets { 133 qp.Set("secret", fmt.Sprintf("%v=%v", k, v)) 134 } 135 _, err := v.client.delete("/v1/volumes/snapshot?"+qp.Encode(), nil, w) 136 return err 137} 138 139// ListSnapshots lists external storage volume snapshots. 140func (v *CSIVolumes) ListSnapshots(pluginID string, q *QueryOptions) (*CSISnapshotListResponse, *QueryMeta, error) { 141 var resp *CSISnapshotListResponse 142 143 qp := url.Values{} 144 if pluginID != "" { 145 qp.Set("plugin_id", pluginID) 146 } 147 if q.NextToken != "" { 148 qp.Set("next_token", q.NextToken) 149 } 150 if q.PerPage != 0 { 151 qp.Set("per_page", fmt.Sprint(q.PerPage)) 152 } 153 154 qm, err := v.client.query("/v1/volumes/snapshot?"+qp.Encode(), &resp, q) 155 if err != nil { 156 return nil, nil, err 157 } 158 159 sort.Sort(CSISnapshotSort(resp.Snapshots)) 160 return resp, qm, nil 161} 162 163// CSIVolumeAttachmentMode chooses the type of storage api that will be used to 164// interact with the device. (Duplicated in nomad/structs/csi.go) 165type CSIVolumeAttachmentMode string 166 167const ( 168 CSIVolumeAttachmentModeUnknown CSIVolumeAttachmentMode = "" 169 CSIVolumeAttachmentModeBlockDevice CSIVolumeAttachmentMode = "block-device" 170 CSIVolumeAttachmentModeFilesystem CSIVolumeAttachmentMode = "file-system" 171) 172 173// CSIVolumeAccessMode indicates how a volume should be used in a storage topology 174// e.g whether the provider should make the volume available concurrently. (Duplicated in nomad/structs/csi.go) 175type CSIVolumeAccessMode string 176 177const ( 178 CSIVolumeAccessModeUnknown CSIVolumeAccessMode = "" 179 CSIVolumeAccessModeSingleNodeReader CSIVolumeAccessMode = "single-node-reader-only" 180 CSIVolumeAccessModeSingleNodeWriter CSIVolumeAccessMode = "single-node-writer" 181 CSIVolumeAccessModeMultiNodeReader CSIVolumeAccessMode = "multi-node-reader-only" 182 CSIVolumeAccessModeMultiNodeSingleWriter CSIVolumeAccessMode = "multi-node-single-writer" 183 CSIVolumeAccessModeMultiNodeMultiWriter CSIVolumeAccessMode = "multi-node-multi-writer" 184) 185 186// CSIMountOptions contain optional additional configuration that can be used 187// when specifying that a Volume should be used with VolumeAccessTypeMount. 188type CSIMountOptions struct { 189 // FSType is an optional field that allows an operator to specify the type 190 // of the filesystem. 191 FSType string `hcl:"fs_type,optional"` 192 193 // MountFlags contains additional options that may be used when mounting the 194 // volume by the plugin. This may contain sensitive data and should not be 195 // leaked. 196 MountFlags []string `hcl:"mount_flags,optional"` 197 198 ExtraKeysHCL []string `hcl1:",unusedKeys" json:"-"` // report unexpected keys 199} 200 201// CSISecrets contain optional additional credentials that may be needed by 202// the storage provider. These values will be redacted when reported in the 203// API or in Nomad's logs. 204type CSISecrets map[string]string 205 206// CSIVolume is used for serialization, see also nomad/structs/csi.go 207type CSIVolume struct { 208 ID string 209 Name string 210 ExternalID string `mapstructure:"external_id" hcl:"external_id"` 211 Namespace string 212 Topologies []*CSITopology 213 AccessMode CSIVolumeAccessMode `hcl:"access_mode"` 214 AttachmentMode CSIVolumeAttachmentMode `hcl:"attachment_mode"` 215 MountOptions *CSIMountOptions `hcl:"mount_options"` 216 Secrets CSISecrets `mapstructure:"secrets" hcl:"secrets"` 217 Parameters map[string]string `mapstructure:"parameters" hcl:"parameters"` 218 Context map[string]string `mapstructure:"context" hcl:"context"` 219 Capacity int64 `hcl:"-"` 220 221 // These fields are used as part of the volume creation request 222 RequestedCapacityMin int64 `hcl:"capacity_min"` 223 RequestedCapacityMax int64 `hcl:"capacity_max"` 224 RequestedCapabilities []*CSIVolumeCapability `hcl:"capability"` 225 CloneID string `mapstructure:"clone_id" hcl:"clone_id"` 226 SnapshotID string `mapstructure:"snapshot_id" hcl:"snapshot_id"` 227 228 // ReadAllocs is a map of allocation IDs for tracking reader claim status. 229 // The Allocation value will always be nil; clients can populate this data 230 // by iterating over the Allocations field. 231 ReadAllocs map[string]*Allocation 232 233 // WriteAllocs is a map of allocation IDs for tracking writer claim 234 // status. The Allocation value will always be nil; clients can populate 235 // this data by iterating over the Allocations field. 236 WriteAllocs map[string]*Allocation 237 238 // Allocations is a combined list of readers and writers 239 Allocations []*AllocationListStub 240 241 // Schedulable is true if all the denormalized plugin health fields are true 242 Schedulable bool 243 PluginID string `mapstructure:"plugin_id" hcl:"plugin_id"` 244 Provider string 245 ProviderVersion string 246 ControllerRequired bool 247 ControllersHealthy int 248 ControllersExpected int 249 NodesHealthy int 250 NodesExpected int 251 ResourceExhausted time.Time 252 253 CreateIndex uint64 254 ModifyIndex uint64 255 256 // ExtraKeysHCL is used by the hcl parser to report unexpected keys 257 ExtraKeysHCL []string `hcl1:",unusedKeys" json:"-"` 258} 259 260// CSIVolumeCapability is a requested attachment and access mode for a 261// volume 262type CSIVolumeCapability struct { 263 AccessMode CSIVolumeAccessMode `mapstructure:"access_mode" hcl:"access_mode"` 264 AttachmentMode CSIVolumeAttachmentMode `mapstructure:"attachment_mode" hcl:"attachment_mode"` 265} 266 267// CSIVolumeIndexSort is a helper used for sorting volume stubs by creation 268// time. 269type CSIVolumeIndexSort []*CSIVolumeListStub 270 271func (v CSIVolumeIndexSort) Len() int { 272 return len(v) 273} 274 275func (v CSIVolumeIndexSort) Less(i, j int) bool { 276 return v[i].CreateIndex > v[j].CreateIndex 277} 278 279func (v CSIVolumeIndexSort) Swap(i, j int) { 280 v[i], v[j] = v[j], v[i] 281} 282 283// CSIVolumeListStub omits allocations. See also nomad/structs/csi.go 284type CSIVolumeListStub struct { 285 ID string 286 Namespace string 287 Name string 288 ExternalID string 289 Topologies []*CSITopology 290 AccessMode CSIVolumeAccessMode 291 AttachmentMode CSIVolumeAttachmentMode 292 Schedulable bool 293 PluginID string 294 Provider string 295 ControllerRequired bool 296 ControllersHealthy int 297 ControllersExpected int 298 NodesHealthy int 299 NodesExpected int 300 ResourceExhausted time.Time 301 302 CreateIndex uint64 303 ModifyIndex uint64 304} 305 306type CSIVolumeListExternalResponse struct { 307 Volumes []*CSIVolumeExternalStub 308 NextToken string 309} 310 311// CSIVolumeExternalStub is the storage provider's view of a volume, as 312// returned from the controller plugin; all IDs are for external resources 313type CSIVolumeExternalStub struct { 314 ExternalID string 315 CapacityBytes int64 316 VolumeContext map[string]string 317 CloneID string 318 SnapshotID string 319 PublishedExternalNodeIDs []string 320 IsAbnormal bool 321 Status string 322} 323 324// CSIVolumeExternalStubSort is a sorting helper for external volumes. We 325// can't sort these by creation time because we don't get that data back from 326// the storage provider. Sort by External ID within this page. 327type CSIVolumeExternalStubSort []*CSIVolumeExternalStub 328 329func (v CSIVolumeExternalStubSort) Len() int { 330 return len(v) 331} 332 333func (v CSIVolumeExternalStubSort) Less(i, j int) bool { 334 return v[i].ExternalID > v[j].ExternalID 335} 336 337func (v CSIVolumeExternalStubSort) Swap(i, j int) { 338 v[i], v[j] = v[j], v[i] 339} 340 341type CSIVolumeCreateRequest struct { 342 Volumes []*CSIVolume 343 WriteRequest 344} 345 346type CSIVolumeCreateResponse struct { 347 Volumes []*CSIVolume 348 QueryMeta 349} 350 351type CSIVolumeRegisterRequest struct { 352 Volumes []*CSIVolume 353 WriteRequest 354} 355 356type CSIVolumeDeregisterRequest struct { 357 VolumeIDs []string 358 WriteRequest 359} 360 361// CSISnapshot is the storage provider's view of a volume snapshot 362type CSISnapshot struct { 363 ID string // storage provider's ID 364 ExternalSourceVolumeID string // storage provider's ID for volume 365 SizeBytes int64 // value from storage provider 366 CreateTime int64 // value from storage provider 367 IsReady bool // value from storage provider 368 SourceVolumeID string // Nomad volume ID 369 PluginID string // CSI plugin ID 370 371 // These field are only used during snapshot creation and will not be 372 // populated when the snapshot is returned 373 Name string // suggested name of the snapshot, used for creation 374 Secrets CSISecrets // secrets needed to create snapshot 375 Parameters map[string]string // secrets needed to create snapshot 376} 377 378// CSISnapshotSort is a helper used for sorting snapshots by creation time. 379type CSISnapshotSort []*CSISnapshot 380 381func (v CSISnapshotSort) Len() int { 382 return len(v) 383} 384 385func (v CSISnapshotSort) Less(i, j int) bool { 386 return v[i].CreateTime > v[j].CreateTime 387} 388 389func (v CSISnapshotSort) Swap(i, j int) { 390 v[i], v[j] = v[j], v[i] 391} 392 393type CSISnapshotCreateRequest struct { 394 Snapshots []*CSISnapshot 395 WriteRequest 396} 397 398type CSISnapshotCreateResponse struct { 399 Snapshots []*CSISnapshot 400 QueryMeta 401} 402 403// CSISnapshotListRequest is a request to a controller plugin to list all the 404// snapshot known to the the storage provider. This request is paginated by 405// the plugin and accepts the QueryOptions.PerPage and QueryOptions.NextToken 406// fields 407type CSISnapshotListRequest struct { 408 PluginID string 409 QueryOptions 410} 411 412type CSISnapshotListResponse struct { 413 Snapshots []*CSISnapshot 414 NextToken string 415 QueryMeta 416} 417 418// CSI Plugins are jobs with plugin specific data 419type CSIPlugins struct { 420 client *Client 421} 422 423// CSIPlugin is used for serialization, see also nomad/structs/csi.go 424type CSIPlugin struct { 425 ID string 426 Provider string 427 Version string 428 ControllerRequired bool 429 // Map Node.ID to CSIInfo fingerprint results 430 Controllers map[string]*CSIInfo 431 Nodes map[string]*CSIInfo 432 Allocations []*AllocationListStub 433 ControllersHealthy int 434 ControllersExpected int 435 NodesHealthy int 436 NodesExpected int 437 CreateIndex uint64 438 ModifyIndex uint64 439} 440 441type CSIPluginListStub struct { 442 ID string 443 Provider string 444 ControllerRequired bool 445 ControllersHealthy int 446 ControllersExpected int 447 NodesHealthy int 448 NodesExpected int 449 CreateIndex uint64 450 ModifyIndex uint64 451} 452 453// CSIPluginIndexSort is a helper used for sorting plugin stubs by creation 454// time. 455type CSIPluginIndexSort []*CSIPluginListStub 456 457func (v CSIPluginIndexSort) Len() int { 458 return len(v) 459} 460 461func (v CSIPluginIndexSort) Less(i, j int) bool { 462 return v[i].CreateIndex > v[j].CreateIndex 463} 464 465func (v CSIPluginIndexSort) Swap(i, j int) { 466 v[i], v[j] = v[j], v[i] 467} 468 469// CSIPlugins returns a handle on the CSIPlugins endpoint 470func (c *Client) CSIPlugins() *CSIPlugins { 471 return &CSIPlugins{client: c} 472} 473 474// List returns all CSI plugins 475func (v *CSIPlugins) List(q *QueryOptions) ([]*CSIPluginListStub, *QueryMeta, error) { 476 var resp []*CSIPluginListStub 477 qm, err := v.client.query("/v1/plugins?type=csi", &resp, q) 478 if err != nil { 479 return nil, nil, err 480 } 481 sort.Sort(CSIPluginIndexSort(resp)) 482 return resp, qm, nil 483} 484 485// Info is used to retrieve a single CSI Plugin Job 486func (v *CSIPlugins) Info(id string, q *QueryOptions) (*CSIPlugin, *QueryMeta, error) { 487 var resp *CSIPlugin 488 qm, err := v.client.query("/v1/plugin/csi/"+id, &resp, q) 489 if err != nil { 490 return nil, nil, err 491 } 492 return resp, qm, nil 493} 494