1package swift 2 3import ( 4 "context" 5 "fmt" 6 "os" 7 "sort" 8 "strconv" 9 "strings" 10 "time" 11 12 log "github.com/hashicorp/go-hclog" 13 14 metrics "github.com/armon/go-metrics" 15 "github.com/hashicorp/errwrap" 16 cleanhttp "github.com/hashicorp/go-cleanhttp" 17 "github.com/hashicorp/vault/sdk/helper/strutil" 18 "github.com/hashicorp/vault/sdk/physical" 19 "github.com/ncw/swift" 20) 21 22// Verify SwiftBackend satisfies the correct interfaces 23var _ physical.Backend = (*SwiftBackend)(nil) 24 25// SwiftBackend is a physical backend that stores data 26// within an OpenStack Swift container. 27type SwiftBackend struct { 28 container string 29 client *swift.Connection 30 logger log.Logger 31 permitPool *physical.PermitPool 32} 33 34// NewSwiftBackend constructs a Swift backend using a pre-existing 35// container. Credentials can be provided to the backend, sourced 36// from the environment. 37func NewSwiftBackend(conf map[string]string, logger log.Logger) (physical.Backend, error) { 38 var ok bool 39 40 username := os.Getenv("OS_USERNAME") 41 if username == "" { 42 username = conf["username"] 43 if username == "" { 44 return nil, fmt.Errorf("missing username") 45 } 46 } 47 password := os.Getenv("OS_PASSWORD") 48 if password == "" { 49 password = conf["password"] 50 if password == "" { 51 return nil, fmt.Errorf("missing password") 52 } 53 } 54 authUrl := os.Getenv("OS_AUTH_URL") 55 if authUrl == "" { 56 authUrl = conf["auth_url"] 57 if authUrl == "" { 58 return nil, fmt.Errorf("missing auth_url") 59 } 60 } 61 container := os.Getenv("OS_CONTAINER") 62 if container == "" { 63 container = conf["container"] 64 if container == "" { 65 return nil, fmt.Errorf("missing container") 66 } 67 } 68 project := os.Getenv("OS_PROJECT_NAME") 69 if project == "" { 70 if project, ok = conf["project"]; !ok { 71 // Check for KeyStone naming prior to V3 72 project = os.Getenv("OS_TENANT_NAME") 73 if project == "" { 74 project = conf["tenant"] 75 } 76 } 77 } 78 79 domain := os.Getenv("OS_USER_DOMAIN_NAME") 80 if domain == "" { 81 domain = conf["domain"] 82 } 83 projectDomain := os.Getenv("OS_PROJECT_DOMAIN_NAME") 84 if projectDomain == "" { 85 projectDomain = conf["project-domain"] 86 } 87 88 region := os.Getenv("OS_REGION_NAME") 89 if region == "" { 90 region = conf["region"] 91 } 92 tenantID := os.Getenv("OS_TENANT_ID") 93 if tenantID == "" { 94 tenantID = conf["tenant_id"] 95 } 96 trustID := os.Getenv("OS_TRUST_ID") 97 if trustID == "" { 98 trustID = conf["trust_id"] 99 } 100 storageUrl := os.Getenv("OS_STORAGE_URL") 101 if storageUrl == "" { 102 storageUrl = conf["storage_url"] 103 } 104 authToken := os.Getenv("OS_AUTH_TOKEN") 105 if authToken == "" { 106 authToken = conf["auth_token"] 107 } 108 109 c := swift.Connection{ 110 Domain: domain, 111 UserName: username, 112 ApiKey: password, 113 AuthUrl: authUrl, 114 Tenant: project, 115 TenantDomain: projectDomain, 116 Region: region, 117 TenantId: tenantID, 118 TrustId: trustID, 119 StorageUrl: storageUrl, 120 AuthToken: authToken, 121 Transport: cleanhttp.DefaultPooledTransport(), 122 } 123 124 err := c.Authenticate() 125 if err != nil { 126 return nil, err 127 } 128 129 _, _, err = c.Container(container) 130 if err != nil { 131 return nil, errwrap.Wrapf(fmt.Sprintf("Unable to access container %q: {{err}}", container), err) 132 } 133 134 maxParStr, ok := conf["max_parallel"] 135 var maxParInt int 136 if ok { 137 maxParInt, err = strconv.Atoi(maxParStr) 138 if err != nil { 139 return nil, errwrap.Wrapf("failed parsing max_parallel parameter: {{err}}", err) 140 } 141 if logger.IsDebug() { 142 logger.Debug("max_parallel set", "max_parallel", maxParInt) 143 } 144 } 145 146 s := &SwiftBackend{ 147 client: &c, 148 container: container, 149 logger: logger, 150 permitPool: physical.NewPermitPool(maxParInt), 151 } 152 return s, nil 153} 154 155// Put is used to insert or update an entry 156func (s *SwiftBackend) Put(ctx context.Context, entry *physical.Entry) error { 157 defer metrics.MeasureSince([]string{"swift", "put"}, time.Now()) 158 159 s.permitPool.Acquire() 160 defer s.permitPool.Release() 161 162 err := s.client.ObjectPutBytes(s.container, entry.Key, entry.Value, "") 163 164 if err != nil { 165 return err 166 } 167 168 return nil 169} 170 171// Get is used to fetch an entry 172func (s *SwiftBackend) Get(ctx context.Context, key string) (*physical.Entry, error) { 173 defer metrics.MeasureSince([]string{"swift", "get"}, time.Now()) 174 175 s.permitPool.Acquire() 176 defer s.permitPool.Release() 177 178 //Do a list of names with the key first since eventual consistency means 179 //it might be deleted, but a node might return a read of bytes which fails 180 //the physical test 181 list, err := s.client.ObjectNames(s.container, &swift.ObjectsOpts{Prefix: key}) 182 if err != nil { 183 return nil, err 184 } 185 if 0 == len(list) { 186 return nil, nil 187 } 188 data, err := s.client.ObjectGetBytes(s.container, key) 189 if err == swift.ObjectNotFound { 190 return nil, nil 191 } 192 if err != nil { 193 return nil, err 194 } 195 ent := &physical.Entry{ 196 Key: key, 197 Value: data, 198 } 199 200 return ent, nil 201} 202 203// Delete is used to permanently delete an entry 204func (s *SwiftBackend) Delete(ctx context.Context, key string) error { 205 defer metrics.MeasureSince([]string{"swift", "delete"}, time.Now()) 206 207 s.permitPool.Acquire() 208 defer s.permitPool.Release() 209 210 err := s.client.ObjectDelete(s.container, key) 211 212 if err != nil && err != swift.ObjectNotFound { 213 return err 214 } 215 216 return nil 217} 218 219// List is used to list all the keys under a given 220// prefix, up to the next prefix. 221func (s *SwiftBackend) List(ctx context.Context, prefix string) ([]string, error) { 222 defer metrics.MeasureSince([]string{"swift", "list"}, time.Now()) 223 224 s.permitPool.Acquire() 225 defer s.permitPool.Release() 226 227 list, err := s.client.ObjectNamesAll(s.container, &swift.ObjectsOpts{Prefix: prefix}) 228 if nil != err { 229 return nil, err 230 } 231 232 keys := []string{} 233 for _, key := range list { 234 key := strings.TrimPrefix(key, prefix) 235 236 if i := strings.Index(key, "/"); i == -1 { 237 // Add objects only from the current 'folder' 238 keys = append(keys, key) 239 } else if i != -1 { 240 // Add truncated 'folder' paths 241 keys = strutil.AppendIfMissing(keys, key[:i+1]) 242 } 243 } 244 245 sort.Strings(keys) 246 247 return keys, nil 248} 249