1// Package storage implements operations for s3 and fs. 2package storage 3 4import ( 5 "context" 6 "encoding/json" 7 "fmt" 8 "os" 9 "time" 10 11 "github.com/peak/s5cmd/storage/url" 12 "github.com/peak/s5cmd/strutil" 13) 14 15var ( 16 // ErrGivenObjectNotFound indicates a specified object is not found. 17 ErrGivenObjectNotFound = fmt.Errorf("given object not found") 18 19 // ErrNoObjectFound indicates there are no objects found from a given directory. 20 ErrNoObjectFound = fmt.Errorf("no object found") 21) 22 23// Storage is an interface for storage operations that is common 24// to local filesystem and remote object storage. 25type Storage interface { 26 // Stat returns the Object structure describing object. If src is not 27 // found, ErrGivenObjectNotFound is returned. 28 Stat(ctx context.Context, src *url.URL) (*Object, error) 29 30 // List the objects and directories/prefixes in the src. 31 List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object 32 33 // Delete deletes the given src. 34 Delete(ctx context.Context, src *url.URL) error 35 36 // MultiDelete deletes all items returned from given urls in batches. 37 MultiDelete(ctx context.Context, urls <-chan *url.URL) <-chan *Object 38 39 // Copy src to dst, optionally setting the given metadata. Src and dst 40 // arguments are of the same type. If src is a remote type, server side 41 // copying will be used. 42 Copy(ctx context.Context, src, dst *url.URL, metadata Metadata) error 43} 44 45func NewLocalClient(opts Options) *Filesystem { 46 return &Filesystem{dryRun: opts.DryRun} 47} 48 49func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, error) { 50 newOpts := Options{ 51 MaxRetries: opts.MaxRetries, 52 Endpoint: opts.Endpoint, 53 NoVerifySSL: opts.NoVerifySSL, 54 DryRun: opts.DryRun, 55 NoSignRequest: opts.NoSignRequest, 56 bucket: url.Bucket, 57 region: opts.region, 58 } 59 return newS3Storage(ctx, newOpts) 60} 61 62func NewClient(ctx context.Context, url *url.URL, opts Options) (Storage, error) { 63 if url.IsRemote() { 64 return NewRemoteClient(ctx, url, opts) 65 } 66 return NewLocalClient(opts), nil 67} 68 69// Options stores configuration for storage. 70type Options struct { 71 MaxRetries int 72 Endpoint string 73 NoVerifySSL bool 74 DryRun bool 75 NoSignRequest bool 76 bucket string 77 region string 78} 79 80func (o *Options) SetRegion(region string) { 81 o.region = region 82} 83 84// Object is a generic type which contains metadata for storage items. 85type Object struct { 86 URL *url.URL `json:"key,omitempty"` 87 Etag string `json:"etag,omitempty"` 88 ModTime *time.Time `json:"last_modified,omitempty"` 89 Type ObjectType `json:"type,omitempty"` 90 Size int64 `json:"size,omitempty"` 91 StorageClass StorageClass `json:"storage_class,omitempty"` 92 Err error `json:"error,omitempty"` 93} 94 95// String returns the string representation of Object. 96func (o *Object) String() string { 97 return o.URL.String() 98} 99 100// JSON returns the JSON representation of Object. 101func (o *Object) JSON() string { 102 return strutil.JSON(o) 103} 104 105// ObjectType is the type of Object. 106type ObjectType struct { 107 mode os.FileMode 108} 109 110// String returns the string representation of ObjectType. 111func (o ObjectType) String() string { 112 switch mode := o.mode; { 113 case mode.IsRegular(): 114 return "file" 115 case mode.IsDir(): 116 return "directory" 117 case mode&os.ModeSymlink != 0: 118 return "symlink" 119 } 120 return "" 121} 122 123// MarshalJSON returns the stringer of ObjectType as a marshalled json. 124func (o ObjectType) MarshalJSON() ([]byte, error) { 125 return json.Marshal(o.String()) 126} 127 128// IsDir checks if the object is a directory. 129func (o ObjectType) IsDir() bool { 130 return o.mode.IsDir() 131} 132 133// IsSymlink checks if the object is a symbolic link. 134func (o ObjectType) IsSymlink() bool { 135 return o.mode&os.ModeSymlink != 0 136} 137 138// ShouldProcessUrl returns true if follow symlinks is enabled. 139// If follow symlinks is disabled we should not process the url. 140// (this check is needed only for local files) 141func ShouldProcessUrl(url *url.URL, followSymlinks bool) bool { 142 if followSymlinks { 143 return true 144 } 145 146 if url.IsRemote() { 147 return true 148 } 149 fi, err := os.Lstat(url.Absolute()) 150 if err != nil { 151 return false 152 } 153 154 // do not process symlinks 155 return fi.Mode()&os.ModeSymlink == 0 156} 157 158// dateFormat is a constant time template for the bucket. 159const dateFormat = "2006/01/02 15:04:05" 160 161// Bucket is a container for storage objects. 162type Bucket struct { 163 CreationDate time.Time `json:"created_at"` 164 Name string `json:"name"` 165} 166 167// String returns the string representation of Bucket. 168func (b Bucket) String() string { 169 return fmt.Sprintf("%s s3://%s", b.CreationDate.Format(dateFormat), b.Name) 170} 171 172// JSON returns the JSON representation of Bucket. 173func (b Bucket) JSON() string { 174 return strutil.JSON(b) 175} 176 177// StorageClass represents the storage used to store an object. 178type StorageClass string 179 180func (s StorageClass) IsGlacier() bool { 181 return s == "GLACIER" 182} 183 184// notImplemented is a structure which is used on the unsupported operations. 185type notImplemented struct { 186 apiType string 187 method string 188} 189 190// Error returns the string representation of Error for notImplemented. 191func (e notImplemented) Error() string { 192 return fmt.Sprintf("%q is not supported on %q storage", e.method, e.apiType) 193} 194 195type Metadata map[string]string 196 197// NewMetadata will return an empty metadata object. 198func NewMetadata() Metadata { 199 return Metadata{} 200} 201 202func (m Metadata) ACL() string { 203 return m["ACL"] 204} 205 206func (m Metadata) SetACL(acl string) Metadata { 207 m["ACL"] = acl 208 return m 209} 210 211func (m Metadata) CacheControl() string { 212 return m["CacheControl"] 213} 214 215func (m Metadata) SetCacheControl(cacheControl string) Metadata { 216 m["CacheControl"] = cacheControl 217 return m 218} 219 220func (m Metadata) Expires() string { 221 return m["Expires"] 222} 223 224func (m Metadata) SetExpires(expires string) Metadata { 225 m["Expires"] = expires 226 return m 227} 228 229func (m Metadata) StorageClass() string { 230 return m["StorageClass"] 231} 232 233func (m Metadata) SetStorageClass(class string) Metadata { 234 m["StorageClass"] = class 235 return m 236} 237 238func (m Metadata) ContentType() string { 239 return m["ContentType"] 240} 241 242func (m Metadata) SetContentType(contentType string) Metadata { 243 m["ContentType"] = contentType 244 return m 245} 246 247func (m Metadata) SSE() string { 248 return m["EncryptionMethod"] 249} 250 251func (m Metadata) SetSSE(sse string) Metadata { 252 m["EncryptionMethod"] = sse 253 return m 254} 255 256func (m Metadata) SSEKeyID() string { 257 return m["EncryptionKeyID"] 258} 259 260func (m Metadata) SetSSEKeyID(kid string) Metadata { 261 m["EncryptionKeyID"] = kid 262 return m 263} 264