1// Package storage implements operations for s3 and fs.
2package storage
3
4import (
5	"context"
6	"encoding/json"
7	"fmt"
8	"os"
9	"time"
10
11	"github.com/peak/s5cmd/storage/url"
12	"github.com/peak/s5cmd/strutil"
13)
14
15var (
16	// ErrGivenObjectNotFound indicates a specified object is not found.
17	ErrGivenObjectNotFound = fmt.Errorf("given object not found")
18
19	// ErrNoObjectFound indicates there are no objects found from a given directory.
20	ErrNoObjectFound = fmt.Errorf("no object found")
21)
22
23// Storage is an interface for storage operations that is common
24// to local filesystem and remote object storage.
25type Storage interface {
26	// Stat returns the Object structure describing object. If src is not
27	// found, ErrGivenObjectNotFound is returned.
28	Stat(ctx context.Context, src *url.URL) (*Object, error)
29
30	// List the objects and directories/prefixes in the src.
31	List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object
32
33	// Delete deletes the given src.
34	Delete(ctx context.Context, src *url.URL) error
35
36	// MultiDelete deletes all items returned from given urls in batches.
37	MultiDelete(ctx context.Context, urls <-chan *url.URL) <-chan *Object
38
39	// Copy src to dst, optionally setting the given metadata. Src and dst
40	// arguments are of the same type. If src is a remote type, server side
41	// copying will be used.
42	Copy(ctx context.Context, src, dst *url.URL, metadata Metadata) error
43}
44
45func NewLocalClient(opts Options) *Filesystem {
46	return &Filesystem{dryRun: opts.DryRun}
47}
48
49func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, error) {
50	newOpts := Options{
51		MaxRetries:    opts.MaxRetries,
52		Endpoint:      opts.Endpoint,
53		NoVerifySSL:   opts.NoVerifySSL,
54		DryRun:        opts.DryRun,
55		NoSignRequest: opts.NoSignRequest,
56		bucket:        url.Bucket,
57		region:        opts.region,
58	}
59	return newS3Storage(ctx, newOpts)
60}
61
62func NewClient(ctx context.Context, url *url.URL, opts Options) (Storage, error) {
63	if url.IsRemote() {
64		return NewRemoteClient(ctx, url, opts)
65	}
66	return NewLocalClient(opts), nil
67}
68
69// Options stores configuration for storage.
70type Options struct {
71	MaxRetries    int
72	Endpoint      string
73	NoVerifySSL   bool
74	DryRun        bool
75	NoSignRequest bool
76	bucket        string
77	region        string
78}
79
80func (o *Options) SetRegion(region string) {
81	o.region = region
82}
83
84// Object is a generic type which contains metadata for storage items.
85type Object struct {
86	URL          *url.URL     `json:"key,omitempty"`
87	Etag         string       `json:"etag,omitempty"`
88	ModTime      *time.Time   `json:"last_modified,omitempty"`
89	Type         ObjectType   `json:"type,omitempty"`
90	Size         int64        `json:"size,omitempty"`
91	StorageClass StorageClass `json:"storage_class,omitempty"`
92	Err          error        `json:"error,omitempty"`
93}
94
95// String returns the string representation of Object.
96func (o *Object) String() string {
97	return o.URL.String()
98}
99
100// JSON returns the JSON representation of Object.
101func (o *Object) JSON() string {
102	return strutil.JSON(o)
103}
104
105// ObjectType is the type of Object.
106type ObjectType struct {
107	mode os.FileMode
108}
109
110// String returns the string representation of ObjectType.
111func (o ObjectType) String() string {
112	switch mode := o.mode; {
113	case mode.IsRegular():
114		return "file"
115	case mode.IsDir():
116		return "directory"
117	case mode&os.ModeSymlink != 0:
118		return "symlink"
119	}
120	return ""
121}
122
123// MarshalJSON returns the stringer of ObjectType as a marshalled json.
124func (o ObjectType) MarshalJSON() ([]byte, error) {
125	return json.Marshal(o.String())
126}
127
128// IsDir checks if the object is a directory.
129func (o ObjectType) IsDir() bool {
130	return o.mode.IsDir()
131}
132
133// IsSymlink checks if the object is a symbolic link.
134func (o ObjectType) IsSymlink() bool {
135	return o.mode&os.ModeSymlink != 0
136}
137
138// ShouldProcessUrl returns true if follow symlinks is enabled.
139// If follow symlinks is disabled we should not process the url.
140// (this check is needed only for local files)
141func ShouldProcessUrl(url *url.URL, followSymlinks bool) bool {
142	if followSymlinks {
143		return true
144	}
145
146	if url.IsRemote() {
147		return true
148	}
149	fi, err := os.Lstat(url.Absolute())
150	if err != nil {
151		return false
152	}
153
154	// do not process symlinks
155	return fi.Mode()&os.ModeSymlink == 0
156}
157
158// dateFormat is a constant time template for the bucket.
159const dateFormat = "2006/01/02 15:04:05"
160
161// Bucket is a container for storage objects.
162type Bucket struct {
163	CreationDate time.Time `json:"created_at"`
164	Name         string    `json:"name"`
165}
166
167// String returns the string representation of Bucket.
168func (b Bucket) String() string {
169	return fmt.Sprintf("%s  s3://%s", b.CreationDate.Format(dateFormat), b.Name)
170}
171
172// JSON returns the JSON representation of Bucket.
173func (b Bucket) JSON() string {
174	return strutil.JSON(b)
175}
176
177// StorageClass represents the storage used to store an object.
178type StorageClass string
179
180func (s StorageClass) IsGlacier() bool {
181	return s == "GLACIER"
182}
183
184// notImplemented is a structure which is used on the unsupported operations.
185type notImplemented struct {
186	apiType string
187	method  string
188}
189
190// Error returns the string representation of Error for notImplemented.
191func (e notImplemented) Error() string {
192	return fmt.Sprintf("%q is not supported on %q storage", e.method, e.apiType)
193}
194
195type Metadata map[string]string
196
197// NewMetadata will return an empty metadata object.
198func NewMetadata() Metadata {
199	return Metadata{}
200}
201
202func (m Metadata) ACL() string {
203	return m["ACL"]
204}
205
206func (m Metadata) SetACL(acl string) Metadata {
207	m["ACL"] = acl
208	return m
209}
210
211func (m Metadata) CacheControl() string {
212	return m["CacheControl"]
213}
214
215func (m Metadata) SetCacheControl(cacheControl string) Metadata {
216	m["CacheControl"] = cacheControl
217	return m
218}
219
220func (m Metadata) Expires() string {
221	return m["Expires"]
222}
223
224func (m Metadata) SetExpires(expires string) Metadata {
225	m["Expires"] = expires
226	return m
227}
228
229func (m Metadata) StorageClass() string {
230	return m["StorageClass"]
231}
232
233func (m Metadata) SetStorageClass(class string) Metadata {
234	m["StorageClass"] = class
235	return m
236}
237
238func (m Metadata) ContentType() string {
239	return m["ContentType"]
240}
241
242func (m Metadata) SetContentType(contentType string) Metadata {
243	m["ContentType"] = contentType
244	return m
245}
246
247func (m Metadata) SSE() string {
248	return m["EncryptionMethod"]
249}
250
251func (m Metadata) SetSSE(sse string) Metadata {
252	m["EncryptionMethod"] = sse
253	return m
254}
255
256func (m Metadata) SSEKeyID() string {
257	return m["EncryptionKeyID"]
258}
259
260func (m Metadata) SetSSEKeyID(kid string) Metadata {
261	m["EncryptionKeyID"] = kid
262	return m
263}
264