1package gofakes3_test 2 3import ( 4 "bufio" 5 "bytes" 6 "encoding/json" 7 "fmt" 8 "log" 9 "os/exec" 10 "path" 11 "reflect" 12 "regexp" 13 "strings" 14 "testing" 15 "time" 16 17 "github.com/johannesboyne/gofakes3" 18) 19 20func TestCLILsBuckets(t *testing.T) { 21 cli := newTestCLI(t, withoutInitialBuckets()) 22 defer cli.Close() 23 24 if len(cli.lsBuckets()) != 0 { 25 t.Fatal() 26 } 27 28 cli.backendCreateBucket("foo") 29 if !reflect.DeepEqual(cli.lsBuckets().Names(), []string{"foo"}) { 30 t.Fatal() 31 } 32 33 cli.backendCreateBucket("bar") 34 if !reflect.DeepEqual(cli.lsBuckets().Names(), []string{"bar", "foo"}) { 35 t.Fatal() 36 } 37} 38 39func TestCLILsFiles(t *testing.T) { 40 cli := newTestCLI(t) 41 defer cli.Close() 42 43 if len(cli.lsFiles(defaultBucket, "")) != 0 { 44 t.Fatal() 45 } 46 47 cli.backendPutString(defaultBucket, "test-one", nil, "hello") 48 cli.assertLsFiles(defaultBucket, "", 49 nil, []string{"test-one"}) 50 51 cli.backendPutString(defaultBucket, "test-two", nil, "hello") 52 cli.assertLsFiles(defaultBucket, "", 53 nil, []string{"test-one", "test-two"}) 54 55 // only "test-one" and "test-two" should pass the prefix match 56 cli.backendPutString(defaultBucket, "no-match", nil, "hello") 57 cli.assertLsFiles(defaultBucket, "test-", 58 nil, []string{"test-one", "test-two"}) 59 60 cli.backendPutString(defaultBucket, "test/yep", nil, "hello") 61 cli.assertLsFiles(defaultBucket, "", 62 []string{"test/"}, []string{"no-match", "test-one", "test-two"}) 63 64 // "test-one" and "test-two" and the directory "test" should pass the prefix match: 65 cli.assertLsFiles(defaultBucket, "test", 66 []string{"test/"}, []string{"test-one", "test-two"}) 67 68 // listing with a trailing slash should list the directory contents: 69 cli.assertLsFiles(defaultBucket, "test/", 70 nil, []string{"yep"}) 71} 72 73func TestCLIRmOne(t *testing.T) { 74 cli := newTestCLI(t) 75 defer cli.Close() 76 77 cli.backendPutString(defaultBucket, "foo", nil, "hello") 78 cli.backendPutString(defaultBucket, "bar", nil, "hello") 79 cli.assertLsFiles(defaultBucket, "", nil, []string{"foo", "bar"}) 80 81 cli.rm(cli.fileArg(defaultBucket, "foo")) 82 cli.assertLsFiles(defaultBucket, "", nil, []string{"bar"}) 83} 84 85func TestCLIRmMulti(t *testing.T) { 86 cli := newTestCLI(t) 87 defer cli.Close() 88 89 cli.backendPutString(defaultBucket, "foo", nil, "hello") 90 cli.backendPutString(defaultBucket, "bar", nil, "hello") 91 cli.backendPutString(defaultBucket, "baz", nil, "hello") 92 cli.assertLsFiles(defaultBucket, "", nil, []string{"foo", "bar", "baz"}) 93 94 cli.rmMulti(defaultBucket, "foo", "bar", "baz") 95 cli.assertLsFiles(defaultBucket, "", nil, nil) 96} 97 98func TestCLIDownload(t *testing.T) { 99 // NOTE: this must be set to the largest value you plan to test in the test cases. 100 var source = randomFileBody(100000000) 101 102 for _, tc := range []struct { 103 in []byte 104 }{ 105 {in: nil}, 106 {in: source[:1]}, 107 108 // FIXME: Beyond a certain size, the AWS client switches to using range 109 // requests and downloads several parts in parallel. This takes a stab 110 // at what that size is, but it isn't an especially robust way to 111 // determine what the spill point is: 112 {in: source[:1000000]}, 113 {in: source[:10000000]}, 114 {in: source[:100000000]}, 115 } { 116 t.Run("", func(t *testing.T) { 117 cli := newTestCLI(t) 118 defer cli.Close() 119 120 cli.backendPutBytes(defaultBucket, "foo", nil, tc.in) 121 out := cli.download(defaultBucket, "foo") 122 if !bytes.Equal(out, tc.in) { 123 t.Fatal() 124 } 125 }) 126 } 127} 128 129type testCLI struct { 130 *testServer 131} 132 133func newTestCLI(t *testing.T, options ...testServerOption) *testCLI { 134 return &testCLI{newTestServer(t, options...)} 135} 136 137func (tc *testCLI) command(method string, subcommand string, args ...string) *exec.Cmd { 138 tc.Helper() 139 140 if method != "s3" && method != "s3api" { 141 panic("expected 's3' or 's3api'") 142 } 143 144 cmdArgs := append([]string{ 145 "--output", "json", 146 method, 147 "--endpoint", tc.server.URL, 148 subcommand, 149 }, args...) 150 151 cmd := exec.Command("aws", cmdArgs...) 152 153 log.Println("cli args:", cmdArgs) 154 155 cmd.Env = []string{ 156 "AWS_ACCESS_KEY_ID=key", 157 "AWS_SECRET_ACCESS_KEY=secret", 158 } 159 return cmd 160} 161 162func (tc *testCLI) run(method string, subcommand string, args ...string) { 163 tc.Helper() 164 err := tc.command(method, subcommand, args...).Run() 165 if _, ok := err.(*exec.Error); ok { 166 tc.Skip("aws cli not found on $PATH") 167 } 168 tc.OK(err) 169} 170 171func (tc *testCLI) output(method string, subcommand string, args ...string) (out []byte) { 172 tc.Helper() 173 out, err := tc.command(method, subcommand, args...).Output() 174 if _, ok := err.(*exec.Error); ok { 175 tc.Skip("aws cli not found on $PATH") 176 } 177 tc.OK(err) 178 return out 179} 180 181func (tc *testCLI) combinedOutput(method string, subcommand string, args ...string) (out []byte) { 182 tc.Helper() 183 out, err := tc.command(method, subcommand, args...).CombinedOutput() 184 if _, ok := err.(*exec.Error); ok { 185 tc.Skip("aws cli not found on $PATH") 186 } 187 tc.OK(err) 188 return out 189} 190 191var cliLsDirMatcher = regexp.MustCompile(`^\s*PRE (.*)$`) 192 193func (tc *testCLI) assertLsFiles(bucket string, prefix string, dirs []string, files []string) (items lsItems) { 194 tc.Helper() 195 items = tc.lsFiles(bucket, prefix) 196 items.assertContents(tc.TT, dirs, files) 197 return items 198} 199 200func (tc *testCLI) lsFiles(bucket string, prefix string) (items lsItems) { 201 tc.Helper() 202 203 prefix = strings.TrimLeft(prefix, "/") 204 out := tc.combinedOutput("s3", "ls", fmt.Sprintf("s3://%s/%s", bucket, prefix)) 205 206 scn := bufio.NewScanner(bytes.NewReader(out)) 207 for scn.Scan() { 208 cur := scn.Text() 209 dir := cliLsDirMatcher.FindStringSubmatch(cur) 210 if dir != nil { 211 items = append(items, lsItem{ 212 isDir: true, 213 name: dir[1], // first submatch 214 }) 215 216 } else { // file matching 217 var ct cliTime 218 var item lsItem 219 tc.OKAll(fmt.Sscan(scn.Text(), &ct, &item.size, &item.name)) 220 item.date = time.Time(ct) 221 items = append(items, item) 222 } 223 } 224 225 return items 226} 227 228func (tc *testCLI) lsBuckets() (buckets gofakes3.Buckets) { 229 tc.Helper() 230 231 out := tc.combinedOutput("s3", "ls") 232 233 scn := bufio.NewScanner(bytes.NewReader(out)) 234 for scn.Scan() { 235 var ct cliTime 236 var b gofakes3.BucketInfo 237 tc.OKAll(fmt.Sscan(scn.Text(), &ct, &b.Name)) 238 b.CreationDate = ct.contentTime() 239 buckets = append(buckets, b) 240 } 241 242 return buckets 243} 244 245func (tc *testCLI) download(bucket, object string) []byte { 246 tc.Helper() 247 return tc.combinedOutput("s3", "cp", fmt.Sprintf("s3://%s/%s", bucket, object), "-") 248} 249 250func (tc *testCLI) rmMulti(bucket string, objects ...string) { 251 tc.Helper() 252 253 // delete-objects --bucket fakes3 --delete 'Objects=[{Key=test},{Key=test2}]' 254 255 var delArg struct{ Objects []gofakes3.ObjectID } 256 for _, obj := range objects { 257 delArg.Objects = append(delArg.Objects, gofakes3.ObjectID{Key: obj}) 258 } 259 bts, err := json.Marshal(delArg) 260 if err != nil { 261 panic(err) 262 } 263 264 args := []string{ 265 "--bucket", bucket, 266 "--delete", string(bts), 267 } 268 tc.run("s3api", "delete-objects", args...) 269} 270 271func (tc *testCLI) rm(fileURL string) { 272 tc.Helper() 273 tc.run("s3", "rm", fileURL) 274} 275 276func (tc *testCLI) fileArg(bucket string, file string) string { 277 return fmt.Sprintf("s3://%s", path.Join(bucket, file)) 278} 279 280func (tc *testCLI) fileArgs(bucket string, files ...string) []string { 281 out := make([]string, len(files)) 282 for i, f := range files { 283 out[i] = tc.fileArg(bucket, f) 284 } 285 return out 286} 287 288type cliTime time.Time 289 290func (c cliTime) contentTime() gofakes3.ContentTime { 291 return gofakes3.NewContentTime(time.Time(c)) 292} 293 294func (c *cliTime) Scan(state fmt.ScanState, verb rune) error { 295 d, err := state.Token(false, nil) 296 if err != nil { 297 return err 298 } 299 ds := string(d) 300 301 t, err := state.Token(true, nil) 302 if err != nil { 303 return err 304 } 305 ts := string(t) 306 307 // CLI returns time in the machine's timezone: 308 tv, err := time.ParseInLocation("2006-01-01 15:04:05", ds+" "+ts, time.Local) 309 if err != nil { 310 return err 311 } 312 313 *c = cliTime(tv.In(time.UTC)) 314 315 return nil 316} 317