1package releasedir_test 2 3import ( 4 "errors" 5 "io/ioutil" 6 "os" 7 "path/filepath" 8 "strings" 9 "syscall" 10 11 fakecrypto "github.com/cloudfoundry/bosh-cli/crypto/fakes" 12 boshcrypto "github.com/cloudfoundry/bosh-utils/crypto" 13 fakelogger "github.com/cloudfoundry/bosh-utils/logger/loggerfakes" 14 fakesys "github.com/cloudfoundry/bosh-utils/system/fakes" 15 . "github.com/onsi/ginkgo" 16 . "github.com/onsi/gomega" 17 18 "fmt" 19 20 . "github.com/cloudfoundry/bosh-cli/releasedir" 21 fakereldir "github.com/cloudfoundry/bosh-cli/releasedir/releasedirfakes" 22) 23 24var _ = Describe("FSBlobsDir", func() { 25 var ( 26 fs *fakesys.FakeFileSystem 27 reporter *fakereldir.FakeBlobsDirReporter 28 blobstore *fakereldir.FakeDigestBlobstore 29 digestCalculator *fakecrypto.FakeDigestCalculator 30 blobsDir FSBlobsDir 31 logger *fakelogger.FakeLogger 32 ) 33 34 BeforeEach(func() { 35 fs = fakesys.NewFakeFileSystem() 36 reporter = &fakereldir.FakeBlobsDirReporter{} 37 blobstore = &fakereldir.FakeDigestBlobstore{} 38 digestCalculator = fakecrypto.NewFakeDigestCalculator() 39 logger = &fakelogger.FakeLogger{} 40 blobsDir = NewFSBlobsDir(filepath.Join("/", "dir"), reporter, blobstore, digestCalculator, fs, logger) 41 }) 42 43 Describe("Blobs", func() { 44 act := func() ([]Blob, error) { 45 return blobsDir.Blobs() 46 } 47 48 It("returns no blobs if blobs.yml is empty", func() { 49 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), "") 50 51 blobs, err := act() 52 Expect(err).ToNot(HaveOccurred()) 53 Expect(blobs).To(BeEmpty()) 54 }) 55 56 It("returns parsed blobs", func() { 57 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), ` 58bosh-116.tgz: 59 size: 133959511 60 sha: 13ebc5850fcbde216ec32ab4354df53df76e4745 61`+filepath.Join("dir", "file.tgz")+`: 62 size: 133959000 63 object_id: ea50bf88-52ca-4230-4ef3-ff22c3975d04 64 sha: 2b86b5850fcbde216ec565b4354df53df76e4745 65file2.tgz: 66 size: 245959511 67 object_id: dc21b23e-1e32-40f4-61fb-5c9db26f7375 68 sha: 3456b5850fcbde216ec32ab4354df53395607042 69`) 70 71 blobs, err := act() 72 Expect(err).ToNot(HaveOccurred()) 73 Expect(blobs).To(Equal([]Blob{ 74 { 75 Path: "bosh-116.tgz", 76 Size: 133959511, 77 SHA1: "13ebc5850fcbde216ec32ab4354df53df76e4745", 78 }, 79 { 80 Path: filepath.Join("dir", "file.tgz"), 81 Size: 133959000, 82 BlobstoreID: "ea50bf88-52ca-4230-4ef3-ff22c3975d04", 83 SHA1: "2b86b5850fcbde216ec565b4354df53df76e4745", 84 }, 85 { 86 Path: "file2.tgz", 87 Size: 245959511, 88 BlobstoreID: "dc21b23e-1e32-40f4-61fb-5c9db26f7375", 89 SHA1: "3456b5850fcbde216ec32ab4354df53395607042", 90 }, 91 })) 92 }) 93 94 It("returns error if blobs.yml is not found so that user initializes it explicitly", func() { 95 _, err := act() 96 Expect(err).To(HaveOccurred()) 97 Expect(err.Error()).To(ContainSubstring("Reading blobs index")) 98 }) 99 100 It("returns error if blobs.yml is not parseable", func() { 101 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), "-") 102 103 _, err := act() 104 Expect(err).To(HaveOccurred()) 105 Expect(err.Error()).To(ContainSubstring("Unmarshalling blobs index")) 106 }) 107 }) 108 109 Describe("SyncBlobs", func() { 110 act := func(numOfParallelWorkers int) error { 111 return blobsDir.SyncBlobs(numOfParallelWorkers) 112 } 113 114 BeforeEach(func() { 115 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), filepath.Join("dir", "file-in-directory.tgz")+":"+` 116 object_id: blob1 117 size: 133 118 sha: blob1sha 119non-uploaded.tgz: 120 size: 245 121 sha: 345 122file-in-root.tgz: 123 object_id: blob2 124 size: 245 125 sha: blob2sha 126already-downloaded.tgz: 127 object_id: blob3 128 size: 245 129 sha: 1da283030f72f285fa9e05d597a528f08780c992 130`) 131 132 fs.WriteFileString(filepath.Join("/", "blob1-tmp"), "blob1-content") 133 fs.WriteFileString(filepath.Join("/", "blob2-tmp"), "blob2-content") 134 fs.WriteFileString(filepath.Join("/", "dir", "blobs", "already-downloaded.tgz"), "blob3-content") 135 136 times := 0 137 blobstore.GetStub = func(blobID string, digest boshcrypto.Digest) (string, error) { 138 defer func() { times += 1 }() 139 return []string{filepath.Join("/", "blob1-tmp"), filepath.Join("/", "blob2-tmp")}[times], nil 140 } 141 }) 142 143 Context("Multiple workers used to download blobs", func() { 144 It("downloads all blobs without local blob copy, skipping non-uploaded blobs", func() { 145 blobstore.GetStub = func(blobID string, digest boshcrypto.Digest) (fileName string, err error) { 146 if blobID == "blob1" && digest.String() == "blob1sha" { 147 return filepath.Join("/", "blob1-tmp"), nil 148 } else if blobID == "blob2" && digest.String() == "blob2sha" { 149 return filepath.Join("/", "blob2-tmp"), nil 150 } else { 151 panic("Received non-matching blobstore.Get call") 152 } 153 } 154 155 blobsDir = NewFSBlobsDir(filepath.Join("/", "dir"), reporter, blobstore, digestCalculator, fs, logger) 156 157 err := act(4) 158 Expect(err).ToNot(HaveOccurred()) 159 160 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "dir"))).To(BeTrue()) 161 Expect(fs.ReadFileString(filepath.Join("/", "dir", "blobs", "dir", "file-in-directory.tgz"))).To(Equal("blob1-content")) 162 Expect(fs.ReadFileString(filepath.Join("/", "dir", "blobs", "file-in-root.tgz"))).To(Equal("blob2-content")) 163 }) 164 }) 165 166 Context("A single worker to download blobs", func() { 167 It("downloads all blobs without local blob copy, skipping non-uploaded blobs", func() { 168 err := act(1) 169 Expect(err).ToNot(HaveOccurred()) 170 171 id1, digest1 := blobstore.GetArgsForCall(0) 172 Expect(id1).To(Equal("blob1")) 173 Expect(digest1).To(Equal(boshcrypto.MustParseMultipleDigest("blob1sha"))) 174 175 id2, digest2 := blobstore.GetArgsForCall(1) 176 Expect(id2).To(Equal("blob2")) 177 Expect(digest2).To(Equal(boshcrypto.MustParseMultipleDigest("blob2sha"))) 178 179 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "dir"))).To(BeTrue()) 180 Expect(fs.ReadFileString(filepath.Join("/", "dir", "blobs", "dir", "file-in-directory.tgz"))).To(Equal("blob1-content")) 181 Expect(fs.ReadFileString(filepath.Join("/", "dir", "blobs", "file-in-root.tgz"))).To(Equal("blob2-content")) 182 }) 183 184 It("reports downloaded blobs skipping already existing ones", func() { 185 err := act(1) 186 Expect(err).ToNot(HaveOccurred()) 187 188 { 189 Expect(reporter.BlobDownloadStartedCallCount()).To(Equal(2)) 190 191 path, size, blobID, sha1 := reporter.BlobDownloadStartedArgsForCall(0) 192 Expect(path).To(Equal(filepath.Join("dir", "file-in-directory.tgz"))) 193 Expect(size).To(Equal(int64(133))) 194 Expect(blobID).To(Equal("blob1")) 195 Expect(sha1).To(Equal("blob1sha")) 196 197 path, size, blobID, sha1 = reporter.BlobDownloadStartedArgsForCall(1) 198 Expect(path).To(Equal("file-in-root.tgz")) 199 Expect(size).To(Equal(int64(245))) 200 Expect(blobID).To(Equal("blob2")) 201 Expect(sha1).To(Equal("blob2sha")) 202 } 203 204 { 205 Expect(reporter.BlobDownloadFinishedCallCount()).To(Equal(2)) 206 207 path, blobID, err := reporter.BlobDownloadFinishedArgsForCall(0) 208 Expect(path).To(Equal(filepath.Join("dir", "file-in-directory.tgz"))) 209 Expect(blobID).To(Equal("blob1")) 210 Expect(err).ToNot(HaveOccurred()) 211 212 path, blobID, err = reporter.BlobDownloadFinishedArgsForCall(1) 213 Expect(path).To(Equal("file-in-root.tgz")) 214 Expect(blobID).To(Equal("blob2")) 215 Expect(err).ToNot(HaveOccurred()) 216 } 217 }) 218 }) 219 220 Context("downloading fails", func() { 221 It("reports error", func() { 222 blobstore.GetReturns("", errors.New("fake-err")) 223 224 err := act(2) 225 Expect(err).To(HaveOccurred()) 226 Expect(err.Error()).To(ContainSubstring("Getting blob 'blob1' for path '" + filepath.Join("dir", "file-in-directory.tgz") + "': fake-err")) 227 228 Expect(reporter.BlobDownloadStartedCallCount()).To(Equal(2)) 229 Expect(reporter.BlobDownloadFinishedCallCount()).To(Equal(2)) 230 }) 231 232 Context("when more than one blob fails to download", func() { 233 It("reports error", func() { 234 blobstore.GetStub = func(blobID string, _ boshcrypto.Digest) (fileName string, err error) { 235 switch blobID { 236 case "blob1": 237 return filepath.Join("/", "blob1-tmp"), errors.New("fake-err1") 238 case "blob2": 239 return filepath.Join("/", "blob2-tmp"), errors.New("fake-err2") 240 } 241 return "", nil 242 } 243 244 err := act(2) 245 Expect(err).To(HaveOccurred()) 246 Expect(err.Error()).To(ContainSubstring("Getting blob 'blob1' for path '" + filepath.Join("dir", "file-in-directory.tgz") + "': fake-err1")) 247 Expect(err.Error()).To(ContainSubstring("Getting blob 'blob2' for path 'file-in-root.tgz': fake-err2")) 248 249 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "dir"))).To(BeFalse()) 250 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "dir", "file-in-directory.tgz"))).To(BeFalse()) 251 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "file-in-root.tgz"))).To(BeFalse()) 252 253 }) 254 }) 255 256 Context("without creating any blob sub-dirs", func() { 257 It("returns error", func() { 258 blobstore.GetReturns("", errors.New("fake-err")) 259 260 err := act(1) 261 Expect(err).To(HaveOccurred()) 262 Expect(err.Error()).To(ContainSubstring("fake-err")) 263 264 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "dir"))).To(BeFalse()) 265 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "dir", "file-in-directory.tgz"))).To(BeFalse()) 266 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "file-in-root.tgz"))).To(BeFalse()) 267 }) 268 }) 269 270 Context("without placing any local blobs", func() { 271 It("returns error", func() { 272 blobstore.GetStub = func(blobID string, _ boshcrypto.Digest) (fileName string, err error) { 273 switch blobID { 274 case "blob1": 275 return filepath.Join("/", "blob1-tmp"), nil 276 case "blob2": 277 return filepath.Join("/", "blob2-tmp"), errors.New("fake-err") 278 } 279 return "", nil 280 } 281 282 err := act(1) 283 Expect(err).To(HaveOccurred()) 284 Expect(err.Error()).To(ContainSubstring("fake-err")) 285 286 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "dir"))).To(BeTrue()) 287 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "dir", "file-in-directory.tgz"))).To(BeTrue()) 288 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "file-in-root.tgz"))).To(BeFalse()) 289 }) 290 }) 291 }) 292 293 Context("parsing digest string for sha fails", func() { 294 BeforeEach(func() { 295 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), ` 296bad-sha-blob.tgz: 297 object_id: blob3 298 size: 245 299 sha: '' 300`) 301 }) 302 303 It("returns descriptive error", func() { 304 err := act(1) 305 Expect(err).To(MatchError(ContainSubstring("No digest algorithm found. Supported algorithms: sha1, sha256, sha512"))) 306 }) 307 }) 308 309 Context("when blobs already on disk have different sha than in index", func() { 310 BeforeEach(func() { 311 fs.WriteFileString(filepath.Join("/", "blob3-tmp"), "blob3-content") 312 fs.WriteFileString(filepath.Join("/", "dir", "blobs", "already-downloaded.tgz"), "incorrect-blob3-content") 313 314 times := 0 315 blobstore.GetStub = func(blobID string, digest boshcrypto.Digest) (string, error) { 316 defer func() { times += 1 }() 317 return []string{filepath.Join("/", "blob3-tmp"), filepath.Join("/", "blob1-tmp"), filepath.Join("/", "blob2-tmp")}[times], nil 318 } 319 }) 320 321 It("downloads new copy from blobstore and logs an error", func() { 322 err := act(1) 323 Expect(err).ToNot(HaveOccurred()) 324 325 id3, digest3 := blobstore.GetArgsForCall(0) 326 Expect(id3).To(Equal("blob3")) 327 Expect(digest3).To(Equal(boshcrypto.MustParseMultipleDigest("1da283030f72f285fa9e05d597a528f08780c992"))) 328 329 id1, digest1 := blobstore.GetArgsForCall(1) 330 Expect(id1).To(Equal("blob1")) 331 Expect(digest1).To(Equal(boshcrypto.MustParseMultipleDigest("blob1sha"))) 332 333 id2, digest2 := blobstore.GetArgsForCall(2) 334 Expect(id2).To(Equal("blob2")) 335 Expect(digest2).To(Equal(boshcrypto.MustParseMultipleDigest("blob2sha"))) 336 337 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "dir"))).To(BeTrue()) 338 Expect(fs.ReadFileString(filepath.Join("/", "dir", "blobs", "dir", "file-in-directory.tgz"))).To(Equal("blob1-content")) 339 Expect(fs.ReadFileString(filepath.Join("/", "dir", "blobs", "file-in-root.tgz"))).To(Equal("blob2-content")) 340 Expect(fs.ReadFileString(filepath.Join("/", "dir", "blobs", "already-downloaded.tgz"))).To(Equal("blob3-content")) 341 342 tag, message, _ := logger.ErrorArgsForCall(0) 343 Expect(tag).To(Equal("releasedir.FSBlobsDir")) 344 Expect(message).To(Equal("Incorrect SHA sum for blob at '" + filepath.Join("/", "dir", "blobs", "already-downloaded.tgz") + "'. Re-downloading from blobstore.")) 345 }) 346 }) 347 348 Context("when creating blob sub-dir fails", func() { 349 It("returns error", func() { 350 fs.MkdirAllError = errors.New("fake-err") 351 352 err := act(1) 353 Expect(err).To(HaveOccurred()) 354 Expect(err.Error()).To(ContainSubstring("fake-err")) 355 }) 356 }) 357 358 Context("when moving temp blob file across devices into its final destination", func() { 359 BeforeEach(func() { 360 fs.RenameError = &os.LinkError{ 361 Err: syscall.Errno(0x12), 362 } 363 }) 364 365 It("downloads all blobs without local blob copy", func() { 366 err := act(1) 367 Expect(err).ToNot(HaveOccurred()) 368 }) 369 370 Context("when copying blobs across devices fails", func() { 371 It("returns error", func() { 372 fs.CopyFileError = errors.New("failed to copy") 373 374 err := act(1) 375 Expect(err).To(HaveOccurred()) 376 Expect(err.Error()).To(ContainSubstring("failed to copy")) 377 }) 378 }) 379 }) 380 381 Context("when moving temp blob file into its final destination fails for an uncaught reason", func() { 382 It("returns error", func() { 383 fs.RenameError = errors.New("fake-err") 384 385 err := act(1) 386 Expect(err).To(HaveOccurred()) 387 Expect(err.Error()).To(ContainSubstring("fake-err")) 388 }) 389 }) 390 391 Context("when blobs exist on the file system which are not in the blobs.yml", func() { 392 BeforeEach(func() { 393 fs.SetGlob(filepath.Join("/", "dir", "blobs", "**", "*"), []string{filepath.Join("/", "dir", "blobs", "dir"), filepath.Join("/", "dir", "blobs", "already-downloaded.tgz"), filepath.Join("/", "dir", "blobs", "extra-blob.tgz")}) 394 fs.MkdirAll(filepath.Join("/", "dir", "blobs", "dir"), os.ModeDir) 395 fs.WriteFileString(filepath.Join("/", "dir", "blobs", "extra-blob.tgz"), "I don't belong here") 396 }) 397 398 It("deletes the blobs in the blob dir, logging a warning for each file deleted, and leaving correct blobs and directories", func() { 399 err := act(1) 400 Expect(err).ToNot(HaveOccurred()) 401 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "extra-blob.tgz"))).To(BeFalse()) 402 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "already-downloaded.tgz"))).To(BeTrue()) 403 404 tag, message, _ := logger.InfoArgsForCall(0) 405 Expect(tag).To(Equal("releasedir.FSBlobsDir")) 406 Expect(message).To(Equal("Deleting blob at '" + filepath.Join("/", "dir", "blobs", "extra-blob.tgz") + "' that is not in the blob index.")) 407 }) 408 409 It("returns an error when the glob fails", func() { 410 fs.GlobStub = func(string) ([]string, error) { 411 return []string{}, errors.New("failed to glob") 412 } 413 err := act(1) 414 Expect(err).To(MatchError("Syncing blobs: Checking for unknown blobs: failed to glob")) 415 }) 416 417 It("returns an error when the unknown blob removal fails", func() { 418 fs.RemoveAllStub = func(filename string) error { 419 return fmt.Errorf("failed to remove %s", filename) 420 } 421 err := act(1) 422 Expect(err).To(MatchError("Syncing blobs: Removing unknown blob: failed to remove " + filepath.Join("/", "dir", "blobs", "extra-blob.tgz"))) 423 }) 424 }) 425 426 Context("when a symlink exists", func() { 427 It("returns an error", func() { 428 missingFilePath := filepath.Join("/", "dir", ".blobs", "does-not-exist") 429 symlink := filepath.Join("/", "dir", "blobs", "fake-symlink") 430 431 fs.Symlink(missingFilePath, symlink) 432 433 fs.SetGlob(filepath.Join("/", "dir", "blobs", "**", "*"), []string{symlink}) 434 435 fs.RegisterOpenFile(symlink, &fakesys.FakeFile{ 436 Stats: &fakesys.FakeFileStats{ 437 FileType: fakesys.FakeFileTypeFile, 438 FileMode: os.FileMode(os.ModeSymlink), 439 SymlinkTarget: missingFilePath, 440 }, 441 }) 442 443 err := act(1) 444 Expect(err).To(MatchError("Bailing because symlinks found in blobs directory. If switching from CLI v1, please use the `reset-release` command.")) 445 }) 446 }) 447 448 Context("when getting the symlink description errors", func() { 449 It("passes the error along", func() { 450 existingFilePath := filepath.Join("/", "dir", "blobs", "does-exist") 451 symlink := filepath.Join("/", "dir", "blobs", "fake-symlink") 452 453 fs.Symlink(existingFilePath, symlink) 454 455 fs.SetGlob(filepath.Join("/", "dir", "blobs", "**", "*"), []string{symlink}) 456 457 fs.RegisterOpenFile(symlink, &fakesys.FakeFile{ 458 StatErr: errors.New("fake-err"), 459 }) 460 461 err := act(1) 462 Expect(err).To(MatchError("Syncing blobs: fake-err")) 463 }) 464 }) 465 466 Context("when no symlink exists", func() { 467 It("succeeds", func() { 468 existingFilePath := filepath.Join("/", "dir", "blobs", "does-exist") 469 470 fs.SetGlob(filepath.Join("/", "dir", "blobs", "**", "*"), []string{existingFilePath}) 471 472 fs.RegisterOpenFile(existingFilePath, &fakesys.FakeFile{ 473 Stats: &fakesys.FakeFileStats{ 474 FileType: fakesys.FakeFileTypeFile, 475 FileMode: os.FileMode(os.ModeDir), 476 }, 477 }) 478 479 err := act(1) 480 Expect(err).ToNot(HaveOccurred()) 481 }) 482 }) 483 }) 484 485 Describe("TrackBlob", func() { 486 act := func() (Blob, error) { 487 content := ioutil.NopCloser(strings.NewReader(string("content"))) 488 return blobsDir.TrackBlob(filepath.Join("dir", "file.tgz"), content) 489 } 490 491 BeforeEach(func() { 492 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), "") 493 494 fs.ReturnTempFile = fakesys.NewFakeFile(filepath.Join("/", "tmp-file"), fs) 495 496 digestCalculator.SetCalculateBehavior(map[string]fakecrypto.CalculateInput{ 497 filepath.Join("/", "tmp-file"): fakecrypto.CalculateInput{DigestStr: "contentsha1"}, 498 }) 499 }) 500 501 It("adds a blob to the list if it's not already tracked", func() { 502 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), ` 503file2.tgz: 504 size: 245 505 sha: 345 506`) 507 508 blob, err := act() 509 Expect(err).ToNot(HaveOccurred()) 510 Expect(blob).To(Equal(Blob{Path: filepath.Join("dir", "file.tgz"), Size: 7, SHA1: "contentsha1"})) 511 512 Expect(blobsDir.Blobs()).To(Equal([]Blob{ 513 {Path: filepath.Join("dir", "file.tgz"), Size: 7, SHA1: "contentsha1"}, 514 {Path: "file2.tgz", Size: 245, SHA1: "345"}, 515 })) 516 517 Expect(fs.ReadFileString(filepath.Join("/", "dir", "blobs", "dir", "file.tgz"))).To(Equal("content")) 518 }) 519 520 It("updates blob record if it's already tracked", func() { 521 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), filepath.Join("dir", "file.tgz")+`: 522 size: 133 523 sha: 13e 524file2.tgz: 525 size: 245 526 sha: 345 527`) 528 529 blob, err := act() 530 Expect(err).ToNot(HaveOccurred()) 531 Expect(blob).To(Equal(Blob{Path: filepath.Join("dir", "file.tgz"), Size: 7, SHA1: "contentsha1"})) 532 533 Expect(blobsDir.Blobs()).To(Equal([]Blob{ 534 {Path: filepath.Join("dir", "file.tgz"), Size: 7, SHA1: "contentsha1"}, 535 {Path: "file2.tgz", Size: 245, SHA1: "345"}, 536 })) 537 538 Expect(fs.ReadFileString(filepath.Join("/", "dir", "blobs", "dir", "file.tgz"))).To(Equal("content")) 539 }) 540 541 It("overrides existing local blob copy", func() { 542 fs.WriteFileString(filepath.Join("/", "dir", "blobs", "dir", "file.tgz"), "prev-content") 543 544 _, err := act() 545 Expect(err).ToNot(HaveOccurred()) 546 547 Expect(fs.ReadFileString(filepath.Join("/", "dir", "blobs", "dir", "file.tgz"))).To(Equal("content")) 548 }) 549 550 It("returns error and does not update blobs.yml if temp file cannot be opened", func() { 551 fs.TempFileError = errors.New("fake-err") 552 553 _, err := act() 554 Expect(err).To(HaveOccurred()) 555 Expect(err.Error()).To(ContainSubstring("fake-err")) 556 557 Expect(blobsDir.Blobs()).To(BeEmpty()) 558 }) 559 560 It("returns error and does not update blobs.yml if copying from src fails", func() { 561 file := fakesys.NewFakeFile(filepath.Join("/", "tmp-file"), fs) 562 file.WriteErr = errors.New("fake-err") 563 fs.ReturnTempFile = file 564 565 _, err := act() 566 Expect(err).To(HaveOccurred()) 567 Expect(err.Error()).To(ContainSubstring("fake-err")) 568 569 Expect(blobsDir.Blobs()).To(BeEmpty()) 570 }) 571 572 It("returns error and does not update blobs.yml if cannot determine size", func() { 573 file := fakesys.NewFakeFile(filepath.Join("/", "tmp-file"), fs) 574 file.StatErr = errors.New("fake-err") 575 fs.ReturnTempFile = file 576 577 _, err := act() 578 Expect(err).To(HaveOccurred()) 579 Expect(err.Error()).To(ContainSubstring("fake-err")) 580 581 Expect(blobsDir.Blobs()).To(BeEmpty()) 582 }) 583 584 It("returns error and does not update blobs.yml if calculating sha1 fails", func() { 585 digestCalculator.SetCalculateBehavior(map[string]fakecrypto.CalculateInput{ 586 filepath.Join("/", "tmp-file"): fakecrypto.CalculateInput{Err: errors.New("fake-err")}, 587 }) 588 589 _, err := act() 590 Expect(err).To(HaveOccurred()) 591 Expect(err.Error()).To(ContainSubstring("fake-err")) 592 593 Expect(blobsDir.Blobs()).To(BeEmpty()) 594 }) 595 }) 596 597 Describe("UntrackBlob", func() { 598 act := func() error { 599 return blobsDir.UntrackBlob(filepath.Join("dir", "file.tgz")) 600 } 601 602 It("removes reference from list of blobs (first)", func() { 603 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), filepath.Join("dir", "file.tgz")+`: 604 size: 133 605 sha: 13e 606file2.tgz: 607 size: 245 608 sha: 345 609`) 610 611 err := act() 612 Expect(err).ToNot(HaveOccurred()) 613 614 Expect(blobsDir.Blobs()).To(Equal([]Blob{ 615 {Path: "file2.tgz", Size: 245, SHA1: "345"}, 616 })) 617 }) 618 619 It("removes reference from list of blobs (middle)", func() { 620 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), ` 621bosh-116.tgz: 622 size: 133 623 sha: 13e 624`+filepath.Join("dir", "file.tgz")+`: 625 size: 133 626 sha: 2b8 627file2.tgz: 628 size: 245 629 sha: 345 630`) 631 632 err := act() 633 Expect(err).ToNot(HaveOccurred()) 634 635 Expect(blobsDir.Blobs()).To(Equal([]Blob{ 636 {Path: "bosh-116.tgz", Size: 133, SHA1: "13e"}, 637 {Path: "file2.tgz", Size: 245, SHA1: "345"}, 638 })) 639 }) 640 641 It("removes reference from list of blobs (last)", func() { 642 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), ` 643bosh-116.tgz: 644 size: 133 645 sha: 13e 646`+filepath.Join("dir", "file.tgz")+`: 647 size: 245 648 sha: 345 649`) 650 651 err := act() 652 Expect(err).ToNot(HaveOccurred()) 653 654 Expect(blobsDir.Blobs()).To(Equal([]Blob{ 655 {Path: "bosh-116.tgz", Size: 133, SHA1: "13e"}, 656 })) 657 }) 658 659 It("succeeds even if record is not found", func() { 660 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), ` 661bosh-116.tgz: 662 size: 133 663 sha: 13e 664`) 665 666 err := act() 667 Expect(err).ToNot(HaveOccurred()) 668 669 Expect(blobsDir.Blobs()).To(Equal([]Blob{ 670 {Path: "bosh-116.tgz", Size: 133, SHA1: "13e"}, 671 })) 672 }) 673 674 It("removes local blob copy", func() { 675 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), "") 676 fs.WriteFileString(filepath.Join("/", "dir", "blobs", "dir", "file.tgz"), "blob") 677 678 err := act() 679 Expect(err).ToNot(HaveOccurred()) 680 681 Expect(fs.FileExists(filepath.Join("/", "dir", "blobs", "dir", "file.tgz"))).To(BeFalse()) 682 }) 683 684 It("returns error if removing local blob copy fails", func() { 685 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), filepath.Join("dir", "file.tgz:")+` 686 size: 133 687 sha: 13e 688`) 689 690 fs.RemoveAllStub = func(_ string) error { 691 return errors.New("fake-err") 692 } 693 694 err := act() 695 Expect(err).To(HaveOccurred()) 696 Expect(err.Error()).To(ContainSubstring("fake-err")) 697 698 Expect(blobsDir.Blobs()).To(Equal([]Blob{ 699 {Path: filepath.Join("dir", "file.tgz"), Size: 133, SHA1: "13e"}, 700 })) 701 }) 702 }) 703 704 Describe("UploadBlobs", func() { 705 act := func() error { 706 return blobsDir.UploadBlobs() 707 } 708 709 BeforeEach(func() { 710 fs.WriteFileString(filepath.Join("/", "dir", "config", "blobs.yml"), filepath.Join("dir", "file-in-directory.tgz")+`: 711 object_id: blob1 712 size: 133 713 sha: blob1sha 714non-uploaded.tgz: 715 size: 243 716 sha: blob2sha 717file-in-root.tgz: 718 object_id: blob3 719 size: 245 720 sha: blob3sha 721already-downloaded.tgz: 722 object_id: blob4 723 size: 245 724 sha: blob4sha 725non-uploaded2.tgz: 726 size: 245 727 sha: blob5sha 728`) 729 730 times := 0 731 blobstore.CreateStub = func(fileName string) (string, boshcrypto.MultipleDigest, error) { 732 defer func() { times += 1 }() 733 multiDigest := boshcrypto.MustNewMultipleDigest( 734 boshcrypto.NewDigest(boshcrypto.DigestAlgorithmSHA1, "whatever"), 735 ) 736 return []string{"blob2", "blob5"}[times], multiDigest, nil 737 } 738 }) 739 740 It("uploads non-uploaded blobs", func() { 741 err := act() 742 Expect(err).ToNot(HaveOccurred()) 743 744 Expect(blobstore.CreateArgsForCall(0)).To(Equal(filepath.Join("/", "dir", "blobs", "non-uploaded.tgz"))) 745 Expect(blobstore.CreateArgsForCall(1)).To(Equal(filepath.Join("/", "dir", "blobs", "non-uploaded2.tgz"))) 746 747 Expect(blobsDir.Blobs()).To(Equal([]Blob{ 748 {Path: "already-downloaded.tgz", Size: 245, BlobstoreID: "blob4", SHA1: "blob4sha"}, 749 {Path: filepath.Join("dir", "file-in-directory.tgz"), Size: 133, BlobstoreID: "blob1", SHA1: "blob1sha"}, 750 {Path: "file-in-root.tgz", Size: 245, BlobstoreID: "blob3", SHA1: "blob3sha"}, 751 {Path: "non-uploaded.tgz", Size: 243, BlobstoreID: "blob2", SHA1: "blob2sha"}, 752 {Path: "non-uploaded2.tgz", Size: 245, BlobstoreID: "blob5", SHA1: "blob5sha"}, 753 })) 754 }) 755 756 It("reports uploaded blobs skipping already existing ones", func() { 757 err := act() 758 Expect(err).ToNot(HaveOccurred()) 759 760 { 761 Expect(reporter.BlobUploadStartedCallCount()).To(Equal(2)) 762 763 path, size, sha1 := reporter.BlobUploadStartedArgsForCall(0) 764 Expect(path).To(Equal("non-uploaded.tgz")) 765 Expect(size).To(Equal(int64(243))) 766 Expect(sha1).To(Equal("blob2sha")) 767 768 path, size, sha1 = reporter.BlobUploadStartedArgsForCall(1) 769 Expect(path).To(Equal("non-uploaded2.tgz")) 770 Expect(size).To(Equal(int64(245))) 771 Expect(sha1).To(Equal("blob5sha")) 772 } 773 774 { 775 Expect(reporter.BlobUploadFinishedCallCount()).To(Equal(2)) 776 777 path, blobID, err := reporter.BlobUploadFinishedArgsForCall(0) 778 Expect(path).To(Equal("non-uploaded.tgz")) 779 Expect(blobID).To(Equal("blob2")) 780 Expect(err).ToNot(HaveOccurred()) 781 782 path, blobID, err = reporter.BlobUploadFinishedArgsForCall(1) 783 Expect(path).To(Equal("non-uploaded2.tgz")) 784 Expect(blobID).To(Equal("blob5")) 785 Expect(err).ToNot(HaveOccurred()) 786 } 787 }) 788 789 It("returns error if uploading fails and does not change blobs.yml", func() { 790 blobstore.CreateReturns("", boshcrypto.MultipleDigest{}, errors.New("fake-err")) 791 792 err := act() 793 Expect(err).To(HaveOccurred()) 794 Expect(err.Error()).To(ContainSubstring("fake-err")) 795 796 Expect(blobsDir.Blobs()).To(Equal([]Blob{ 797 {Path: "already-downloaded.tgz", Size: 245, BlobstoreID: "blob4", SHA1: "blob4sha"}, 798 {Path: filepath.Join("dir", "file-in-directory.tgz"), Size: 133, BlobstoreID: "blob1", SHA1: "blob1sha"}, 799 {Path: "file-in-root.tgz", Size: 245, BlobstoreID: "blob3", SHA1: "blob3sha"}, 800 {Path: "non-uploaded.tgz", Size: 243, SHA1: "blob2sha"}, 801 {Path: "non-uploaded2.tgz", Size: 245, SHA1: "blob5sha"}, 802 })) 803 }) 804 805 It("reports error if uploading fails", func() { 806 blobstore.CreateReturns("", boshcrypto.MultipleDigest{}, errors.New("fake-err")) 807 808 err := act() 809 Expect(err).To(HaveOccurred()) 810 Expect(err.Error()).To(ContainSubstring("fake-err")) 811 812 Expect(reporter.BlobUploadStartedCallCount()).To(Equal(1)) 813 Expect(reporter.BlobUploadFinishedCallCount()).To(Equal(1)) 814 815 path, size, sha1 := reporter.BlobUploadStartedArgsForCall(0) 816 Expect(path).To(Equal("non-uploaded.tgz")) 817 Expect(size).To(Equal(int64(243))) 818 Expect(sha1).To(Equal("blob2sha")) 819 820 path, blobID, err := reporter.BlobUploadFinishedArgsForCall(0) 821 Expect(path).To(Equal("non-uploaded.tgz")) 822 Expect(blobID).To(Equal("")) 823 Expect(err).To(HaveOccurred()) 824 }) 825 826 It("returns if saving blobstore id fails and does not continue to upload other blobs", func() { 827 fs.WriteFileError = errors.New("fake-err") 828 829 err := act() 830 Expect(err).To(HaveOccurred()) 831 Expect(err.Error()).To(ContainSubstring("fake-err")) 832 833 // Include blobstore id in error message for cleanup purposes 834 Expect(err.Error()).To(ContainSubstring("Saving newly created blob 'blob2'")) 835 836 Expect(reporter.BlobUploadStartedCallCount()).To(Equal(1)) 837 }) 838 839 It("returns error if uploading fails and saves blob id for successfully uploaded blobs", func() { 840 times := 0 841 blobstore.CreateStub = func(fileName string) (string, boshcrypto.MultipleDigest, error) { 842 defer func() { times += 1 }() 843 blobID := []string{"blob2", "blob5"}[times] 844 err := []error{nil, errors.New("fake-err")}[times] 845 return blobID, boshcrypto.MultipleDigest{}, err 846 } 847 848 err := act() 849 Expect(err).To(HaveOccurred()) 850 Expect(err.Error()).To(ContainSubstring("fake-err")) 851 852 Expect(blobsDir.Blobs()).To(Equal([]Blob{ 853 {Path: "already-downloaded.tgz", Size: 245, BlobstoreID: "blob4", SHA1: "blob4sha"}, 854 {Path: filepath.Join("dir", "file-in-directory.tgz"), Size: 133, BlobstoreID: "blob1", SHA1: "blob1sha"}, 855 {Path: "file-in-root.tgz", Size: 245, BlobstoreID: "blob3", SHA1: "blob3sha"}, 856 {Path: "non-uploaded.tgz", Size: 243, BlobstoreID: "blob2", SHA1: "blob2sha"}, 857 {Path: "non-uploaded2.tgz", Size: 245, SHA1: "blob5sha"}, 858 })) 859 }) 860 }) 861}) 862