1/* 2Copyright 2011 The Perkeep Authors 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package blob 18 19import ( 20 "context" 21 "errors" 22 "fmt" 23 "io" 24 "math" 25 "os" 26) 27 28var ( 29 ErrNegativeSubFetch = errors.New("invalid negative subfetch parameters") 30 ErrOutOfRangeOffsetSubFetch = errors.New("subfetch offset greater than blob size") 31) 32 33// Fetcher is the minimal interface for retrieving a blob from storage. 34// The full storage interface is blobserver.Storage. 35type Fetcher interface { 36 // Fetch returns a blob. If the blob is not found then 37 // os.ErrNotExist should be returned for the error (not a wrapped 38 // error with a ErrNotExist inside) 39 // 40 // The contents are not guaranteed to match the digest of the 41 // provided Ref (e.g. when streamed over HTTP). Paranoid 42 // callers should verify them. 43 // 44 // The caller must close blob. 45 // 46 // The provided context is used until blob is closed and its 47 // cancelation should but may not necessarily cause reads from 48 // blob to fail with an error. 49 Fetch(context.Context, Ref) (blob io.ReadCloser, size uint32, err error) 50} 51 52// ErrUnimplemented is returned by optional interfaces when their 53// wrapped values don't implemented the optional interface. 54var ErrUnimplemented = errors.New("optional method not implemented") 55 56// A SubFetcher is a Fetcher that can retrieve part of a blob. 57type SubFetcher interface { 58 // SubFetch returns part of a blob. 59 // The caller must close the returned io.ReadCloser. 60 // The Reader may return fewer than 'length' bytes. Callers should 61 // check. The returned error should be: ErrNegativeSubFetch if any of 62 // offset or length is negative, or os.ErrNotExist if the blob 63 // doesn't exist, or ErrOutOfRangeOffsetSubFetch if offset goes over 64 // the size of the blob. If the error is ErrUnimplemented, the caller should 65 // treat this Fetcher as if it doesn't implement SubFetcher. 66 SubFetch(ctx context.Context, ref Ref, offset, length int64) (io.ReadCloser, error) 67} 68 69func NewSerialFetcher(fetchers ...Fetcher) Fetcher { 70 return &serialFetcher{fetchers} 71} 72 73func NewSimpleDirectoryFetcher(dir string) *DirFetcher { 74 return &DirFetcher{dir, "camli"} 75} 76 77type serialFetcher struct { 78 fetchers []Fetcher 79} 80 81func (sf *serialFetcher) Fetch(ctx context.Context, r Ref) (file io.ReadCloser, size uint32, err error) { 82 for _, fetcher := range sf.fetchers { 83 file, size, err = fetcher.Fetch(ctx, r) 84 if err == nil { 85 return 86 } 87 } 88 return 89} 90 91type DirFetcher struct { 92 directory, extension string 93} 94 95func (df *DirFetcher) Fetch(ctx context.Context, r Ref) (file io.ReadCloser, size uint32, err error) { 96 fileName := fmt.Sprintf("%s/%s.%s", df.directory, r.String(), df.extension) 97 var stat os.FileInfo 98 stat, err = os.Stat(fileName) 99 if err != nil { 100 return 101 } 102 if stat.Size() > math.MaxUint32 { 103 err = errors.New("file size too big") 104 return 105 } 106 file, err = os.Open(fileName) 107 if err != nil { 108 return 109 } 110 size = uint32(stat.Size()) 111 return 112} 113 114// ReaderAt returns an io.ReaderAt of br, fetching against sf. 115// The context is stored in and used by the returned ReaderAt. 116func ReaderAt(ctx context.Context, sf SubFetcher, br Ref) io.ReaderAt { 117 return readerAt{ctx, sf, br} 118} 119 120type readerAt struct { 121 ctx context.Context 122 sf SubFetcher 123 br Ref 124} 125 126func (ra readerAt) ReadAt(p []byte, off int64) (n int, err error) { 127 rc, err := ra.sf.SubFetch(ra.ctx, ra.br, off, int64(len(p))) 128 if err != nil { 129 return 0, err 130 } 131 defer rc.Close() 132 return io.ReadFull(rc, p) 133} 134