1package storage 2 3// Copyright 2017 Microsoft Corporation 4// 5// Licensed under the Apache License, Version 2.0 (the "License"); 6// you may not use this file except in compliance with the License. 7// You may obtain a copy of the License at 8// 9// http://www.apache.org/licenses/LICENSE-2.0 10// 11// Unless required by applicable law or agreed to in writing, software 12// distributed under the License is distributed on an "AS IS" BASIS, 13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14// See the License for the specific language governing permissions and 15// limitations under the License. 16 17import ( 18 "bytes" 19 "encoding/xml" 20 "fmt" 21 "io" 22 "net/http" 23 "net/url" 24 "strconv" 25 "strings" 26 "time" 27) 28 29// BlockListType is used to filter out types of blocks in a Get Blocks List call 30// for a block blob. 31// 32// See https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx for all 33// block types. 34type BlockListType string 35 36// Filters for listing blocks in block blobs 37const ( 38 BlockListTypeAll BlockListType = "all" 39 BlockListTypeCommitted BlockListType = "committed" 40 BlockListTypeUncommitted BlockListType = "uncommitted" 41) 42 43// Maximum sizes (per REST API) for various concepts 44const ( 45 MaxBlobBlockSize = 100 * 1024 * 1024 46 MaxBlobPageSize = 4 * 1024 * 1024 47) 48 49// BlockStatus defines states a block for a block blob can 50// be in. 51type BlockStatus string 52 53// List of statuses that can be used to refer to a block in a block list 54const ( 55 BlockStatusUncommitted BlockStatus = "Uncommitted" 56 BlockStatusCommitted BlockStatus = "Committed" 57 BlockStatusLatest BlockStatus = "Latest" 58) 59 60// Block is used to create Block entities for Put Block List 61// call. 62type Block struct { 63 ID string 64 Status BlockStatus 65} 66 67// BlockListResponse contains the response fields from Get Block List call. 68// 69// See https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx 70type BlockListResponse struct { 71 XMLName xml.Name `xml:"BlockList"` 72 CommittedBlocks []BlockResponse `xml:"CommittedBlocks>Block"` 73 UncommittedBlocks []BlockResponse `xml:"UncommittedBlocks>Block"` 74} 75 76// BlockResponse contains the block information returned 77// in the GetBlockListCall. 78type BlockResponse struct { 79 Name string `xml:"Name"` 80 Size int64 `xml:"Size"` 81} 82 83// CreateBlockBlob initializes an empty block blob with no blocks. 84// 85// See CreateBlockBlobFromReader for more info on creating blobs. 86// 87// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob 88func (b *Blob) CreateBlockBlob(options *PutBlobOptions) error { 89 return b.CreateBlockBlobFromReader(nil, options) 90} 91 92// CreateBlockBlobFromReader initializes a block blob using data from 93// reader. Size must be the number of bytes read from reader. To 94// create an empty blob, use size==0 and reader==nil. 95// 96// Any headers set in blob.Properties or metadata in blob.Metadata 97// will be set on the blob. 98// 99// The API rejects requests with size > 256 MiB (but this limit is not 100// checked by the SDK). To write a larger blob, use CreateBlockBlob, 101// PutBlock, and PutBlockList. 102// 103// To create a blob from scratch, call container.GetBlobReference() to 104// get an empty blob, fill in blob.Properties and blob.Metadata as 105// appropriate then call this method. 106// 107// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob 108func (b *Blob) CreateBlockBlobFromReader(blob io.Reader, options *PutBlobOptions) error { 109 params := url.Values{} 110 headers := b.Container.bsc.client.getStandardHeaders() 111 headers["x-ms-blob-type"] = string(BlobTypeBlock) 112 113 headers["Content-Length"] = "0" 114 var n int64 115 var err error 116 if blob != nil { 117 type lener interface { 118 Len() int 119 } 120 // TODO(rjeczalik): handle io.ReadSeeker, in case blob is *os.File etc. 121 if l, ok := blob.(lener); ok { 122 n = int64(l.Len()) 123 } else { 124 var buf bytes.Buffer 125 n, err = io.Copy(&buf, blob) 126 if err != nil { 127 return err 128 } 129 blob = &buf 130 } 131 132 headers["Content-Length"] = strconv.FormatInt(n, 10) 133 } 134 b.Properties.ContentLength = n 135 136 headers = mergeHeaders(headers, headersFromStruct(b.Properties)) 137 headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata) 138 139 if options != nil { 140 params = addTimeout(params, options.Timeout) 141 headers = mergeHeaders(headers, headersFromStruct(*options)) 142 } 143 uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params) 144 145 resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, blob, b.Container.bsc.auth) 146 if err != nil { 147 return err 148 } 149 return b.respondCreation(resp, BlobTypeBlock) 150} 151 152// PutBlockOptions includes the options for a put block operation 153type PutBlockOptions struct { 154 Timeout uint 155 LeaseID string `header:"x-ms-lease-id"` 156 ContentMD5 string `header:"Content-MD5"` 157 RequestID string `header:"x-ms-client-request-id"` 158} 159 160// PutBlock saves the given data chunk to the specified block blob with 161// given ID. 162// 163// The API rejects chunks larger than 100 MiB (but this limit is not 164// checked by the SDK). 165// 166// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Block 167func (b *Blob) PutBlock(blockID string, chunk []byte, options *PutBlockOptions) error { 168 return b.PutBlockWithLength(blockID, uint64(len(chunk)), bytes.NewReader(chunk), options) 169} 170 171// PutBlockWithLength saves the given data stream of exactly specified size to 172// the block blob with given ID. It is an alternative to PutBlocks where data 173// comes as stream but the length is known in advance. 174// 175// The API rejects requests with size > 100 MiB (but this limit is not 176// checked by the SDK). 177// 178// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Block 179func (b *Blob) PutBlockWithLength(blockID string, size uint64, blob io.Reader, options *PutBlockOptions) error { 180 query := url.Values{ 181 "comp": {"block"}, 182 "blockid": {blockID}, 183 } 184 headers := b.Container.bsc.client.getStandardHeaders() 185 headers["Content-Length"] = fmt.Sprintf("%v", size) 186 187 if options != nil { 188 query = addTimeout(query, options.Timeout) 189 headers = mergeHeaders(headers, headersFromStruct(*options)) 190 } 191 uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), query) 192 193 resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, blob, b.Container.bsc.auth) 194 if err != nil { 195 return err 196 } 197 return b.respondCreation(resp, BlobTypeBlock) 198} 199 200// PutBlockListOptions includes the options for a put block list operation 201type PutBlockListOptions struct { 202 Timeout uint 203 LeaseID string `header:"x-ms-lease-id"` 204 IfModifiedSince *time.Time `header:"If-Modified-Since"` 205 IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"` 206 IfMatch string `header:"If-Match"` 207 IfNoneMatch string `header:"If-None-Match"` 208 RequestID string `header:"x-ms-client-request-id"` 209} 210 211// PutBlockList saves list of blocks to the specified block blob. 212// 213// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Block-List 214func (b *Blob) PutBlockList(blocks []Block, options *PutBlockListOptions) error { 215 params := url.Values{"comp": {"blocklist"}} 216 blockListXML := prepareBlockListRequest(blocks) 217 headers := b.Container.bsc.client.getStandardHeaders() 218 headers["Content-Length"] = fmt.Sprintf("%v", len(blockListXML)) 219 headers = mergeHeaders(headers, headersFromStruct(b.Properties)) 220 headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata) 221 222 if options != nil { 223 params = addTimeout(params, options.Timeout) 224 headers = mergeHeaders(headers, headersFromStruct(*options)) 225 } 226 uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params) 227 228 resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, strings.NewReader(blockListXML), b.Container.bsc.auth) 229 if err != nil { 230 return err 231 } 232 defer drainRespBody(resp) 233 return checkRespCode(resp, []int{http.StatusCreated}) 234} 235 236// GetBlockListOptions includes the options for a get block list operation 237type GetBlockListOptions struct { 238 Timeout uint 239 Snapshot *time.Time 240 LeaseID string `header:"x-ms-lease-id"` 241 RequestID string `header:"x-ms-client-request-id"` 242} 243 244// GetBlockList retrieves list of blocks in the specified block blob. 245// 246// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Get-Block-List 247func (b *Blob) GetBlockList(blockType BlockListType, options *GetBlockListOptions) (BlockListResponse, error) { 248 params := url.Values{ 249 "comp": {"blocklist"}, 250 "blocklisttype": {string(blockType)}, 251 } 252 headers := b.Container.bsc.client.getStandardHeaders() 253 254 if options != nil { 255 params = addTimeout(params, options.Timeout) 256 params = addSnapshot(params, options.Snapshot) 257 headers = mergeHeaders(headers, headersFromStruct(*options)) 258 } 259 uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params) 260 261 var out BlockListResponse 262 resp, err := b.Container.bsc.client.exec(http.MethodGet, uri, headers, nil, b.Container.bsc.auth) 263 if err != nil { 264 return out, err 265 } 266 defer resp.Body.Close() 267 268 err = xmlUnmarshal(resp.Body, &out) 269 return out, err 270} 271