azure-sdk-for-go/storage/file.go

package storage

// Copyright 2017 Microsoft Corporation
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"sync"
)

const fourMB = uint64(4194304)
const oneTB = uint64(1099511627776)

// Export maximum range and file sizes
const MaxRangeSize = fourMB
const MaxFileSize = oneTB

// File represents a file on a share.
type File struct {
	fsc                *FileServiceClient
	Metadata           map[string]string
	Name               string `xml:"Name"`
	parent             *Directory
	Properties         FileProperties `xml:"Properties"`
	share              *Share
	FileCopyProperties FileCopyState
	mutex              *sync.Mutex
}

// FileProperties contains various properties of a file.
type FileProperties struct {
	CacheControl string `header:"x-ms-cache-control"`
	Disposition  string `header:"x-ms-content-disposition"`
	Encoding     string `header:"x-ms-content-encoding"`
	Etag         string
	Language     string `header:"x-ms-content-language"`
	LastModified string
	Length       uint64 `xml:"Content-Length" header:"x-ms-content-length"`
	MD5          string `header:"x-ms-content-md5"`
	Type         string `header:"x-ms-content-type"`
}

// FileCopyState contains various properties of a file copy operation.
type FileCopyState struct {
	CompletionTime string
	ID             string `header:"x-ms-copy-id"`
	Progress       string
	Source         string
	Status         string `header:"x-ms-copy-status"`
	StatusDesc     string
}

// FileStream contains file data returned from a call to GetFile.
type FileStream struct {
	Body       io.ReadCloser
	ContentMD5 string
}

// FileRequestOptions will be passed to misc file operations.
// Currently just Timeout (in seconds) but could expand.
type FileRequestOptions struct {
	Timeout uint // timeout duration in seconds.
}

func prepareOptions(options *FileRequestOptions) url.Values {
	params := url.Values{}
	if options != nil {
		params = addTimeout(params, options.Timeout)
	}
	return params
}

// FileRanges contains a list of file range information for a file.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Ranges
type FileRanges struct {
	ContentLength uint64
	LastModified  string
	ETag          string
	FileRanges    []FileRange `xml:"Range"`
}

// FileRange contains range information for a file.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Ranges
type FileRange struct {
	Start uint64 `xml:"Start"`
	End   uint64 `xml:"End"`
}

func (fr FileRange) String() string {
	return fmt.Sprintf("bytes=%d-%d", fr.Start, fr.End)
}

// builds the complete file path for this file object
func (f *File) buildPath() string {
	return f.parent.buildPath() + "/" + f.Name
}

// ClearRange releases the specified range of space in a file.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Range
func (f *File) ClearRange(fileRange FileRange, options *FileRequestOptions) error {
	var timeout *uint
	if options != nil {
		timeout = &options.Timeout
	}
	headers, err := f.modifyRange(nil, fileRange, timeout, nil)
	if err != nil {
		return err
	}

	f.updateEtagAndLastModified(headers)
	return nil
}

// Create creates a new file or replaces an existing one.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-File
func (f *File) Create(maxSize uint64, options *FileRequestOptions) error {
	if maxSize > oneTB {
		return fmt.Errorf("max file size is 1TB")
	}
	params := prepareOptions(options)
	headers := headersFromStruct(f.Properties)
	headers["x-ms-content-length"] = strconv.FormatUint(maxSize, 10)
	headers["x-ms-type"] = "file"

	outputHeaders, err := f.fsc.createResource(f.buildPath(), resourceFile, params, mergeMDIntoExtraHeaders(f.Metadata, headers), []int{http.StatusCreated})
	if err != nil {
		return err
	}

	f.Properties.Length = maxSize
	f.updateEtagAndLastModified(outputHeaders)
	return nil
}

// CopyFile operation copied a file/blob from the sourceURL to the path provided.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/copy-file
func (f *File) CopyFile(sourceURL string, options *FileRequestOptions) error {
	extraHeaders := map[string]string{
		"x-ms-type":        "file",
		"x-ms-copy-source": sourceURL,
	}
	params := prepareOptions(options)

	headers, err := f.fsc.createResource(f.buildPath(), resourceFile, params, mergeMDIntoExtraHeaders(f.Metadata, extraHeaders), []int{http.StatusAccepted})
	if err != nil {
		return err
	}

	f.updateEtagAndLastModified(headers)
	f.FileCopyProperties.ID = headers.Get("X-Ms-Copy-Id")
	f.FileCopyProperties.Status = headers.Get("X-Ms-Copy-Status")
	return nil
}

// Delete immediately removes this file from the storage account.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-File2
func (f *File) Delete(options *FileRequestOptions) error {
	return f.fsc.deleteResource(f.buildPath(), resourceFile, options)
}

// DeleteIfExists removes this file if it exists.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-File2
func (f *File) DeleteIfExists(options *FileRequestOptions) (bool, error) {
	resp, err := f.fsc.deleteResourceNoClose(f.buildPath(), resourceFile, options)
	if resp != nil {
		defer drainRespBody(resp)
		if resp.StatusCode == http.StatusAccepted || resp.StatusCode == http.StatusNotFound {
			return resp.StatusCode == http.StatusAccepted, nil
		}
	}
	return false, err
}

// GetFileOptions includes options for a get file operation
type GetFileOptions struct {
	Timeout       uint
	GetContentMD5 bool
}

// DownloadToStream operation downloads the file.
//
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file
func (f *File) DownloadToStream(options *FileRequestOptions) (io.ReadCloser, error) {
	params := prepareOptions(options)
	resp, err := f.fsc.getResourceNoClose(f.buildPath(), compNone, resourceFile, params, http.MethodGet, nil)
	if err != nil {
		return nil, err
	}

	if err = checkRespCode(resp, []int{http.StatusOK}); err != nil {
		drainRespBody(resp)
		return nil, err
	}
	return resp.Body, nil
}

// DownloadRangeToStream operation downloads the specified range of this file with optional MD5 hash.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file
func (f *File) DownloadRangeToStream(fileRange FileRange, options *GetFileOptions) (fs FileStream, err error) {
	extraHeaders := map[string]string{
		"Range": fileRange.String(),
	}
	params := url.Values{}
	if options != nil {
		if options.GetContentMD5 {
			if isRangeTooBig(fileRange) {
				return fs, fmt.Errorf("must specify a range less than or equal to 4MB when getContentMD5 is true")
			}
			extraHeaders["x-ms-range-get-content-md5"] = "true"
		}
		params = addTimeout(params, options.Timeout)
	}

	resp, err := f.fsc.getResourceNoClose(f.buildPath(), compNone, resourceFile, params, http.MethodGet, extraHeaders)
	if err != nil {
		return fs, err
	}

	if err = checkRespCode(resp, []int{http.StatusOK, http.StatusPartialContent}); err != nil {
		drainRespBody(resp)
		return fs, err
	}

	fs.Body = resp.Body
	if options != nil && options.GetContentMD5 {
		fs.ContentMD5 = resp.Header.Get("Content-MD5")
	}
	return fs, nil
}

// Exists returns true if this file exists.
func (f *File) Exists() (bool, error) {
	exists, headers, err := f.fsc.resourceExists(f.buildPath(), resourceFile)
	if exists {
		f.updateEtagAndLastModified(headers)
		f.updateProperties(headers)
	}
	return exists, err
}

// FetchAttributes updates metadata and properties for this file.
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file-properties
func (f *File) FetchAttributes(options *FileRequestOptions) error {
	params := prepareOptions(options)
	headers, err := f.fsc.getResourceHeaders(f.buildPath(), compNone, resourceFile, params, http.MethodHead)
	if err != nil {
		return err
	}

	f.updateEtagAndLastModified(headers)
	f.updateProperties(headers)
	f.Metadata = getMetadataFromHeaders(headers)
	return nil
}

// returns true if the range is larger than 4MB
func isRangeTooBig(fileRange FileRange) bool {
	if fileRange.End-fileRange.Start > fourMB {
		return true
	}

	return false
}

// ListRangesOptions includes options for a list file ranges operation
type ListRangesOptions struct {
	Timeout   uint
	ListRange *FileRange
}

// ListRanges returns the list of valid ranges for this file.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Ranges
func (f *File) ListRanges(options *ListRangesOptions) (*FileRanges, error) {
	params := url.Values{"comp": {"rangelist"}}

	// add optional range to list
	var headers map[string]string
	if options != nil {
		params = addTimeout(params, options.Timeout)
		if options.ListRange != nil {
			headers = make(map[string]string)
			headers["Range"] = options.ListRange.String()
		}
	}

	resp, err := f.fsc.listContent(f.buildPath(), params, headers)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	var cl uint64
	cl, err = strconv.ParseUint(resp.Header.Get("x-ms-content-length"), 10, 64)
	if err != nil {
		ioutil.ReadAll(resp.Body)
		return nil, err
	}

	var out FileRanges
	out.ContentLength = cl
	out.ETag = resp.Header.Get("ETag")
	out.LastModified = resp.Header.Get("Last-Modified")

	err = xmlUnmarshal(resp.Body, &out)
	return &out, err
}

// modifies a range of bytes in this file
func (f *File) modifyRange(bytes io.Reader, fileRange FileRange, timeout *uint, contentMD5 *string) (http.Header, error) {
	if err := f.fsc.checkForStorageEmulator(); err != nil {
		return nil, err
	}
	if fileRange.End < fileRange.Start {
		return nil, errors.New("the value for rangeEnd must be greater than or equal to rangeStart")
	}
	if bytes != nil && isRangeTooBig(fileRange) {
		return nil, errors.New("range cannot exceed 4MB in size")
	}

	params := url.Values{"comp": {"range"}}
	if timeout != nil {
		params = addTimeout(params, *timeout)
	}

	uri := f.fsc.client.getEndpoint(fileServiceName, f.buildPath(), params)

	// default to clear
	write := "clear"
	cl := uint64(0)

	// if bytes is not nil then this is an update operation
	if bytes != nil {
		write = "update"
		cl = (fileRange.End - fileRange.Start) + 1
	}

	extraHeaders := map[string]string{
		"Content-Length": strconv.FormatUint(cl, 10),
		"Range":          fileRange.String(),
		"x-ms-write":     write,
	}

	if contentMD5 != nil {
		extraHeaders["Content-MD5"] = *contentMD5
	}

	headers := mergeHeaders(f.fsc.client.getStandardHeaders(), extraHeaders)
	resp, err := f.fsc.client.exec(http.MethodPut, uri, headers, bytes, f.fsc.auth)
	if err != nil {
		return nil, err
	}
	defer drainRespBody(resp)
	return resp.Header, checkRespCode(resp, []int{http.StatusCreated})
}

// SetMetadata replaces the metadata for this file.
//
// Some keys may be converted to Camel-Case before sending. All keys
// are returned in lower case by GetFileMetadata. HTTP header names
// are case-insensitive so case munging should not matter to other
// applications either.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-File-Metadata
func (f *File) SetMetadata(options *FileRequestOptions) error {
	headers, err := f.fsc.setResourceHeaders(f.buildPath(), compMetadata, resourceFile, mergeMDIntoExtraHeaders(f.Metadata, nil), options)
	if err != nil {
		return err
	}

	f.updateEtagAndLastModified(headers)
	return nil
}

// SetProperties sets system properties on this file.
//
// Some keys may be converted to Camel-Case before sending. All keys
// are returned in lower case by SetFileProperties. HTTP header names
// are case-insensitive so case munging should not matter to other
// applications either.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-File-Properties
func (f *File) SetProperties(options *FileRequestOptions) error {
	headers, err := f.fsc.setResourceHeaders(f.buildPath(), compProperties, resourceFile, headersFromStruct(f.Properties), options)
	if err != nil {
		return err
	}

	f.updateEtagAndLastModified(headers)
	return nil
}

// updates Etag and last modified date
func (f *File) updateEtagAndLastModified(headers http.Header) {
	f.Properties.Etag = headers.Get("Etag")
	f.Properties.LastModified = headers.Get("Last-Modified")
}

// updates file properties from the specified HTTP header
func (f *File) updateProperties(header http.Header) {
	size, err := strconv.ParseUint(header.Get("Content-Length"), 10, 64)
	if err == nil {
		f.Properties.Length = size
	}

	f.updateEtagAndLastModified(header)
	f.Properties.CacheControl = header.Get("Cache-Control")
	f.Properties.Disposition = header.Get("Content-Disposition")
	f.Properties.Encoding = header.Get("Content-Encoding")
	f.Properties.Language = header.Get("Content-Language")
	f.Properties.MD5 = header.Get("Content-MD5")
	f.Properties.Type = header.Get("Content-Type")
}

// URL gets the canonical URL to this file.
// This method does not create a publicly accessible URL if the file
// is private and this method does not check if the file exists.
func (f *File) URL() string {
	return f.fsc.client.getEndpoint(fileServiceName, f.buildPath(), nil)
}

// WriteRangeOptions includes options for a write file range operation
type WriteRangeOptions struct {
	Timeout    uint
	ContentMD5 string
}

// WriteRange writes a range of bytes to this file with an optional MD5 hash of the content (inside
// options parameter). Note that the length of bytes must match (rangeEnd - rangeStart) + 1 with
// a maximum size of 4MB.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Range
func (f *File) WriteRange(bytes io.Reader, fileRange FileRange, options *WriteRangeOptions) error {
	if bytes == nil {
		return errors.New("bytes cannot be nil")
	}
	var timeout *uint
	var md5 *string
	if options != nil {
		timeout = &options.Timeout
		md5 = &options.ContentMD5
	}

	headers, err := f.modifyRange(bytes, fileRange, timeout, md5)
	if err != nil {
		return err
	}
	// it's perfectly legal for multiple go routines to call WriteRange
	// on the same *File (e.g. concurrently writing non-overlapping ranges)
	// so we must take the file mutex before updating our properties.
	f.mutex.Lock()
	f.updateEtagAndLastModified(headers)
	f.mutex.Unlock()
	return nil
}