1// Copyright 2018 The Go Cloud Development Kit Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// https://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15// Package fileblob provides a blob implementation that uses the filesystem. 16// Use OpenBucket to construct a *blob.Bucket. 17// 18// URLs 19// 20// For blob.OpenBucket, fileblob registers for the scheme "file". 21// To customize the URL opener, or for more details on the URL format, 22// see URLOpener. 23// See https://gocloud.dev/concepts/urls/ for background information. 24// 25// Escaping 26// 27// Go CDK supports all UTF-8 strings; to make this work with services lacking 28// full UTF-8 support, strings must be escaped (during writes) and unescaped 29// (during reads). The following escapes are performed for fileblob: 30// - Blob keys: ASCII characters 0-31 are escaped to "__0x<hex>__". 31// If os.PathSeparator != "/", it is also escaped. 32// Additionally, the "/" in "../", the trailing "/" in "//", and a trailing 33// "/" is key names are escaped in the same way. 34// On Windows, the characters "<>:"|?*" are also escaped. 35// 36// As 37// 38// fileblob exposes the following types for As: 39// - Bucket: os.FileInfo 40// - Error: *os.PathError 41// - ListObject: os.FileInfo 42// - Reader: io.Reader 43// - ReaderOptions.BeforeRead: *os.File 44// - Attributes: os.FileInfo 45// - CopyOptions.BeforeCopy: *os.File 46// - WriterOptions.BeforeWrite: *os.File 47 48package fileblob // import "gocloud.dev/blob/fileblob" 49 50import ( 51 "context" 52 "crypto/hmac" 53 "crypto/md5" 54 "crypto/sha256" 55 "encoding/base64" 56 "errors" 57 "fmt" 58 "hash" 59 "io" 60 "io/ioutil" 61 "net/url" 62 "os" 63 "path/filepath" 64 "strconv" 65 "strings" 66 "time" 67 68 "gocloud.dev/blob" 69 "gocloud.dev/blob/driver" 70 "gocloud.dev/gcerrors" 71 "gocloud.dev/internal/escape" 72 "gocloud.dev/internal/gcerr" 73) 74 75const defaultPageSize = 1000 76 77func init() { 78 blob.DefaultURLMux().RegisterBucket(Scheme, &URLOpener{}) 79} 80 81// Scheme is the URL scheme fileblob registers its URLOpener under on 82// blob.DefaultMux. 83const Scheme = "file" 84 85// URLOpener opens file bucket URLs like "file:///foo/bar/baz". 86// 87// The URL's host is ignored unless it is ".", which is used to signal a 88// relative path. For example, "file://./../.." uses "../.." as the path. 89// 90// If os.PathSeparator != "/", any leading "/" from the path is dropped 91// and remaining '/' characters are converted to os.PathSeparator. 92// 93// The following query parameters are supported: 94// 95// - create_dir: (any non-empty value) the directory is created (using os.MkDirAll) 96// if it does not already exist. 97// - base_url: the base URL to use to construct signed URLs; see URLSignerHMAC 98// - secret_key_path: path to read for the secret key used to construct signed URLs; 99// see URLSignerHMAC 100// 101// If either of base_url / secret_key_path are provided, both must be. 102// 103// - file:///a/directory 104// -> Passes "/a/directory" to OpenBucket. 105// - file://localhost/a/directory 106// -> Also passes "/a/directory". 107// - file://./../.. 108// -> The hostname is ".", signaling a relative path; passes "../..". 109// - file:///c:/foo/bar on Windows. 110// -> Passes "c:\foo\bar". 111// - file://localhost/c:/foo/bar on Windows. 112// -> Also passes "c:\foo\bar". 113// - file:///a/directory?base_url=/show&secret_key_path=secret.key 114// -> Passes "/a/directory" to OpenBucket, and sets Options.URLSigner 115// to a URLSignerHMAC initialized with base URL "/show" and secret key 116// bytes read from the file "secret.key". 117type URLOpener struct { 118 // Options specifies the default options to pass to OpenBucket. 119 Options Options 120} 121 122// OpenBucketURL opens a blob.Bucket based on u. 123func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket, error) { 124 path := u.Path 125 // Hostname == "." means a relative path, so drop the leading "/". 126 // Also drop the leading "/" on Windows. 127 if u.Host == "." || os.PathSeparator != '/' { 128 path = strings.TrimPrefix(path, "/") 129 } 130 opts, err := o.forParams(ctx, u.Query()) 131 if err != nil { 132 return nil, fmt.Errorf("open bucket %v: %v", u, err) 133 } 134 return OpenBucket(filepath.FromSlash(path), opts) 135} 136 137func (o *URLOpener) forParams(ctx context.Context, q url.Values) (*Options, error) { 138 for k := range q { 139 if k != "create_dir" && k != "base_url" && k != "secret_key_path" { 140 return nil, fmt.Errorf("invalid query parameter %q", k) 141 } 142 } 143 opts := new(Options) 144 *opts = o.Options 145 146 if q.Get("create_dir") != "" { 147 opts.CreateDir = true 148 } 149 baseURL := q.Get("base_url") 150 keyPath := q.Get("secret_key_path") 151 if (baseURL == "") != (keyPath == "") { 152 return nil, errors.New("must supply both base_url and secret_key_path query parameters") 153 } 154 if baseURL != "" { 155 burl, err := url.Parse(baseURL) 156 if err != nil { 157 return nil, err 158 } 159 sk, err := ioutil.ReadFile(keyPath) 160 if err != nil { 161 return nil, err 162 } 163 opts.URLSigner = NewURLSignerHMAC(burl, sk) 164 } 165 return opts, nil 166} 167 168// Options sets options for constructing a *blob.Bucket backed by fileblob. 169type Options struct { 170 // URLSigner implements signing URLs (to allow access to a resource without 171 // further authorization) and verifying that a given URL is unexpired and 172 // contains a signature produced by the URLSigner. 173 // URLSigner is only required for utilizing the SignedURL API. 174 URLSigner URLSigner 175 176 // If true, create the directory backing the Bucket if it does not exist 177 // (using os.MkdirAll). 178 CreateDir bool 179} 180 181type bucket struct { 182 dir string 183 opts *Options 184} 185 186// openBucket creates a driver.Bucket that reads and writes to dir. 187// dir must exist. 188func openBucket(dir string, opts *Options) (driver.Bucket, error) { 189 if opts == nil { 190 opts = &Options{} 191 } 192 absdir, err := filepath.Abs(dir) 193 if err != nil { 194 return nil, fmt.Errorf("failed to convert %s into an absolute path: %v", dir, err) 195 } 196 info, err := os.Stat(absdir) 197 198 // Optionally, create the directory if it does not already exist. 199 if err != nil && opts.CreateDir && os.IsNotExist(err) { 200 err = os.MkdirAll(absdir, os.ModeDir) 201 if err != nil { 202 return nil, fmt.Errorf("tried to create directory but failed: %v", err) 203 } 204 info, err = os.Stat(absdir) 205 } 206 if err != nil { 207 return nil, err 208 } 209 if !info.IsDir() { 210 return nil, fmt.Errorf("%s is not a directory", absdir) 211 } 212 return &bucket{dir: absdir, opts: opts}, nil 213} 214 215// OpenBucket creates a *blob.Bucket backed by the filesystem and rooted at 216// dir, which must exist. See the package documentation for an example. 217func OpenBucket(dir string, opts *Options) (*blob.Bucket, error) { 218 drv, err := openBucket(dir, opts) 219 if err != nil { 220 return nil, err 221 } 222 return blob.NewBucket(drv), nil 223} 224 225func (b *bucket) Close() error { 226 return nil 227} 228 229// escapeKey does all required escaping for UTF-8 strings to work the filesystem. 230func escapeKey(s string) string { 231 s = escape.HexEscape(s, func(r []rune, i int) bool { 232 c := r[i] 233 switch { 234 case c < 32: 235 return true 236 // We're going to replace '/' with os.PathSeparator below. In order for this 237 // to be reversible, we need to escape raw os.PathSeparators. 238 case os.PathSeparator != '/' && c == os.PathSeparator: 239 return true 240 // For "../", escape the trailing slash. 241 case i > 1 && c == '/' && r[i-1] == '.' && r[i-2] == '.': 242 return true 243 // For "//", escape the trailing slash. 244 case i > 0 && c == '/' && r[i-1] == '/': 245 return true 246 // Escape the trailing slash in a key. 247 case c == '/' && i == len(r)-1: 248 return true 249 // https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file 250 case os.PathSeparator == '\\' && (c == '>' || c == '<' || c == ':' || c == '"' || c == '|' || c == '?' || c == '*'): 251 return true 252 } 253 return false 254 }) 255 // Replace "/" with os.PathSeparator if needed, so that the local filesystem 256 // can use subdirectories. 257 if os.PathSeparator != '/' { 258 s = strings.Replace(s, "/", string(os.PathSeparator), -1) 259 } 260 return s 261} 262 263// unescapeKey reverses escapeKey. 264func unescapeKey(s string) string { 265 if os.PathSeparator != '/' { 266 s = strings.Replace(s, string(os.PathSeparator), "/", -1) 267 } 268 s = escape.HexUnescape(s) 269 return s 270} 271 272func (b *bucket) ErrorCode(err error) gcerrors.ErrorCode { 273 switch { 274 case os.IsNotExist(err): 275 return gcerrors.NotFound 276 default: 277 return gcerrors.Unknown 278 } 279} 280 281// path returns the full path for a key 282func (b *bucket) path(key string) (string, error) { 283 path := filepath.Join(b.dir, escapeKey(key)) 284 if strings.HasSuffix(path, attrsExt) { 285 return "", errAttrsExt 286 } 287 return path, nil 288} 289 290// forKey returns the full path, os.FileInfo, and attributes for key. 291func (b *bucket) forKey(key string) (string, os.FileInfo, *xattrs, error) { 292 path, err := b.path(key) 293 if err != nil { 294 return "", nil, nil, err 295 } 296 info, err := os.Stat(path) 297 if err != nil { 298 return "", nil, nil, err 299 } 300 if info.IsDir() { 301 return "", nil, nil, os.ErrNotExist 302 } 303 xa, err := getAttrs(path) 304 if err != nil { 305 return "", nil, nil, err 306 } 307 return path, info, &xa, nil 308} 309 310// ListPaged implements driver.ListPaged. 311func (b *bucket) ListPaged(ctx context.Context, opts *driver.ListOptions) (*driver.ListPage, error) { 312 313 var pageToken string 314 if len(opts.PageToken) > 0 { 315 pageToken = string(opts.PageToken) 316 } 317 pageSize := opts.PageSize 318 if pageSize == 0 { 319 pageSize = defaultPageSize 320 } 321 // If opts.Delimiter != "", lastPrefix contains the last "directory" key we 322 // added. It is used to avoid adding it again; all files in this "directory" 323 // are collapsed to the single directory entry. 324 var lastPrefix string 325 326 // If the Prefix contains a "/", we can set the root of the Walk 327 // to the path specified by the Prefix as any files below the path will not 328 // match the Prefix. 329 // Note that we use "/" explicitly and not os.PathSeparator, as the opts.Prefix 330 // is in the unescaped form. 331 root := b.dir 332 if i := strings.LastIndex(opts.Prefix, "/"); i > -1 { 333 root = filepath.Join(root, opts.Prefix[:i]) 334 } 335 336 // Do a full recursive scan of the root directory. 337 var result driver.ListPage 338 err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { 339 if err != nil { 340 // Couldn't read this file/directory for some reason; just skip it. 341 return nil 342 } 343 // Skip the self-generated attribute files. 344 if strings.HasSuffix(path, attrsExt) { 345 return nil 346 } 347 // os.Walk returns the root directory; skip it. 348 if path == b.dir { 349 return nil 350 } 351 // Strip the <b.dir> prefix from path. 352 prefixLen := len(b.dir) 353 // Include the separator for non-root. 354 if b.dir != "/" { 355 prefixLen++ 356 } 357 path = path[prefixLen:] 358 // Unescape the path to get the key. 359 key := unescapeKey(path) 360 // Skip all directories. If opts.Delimiter is set, we'll create 361 // pseudo-directories later. 362 // Note that returning nil means that we'll still recurse into it; 363 // we're just not adding a result for the directory itself. 364 if info.IsDir() { 365 key += "/" 366 // Avoid recursing into subdirectories if the directory name already 367 // doesn't match the prefix; any files in it are guaranteed not to match. 368 if len(key) > len(opts.Prefix) && !strings.HasPrefix(key, opts.Prefix) { 369 return filepath.SkipDir 370 } 371 // Similarly, avoid recursing into subdirectories if we're making 372 // "directories" and all of the files in this subdirectory are guaranteed 373 // to collapse to a "directory" that we've already added. 374 if lastPrefix != "" && strings.HasPrefix(key, lastPrefix) { 375 return filepath.SkipDir 376 } 377 return nil 378 } 379 // Skip files/directories that don't match the Prefix. 380 if !strings.HasPrefix(key, opts.Prefix) { 381 return nil 382 } 383 var md5 []byte 384 if xa, err := getAttrs(path); err == nil { 385 // Note: we only have the MD5 hash for blobs that we wrote. 386 // For other blobs, md5 will remain nil. 387 md5 = xa.MD5 388 } 389 asFunc := func(i interface{}) bool { 390 p, ok := i.(*os.FileInfo) 391 if !ok { 392 return false 393 } 394 *p = info 395 return true 396 } 397 obj := &driver.ListObject{ 398 Key: key, 399 ModTime: info.ModTime(), 400 Size: info.Size(), 401 MD5: md5, 402 AsFunc: asFunc, 403 } 404 // If using Delimiter, collapse "directories". 405 if opts.Delimiter != "" { 406 // Strip the prefix, which may contain Delimiter. 407 keyWithoutPrefix := key[len(opts.Prefix):] 408 // See if the key still contains Delimiter. 409 // If no, it's a file and we just include it. 410 // If yes, it's a file in a "sub-directory" and we want to collapse 411 // all files in that "sub-directory" into a single "directory" result. 412 if idx := strings.Index(keyWithoutPrefix, opts.Delimiter); idx != -1 { 413 prefix := opts.Prefix + keyWithoutPrefix[0:idx+len(opts.Delimiter)] 414 // We've already included this "directory"; don't add it. 415 if prefix == lastPrefix { 416 return nil 417 } 418 // Update the object to be a "directory". 419 obj = &driver.ListObject{ 420 Key: prefix, 421 IsDir: true, 422 AsFunc: asFunc, 423 } 424 lastPrefix = prefix 425 } 426 } 427 // If there's a pageToken, skip anything before it. 428 if pageToken != "" && obj.Key <= pageToken { 429 return nil 430 } 431 // If we've already got a full page of results, set NextPageToken and stop. 432 if len(result.Objects) == pageSize { 433 result.NextPageToken = []byte(result.Objects[pageSize-1].Key) 434 return io.EOF 435 } 436 result.Objects = append(result.Objects, obj) 437 return nil 438 }) 439 if err != nil && err != io.EOF { 440 return nil, err 441 } 442 return &result, nil 443} 444 445// As implements driver.As. 446func (b *bucket) As(i interface{}) bool { 447 p, ok := i.(*os.FileInfo) 448 if !ok { 449 return false 450 } 451 fi, err := os.Stat(b.dir) 452 if err != nil { 453 return false 454 } 455 *p = fi 456 return true 457} 458 459// As implements driver.ErrorAs. 460func (b *bucket) ErrorAs(err error, i interface{}) bool { 461 if perr, ok := err.(*os.PathError); ok { 462 if p, ok := i.(**os.PathError); ok { 463 *p = perr 464 return true 465 } 466 } 467 return false 468} 469 470// Attributes implements driver.Attributes. 471func (b *bucket) Attributes(ctx context.Context, key string) (*driver.Attributes, error) { 472 _, info, xa, err := b.forKey(key) 473 if err != nil { 474 return nil, err 475 } 476 return &driver.Attributes{ 477 CacheControl: xa.CacheControl, 478 ContentDisposition: xa.ContentDisposition, 479 ContentEncoding: xa.ContentEncoding, 480 ContentLanguage: xa.ContentLanguage, 481 ContentType: xa.ContentType, 482 Metadata: xa.Metadata, 483 // CreateTime left as the zero time. 484 ModTime: info.ModTime(), 485 Size: info.Size(), 486 MD5: xa.MD5, 487 ETag: fmt.Sprintf("\"%x-%x\"", info.ModTime().UnixNano(), info.Size()), 488 AsFunc: func(i interface{}) bool { 489 p, ok := i.(*os.FileInfo) 490 if !ok { 491 return false 492 } 493 *p = info 494 return true 495 }, 496 }, nil 497} 498 499// NewRangeReader implements driver.NewRangeReader. 500func (b *bucket) NewRangeReader(ctx context.Context, key string, offset, length int64, opts *driver.ReaderOptions) (driver.Reader, error) { 501 path, info, xa, err := b.forKey(key) 502 if err != nil { 503 return nil, err 504 } 505 f, err := os.Open(path) 506 if err != nil { 507 return nil, err 508 } 509 if opts.BeforeRead != nil { 510 if err := opts.BeforeRead(func(i interface{}) bool { 511 p, ok := i.(**os.File) 512 if !ok { 513 return false 514 } 515 *p = f 516 return true 517 }); err != nil { 518 return nil, err 519 } 520 } 521 if offset > 0 { 522 if _, err := f.Seek(offset, io.SeekStart); err != nil { 523 return nil, err 524 } 525 } 526 r := io.Reader(f) 527 if length >= 0 { 528 r = io.LimitReader(r, length) 529 } 530 return &reader{ 531 r: r, 532 c: f, 533 attrs: driver.ReaderAttributes{ 534 ContentType: xa.ContentType, 535 ModTime: info.ModTime(), 536 Size: info.Size(), 537 }, 538 }, nil 539} 540 541type reader struct { 542 r io.Reader 543 c io.Closer 544 attrs driver.ReaderAttributes 545} 546 547func (r *reader) Read(p []byte) (int, error) { 548 if r.r == nil { 549 return 0, io.EOF 550 } 551 return r.r.Read(p) 552} 553 554func (r *reader) Close() error { 555 if r.c == nil { 556 return nil 557 } 558 return r.c.Close() 559} 560 561func (r *reader) Attributes() *driver.ReaderAttributes { 562 return &r.attrs 563} 564 565func (r *reader) As(i interface{}) bool { 566 p, ok := i.(*io.Reader) 567 if !ok { 568 return false 569 } 570 *p = r.r 571 return true 572} 573 574// NewTypedWriter implements driver.NewTypedWriter. 575func (b *bucket) NewTypedWriter(ctx context.Context, key string, contentType string, opts *driver.WriterOptions) (driver.Writer, error) { 576 path, err := b.path(key) 577 if err != nil { 578 return nil, err 579 } 580 if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil { 581 return nil, err 582 } 583 f, err := ioutil.TempFile(filepath.Dir(path), "fileblob") 584 if err != nil { 585 return nil, err 586 } 587 if opts.BeforeWrite != nil { 588 if err := opts.BeforeWrite(func(i interface{}) bool { 589 p, ok := i.(**os.File) 590 if !ok { 591 return false 592 } 593 *p = f 594 return true 595 }); err != nil { 596 return nil, err 597 } 598 } 599 var metadata map[string]string 600 if len(opts.Metadata) > 0 { 601 metadata = opts.Metadata 602 } 603 attrs := xattrs{ 604 CacheControl: opts.CacheControl, 605 ContentDisposition: opts.ContentDisposition, 606 ContentEncoding: opts.ContentEncoding, 607 ContentLanguage: opts.ContentLanguage, 608 ContentType: contentType, 609 Metadata: metadata, 610 } 611 w := &writer{ 612 ctx: ctx, 613 f: f, 614 path: path, 615 attrs: attrs, 616 contentMD5: opts.ContentMD5, 617 md5hash: md5.New(), 618 } 619 return w, nil 620} 621 622type writer struct { 623 ctx context.Context 624 f *os.File 625 path string 626 attrs xattrs 627 contentMD5 []byte 628 // We compute the MD5 hash so that we can store it with the file attributes, 629 // not for verification. 630 md5hash hash.Hash 631} 632 633func (w *writer) Write(p []byte) (n int, err error) { 634 if _, err := w.md5hash.Write(p); err != nil { 635 return 0, err 636 } 637 return w.f.Write(p) 638} 639 640func (w *writer) Close() error { 641 err := w.f.Close() 642 if err != nil { 643 return err 644 } 645 // Always delete the temp file. On success, it will have been renamed so 646 // the Remove will fail. 647 defer func() { 648 _ = os.Remove(w.f.Name()) 649 }() 650 651 // Check if the write was cancelled. 652 if err := w.ctx.Err(); err != nil { 653 return err 654 } 655 656 md5sum := w.md5hash.Sum(nil) 657 w.attrs.MD5 = md5sum 658 659 // Write the attributes file. 660 if err := setAttrs(w.path, w.attrs); err != nil { 661 return err 662 } 663 // Rename the temp file to path. 664 if err := os.Rename(w.f.Name(), w.path); err != nil { 665 _ = os.Remove(w.path + attrsExt) 666 return err 667 } 668 return nil 669} 670 671// Copy implements driver.Copy. 672func (b *bucket) Copy(ctx context.Context, dstKey, srcKey string, opts *driver.CopyOptions) error { 673 // Note: we could use NewRangeReader here, but since we need to copy all of 674 // the metadata (from xa), it's more efficient to do it directly. 675 srcPath, _, xa, err := b.forKey(srcKey) 676 if err != nil { 677 return err 678 } 679 f, err := os.Open(srcPath) 680 if err != nil { 681 return err 682 } 683 defer f.Close() 684 685 // We'll write the copy using Writer, to avoid re-implementing making of a 686 // temp file, cleaning up after partial failures, etc. 687 wopts := driver.WriterOptions{ 688 CacheControl: xa.CacheControl, 689 ContentDisposition: xa.ContentDisposition, 690 ContentEncoding: xa.ContentEncoding, 691 ContentLanguage: xa.ContentLanguage, 692 Metadata: xa.Metadata, 693 BeforeWrite: opts.BeforeCopy, 694 } 695 // Create a cancelable context so we can cancel the write if there are 696 // problems. 697 writeCtx, cancel := context.WithCancel(ctx) 698 defer cancel() 699 w, err := b.NewTypedWriter(writeCtx, dstKey, xa.ContentType, &wopts) 700 if err != nil { 701 return err 702 } 703 _, err = io.Copy(w, f) 704 if err != nil { 705 cancel() // cancel before Close cancels the write 706 w.Close() 707 return err 708 } 709 return w.Close() 710} 711 712// Delete implements driver.Delete. 713func (b *bucket) Delete(ctx context.Context, key string) error { 714 path, err := b.path(key) 715 if err != nil { 716 return err 717 } 718 err = os.Remove(path) 719 if err != nil { 720 return err 721 } 722 if err = os.Remove(path + attrsExt); err != nil && !os.IsNotExist(err) { 723 return err 724 } 725 return nil 726} 727 728// SignedURL implements driver.SignedURL 729func (b *bucket) SignedURL(ctx context.Context, key string, opts *driver.SignedURLOptions) (string, error) { 730 if b.opts.URLSigner == nil { 731 return "", gcerr.New(gcerr.Unimplemented, nil, 1, "fileblob.SignedURL: bucket does not have an Options.URLSigner") 732 } 733 if opts.BeforeSign != nil { 734 if err := opts.BeforeSign(func(interface{}) bool { return false }); err != nil { 735 return "", err 736 } 737 } 738 surl, err := b.opts.URLSigner.URLFromKey(ctx, key, opts) 739 if err != nil { 740 return "", err 741 } 742 return surl.String(), nil 743} 744 745// URLSigner defines an interface for creating and verifying a signed URL for 746// objects in a fileblob bucket. Signed URLs are typically used for granting 747// access to an otherwise-protected resource without requiring further 748// authentication, and callers should take care to restrict the creation of 749// signed URLs as is appropriate for their application. 750type URLSigner interface { 751 // URLFromKey defines how the bucket's object key will be turned 752 // into a signed URL. URLFromKey must be safe to call from multiple goroutines. 753 URLFromKey(ctx context.Context, key string, opts *driver.SignedURLOptions) (*url.URL, error) 754 755 // KeyFromURL must be able to validate a URL returned from URLFromKey. 756 // KeyFromURL must only return the object if if the URL is 757 // both unexpired and authentic. KeyFromURL must be safe to call from 758 // multiple goroutines. Implementations of KeyFromURL should not modify 759 // the URL argument. 760 KeyFromURL(ctx context.Context, surl *url.URL) (string, error) 761} 762 763// URLSignerHMAC signs URLs by adding the object key, expiration time, and a 764// hash-based message authentication code (HMAC) into the query parameters. 765// Values of URLSignerHMAC with the same secret key will accept URLs produced by 766// others as valid. 767type URLSignerHMAC struct { 768 baseURL *url.URL 769 secretKey []byte 770} 771 772// NewURLSignerHMAC creates a URLSignerHMAC. If the secret key is empty, 773// then NewURLSignerHMAC panics. 774func NewURLSignerHMAC(baseURL *url.URL, secretKey []byte) *URLSignerHMAC { 775 if len(secretKey) == 0 { 776 panic("creating URLSignerHMAC: secretKey is required") 777 } 778 uc := new(url.URL) 779 *uc = *baseURL 780 return &URLSignerHMAC{ 781 baseURL: uc, 782 secretKey: secretKey, 783 } 784} 785 786// URLFromKey creates a signed URL by copying the baseURL and appending the 787// object key, expiry, and signature as a query params. 788func (h *URLSignerHMAC) URLFromKey(ctx context.Context, key string, opts *driver.SignedURLOptions) (*url.URL, error) { 789 sURL := new(url.URL) 790 *sURL = *h.baseURL 791 792 q := sURL.Query() 793 q.Set("obj", key) 794 q.Set("expiry", strconv.FormatInt(time.Now().Add(opts.Expiry).Unix(), 10)) 795 q.Set("method", opts.Method) 796 if opts.ContentType != "" { 797 q.Set("contentType", opts.ContentType) 798 } 799 q.Set("signature", h.getMAC(q)) 800 sURL.RawQuery = q.Encode() 801 802 return sURL, nil 803} 804 805func (h *URLSignerHMAC) getMAC(q url.Values) string { 806 signedVals := url.Values{} 807 signedVals.Set("obj", q.Get("obj")) 808 signedVals.Set("expiry", q.Get("expiry")) 809 signedVals.Set("method", q.Get("method")) 810 if contentType := q.Get("contentType"); contentType != "" { 811 signedVals.Set("contentType", contentType) 812 } 813 msg := signedVals.Encode() 814 815 hsh := hmac.New(sha256.New, h.secretKey) 816 hsh.Write([]byte(msg)) 817 return base64.RawURLEncoding.EncodeToString(hsh.Sum(nil)) 818} 819 820// KeyFromURL checks expiry and signature, and returns the object key 821// only if the signed URL is both authentic and unexpired. 822func (h *URLSignerHMAC) KeyFromURL(ctx context.Context, sURL *url.URL) (string, error) { 823 q := sURL.Query() 824 825 exp, err := strconv.ParseInt(q.Get("expiry"), 10, 64) 826 if err != nil || time.Now().Unix() > exp { 827 return "", errors.New("retrieving blob key from URL: key cannot be retrieved") 828 } 829 830 if !h.checkMAC(q) { 831 return "", errors.New("retrieving blob key from URL: key cannot be retrieved") 832 } 833 return q.Get("obj"), nil 834} 835 836func (h *URLSignerHMAC) checkMAC(q url.Values) bool { 837 mac := q.Get("signature") 838 expected := h.getMAC(q) 839 // This compares the Base-64 encoded MACs 840 return hmac.Equal([]byte(mac), []byte(expected)) 841} 842