1// Copyright 2020 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package storage 16 17import ( 18 "crypto" 19 "crypto/rand" 20 "crypto/rsa" 21 "crypto/sha256" 22 "encoding/base64" 23 "encoding/json" 24 "errors" 25 "fmt" 26 "net/url" 27 "strings" 28 "time" 29) 30 31// PostPolicyV4Options are used to construct a signed post policy. 32// Please see https://cloud.google.com/storage/docs/xml-api/post-object 33// for reference about the fields. 34type PostPolicyV4Options struct { 35 // GoogleAccessID represents the authorizer of the signed URL generation. 36 // It is typically the Google service account client email address from 37 // the Google Developers Console in the form of "xxx@developer.gserviceaccount.com". 38 // Required. 39 GoogleAccessID string 40 41 // PrivateKey is the Google service account private key. It is obtainable 42 // from the Google Developers Console. 43 // At https://console.developers.google.com/project/<your-project-id>/apiui/credential, 44 // create a service account client ID or reuse one of your existing service account 45 // credentials. Click on the "Generate new P12 key" to generate and download 46 // a new private key. Once you download the P12 file, use the following command 47 // to convert it into a PEM file. 48 // 49 // $ openssl pkcs12 -in key.p12 -passin pass:notasecret -out key.pem -nodes 50 // 51 // Provide the contents of the PEM file as a byte slice. 52 // Exactly one of PrivateKey or SignBytes must be non-nil. 53 PrivateKey []byte 54 55 // SignBytes is a function for implementing custom signing. For example, if 56 // your application is running on Google App Engine, you can use 57 // appengine's internal signing function: 58 // ctx := appengine.NewContext(request) 59 // acc, _ := appengine.ServiceAccount(ctx) 60 // url, err := SignedURL("bucket", "object", &SignedURLOptions{ 61 // GoogleAccessID: acc, 62 // SignBytes: func(b []byte) ([]byte, error) { 63 // _, signedBytes, err := appengine.SignBytes(ctx, b) 64 // return signedBytes, err 65 // }, 66 // // etc. 67 // }) 68 // 69 // Exactly one of PrivateKey or SignBytes must be non-nil. 70 SignBytes func(hashBytes []byte) (signature []byte, err error) 71 72 // Expires is the expiration time on the signed URL. 73 // It must be a time in the future. 74 // Required. 75 Expires time.Time 76 77 // Style provides options for the type of URL to use. Options are 78 // PathStyle (default), BucketBoundHostname, and VirtualHostedStyle. See 79 // https://cloud.google.com/storage/docs/request-endpoints for details. 80 // Optional. 81 Style URLStyle 82 83 // Insecure when set indicates that the generated URL's scheme 84 // will use "http" instead of "https" (default). 85 // Optional. 86 Insecure bool 87 88 // Fields specifies the attributes of a PostPolicyV4 request. 89 // When Fields is non-nil, its attributes must match those that will 90 // passed into field Conditions. 91 // Optional. 92 Fields *PolicyV4Fields 93 94 // The conditions that the uploaded file will be expected to conform to. 95 // When used, the failure of an upload to satisfy a condition will result in 96 // a 4XX status code, back with the message describing the problem. 97 // Optional. 98 Conditions []PostPolicyV4Condition 99} 100 101// PolicyV4Fields describes the attributes for a PostPolicyV4 request. 102type PolicyV4Fields struct { 103 // ACL specifies the access control permissions for the object. 104 // Optional. 105 ACL string 106 // CacheControl specifies the caching directives for the object. 107 // Optional. 108 CacheControl string 109 // ContentType specifies the media type of the object. 110 // Optional. 111 ContentType string 112 // ContentDisposition specifies how the file will be served back to requesters. 113 // Optional. 114 ContentDisposition string 115 // ContentEncoding specifies the decompressive transcoding that the object. 116 // This field is complementary to ContentType in that the file could be 117 // compressed but ContentType specifies the file's original media type. 118 // Optional. 119 ContentEncoding string 120 // Metadata specifies custom metadata for the object. 121 // If any key doesn't begin with "x-goog-meta-", an error will be returned. 122 // Optional. 123 Metadata map[string]string 124 // StatusCodeOnSuccess when set, specifies the status code that Cloud Storage 125 // will serve back on successful upload of the object. 126 // Optional. 127 StatusCodeOnSuccess int 128 // RedirectToURLOnSuccess when set, specifies the URL that Cloud Storage 129 // will serve back on successful upload of the object. 130 // Optional. 131 RedirectToURLOnSuccess string 132} 133 134// PostPolicyV4 describes the URL and respective form fields for a generated PostPolicyV4 request. 135type PostPolicyV4 struct { 136 // URL is the generated URL that the file upload will be made to. 137 URL string 138 // Fields specifies the generated key-values that the file uploader 139 // must include in their multipart upload form. 140 Fields map[string]string 141} 142 143// PostPolicyV4Condition describes the constraints that the subsequent 144// object upload's multipart form fields will be expected to conform to. 145type PostPolicyV4Condition interface { 146 isEmpty() bool 147 json.Marshaler 148} 149 150type startsWith struct { 151 key, value string 152} 153 154func (sw *startsWith) MarshalJSON() ([]byte, error) { 155 return json.Marshal([]string{"starts-with", sw.key, sw.value}) 156} 157func (sw *startsWith) isEmpty() bool { 158 return sw.value == "" 159} 160 161// ConditionStartsWith checks that an attributes starts with value. 162// An empty value will cause this condition to be ignored. 163func ConditionStartsWith(key, value string) PostPolicyV4Condition { 164 return &startsWith{key, value} 165} 166 167type contentLengthRangeCondition struct { 168 start, end uint64 169} 170 171func (clr *contentLengthRangeCondition) MarshalJSON() ([]byte, error) { 172 return json.Marshal([]interface{}{"content-length-range", clr.start, clr.end}) 173} 174func (clr *contentLengthRangeCondition) isEmpty() bool { 175 return clr.start == 0 && clr.end == 0 176} 177 178type singleValueCondition struct { 179 name, value string 180} 181 182func (svc *singleValueCondition) MarshalJSON() ([]byte, error) { 183 return json.Marshal(map[string]string{svc.name: svc.value}) 184} 185func (svc *singleValueCondition) isEmpty() bool { 186 return svc.value == "" 187} 188 189// ConditionContentLengthRange constraints the limits that the 190// multipart upload's range header will be expected to be within. 191func ConditionContentLengthRange(start, end uint64) PostPolicyV4Condition { 192 return &contentLengthRangeCondition{start, end} 193} 194 195func conditionRedirectToURLOnSuccess(redirectURL string) PostPolicyV4Condition { 196 return &singleValueCondition{"success_action_redirect", redirectURL} 197} 198 199func conditionStatusCodeOnSuccess(statusCode int) PostPolicyV4Condition { 200 svc := &singleValueCondition{name: "success_action_status"} 201 if statusCode > 0 { 202 svc.value = fmt.Sprintf("%d", statusCode) 203 } 204 return svc 205} 206 207// GenerateSignedPostPolicyV4 generates a PostPolicyV4 value from bucket, object and opts. 208// The generated URL and fields will then allow an unauthenticated client to perform multipart uploads. 209func GenerateSignedPostPolicyV4(bucket, object string, opts *PostPolicyV4Options) (*PostPolicyV4, error) { 210 if bucket == "" { 211 return nil, errors.New("storage: bucket must be non-empty") 212 } 213 if object == "" { 214 return nil, errors.New("storage: object must be non-empty") 215 } 216 now := utcNow() 217 if err := validatePostPolicyV4Options(opts, now); err != nil { 218 return nil, err 219 } 220 221 var signingFn func(hashedBytes []byte) ([]byte, error) 222 switch { 223 case opts.SignBytes != nil: 224 signingFn = opts.SignBytes 225 226 case len(opts.PrivateKey) != 0: 227 parsedRSAPrivKey, err := parseKey(opts.PrivateKey) 228 if err != nil { 229 return nil, err 230 } 231 signingFn = func(hashedBytes []byte) ([]byte, error) { 232 return rsa.SignPKCS1v15(rand.Reader, parsedRSAPrivKey, crypto.SHA256, hashedBytes) 233 } 234 235 default: 236 return nil, errors.New("storage: exactly one of PrivateKey or SignedBytes must be set") 237 } 238 239 var descFields PolicyV4Fields 240 if opts.Fields != nil { 241 descFields = *opts.Fields 242 } 243 244 if err := validateMetadata(descFields.Metadata); err != nil { 245 return nil, err 246 } 247 248 // Build the policy. 249 conds := make([]PostPolicyV4Condition, len(opts.Conditions)) 250 copy(conds, opts.Conditions) 251 conds = append(conds, 252 // These are ordered lexicographically. Technically the order doesn't matter 253 // for creating the policy, but we use this order to match the 254 // cross-language conformance tests for this feature. 255 &singleValueCondition{"acl", descFields.ACL}, 256 &singleValueCondition{"cache-control", descFields.CacheControl}, 257 &singleValueCondition{"content-disposition", descFields.ContentDisposition}, 258 &singleValueCondition{"content-encoding", descFields.ContentEncoding}, 259 &singleValueCondition{"content-type", descFields.ContentType}, 260 conditionRedirectToURLOnSuccess(descFields.RedirectToURLOnSuccess), 261 conditionStatusCodeOnSuccess(descFields.StatusCodeOnSuccess), 262 ) 263 264 YYYYMMDD := now.Format(yearMonthDay) 265 policyFields := map[string]string{ 266 "key": object, 267 "x-goog-date": now.Format(iso8601), 268 "x-goog-credential": opts.GoogleAccessID + "/" + YYYYMMDD + "/auto/storage/goog4_request", 269 "x-goog-algorithm": "GOOG4-RSA-SHA256", 270 "acl": descFields.ACL, 271 "cache-control": descFields.CacheControl, 272 "content-disposition": descFields.ContentDisposition, 273 "content-encoding": descFields.ContentEncoding, 274 "content-type": descFields.ContentType, 275 "success_action_redirect": descFields.RedirectToURLOnSuccess, 276 } 277 for key, value := range descFields.Metadata { 278 conds = append(conds, &singleValueCondition{key, value}) 279 policyFields[key] = value 280 } 281 282 // Following from the order expected by the conformance test cases, 283 // hence manually inserting these fields in a specific order. 284 conds = append(conds, 285 &singleValueCondition{"bucket", bucket}, 286 &singleValueCondition{"key", object}, 287 &singleValueCondition{"x-goog-date", now.Format(iso8601)}, 288 &singleValueCondition{ 289 name: "x-goog-credential", 290 value: opts.GoogleAccessID + "/" + YYYYMMDD + "/auto/storage/goog4_request", 291 }, 292 &singleValueCondition{"x-goog-algorithm", "GOOG4-RSA-SHA256"}, 293 ) 294 295 nonEmptyConds := make([]PostPolicyV4Condition, 0, len(opts.Conditions)) 296 for _, cond := range conds { 297 if cond == nil || !cond.isEmpty() { 298 nonEmptyConds = append(nonEmptyConds, cond) 299 } 300 } 301 condsAsJSON, err := json.Marshal(map[string]interface{}{ 302 "conditions": nonEmptyConds, 303 "expiration": opts.Expires.Format(time.RFC3339), 304 }) 305 if err != nil { 306 return nil, fmt.Errorf("storage: PostPolicyV4 JSON serialization failed: %v", err) 307 } 308 309 b64Policy := base64.StdEncoding.EncodeToString(condsAsJSON) 310 shaSum := sha256.Sum256([]byte(b64Policy)) 311 signature, err := signingFn(shaSum[:]) 312 if err != nil { 313 return nil, err 314 } 315 316 policyFields["policy"] = b64Policy 317 policyFields["x-goog-signature"] = fmt.Sprintf("%x", signature) 318 319 // Construct the URL. 320 scheme := "https" 321 if opts.Insecure { 322 scheme = "http" 323 } 324 path := opts.Style.path(bucket, "") + "/" 325 u := &url.URL{ 326 Path: path, 327 RawPath: pathEncodeV4(path), 328 Host: opts.Style.host(bucket), 329 Scheme: scheme, 330 } 331 332 if descFields.StatusCodeOnSuccess > 0 { 333 policyFields["success_action_status"] = fmt.Sprintf("%d", descFields.StatusCodeOnSuccess) 334 } 335 336 // Clear out fields with blanks values. 337 for key, value := range policyFields { 338 if value == "" { 339 delete(policyFields, key) 340 } 341 } 342 pp4 := &PostPolicyV4{ 343 Fields: policyFields, 344 URL: u.String(), 345 } 346 return pp4, nil 347} 348 349// validatePostPolicyV4Options checks that: 350// * GoogleAccessID is set 351// * either but not both PrivateKey and SignBytes are set or nil, but not both 352// * Expires, the deadline is not in the past 353// * if Style is not set, it'll use PathStyle 354func validatePostPolicyV4Options(opts *PostPolicyV4Options, now time.Time) error { 355 if opts == nil || opts.GoogleAccessID == "" { 356 return errors.New("storage: missing required GoogleAccessID") 357 } 358 if privBlank, signBlank := len(opts.PrivateKey) == 0, opts.SignBytes == nil; privBlank == signBlank { 359 return errors.New("storage: exactly one of PrivateKey or SignedBytes must be set") 360 } 361 if opts.Expires.Before(now) { 362 return errors.New("storage: expecting Expires to be in the future") 363 } 364 if opts.Style == nil { 365 opts.Style = PathStyle() 366 } 367 return nil 368} 369 370// validateMetadata ensures that all keys passed in have a prefix of "x-goog-meta-", 371// otherwise it will return an error. 372func validateMetadata(hdrs map[string]string) (err error) { 373 if len(hdrs) == 0 { 374 return nil 375 } 376 377 badKeys := make([]string, 0, len(hdrs)) 378 for key := range hdrs { 379 if !strings.HasPrefix(key, "x-goog-meta-") { 380 badKeys = append(badKeys, key) 381 } 382 } 383 if len(badKeys) != 0 { 384 err = errors.New("storage: expected metadata to begin with x-goog-meta-, got " + strings.Join(badKeys, ", ")) 385 } 386 return 387} 388