1# Copyright 2020 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import json 16import logging 17 18import database 19import flask 20import gcs as gcs_type 21import grpc_server 22import httpbin 23import utils 24from werkzeug import serving 25from werkzeug.middleware.dispatcher import DispatcherMiddleware 26 27from google.cloud.storage_v1.proto import storage_resources_pb2 as resources_pb2 28from google.cloud.storage_v1.proto.storage_resources_pb2 import CommonEnums 29from google.protobuf import json_format 30 31db = None 32grpc_port = 0 33 34# === DEFAULT ENTRY FOR REST SERVER === # 35root = flask.Flask(__name__) 36root.debug = True 37 38 39@root.route("/") 40def index(): 41 return "OK" 42 43 44@root.route("/start_grpc") 45def start_grpc(): 46 # We need to do this because `gunicorn` will spawn a new subprocess ( a worker ) 47 # when running `Flask` server. If we start `gRPC` server before the spawn of 48 # the subprocess, it's nearly impossible to share the `database` with the new 49 # subprocess because Python will copy everything in the memory from the parent 50 # process to the subprocess ( So we have 2 separate instance of `database` ). 51 # The endpoint will start the `gRPC` server in the same subprocess so there is 52 # only one instance of `database`. 53 global grpc_port 54 if grpc_port == 0: 55 port = flask.request.args.get("port", "0") 56 grpc_port = grpc_server.run(port, db) 57 return str(grpc_port) 58 return str(grpc_port) 59 60 61@root.route("/raise_error") 62def raise_error(): 63 etype = flask.request.args.get("etype") 64 msg = flask.request.args.get("msg", "") 65 if etype is not None: 66 raise TypeError(msg) 67 else: 68 raise Exception(msg) 69 70 71@root.route("/<path:object_name>", subdomain="<bucket_name>") 72def root_get_object(bucket_name, object_name): 73 return xml_get_object(bucket_name, object_name) 74 75 76@root.route("/<bucket_name>/<path:object_name>", subdomain="") 77def root_get_object_with_bucket(bucket_name, object_name): 78 return xml_get_object(bucket_name, object_name) 79 80 81@root.route("/<path:object_name>", subdomain="<bucket_name>", methods=["PUT"]) 82def root_put_object(bucket_name, object_name): 83 return xml_put_object(bucket_name, object_name) 84 85 86@root.route("/<bucket_name>/<path:object_name>", subdomain="", methods=["PUT"]) 87def root_put_object_with_bucket(bucket_name, object_name): 88 return xml_put_object(bucket_name, object_name) 89 90 91# === WSGI APP TO HANDLE JSON API === # 92GCS_HANDLER_PATH = "/storage/v1" 93gcs = flask.Flask(__name__) 94gcs.debug = True 95 96 97# === BUCKET === # 98 99 100@gcs.route("/b", methods=["GET"]) 101def bucket_list(): 102 db.insert_test_bucket(None) 103 project = flask.request.args.get("project") 104 projection = flask.request.args.get("projection", "noAcl") 105 fields = flask.request.args.get("fields", None) 106 response = { 107 "kind": "storage#buckets", 108 "nextPageToken": "", 109 "items": [ 110 bucket.rest() for bucket in db.list_bucket(flask.request, project, None) 111 ], 112 } 113 return utils.common.filter_response_rest(response, projection, fields) 114 115 116@gcs.route("/b", methods=["POST"]) 117def bucket_insert(): 118 db.insert_test_bucket(None) 119 bucket, projection = gcs_type.bucket.Bucket.init(flask.request, None) 120 fields = flask.request.args.get("fields", None) 121 db.insert_bucket(flask.request, bucket, None) 122 return utils.common.filter_response_rest(bucket.rest(), projection, fields) 123 124 125@gcs.route("/b/<bucket_name>") 126def bucket_get(bucket_name): 127 db.insert_test_bucket(None) 128 db.insert_test_bucket(None) 129 bucket = db.get_bucket(flask.request, bucket_name, None) 130 projection = utils.common.extract_projection( 131 flask.request, CommonEnums.Projection.NO_ACL, None 132 ) 133 fields = flask.request.args.get("fields", None) 134 return utils.common.filter_response_rest(bucket.rest(), projection, fields) 135 136 137@gcs.route("/b/<bucket_name>", methods=["PUT"]) 138def bucket_update(bucket_name): 139 db.insert_test_bucket(None) 140 bucket = db.get_bucket(flask.request, bucket_name, None) 141 bucket.update(flask.request, None) 142 projection = utils.common.extract_projection( 143 flask.request, CommonEnums.Projection.FULL, None 144 ) 145 fields = flask.request.args.get("fields", None) 146 return utils.common.filter_response_rest(bucket.rest(), projection, fields) 147 148 149@gcs.route("/b/<bucket_name>", methods=["PATCH"]) 150def bucket_patch(bucket_name): 151 bucket = db.get_bucket(flask.request, bucket_name, None) 152 bucket.patch(flask.request, None) 153 projection = utils.common.extract_projection( 154 flask.request, CommonEnums.Projection.FULL, None 155 ) 156 fields = flask.request.args.get("fields", None) 157 return utils.common.filter_response_rest(bucket.rest(), projection, fields) 158 159 160@gcs.route("/b/<bucket_name>", methods=["DELETE"]) 161def bucket_delete(bucket_name): 162 db.delete_bucket(flask.request, bucket_name, None) 163 return "" 164 165 166# === BUCKET ACL === # 167 168 169@gcs.route("/b/<bucket_name>/acl") 170def bucket_acl_list(bucket_name): 171 bucket = db.get_bucket(flask.request, bucket_name, None) 172 response = {"kind": "storage#bucketAccessControls", "items": []} 173 for acl in bucket.metadata.acl: 174 acl_rest = json_format.MessageToDict(acl) 175 acl_rest["kind"] = "storage#bucketAccessControl" 176 response["items"].append(acl_rest) 177 fields = flask.request.args.get("fields", None) 178 return utils.common.filter_response_rest(response, None, fields) 179 180 181@gcs.route("/b/<bucket_name>/acl", methods=["POST"]) 182def bucket_acl_insert(bucket_name): 183 bucket = db.get_bucket(flask.request, bucket_name, None) 184 acl = bucket.insert_acl(flask.request, None) 185 response = json_format.MessageToDict(acl) 186 response["kind"] = "storage#bucketAccessControl" 187 fields = flask.request.args.get("fields", None) 188 return utils.common.filter_response_rest(response, None, fields) 189 190 191@gcs.route("/b/<bucket_name>/acl/<entity>") 192def bucket_acl_get(bucket_name, entity): 193 bucket = db.get_bucket(flask.request, bucket_name, None) 194 acl = bucket.get_acl(entity, None) 195 response = json_format.MessageToDict(acl) 196 response["kind"] = "storage#bucketAccessControl" 197 fields = flask.request.args.get("fields", None) 198 return utils.common.filter_response_rest(response, None, fields) 199 200 201@gcs.route("/b/<bucket_name>/acl/<entity>", methods=["PUT"]) 202def bucket_acl_update(bucket_name, entity): 203 bucket = db.get_bucket(flask.request, bucket_name, None) 204 acl = bucket.update_acl(flask.request, entity, None) 205 response = json_format.MessageToDict(acl) 206 response["kind"] = "storage#bucketAccessControl" 207 fields = flask.request.args.get("fields", None) 208 return utils.common.filter_response_rest(response, None, fields) 209 210 211@gcs.route("/b/<bucket_name>/acl/<entity>", methods=["PATCH"]) 212def bucket_acl_patch(bucket_name, entity): 213 bucket = db.get_bucket(flask.request, bucket_name, None) 214 acl = bucket.patch_acl(flask.request, entity, None) 215 response = json_format.MessageToDict(acl) 216 response["kind"] = "storage#bucketAccessControl" 217 fields = flask.request.args.get("fields", None) 218 return utils.common.filter_response_rest(response, None, fields) 219 220 221@gcs.route("/b/<bucket_name>/acl/<entity>", methods=["DELETE"]) 222def bucket_acl_delete(bucket_name, entity): 223 bucket = db.get_bucket(flask.request, bucket_name, None) 224 bucket.delete_acl(entity, None) 225 return "" 226 227 228@gcs.route("/b/<bucket_name>/defaultObjectAcl") 229def bucket_default_object_acl_list(bucket_name): 230 bucket = db.get_bucket(flask.request, bucket_name, None) 231 response = {"kind": "storage#objectAccessControls", "items": []} 232 for acl in bucket.metadata.default_object_acl: 233 acl_rest = json_format.MessageToDict(acl) 234 acl_rest["kind"] = "storage#objectAccessControl" 235 response["items"].append(acl_rest) 236 fields = flask.request.args.get("fields", None) 237 return utils.common.filter_response_rest(response, None, fields) 238 239 240@gcs.route("/b/<bucket_name>/defaultObjectAcl", methods=["POST"]) 241def bucket_default_object_acl_insert(bucket_name): 242 bucket = db.get_bucket(flask.request, bucket_name, None) 243 acl = bucket.insert_default_object_acl(flask.request, None) 244 response = json_format.MessageToDict(acl) 245 response["kind"] = "storage#objectAccessControl" 246 fields = flask.request.args.get("fields", None) 247 return utils.common.filter_response_rest(response, None, fields) 248 249 250@gcs.route("/b/<bucket_name>/defaultObjectAcl/<entity>") 251def bucket_default_object_acl_get(bucket_name, entity): 252 bucket = db.get_bucket(flask.request, bucket_name, None) 253 acl = bucket.get_default_object_acl(entity, None) 254 response = json_format.MessageToDict(acl) 255 response["kind"] = "storage#objectAccessControl" 256 fields = flask.request.args.get("fields", None) 257 return utils.common.filter_response_rest(response, None, fields) 258 259 260@gcs.route("/b/<bucket_name>/defaultObjectAcl/<entity>", methods=["PUT"]) 261def bucket_default_object_acl_update(bucket_name, entity): 262 bucket = db.get_bucket(flask.request, bucket_name, None) 263 acl = bucket.update_default_object_acl(flask.request, entity, None) 264 response = json_format.MessageToDict(acl) 265 response["kind"] = "storage#objectAccessControl" 266 fields = flask.request.args.get("fields", None) 267 return utils.common.filter_response_rest(response, None, fields) 268 269 270@gcs.route("/b/<bucket_name>/defaultObjectAcl/<entity>", methods=["PATCH"]) 271def bucket_default_object_acl_patch(bucket_name, entity): 272 bucket = db.get_bucket(flask.request, bucket_name, None) 273 acl = bucket.patch_default_object_acl(flask.request, entity, None) 274 response = json_format.MessageToDict(acl) 275 response["kind"] = "storage#objectAccessControl" 276 fields = flask.request.args.get("fields", None) 277 return utils.common.filter_response_rest(response, None, fields) 278 279 280@gcs.route("/b/<bucket_name>/defaultObjectAcl/<entity>", methods=["DELETE"]) 281def bucket_default_object_acl_delete(bucket_name, entity): 282 bucket = db.get_bucket(flask.request, bucket_name, None) 283 bucket.delete_default_object_acl(entity, None) 284 return "" 285 286 287@gcs.route("/b/<bucket_name>/notificationConfigs") 288def bucket_notification_list(bucket_name): 289 bucket = db.get_bucket(flask.request, bucket_name, None) 290 response = {"kind": "storage#notifications", "items": []} 291 for notification in bucket.notifications.values(): 292 response["items"].append( 293 json_format.MessageToDict(notification, preserving_proto_field_name=True) 294 ) 295 return response 296 297 298@gcs.route("/b/<bucket_name>/notificationConfigs", methods=["POST"]) 299def bucket_notification_insert(bucket_name): 300 bucket = db.get_bucket(flask.request, bucket_name, None) 301 notification = bucket.insert_notification(flask.request, None) 302 response = json_format.MessageToDict(notification, preserving_proto_field_name=True) 303 response["kind"] = "storage#notification" 304 return response 305 306 307@gcs.route("/b/<bucket_name>/notificationConfigs/<notification_id>") 308def bucket_notification_get(bucket_name, notification_id): 309 bucket = db.get_bucket(flask.request, bucket_name, None) 310 notification = bucket.get_notification(notification_id, None) 311 response = json_format.MessageToDict(notification, preserving_proto_field_name=True) 312 response["kind"] = "storage#notification" 313 return response 314 315 316@gcs.route("/b/<bucket_name>/notificationConfigs/<notification_id>", methods=["DELETE"]) 317def bucket_notification_delete(bucket_name, notification_id): 318 bucket = db.get_bucket(flask.request, bucket_name, None) 319 bucket.delete_notification(notification_id, None) 320 return "" 321 322 323@gcs.route("/b/<bucket_name>/iam") 324def bucket_get_iam_policy(bucket_name): 325 bucket = db.get_bucket(flask.request, bucket_name, None) 326 response = json_format.MessageToDict(bucket.iam_policy) 327 response["kind"] = "storage#policy" 328 return response 329 330 331@gcs.route("/b/<bucket_name>/iam", methods=["PUT"]) 332def bucket_set_iam_policy(bucket_name): 333 bucket = db.get_bucket(flask.request, bucket_name, None) 334 bucket.set_iam_policy(flask.request, None) 335 response = json_format.MessageToDict(bucket.iam_policy) 336 response["kind"] = "storage#policy" 337 return response 338 339 340@gcs.route("/b/<bucket_name>/iam/testPermissions") 341def bucket_test_iam_permissions(bucket_name): 342 db.get_bucket(flask.request, bucket_name, None) 343 permissions = flask.request.args.getlist("permissions") 344 result = {"kind": "storage#testIamPermissionsResponse", "permissions": permissions} 345 return result 346 347 348@gcs.route("/b/<bucket_name>/lockRetentionPolicy", methods=["POST"]) 349def bucket_lock_retention_policy(bucket_name): 350 bucket = db.get_bucket(flask.request, bucket_name, None) 351 bucket.metadata.retention_policy.is_locked = True 352 return bucket.rest() 353 354 355# === OBJECT === # 356 357 358@gcs.route("/b/<bucket_name>/o") 359def object_list(bucket_name): 360 db.insert_test_bucket(None) 361 items, prefixes, rest_onlys = db.list_object(flask.request, bucket_name, None) 362 response = { 363 "kind": "storage#objects", 364 "nextPageToken": "", 365 "items": [ 366 gcs_type.object.Object.rest(blob, rest_only) 367 for blob, rest_only in zip(items, rest_onlys) 368 ], 369 "prefixes": prefixes, 370 } 371 fields = flask.request.args.get("fields", None) 372 return utils.common.filter_response_rest(response, None, fields) 373 374 375@gcs.route("/b/<bucket_name>/o/<path:object_name>", methods=["PUT"]) 376def object_update(bucket_name, object_name): 377 blob = db.get_object(flask.request, bucket_name, object_name, False, None) 378 blob.update(flask.request, None) 379 projection = utils.common.extract_projection( 380 flask.request, CommonEnums.Projection.FULL, None 381 ) 382 fields = flask.request.args.get("fields", None) 383 return utils.common.filter_response_rest(blob.rest_metadata(), projection, fields) 384 385 386@gcs.route("/b/<bucket_name>/o/<path:object_name>", methods=["PATCH"]) 387def object_patch(bucket_name, object_name): 388 blob = db.get_object(flask.request, bucket_name, object_name, False, None) 389 blob.patch(flask.request, None) 390 projection = utils.common.extract_projection( 391 flask.request, CommonEnums.Projection.FULL, None 392 ) 393 fields = flask.request.args.get("fields", None) 394 return utils.common.filter_response_rest(blob.rest_metadata(), projection, fields) 395 396 397@gcs.route("/b/<bucket_name>/o/<path:object_name>", methods=["DELETE"]) 398def object_delete(bucket_name, object_name): 399 db.delete_object(flask.request, bucket_name, object_name, None) 400 return "" 401 402 403# === OBJECT SPECIAL OPERATIONS === # 404 405 406@gcs.route("/b/<bucket_name>/o/<path:object_name>/compose", methods=["POST"]) 407def objects_compose(bucket_name, object_name): 408 bucket = db.get_bucket_without_generation(bucket_name, None).metadata 409 payload = json.loads(flask.request.data) 410 source_objects = payload["sourceObjects"] 411 if source_objects is None: 412 utils.error.missing("source component", None) 413 if len(source_objects) > 32: 414 utils.error.invalid( 415 "The number of source components provided (%d > 32)" % len(source_objects), 416 None, 417 ) 418 composed_media = b"" 419 for source_object in source_objects: 420 source_object_name = source_object.get("name") 421 if source_object_name is None: 422 utils.error.missing("Name of source compose object", None) 423 generation = source_object.get("generation", None) 424 if_generation_match = ( 425 source_object.get("objectPreconditions").get("ifGenerationMatch") 426 if source_object.get("objectPreconditions") is not None 427 else None 428 ) 429 fake_request = utils.common.FakeRequest(args=dict(), headers={}) 430 if generation is not None: 431 fake_request.args["generation"] = generation 432 if if_generation_match is not None: 433 fake_request.args["ifGenerationMatch"] = if_generation_match 434 source_object = db.get_object( 435 fake_request, bucket_name, source_object_name, False, None 436 ) 437 composed_media += source_object.media 438 metadata = {"name": object_name, "bucket": bucket_name} 439 metadata.update(payload.get("destination", {})) 440 composed_object, _ = gcs_type.object.Object.init_dict( 441 flask.request, metadata, composed_media, bucket, True 442 ) 443 db.insert_object(flask.request, bucket_name, composed_object, None) 444 return composed_object.rest_metadata() 445 446 447@gcs.route( 448 "/b/<src_bucket_name>/o/<path:src_object_name>/copyTo/b/<dst_bucket_name>/o/<path:dst_object_name>", 449 methods=["POST"], 450) 451def objects_copy(src_bucket_name, src_object_name, dst_bucket_name, dst_object_name): 452 db.insert_test_bucket(None) 453 dst_bucket = db.get_bucket_without_generation(dst_bucket_name, None).metadata 454 src_object = db.get_object( 455 flask.request, src_bucket_name, src_object_name, True, None 456 ) 457 utils.csek.validation( 458 flask.request, src_object.metadata.customer_encryption.key_sha256, False, None 459 ) 460 dst_metadata = resources_pb2.Object() 461 dst_metadata.CopyFrom(src_object.metadata) 462 del dst_metadata.acl[:] 463 dst_metadata.bucket = dst_bucket_name 464 dst_metadata.name = dst_object_name 465 dst_media = b"" 466 dst_media += src_object.media 467 dst_rest_only = dict(src_object.rest_only) 468 dst_object, _ = gcs_type.object.Object.init( 469 flask.request, dst_metadata, dst_media, dst_bucket, True, None, dst_rest_only 470 ) 471 db.insert_object(flask.request, dst_bucket_name, dst_object, None) 472 dst_object.patch(flask.request, None) 473 dst_object.metadata.metageneration = 1 474 dst_object.metadata.updated.FromDatetime( 475 dst_object.metadata.time_created.ToDatetime() 476 ) 477 return dst_object.rest_metadata() 478 479 480@gcs.route( 481 "/b/<src_bucket_name>/o/<path:src_object_name>/rewriteTo/b/<dst_bucket_name>/o/<path:dst_object_name>", 482 methods=["POST"], 483) 484def objects_rewrite(src_bucket_name, src_object_name, dst_bucket_name, dst_object_name): 485 db.insert_test_bucket(None) 486 token, rewrite = flask.request.args.get("rewriteToken"), None 487 src_object = None 488 if token is None: 489 rewrite = gcs_type.holder.DataHolder.init_rewrite_rest( 490 flask.request, 491 src_bucket_name, 492 src_object_name, 493 dst_bucket_name, 494 dst_object_name, 495 ) 496 db.insert_rewrite(rewrite) 497 else: 498 rewrite = db.get_rewrite(token, None) 499 src_object = db.get_object( 500 rewrite.request, src_bucket_name, src_object_name, True, None 501 ) 502 utils.csek.validation( 503 rewrite.request, src_object.metadata.customer_encryption.key_sha256, True, None 504 ) 505 total_bytes_rewritten = len(rewrite.media) 506 total_bytes_rewritten += min( 507 rewrite.max_bytes_rewritten_per_call, len(src_object.media) - len(rewrite.media) 508 ) 509 rewrite.media += src_object.media[len(rewrite.media) : total_bytes_rewritten] 510 done, dst_object = total_bytes_rewritten == len(src_object.media), None 511 response = { 512 "kind": "storage#rewriteResponse", 513 "totalBytesRewritten": len(rewrite.media), 514 "objectSize": len(src_object.media), 515 "done": done, 516 } 517 if done: 518 dst_bucket = db.get_bucket_without_generation(dst_bucket_name, None).metadata 519 dst_metadata = resources_pb2.Object() 520 dst_metadata.CopyFrom(src_object.metadata) 521 dst_rest_only = dict(src_object.rest_only) 522 dst_metadata.bucket = dst_bucket_name 523 dst_metadata.name = dst_object_name 524 dst_media = rewrite.media 525 dst_object, _ = gcs_type.object.Object.init( 526 flask.request, 527 dst_metadata, 528 dst_media, 529 dst_bucket, 530 True, 531 None, 532 dst_rest_only, 533 ) 534 db.insert_object(flask.request, dst_bucket_name, dst_object, None) 535 dst_object.patch(rewrite.request, None) 536 dst_object.metadata.metageneration = 1 537 dst_object.metadata.updated.FromDatetime( 538 dst_object.metadata.time_created.ToDatetime() 539 ) 540 resources = dst_object.rest_metadata() 541 response["resource"] = resources 542 else: 543 response["rewriteToken"] = rewrite.token 544 return response 545 546 547# === OBJECT ACCESS CONTROL === # 548 549 550@gcs.route("/b/<bucket_name>/o/<path:object_name>/acl") 551def object_acl_list(bucket_name, object_name): 552 blob = db.get_object(flask.request, bucket_name, object_name, False, None) 553 response = {"kind": "storage#objectAccessControls", "items": []} 554 for acl in blob.metadata.acl: 555 acl_rest = json_format.MessageToDict(acl) 556 acl_rest["kind"] = "storage#objectAccessControl" 557 response["items"].append(acl_rest) 558 fields = flask.request.args.get("fields", None) 559 return utils.common.filter_response_rest(response, None, fields) 560 561 562@gcs.route("/b/<bucket_name>/o/<path:object_name>/acl", methods=["POST"]) 563def object_acl_insert(bucket_name, object_name): 564 blob = db.get_object(flask.request, bucket_name, object_name, False, None) 565 acl = blob.insert_acl(flask.request, None) 566 response = json_format.MessageToDict(acl) 567 response["kind"] = "storage#objectAccessControl" 568 fields = flask.request.args.get("fields", None) 569 return utils.common.filter_response_rest(response, None, fields) 570 571 572@gcs.route("/b/<bucket_name>/o/<path:object_name>/acl/<entity>") 573def object_acl_get(bucket_name, object_name, entity): 574 blob = db.get_object(flask.request, bucket_name, object_name, False, None) 575 acl = blob.get_acl(entity, None) 576 response = json_format.MessageToDict(acl) 577 response["kind"] = "storage#objectAccessControl" 578 fields = flask.request.args.get("fields", None) 579 return utils.common.filter_response_rest(response, None, fields) 580 581 582@gcs.route("/b/<bucket_name>/o/<path:object_name>/acl/<entity>", methods=["PUT"]) 583def object_acl_update(bucket_name, object_name, entity): 584 blob = db.get_object(flask.request, bucket_name, object_name, False, None) 585 acl = blob.update_acl(flask.request, entity, None) 586 response = json_format.MessageToDict(acl) 587 response["kind"] = "storage#objectAccessControl" 588 fields = flask.request.args.get("fields", None) 589 return utils.common.filter_response_rest(response, None, fields) 590 591 592@gcs.route("/b/<bucket_name>/o/<path:object_name>/acl/<entity>", methods=["PATCH"]) 593def object_acl_patch(bucket_name, object_name, entity): 594 blob = db.get_object(flask.request, bucket_name, object_name, False, None) 595 acl = blob.patch_acl(flask.request, entity, None) 596 response = json_format.MessageToDict(acl) 597 response["kind"] = "storage#objectAccessControl" 598 fields = flask.request.args.get("fields", None) 599 return utils.common.filter_response_rest(response, None, fields) 600 601 602@gcs.route("/b/<bucket_name>/o/<path:object_name>/acl/<entity>", methods=["DELETE"]) 603def object_acl_delete(bucket_name, object_name, entity): 604 blob = db.get_object(flask.request, bucket_name, object_name, False, None) 605 blob.delete_acl(entity, None) 606 return "" 607 608 609# Define the WSGI application to handle bucket requests. 610DOWNLOAD_HANDLER_PATH = "/download/storage/v1" 611download = flask.Flask(__name__) 612download.debug = True 613 614 615@gcs.route("/b/<bucket_name>/o/<path:object_name>") 616@download.route("/b/<bucket_name>/o/<path:object_name>") 617def object_get(bucket_name, object_name): 618 blob = db.get_object(flask.request, bucket_name, object_name, False, None) 619 media = flask.request.args.get("alt", None) 620 if media is None or media == "json": 621 projection = utils.common.extract_projection( 622 flask.request, CommonEnums.Projection.NO_ACL, None 623 ) 624 fields = flask.request.args.get("fields", None) 625 return utils.common.filter_response_rest( 626 blob.rest_metadata(), projection, fields 627 ) 628 if media != "media": 629 utils.error.invalid("Alt %s") 630 utils.csek.validation( 631 flask.request, blob.metadata.customer_encryption.key_sha256, False, None 632 ) 633 return blob.rest_media(flask.request) 634 635 636# Define the WSGI application to handle bucket requests. 637UPLOAD_HANDLER_PATH = "/upload/storage/v1" 638upload = flask.Flask(__name__) 639upload.debug = True 640 641 642@upload.route("/b/<bucket_name>/o", methods=["POST"]) 643def object_insert(bucket_name): 644 db.insert_test_bucket(None) 645 bucket = db.get_bucket_without_generation(bucket_name, None).metadata 646 upload_type = flask.request.args.get("uploadType") 647 if upload_type is None: 648 utils.error.missing("uploadType", None) 649 elif upload_type not in {"multipart", "media", "resumable"}: 650 utils.error.invalid("uploadType %s" % upload_type) 651 if upload_type == "resumable": 652 upload = gcs_type.holder.DataHolder.init_resumable_rest(flask.request, bucket) 653 db.insert_upload(upload) 654 response = flask.make_response("") 655 response.headers["Location"] = upload.location 656 return response 657 blob, projection = None, "" 658 if upload_type == "media": 659 blob, projection = gcs_type.object.Object.init_media(flask.request, bucket) 660 elif upload_type == "multipart": 661 blob, projection = gcs_type.object.Object.init_multipart(flask.request, bucket) 662 db.insert_object(flask.request, bucket.name, blob, None) 663 fields = flask.request.args.get("fields", None) 664 return utils.common.filter_response_rest(blob.rest_metadata(), projection, fields) 665 666 667@upload.route("/b/<bucket_name>/o", methods=["PUT"]) 668def resumable_upload_chunk(bucket_name): 669 request = flask.request 670 upload_id = request.args.get("upload_id") 671 if upload_id is None: 672 utils.error.missing("upload_id in resumable_upload_chunk", None) 673 upload = db.get_upload(upload_id, None) 674 if upload.complete: 675 return gcs_type.object.Object.rest(upload.metadata, upload.rest_only) 676 upload.transfer.add(request.environ.get("HTTP_TRANSFER_ENCODING", "")) 677 content_length = int(request.headers.get("content-length", 0)) 678 data = utils.common.extract_media(request) 679 if content_length != len(data): 680 utils.error.invalid("content-length header", None) 681 content_range = request.headers.get("content-range") 682 if content_range is not None: 683 items = list(utils.common.content_range_split.match(content_range).groups()) 684 if len(items) != 2 or (items[0] == items[1] and items[0] != "*"): 685 utils.error.invalid("content-range header", None) 686 if items[0] == "*": 687 if upload.complete: 688 return gcs_type.object.Object.rest(upload.metadata, upload.rest_only) 689 if items[1] != "*" and int(items[1]) == len(upload.media): 690 upload.complete = True 691 blob, _ = gcs_type.object.Object.init( 692 upload.request, 693 upload.metadata, 694 upload.media, 695 upload.bucket, 696 False, 697 None, 698 ) 699 blob.metadata.metadata["x_emulator_transfer_encoding"] = ":".join( 700 upload.transfer 701 ) 702 blob.metadata.metadata["x_emulator_upload"] = "resumable" 703 db.insert_object(upload.request, bucket_name, blob, None) 704 projection = utils.common.extract_projection( 705 upload.request, CommonEnums.Projection.NO_ACL, None 706 ) 707 fields = upload.request.args.get("fields", None) 708 return utils.common.filter_response_rest( 709 blob.rest_metadata(), projection, fields 710 ) 711 return upload.resumable_status_rest() 712 _, chunk_last_byte = [int(v) for v in items[0].split("-")] 713 x_upload_content_length = int( 714 upload.request.headers.get("x-upload-content-length", 0) 715 ) 716 total_object_size = ( 717 int(items[1]) if items[1] != "*" else x_upload_content_length 718 ) 719 if ( 720 x_upload_content_length != 0 721 and x_upload_content_length != total_object_size 722 ): 723 utils.error.mismatch( 724 "X-Upload-Content-Length", 725 x_upload_content_length, 726 total_object_size, 727 None, 728 rest_code=400, 729 ) 730 upload.media += data 731 upload.complete = total_object_size == len(upload.media) or ( 732 chunk_last_byte + 1 == total_object_size 733 ) 734 else: 735 upload.media += data 736 upload.complete = True 737 if upload.complete: 738 blob, _ = gcs_type.object.Object.init( 739 upload.request, 740 upload.metadata, 741 upload.media, 742 upload.bucket, 743 False, 744 None, 745 upload.rest_only, 746 ) 747 blob.metadata.metadata["x_emulator_transfer_encoding"] = ":".join( 748 upload.transfer 749 ) 750 blob.metadata.metadata["x_emulator_upload"] = "resumable" 751 db.insert_object(upload.request, bucket_name, blob, None) 752 projection = utils.common.extract_projection( 753 upload.request, CommonEnums.Projection.NO_ACL, None 754 ) 755 fields = upload.request.args.get("fields", None) 756 return utils.common.filter_response_rest( 757 blob.rest_metadata(), projection, fields 758 ) 759 else: 760 return upload.resumable_status_rest() 761 762 763@upload.route("/b/<bucket_name>/o", methods=["DELETE"]) 764def delete_resumable_upload(bucket_name): 765 upload_id = flask.request.args.get("upload_id") 766 db.delete_upload(upload_id, None) 767 return flask.make_response("", 499, {"content-length": 0}) 768 769 770def xml_put_object(bucket_name, object_name): 771 db.insert_test_bucket(None) 772 bucket = db.get_bucket_without_generation(bucket_name, None).metadata 773 blob, fake_request = gcs_type.object.Object.init_xml( 774 flask.request, bucket, object_name 775 ) 776 db.insert_object(fake_request, bucket_name, blob, None) 777 response = flask.make_response("") 778 response.headers["x-goog-hash"] = fake_request.headers.get("x-goog-hash") 779 return response 780 781 782def xml_get_object(bucket_name, object_name): 783 fake_request = utils.common.FakeRequest.init_xml(flask.request) 784 blob = db.get_object(fake_request, bucket_name, object_name, False, None) 785 return blob.rest_media(fake_request) 786 787 788# === SERVER === # 789 790# Define the WSGI application to handle HMAC key requests 791(PROJECTS_HANDLER_PATH, projects_app) = gcs_type.project.get_projects_app() 792 793# Define the WSGI application to handle IAM requests 794(IAM_HANDLER_PATH, iam_app) = gcs_type.iam.get_iam_app() 795 796 797server = DispatcherMiddleware( 798 root, 799 { 800 "/httpbin": httpbin.app, 801 GCS_HANDLER_PATH: gcs, 802 DOWNLOAD_HANDLER_PATH: download, 803 UPLOAD_HANDLER_PATH: upload, 804 PROJECTS_HANDLER_PATH: projects_app, 805 IAM_HANDLER_PATH: iam_app, 806 }, 807) 808 809root.register_error_handler(Exception, utils.error.RestException.handler) 810httpbin.app.register_error_handler(Exception, utils.error.RestException.handler) 811gcs.register_error_handler(Exception, utils.error.RestException.handler) 812download.register_error_handler(Exception, utils.error.RestException.handler) 813upload.register_error_handler(Exception, utils.error.RestException.handler) 814projects_app.register_error_handler(Exception, utils.error.RestException.handler) 815iam_app.register_error_handler(Exception, utils.error.RestException.handler) 816 817 818def run(): 819 global db 820 logging.basicConfig() 821 db = database.Database.init() 822 return server 823 824 825def run_without_gunicorn(port, database): 826 global db 827 db = database 828 serving.run_simple( 829 "localhost", int(port), server, use_reloader=False, threaded=True 830 ) 831