1# Copyright 2020 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import json
16import logging
17
18import database
19import flask
20import gcs as gcs_type
21import grpc_server
22import httpbin
23import utils
24from werkzeug import serving
25from werkzeug.middleware.dispatcher import DispatcherMiddleware
26
27from google.cloud.storage_v1.proto import storage_resources_pb2 as resources_pb2
28from google.cloud.storage_v1.proto.storage_resources_pb2 import CommonEnums
29from google.protobuf import json_format
30
31db = None
32grpc_port = 0
33
34# === DEFAULT ENTRY FOR REST SERVER === #
35root = flask.Flask(__name__)
36root.debug = True
37
38
39@root.route("/")
40def index():
41    return "OK"
42
43
44@root.route("/start_grpc")
45def start_grpc():
46    # We need to do this because `gunicorn` will spawn a new subprocess ( a worker )
47    # when running `Flask` server. If we start `gRPC` server before the spawn of
48    # the subprocess, it's nearly impossible to share the `database` with the new
49    # subprocess because Python will copy everything in the memory from the parent
50    # process to the subprocess ( So we have 2 separate instance of `database` ).
51    # The endpoint will start the `gRPC` server in the same subprocess so there is
52    # only one instance of `database`.
53    global grpc_port
54    if grpc_port == 0:
55        port = flask.request.args.get("port", "0")
56        grpc_port = grpc_server.run(port, db)
57        return str(grpc_port)
58    return str(grpc_port)
59
60
61@root.route("/raise_error")
62def raise_error():
63    etype = flask.request.args.get("etype")
64    msg = flask.request.args.get("msg", "")
65    if etype is not None:
66        raise TypeError(msg)
67    else:
68        raise Exception(msg)
69
70
71@root.route("/<path:object_name>", subdomain="<bucket_name>")
72def root_get_object(bucket_name, object_name):
73    return xml_get_object(bucket_name, object_name)
74
75
76@root.route("/<bucket_name>/<path:object_name>", subdomain="")
77def root_get_object_with_bucket(bucket_name, object_name):
78    return xml_get_object(bucket_name, object_name)
79
80
81@root.route("/<path:object_name>", subdomain="<bucket_name>", methods=["PUT"])
82def root_put_object(bucket_name, object_name):
83    return xml_put_object(bucket_name, object_name)
84
85
86@root.route("/<bucket_name>/<path:object_name>", subdomain="", methods=["PUT"])
87def root_put_object_with_bucket(bucket_name, object_name):
88    return xml_put_object(bucket_name, object_name)
89
90
91# === WSGI APP TO HANDLE JSON API === #
92GCS_HANDLER_PATH = "/storage/v1"
93gcs = flask.Flask(__name__)
94gcs.debug = True
95
96
97# === BUCKET === #
98
99
100@gcs.route("/b", methods=["GET"])
101def bucket_list():
102    db.insert_test_bucket(None)
103    project = flask.request.args.get("project")
104    projection = flask.request.args.get("projection", "noAcl")
105    fields = flask.request.args.get("fields", None)
106    response = {
107        "kind": "storage#buckets",
108        "nextPageToken": "",
109        "items": [
110            bucket.rest() for bucket in db.list_bucket(flask.request, project, None)
111        ],
112    }
113    return utils.common.filter_response_rest(response, projection, fields)
114
115
116@gcs.route("/b", methods=["POST"])
117def bucket_insert():
118    db.insert_test_bucket(None)
119    bucket, projection = gcs_type.bucket.Bucket.init(flask.request, None)
120    fields = flask.request.args.get("fields", None)
121    db.insert_bucket(flask.request, bucket, None)
122    return utils.common.filter_response_rest(bucket.rest(), projection, fields)
123
124
125@gcs.route("/b/<bucket_name>")
126def bucket_get(bucket_name):
127    db.insert_test_bucket(None)
128    db.insert_test_bucket(None)
129    bucket = db.get_bucket(flask.request, bucket_name, None)
130    projection = utils.common.extract_projection(
131        flask.request, CommonEnums.Projection.NO_ACL, None
132    )
133    fields = flask.request.args.get("fields", None)
134    return utils.common.filter_response_rest(bucket.rest(), projection, fields)
135
136
137@gcs.route("/b/<bucket_name>", methods=["PUT"])
138def bucket_update(bucket_name):
139    db.insert_test_bucket(None)
140    bucket = db.get_bucket(flask.request, bucket_name, None)
141    bucket.update(flask.request, None)
142    projection = utils.common.extract_projection(
143        flask.request, CommonEnums.Projection.FULL, None
144    )
145    fields = flask.request.args.get("fields", None)
146    return utils.common.filter_response_rest(bucket.rest(), projection, fields)
147
148
149@gcs.route("/b/<bucket_name>", methods=["PATCH"])
150def bucket_patch(bucket_name):
151    bucket = db.get_bucket(flask.request, bucket_name, None)
152    bucket.patch(flask.request, None)
153    projection = utils.common.extract_projection(
154        flask.request, CommonEnums.Projection.FULL, None
155    )
156    fields = flask.request.args.get("fields", None)
157    return utils.common.filter_response_rest(bucket.rest(), projection, fields)
158
159
160@gcs.route("/b/<bucket_name>", methods=["DELETE"])
161def bucket_delete(bucket_name):
162    db.delete_bucket(flask.request, bucket_name, None)
163    return ""
164
165
166# === BUCKET ACL === #
167
168
169@gcs.route("/b/<bucket_name>/acl")
170def bucket_acl_list(bucket_name):
171    bucket = db.get_bucket(flask.request, bucket_name, None)
172    response = {"kind": "storage#bucketAccessControls", "items": []}
173    for acl in bucket.metadata.acl:
174        acl_rest = json_format.MessageToDict(acl)
175        acl_rest["kind"] = "storage#bucketAccessControl"
176        response["items"].append(acl_rest)
177    fields = flask.request.args.get("fields", None)
178    return utils.common.filter_response_rest(response, None, fields)
179
180
181@gcs.route("/b/<bucket_name>/acl", methods=["POST"])
182def bucket_acl_insert(bucket_name):
183    bucket = db.get_bucket(flask.request, bucket_name, None)
184    acl = bucket.insert_acl(flask.request, None)
185    response = json_format.MessageToDict(acl)
186    response["kind"] = "storage#bucketAccessControl"
187    fields = flask.request.args.get("fields", None)
188    return utils.common.filter_response_rest(response, None, fields)
189
190
191@gcs.route("/b/<bucket_name>/acl/<entity>")
192def bucket_acl_get(bucket_name, entity):
193    bucket = db.get_bucket(flask.request, bucket_name, None)
194    acl = bucket.get_acl(entity, None)
195    response = json_format.MessageToDict(acl)
196    response["kind"] = "storage#bucketAccessControl"
197    fields = flask.request.args.get("fields", None)
198    return utils.common.filter_response_rest(response, None, fields)
199
200
201@gcs.route("/b/<bucket_name>/acl/<entity>", methods=["PUT"])
202def bucket_acl_update(bucket_name, entity):
203    bucket = db.get_bucket(flask.request, bucket_name, None)
204    acl = bucket.update_acl(flask.request, entity, None)
205    response = json_format.MessageToDict(acl)
206    response["kind"] = "storage#bucketAccessControl"
207    fields = flask.request.args.get("fields", None)
208    return utils.common.filter_response_rest(response, None, fields)
209
210
211@gcs.route("/b/<bucket_name>/acl/<entity>", methods=["PATCH"])
212def bucket_acl_patch(bucket_name, entity):
213    bucket = db.get_bucket(flask.request, bucket_name, None)
214    acl = bucket.patch_acl(flask.request, entity, None)
215    response = json_format.MessageToDict(acl)
216    response["kind"] = "storage#bucketAccessControl"
217    fields = flask.request.args.get("fields", None)
218    return utils.common.filter_response_rest(response, None, fields)
219
220
221@gcs.route("/b/<bucket_name>/acl/<entity>", methods=["DELETE"])
222def bucket_acl_delete(bucket_name, entity):
223    bucket = db.get_bucket(flask.request, bucket_name, None)
224    bucket.delete_acl(entity, None)
225    return ""
226
227
228@gcs.route("/b/<bucket_name>/defaultObjectAcl")
229def bucket_default_object_acl_list(bucket_name):
230    bucket = db.get_bucket(flask.request, bucket_name, None)
231    response = {"kind": "storage#objectAccessControls", "items": []}
232    for acl in bucket.metadata.default_object_acl:
233        acl_rest = json_format.MessageToDict(acl)
234        acl_rest["kind"] = "storage#objectAccessControl"
235        response["items"].append(acl_rest)
236    fields = flask.request.args.get("fields", None)
237    return utils.common.filter_response_rest(response, None, fields)
238
239
240@gcs.route("/b/<bucket_name>/defaultObjectAcl", methods=["POST"])
241def bucket_default_object_acl_insert(bucket_name):
242    bucket = db.get_bucket(flask.request, bucket_name, None)
243    acl = bucket.insert_default_object_acl(flask.request, None)
244    response = json_format.MessageToDict(acl)
245    response["kind"] = "storage#objectAccessControl"
246    fields = flask.request.args.get("fields", None)
247    return utils.common.filter_response_rest(response, None, fields)
248
249
250@gcs.route("/b/<bucket_name>/defaultObjectAcl/<entity>")
251def bucket_default_object_acl_get(bucket_name, entity):
252    bucket = db.get_bucket(flask.request, bucket_name, None)
253    acl = bucket.get_default_object_acl(entity, None)
254    response = json_format.MessageToDict(acl)
255    response["kind"] = "storage#objectAccessControl"
256    fields = flask.request.args.get("fields", None)
257    return utils.common.filter_response_rest(response, None, fields)
258
259
260@gcs.route("/b/<bucket_name>/defaultObjectAcl/<entity>", methods=["PUT"])
261def bucket_default_object_acl_update(bucket_name, entity):
262    bucket = db.get_bucket(flask.request, bucket_name, None)
263    acl = bucket.update_default_object_acl(flask.request, entity, None)
264    response = json_format.MessageToDict(acl)
265    response["kind"] = "storage#objectAccessControl"
266    fields = flask.request.args.get("fields", None)
267    return utils.common.filter_response_rest(response, None, fields)
268
269
270@gcs.route("/b/<bucket_name>/defaultObjectAcl/<entity>", methods=["PATCH"])
271def bucket_default_object_acl_patch(bucket_name, entity):
272    bucket = db.get_bucket(flask.request, bucket_name, None)
273    acl = bucket.patch_default_object_acl(flask.request, entity, None)
274    response = json_format.MessageToDict(acl)
275    response["kind"] = "storage#objectAccessControl"
276    fields = flask.request.args.get("fields", None)
277    return utils.common.filter_response_rest(response, None, fields)
278
279
280@gcs.route("/b/<bucket_name>/defaultObjectAcl/<entity>", methods=["DELETE"])
281def bucket_default_object_acl_delete(bucket_name, entity):
282    bucket = db.get_bucket(flask.request, bucket_name, None)
283    bucket.delete_default_object_acl(entity, None)
284    return ""
285
286
287@gcs.route("/b/<bucket_name>/notificationConfigs")
288def bucket_notification_list(bucket_name):
289    bucket = db.get_bucket(flask.request, bucket_name, None)
290    response = {"kind": "storage#notifications", "items": []}
291    for notification in bucket.notifications.values():
292        response["items"].append(
293            json_format.MessageToDict(notification, preserving_proto_field_name=True)
294        )
295    return response
296
297
298@gcs.route("/b/<bucket_name>/notificationConfigs", methods=["POST"])
299def bucket_notification_insert(bucket_name):
300    bucket = db.get_bucket(flask.request, bucket_name, None)
301    notification = bucket.insert_notification(flask.request, None)
302    response = json_format.MessageToDict(notification, preserving_proto_field_name=True)
303    response["kind"] = "storage#notification"
304    return response
305
306
307@gcs.route("/b/<bucket_name>/notificationConfigs/<notification_id>")
308def bucket_notification_get(bucket_name, notification_id):
309    bucket = db.get_bucket(flask.request, bucket_name, None)
310    notification = bucket.get_notification(notification_id, None)
311    response = json_format.MessageToDict(notification, preserving_proto_field_name=True)
312    response["kind"] = "storage#notification"
313    return response
314
315
316@gcs.route("/b/<bucket_name>/notificationConfigs/<notification_id>", methods=["DELETE"])
317def bucket_notification_delete(bucket_name, notification_id):
318    bucket = db.get_bucket(flask.request, bucket_name, None)
319    bucket.delete_notification(notification_id, None)
320    return ""
321
322
323@gcs.route("/b/<bucket_name>/iam")
324def bucket_get_iam_policy(bucket_name):
325    bucket = db.get_bucket(flask.request, bucket_name, None)
326    response = json_format.MessageToDict(bucket.iam_policy)
327    response["kind"] = "storage#policy"
328    return response
329
330
331@gcs.route("/b/<bucket_name>/iam", methods=["PUT"])
332def bucket_set_iam_policy(bucket_name):
333    bucket = db.get_bucket(flask.request, bucket_name, None)
334    bucket.set_iam_policy(flask.request, None)
335    response = json_format.MessageToDict(bucket.iam_policy)
336    response["kind"] = "storage#policy"
337    return response
338
339
340@gcs.route("/b/<bucket_name>/iam/testPermissions")
341def bucket_test_iam_permissions(bucket_name):
342    db.get_bucket(flask.request, bucket_name, None)
343    permissions = flask.request.args.getlist("permissions")
344    result = {"kind": "storage#testIamPermissionsResponse", "permissions": permissions}
345    return result
346
347
348@gcs.route("/b/<bucket_name>/lockRetentionPolicy", methods=["POST"])
349def bucket_lock_retention_policy(bucket_name):
350    bucket = db.get_bucket(flask.request, bucket_name, None)
351    bucket.metadata.retention_policy.is_locked = True
352    return bucket.rest()
353
354
355# === OBJECT === #
356
357
358@gcs.route("/b/<bucket_name>/o")
359def object_list(bucket_name):
360    db.insert_test_bucket(None)
361    items, prefixes, rest_onlys = db.list_object(flask.request, bucket_name, None)
362    response = {
363        "kind": "storage#objects",
364        "nextPageToken": "",
365        "items": [
366            gcs_type.object.Object.rest(blob, rest_only)
367            for blob, rest_only in zip(items, rest_onlys)
368        ],
369        "prefixes": prefixes,
370    }
371    fields = flask.request.args.get("fields", None)
372    return utils.common.filter_response_rest(response, None, fields)
373
374
375@gcs.route("/b/<bucket_name>/o/<path:object_name>", methods=["PUT"])
376def object_update(bucket_name, object_name):
377    blob = db.get_object(flask.request, bucket_name, object_name, False, None)
378    blob.update(flask.request, None)
379    projection = utils.common.extract_projection(
380        flask.request, CommonEnums.Projection.FULL, None
381    )
382    fields = flask.request.args.get("fields", None)
383    return utils.common.filter_response_rest(blob.rest_metadata(), projection, fields)
384
385
386@gcs.route("/b/<bucket_name>/o/<path:object_name>", methods=["PATCH"])
387def object_patch(bucket_name, object_name):
388    blob = db.get_object(flask.request, bucket_name, object_name, False, None)
389    blob.patch(flask.request, None)
390    projection = utils.common.extract_projection(
391        flask.request, CommonEnums.Projection.FULL, None
392    )
393    fields = flask.request.args.get("fields", None)
394    return utils.common.filter_response_rest(blob.rest_metadata(), projection, fields)
395
396
397@gcs.route("/b/<bucket_name>/o/<path:object_name>", methods=["DELETE"])
398def object_delete(bucket_name, object_name):
399    db.delete_object(flask.request, bucket_name, object_name, None)
400    return ""
401
402
403# === OBJECT SPECIAL OPERATIONS === #
404
405
406@gcs.route("/b/<bucket_name>/o/<path:object_name>/compose", methods=["POST"])
407def objects_compose(bucket_name, object_name):
408    bucket = db.get_bucket_without_generation(bucket_name, None).metadata
409    payload = json.loads(flask.request.data)
410    source_objects = payload["sourceObjects"]
411    if source_objects is None:
412        utils.error.missing("source component", None)
413    if len(source_objects) > 32:
414        utils.error.invalid(
415            "The number of source components provided (%d > 32)" % len(source_objects),
416            None,
417        )
418    composed_media = b""
419    for source_object in source_objects:
420        source_object_name = source_object.get("name")
421        if source_object_name is None:
422            utils.error.missing("Name of source compose object", None)
423        generation = source_object.get("generation", None)
424        if_generation_match = (
425            source_object.get("objectPreconditions").get("ifGenerationMatch")
426            if source_object.get("objectPreconditions") is not None
427            else None
428        )
429        fake_request = utils.common.FakeRequest(args=dict(), headers={})
430        if generation is not None:
431            fake_request.args["generation"] = generation
432        if if_generation_match is not None:
433            fake_request.args["ifGenerationMatch"] = if_generation_match
434        source_object = db.get_object(
435            fake_request, bucket_name, source_object_name, False, None
436        )
437        composed_media += source_object.media
438    metadata = {"name": object_name, "bucket": bucket_name}
439    metadata.update(payload.get("destination", {}))
440    composed_object, _ = gcs_type.object.Object.init_dict(
441        flask.request, metadata, composed_media, bucket, True
442    )
443    db.insert_object(flask.request, bucket_name, composed_object, None)
444    return composed_object.rest_metadata()
445
446
447@gcs.route(
448    "/b/<src_bucket_name>/o/<path:src_object_name>/copyTo/b/<dst_bucket_name>/o/<path:dst_object_name>",
449    methods=["POST"],
450)
451def objects_copy(src_bucket_name, src_object_name, dst_bucket_name, dst_object_name):
452    db.insert_test_bucket(None)
453    dst_bucket = db.get_bucket_without_generation(dst_bucket_name, None).metadata
454    src_object = db.get_object(
455        flask.request, src_bucket_name, src_object_name, True, None
456    )
457    utils.csek.validation(
458        flask.request, src_object.metadata.customer_encryption.key_sha256, False, None
459    )
460    dst_metadata = resources_pb2.Object()
461    dst_metadata.CopyFrom(src_object.metadata)
462    del dst_metadata.acl[:]
463    dst_metadata.bucket = dst_bucket_name
464    dst_metadata.name = dst_object_name
465    dst_media = b""
466    dst_media += src_object.media
467    dst_rest_only = dict(src_object.rest_only)
468    dst_object, _ = gcs_type.object.Object.init(
469        flask.request, dst_metadata, dst_media, dst_bucket, True, None, dst_rest_only
470    )
471    db.insert_object(flask.request, dst_bucket_name, dst_object, None)
472    dst_object.patch(flask.request, None)
473    dst_object.metadata.metageneration = 1
474    dst_object.metadata.updated.FromDatetime(
475        dst_object.metadata.time_created.ToDatetime()
476    )
477    return dst_object.rest_metadata()
478
479
480@gcs.route(
481    "/b/<src_bucket_name>/o/<path:src_object_name>/rewriteTo/b/<dst_bucket_name>/o/<path:dst_object_name>",
482    methods=["POST"],
483)
484def objects_rewrite(src_bucket_name, src_object_name, dst_bucket_name, dst_object_name):
485    db.insert_test_bucket(None)
486    token, rewrite = flask.request.args.get("rewriteToken"), None
487    src_object = None
488    if token is None:
489        rewrite = gcs_type.holder.DataHolder.init_rewrite_rest(
490            flask.request,
491            src_bucket_name,
492            src_object_name,
493            dst_bucket_name,
494            dst_object_name,
495        )
496        db.insert_rewrite(rewrite)
497    else:
498        rewrite = db.get_rewrite(token, None)
499    src_object = db.get_object(
500        rewrite.request, src_bucket_name, src_object_name, True, None
501    )
502    utils.csek.validation(
503        rewrite.request, src_object.metadata.customer_encryption.key_sha256, True, None
504    )
505    total_bytes_rewritten = len(rewrite.media)
506    total_bytes_rewritten += min(
507        rewrite.max_bytes_rewritten_per_call, len(src_object.media) - len(rewrite.media)
508    )
509    rewrite.media += src_object.media[len(rewrite.media) : total_bytes_rewritten]
510    done, dst_object = total_bytes_rewritten == len(src_object.media), None
511    response = {
512        "kind": "storage#rewriteResponse",
513        "totalBytesRewritten": len(rewrite.media),
514        "objectSize": len(src_object.media),
515        "done": done,
516    }
517    if done:
518        dst_bucket = db.get_bucket_without_generation(dst_bucket_name, None).metadata
519        dst_metadata = resources_pb2.Object()
520        dst_metadata.CopyFrom(src_object.metadata)
521        dst_rest_only = dict(src_object.rest_only)
522        dst_metadata.bucket = dst_bucket_name
523        dst_metadata.name = dst_object_name
524        dst_media = rewrite.media
525        dst_object, _ = gcs_type.object.Object.init(
526            flask.request,
527            dst_metadata,
528            dst_media,
529            dst_bucket,
530            True,
531            None,
532            dst_rest_only,
533        )
534        db.insert_object(flask.request, dst_bucket_name, dst_object, None)
535        dst_object.patch(rewrite.request, None)
536        dst_object.metadata.metageneration = 1
537        dst_object.metadata.updated.FromDatetime(
538            dst_object.metadata.time_created.ToDatetime()
539        )
540        resources = dst_object.rest_metadata()
541        response["resource"] = resources
542    else:
543        response["rewriteToken"] = rewrite.token
544    return response
545
546
547# === OBJECT ACCESS CONTROL === #
548
549
550@gcs.route("/b/<bucket_name>/o/<path:object_name>/acl")
551def object_acl_list(bucket_name, object_name):
552    blob = db.get_object(flask.request, bucket_name, object_name, False, None)
553    response = {"kind": "storage#objectAccessControls", "items": []}
554    for acl in blob.metadata.acl:
555        acl_rest = json_format.MessageToDict(acl)
556        acl_rest["kind"] = "storage#objectAccessControl"
557        response["items"].append(acl_rest)
558    fields = flask.request.args.get("fields", None)
559    return utils.common.filter_response_rest(response, None, fields)
560
561
562@gcs.route("/b/<bucket_name>/o/<path:object_name>/acl", methods=["POST"])
563def object_acl_insert(bucket_name, object_name):
564    blob = db.get_object(flask.request, bucket_name, object_name, False, None)
565    acl = blob.insert_acl(flask.request, None)
566    response = json_format.MessageToDict(acl)
567    response["kind"] = "storage#objectAccessControl"
568    fields = flask.request.args.get("fields", None)
569    return utils.common.filter_response_rest(response, None, fields)
570
571
572@gcs.route("/b/<bucket_name>/o/<path:object_name>/acl/<entity>")
573def object_acl_get(bucket_name, object_name, entity):
574    blob = db.get_object(flask.request, bucket_name, object_name, False, None)
575    acl = blob.get_acl(entity, None)
576    response = json_format.MessageToDict(acl)
577    response["kind"] = "storage#objectAccessControl"
578    fields = flask.request.args.get("fields", None)
579    return utils.common.filter_response_rest(response, None, fields)
580
581
582@gcs.route("/b/<bucket_name>/o/<path:object_name>/acl/<entity>", methods=["PUT"])
583def object_acl_update(bucket_name, object_name, entity):
584    blob = db.get_object(flask.request, bucket_name, object_name, False, None)
585    acl = blob.update_acl(flask.request, entity, None)
586    response = json_format.MessageToDict(acl)
587    response["kind"] = "storage#objectAccessControl"
588    fields = flask.request.args.get("fields", None)
589    return utils.common.filter_response_rest(response, None, fields)
590
591
592@gcs.route("/b/<bucket_name>/o/<path:object_name>/acl/<entity>", methods=["PATCH"])
593def object_acl_patch(bucket_name, object_name, entity):
594    blob = db.get_object(flask.request, bucket_name, object_name, False, None)
595    acl = blob.patch_acl(flask.request, entity, None)
596    response = json_format.MessageToDict(acl)
597    response["kind"] = "storage#objectAccessControl"
598    fields = flask.request.args.get("fields", None)
599    return utils.common.filter_response_rest(response, None, fields)
600
601
602@gcs.route("/b/<bucket_name>/o/<path:object_name>/acl/<entity>", methods=["DELETE"])
603def object_acl_delete(bucket_name, object_name, entity):
604    blob = db.get_object(flask.request, bucket_name, object_name, False, None)
605    blob.delete_acl(entity, None)
606    return ""
607
608
609# Define the WSGI application to handle bucket requests.
610DOWNLOAD_HANDLER_PATH = "/download/storage/v1"
611download = flask.Flask(__name__)
612download.debug = True
613
614
615@gcs.route("/b/<bucket_name>/o/<path:object_name>")
616@download.route("/b/<bucket_name>/o/<path:object_name>")
617def object_get(bucket_name, object_name):
618    blob = db.get_object(flask.request, bucket_name, object_name, False, None)
619    media = flask.request.args.get("alt", None)
620    if media is None or media == "json":
621        projection = utils.common.extract_projection(
622            flask.request, CommonEnums.Projection.NO_ACL, None
623        )
624        fields = flask.request.args.get("fields", None)
625        return utils.common.filter_response_rest(
626            blob.rest_metadata(), projection, fields
627        )
628    if media != "media":
629        utils.error.invalid("Alt %s")
630    utils.csek.validation(
631        flask.request, blob.metadata.customer_encryption.key_sha256, False, None
632    )
633    return blob.rest_media(flask.request)
634
635
636# Define the WSGI application to handle bucket requests.
637UPLOAD_HANDLER_PATH = "/upload/storage/v1"
638upload = flask.Flask(__name__)
639upload.debug = True
640
641
642@upload.route("/b/<bucket_name>/o", methods=["POST"])
643def object_insert(bucket_name):
644    db.insert_test_bucket(None)
645    bucket = db.get_bucket_without_generation(bucket_name, None).metadata
646    upload_type = flask.request.args.get("uploadType")
647    if upload_type is None:
648        utils.error.missing("uploadType", None)
649    elif upload_type not in {"multipart", "media", "resumable"}:
650        utils.error.invalid("uploadType %s" % upload_type)
651    if upload_type == "resumable":
652        upload = gcs_type.holder.DataHolder.init_resumable_rest(flask.request, bucket)
653        db.insert_upload(upload)
654        response = flask.make_response("")
655        response.headers["Location"] = upload.location
656        return response
657    blob, projection = None, ""
658    if upload_type == "media":
659        blob, projection = gcs_type.object.Object.init_media(flask.request, bucket)
660    elif upload_type == "multipart":
661        blob, projection = gcs_type.object.Object.init_multipart(flask.request, bucket)
662    db.insert_object(flask.request, bucket.name, blob, None)
663    fields = flask.request.args.get("fields", None)
664    return utils.common.filter_response_rest(blob.rest_metadata(), projection, fields)
665
666
667@upload.route("/b/<bucket_name>/o", methods=["PUT"])
668def resumable_upload_chunk(bucket_name):
669    request = flask.request
670    upload_id = request.args.get("upload_id")
671    if upload_id is None:
672        utils.error.missing("upload_id in resumable_upload_chunk", None)
673    upload = db.get_upload(upload_id, None)
674    if upload.complete:
675        return gcs_type.object.Object.rest(upload.metadata, upload.rest_only)
676    upload.transfer.add(request.environ.get("HTTP_TRANSFER_ENCODING", ""))
677    content_length = int(request.headers.get("content-length", 0))
678    data = utils.common.extract_media(request)
679    if content_length != len(data):
680        utils.error.invalid("content-length header", None)
681    content_range = request.headers.get("content-range")
682    if content_range is not None:
683        items = list(utils.common.content_range_split.match(content_range).groups())
684        if len(items) != 2 or (items[0] == items[1] and items[0] != "*"):
685            utils.error.invalid("content-range header", None)
686        if items[0] == "*":
687            if upload.complete:
688                return gcs_type.object.Object.rest(upload.metadata, upload.rest_only)
689            if items[1] != "*" and int(items[1]) == len(upload.media):
690                upload.complete = True
691                blob, _ = gcs_type.object.Object.init(
692                    upload.request,
693                    upload.metadata,
694                    upload.media,
695                    upload.bucket,
696                    False,
697                    None,
698                )
699                blob.metadata.metadata["x_emulator_transfer_encoding"] = ":".join(
700                    upload.transfer
701                )
702                blob.metadata.metadata["x_emulator_upload"] = "resumable"
703                db.insert_object(upload.request, bucket_name, blob, None)
704                projection = utils.common.extract_projection(
705                    upload.request, CommonEnums.Projection.NO_ACL, None
706                )
707                fields = upload.request.args.get("fields", None)
708                return utils.common.filter_response_rest(
709                    blob.rest_metadata(), projection, fields
710                )
711            return upload.resumable_status_rest()
712        _, chunk_last_byte = [int(v) for v in items[0].split("-")]
713        x_upload_content_length = int(
714            upload.request.headers.get("x-upload-content-length", 0)
715        )
716        total_object_size = (
717            int(items[1]) if items[1] != "*" else x_upload_content_length
718        )
719        if (
720            x_upload_content_length != 0
721            and x_upload_content_length != total_object_size
722        ):
723            utils.error.mismatch(
724                "X-Upload-Content-Length",
725                x_upload_content_length,
726                total_object_size,
727                None,
728                rest_code=400,
729            )
730        upload.media += data
731        upload.complete = total_object_size == len(upload.media) or (
732            chunk_last_byte + 1 == total_object_size
733        )
734    else:
735        upload.media += data
736        upload.complete = True
737    if upload.complete:
738        blob, _ = gcs_type.object.Object.init(
739            upload.request,
740            upload.metadata,
741            upload.media,
742            upload.bucket,
743            False,
744            None,
745            upload.rest_only,
746        )
747        blob.metadata.metadata["x_emulator_transfer_encoding"] = ":".join(
748            upload.transfer
749        )
750        blob.metadata.metadata["x_emulator_upload"] = "resumable"
751        db.insert_object(upload.request, bucket_name, blob, None)
752        projection = utils.common.extract_projection(
753            upload.request, CommonEnums.Projection.NO_ACL, None
754        )
755        fields = upload.request.args.get("fields", None)
756        return utils.common.filter_response_rest(
757            blob.rest_metadata(), projection, fields
758        )
759    else:
760        return upload.resumable_status_rest()
761
762
763@upload.route("/b/<bucket_name>/o", methods=["DELETE"])
764def delete_resumable_upload(bucket_name):
765    upload_id = flask.request.args.get("upload_id")
766    db.delete_upload(upload_id, None)
767    return flask.make_response("", 499, {"content-length": 0})
768
769
770def xml_put_object(bucket_name, object_name):
771    db.insert_test_bucket(None)
772    bucket = db.get_bucket_without_generation(bucket_name, None).metadata
773    blob, fake_request = gcs_type.object.Object.init_xml(
774        flask.request, bucket, object_name
775    )
776    db.insert_object(fake_request, bucket_name, blob, None)
777    response = flask.make_response("")
778    response.headers["x-goog-hash"] = fake_request.headers.get("x-goog-hash")
779    return response
780
781
782def xml_get_object(bucket_name, object_name):
783    fake_request = utils.common.FakeRequest.init_xml(flask.request)
784    blob = db.get_object(fake_request, bucket_name, object_name, False, None)
785    return blob.rest_media(fake_request)
786
787
788# === SERVER === #
789
790# Define the WSGI application to handle HMAC key requests
791(PROJECTS_HANDLER_PATH, projects_app) = gcs_type.project.get_projects_app()
792
793# Define the WSGI application to handle IAM requests
794(IAM_HANDLER_PATH, iam_app) = gcs_type.iam.get_iam_app()
795
796
797server = DispatcherMiddleware(
798    root,
799    {
800        "/httpbin": httpbin.app,
801        GCS_HANDLER_PATH: gcs,
802        DOWNLOAD_HANDLER_PATH: download,
803        UPLOAD_HANDLER_PATH: upload,
804        PROJECTS_HANDLER_PATH: projects_app,
805        IAM_HANDLER_PATH: iam_app,
806    },
807)
808
809root.register_error_handler(Exception, utils.error.RestException.handler)
810httpbin.app.register_error_handler(Exception, utils.error.RestException.handler)
811gcs.register_error_handler(Exception, utils.error.RestException.handler)
812download.register_error_handler(Exception, utils.error.RestException.handler)
813upload.register_error_handler(Exception, utils.error.RestException.handler)
814projects_app.register_error_handler(Exception, utils.error.RestException.handler)
815iam_app.register_error_handler(Exception, utils.error.RestException.handler)
816
817
818def run():
819    global db
820    logging.basicConfig()
821    db = database.Database.init()
822    return server
823
824
825def run_without_gunicorn(port, database):
826    global db
827    db = database
828    serving.run_simple(
829        "localhost", int(port), server, use_reloader=False, threaded=True
830    )
831