1 // Copyright 2018 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "google/cloud/storage/internal/object_requests.h"
16 #include "google/cloud/storage/internal/binary_data_as_debug_string.h"
17 #include "google/cloud/storage/internal/common_metadata_parser.h"
18 #include "google/cloud/storage/internal/metadata_parser.h"
19 #include "google/cloud/storage/internal/object_acl_requests.h"
20 #include "google/cloud/storage/internal/object_metadata_parser.h"
21 #include "google/cloud/storage/object_metadata.h"
22 #include <cinttypes>
23 #include <sstream>
24
25 namespace google {
26 namespace cloud {
27 namespace storage {
28 inline namespace STORAGE_CLIENT_NS {
29 namespace internal {
json_payload() const30 std::string UpdateObjectRequest::json_payload() const {
31 return ObjectMetadataJsonForUpdate(metadata_).dump();
32 }
33
operator <<(std::ostream & os,ListObjectsRequest const & r)34 std::ostream& operator<<(std::ostream& os, ListObjectsRequest const& r) {
35 os << "ListObjectsRequest={bucket_name=" << r.bucket_name();
36 r.DumpOptions(os, ", ");
37 return os << "}";
38 }
39
FromHttpResponse(std::string const & payload)40 StatusOr<ListObjectsResponse> ListObjectsResponse::FromHttpResponse(
41 std::string const& payload) {
42 auto json = nlohmann::json::parse(payload, nullptr, false);
43 if (!json.is_object()) {
44 return Status(StatusCode::kInvalidArgument, __func__);
45 }
46
47 ListObjectsResponse result;
48 result.next_page_token = json.value("nextPageToken", "");
49
50 for (auto const& kv : json["items"].items()) {
51 auto parsed = internal::ObjectMetadataParser::FromJson(kv.value());
52 if (!parsed.ok()) {
53 return std::move(parsed).status();
54 }
55 result.items.emplace_back(std::move(*parsed));
56 }
57
58 for (auto const& prefix_iterator : json["prefixes"].items()) {
59 auto const& prefix = prefix_iterator.value();
60 if (!prefix.is_string()) {
61 return Status(StatusCode::kInternal,
62 "List Objects Response's 'prefix' is not a string.");
63 }
64 result.prefixes.emplace_back(prefix.get<std::string>());
65 }
66
67 return result;
68 }
69
operator <<(std::ostream & os,ListObjectsResponse const & r)70 std::ostream& operator<<(std::ostream& os, ListObjectsResponse const& r) {
71 os << "ListObjectsResponse={next_page_token=" << r.next_page_token
72 << ", items={";
73 std::copy(r.items.begin(), r.items.end(),
74 std::ostream_iterator<ObjectMetadata>(os, "\n "));
75 os << "}, prefixes={";
76 std::copy(r.prefixes.begin(), r.prefixes.end(),
77 std::ostream_iterator<std::string>(os, "\n "));
78 return os << "}}";
79 }
80
operator <<(std::ostream & os,GetObjectMetadataRequest const & r)81 std::ostream& operator<<(std::ostream& os, GetObjectMetadataRequest const& r) {
82 os << "GetObjectMetadataRequest={bucket_name=" << r.bucket_name()
83 << ", object_name=" << r.object_name();
84 r.DumpOptions(os, ", ");
85 return os << "}";
86 }
87
operator <<(std::ostream & os,InsertObjectMediaRequest const & r)88 std::ostream& operator<<(std::ostream& os, InsertObjectMediaRequest const& r) {
89 os << "InsertObjectMediaRequest={bucket_name=" << r.bucket_name()
90 << ", object_name=" << r.object_name();
91 r.DumpOptions(os, ", ");
92 std::size_t constexpr kMaxDumpSize = 1024;
93 if (r.contents().size() > kMaxDumpSize) {
94 os << ", contents[0..1024]=\n"
95 << BinaryDataAsDebugString(r.contents().data(), kMaxDumpSize);
96 } else {
97 os << ", contents=\n"
98 << BinaryDataAsDebugString(r.contents().data(), r.contents().size());
99 }
100 return os << "}";
101 }
102
operator <<(std::ostream & os,CopyObjectRequest const & r)103 std::ostream& operator<<(std::ostream& os, CopyObjectRequest const& r) {
104 os << "CopyObjectRequest={destination_bucket=" << r.destination_bucket()
105 << ", destination_object=" << r.destination_object()
106 << ", source_bucket=" << r.source_bucket()
107 << ", source_object=" << r.source_object();
108 r.DumpOptions(os, ", ");
109 return os << "}";
110 }
111
RequiresNoCache() const112 bool ReadObjectRangeRequest::RequiresNoCache() const {
113 if (HasOption<ReadRange>()) {
114 return true;
115 }
116 if (HasOption<ReadFromOffset>() && GetOption<ReadFromOffset>().value() != 0) {
117 return true;
118 }
119 return HasOption<ReadLast>();
120 }
121
RequiresRangeHeader() const122 bool ReadObjectRangeRequest::RequiresRangeHeader() const {
123 return RequiresNoCache();
124 }
125
RangeHeader() const126 std::string ReadObjectRangeRequest::RangeHeader() const {
127 if (HasOption<ReadRange>() && HasOption<ReadFromOffset>()) {
128 auto range = GetOption<ReadRange>().value();
129 auto offset = GetOption<ReadFromOffset>().value();
130 auto begin = (std::max)(range.begin, offset);
131 return "Range: bytes=" + std::to_string(begin) + "-" +
132 std::to_string(range.end - 1);
133 }
134 if (HasOption<ReadRange>()) {
135 auto range = GetOption<ReadRange>().value();
136 return "Range: bytes=" + std::to_string(range.begin) + "-" +
137 std::to_string(range.end - 1);
138 }
139 if (HasOption<ReadFromOffset>()) {
140 auto offset = GetOption<ReadFromOffset>().value();
141 if (offset != 0) {
142 return "Range: bytes=" + std::to_string(offset) + "-";
143 }
144 }
145 if (HasOption<ReadLast>()) {
146 auto last = GetOption<ReadLast>().value();
147 return "Range: bytes=-" + std::to_string(last);
148 }
149 return "";
150 }
151
StartingByte() const152 std::int64_t ReadObjectRangeRequest::StartingByte() const {
153 std::int64_t result = 0;
154 if (HasOption<ReadRange>()) {
155 result = (std::max)(result, GetOption<ReadRange>().value().begin);
156 }
157 if (HasOption<ReadFromOffset>()) {
158 result = (std::max)(result, GetOption<ReadFromOffset>().value());
159 }
160 if (HasOption<ReadLast>()) {
161 // The value of `StartingByte()` is unknown if `ReadLast` is set
162 result = -1;
163 }
164 return result;
165 }
166
operator <<(std::ostream & os,ReadObjectRangeRequest const & r)167 std::ostream& operator<<(std::ostream& os, ReadObjectRangeRequest const& r) {
168 os << "ReadObjectRangeRequest={bucket_name=" << r.bucket_name()
169 << ", object_name=" << r.object_name();
170 r.DumpOptions(os, ", ");
171 return os << "}";
172 }
173
174 #include "google/cloud/internal/disable_msvc_crt_secure_warnings.inc"
FromHttpResponse(HttpResponse && response)175 ReadObjectRangeResponse ReadObjectRangeResponse::FromHttpResponse(
176 HttpResponse&& response) {
177 auto loc = response.headers.find(std::string("content-range"));
178 if (response.headers.end() == loc) {
179 google::cloud::internal::ThrowInvalidArgument(
180 "invalid http response for ReadObjectRange");
181 }
182
183 std::string const& content_range_value = loc->second;
184 // capture this function name, not the lambda's
185 auto const* function = __func__;
186 auto raise_error = [&content_range_value, &function]() {
187 std::ostringstream os;
188 os << static_cast<char const*>(function)
189 << " invalid format for content-range header <" << content_range_value
190 << ">";
191 google::cloud::internal::ThrowInvalidArgument(os.str());
192 };
193 char const unit_descriptor[] = "bytes";
194 if (content_range_value.rfind(unit_descriptor, 0) != 0) {
195 raise_error();
196 }
197 char const* buffer = content_range_value.data();
198 auto size = content_range_value.size();
199 // skip the initial "bytes " string.
200 buffer += sizeof(unit_descriptor);
201
202 if (size < 2) {
203 raise_error();
204 }
205
206 if (buffer[0] == '*' && buffer[1] == '/') {
207 // The header is just the indication of size ('bytes */<size>'), parse that.
208 buffer += 2;
209 std::int64_t object_size;
210 auto count = std::sscanf(buffer, "%" PRId64, &object_size);
211 if (count != 1) {
212 raise_error();
213 }
214 return ReadObjectRangeResponse{std::move(response.payload), 0, 0,
215 object_size};
216 }
217
218 std::int64_t first_byte;
219 std::int64_t last_byte;
220 std::int64_t object_size;
221 auto count = std::sscanf(buffer, "%" PRId64 "-%" PRId64 "/%" PRId64,
222 &first_byte, &last_byte, &object_size);
223 if (count != 3) {
224 raise_error();
225 }
226
227 return ReadObjectRangeResponse{std::move(response.payload), first_byte,
228 last_byte, object_size};
229 }
230 #include "google/cloud/internal/diagnostics_pop.inc"
231
operator <<(std::ostream & os,ReadObjectRangeResponse const & r)232 std::ostream& operator<<(std::ostream& os, ReadObjectRangeResponse const& r) {
233 return os << "ReadObjectRangeResponse={range=" << r.first_byte << "-"
234 << r.last_byte << "/" << r.object_size << ", contents=\n"
235 << BinaryDataAsDebugString(r.contents.data(), r.contents.size())
236 << "}";
237 }
238
operator <<(std::ostream & os,DeleteObjectRequest const & r)239 std::ostream& operator<<(std::ostream& os, DeleteObjectRequest const& r) {
240 os << "DeleteObjectRequest={bucket_name=" << r.bucket_name()
241 << ", object_name=" << r.object_name();
242 r.DumpOptions(os, ", ");
243 return os << "}";
244 }
245
operator <<(std::ostream & os,UpdateObjectRequest const & r)246 std::ostream& operator<<(std::ostream& os, UpdateObjectRequest const& r) {
247 os << "UpdateObjectRequest={bucket_name=" << r.bucket_name()
248 << ", object_name=" << r.object_name() << ", metadata=" << r.metadata();
249 r.DumpOptions(os, ", ");
250 return os << "}";
251 }
252
ComposeObjectRequest(std::string bucket_name,std::vector<ComposeSourceObject> source_objects,std::string destination_object_name)253 ComposeObjectRequest::ComposeObjectRequest(
254 std::string bucket_name, std::vector<ComposeSourceObject> source_objects,
255 std::string destination_object_name)
256 : GenericObjectRequest(std::move(bucket_name),
257 std::move(destination_object_name)),
258 source_objects_(std::move(source_objects)) {}
259
JsonPayload() const260 std::string ComposeObjectRequest::JsonPayload() const {
261 nlohmann::json compose_object_payload_json;
262 compose_object_payload_json["kind"] = "storage#composeRequest";
263 nlohmann::json destination_metadata_payload;
264 if (HasOption<WithObjectMetadata>()) {
265 destination_metadata_payload =
266 ObjectMetadataJsonForCompose(GetOption<WithObjectMetadata>().value());
267 }
268 if (!destination_metadata_payload.is_null()) {
269 compose_object_payload_json["destination"] = destination_metadata_payload;
270 }
271 nlohmann::json source_object_list;
272 for (auto const& source_object : source_objects_) {
273 nlohmann::json source_object_json;
274 source_object_json["name"] = source_object.object_name;
275 if (source_object.generation.has_value()) {
276 source_object_json["generation"] = source_object.generation.value();
277 }
278 if (source_object.if_generation_match.has_value()) {
279 source_object_json["ifGenerationMatch"] =
280 source_object.if_generation_match.value();
281 }
282 source_object_list.emplace_back(std::move(source_object_json));
283 }
284 compose_object_payload_json["sourceObjects"] = source_object_list;
285
286 return compose_object_payload_json.dump();
287 }
288
operator <<(std::ostream & os,ComposeObjectRequest const & r)289 std::ostream& operator<<(std::ostream& os, ComposeObjectRequest const& r) {
290 os << "ComposeObjectRequest={bucket_name=" << r.bucket_name()
291 << ", destination_object_name=" << r.object_name();
292 r.DumpOptions(os, ", ");
293 return os << ", payload=" << r.JsonPayload() << "}";
294 }
295
PatchObjectRequest(std::string bucket_name,std::string object_name,ObjectMetadata const & original,ObjectMetadata const & updated)296 PatchObjectRequest::PatchObjectRequest(std::string bucket_name,
297 std::string object_name,
298 ObjectMetadata const& original,
299 ObjectMetadata const& updated)
300 : GenericObjectRequest(std::move(bucket_name), std::move(object_name)) {
301 // Compare each writeable field to build the patch.
302 ObjectMetadataPatchBuilder builder;
303
304 if (original.acl() != updated.acl()) {
305 builder.SetAcl(updated.acl());
306 }
307 if (original.cache_control() != updated.cache_control()) {
308 builder.SetCacheControl(updated.cache_control());
309 }
310 if (original.content_disposition() != updated.content_disposition()) {
311 builder.SetContentDisposition(updated.content_disposition());
312 }
313 if (original.content_encoding() != updated.content_encoding()) {
314 builder.SetContentEncoding(updated.content_encoding());
315 }
316 if (original.content_language() != updated.content_language()) {
317 builder.SetContentLanguage(updated.content_language());
318 }
319 if (original.content_type() != updated.content_type()) {
320 builder.SetContentType(updated.content_type());
321 }
322 if (original.event_based_hold() != updated.event_based_hold()) {
323 builder.SetEventBasedHold(updated.event_based_hold());
324 }
325
326 if (original.metadata() != updated.metadata()) {
327 if (updated.metadata().empty()) {
328 builder.ResetMetadata();
329 } else {
330 std::map<std::string, std::string> difference;
331 // Find the keys in the original map that are not in the new map. Using
332 // `std::set_difference()` works because, unlike `std::unordered_map` the
333 // `std::map` iterators return elements ordered by key:
334 std::set_difference(original.metadata().begin(),
335 original.metadata().end(), updated.metadata().begin(),
336 updated.metadata().end(),
337 std::inserter(difference, difference.end()),
338 // We want to compare just keys and ignore values, the
339 // map class provides such a function, so use it:
340 original.metadata().value_comp());
341 for (auto&& d : difference) {
342 builder.ResetMetadata(d.first);
343 }
344
345 // Find the elements (comparing key and value) in the updated map that
346 // are not in the original map:
347 difference.clear();
348 std::set_difference(updated.metadata().begin(), updated.metadata().end(),
349 original.metadata().begin(),
350 original.metadata().end(),
351 std::inserter(difference, difference.end()));
352 for (auto&& d : difference) {
353 builder.SetMetadata(d.first, d.second);
354 }
355 }
356 }
357
358 if (original.temporary_hold() != updated.temporary_hold()) {
359 builder.SetTemporaryHold(updated.temporary_hold());
360 }
361
362 payload_ = builder.BuildPatch();
363 }
364
PatchObjectRequest(std::string bucket_name,std::string object_name,ObjectMetadataPatchBuilder const & patch)365 PatchObjectRequest::PatchObjectRequest(std::string bucket_name,
366 std::string object_name,
367 ObjectMetadataPatchBuilder const& patch)
368 : GenericObjectRequest(std::move(bucket_name), std::move(object_name)),
369 payload_(patch.BuildPatch()) {}
370
operator <<(std::ostream & os,PatchObjectRequest const & r)371 std::ostream& operator<<(std::ostream& os, PatchObjectRequest const& r) {
372 os << "PatchObjectRequest={bucket_name=" << r.bucket_name()
373 << ", object_name=" << r.object_name();
374 r.DumpOptions(os, ", ");
375 return os << ", payload=" << r.payload() << "}";
376 }
377
operator <<(std::ostream & os,RewriteObjectRequest const & r)378 std::ostream& operator<<(std::ostream& os, RewriteObjectRequest const& r) {
379 os << "RewriteObjectRequest={destination_bucket=" << r.destination_bucket()
380 << ", destination_object=" << r.destination_object()
381 << ", source_bucket=" << r.source_bucket()
382 << ", source_object=" << r.source_object()
383 << ", rewrite_token=" << r.rewrite_token();
384 r.DumpOptions(os, ", ");
385 return os << "}";
386 }
387
FromHttpResponse(std::string const & payload)388 StatusOr<RewriteObjectResponse> RewriteObjectResponse::FromHttpResponse(
389 std::string const& payload) {
390 auto object = nlohmann::json::parse(payload, nullptr, false);
391 if (!object.is_object()) {
392 return Status(StatusCode::kInvalidArgument, __func__);
393 }
394
395 RewriteObjectResponse result;
396 result.total_bytes_rewritten =
397 ParseUnsignedLongField(object, "totalBytesRewritten");
398 result.object_size = ParseUnsignedLongField(object, "objectSize");
399 result.done = object.value("done", false);
400 result.rewrite_token = object.value("rewriteToken", "");
401 if (object.count("resource") != 0) {
402 auto parsed = internal::ObjectMetadataParser::FromJson(object["resource"]);
403 if (!parsed.ok()) {
404 return std::move(parsed).status();
405 }
406 result.resource = std::move(*parsed);
407 }
408 return result;
409 }
410
operator <<(std::ostream & os,RewriteObjectResponse const & r)411 std::ostream& operator<<(std::ostream& os, RewriteObjectResponse const& r) {
412 return os << "RewriteObjectResponse={total_bytes_rewritten="
413 << r.total_bytes_rewritten << ", object_size=" << r.object_size
414 << ", done=" << std::boolalpha << r.done
415 << ", rewrite_token=" << r.rewrite_token
416 << ", resource=" << r.resource << "}";
417 }
418
operator <<(std::ostream & os,ResumableUploadRequest const & r)419 std::ostream& operator<<(std::ostream& os, ResumableUploadRequest const& r) {
420 os << "ResumableUploadRequest={bucket_name=" << r.bucket_name()
421 << ", object_name=" << r.object_name();
422 r.DumpOptions(os, ", ");
423 return os << "}";
424 }
425
operator <<(std::ostream & os,DeleteResumableUploadRequest const & r)426 std::ostream& operator<<(std::ostream& os,
427 DeleteResumableUploadRequest const& r) {
428 os << "DeleteResumableUploadRequest={upload_session_url="
429 << r.upload_session_url();
430 r.DumpOptions(os, ", ");
431 return os << "}";
432 }
433
RangeHeader() const434 std::string UploadChunkRequest::RangeHeader() const {
435 std::ostringstream os;
436 os << "Content-Range: bytes ";
437 auto const size = payload_size();
438 if (size == 0) {
439 // This typically happens when the sender realizes too late that the
440 // previous chunk was really the last chunk (e.g. the file is exactly a
441 // multiple of the quantum, reading the last chunk from a file, or sending
442 // it as part of a stream that does not detect the EOF), the formatting of
443 // the range is special in this case.
444 os << "*";
445 } else {
446 os << range_begin() << "-" << range_begin() + size - 1;
447 }
448 if (!last_chunk_) {
449 os << "/*";
450 } else {
451 os << "/" << source_size();
452 }
453 return std::move(os).str();
454 }
455
operator <<(std::ostream & os,UploadChunkRequest const & r)456 std::ostream& operator<<(std::ostream& os, UploadChunkRequest const& r) {
457 os << "UploadChunkRequest={upload_session_url=" << r.upload_session_url()
458 << ", range=<" << r.RangeHeader() << ">";
459 r.DumpOptions(os, ", ");
460 os << ", payload={";
461 auto constexpr kMaxOutputBytes = 128;
462 char const* sep = "";
463 for (auto const& b : r.payload()) {
464 os << sep << "{"
465 << BinaryDataAsDebugString(b.data(), b.size(), kMaxOutputBytes) << "}";
466 sep = ", ";
467 }
468 return os << "}}";
469 }
470
operator <<(std::ostream & os,QueryResumableUploadRequest const & r)471 std::ostream& operator<<(std::ostream& os,
472 QueryResumableUploadRequest const& r) {
473 os << "QueryResumableUploadRequest={upload_session_url="
474 << r.upload_session_url();
475 r.DumpOptions(os, ", ");
476 return os << "}";
477 }
478
479 } // namespace internal
480 } // namespace STORAGE_CLIENT_NS
481 } // namespace storage
482 } // namespace cloud
483 } // namespace google
484