1 // Copyright 2019 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //     http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "google/cloud/storage/client.h"
16 #include "google/cloud/storage/object_stream.h"
17 #include "google/cloud/storage/testing/storage_integration_test.h"
18 #include "google/cloud/internal/getenv.h"
19 #include "google/cloud/testing_util/assert_ok.h"
20 #include "google/cloud/testing_util/scoped_environment.h"
21 #include <crc32c/crc32c.h>
22 #include <gmock/gmock.h>
23 #include <nlohmann/json.hpp>
24 #include <algorithm>
25 #include <vector>
26 
27 #if GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
28 #include "google/cloud/storage/internal/grpc_client.h"
29 #include "google/cloud/grpc_error_delegate.h"
30 #include <grpcpp/grpcpp.h>
31 #endif  // GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
32 
33 namespace google {
34 namespace cloud {
35 namespace storage {
36 inline namespace STORAGE_CLIENT_NS {
37 namespace internal {
38 namespace {
39 
40 using ::testing::Contains;
41 using ::testing::Not;
42 
43 // When GOOGLE_CLOUD_CPP_HAVE_GRPC is not set these tests compile, but they
44 // actually just run against the regular GCS REST API. That is fine.
45 class GrpcIntegrationTest
46     : public google::cloud::storage::testing::StorageIntegrationTest {
47  protected:
SetUp()48   void SetUp() override {
49     project_id_ =
50         google::cloud::internal::GetEnv("GOOGLE_CLOUD_PROJECT").value_or("");
51     ASSERT_FALSE(project_id_.empty()) << "GOOGLE_CLOUD_PROJECT is not set";
52   }
project_id() const53   std::string project_id() const { return project_id_; }
54 
55  private:
56   std::string project_id_;
57   testing_util::ScopedEnvironment grpc_config_{
58       "GOOGLE_CLOUD_CPP_STORAGE_GRPC_CONFIG", "metadata"};
59 };
60 
TEST_F(GrpcIntegrationTest,BucketCRUD)61 TEST_F(GrpcIntegrationTest, BucketCRUD) {
62   auto client = MakeBucketIntegrationTestClient();
63   ASSERT_STATUS_OK(client);
64 
65   auto bucket_name = MakeRandomBucketName("cloud-cpp-testing-");
66   auto bucket_metadata = client->CreateBucketForProject(
67       bucket_name, project_id(), BucketMetadata());
68   ASSERT_STATUS_OK(bucket_metadata);
69 
70   EXPECT_EQ(bucket_name, bucket_metadata->name());
71 
72   auto list_bucket_names = [&client, this] {
73     std::vector<std::string> names;
74     for (auto b : client->ListBucketsForProject(project_id())) {
75       EXPECT_STATUS_OK(b);
76       if (!b) break;
77       names.push_back(b->name());
78     }
79     return names;
80   };
81   EXPECT_THAT(list_bucket_names(), Contains(bucket_name));
82 
83   auto get = client->GetBucketMetadata(bucket_name);
84   ASSERT_STATUS_OK(get);
85   EXPECT_EQ(bucket_name, get->id());
86   EXPECT_EQ(bucket_name, get->name());
87 
88   auto delete_status = client->DeleteBucket(bucket_name);
89   EXPECT_STATUS_OK(delete_status);
90   EXPECT_THAT(list_bucket_names(), Not(Contains(bucket_name)));
91 }
92 
TEST_F(GrpcIntegrationTest,ObjectCRUD)93 TEST_F(GrpcIntegrationTest, ObjectCRUD) {
94   auto bucket_client = MakeBucketIntegrationTestClient();
95   ASSERT_STATUS_OK(bucket_client);
96 
97   auto client = MakeIntegrationTestClient();
98   ASSERT_STATUS_OK(client);
99 
100   auto bucket_name = MakeRandomBucketName("cloud-cpp-testing-");
101   auto object_name = MakeRandomObjectName();
102   auto bucket_metadata = bucket_client->CreateBucketForProject(
103       bucket_name, project_id(), BucketMetadata());
104   ASSERT_STATUS_OK(bucket_metadata);
105 
106   EXPECT_EQ(bucket_name, bucket_metadata->name());
107 
108   auto object_metadata = client->InsertObject(
109       bucket_name, object_name, LoremIpsum(), IfGenerationMatch(0));
110   ASSERT_STATUS_OK(object_metadata);
111 
112   auto stream = client->ReadObject(bucket_name, object_name);
113 
114   std::string actual(std::istreambuf_iterator<char>{stream}, {});
115   EXPECT_EQ(LoremIpsum(), actual);
116   EXPECT_STATUS_OK(stream.status());
117 
118   auto delete_object_status = client->DeleteObject(
119       bucket_name, object_name, Generation(object_metadata->generation()));
120   EXPECT_STATUS_OK(delete_object_status);
121 
122   auto delete_bucket_status = bucket_client->DeleteBucket(bucket_name);
123   EXPECT_STATUS_OK(delete_bucket_status);
124 }
125 
TEST_F(GrpcIntegrationTest,WriteResume)126 TEST_F(GrpcIntegrationTest, WriteResume) {
127   auto client = MakeIntegrationTestClient();
128   ASSERT_STATUS_OK(client);
129 
130   auto bucket_name = MakeRandomBucketName("cloud-cpp-testing-");
131   auto object_name = MakeRandomObjectName();
132   auto bucket_metadata = client->CreateBucketForProject(
133       bucket_name, project_id(), BucketMetadata());
134   ASSERT_STATUS_OK(bucket_metadata);
135 
136   // We will construct the expected response while streaming the data up.
137   std::ostringstream expected;
138 
139   // Create the object, but only if it does not exist already.
140   std::string session_id;
141   {
142     auto old_os =
143         client->WriteObject(bucket_name, object_name, IfGenerationMatch(0),
144                             NewResumableUploadSession());
145     ASSERT_TRUE(old_os.good()) << "status=" << old_os.metadata().status();
146     session_id = old_os.resumable_session_id();
147     std::move(old_os).Suspend();
148   }
149 
150   auto os = client->WriteObject(bucket_name, object_name,
151                                 RestoreResumableUploadSession(session_id));
152   ASSERT_TRUE(os.good()) << "status=" << os.metadata().status();
153   EXPECT_EQ(session_id, os.resumable_session_id());
154   os << LoremIpsum();
155   os.Close();
156   ASSERT_STATUS_OK(os.metadata());
157   ObjectMetadata meta = os.metadata().value();
158   EXPECT_EQ(object_name, meta.name());
159   EXPECT_EQ(bucket_name, meta.bucket());
160   if (UsingTestbench()) {
161     EXPECT_TRUE(meta.has_metadata("x_testbench_upload"));
162     EXPECT_EQ("resumable", meta.metadata("x_testbench_upload"));
163   }
164 
165   auto status = client->DeleteObject(bucket_name, object_name);
166   EXPECT_STATUS_OK(status);
167 
168   auto delete_bucket_status = client->DeleteBucket(bucket_name);
169   EXPECT_STATUS_OK(delete_bucket_status);
170 }
171 
172 #if GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
173 /// @test Verify that NOT_FOUND is returned for missing objects
TEST_F(GrpcIntegrationTest,GetObjectMediaNotFound)174 TEST_F(GrpcIntegrationTest, GetObjectMediaNotFound) {
175   auto bucket_client = MakeBucketIntegrationTestClient();
176   ASSERT_STATUS_OK(bucket_client);
177 
178   auto client = Client::CreateDefaultClient();
179   ASSERT_STATUS_OK(client);
180 
181   auto bucket_name = MakeRandomBucketName("cloud-cpp-testing-");
182   auto bucket_metadata = bucket_client->CreateBucketForProject(
183       bucket_name, project_id(), BucketMetadata());
184   ASSERT_STATUS_OK(bucket_metadata);
185 
186   auto object_name = MakeRandomObjectName();
187 
188   auto channel = grpc::CreateChannel("storage.googleapis.com",
189                                      grpc::GoogleDefaultCredentials());
190   auto stub = google::storage::v1::Storage::NewStub(channel);
191 
192   grpc::ClientContext context;
193   google::storage::v1::GetObjectMediaRequest request;
194   request.set_bucket(bucket_name);
195   request.set_object(object_name);
196   auto stream = stub->GetObjectMedia(&context, request);
197   google::storage::v1::GetObjectMediaResponse response;
198   auto open = true;
199   for (int i = 0; i != 100 && open; ++i) {
200     open = stream->Read(&response);
201   }
202   EXPECT_FALSE(open);
203 
204   auto status = stream->Finish();
205   ASSERT_EQ(grpc::StatusCode::NOT_FOUND, status.error_code())
206       << "message = " << status.error_message();
207   auto delete_bucket_status = bucket_client->DeleteBucket(bucket_name);
208   EXPECT_STATUS_OK(delete_bucket_status);
209 }
210 #endif  // GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
211 
TEST_F(GrpcIntegrationTest,InsertLarge)212 TEST_F(GrpcIntegrationTest, InsertLarge) {
213   auto bucket_client = MakeBucketIntegrationTestClient();
214   ASSERT_STATUS_OK(bucket_client);
215 
216   auto client = MakeIntegrationTestClient();
217   ASSERT_STATUS_OK(client);
218 
219   auto bucket_name = MakeRandomBucketName("cloud-cpp-testing-");
220   auto object_name = MakeRandomObjectName();
221   auto bucket_metadata = bucket_client->CreateBucketForProject(
222       bucket_name, project_id(), BucketMetadata());
223   ASSERT_STATUS_OK(bucket_metadata);
224 
225   // Insert an object that is larger than 4 MiB, and its size is not a multiple
226   // of 256 KiB.
227   auto const desired_size = 8 * 1024 * 1024L + 253 * 1024 + 15;
228   auto data = MakeRandomData(desired_size);
229   auto metadata = client->InsertObject(bucket_name, object_name, data,
230                                        IfGenerationMatch(0));
231   EXPECT_STATUS_OK(metadata);
232   if (!metadata) {
233     auto delete_bucket_status = client->DeleteBucket(bucket_name);
234     EXPECT_STATUS_OK(delete_bucket_status);
235     return;
236   }
237   EXPECT_EQ(desired_size, metadata->size());
238 
239   auto status = client->DeleteObject(bucket_name, object_name);
240   EXPECT_STATUS_OK(status);
241 
242   auto delete_bucket_status = bucket_client->DeleteBucket(bucket_name);
243   EXPECT_STATUS_OK(delete_bucket_status);
244 }
245 
TEST_F(GrpcIntegrationTest,StreamLargeChunks)246 TEST_F(GrpcIntegrationTest, StreamLargeChunks) {
247   auto bucket_client = MakeBucketIntegrationTestClient();
248   ASSERT_STATUS_OK(bucket_client);
249 
250   auto client = MakeIntegrationTestClient();
251   ASSERT_STATUS_OK(client);
252 
253   auto bucket_name = MakeRandomBucketName("cloud-cpp-testing-");
254   auto object_name = MakeRandomObjectName();
255   auto bucket_metadata = bucket_client->CreateBucketForProject(
256       bucket_name, project_id(), BucketMetadata());
257   ASSERT_STATUS_OK(bucket_metadata);
258 
259   // Insert an object in chunks larger than 4 MiB each.
260   auto const desired_size = 8 * 1024 * 1024L;
261   auto data = MakeRandomData(desired_size);
262   auto stream =
263       client->WriteObject(bucket_name, object_name, IfGenerationMatch(0));
264   stream.write(data.data(), data.size());
265   EXPECT_TRUE(stream.good());
266   stream.write(data.data(), data.size());
267   EXPECT_TRUE(stream.good());
268   stream.Close();
269   EXPECT_FALSE(stream.bad());
270   EXPECT_STATUS_OK(stream.metadata());
271 
272   EXPECT_EQ(2 * desired_size, stream.metadata()->size());
273 
274   auto status = client->DeleteObject(bucket_name, object_name);
275   EXPECT_STATUS_OK(status);
276 
277   auto delete_bucket_status = bucket_client->DeleteBucket(bucket_name);
278   EXPECT_STATUS_OK(delete_bucket_status);
279 }
280 
281 }  // namespace
282 }  // namespace internal
283 }  // namespace STORAGE_CLIENT_NS
284 }  // namespace storage
285 }  // namespace cloud
286 }  // namespace google
287