1 // Copyright 2019 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "google/cloud/storage/client.h"
16 #include "google/cloud/storage/object_stream.h"
17 #include "google/cloud/storage/testing/storage_integration_test.h"
18 #include "google/cloud/internal/getenv.h"
19 #include "google/cloud/testing_util/assert_ok.h"
20 #include "google/cloud/testing_util/scoped_environment.h"
21 #include <crc32c/crc32c.h>
22 #include <gmock/gmock.h>
23 #include <nlohmann/json.hpp>
24 #include <algorithm>
25 #include <vector>
26
27 #if GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
28 #include "google/cloud/storage/internal/grpc_client.h"
29 #include "google/cloud/grpc_error_delegate.h"
30 #include <grpcpp/grpcpp.h>
31 #endif // GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
32
33 namespace google {
34 namespace cloud {
35 namespace storage {
36 inline namespace STORAGE_CLIENT_NS {
37 namespace internal {
38 namespace {
39
40 using ::testing::Contains;
41 using ::testing::Not;
42
43 // When GOOGLE_CLOUD_CPP_HAVE_GRPC is not set these tests compile, but they
44 // actually just run against the regular GCS REST API. That is fine.
45 class GrpcIntegrationTest
46 : public google::cloud::storage::testing::StorageIntegrationTest {
47 protected:
SetUp()48 void SetUp() override {
49 project_id_ =
50 google::cloud::internal::GetEnv("GOOGLE_CLOUD_PROJECT").value_or("");
51 ASSERT_FALSE(project_id_.empty()) << "GOOGLE_CLOUD_PROJECT is not set";
52 }
project_id() const53 std::string project_id() const { return project_id_; }
54
55 private:
56 std::string project_id_;
57 testing_util::ScopedEnvironment grpc_config_{
58 "GOOGLE_CLOUD_CPP_STORAGE_GRPC_CONFIG", "metadata"};
59 };
60
TEST_F(GrpcIntegrationTest,BucketCRUD)61 TEST_F(GrpcIntegrationTest, BucketCRUD) {
62 auto client = MakeBucketIntegrationTestClient();
63 ASSERT_STATUS_OK(client);
64
65 auto bucket_name = MakeRandomBucketName("cloud-cpp-testing-");
66 auto bucket_metadata = client->CreateBucketForProject(
67 bucket_name, project_id(), BucketMetadata());
68 ASSERT_STATUS_OK(bucket_metadata);
69
70 EXPECT_EQ(bucket_name, bucket_metadata->name());
71
72 auto list_bucket_names = [&client, this] {
73 std::vector<std::string> names;
74 for (auto b : client->ListBucketsForProject(project_id())) {
75 EXPECT_STATUS_OK(b);
76 if (!b) break;
77 names.push_back(b->name());
78 }
79 return names;
80 };
81 EXPECT_THAT(list_bucket_names(), Contains(bucket_name));
82
83 auto get = client->GetBucketMetadata(bucket_name);
84 ASSERT_STATUS_OK(get);
85 EXPECT_EQ(bucket_name, get->id());
86 EXPECT_EQ(bucket_name, get->name());
87
88 auto delete_status = client->DeleteBucket(bucket_name);
89 EXPECT_STATUS_OK(delete_status);
90 EXPECT_THAT(list_bucket_names(), Not(Contains(bucket_name)));
91 }
92
TEST_F(GrpcIntegrationTest,ObjectCRUD)93 TEST_F(GrpcIntegrationTest, ObjectCRUD) {
94 auto bucket_client = MakeBucketIntegrationTestClient();
95 ASSERT_STATUS_OK(bucket_client);
96
97 auto client = MakeIntegrationTestClient();
98 ASSERT_STATUS_OK(client);
99
100 auto bucket_name = MakeRandomBucketName("cloud-cpp-testing-");
101 auto object_name = MakeRandomObjectName();
102 auto bucket_metadata = bucket_client->CreateBucketForProject(
103 bucket_name, project_id(), BucketMetadata());
104 ASSERT_STATUS_OK(bucket_metadata);
105
106 EXPECT_EQ(bucket_name, bucket_metadata->name());
107
108 auto object_metadata = client->InsertObject(
109 bucket_name, object_name, LoremIpsum(), IfGenerationMatch(0));
110 ASSERT_STATUS_OK(object_metadata);
111
112 auto stream = client->ReadObject(bucket_name, object_name);
113
114 std::string actual(std::istreambuf_iterator<char>{stream}, {});
115 EXPECT_EQ(LoremIpsum(), actual);
116 EXPECT_STATUS_OK(stream.status());
117
118 auto delete_object_status = client->DeleteObject(
119 bucket_name, object_name, Generation(object_metadata->generation()));
120 EXPECT_STATUS_OK(delete_object_status);
121
122 auto delete_bucket_status = bucket_client->DeleteBucket(bucket_name);
123 EXPECT_STATUS_OK(delete_bucket_status);
124 }
125
TEST_F(GrpcIntegrationTest,WriteResume)126 TEST_F(GrpcIntegrationTest, WriteResume) {
127 auto client = MakeIntegrationTestClient();
128 ASSERT_STATUS_OK(client);
129
130 auto bucket_name = MakeRandomBucketName("cloud-cpp-testing-");
131 auto object_name = MakeRandomObjectName();
132 auto bucket_metadata = client->CreateBucketForProject(
133 bucket_name, project_id(), BucketMetadata());
134 ASSERT_STATUS_OK(bucket_metadata);
135
136 // We will construct the expected response while streaming the data up.
137 std::ostringstream expected;
138
139 // Create the object, but only if it does not exist already.
140 std::string session_id;
141 {
142 auto old_os =
143 client->WriteObject(bucket_name, object_name, IfGenerationMatch(0),
144 NewResumableUploadSession());
145 ASSERT_TRUE(old_os.good()) << "status=" << old_os.metadata().status();
146 session_id = old_os.resumable_session_id();
147 std::move(old_os).Suspend();
148 }
149
150 auto os = client->WriteObject(bucket_name, object_name,
151 RestoreResumableUploadSession(session_id));
152 ASSERT_TRUE(os.good()) << "status=" << os.metadata().status();
153 EXPECT_EQ(session_id, os.resumable_session_id());
154 os << LoremIpsum();
155 os.Close();
156 ASSERT_STATUS_OK(os.metadata());
157 ObjectMetadata meta = os.metadata().value();
158 EXPECT_EQ(object_name, meta.name());
159 EXPECT_EQ(bucket_name, meta.bucket());
160 if (UsingTestbench()) {
161 EXPECT_TRUE(meta.has_metadata("x_testbench_upload"));
162 EXPECT_EQ("resumable", meta.metadata("x_testbench_upload"));
163 }
164
165 auto status = client->DeleteObject(bucket_name, object_name);
166 EXPECT_STATUS_OK(status);
167
168 auto delete_bucket_status = client->DeleteBucket(bucket_name);
169 EXPECT_STATUS_OK(delete_bucket_status);
170 }
171
172 #if GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
173 /// @test Verify that NOT_FOUND is returned for missing objects
TEST_F(GrpcIntegrationTest,GetObjectMediaNotFound)174 TEST_F(GrpcIntegrationTest, GetObjectMediaNotFound) {
175 auto bucket_client = MakeBucketIntegrationTestClient();
176 ASSERT_STATUS_OK(bucket_client);
177
178 auto client = Client::CreateDefaultClient();
179 ASSERT_STATUS_OK(client);
180
181 auto bucket_name = MakeRandomBucketName("cloud-cpp-testing-");
182 auto bucket_metadata = bucket_client->CreateBucketForProject(
183 bucket_name, project_id(), BucketMetadata());
184 ASSERT_STATUS_OK(bucket_metadata);
185
186 auto object_name = MakeRandomObjectName();
187
188 auto channel = grpc::CreateChannel("storage.googleapis.com",
189 grpc::GoogleDefaultCredentials());
190 auto stub = google::storage::v1::Storage::NewStub(channel);
191
192 grpc::ClientContext context;
193 google::storage::v1::GetObjectMediaRequest request;
194 request.set_bucket(bucket_name);
195 request.set_object(object_name);
196 auto stream = stub->GetObjectMedia(&context, request);
197 google::storage::v1::GetObjectMediaResponse response;
198 auto open = true;
199 for (int i = 0; i != 100 && open; ++i) {
200 open = stream->Read(&response);
201 }
202 EXPECT_FALSE(open);
203
204 auto status = stream->Finish();
205 ASSERT_EQ(grpc::StatusCode::NOT_FOUND, status.error_code())
206 << "message = " << status.error_message();
207 auto delete_bucket_status = bucket_client->DeleteBucket(bucket_name);
208 EXPECT_STATUS_OK(delete_bucket_status);
209 }
210 #endif // GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
211
TEST_F(GrpcIntegrationTest,InsertLarge)212 TEST_F(GrpcIntegrationTest, InsertLarge) {
213 auto bucket_client = MakeBucketIntegrationTestClient();
214 ASSERT_STATUS_OK(bucket_client);
215
216 auto client = MakeIntegrationTestClient();
217 ASSERT_STATUS_OK(client);
218
219 auto bucket_name = MakeRandomBucketName("cloud-cpp-testing-");
220 auto object_name = MakeRandomObjectName();
221 auto bucket_metadata = bucket_client->CreateBucketForProject(
222 bucket_name, project_id(), BucketMetadata());
223 ASSERT_STATUS_OK(bucket_metadata);
224
225 // Insert an object that is larger than 4 MiB, and its size is not a multiple
226 // of 256 KiB.
227 auto const desired_size = 8 * 1024 * 1024L + 253 * 1024 + 15;
228 auto data = MakeRandomData(desired_size);
229 auto metadata = client->InsertObject(bucket_name, object_name, data,
230 IfGenerationMatch(0));
231 EXPECT_STATUS_OK(metadata);
232 if (!metadata) {
233 auto delete_bucket_status = client->DeleteBucket(bucket_name);
234 EXPECT_STATUS_OK(delete_bucket_status);
235 return;
236 }
237 EXPECT_EQ(desired_size, metadata->size());
238
239 auto status = client->DeleteObject(bucket_name, object_name);
240 EXPECT_STATUS_OK(status);
241
242 auto delete_bucket_status = bucket_client->DeleteBucket(bucket_name);
243 EXPECT_STATUS_OK(delete_bucket_status);
244 }
245
TEST_F(GrpcIntegrationTest,StreamLargeChunks)246 TEST_F(GrpcIntegrationTest, StreamLargeChunks) {
247 auto bucket_client = MakeBucketIntegrationTestClient();
248 ASSERT_STATUS_OK(bucket_client);
249
250 auto client = MakeIntegrationTestClient();
251 ASSERT_STATUS_OK(client);
252
253 auto bucket_name = MakeRandomBucketName("cloud-cpp-testing-");
254 auto object_name = MakeRandomObjectName();
255 auto bucket_metadata = bucket_client->CreateBucketForProject(
256 bucket_name, project_id(), BucketMetadata());
257 ASSERT_STATUS_OK(bucket_metadata);
258
259 // Insert an object in chunks larger than 4 MiB each.
260 auto const desired_size = 8 * 1024 * 1024L;
261 auto data = MakeRandomData(desired_size);
262 auto stream =
263 client->WriteObject(bucket_name, object_name, IfGenerationMatch(0));
264 stream.write(data.data(), data.size());
265 EXPECT_TRUE(stream.good());
266 stream.write(data.data(), data.size());
267 EXPECT_TRUE(stream.good());
268 stream.Close();
269 EXPECT_FALSE(stream.bad());
270 EXPECT_STATUS_OK(stream.metadata());
271
272 EXPECT_EQ(2 * desired_size, stream.metadata()->size());
273
274 auto status = client->DeleteObject(bucket_name, object_name);
275 EXPECT_STATUS_OK(status);
276
277 auto delete_bucket_status = bucket_client->DeleteBucket(bucket_name);
278 EXPECT_STATUS_OK(delete_bucket_status);
279 }
280
281 } // namespace
282 } // namespace internal
283 } // namespace STORAGE_CLIENT_NS
284 } // namespace storage
285 } // namespace cloud
286 } // namespace google
287