1 // Copyright 2019 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "google/cloud/storage/testing/storage_integration_test.h"
16 #if GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
17 #include "google/cloud/storage/client.h"
18 #include "google/cloud/storage/internal/openssl_util.h"
19 #include "google/cloud/storage/internal/signed_url_requests.h"
20 #include "google/cloud/storage/list_objects_reader.h"
21 #include "google/cloud/storage/tests/conformance_tests.pb.h"
22 #include "google/cloud/internal/format_time_point.h"
23 #include "google/cloud/internal/getenv.h"
24 #include "google/cloud/internal/time_utils.h"
25 #include "google/cloud/terminate_handler.h"
26 #include "google/cloud/testing_util/assert_ok.h"
27 #include "absl/memory/memory.h"
28 #include <google/protobuf/util/json_util.h>
29 #include <gmock/gmock.h>
30 #include <fstream>
31 #include <type_traits>
32
33 /**
34 * @file
35 *
36 * Executes V4 signed URLs conformance tests described in an external file.
37 *
38 * We have a common set of conformance tests for V4 signed URLs used in all the
39 * GCS client libraries. The tests are stored in an external JSON file. This
40 * program receives the file name as an input parameter, loads it, and executes
41 * the tests described in the file.
42 *
43 * A separate command-line argument is the name of a (invalidated) service
44 * account key file used to create the signed URLs.
45 */
46
47 namespace google {
48 namespace cloud {
49 namespace storage {
50 inline namespace STORAGE_CLIENT_NS {
51 namespace {
52 using google::cloud::conformance::storage::v1::PostPolicyV4Test;
53 using google::cloud::conformance::storage::v1::SigningV4Test;
54 using google::cloud::conformance::storage::v1::UrlStyle;
55 using ::testing::HasSubstr;
56
57 // Initialized in main() below.
58 std::map<std::string, SigningV4Test>* signing_tests;
59 std::map<std::string, PostPolicyV4Test>* post_policy_tests;
60
61 class V4SignedUrlConformanceTest
62 : public google::cloud::storage::testing::StorageIntegrationTest,
63 public ::testing::WithParamInterface<std::string> {
64 protected:
SetUp()65 void SetUp() override {
66 service_account_key_filename_ =
67 google::cloud::internal::GetEnv(
68 "GOOGLE_CLOUD_CPP_STORAGE_TEST_SIGNING_KEYFILE")
69 .value_or("");
70 ASSERT_FALSE(service_account_key_filename_.empty());
71 }
72
73 std::string service_account_key_filename_;
74 };
75
76 class V4PostPolicyConformanceTest : public V4SignedUrlConformanceTest {};
77
TEST_P(V4SignedUrlConformanceTest,V4SignJson)78 TEST_P(V4SignedUrlConformanceTest, V4SignJson) {
79 auto creds = oauth2::CreateServiceAccountCredentialsFromJsonFilePath(
80 service_account_key_filename_);
81
82 ASSERT_STATUS_OK(creds);
83 std::string account_email = creds->get()->AccountEmail();
84 Client client(*creds);
85 std::string actual_canonical_request;
86 std::string actual_string_to_sign;
87
88 auto const& test_params = (*signing_tests)[GetParam()];
89 auto const& method_name = test_params.method();
90 auto const& bucket_name = test_params.bucket();
91 auto const& object_name = test_params.object();
92 auto const& scheme = test_params.scheme();
93 auto const& url_style = test_params.urlstyle();
94 auto const date =
95 ::google::cloud::internal::ToChronoTimePoint(test_params.timestamp());
96 auto const valid_for = std::chrono::seconds(test_params.expiration());
97 auto const& expected = test_params.expectedurl();
98 auto const& expected_canonical_request =
99 test_params.expectedcanonicalrequest();
100 auto const& expected_string_to_sign = test_params.expectedstringtosign();
101
102 // Extract the headers for each object
103 auto headers = test_params.headers();
104 auto params = test_params.query_parameters();
105
106 google::cloud::storage::internal::V4SignUrlRequest request(
107 method_name, bucket_name, object_name);
108 request.set_multiple_options(SignedUrlTimestamp(date),
109 SignedUrlDuration(valid_for));
110
111 std::vector<AddExtensionHeaderOption> header_extensions(5);
112 ASSERT_LE(headers.size(), header_extensions.size());
113 std::size_t idx = 0;
114 for (auto const& name_val : headers) {
115 request.set_multiple_options(
116 AddExtensionHeader(name_val.first, name_val.second));
117 header_extensions[idx++] =
118 AddExtensionHeader(name_val.first, name_val.second);
119 }
120
121 std::vector<AddQueryParameterOption> query_params(5);
122 ASSERT_LE(params.size(), query_params.size());
123 idx = 0;
124 for (auto const& name_val : params) {
125 request.set_multiple_options(
126 AddQueryParameterOption(name_val.first, name_val.second));
127 query_params[idx++] =
128 AddQueryParameterOption(name_val.first, name_val.second);
129 }
130
131 VirtualHostname virtual_hotname;
132 if (url_style == UrlStyle::VIRTUAL_HOSTED_STYLE) {
133 virtual_hotname = VirtualHostname(true);
134 request.set_multiple_options(VirtualHostname(true));
135 }
136
137 BucketBoundHostname domain_named_bucket;
138 if (url_style == UrlStyle::BUCKET_BOUND_HOSTNAME) {
139 domain_named_bucket =
140 BucketBoundHostname(test_params.bucketboundhostname());
141 request.set_multiple_options(
142 BucketBoundHostname(test_params.bucketboundhostname()));
143 }
144
145 auto actual = client.CreateV4SignedUrl(
146 method_name, bucket_name, object_name, SignedUrlTimestamp(date),
147 SignedUrlDuration(valid_for), header_extensions[0], header_extensions[1],
148 header_extensions[2], header_extensions[3], header_extensions[4],
149 query_params[0], query_params[1], query_params[2], query_params[3],
150 query_params[4], virtual_hotname, domain_named_bucket, Scheme(scheme));
151 ASSERT_STATUS_OK(request.Validate());
152 request.AddMissingRequiredHeaders();
153 ASSERT_STATUS_OK(request.Validate());
154
155 actual_string_to_sign = request.StringToSign(account_email);
156 actual_canonical_request = request.CanonicalRequest(account_email);
157
158 ASSERT_STATUS_OK(actual);
159 if (!domain_named_bucket.has_value()) {
160 EXPECT_THAT(*actual, HasSubstr(bucket_name));
161 }
162 EXPECT_EQ(expected, *actual);
163 EXPECT_EQ(expected_canonical_request, actual_canonical_request);
164 EXPECT_EQ(expected_string_to_sign, actual_string_to_sign);
165 }
166
167 INSTANTIATE_TEST_SUITE_P(
168 V4SignedUrlConformanceTest, V4SignedUrlConformanceTest,
__anon028cd7b70202null169 ::testing::ValuesIn([] {
170 std::vector<std::string> res;
171 std::transform(signing_tests->begin(), signing_tests->end(),
172 std::back_inserter(res),
173 [](std::pair<std::string, SigningV4Test> const& p) {
174 return p.first;
175 });
176 return res;
177 }()));
178
TEST_P(V4PostPolicyConformanceTest,V4PostPolicy)179 TEST_P(V4PostPolicyConformanceTest, V4PostPolicy) {
180 auto creds = oauth2::CreateServiceAccountCredentialsFromJsonFilePath(
181 service_account_key_filename_);
182
183 ASSERT_STATUS_OK(creds);
184 std::string account_email = creds->get()->AccountEmail();
185 Client client(*creds);
186
187 auto const& test_params = (*post_policy_tests)[GetParam()];
188 auto const& input = test_params.policyinput();
189 auto const& output = test_params.policyoutput();
190 auto const& bucket_name = input.bucket();
191 auto const& object_name = input.object();
192 auto const valid_for = std::chrono::seconds(input.expiration());
193 auto const timestamp =
194 ::google::cloud::internal::ToChronoTimePoint(input.timestamp());
195 auto const& scheme = input.scheme();
196 BucketBoundHostname domain_named_bucket;
197 auto const& url_style = input.urlstyle();
198 if (url_style == UrlStyle::BUCKET_BOUND_HOSTNAME) {
199 domain_named_bucket = BucketBoundHostname(input.bucketboundhostname());
200 }
201 VirtualHostname virtual_hotname;
202 if (url_style == UrlStyle::VIRTUAL_HOSTED_STYLE) {
203 virtual_hotname = VirtualHostname(true);
204 }
205
206 std::vector<PolicyDocumentCondition> conditions;
207 auto const& condition = input.conditions();
208
209 ASSERT_TRUE(condition.startswith().empty() ||
210 condition.startswith().size() == 2U)
211 << condition.startswith().size();
212 if (condition.startswith().size() == 2U) {
213 conditions.emplace_back(PolicyDocumentCondition::StartsWith(
214 condition.startswith()[0].substr(1), condition.startswith()[1]));
215 }
216 ASSERT_TRUE(condition.contentlengthrange().empty() ||
217 condition.contentlengthrange().size() == 2U)
218 << condition.contentlengthrange().size();
219 if (condition.contentlengthrange().size() == 2U) {
220 conditions.emplace_back(PolicyDocumentCondition::ContentLengthRange(
221 condition.contentlengthrange()[0], condition.contentlengthrange()[1]));
222 }
223
224 auto const& expected_url = output.url();
225 auto const& fields = output.fields();
226 auto const& expected_algorithm = fields.at("x-goog-algorithm");
227 auto const& expected_credential = fields.at("x-goog-credential");
228 auto const& expected_date = fields.at("x-goog-date");
229 auto const& expected_signature = fields.at("x-goog-signature");
230 auto const& expected_policy = fields.at("policy");
231
232 // We need to escape it because `nlohmann::json` interprets the escaped
233 // characters.
234 std::string const expected_decoded_policy =
235 *internal::PostPolicyV4Escape(output.expecteddecodedpolicy());
236
237 std::vector<AddExtensionFieldOption> extension_fields(5);
238 ASSERT_LE(input.fields().size(), extension_fields.size());
239 std::size_t idx = 0;
240 for (auto const& name_val : input.fields()) {
241 extension_fields[idx++] =
242 AddExtensionField(name_val.first, name_val.second);
243 }
244 PolicyDocumentV4 doc{bucket_name, object_name, valid_for, timestamp,
245 std::move(conditions)};
246 auto doc_res = client.GenerateSignedPostPolicyV4(
247 doc, extension_fields[0], extension_fields[1], extension_fields[2],
248 extension_fields[3], extension_fields[4], Scheme(scheme),
249 domain_named_bucket, virtual_hotname);
250 ASSERT_STATUS_OK(doc_res);
251 EXPECT_EQ(expected_policy, doc_res->policy);
252 auto actual_policy_vec = internal::Base64Decode(doc_res->policy);
253 std::string actual_policy(actual_policy_vec.begin(), actual_policy_vec.end());
254 EXPECT_EQ(expected_decoded_policy, actual_policy);
255 EXPECT_EQ(expected_url, doc_res->url);
256 EXPECT_EQ(expected_credential, doc_res->access_id);
257 EXPECT_EQ(expected_date, google::cloud::internal::FormatV4SignedUrlTimestamp(
258 doc_res->expiration - valid_for));
259 EXPECT_EQ(expected_algorithm, doc_res->signing_algorithm);
260 EXPECT_EQ(expected_signature, doc_res->signature);
261 EXPECT_EQ((std::map<std::string, std::string>(fields.begin(), fields.end())),
262 doc_res->required_form_fields);
263 }
264
265 INSTANTIATE_TEST_SUITE_P(
266 V4PostPolicyConformanceTest, V4PostPolicyConformanceTest,
__anon028cd7b70402null267 ::testing::ValuesIn([] {
268 std::vector<std::string> res;
269 std::transform(post_policy_tests->begin(), post_policy_tests->end(),
270 std::back_inserter(res),
271 [](std::pair<std::string, PostPolicyV4Test> const& p) {
272 return p.first;
273 });
274 return res;
275 }()));
276
277 } // namespace
278 } // namespace STORAGE_CLIENT_NS
279 } // namespace storage
280 } // namespace cloud
281 } // namespace google
282
283 using google::cloud::conformance::storage::v1::PostPolicyV4Test;
284 using google::cloud::conformance::storage::v1::SigningV4Test;
285
main(int argc,char * argv[])286 int main(int argc, char* argv[]) { // NOLINT(bugprone-exception-escape)
287 auto conformance_tests_file =
288 google::cloud::internal::GetEnv(
289 "GOOGLE_CLOUD_CPP_STORAGE_TEST_SIGNING_CONFORMANCE_FILENAME")
290 .value_or("");
291 if (conformance_tests_file.empty()) {
292 std::cerr
293 << "The GOOGLE_CLOUD_CPP_STORAGE_TEST_SIGNING_CONFORMANCE_FILENAME"
294 << " environment variable must be set and not empty.\n";
295 return 1;
296 }
297
298 std::ifstream ifstr(conformance_tests_file);
299 if (!ifstr.is_open()) {
300 std::cerr << "Failed to open data file: \"" << conformance_tests_file
301 << "\"\n";
302 return 1;
303 }
304 std::string json_rep(std::istreambuf_iterator<char>{ifstr}, {});
305 google::cloud::conformance::storage::v1::TestFile tests;
306 auto status = google::protobuf::util::JsonStringToMessage(json_rep, &tests);
307 if (!status.ok()) {
308 std::cerr << "Failed to parse conformance tests: " << status.ToString()
309 << "\n";
310 return 1;
311 }
312
313 auto signing_tests_destroyer =
314 absl::make_unique<std::map<std::string, SigningV4Test>>();
315 google::cloud::storage::signing_tests = signing_tests_destroyer.get();
316
317 // The implementation is not yet completed and these tests still fail, so skip
318 // them so far.
319 std::set<std::string> nonconformant_url_tests{"ListObjects"};
320
321 for (auto const& signing_test : tests.signing_v4_tests()) {
322 std::string name_with_spaces = signing_test.description();
323 std::string name;
324 // gtest doesn't allow for anything other than [a-zA-Z]
325 std::copy_if(name_with_spaces.begin(), name_with_spaces.end(),
326 back_inserter(name), [](char c) {
327 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
328 });
329 if (nonconformant_url_tests.find(name) != nonconformant_url_tests.end()) {
330 continue;
331 }
332 bool inserted =
333 google::cloud::storage::signing_tests->emplace(name, signing_test)
334 .second;
335 if (!inserted) {
336 std::cerr << "Duplicate test description: " << name << "\n";
337 }
338 }
339
340 auto post_policy_tests_destroyer =
341 absl::make_unique<std::map<std::string, PostPolicyV4Test>>();
342 google::cloud::storage::post_policy_tests = post_policy_tests_destroyer.get();
343
344 for (auto const& policy_test : tests.post_policy_v4_tests()) {
345 std::string name_with_spaces = policy_test.description();
346 std::string name;
347 // gtest doesn't allow for anything other than [a-zA-Z]
348 std::copy_if(name_with_spaces.begin(), name_with_spaces.end(),
349 back_inserter(name), [](char c) {
350 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
351 });
352 #if !GOOGLE_CLOUD_CPP_HAVE_EXCEPTIONS
353 if (name == "POSTPolicyCharacterEscaping" ||
354 name == "POSTPolicyWithAdditionalMetadata") {
355 // Escaping is not supported if exceptions are unavailable.
356 continue;
357 }
358 #endif // !GOOGLE_CLOUD_CPP_HAVE_EXCEPTIONS
359 bool inserted =
360 google::cloud::storage::post_policy_tests->emplace(name, policy_test)
361 .second;
362 if (!inserted) {
363 std::cerr << "Duplicate test description: " << name << "\n";
364 }
365 }
366
367 ::testing::InitGoogleMock(&argc, argv);
368
369 return RUN_ALL_TESTS();
370 }
371 #endif // GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
372