1 // Copyright 2019 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //     http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "google/cloud/storage/testing/storage_integration_test.h"
16 #if GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
17 #include "google/cloud/storage/client.h"
18 #include "google/cloud/storage/internal/openssl_util.h"
19 #include "google/cloud/storage/internal/signed_url_requests.h"
20 #include "google/cloud/storage/list_objects_reader.h"
21 #include "google/cloud/storage/tests/conformance_tests.pb.h"
22 #include "google/cloud/internal/format_time_point.h"
23 #include "google/cloud/internal/getenv.h"
24 #include "google/cloud/internal/time_utils.h"
25 #include "google/cloud/terminate_handler.h"
26 #include "google/cloud/testing_util/assert_ok.h"
27 #include "absl/memory/memory.h"
28 #include <google/protobuf/util/json_util.h>
29 #include <gmock/gmock.h>
30 #include <fstream>
31 #include <type_traits>
32 
33 /**
34  * @file
35  *
36  * Executes V4 signed URLs conformance tests described in an external file.
37  *
38  * We have a common set of conformance tests for V4 signed URLs used in all the
39  * GCS client libraries. The tests are stored in an external JSON file. This
40  * program receives the file name as an input parameter, loads it, and executes
41  * the tests described in the file.
42  *
43  * A separate command-line argument is the name of a (invalidated) service
44  * account key file used to create the signed URLs.
45  */
46 
47 namespace google {
48 namespace cloud {
49 namespace storage {
50 inline namespace STORAGE_CLIENT_NS {
51 namespace {
52 using google::cloud::conformance::storage::v1::PostPolicyV4Test;
53 using google::cloud::conformance::storage::v1::SigningV4Test;
54 using google::cloud::conformance::storage::v1::UrlStyle;
55 using ::testing::HasSubstr;
56 
57 // Initialized in main() below.
58 std::map<std::string, SigningV4Test>* signing_tests;
59 std::map<std::string, PostPolicyV4Test>* post_policy_tests;
60 
61 class V4SignedUrlConformanceTest
62     : public google::cloud::storage::testing::StorageIntegrationTest,
63       public ::testing::WithParamInterface<std::string> {
64  protected:
SetUp()65   void SetUp() override {
66     service_account_key_filename_ =
67         google::cloud::internal::GetEnv(
68             "GOOGLE_CLOUD_CPP_STORAGE_TEST_SIGNING_KEYFILE")
69             .value_or("");
70     ASSERT_FALSE(service_account_key_filename_.empty());
71   }
72 
73   std::string service_account_key_filename_;
74 };
75 
76 class V4PostPolicyConformanceTest : public V4SignedUrlConformanceTest {};
77 
TEST_P(V4SignedUrlConformanceTest,V4SignJson)78 TEST_P(V4SignedUrlConformanceTest, V4SignJson) {
79   auto creds = oauth2::CreateServiceAccountCredentialsFromJsonFilePath(
80       service_account_key_filename_);
81 
82   ASSERT_STATUS_OK(creds);
83   std::string account_email = creds->get()->AccountEmail();
84   Client client(*creds);
85   std::string actual_canonical_request;
86   std::string actual_string_to_sign;
87 
88   auto const& test_params = (*signing_tests)[GetParam()];
89   auto const& method_name = test_params.method();
90   auto const& bucket_name = test_params.bucket();
91   auto const& object_name = test_params.object();
92   auto const& scheme = test_params.scheme();
93   auto const& url_style = test_params.urlstyle();
94   auto const date =
95       ::google::cloud::internal::ToChronoTimePoint(test_params.timestamp());
96   auto const valid_for = std::chrono::seconds(test_params.expiration());
97   auto const& expected = test_params.expectedurl();
98   auto const& expected_canonical_request =
99       test_params.expectedcanonicalrequest();
100   auto const& expected_string_to_sign = test_params.expectedstringtosign();
101 
102   // Extract the headers for each object
103   auto headers = test_params.headers();
104   auto params = test_params.query_parameters();
105 
106   google::cloud::storage::internal::V4SignUrlRequest request(
107       method_name, bucket_name, object_name);
108   request.set_multiple_options(SignedUrlTimestamp(date),
109                                SignedUrlDuration(valid_for));
110 
111   std::vector<AddExtensionHeaderOption> header_extensions(5);
112   ASSERT_LE(headers.size(), header_extensions.size());
113   std::size_t idx = 0;
114   for (auto const& name_val : headers) {
115     request.set_multiple_options(
116         AddExtensionHeader(name_val.first, name_val.second));
117     header_extensions[idx++] =
118         AddExtensionHeader(name_val.first, name_val.second);
119   }
120 
121   std::vector<AddQueryParameterOption> query_params(5);
122   ASSERT_LE(params.size(), query_params.size());
123   idx = 0;
124   for (auto const& name_val : params) {
125     request.set_multiple_options(
126         AddQueryParameterOption(name_val.first, name_val.second));
127     query_params[idx++] =
128         AddQueryParameterOption(name_val.first, name_val.second);
129   }
130 
131   VirtualHostname virtual_hotname;
132   if (url_style == UrlStyle::VIRTUAL_HOSTED_STYLE) {
133     virtual_hotname = VirtualHostname(true);
134     request.set_multiple_options(VirtualHostname(true));
135   }
136 
137   BucketBoundHostname domain_named_bucket;
138   if (url_style == UrlStyle::BUCKET_BOUND_HOSTNAME) {
139     domain_named_bucket =
140         BucketBoundHostname(test_params.bucketboundhostname());
141     request.set_multiple_options(
142         BucketBoundHostname(test_params.bucketboundhostname()));
143   }
144 
145   auto actual = client.CreateV4SignedUrl(
146       method_name, bucket_name, object_name, SignedUrlTimestamp(date),
147       SignedUrlDuration(valid_for), header_extensions[0], header_extensions[1],
148       header_extensions[2], header_extensions[3], header_extensions[4],
149       query_params[0], query_params[1], query_params[2], query_params[3],
150       query_params[4], virtual_hotname, domain_named_bucket, Scheme(scheme));
151   ASSERT_STATUS_OK(request.Validate());
152   request.AddMissingRequiredHeaders();
153   ASSERT_STATUS_OK(request.Validate());
154 
155   actual_string_to_sign = request.StringToSign(account_email);
156   actual_canonical_request = request.CanonicalRequest(account_email);
157 
158   ASSERT_STATUS_OK(actual);
159   if (!domain_named_bucket.has_value()) {
160     EXPECT_THAT(*actual, HasSubstr(bucket_name));
161   }
162   EXPECT_EQ(expected, *actual);
163   EXPECT_EQ(expected_canonical_request, actual_canonical_request);
164   EXPECT_EQ(expected_string_to_sign, actual_string_to_sign);
165 }
166 
167 INSTANTIATE_TEST_SUITE_P(
168     V4SignedUrlConformanceTest, V4SignedUrlConformanceTest,
__anon028cd7b70202null169     ::testing::ValuesIn([] {
170       std::vector<std::string> res;
171       std::transform(signing_tests->begin(), signing_tests->end(),
172                      std::back_inserter(res),
173                      [](std::pair<std::string, SigningV4Test> const& p) {
174                        return p.first;
175                      });
176       return res;
177     }()));
178 
TEST_P(V4PostPolicyConformanceTest,V4PostPolicy)179 TEST_P(V4PostPolicyConformanceTest, V4PostPolicy) {
180   auto creds = oauth2::CreateServiceAccountCredentialsFromJsonFilePath(
181       service_account_key_filename_);
182 
183   ASSERT_STATUS_OK(creds);
184   std::string account_email = creds->get()->AccountEmail();
185   Client client(*creds);
186 
187   auto const& test_params = (*post_policy_tests)[GetParam()];
188   auto const& input = test_params.policyinput();
189   auto const& output = test_params.policyoutput();
190   auto const& bucket_name = input.bucket();
191   auto const& object_name = input.object();
192   auto const valid_for = std::chrono::seconds(input.expiration());
193   auto const timestamp =
194       ::google::cloud::internal::ToChronoTimePoint(input.timestamp());
195   auto const& scheme = input.scheme();
196   BucketBoundHostname domain_named_bucket;
197   auto const& url_style = input.urlstyle();
198   if (url_style == UrlStyle::BUCKET_BOUND_HOSTNAME) {
199     domain_named_bucket = BucketBoundHostname(input.bucketboundhostname());
200   }
201   VirtualHostname virtual_hotname;
202   if (url_style == UrlStyle::VIRTUAL_HOSTED_STYLE) {
203     virtual_hotname = VirtualHostname(true);
204   }
205 
206   std::vector<PolicyDocumentCondition> conditions;
207   auto const& condition = input.conditions();
208 
209   ASSERT_TRUE(condition.startswith().empty() ||
210               condition.startswith().size() == 2U)
211       << condition.startswith().size();
212   if (condition.startswith().size() == 2U) {
213     conditions.emplace_back(PolicyDocumentCondition::StartsWith(
214         condition.startswith()[0].substr(1), condition.startswith()[1]));
215   }
216   ASSERT_TRUE(condition.contentlengthrange().empty() ||
217               condition.contentlengthrange().size() == 2U)
218       << condition.contentlengthrange().size();
219   if (condition.contentlengthrange().size() == 2U) {
220     conditions.emplace_back(PolicyDocumentCondition::ContentLengthRange(
221         condition.contentlengthrange()[0], condition.contentlengthrange()[1]));
222   }
223 
224   auto const& expected_url = output.url();
225   auto const& fields = output.fields();
226   auto const& expected_algorithm = fields.at("x-goog-algorithm");
227   auto const& expected_credential = fields.at("x-goog-credential");
228   auto const& expected_date = fields.at("x-goog-date");
229   auto const& expected_signature = fields.at("x-goog-signature");
230   auto const& expected_policy = fields.at("policy");
231 
232   // We need to escape it because `nlohmann::json` interprets the escaped
233   // characters.
234   std::string const expected_decoded_policy =
235       *internal::PostPolicyV4Escape(output.expecteddecodedpolicy());
236 
237   std::vector<AddExtensionFieldOption> extension_fields(5);
238   ASSERT_LE(input.fields().size(), extension_fields.size());
239   std::size_t idx = 0;
240   for (auto const& name_val : input.fields()) {
241     extension_fields[idx++] =
242         AddExtensionField(name_val.first, name_val.second);
243   }
244   PolicyDocumentV4 doc{bucket_name, object_name, valid_for, timestamp,
245                        std::move(conditions)};
246   auto doc_res = client.GenerateSignedPostPolicyV4(
247       doc, extension_fields[0], extension_fields[1], extension_fields[2],
248       extension_fields[3], extension_fields[4], Scheme(scheme),
249       domain_named_bucket, virtual_hotname);
250   ASSERT_STATUS_OK(doc_res);
251   EXPECT_EQ(expected_policy, doc_res->policy);
252   auto actual_policy_vec = internal::Base64Decode(doc_res->policy);
253   std::string actual_policy(actual_policy_vec.begin(), actual_policy_vec.end());
254   EXPECT_EQ(expected_decoded_policy, actual_policy);
255   EXPECT_EQ(expected_url, doc_res->url);
256   EXPECT_EQ(expected_credential, doc_res->access_id);
257   EXPECT_EQ(expected_date, google::cloud::internal::FormatV4SignedUrlTimestamp(
258                                doc_res->expiration - valid_for));
259   EXPECT_EQ(expected_algorithm, doc_res->signing_algorithm);
260   EXPECT_EQ(expected_signature, doc_res->signature);
261   EXPECT_EQ((std::map<std::string, std::string>(fields.begin(), fields.end())),
262             doc_res->required_form_fields);
263 }
264 
265 INSTANTIATE_TEST_SUITE_P(
266     V4PostPolicyConformanceTest, V4PostPolicyConformanceTest,
__anon028cd7b70402null267     ::testing::ValuesIn([] {
268       std::vector<std::string> res;
269       std::transform(post_policy_tests->begin(), post_policy_tests->end(),
270                      std::back_inserter(res),
271                      [](std::pair<std::string, PostPolicyV4Test> const& p) {
272                        return p.first;
273                      });
274       return res;
275     }()));
276 
277 }  // namespace
278 }  // namespace STORAGE_CLIENT_NS
279 }  // namespace storage
280 }  // namespace cloud
281 }  // namespace google
282 
283 using google::cloud::conformance::storage::v1::PostPolicyV4Test;
284 using google::cloud::conformance::storage::v1::SigningV4Test;
285 
main(int argc,char * argv[])286 int main(int argc, char* argv[]) {  // NOLINT(bugprone-exception-escape)
287   auto conformance_tests_file =
288       google::cloud::internal::GetEnv(
289           "GOOGLE_CLOUD_CPP_STORAGE_TEST_SIGNING_CONFORMANCE_FILENAME")
290           .value_or("");
291   if (conformance_tests_file.empty()) {
292     std::cerr
293         << "The GOOGLE_CLOUD_CPP_STORAGE_TEST_SIGNING_CONFORMANCE_FILENAME"
294         << " environment variable must be set and not empty.\n";
295     return 1;
296   }
297 
298   std::ifstream ifstr(conformance_tests_file);
299   if (!ifstr.is_open()) {
300     std::cerr << "Failed to open data file: \"" << conformance_tests_file
301               << "\"\n";
302     return 1;
303   }
304   std::string json_rep(std::istreambuf_iterator<char>{ifstr}, {});
305   google::cloud::conformance::storage::v1::TestFile tests;
306   auto status = google::protobuf::util::JsonStringToMessage(json_rep, &tests);
307   if (!status.ok()) {
308     std::cerr << "Failed to parse conformance tests: " << status.ToString()
309               << "\n";
310     return 1;
311   }
312 
313   auto signing_tests_destroyer =
314       absl::make_unique<std::map<std::string, SigningV4Test>>();
315   google::cloud::storage::signing_tests = signing_tests_destroyer.get();
316 
317   // The implementation is not yet completed and these tests still fail, so skip
318   // them so far.
319   std::set<std::string> nonconformant_url_tests{"ListObjects"};
320 
321   for (auto const& signing_test : tests.signing_v4_tests()) {
322     std::string name_with_spaces = signing_test.description();
323     std::string name;
324     // gtest doesn't allow for anything other than [a-zA-Z]
325     std::copy_if(name_with_spaces.begin(), name_with_spaces.end(),
326                  back_inserter(name), [](char c) {
327                    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
328                  });
329     if (nonconformant_url_tests.find(name) != nonconformant_url_tests.end()) {
330       continue;
331     }
332     bool inserted =
333         google::cloud::storage::signing_tests->emplace(name, signing_test)
334             .second;
335     if (!inserted) {
336       std::cerr << "Duplicate test description: " << name << "\n";
337     }
338   }
339 
340   auto post_policy_tests_destroyer =
341       absl::make_unique<std::map<std::string, PostPolicyV4Test>>();
342   google::cloud::storage::post_policy_tests = post_policy_tests_destroyer.get();
343 
344   for (auto const& policy_test : tests.post_policy_v4_tests()) {
345     std::string name_with_spaces = policy_test.description();
346     std::string name;
347     // gtest doesn't allow for anything other than [a-zA-Z]
348     std::copy_if(name_with_spaces.begin(), name_with_spaces.end(),
349                  back_inserter(name), [](char c) {
350                    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
351                  });
352 #if !GOOGLE_CLOUD_CPP_HAVE_EXCEPTIONS
353     if (name == "POSTPolicyCharacterEscaping" ||
354         name == "POSTPolicyWithAdditionalMetadata") {
355       // Escaping is not supported if exceptions are unavailable.
356       continue;
357     }
358 #endif  // !GOOGLE_CLOUD_CPP_HAVE_EXCEPTIONS
359     bool inserted =
360         google::cloud::storage::post_policy_tests->emplace(name, policy_test)
361             .second;
362     if (!inserted) {
363       std::cerr << "Duplicate test description: " << name << "\n";
364     }
365   }
366 
367   ::testing::InitGoogleMock(&argc, argv);
368 
369   return RUN_ALL_TESTS();
370 }
371 #endif  // GOOGLE_CLOUD_CPP_STORAGE_HAVE_GRPC
372