1 // Copyright 2018 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "google/cloud/bigtable/instance_admin.h"
16 #include "google/cloud/bigtable/testing/table_integration_test.h"
17 #include "google/cloud/internal/getenv.h"
18 #include "google/cloud/internal/time_utils.h"
19 #include "google/cloud/testing_util/assert_ok.h"
20 #include "google/cloud/testing_util/chrono_literals.h"
21 #include <google/protobuf/util/time_util.h>
22 #include <gmock/gmock.h>
23 
24 namespace google {
25 namespace cloud {
26 namespace bigtable {
27 inline namespace BIGTABLE_CLIENT_NS {
28 namespace {
29 
30 using ::testing::Contains;
31 using ::testing::Not;
32 namespace btadmin = google::bigtable::admin::v2;
33 namespace bigtable = google::cloud::bigtable;
34 
35 class AdminAsyncFutureIntegrationTest
36     : public bigtable::testing::TableIntegrationTest {
37  protected:
38   std::shared_ptr<AdminClient> admin_client_;
39   std::unique_ptr<TableAdmin> table_admin_;
40 
SetUp()41   void SetUp() override {
42     if (google::cloud::internal::GetEnv(
43             "ENABLE_BIGTABLE_ADMIN_INTEGRATION_TESTS")
44             .value_or("") != "yes") {
45       GTEST_SKIP();
46     }
47 
48     TableIntegrationTest::SetUp();
49     admin_client_ = CreateDefaultAdminClient(
50         testing::TableTestEnvironment::project_id(), ClientOptions());
51     table_admin_ = absl::make_unique<TableAdmin>(
52         admin_client_, bigtable::testing::TableTestEnvironment::instance_id());
53   }
54 };
55 
56 /// @test Verify that `bigtable::TableAdmin` Async CRUD operations work as
57 /// expected.
TEST_F(AdminAsyncFutureIntegrationTest,CreateListGetDeleteTableTest)58 TEST_F(AdminAsyncFutureIntegrationTest, CreateListGetDeleteTableTest) {
59   // Currently this test uses mostly synchronous operations, as we implement
60   // async versions we should replace them in this function.
61 
62   std::string const table_id = RandomTableId();
63 
64   CompletionQueue cq;
65   std::thread pool([&cq] { cq.Run(); });
66 
67   // AsyncCreateTable()
68   TableConfig table_config({{"fam", GcRule::MaxNumVersions(5)},
69                             {"foo", GcRule::MaxAge(std::chrono::hours(24))}},
70                            {"a1000", "a2000", "b3000", "m5000"});
71 
72   auto count_matching_families = [](btadmin::Table const& table,
73                                     std::string const& name) {
74     int count = 0;
75     for (auto const& kv : table.column_families()) {
76       if (kv.first == name) {
77         ++count;
78       }
79     }
80     return count;
81   };
82 
83   future<void> chain =
84       table_admin_->AsyncListTables(cq, btadmin::Table::NAME_ONLY)
85           .then([&](future<StatusOr<std::vector<btadmin::Table>>> fut) {
86             StatusOr<std::vector<btadmin::Table>> result = fut.get();
87             EXPECT_STATUS_OK(result);
88             EXPECT_THAT(TableNames(*result),
89                         Not(Contains(table_admin_->instance_name() +
90                                      "/tables/" + table_id)))
91                 << "Table (" << table_id << ") already exists."
92                 << " This is unexpected, as the table ids are"
93                 << " generated at random.";
94             return table_admin_->AsyncCreateTable(cq, table_id, table_config);
95           })
96           .then([&](future<StatusOr<btadmin::Table>> fut) {
97             StatusOr<btadmin::Table> result = fut.get();
98             EXPECT_STATUS_OK(result);
99             EXPECT_THAT(result->name(), ::testing::HasSubstr(table_id));
100             return table_admin_->AsyncGetTable(cq, table_id,
101                                                btadmin::Table::FULL);
102           })
103           .then([&](future<StatusOr<btadmin::Table>> fut) {
104             StatusOr<btadmin::Table> get_result = fut.get();
105             EXPECT_STATUS_OK(get_result);
106 
107             EXPECT_EQ(1, count_matching_families(*get_result, "fam"));
108             EXPECT_EQ(1, count_matching_families(*get_result, "foo"));
109 
110             // update table
111             std::vector<bigtable::ColumnFamilyModification>
112                 column_modification_list = {
113                     bigtable::ColumnFamilyModification::Create(
114                         "newfam",
115                         GcRule::Intersection(
116                             GcRule::MaxAge(std::chrono::hours(7 * 24)),
117                             GcRule::MaxNumVersions(1))),
118                     bigtable::ColumnFamilyModification::Update(
119                         "fam", GcRule::MaxNumVersions(2)),
120                     bigtable::ColumnFamilyModification::Drop("foo")};
121             return table_admin_->AsyncModifyColumnFamilies(
122                 cq, table_id, column_modification_list);
123           })
124           .then([&](future<StatusOr<btadmin::Table>> fut) {
125             StatusOr<btadmin::Table> get_result = fut.get();
126             EXPECT_EQ(1, count_matching_families(*get_result, "fam"));
127             EXPECT_EQ(0, count_matching_families(*get_result, "foo"));
128             EXPECT_EQ(1, count_matching_families(*get_result, "newfam"));
129             auto const& gc =
130                 get_result->column_families().at("newfam").gc_rule();
131             EXPECT_TRUE(gc.has_intersection());
132             EXPECT_EQ(2, gc.intersection().rules_size());
133 
134             return table_admin_->AsyncDeleteTable(cq, table_id);
135           })
136           .then([&](future<Status> fut) {
137             Status delete_result = fut.get();
138             EXPECT_STATUS_OK(delete_result);
139             return table_admin_->AsyncListTables(cq, btadmin::Table::NAME_ONLY);
140           })
141           .then([&](future<StatusOr<std::vector<btadmin::Table>>> fut) {
142             StatusOr<std::vector<btadmin::Table>> result = fut.get();
143             EXPECT_STATUS_OK(result);
144             ASSERT_THAT(TableNames(*result),
145                         Not(Contains(table_admin_->instance_name() +
146                                      "/tables/" + table_id)))
147                 << "Table (" << table_id << ") already exists."
148                 << " This is unexpected, as the table ids are"
149                 << " generated at random.";
150           });
151 
152   chain.get();
153   SUCCEED();  // we expect that previous operations do not fail.
154 
155   cq.Shutdown();
156   pool.join();
157 }
158 
159 /// @test Verify that `bigtable::TableAdmin` AsyncDropRowsByPrefix works
TEST_F(AdminAsyncFutureIntegrationTest,AsyncDropRowsByPrefixTest)160 TEST_F(AdminAsyncFutureIntegrationTest, AsyncDropRowsByPrefixTest) {
161   auto table = GetTable();
162 
163   CompletionQueue cq;
164   std::thread pool([&cq] { cq.Run(); });
165 
166   // Create a vector of cell which will be inserted into bigtable
167   std::string const row_key1_prefix = "DropRowPrefix1";
168   std::string const row_key2_prefix = "DropRowPrefix2";
169   std::string const row_key1 = row_key1_prefix + "-Key1";
170   std::string const row_key1_1 = row_key1_prefix + "_1-Key1";
171   std::string const row_key2 = row_key2_prefix + "-Key2";
172   std::vector<bigtable::Cell> created_cells{
173       {row_key1, "family1", "column_id1", 0, "v-c-0-0"},
174       {row_key1, "family1", "column_id1", 1000, "v-c-0-1"},
175       {row_key1, "family2", "column_id3", 2000, "v-c-0-2"},
176       {row_key1_1, "family2", "column_id3", 2000, "v-c-0-2"},
177       {row_key1_1, "family2", "column_id3", 3000, "v-c-0-2"},
178       {row_key2, "family2", "column_id2", 2000, "v-c0-0-0"},
179       {row_key2, "family3", "column_id3", 3000, "v-c1-0-2"},
180   };
181   std::vector<bigtable::Cell> expected_cells{
182       {row_key2, "family2", "column_id2", 2000, "v-c0-0-0"},
183       {row_key2, "family3", "column_id3", 3000, "v-c1-0-2"}};
184 
185   CreateCells(table, created_cells);
186 
187   future<void> chain =
188       table_admin_
189           ->AsyncDropRowsByPrefix(
190               cq, bigtable::testing::TableTestEnvironment::table_id(),
191               row_key1_prefix)
192           .then([&](future<Status> fut) {
193             Status delete_result = fut.get();
194             EXPECT_STATUS_OK(delete_result);
195             auto actual_cells =
196                 ReadRows(table, bigtable::Filter::PassAllFilter());
197             CheckEqualUnordered(expected_cells, actual_cells);
198           });
199 
200   chain.get();
201   SUCCEED();
202   cq.Shutdown();
203   pool.join();
204 }
205 
206 /// @test Verify that `bigtable::TableAdmin` AsyncDropAllRows works
TEST_F(AdminAsyncFutureIntegrationTest,AsyncDropAllRowsTest)207 TEST_F(AdminAsyncFutureIntegrationTest, AsyncDropAllRowsTest) {
208   auto table = GetTable();
209 
210   CompletionQueue cq;
211   std::thread pool([&cq] { cq.Run(); });
212 
213   // Create a vector of cell which will be inserted into bigtable
214   std::string const row_key1 = "DropRowKey1";
215   std::string const row_key2 = "DropRowKey2";
216   std::vector<bigtable::Cell> created_cells{
217       {row_key1, "family1", "column_id1", 0, "v-c-0-0"},
218       {row_key1, "family1", "column_id1", 1000, "v-c-0-1"},
219       {row_key1, "family2", "column_id3", 2000, "v-c-0-2"},
220       {row_key2, "family2", "column_id2", 2000, "v-c0-0-0"},
221       {row_key2, "family3", "column_id3", 3000, "v-c1-0-2"},
222   };
223 
224   CreateCells(table, created_cells);
225 
226   future<void> chain =
227       table_admin_
228           ->AsyncDropAllRows(
229               cq, bigtable::testing::TableTestEnvironment::table_id())
230           .then([&](future<Status> fut) {
231             Status delete_result = fut.get();
232             EXPECT_STATUS_OK(delete_result);
233             auto actual_cells =
234                 ReadRows(table, bigtable::Filter::PassAllFilter());
235             ASSERT_TRUE(actual_cells.empty());
236           });
237 
238   chain.get();
239   SUCCEED();
240   cq.Shutdown();
241   pool.join();
242 }
243 
244 /// @test Verify that `bigtable::TableAdmin` AsyncCheckConsistency works as
245 /// expected.
TEST_F(AdminAsyncFutureIntegrationTest,AsyncCheckConsistencyIntegrationTest)246 TEST_F(AdminAsyncFutureIntegrationTest, AsyncCheckConsistencyIntegrationTest) {
247   std::string id = bigtable::testing::TableTestEnvironment::RandomInstanceId();
248   std::string const table_id = RandomTableId();
249 
250   auto project_id = bigtable::testing::TableTestEnvironment::project_id();
251 
252   auto instance_admin_client = bigtable::CreateDefaultInstanceAdminClient(
253       project_id, bigtable::ClientOptions());
254   bigtable::InstanceAdmin instance_admin(instance_admin_client);
255 
256   // need to create table_admin for dynamically created instance
257   auto admin_client =
258       bigtable::CreateDefaultAdminClient(project_id, bigtable::ClientOptions());
259   bigtable::TableAdmin table_admin(admin_client, id);
260 
261   auto data_client = bigtable::CreateDefaultDataClient(
262       project_id, id, bigtable::ClientOptions());
263   bigtable::Table table(data_client, table_id);
264 
265   // Abbreviate "Integration Test" as "IT" because the display name cannot be
266   // longer than 30 characters.
267   auto display_name = ("IT " + id).substr(0, 30);
268 
269   // Replication needs at least two clusters
270   auto cluster_config_1 =
271       bigtable::ClusterConfig(bigtable::testing::TableTestEnvironment::zone_a(),
272                               3, bigtable::ClusterConfig::HDD);
273   auto cluster_config_2 =
274       bigtable::ClusterConfig(bigtable::testing::TableTestEnvironment::zone_b(),
275                               3, bigtable::ClusterConfig::HDD);
276   bigtable::InstanceConfig config(
277       id, display_name,
278       {{id + "-c1", cluster_config_1}, {id + "-c2", cluster_config_2}});
279 
280   std::string const column_family1 = "family1";
281   std::string const column_family2 = "family2";
282   std::string const column_family3 = "family3";
283   bigtable::TableConfig table_config = bigtable::TableConfig(
284       {{column_family1, bigtable::GcRule::MaxNumVersions(10)},
285        {column_family2, bigtable::GcRule::MaxNumVersions(10)},
286        {column_family3, bigtable::GcRule::MaxNumVersions(10)}},
287       {});
288 
289   // Create a vector of cell which will be inserted into bigtable
290   std::string const row_key1 = "DropRowKey1";
291   std::string const row_key2 = "DropRowKey2";
292   std::vector<bigtable::Cell> created_cells{
293       {row_key1, column_family1, "column_id1", 1000, "v-c-0-0"},
294       {row_key1, column_family1, "column_id2", 1000, "v-c-0-1"},
295       {row_key1, column_family2, "column_id3", 2000, "v-c-0-2"},
296       {row_key2, column_family2, "column_id2", 2000, "v-c0-0-0"},
297       {row_key2, column_family3, "column_id3", 3000, "v-c1-0-2"},
298   };
299 
300   CompletionQueue cq;
301   std::thread pool([&cq] { cq.Run(); });
302 
303   future<Status> chain =
304       instance_admin.AsyncCreateInstance(cq, config)
305           .then([&](future<StatusOr<btadmin::Instance>> fut) {
306             StatusOr<btadmin::Instance> result = fut.get();
307             EXPECT_STATUS_OK(result);
308             if (!result) {
309               return make_ready_future(
310                   StatusOr<btadmin::Table>(result.status()));
311             }
312             return table_admin.AsyncCreateTable(cq, table_id, table_config);
313           })
314           .then([&](future<StatusOr<btadmin::Table>> fut) {
315             StatusOr<btadmin::Table> result = fut.get();
316             EXPECT_STATUS_OK(result);
317             if (!result) {
318               return make_ready_future(StatusOr<std::string>(result.status()));
319             }
320             EXPECT_THAT(result->name(), ::testing::HasSubstr(table_id));
321             CreateCells(table, created_cells);
322             return table_admin.AsyncGenerateConsistencyToken(cq, table_id);
323           })
324           .then([&](future<StatusOr<std::string>> fut) {
325             auto token = fut.get();
326             EXPECT_STATUS_OK(token);
327             if (!token) {
328               return make_ready_future(StatusOr<Consistency>(token.status()));
329             }
330             return table_admin.AsyncWaitForConsistency(cq, table_id, *token);
331           })
332           .then([&](future<StatusOr<Consistency>> fut) {
333             auto result = fut.get();
334             EXPECT_STATUS_OK(result);
335             if (!result) {
336               return google::cloud::make_ready_future(result.status());
337             }
338             // If there is an error we cannot check the result, but
339             // we want to delete the table and continue.
340             EXPECT_EQ(*result, Consistency::kConsistent);
341             return table_admin.AsyncDeleteTable(cq, table_id);
342           })
343           .then([&](future<Status> fut) {
344             Status delete_result = fut.get();
345             EXPECT_STATUS_OK(delete_result);
346             return instance_admin.AsyncDeleteInstance(id, cq);
347           });
348 
349   auto status = chain.get();
350   EXPECT_STATUS_OK(status);
351   cq.Shutdown();
352   pool.join();
353 }
354 
355 }  // namespace
356 }  // namespace BIGTABLE_CLIENT_NS
357 }  // namespace bigtable
358 }  // namespace cloud
359 }  // namespace google
360 
main(int argc,char * argv[])361 int main(int argc, char* argv[]) {
362   ::testing::InitGoogleMock(&argc, argv);
363   (void)::testing::AddGlobalTestEnvironment(
364       new google::cloud::bigtable::testing::TableTestEnvironment);
365   return RUN_ALL_TESTS();
366 }
367