1 // Copyright 2017 Google Inc.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "google/cloud/bigtable/instance_admin.h"
16 #include "google/cloud/bigtable/testing/table_integration_test.h"
17 #include "google/cloud/internal/getenv.h"
18 #include "google/cloud/internal/random.h"
19 #include "google/cloud/testing_util/assert_ok.h"
20 #include "google/cloud/testing_util/chrono_literals.h"
21 #include "google/cloud/testing_util/contains_once.h"
22 #include "absl/memory/memory.h"
23 #include <gmock/gmock.h>
24 #include <string>
25 #include <vector>
26 
27 namespace google {
28 namespace cloud {
29 namespace bigtable {
30 inline namespace BIGTABLE_CLIENT_NS {
31 namespace {
32 
33 using ::google::cloud::testing_util::ContainsOnce;
34 using ::testing::Contains;
35 using ::testing::Not;
36 namespace btadmin = google::bigtable::admin::v2;
37 
38 class AdminIntegrationTest : public bigtable::testing::TableIntegrationTest {
39  protected:
40   std::unique_ptr<bigtable::TableAdmin> table_admin_;
41 
SetUp()42   void SetUp() override {
43     if (google::cloud::internal::GetEnv(
44             "ENABLE_BIGTABLE_ADMIN_INTEGRATION_TESTS")
45             .value_or("") != "yes") {
46       GTEST_SKIP();
47     }
48     TableIntegrationTest::SetUp();
49 
50     std::shared_ptr<bigtable::AdminClient> admin_client =
51         bigtable::CreateDefaultAdminClient(
52             bigtable::testing::TableTestEnvironment::project_id(),
53             bigtable::ClientOptions());
54     table_admin_ = absl::make_unique<bigtable::TableAdmin>(
55         admin_client, bigtable::testing::TableTestEnvironment::instance_id());
56   }
57 };
58 
TEST_F(AdminIntegrationTest,TableListWithMultipleTables)59 TEST_F(AdminIntegrationTest, TableListWithMultipleTables) {
60   std::vector<std::string> expected_table_list;
61   auto table_config = bigtable::TableConfig();
62 
63   // Get the current list of tables.
64   auto previous_table_list =
65       table_admin_->ListTables(btadmin::Table::NAME_ONLY);
66   ASSERT_STATUS_OK(previous_table_list);
67 
68   int constexpr kTableCount = 5;
69   for (int index = 0; index < kTableCount; ++index) {
70     std::string table_id = RandomTableId();
71     ASSERT_THAT(
72         TableNames(*previous_table_list),
73         Not(Contains(table_admin_->instance_name() + "/tables/" + table_id)))
74         << "Table (" << table_id << ") already exists."
75         << " This is unexpected, as the table ids are generated at random.";
76     EXPECT_STATUS_OK(table_admin_->CreateTable(table_id, table_config));
77 
78     expected_table_list.emplace_back(table_id);
79   }
80   auto current_table_list = table_admin_->ListTables(btadmin::Table::NAME_ONLY);
81   ASSERT_STATUS_OK(current_table_list);
82   // Delete the tables so future tests have a clean slate.
83   for (auto const& table_id : expected_table_list) {
84     EXPECT_THAT(
85         TableNames(*current_table_list),
86         ContainsOnce(table_admin_->instance_name() + "/tables/" + table_id));
87   }
88   for (auto const& table_id : expected_table_list) {
89     EXPECT_STATUS_OK(table_admin_->DeleteTable(table_id));
90   }
91   current_table_list = table_admin_->ListTables(btadmin::Table::NAME_ONLY);
92   ASSERT_STATUS_OK(current_table_list);
93   // Delete the tables so future tests have a clean slate.
94   for (auto const& table_id : expected_table_list) {
95     EXPECT_THAT(
96         TableNames(*current_table_list),
97         Not(Contains(table_admin_->instance_name() + "/tables/" + table_id)));
98   }
99 }
100 
TEST_F(AdminIntegrationTest,DropRowsByPrefix)101 TEST_F(AdminIntegrationTest, DropRowsByPrefix) {
102   auto table = GetTable();
103 
104   // Create a vector of cell which will be inserted into bigtable
105   std::string const row_key1_prefix = "DropRowPrefix1";
106   std::string const row_key2_prefix = "DropRowPrefix2";
107   std::string const row_key1 = row_key1_prefix + "-Key1";
108   std::string const row_key1_1 = row_key1_prefix + "_1-Key1";
109   std::string const row_key2 = row_key2_prefix + "-Key2";
110   std::vector<bigtable::Cell> created_cells{
111       {row_key1, "family1", "column_id1", 0, "v-c-0-0"},
112       {row_key1, "family1", "column_id1", 1000, "v-c-0-1"},
113       {row_key1, "family2", "column_id3", 2000, "v-c-0-2"},
114       {row_key1_1, "family2", "column_id3", 2000, "v-c-0-2"},
115       {row_key1_1, "family2", "column_id3", 3000, "v-c-0-2"},
116       {row_key2, "family2", "column_id2", 2000, "v-c0-0-0"},
117       {row_key2, "family3", "column_id3", 3000, "v-c1-0-2"},
118   };
119   std::vector<bigtable::Cell> expected_cells{
120       {row_key2, "family2", "column_id2", 2000, "v-c0-0-0"},
121       {row_key2, "family3", "column_id3", 3000, "v-c1-0-2"}};
122 
123   // Create records
124   CreateCells(table, created_cells);
125   // Delete all the records for a row
126   EXPECT_STATUS_OK(table_admin_->DropRowsByPrefix(
127       bigtable::testing::TableTestEnvironment::table_id(), row_key1_prefix));
128   auto actual_cells = ReadRows(table, bigtable::Filter::PassAllFilter());
129 
130   CheckEqualUnordered(expected_cells, actual_cells);
131 }
132 
TEST_F(AdminIntegrationTest,DropAllRows)133 TEST_F(AdminIntegrationTest, DropAllRows) {
134   auto table = GetTable();
135 
136   // Create a vector of cell which will be inserted into bigtable
137   std::string const row_key1 = "DropRowKey1";
138   std::string const row_key2 = "DropRowKey2";
139   std::vector<bigtable::Cell> created_cells{
140       {row_key1, "family1", "column_id1", 0, "v-c-0-0"},
141       {row_key1, "family1", "column_id1", 1000, "v-c-0-1"},
142       {row_key1, "family2", "column_id3", 2000, "v-c-0-2"},
143       {row_key2, "family2", "column_id2", 2000, "v-c0-0-0"},
144       {row_key2, "family3", "column_id3", 3000, "v-c1-0-2"},
145   };
146 
147   // Create records
148   CreateCells(table, created_cells);
149   // Delete all the records from a table
150   EXPECT_STATUS_OK(table_admin_->DropAllRows(
151       bigtable::testing::TableTestEnvironment::table_id()));
152   auto actual_cells = ReadRows(table, bigtable::Filter::PassAllFilter());
153 
154   ASSERT_TRUE(actual_cells.empty());
155 }
156 
157 /// @test Verify that `bigtable::TableAdmin` CRUD operations work as expected.
TEST_F(AdminIntegrationTest,CreateListGetDeleteTable)158 TEST_F(AdminIntegrationTest, CreateListGetDeleteTable) {
159   using GC = bigtable::GcRule;
160   std::string const table_id = RandomTableId();
161 
162   // verify new table id in current table list
163   auto previous_table_list =
164       table_admin_->ListTables(btadmin::Table::NAME_ONLY);
165   ASSERT_STATUS_OK(previous_table_list);
166   ASSERT_THAT(
167       TableNames(*previous_table_list),
168       Not(Contains(table_admin_->instance_name() + "/tables/" + table_id)))
169       << "Table (" << table_id << ") already exists."
170       << " This is unexpected, as the table ids are generated at random.";
171 
172   // create table config
173   bigtable::TableConfig table_config(
174       {{"fam", GC::MaxNumVersions(5)},
175        {"foo", GC::MaxAge(std::chrono::hours(24))}},
176       {"a1000", "a2000", "b3000", "m5000"});
177 
178   // create table
179   ASSERT_STATUS_OK(table_admin_->CreateTable(table_id, table_config));
180   bigtable::Table table(data_client_, table_id);
181 
182   // verify new table was created
183   auto table_result = table_admin_->GetTable(table_id);
184   ASSERT_STATUS_OK(table_result);
185   EXPECT_EQ(table.table_name(), table_result->name())
186       << "Mismatched names for GetTable(" << table_id
187       << "): " << table.table_name() << " != " << table_result->name();
188 
189   // get table
190   auto table_detailed = table_admin_->GetTable(table_id, btadmin::Table::FULL);
191   ASSERT_STATUS_OK(table_detailed);
192   auto count_matching_families = [](btadmin::Table const& table,
193                                     std::string const& name) {
194     int count = 0;
195     for (auto const& kv : table.column_families()) {
196       if (kv.first == name) {
197         ++count;
198       }
199     }
200     return count;
201   };
202   EXPECT_EQ(1, count_matching_families(*table_detailed, "fam"));
203   EXPECT_EQ(1, count_matching_families(*table_detailed, "foo"));
204 
205   // update table
206   std::vector<bigtable::ColumnFamilyModification> column_modification_list = {
207       bigtable::ColumnFamilyModification::Create(
208           "newfam", GC::Intersection(GC::MaxAge(std::chrono::hours(7 * 24)),
209                                      GC::MaxNumVersions(1))),
210       bigtable::ColumnFamilyModification::Update("fam", GC::MaxNumVersions(2)),
211       bigtable::ColumnFamilyModification::Drop("foo")};
212 
213   auto table_modified =
214       table_admin_->ModifyColumnFamilies(table_id, column_modification_list);
215   ASSERT_STATUS_OK(table_modified);
216   EXPECT_EQ(1, count_matching_families(*table_modified, "fam"));
217   EXPECT_EQ(0, count_matching_families(*table_modified, "foo"));
218   EXPECT_EQ(1, count_matching_families(*table_modified, "newfam"));
219   auto const& gc = table_modified->column_families().at("newfam").gc_rule();
220   EXPECT_TRUE(gc.has_intersection());
221   EXPECT_EQ(2, gc.intersection().rules_size());
222 
223   // delete table
224   EXPECT_STATUS_OK(table_admin_->DeleteTable(table_id));
225   // List to verify it is no longer there
226   auto current_table_list = table_admin_->ListTables(btadmin::Table::NAME_ONLY);
227   ASSERT_STATUS_OK(current_table_list);
228   EXPECT_THAT(
229       TableNames(*current_table_list),
230       Not(Contains(table_admin_->instance_name() + "/tables/" + table_id)));
231 }
232 
233 /// @test Verify that `bigtable::TableAdmin` WaitForConsistencyCheck works as
234 /// expected.
TEST_F(AdminIntegrationTest,WaitForConsistencyCheck)235 TEST_F(AdminIntegrationTest, WaitForConsistencyCheck) {
236   // WaitForConsistencyCheck() only makes sense on a replicated table, we need
237   // to create an instance with at least 2 clusters to test it.
238   auto project_id = bigtable::testing::TableTestEnvironment::project_id();
239   std::string id = bigtable::testing::TableTestEnvironment::RandomInstanceId();
240   std::string const random_table_id = RandomTableId();
241 
242   // Create a bigtable::InstanceAdmin and a bigtable::TableAdmin to create the
243   // new instance and the new table.
244   auto instance_admin_client = bigtable::CreateDefaultInstanceAdminClient(
245       project_id, bigtable::ClientOptions());
246   bigtable::InstanceAdmin instance_admin(instance_admin_client);
247 
248   auto admin_client =
249       bigtable::CreateDefaultAdminClient(project_id, bigtable::ClientOptions());
250   bigtable::TableAdmin table_admin(admin_client, id);
251 
252   // The instance configuration is involved, it needs two clusters, which must
253   // be production clusters (and therefore have at least 3 nodes each), and
254   // they must be in different zones. Also, the display name cannot be longer
255   // than 30 characters.
256   auto display_name = ("IT " + id).substr(0, 30);
257   auto cluster_config_1 =
258       bigtable::ClusterConfig(bigtable::testing::TableTestEnvironment::zone_a(),
259                               3, bigtable::ClusterConfig::HDD);
260   auto cluster_config_2 =
261       bigtable::ClusterConfig(bigtable::testing::TableTestEnvironment::zone_b(),
262                               3, bigtable::ClusterConfig::HDD);
263   bigtable::InstanceConfig config(
264       id, display_name,
265       {{id + "-c1", cluster_config_1}, {id + "-c2", cluster_config_2}});
266 
267   // Create the new instance.
268   auto instance = instance_admin.CreateInstance(config).get();
269   ASSERT_STATUS_OK(instance);
270 
271   // The table is going to be very simple, just one column family.
272   std::string const family = "column_family";
273   bigtable::TableConfig table_config = bigtable::TableConfig(
274       {{family, bigtable::GcRule::MaxNumVersions(10)}}, {});
275 
276   // Create the new table.
277   auto table_created = table_admin.CreateTable(random_table_id, table_config);
278   ASSERT_STATUS_OK(table_created);
279 
280   // We need to mutate the data in the table and then wait for those mutations
281   // to propagate to both clusters. First create a `bigtable::Table` object.
282   auto data_client = bigtable::CreateDefaultDataClient(
283       project_id, id, bigtable::ClientOptions());
284   bigtable::Table table(data_client, random_table_id);
285 
286   // Insert some cells into the table.
287   std::string const row_key1 = "check-consistency-row1";
288   std::string const row_key2 = "check-consistency-row2";
289   std::vector<bigtable::Cell> created_cells{
290       {row_key1, family, "column1", 1000, "not interesting"},
291       {row_key1, family, "column2", 1000, "not interesting"},
292       {row_key1, family, "column1", 2000, "not interesting"},
293       {row_key2, family, "column2", 2000, "not interesting"},
294       {row_key2, family, "column1", 3000, "not interesting"},
295   };
296   CreateCells(table, created_cells);
297 
298   // Create a consistency token after modifying the table.
299   auto consistency_token =
300       table_admin.GenerateConsistencyToken(random_table_id);
301   ASSERT_STATUS_OK(consistency_token);
302 
303   // Wait until all the mutations before the `consistency_token` have propagated
304   // everywhere.
305   google::cloud::future<google::cloud::StatusOr<bigtable::Consistency>> result =
306       table_admin.WaitForConsistency(random_table_id, *consistency_token);
307   auto is_consistent = result.get();
308   ASSERT_STATUS_OK(is_consistent);
309   EXPECT_EQ(bigtable::Consistency::kConsistent, *is_consistent);
310 
311   // Cleanup the table and the instance.
312   EXPECT_STATUS_OK(table_admin.DeleteTable(random_table_id));
313   EXPECT_STATUS_OK(instance_admin.DeleteInstance(id));
314 }
315 
316 }  // namespace
317 }  // namespace BIGTABLE_CLIENT_NS
318 }  // namespace bigtable
319 }  // namespace cloud
320 }  // namespace google
321 
main(int argc,char * argv[])322 int main(int argc, char* argv[]) {
323   ::testing::InitGoogleMock(&argc, argv);
324   (void)::testing::AddGlobalTestEnvironment(
325       new google::cloud::bigtable::testing::TableTestEnvironment);
326   return RUN_ALL_TESTS();
327 }
328