1 // Copyright 2017 Google Inc.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "google/cloud/bigtable/instance_admin.h"
16 #include "google/cloud/bigtable/testing/table_integration_test.h"
17 #include "google/cloud/internal/getenv.h"
18 #include "google/cloud/internal/random.h"
19 #include "google/cloud/testing_util/assert_ok.h"
20 #include "google/cloud/testing_util/chrono_literals.h"
21 #include "google/cloud/testing_util/contains_once.h"
22 #include "absl/memory/memory.h"
23 #include <gmock/gmock.h>
24 #include <string>
25 #include <vector>
26
27 namespace google {
28 namespace cloud {
29 namespace bigtable {
30 inline namespace BIGTABLE_CLIENT_NS {
31 namespace {
32
33 using ::google::cloud::testing_util::ContainsOnce;
34 using ::testing::Contains;
35 using ::testing::Not;
36 namespace btadmin = google::bigtable::admin::v2;
37
38 class AdminIntegrationTest : public bigtable::testing::TableIntegrationTest {
39 protected:
40 std::unique_ptr<bigtable::TableAdmin> table_admin_;
41
SetUp()42 void SetUp() override {
43 if (google::cloud::internal::GetEnv(
44 "ENABLE_BIGTABLE_ADMIN_INTEGRATION_TESTS")
45 .value_or("") != "yes") {
46 GTEST_SKIP();
47 }
48 TableIntegrationTest::SetUp();
49
50 std::shared_ptr<bigtable::AdminClient> admin_client =
51 bigtable::CreateDefaultAdminClient(
52 bigtable::testing::TableTestEnvironment::project_id(),
53 bigtable::ClientOptions());
54 table_admin_ = absl::make_unique<bigtable::TableAdmin>(
55 admin_client, bigtable::testing::TableTestEnvironment::instance_id());
56 }
57 };
58
TEST_F(AdminIntegrationTest,TableListWithMultipleTables)59 TEST_F(AdminIntegrationTest, TableListWithMultipleTables) {
60 std::vector<std::string> expected_table_list;
61 auto table_config = bigtable::TableConfig();
62
63 // Get the current list of tables.
64 auto previous_table_list =
65 table_admin_->ListTables(btadmin::Table::NAME_ONLY);
66 ASSERT_STATUS_OK(previous_table_list);
67
68 int constexpr kTableCount = 5;
69 for (int index = 0; index < kTableCount; ++index) {
70 std::string table_id = RandomTableId();
71 ASSERT_THAT(
72 TableNames(*previous_table_list),
73 Not(Contains(table_admin_->instance_name() + "/tables/" + table_id)))
74 << "Table (" << table_id << ") already exists."
75 << " This is unexpected, as the table ids are generated at random.";
76 EXPECT_STATUS_OK(table_admin_->CreateTable(table_id, table_config));
77
78 expected_table_list.emplace_back(table_id);
79 }
80 auto current_table_list = table_admin_->ListTables(btadmin::Table::NAME_ONLY);
81 ASSERT_STATUS_OK(current_table_list);
82 // Delete the tables so future tests have a clean slate.
83 for (auto const& table_id : expected_table_list) {
84 EXPECT_THAT(
85 TableNames(*current_table_list),
86 ContainsOnce(table_admin_->instance_name() + "/tables/" + table_id));
87 }
88 for (auto const& table_id : expected_table_list) {
89 EXPECT_STATUS_OK(table_admin_->DeleteTable(table_id));
90 }
91 current_table_list = table_admin_->ListTables(btadmin::Table::NAME_ONLY);
92 ASSERT_STATUS_OK(current_table_list);
93 // Delete the tables so future tests have a clean slate.
94 for (auto const& table_id : expected_table_list) {
95 EXPECT_THAT(
96 TableNames(*current_table_list),
97 Not(Contains(table_admin_->instance_name() + "/tables/" + table_id)));
98 }
99 }
100
TEST_F(AdminIntegrationTest,DropRowsByPrefix)101 TEST_F(AdminIntegrationTest, DropRowsByPrefix) {
102 auto table = GetTable();
103
104 // Create a vector of cell which will be inserted into bigtable
105 std::string const row_key1_prefix = "DropRowPrefix1";
106 std::string const row_key2_prefix = "DropRowPrefix2";
107 std::string const row_key1 = row_key1_prefix + "-Key1";
108 std::string const row_key1_1 = row_key1_prefix + "_1-Key1";
109 std::string const row_key2 = row_key2_prefix + "-Key2";
110 std::vector<bigtable::Cell> created_cells{
111 {row_key1, "family1", "column_id1", 0, "v-c-0-0"},
112 {row_key1, "family1", "column_id1", 1000, "v-c-0-1"},
113 {row_key1, "family2", "column_id3", 2000, "v-c-0-2"},
114 {row_key1_1, "family2", "column_id3", 2000, "v-c-0-2"},
115 {row_key1_1, "family2", "column_id3", 3000, "v-c-0-2"},
116 {row_key2, "family2", "column_id2", 2000, "v-c0-0-0"},
117 {row_key2, "family3", "column_id3", 3000, "v-c1-0-2"},
118 };
119 std::vector<bigtable::Cell> expected_cells{
120 {row_key2, "family2", "column_id2", 2000, "v-c0-0-0"},
121 {row_key2, "family3", "column_id3", 3000, "v-c1-0-2"}};
122
123 // Create records
124 CreateCells(table, created_cells);
125 // Delete all the records for a row
126 EXPECT_STATUS_OK(table_admin_->DropRowsByPrefix(
127 bigtable::testing::TableTestEnvironment::table_id(), row_key1_prefix));
128 auto actual_cells = ReadRows(table, bigtable::Filter::PassAllFilter());
129
130 CheckEqualUnordered(expected_cells, actual_cells);
131 }
132
TEST_F(AdminIntegrationTest,DropAllRows)133 TEST_F(AdminIntegrationTest, DropAllRows) {
134 auto table = GetTable();
135
136 // Create a vector of cell which will be inserted into bigtable
137 std::string const row_key1 = "DropRowKey1";
138 std::string const row_key2 = "DropRowKey2";
139 std::vector<bigtable::Cell> created_cells{
140 {row_key1, "family1", "column_id1", 0, "v-c-0-0"},
141 {row_key1, "family1", "column_id1", 1000, "v-c-0-1"},
142 {row_key1, "family2", "column_id3", 2000, "v-c-0-2"},
143 {row_key2, "family2", "column_id2", 2000, "v-c0-0-0"},
144 {row_key2, "family3", "column_id3", 3000, "v-c1-0-2"},
145 };
146
147 // Create records
148 CreateCells(table, created_cells);
149 // Delete all the records from a table
150 EXPECT_STATUS_OK(table_admin_->DropAllRows(
151 bigtable::testing::TableTestEnvironment::table_id()));
152 auto actual_cells = ReadRows(table, bigtable::Filter::PassAllFilter());
153
154 ASSERT_TRUE(actual_cells.empty());
155 }
156
157 /// @test Verify that `bigtable::TableAdmin` CRUD operations work as expected.
TEST_F(AdminIntegrationTest,CreateListGetDeleteTable)158 TEST_F(AdminIntegrationTest, CreateListGetDeleteTable) {
159 using GC = bigtable::GcRule;
160 std::string const table_id = RandomTableId();
161
162 // verify new table id in current table list
163 auto previous_table_list =
164 table_admin_->ListTables(btadmin::Table::NAME_ONLY);
165 ASSERT_STATUS_OK(previous_table_list);
166 ASSERT_THAT(
167 TableNames(*previous_table_list),
168 Not(Contains(table_admin_->instance_name() + "/tables/" + table_id)))
169 << "Table (" << table_id << ") already exists."
170 << " This is unexpected, as the table ids are generated at random.";
171
172 // create table config
173 bigtable::TableConfig table_config(
174 {{"fam", GC::MaxNumVersions(5)},
175 {"foo", GC::MaxAge(std::chrono::hours(24))}},
176 {"a1000", "a2000", "b3000", "m5000"});
177
178 // create table
179 ASSERT_STATUS_OK(table_admin_->CreateTable(table_id, table_config));
180 bigtable::Table table(data_client_, table_id);
181
182 // verify new table was created
183 auto table_result = table_admin_->GetTable(table_id);
184 ASSERT_STATUS_OK(table_result);
185 EXPECT_EQ(table.table_name(), table_result->name())
186 << "Mismatched names for GetTable(" << table_id
187 << "): " << table.table_name() << " != " << table_result->name();
188
189 // get table
190 auto table_detailed = table_admin_->GetTable(table_id, btadmin::Table::FULL);
191 ASSERT_STATUS_OK(table_detailed);
192 auto count_matching_families = [](btadmin::Table const& table,
193 std::string const& name) {
194 int count = 0;
195 for (auto const& kv : table.column_families()) {
196 if (kv.first == name) {
197 ++count;
198 }
199 }
200 return count;
201 };
202 EXPECT_EQ(1, count_matching_families(*table_detailed, "fam"));
203 EXPECT_EQ(1, count_matching_families(*table_detailed, "foo"));
204
205 // update table
206 std::vector<bigtable::ColumnFamilyModification> column_modification_list = {
207 bigtable::ColumnFamilyModification::Create(
208 "newfam", GC::Intersection(GC::MaxAge(std::chrono::hours(7 * 24)),
209 GC::MaxNumVersions(1))),
210 bigtable::ColumnFamilyModification::Update("fam", GC::MaxNumVersions(2)),
211 bigtable::ColumnFamilyModification::Drop("foo")};
212
213 auto table_modified =
214 table_admin_->ModifyColumnFamilies(table_id, column_modification_list);
215 ASSERT_STATUS_OK(table_modified);
216 EXPECT_EQ(1, count_matching_families(*table_modified, "fam"));
217 EXPECT_EQ(0, count_matching_families(*table_modified, "foo"));
218 EXPECT_EQ(1, count_matching_families(*table_modified, "newfam"));
219 auto const& gc = table_modified->column_families().at("newfam").gc_rule();
220 EXPECT_TRUE(gc.has_intersection());
221 EXPECT_EQ(2, gc.intersection().rules_size());
222
223 // delete table
224 EXPECT_STATUS_OK(table_admin_->DeleteTable(table_id));
225 // List to verify it is no longer there
226 auto current_table_list = table_admin_->ListTables(btadmin::Table::NAME_ONLY);
227 ASSERT_STATUS_OK(current_table_list);
228 EXPECT_THAT(
229 TableNames(*current_table_list),
230 Not(Contains(table_admin_->instance_name() + "/tables/" + table_id)));
231 }
232
233 /// @test Verify that `bigtable::TableAdmin` WaitForConsistencyCheck works as
234 /// expected.
TEST_F(AdminIntegrationTest,WaitForConsistencyCheck)235 TEST_F(AdminIntegrationTest, WaitForConsistencyCheck) {
236 // WaitForConsistencyCheck() only makes sense on a replicated table, we need
237 // to create an instance with at least 2 clusters to test it.
238 auto project_id = bigtable::testing::TableTestEnvironment::project_id();
239 std::string id = bigtable::testing::TableTestEnvironment::RandomInstanceId();
240 std::string const random_table_id = RandomTableId();
241
242 // Create a bigtable::InstanceAdmin and a bigtable::TableAdmin to create the
243 // new instance and the new table.
244 auto instance_admin_client = bigtable::CreateDefaultInstanceAdminClient(
245 project_id, bigtable::ClientOptions());
246 bigtable::InstanceAdmin instance_admin(instance_admin_client);
247
248 auto admin_client =
249 bigtable::CreateDefaultAdminClient(project_id, bigtable::ClientOptions());
250 bigtable::TableAdmin table_admin(admin_client, id);
251
252 // The instance configuration is involved, it needs two clusters, which must
253 // be production clusters (and therefore have at least 3 nodes each), and
254 // they must be in different zones. Also, the display name cannot be longer
255 // than 30 characters.
256 auto display_name = ("IT " + id).substr(0, 30);
257 auto cluster_config_1 =
258 bigtable::ClusterConfig(bigtable::testing::TableTestEnvironment::zone_a(),
259 3, bigtable::ClusterConfig::HDD);
260 auto cluster_config_2 =
261 bigtable::ClusterConfig(bigtable::testing::TableTestEnvironment::zone_b(),
262 3, bigtable::ClusterConfig::HDD);
263 bigtable::InstanceConfig config(
264 id, display_name,
265 {{id + "-c1", cluster_config_1}, {id + "-c2", cluster_config_2}});
266
267 // Create the new instance.
268 auto instance = instance_admin.CreateInstance(config).get();
269 ASSERT_STATUS_OK(instance);
270
271 // The table is going to be very simple, just one column family.
272 std::string const family = "column_family";
273 bigtable::TableConfig table_config = bigtable::TableConfig(
274 {{family, bigtable::GcRule::MaxNumVersions(10)}}, {});
275
276 // Create the new table.
277 auto table_created = table_admin.CreateTable(random_table_id, table_config);
278 ASSERT_STATUS_OK(table_created);
279
280 // We need to mutate the data in the table and then wait for those mutations
281 // to propagate to both clusters. First create a `bigtable::Table` object.
282 auto data_client = bigtable::CreateDefaultDataClient(
283 project_id, id, bigtable::ClientOptions());
284 bigtable::Table table(data_client, random_table_id);
285
286 // Insert some cells into the table.
287 std::string const row_key1 = "check-consistency-row1";
288 std::string const row_key2 = "check-consistency-row2";
289 std::vector<bigtable::Cell> created_cells{
290 {row_key1, family, "column1", 1000, "not interesting"},
291 {row_key1, family, "column2", 1000, "not interesting"},
292 {row_key1, family, "column1", 2000, "not interesting"},
293 {row_key2, family, "column2", 2000, "not interesting"},
294 {row_key2, family, "column1", 3000, "not interesting"},
295 };
296 CreateCells(table, created_cells);
297
298 // Create a consistency token after modifying the table.
299 auto consistency_token =
300 table_admin.GenerateConsistencyToken(random_table_id);
301 ASSERT_STATUS_OK(consistency_token);
302
303 // Wait until all the mutations before the `consistency_token` have propagated
304 // everywhere.
305 google::cloud::future<google::cloud::StatusOr<bigtable::Consistency>> result =
306 table_admin.WaitForConsistency(random_table_id, *consistency_token);
307 auto is_consistent = result.get();
308 ASSERT_STATUS_OK(is_consistent);
309 EXPECT_EQ(bigtable::Consistency::kConsistent, *is_consistent);
310
311 // Cleanup the table and the instance.
312 EXPECT_STATUS_OK(table_admin.DeleteTable(random_table_id));
313 EXPECT_STATUS_OK(instance_admin.DeleteInstance(id));
314 }
315
316 } // namespace
317 } // namespace BIGTABLE_CLIENT_NS
318 } // namespace bigtable
319 } // namespace cloud
320 } // namespace google
321
main(int argc,char * argv[])322 int main(int argc, char* argv[]) {
323 ::testing::InitGoogleMock(&argc, argv);
324 (void)::testing::AddGlobalTestEnvironment(
325 new google::cloud::bigtable::testing::TableTestEnvironment);
326 return RUN_ALL_TESTS();
327 }
328