1 // Copyright 2018 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "google/cloud/bigtable/instance_admin.h"
16 #include "google/cloud/bigtable/testing/table_integration_test.h"
17 #include "google/cloud/internal/getenv.h"
18 #include "google/cloud/internal/time_utils.h"
19 #include "google/cloud/testing_util/assert_ok.h"
20 #include "google/cloud/testing_util/chrono_literals.h"
21 #include <google/protobuf/util/time_util.h>
22 #include <gmock/gmock.h>
23
24 namespace google {
25 namespace cloud {
26 namespace bigtable {
27 inline namespace BIGTABLE_CLIENT_NS {
28 namespace {
29
30 using ::testing::Contains;
31 using ::testing::Not;
32 namespace btadmin = google::bigtable::admin::v2;
33 namespace bigtable = google::cloud::bigtable;
34
35 class AdminAsyncFutureIntegrationTest
36 : public bigtable::testing::TableIntegrationTest {
37 protected:
38 std::shared_ptr<AdminClient> admin_client_;
39 std::unique_ptr<TableAdmin> table_admin_;
40
SetUp()41 void SetUp() override {
42 if (google::cloud::internal::GetEnv(
43 "ENABLE_BIGTABLE_ADMIN_INTEGRATION_TESTS")
44 .value_or("") != "yes") {
45 GTEST_SKIP();
46 }
47
48 TableIntegrationTest::SetUp();
49 admin_client_ = CreateDefaultAdminClient(
50 testing::TableTestEnvironment::project_id(), ClientOptions());
51 table_admin_ = absl::make_unique<TableAdmin>(
52 admin_client_, bigtable::testing::TableTestEnvironment::instance_id());
53 }
54 };
55
56 /// @test Verify that `bigtable::TableAdmin` Async CRUD operations work as
57 /// expected.
TEST_F(AdminAsyncFutureIntegrationTest,CreateListGetDeleteTableTest)58 TEST_F(AdminAsyncFutureIntegrationTest, CreateListGetDeleteTableTest) {
59 // Currently this test uses mostly synchronous operations, as we implement
60 // async versions we should replace them in this function.
61
62 std::string const table_id = RandomTableId();
63
64 CompletionQueue cq;
65 std::thread pool([&cq] { cq.Run(); });
66
67 // AsyncCreateTable()
68 TableConfig table_config({{"fam", GcRule::MaxNumVersions(5)},
69 {"foo", GcRule::MaxAge(std::chrono::hours(24))}},
70 {"a1000", "a2000", "b3000", "m5000"});
71
72 auto count_matching_families = [](btadmin::Table const& table,
73 std::string const& name) {
74 int count = 0;
75 for (auto const& kv : table.column_families()) {
76 if (kv.first == name) {
77 ++count;
78 }
79 }
80 return count;
81 };
82
83 future<void> chain =
84 table_admin_->AsyncListTables(cq, btadmin::Table::NAME_ONLY)
85 .then([&](future<StatusOr<std::vector<btadmin::Table>>> fut) {
86 StatusOr<std::vector<btadmin::Table>> result = fut.get();
87 EXPECT_STATUS_OK(result);
88 EXPECT_THAT(TableNames(*result),
89 Not(Contains(table_admin_->instance_name() +
90 "/tables/" + table_id)))
91 << "Table (" << table_id << ") already exists."
92 << " This is unexpected, as the table ids are"
93 << " generated at random.";
94 return table_admin_->AsyncCreateTable(cq, table_id, table_config);
95 })
96 .then([&](future<StatusOr<btadmin::Table>> fut) {
97 StatusOr<btadmin::Table> result = fut.get();
98 EXPECT_STATUS_OK(result);
99 EXPECT_THAT(result->name(), ::testing::HasSubstr(table_id));
100 return table_admin_->AsyncGetTable(cq, table_id,
101 btadmin::Table::FULL);
102 })
103 .then([&](future<StatusOr<btadmin::Table>> fut) {
104 StatusOr<btadmin::Table> get_result = fut.get();
105 EXPECT_STATUS_OK(get_result);
106
107 EXPECT_EQ(1, count_matching_families(*get_result, "fam"));
108 EXPECT_EQ(1, count_matching_families(*get_result, "foo"));
109
110 // update table
111 std::vector<bigtable::ColumnFamilyModification>
112 column_modification_list = {
113 bigtable::ColumnFamilyModification::Create(
114 "newfam",
115 GcRule::Intersection(
116 GcRule::MaxAge(std::chrono::hours(7 * 24)),
117 GcRule::MaxNumVersions(1))),
118 bigtable::ColumnFamilyModification::Update(
119 "fam", GcRule::MaxNumVersions(2)),
120 bigtable::ColumnFamilyModification::Drop("foo")};
121 return table_admin_->AsyncModifyColumnFamilies(
122 cq, table_id, column_modification_list);
123 })
124 .then([&](future<StatusOr<btadmin::Table>> fut) {
125 StatusOr<btadmin::Table> get_result = fut.get();
126 EXPECT_EQ(1, count_matching_families(*get_result, "fam"));
127 EXPECT_EQ(0, count_matching_families(*get_result, "foo"));
128 EXPECT_EQ(1, count_matching_families(*get_result, "newfam"));
129 auto const& gc =
130 get_result->column_families().at("newfam").gc_rule();
131 EXPECT_TRUE(gc.has_intersection());
132 EXPECT_EQ(2, gc.intersection().rules_size());
133
134 return table_admin_->AsyncDeleteTable(cq, table_id);
135 })
136 .then([&](future<Status> fut) {
137 Status delete_result = fut.get();
138 EXPECT_STATUS_OK(delete_result);
139 return table_admin_->AsyncListTables(cq, btadmin::Table::NAME_ONLY);
140 })
141 .then([&](future<StatusOr<std::vector<btadmin::Table>>> fut) {
142 StatusOr<std::vector<btadmin::Table>> result = fut.get();
143 EXPECT_STATUS_OK(result);
144 ASSERT_THAT(TableNames(*result),
145 Not(Contains(table_admin_->instance_name() +
146 "/tables/" + table_id)))
147 << "Table (" << table_id << ") already exists."
148 << " This is unexpected, as the table ids are"
149 << " generated at random.";
150 });
151
152 chain.get();
153 SUCCEED(); // we expect that previous operations do not fail.
154
155 cq.Shutdown();
156 pool.join();
157 }
158
159 /// @test Verify that `bigtable::TableAdmin` AsyncDropRowsByPrefix works
TEST_F(AdminAsyncFutureIntegrationTest,AsyncDropRowsByPrefixTest)160 TEST_F(AdminAsyncFutureIntegrationTest, AsyncDropRowsByPrefixTest) {
161 auto table = GetTable();
162
163 CompletionQueue cq;
164 std::thread pool([&cq] { cq.Run(); });
165
166 // Create a vector of cell which will be inserted into bigtable
167 std::string const row_key1_prefix = "DropRowPrefix1";
168 std::string const row_key2_prefix = "DropRowPrefix2";
169 std::string const row_key1 = row_key1_prefix + "-Key1";
170 std::string const row_key1_1 = row_key1_prefix + "_1-Key1";
171 std::string const row_key2 = row_key2_prefix + "-Key2";
172 std::vector<bigtable::Cell> created_cells{
173 {row_key1, "family1", "column_id1", 0, "v-c-0-0"},
174 {row_key1, "family1", "column_id1", 1000, "v-c-0-1"},
175 {row_key1, "family2", "column_id3", 2000, "v-c-0-2"},
176 {row_key1_1, "family2", "column_id3", 2000, "v-c-0-2"},
177 {row_key1_1, "family2", "column_id3", 3000, "v-c-0-2"},
178 {row_key2, "family2", "column_id2", 2000, "v-c0-0-0"},
179 {row_key2, "family3", "column_id3", 3000, "v-c1-0-2"},
180 };
181 std::vector<bigtable::Cell> expected_cells{
182 {row_key2, "family2", "column_id2", 2000, "v-c0-0-0"},
183 {row_key2, "family3", "column_id3", 3000, "v-c1-0-2"}};
184
185 CreateCells(table, created_cells);
186
187 future<void> chain =
188 table_admin_
189 ->AsyncDropRowsByPrefix(
190 cq, bigtable::testing::TableTestEnvironment::table_id(),
191 row_key1_prefix)
192 .then([&](future<Status> fut) {
193 Status delete_result = fut.get();
194 EXPECT_STATUS_OK(delete_result);
195 auto actual_cells =
196 ReadRows(table, bigtable::Filter::PassAllFilter());
197 CheckEqualUnordered(expected_cells, actual_cells);
198 });
199
200 chain.get();
201 SUCCEED();
202 cq.Shutdown();
203 pool.join();
204 }
205
206 /// @test Verify that `bigtable::TableAdmin` AsyncDropAllRows works
TEST_F(AdminAsyncFutureIntegrationTest,AsyncDropAllRowsTest)207 TEST_F(AdminAsyncFutureIntegrationTest, AsyncDropAllRowsTest) {
208 auto table = GetTable();
209
210 CompletionQueue cq;
211 std::thread pool([&cq] { cq.Run(); });
212
213 // Create a vector of cell which will be inserted into bigtable
214 std::string const row_key1 = "DropRowKey1";
215 std::string const row_key2 = "DropRowKey2";
216 std::vector<bigtable::Cell> created_cells{
217 {row_key1, "family1", "column_id1", 0, "v-c-0-0"},
218 {row_key1, "family1", "column_id1", 1000, "v-c-0-1"},
219 {row_key1, "family2", "column_id3", 2000, "v-c-0-2"},
220 {row_key2, "family2", "column_id2", 2000, "v-c0-0-0"},
221 {row_key2, "family3", "column_id3", 3000, "v-c1-0-2"},
222 };
223
224 CreateCells(table, created_cells);
225
226 future<void> chain =
227 table_admin_
228 ->AsyncDropAllRows(
229 cq, bigtable::testing::TableTestEnvironment::table_id())
230 .then([&](future<Status> fut) {
231 Status delete_result = fut.get();
232 EXPECT_STATUS_OK(delete_result);
233 auto actual_cells =
234 ReadRows(table, bigtable::Filter::PassAllFilter());
235 ASSERT_TRUE(actual_cells.empty());
236 });
237
238 chain.get();
239 SUCCEED();
240 cq.Shutdown();
241 pool.join();
242 }
243
244 /// @test Verify that `bigtable::TableAdmin` AsyncCheckConsistency works as
245 /// expected.
TEST_F(AdminAsyncFutureIntegrationTest,AsyncCheckConsistencyIntegrationTest)246 TEST_F(AdminAsyncFutureIntegrationTest, AsyncCheckConsistencyIntegrationTest) {
247 std::string id = bigtable::testing::TableTestEnvironment::RandomInstanceId();
248 std::string const table_id = RandomTableId();
249
250 auto project_id = bigtable::testing::TableTestEnvironment::project_id();
251
252 auto instance_admin_client = bigtable::CreateDefaultInstanceAdminClient(
253 project_id, bigtable::ClientOptions());
254 bigtable::InstanceAdmin instance_admin(instance_admin_client);
255
256 // need to create table_admin for dynamically created instance
257 auto admin_client =
258 bigtable::CreateDefaultAdminClient(project_id, bigtable::ClientOptions());
259 bigtable::TableAdmin table_admin(admin_client, id);
260
261 auto data_client = bigtable::CreateDefaultDataClient(
262 project_id, id, bigtable::ClientOptions());
263 bigtable::Table table(data_client, table_id);
264
265 // Abbreviate "Integration Test" as "IT" because the display name cannot be
266 // longer than 30 characters.
267 auto display_name = ("IT " + id).substr(0, 30);
268
269 // Replication needs at least two clusters
270 auto cluster_config_1 =
271 bigtable::ClusterConfig(bigtable::testing::TableTestEnvironment::zone_a(),
272 3, bigtable::ClusterConfig::HDD);
273 auto cluster_config_2 =
274 bigtable::ClusterConfig(bigtable::testing::TableTestEnvironment::zone_b(),
275 3, bigtable::ClusterConfig::HDD);
276 bigtable::InstanceConfig config(
277 id, display_name,
278 {{id + "-c1", cluster_config_1}, {id + "-c2", cluster_config_2}});
279
280 std::string const column_family1 = "family1";
281 std::string const column_family2 = "family2";
282 std::string const column_family3 = "family3";
283 bigtable::TableConfig table_config = bigtable::TableConfig(
284 {{column_family1, bigtable::GcRule::MaxNumVersions(10)},
285 {column_family2, bigtable::GcRule::MaxNumVersions(10)},
286 {column_family3, bigtable::GcRule::MaxNumVersions(10)}},
287 {});
288
289 // Create a vector of cell which will be inserted into bigtable
290 std::string const row_key1 = "DropRowKey1";
291 std::string const row_key2 = "DropRowKey2";
292 std::vector<bigtable::Cell> created_cells{
293 {row_key1, column_family1, "column_id1", 1000, "v-c-0-0"},
294 {row_key1, column_family1, "column_id2", 1000, "v-c-0-1"},
295 {row_key1, column_family2, "column_id3", 2000, "v-c-0-2"},
296 {row_key2, column_family2, "column_id2", 2000, "v-c0-0-0"},
297 {row_key2, column_family3, "column_id3", 3000, "v-c1-0-2"},
298 };
299
300 CompletionQueue cq;
301 std::thread pool([&cq] { cq.Run(); });
302
303 future<Status> chain =
304 instance_admin.AsyncCreateInstance(cq, config)
305 .then([&](future<StatusOr<btadmin::Instance>> fut) {
306 StatusOr<btadmin::Instance> result = fut.get();
307 EXPECT_STATUS_OK(result);
308 if (!result) {
309 return make_ready_future(
310 StatusOr<btadmin::Table>(result.status()));
311 }
312 return table_admin.AsyncCreateTable(cq, table_id, table_config);
313 })
314 .then([&](future<StatusOr<btadmin::Table>> fut) {
315 StatusOr<btadmin::Table> result = fut.get();
316 EXPECT_STATUS_OK(result);
317 if (!result) {
318 return make_ready_future(StatusOr<std::string>(result.status()));
319 }
320 EXPECT_THAT(result->name(), ::testing::HasSubstr(table_id));
321 CreateCells(table, created_cells);
322 return table_admin.AsyncGenerateConsistencyToken(cq, table_id);
323 })
324 .then([&](future<StatusOr<std::string>> fut) {
325 auto token = fut.get();
326 EXPECT_STATUS_OK(token);
327 if (!token) {
328 return make_ready_future(StatusOr<Consistency>(token.status()));
329 }
330 return table_admin.AsyncWaitForConsistency(cq, table_id, *token);
331 })
332 .then([&](future<StatusOr<Consistency>> fut) {
333 auto result = fut.get();
334 EXPECT_STATUS_OK(result);
335 if (!result) {
336 return google::cloud::make_ready_future(result.status());
337 }
338 // If there is an error we cannot check the result, but
339 // we want to delete the table and continue.
340 EXPECT_EQ(*result, Consistency::kConsistent);
341 return table_admin.AsyncDeleteTable(cq, table_id);
342 })
343 .then([&](future<Status> fut) {
344 Status delete_result = fut.get();
345 EXPECT_STATUS_OK(delete_result);
346 return instance_admin.AsyncDeleteInstance(id, cq);
347 });
348
349 auto status = chain.get();
350 EXPECT_STATUS_OK(status);
351 cq.Shutdown();
352 pool.join();
353 }
354
355 } // namespace
356 } // namespace BIGTABLE_CLIENT_NS
357 } // namespace bigtable
358 } // namespace cloud
359 } // namespace google
360
main(int argc,char * argv[])361 int main(int argc, char* argv[]) {
362 ::testing::InitGoogleMock(&argc, argv);
363 (void)::testing::AddGlobalTestEnvironment(
364 new google::cloud::bigtable::testing::TableTestEnvironment);
365 return RUN_ALL_TESTS();
366 }
367