1 /*
2 * backup.actor.cpp
3 *
4 * This source file is part of the FoundationDB open source project
5 *
6 * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 */
20
21 #define BOOST_DATE_TIME_NO_LIB
22 #include <boost/interprocess/managed_shared_memory.hpp>
23
24 #include "flow/flow.h"
25 #include "flow/FastAlloc.h"
26 #include "flow/serialize.h"
27 #include "flow/IRandom.h"
28 #include "flow/genericactors.actor.h"
29 #include "flow/SignalSafeUnwind.h"
30
31 #include "fdbclient/FDBTypes.h"
32 #include "fdbclient/BackupAgent.actor.h"
33 #include "fdbclient/Status.h"
34 #include "fdbclient/BackupContainer.h"
35 #include "fdbclient/KeyBackedTypes.h"
36 #include "fdbclient/RunTransaction.actor.h"
37 #include "fdbclient/BlobStore.h"
38 #include "fdbclient/json_spirit/json_spirit_writer_template.h"
39
40 #include "fdbrpc/Platform.h"
41 #include "fdbrpc/TLSConnection.h"
42
43 #include <stdarg.h>
44 #include <stdio.h>
45 #include <algorithm> // std::transform
46 #include <string>
47 #include <iostream>
48 #include <ctime>
49 using std::cout;
50 using std::endl;
51
52 #ifdef _WIN32
53 #define WIN32_LEAN_AND_MEAN
54 #define NOMINMAX
55 #include <Windows.h>
56 #endif
57 #include <time.h>
58
59 #ifdef __linux__
60 #include <execinfo.h>
61 #ifdef ALLOC_INSTRUMENTATION
62 #include <cxxabi.h>
63 #endif
64 #endif
65
66 #if defined(CMAKE_BUILD) || !defined(WIN32)
67 #include "versions.h"
68 #endif
69 #include "flow/SimpleOpt.h"
70 #include "flow/actorcompiler.h" // This must be the last #include.
71
72
73 // Type of program being executed
74 enum enumProgramExe {
75 EXE_AGENT, EXE_BACKUP, EXE_RESTORE, EXE_DR_AGENT, EXE_DB_BACKUP, EXE_UNDEFINED
76 };
77
78 enum enumBackupType {
79 BACKUP_UNDEFINED=0, BACKUP_START, BACKUP_MODIFY, BACKUP_STATUS, BACKUP_ABORT, BACKUP_WAIT, BACKUP_DISCONTINUE, BACKUP_PAUSE, BACKUP_RESUME, BACKUP_EXPIRE, BACKUP_DELETE, BACKUP_DESCRIBE, BACKUP_LIST, BACKUP_DUMP
80 };
81
82 enum enumDBType {
83 DB_UNDEFINED=0, DB_START, DB_STATUS, DB_SWITCH, DB_ABORT, DB_PAUSE, DB_RESUME
84 };
85
86 enum enumRestoreType {
87 RESTORE_UNKNOWN, RESTORE_START, RESTORE_STATUS, RESTORE_ABORT, RESTORE_WAIT
88 };
89
90 //
91 enum {
92 // Backup constants
93 OPT_DESTCONTAINER, OPT_SNAPSHOTINTERVAL, OPT_ERRORLIMIT, OPT_NOSTOPWHENDONE,
94 OPT_EXPIRE_BEFORE_VERSION, OPT_EXPIRE_BEFORE_DATETIME, OPT_EXPIRE_DELETE_BEFORE_DAYS,
95 OPT_EXPIRE_RESTORABLE_AFTER_VERSION, OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, OPT_EXPIRE_MIN_RESTORABLE_DAYS,
96 OPT_BASEURL, OPT_BLOB_CREDENTIALS, OPT_DESCRIBE_DEEP, OPT_DESCRIBE_TIMESTAMPS,
97 OPT_DUMP_BEGIN, OPT_DUMP_END, OPT_JSON,
98
99 // Backup and Restore constants
100 OPT_TAGNAME, OPT_BACKUPKEYS, OPT_WAITFORDONE,
101
102 // Backup Modify
103 OPT_MOD_ACTIVE_INTERVAL, OPT_MOD_VERIFY_UID,
104
105 // Restore constants
106 OPT_RESTORECONTAINER, OPT_RESTORE_VERSION, OPT_RESTORE_TIMESTAMP, OPT_PREFIX_ADD, OPT_PREFIX_REMOVE, OPT_RESTORE_CLUSTERFILE_DEST, OPT_RESTORE_CLUSTERFILE_ORIG,
107
108 // Shared constants
109 OPT_CLUSTERFILE, OPT_QUIET, OPT_DRYRUN, OPT_FORCE,
110 OPT_HELP, OPT_DEVHELP, OPT_VERSION, OPT_PARENTPID, OPT_CRASHONERROR,
111 OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR, OPT_TRACE, OPT_TRACE_DIR,
112 OPT_KNOB, OPT_TRACE_LOG_GROUP, OPT_MEMLIMIT, OPT_LOCALITY,
113
114 //DB constants
115 OPT_SOURCE_CLUSTER,
116 OPT_DEST_CLUSTER,
117 OPT_CLEANUP,
118
119 OPT_TRACE_FORMAT
120 };
121
122 CSimpleOpt::SOption g_rgAgentOptions[] = {
123 #ifdef _WIN32
124 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
125 #endif
126 { OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
127 { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP },
128 { OPT_KNOB, "--knob_", SO_REQ_SEP },
129 { OPT_VERSION, "--version", SO_NONE },
130 { OPT_VERSION, "-v", SO_NONE },
131 { OPT_QUIET, "-q", SO_NONE },
132 { OPT_QUIET, "--quiet", SO_NONE },
133 { OPT_TRACE, "--log", SO_NONE },
134 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
135 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
136 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
137 { OPT_CRASHONERROR, "--crash", SO_NONE },
138 { OPT_LOCALITY, "--locality_", SO_REQ_SEP },
139 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
140 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
141 { OPT_HELP, "-?", SO_NONE },
142 { OPT_HELP, "-h", SO_NONE },
143 { OPT_HELP, "--help", SO_NONE },
144 { OPT_DEVHELP, "--dev-help", SO_NONE },
145 { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
146 #ifndef TLS_DISABLED
147 TLS_OPTION_FLAGS
148 #endif
149 SO_END_OF_OPTIONS
150 };
151
152 CSimpleOpt::SOption g_rgBackupStartOptions[] = {
153 #ifdef _WIN32
154 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
155 #endif
156 { OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
157 { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP },
158 { OPT_WAITFORDONE, "-w", SO_NONE },
159 { OPT_WAITFORDONE, "--waitfordone", SO_NONE },
160 { OPT_NOSTOPWHENDONE, "-z", SO_NONE },
161 { OPT_NOSTOPWHENDONE, "--no-stop-when-done",SO_NONE },
162 { OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
163 { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
164 { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP },
165 { OPT_SNAPSHOTINTERVAL, "--snapshot_interval", SO_REQ_SEP },
166 { OPT_TAGNAME, "-t", SO_REQ_SEP },
167 { OPT_TAGNAME, "--tagname", SO_REQ_SEP },
168 { OPT_BACKUPKEYS, "-k", SO_REQ_SEP },
169 { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP },
170 { OPT_DRYRUN, "-n", SO_NONE },
171 { OPT_DRYRUN, "--dryrun", SO_NONE },
172 { OPT_TRACE, "--log", SO_NONE },
173 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
174 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
175 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
176 { OPT_QUIET, "-q", SO_NONE },
177 { OPT_QUIET, "--quiet", SO_NONE },
178 { OPT_VERSION, "--version", SO_NONE },
179 { OPT_VERSION, "-v", SO_NONE },
180 { OPT_CRASHONERROR, "--crash", SO_NONE },
181 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
182 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
183 { OPT_HELP, "-?", SO_NONE },
184 { OPT_HELP, "-h", SO_NONE },
185 { OPT_HELP, "--help", SO_NONE },
186 { OPT_DEVHELP, "--dev-help", SO_NONE },
187 { OPT_KNOB, "--knob_", SO_REQ_SEP },
188 { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
189 #ifndef TLS_DISABLED
190 TLS_OPTION_FLAGS
191 #endif
192 SO_END_OF_OPTIONS
193 };
194
195 CSimpleOpt::SOption g_rgBackupModifyOptions[] = {
196 #ifdef _WIN32
197 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
198 #endif
199 { OPT_TRACE, "--log", SO_NONE },
200 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
201 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
202 { OPT_QUIET, "-q", SO_NONE },
203 { OPT_QUIET, "--quiet", SO_NONE },
204 { OPT_VERSION, "-v", SO_NONE },
205 { OPT_VERSION, "--version", SO_NONE },
206 { OPT_CRASHONERROR, "--crash", SO_NONE },
207 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
208 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
209 { OPT_HELP, "-?", SO_NONE },
210 { OPT_HELP, "-h", SO_NONE },
211 { OPT_HELP, "--help", SO_NONE },
212 { OPT_DEVHELP, "--dev-help", SO_NONE },
213 { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
214 { OPT_KNOB, "--knob_", SO_REQ_SEP },
215 { OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
216 { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP },
217 { OPT_TAGNAME, "-t", SO_REQ_SEP },
218 { OPT_TAGNAME, "--tagname", SO_REQ_SEP },
219 { OPT_MOD_VERIFY_UID, "--verify_uid", SO_REQ_SEP },
220 { OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
221 { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
222 { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP },
223 { OPT_SNAPSHOTINTERVAL, "--snapshot_interval", SO_REQ_SEP },
224 { OPT_MOD_ACTIVE_INTERVAL, "--active_snapshot_interval", SO_REQ_SEP },
225
226 SO_END_OF_OPTIONS
227 };
228
229 CSimpleOpt::SOption g_rgBackupStatusOptions[] = {
230 #ifdef _WIN32
231 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
232 #endif
233 { OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
234 { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP },
235 { OPT_ERRORLIMIT, "-e", SO_REQ_SEP },
236 { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP },
237 { OPT_TAGNAME, "-t", SO_REQ_SEP },
238 { OPT_TAGNAME, "--tagname", SO_REQ_SEP },
239 { OPT_TRACE, "--log", SO_NONE },
240 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
241 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
242 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
243 { OPT_VERSION, "--version", SO_NONE },
244 { OPT_VERSION, "-v", SO_NONE },
245 { OPT_QUIET, "-q", SO_NONE },
246 { OPT_QUIET, "--quiet", SO_NONE },
247 { OPT_CRASHONERROR, "--crash", SO_NONE },
248 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
249 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
250 { OPT_HELP, "-?", SO_NONE },
251 { OPT_HELP, "-h", SO_NONE },
252 { OPT_HELP, "--help", SO_NONE },
253 { OPT_DEVHELP, "--dev-help", SO_NONE },
254 { OPT_JSON, "--json", SO_NONE},
255 #ifndef TLS_DISABLED
256 TLS_OPTION_FLAGS
257 #endif
258 SO_END_OF_OPTIONS
259 };
260
261 CSimpleOpt::SOption g_rgBackupAbortOptions[] = {
262 #ifdef _WIN32
263 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
264 #endif
265 { OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
266 { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP },
267 { OPT_TAGNAME, "-t", SO_REQ_SEP },
268 { OPT_TAGNAME, "--tagname", SO_REQ_SEP },
269 { OPT_TRACE, "--log", SO_NONE },
270 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
271 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
272 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
273 { OPT_QUIET, "-q", SO_NONE },
274 { OPT_QUIET, "--quiet", SO_NONE },
275 { OPT_VERSION, "--version", SO_NONE },
276 { OPT_VERSION, "-v", SO_NONE },
277 { OPT_CRASHONERROR, "--crash", SO_NONE },
278 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
279 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
280 { OPT_HELP, "-?", SO_NONE },
281 { OPT_HELP, "-h", SO_NONE },
282 { OPT_HELP, "--help", SO_NONE },
283 { OPT_DEVHELP, "--dev-help", SO_NONE },
284 #ifndef TLS_DISABLED
285 TLS_OPTION_FLAGS
286 #endif
287 SO_END_OF_OPTIONS
288 };
289
290 CSimpleOpt::SOption g_rgBackupDiscontinueOptions[] = {
291 #ifdef _WIN32
292 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
293 #endif
294 { OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
295 { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP },
296 { OPT_TAGNAME, "-t", SO_REQ_SEP },
297 { OPT_TAGNAME, "--tagname", SO_REQ_SEP },
298 { OPT_WAITFORDONE, "-w", SO_NONE },
299 { OPT_WAITFORDONE, "--waitfordone", SO_NONE },
300 { OPT_TRACE, "--log", SO_NONE },
301 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
302 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
303 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
304 { OPT_QUIET, "-q", SO_NONE },
305 { OPT_QUIET, "--quiet", SO_NONE },
306 { OPT_VERSION, "--version", SO_NONE },
307 { OPT_VERSION, "-v", SO_NONE },
308 { OPT_CRASHONERROR, "--crash", SO_NONE },
309 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
310 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
311 { OPT_HELP, "-?", SO_NONE },
312 { OPT_HELP, "-h", SO_NONE },
313 { OPT_HELP, "--help", SO_NONE },
314 { OPT_DEVHELP, "--dev-help", SO_NONE },
315 #ifndef TLS_DISABLED
316 TLS_OPTION_FLAGS
317 #endif
318 SO_END_OF_OPTIONS
319 };
320
321 CSimpleOpt::SOption g_rgBackupWaitOptions[] = {
322 #ifdef _WIN32
323 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
324 #endif
325 { OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
326 { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP },
327 { OPT_TAGNAME, "-t", SO_REQ_SEP },
328 { OPT_TAGNAME, "--tagname", SO_REQ_SEP },
329 { OPT_NOSTOPWHENDONE, "-z", SO_NONE },
330 { OPT_NOSTOPWHENDONE, "--no-stop-when-done",SO_NONE },
331 { OPT_TRACE, "--log", SO_NONE },
332 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
333 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
334 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
335 { OPT_QUIET, "-q", SO_NONE },
336 { OPT_QUIET, "--quiet", SO_NONE },
337 { OPT_VERSION, "--version", SO_NONE },
338 { OPT_VERSION, "-v", SO_NONE },
339 { OPT_CRASHONERROR, "--crash", SO_NONE },
340 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
341 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
342 { OPT_HELP, "-?", SO_NONE },
343 { OPT_HELP, "-h", SO_NONE },
344 { OPT_HELP, "--help", SO_NONE },
345 { OPT_DEVHELP, "--dev-help", SO_NONE },
346 #ifndef TLS_DISABLED
347 TLS_OPTION_FLAGS
348 #endif
349 SO_END_OF_OPTIONS
350 };
351
352 CSimpleOpt::SOption g_rgBackupPauseOptions[] = {
353 #ifdef _WIN32
354 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
355 #endif
356 { OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
357 { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP },
358 { OPT_TRACE, "--log", SO_NONE },
359 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
360 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
361 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
362 { OPT_QUIET, "-q", SO_NONE },
363 { OPT_QUIET, "--quiet", SO_NONE },
364 { OPT_VERSION, "--version", SO_NONE },
365 { OPT_VERSION, "-v", SO_NONE },
366 { OPT_CRASHONERROR, "--crash", SO_NONE },
367 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
368 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
369 { OPT_HELP, "-?", SO_NONE },
370 { OPT_HELP, "-h", SO_NONE },
371 { OPT_HELP, "--help", SO_NONE },
372 { OPT_DEVHELP, "--dev-help", SO_NONE },
373 #ifndef TLS_DISABLED
374 TLS_OPTION_FLAGS
375 #endif
376 SO_END_OF_OPTIONS
377 };
378
379 CSimpleOpt::SOption g_rgBackupExpireOptions[] = {
380 #ifdef _WIN32
381 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
382 #endif
383 { OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
384 { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP },
385 { OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
386 { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
387 { OPT_TRACE, "--log", SO_NONE },
388 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
389 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
390 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
391 { OPT_QUIET, "-q", SO_NONE },
392 { OPT_QUIET, "--quiet", SO_NONE },
393 { OPT_VERSION, "-v", SO_NONE },
394 { OPT_VERSION, "--version", SO_NONE },
395 { OPT_CRASHONERROR, "--crash", SO_NONE },
396 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
397 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
398 { OPT_HELP, "-?", SO_NONE },
399 { OPT_HELP, "-h", SO_NONE },
400 { OPT_HELP, "--help", SO_NONE },
401 { OPT_DEVHELP, "--dev-help", SO_NONE },
402 { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
403 { OPT_KNOB, "--knob_", SO_REQ_SEP },
404 { OPT_FORCE, "-f", SO_NONE },
405 { OPT_FORCE, "--force", SO_NONE },
406 { OPT_EXPIRE_RESTORABLE_AFTER_VERSION, "--restorable_after_version", SO_REQ_SEP },
407 { OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, "--restorable_after_timestamp", SO_REQ_SEP },
408 { OPT_EXPIRE_BEFORE_VERSION, "--expire_before_version", SO_REQ_SEP },
409 { OPT_EXPIRE_BEFORE_DATETIME, "--expire_before_timestamp", SO_REQ_SEP },
410 { OPT_EXPIRE_MIN_RESTORABLE_DAYS, "--min_restorable_days", SO_REQ_SEP },
411 { OPT_EXPIRE_DELETE_BEFORE_DAYS, "--delete_before_days", SO_REQ_SEP },
412 #ifndef TLS_DISABLED
413 TLS_OPTION_FLAGS
414 #endif
415 SO_END_OF_OPTIONS
416 };
417
418 CSimpleOpt::SOption g_rgBackupDeleteOptions[] = {
419 #ifdef _WIN32
420 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
421 #endif
422 { OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
423 { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
424 { OPT_TRACE, "--log", SO_NONE },
425 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
426 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
427 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
428 { OPT_QUIET, "-q", SO_NONE },
429 { OPT_QUIET, "--quiet", SO_NONE },
430 { OPT_VERSION, "-v", SO_NONE },
431 { OPT_VERSION, "--version", SO_NONE },
432 { OPT_CRASHONERROR, "--crash", SO_NONE },
433 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
434 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
435 { OPT_HELP, "-?", SO_NONE },
436 { OPT_HELP, "-h", SO_NONE },
437 { OPT_HELP, "--help", SO_NONE },
438 { OPT_DEVHELP, "--dev-help", SO_NONE },
439 { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
440 { OPT_KNOB, "--knob_", SO_REQ_SEP },
441 #ifndef TLS_DISABLED
442 TLS_OPTION_FLAGS
443 #endif
444 SO_END_OF_OPTIONS
445 };
446
447 CSimpleOpt::SOption g_rgBackupDescribeOptions[] = {
448 #ifdef _WIN32
449 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
450 #endif
451 { OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
452 { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP },
453 { OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
454 { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
455 { OPT_TRACE, "--log", SO_NONE },
456 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
457 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
458 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
459 { OPT_QUIET, "-q", SO_NONE },
460 { OPT_QUIET, "--quiet", SO_NONE },
461 { OPT_VERSION, "-v", SO_NONE },
462 { OPT_VERSION, "--version", SO_NONE },
463 { OPT_CRASHONERROR, "--crash", SO_NONE },
464 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
465 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
466 { OPT_HELP, "-?", SO_NONE },
467 { OPT_HELP, "-h", SO_NONE },
468 { OPT_HELP, "--help", SO_NONE },
469 { OPT_DEVHELP, "--dev-help", SO_NONE },
470 { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
471 { OPT_KNOB, "--knob_", SO_REQ_SEP },
472 { OPT_DESCRIBE_DEEP, "--deep", SO_NONE },
473 { OPT_DESCRIBE_TIMESTAMPS, "--version_timestamps", SO_NONE },
474 { OPT_JSON, "--json", SO_NONE},
475 #ifndef TLS_DISABLED
476 TLS_OPTION_FLAGS
477 #endif
478 SO_END_OF_OPTIONS
479 };
480
481 CSimpleOpt::SOption g_rgBackupDumpOptions[] = {
482 #ifdef _WIN32
483 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
484 #endif
485 { OPT_CLUSTERFILE, "-C", SO_REQ_SEP },
486 { OPT_CLUSTERFILE, "--cluster_file", SO_REQ_SEP },
487 { OPT_DESTCONTAINER, "-d", SO_REQ_SEP },
488 { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP },
489 { OPT_TRACE, "--log", SO_NONE },
490 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
491 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
492 { OPT_QUIET, "-q", SO_NONE },
493 { OPT_QUIET, "--quiet", SO_NONE },
494 { OPT_VERSION, "-v", SO_NONE },
495 { OPT_VERSION, "--version", SO_NONE },
496 { OPT_CRASHONERROR, "--crash", SO_NONE },
497 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
498 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
499 { OPT_HELP, "-?", SO_NONE },
500 { OPT_HELP, "-h", SO_NONE },
501 { OPT_HELP, "--help", SO_NONE },
502 { OPT_DEVHELP, "--dev-help", SO_NONE },
503 { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
504 { OPT_KNOB, "--knob_", SO_REQ_SEP },
505 { OPT_DUMP_BEGIN, "--begin", SO_REQ_SEP },
506 { OPT_DUMP_END, "--end", SO_REQ_SEP },
507 #ifndef TLS_DISABLED
508 TLS_OPTION_FLAGS
509 #endif
510 SO_END_OF_OPTIONS
511 };
512
513 CSimpleOpt::SOption g_rgBackupListOptions[] = {
514 #ifdef _WIN32
515 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
516 #endif
517 { OPT_BASEURL, "-b", SO_REQ_SEP },
518 { OPT_BASEURL, "--base_url", SO_REQ_SEP },
519 { OPT_TRACE, "--log", SO_NONE },
520 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
521 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
522 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
523 { OPT_QUIET, "-q", SO_NONE },
524 { OPT_QUIET, "--quiet", SO_NONE },
525 { OPT_VERSION, "-v", SO_NONE },
526 { OPT_VERSION, "--version", SO_NONE },
527 { OPT_CRASHONERROR, "--crash", SO_NONE },
528 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
529 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
530 { OPT_HELP, "-?", SO_NONE },
531 { OPT_HELP, "-h", SO_NONE },
532 { OPT_HELP, "--help", SO_NONE },
533 { OPT_DEVHELP, "--dev-help", SO_NONE },
534 { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
535 { OPT_KNOB, "--knob_", SO_REQ_SEP },
536 #ifndef TLS_DISABLED
537 TLS_OPTION_FLAGS
538 #endif
539 SO_END_OF_OPTIONS
540 };
541
542 CSimpleOpt::SOption g_rgRestoreOptions[] = {
543 #ifdef _WIN32
544 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
545 #endif
546 { OPT_RESTORE_CLUSTERFILE_DEST, "--dest_cluster_file", SO_REQ_SEP },
547 { OPT_RESTORE_CLUSTERFILE_ORIG, "--orig_cluster_file", SO_REQ_SEP },
548 { OPT_RESTORE_TIMESTAMP, "--timestamp", SO_REQ_SEP },
549 { OPT_KNOB, "--knob_", SO_REQ_SEP },
550 { OPT_RESTORECONTAINER,"-r", SO_REQ_SEP },
551 { OPT_PREFIX_ADD, "-add_prefix", SO_REQ_SEP },
552 { OPT_PREFIX_REMOVE, "-remove_prefix", SO_REQ_SEP },
553 { OPT_TAGNAME, "-t", SO_REQ_SEP },
554 { OPT_TAGNAME, "--tagname", SO_REQ_SEP },
555 { OPT_BACKUPKEYS, "-k", SO_REQ_SEP },
556 { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP },
557 { OPT_WAITFORDONE, "-w", SO_NONE },
558 { OPT_WAITFORDONE, "--waitfordone", SO_NONE },
559 { OPT_RESTORE_VERSION, "--version", SO_REQ_SEP },
560 { OPT_RESTORE_VERSION, "-v", SO_REQ_SEP },
561 { OPT_TRACE, "--log", SO_NONE },
562 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
563 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
564 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
565 { OPT_QUIET, "-q", SO_NONE },
566 { OPT_QUIET, "--quiet", SO_NONE },
567 { OPT_DRYRUN, "-n", SO_NONE },
568 { OPT_DRYRUN, "--dryrun", SO_NONE },
569 { OPT_FORCE, "-f", SO_NONE },
570 { OPT_CRASHONERROR, "--crash", SO_NONE },
571 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
572 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
573 { OPT_HELP, "-?", SO_NONE },
574 { OPT_HELP, "-h", SO_NONE },
575 { OPT_HELP, "--help", SO_NONE },
576 { OPT_DEVHELP, "--dev-help", SO_NONE },
577 { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP },
578 #ifndef TLS_DISABLED
579 TLS_OPTION_FLAGS
580 #endif
581 SO_END_OF_OPTIONS
582 };
583
584 CSimpleOpt::SOption g_rgDBAgentOptions[] = {
585 #ifdef _WIN32
586 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
587 #endif
588 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
589 { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP },
590 { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP },
591 { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP },
592 { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP },
593 { OPT_KNOB, "--knob_", SO_REQ_SEP },
594 { OPT_VERSION, "--version", SO_NONE },
595 { OPT_VERSION, "-v", SO_NONE },
596 { OPT_QUIET, "-q", SO_NONE },
597 { OPT_QUIET, "--quiet", SO_NONE },
598 { OPT_TRACE, "--log", SO_NONE },
599 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
600 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
601 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
602 { OPT_CRASHONERROR, "--crash", SO_NONE },
603 { OPT_LOCALITY, "--locality_", SO_REQ_SEP },
604 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
605 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
606 { OPT_HELP, "-?", SO_NONE },
607 { OPT_HELP, "-h", SO_NONE },
608 { OPT_HELP, "--help", SO_NONE },
609 { OPT_DEVHELP, "--dev-help", SO_NONE },
610 #ifndef TLS_DISABLED
611 TLS_OPTION_FLAGS
612 #endif
613 SO_END_OF_OPTIONS
614 };
615
616 CSimpleOpt::SOption g_rgDBStartOptions[] = {
617 #ifdef _WIN32
618 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
619 #endif
620 { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP },
621 { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP },
622 { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP },
623 { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP },
624 { OPT_TAGNAME, "-t", SO_REQ_SEP },
625 { OPT_TAGNAME, "--tagname", SO_REQ_SEP },
626 { OPT_BACKUPKEYS, "-k", SO_REQ_SEP },
627 { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP },
628 { OPT_TRACE, "--log", SO_NONE },
629 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
630 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
631 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
632 { OPT_QUIET, "-q", SO_NONE },
633 { OPT_QUIET, "--quiet", SO_NONE },
634 { OPT_VERSION, "--version", SO_NONE },
635 { OPT_VERSION, "-v", SO_NONE },
636 { OPT_CRASHONERROR, "--crash", SO_NONE },
637 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
638 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
639 { OPT_HELP, "-?", SO_NONE },
640 { OPT_HELP, "-h", SO_NONE },
641 { OPT_HELP, "--help", SO_NONE },
642 { OPT_DEVHELP, "--dev-help", SO_NONE },
643 #ifndef TLS_DISABLED
644 TLS_OPTION_FLAGS
645 #endif
646 SO_END_OF_OPTIONS
647 };
648
649 CSimpleOpt::SOption g_rgDBStatusOptions[] = {
650 #ifdef _WIN32
651 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
652 #endif
653 { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP },
654 { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP },
655 { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP },
656 { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP },
657 { OPT_ERRORLIMIT, "-e", SO_REQ_SEP },
658 { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP },
659 { OPT_TAGNAME, "-t", SO_REQ_SEP },
660 { OPT_TAGNAME, "--tagname", SO_REQ_SEP },
661 { OPT_TRACE, "--log", SO_NONE },
662 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
663 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
664 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
665 { OPT_VERSION, "--version", SO_NONE },
666 { OPT_VERSION, "-v", SO_NONE },
667 { OPT_QUIET, "-q", SO_NONE },
668 { OPT_QUIET, "--quiet", SO_NONE },
669 { OPT_CRASHONERROR, "--crash", SO_NONE },
670 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
671 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
672 { OPT_HELP, "-?", SO_NONE },
673 { OPT_HELP, "-h", SO_NONE },
674 { OPT_HELP, "--help", SO_NONE },
675 { OPT_DEVHELP, "--dev-help", SO_NONE },
676 #ifndef TLS_DISABLED
677 TLS_OPTION_FLAGS
678 #endif
679 SO_END_OF_OPTIONS
680 };
681
682 CSimpleOpt::SOption g_rgDBSwitchOptions[] = {
683 #ifdef _WIN32
684 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
685 #endif
686 { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP },
687 { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP },
688 { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP },
689 { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP },
690 { OPT_TAGNAME, "-t", SO_REQ_SEP },
691 { OPT_TAGNAME, "--tagname", SO_REQ_SEP },
692 { OPT_TRACE, "--log", SO_NONE },
693 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
694 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
695 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
696 { OPT_QUIET, "-q", SO_NONE },
697 { OPT_QUIET, "--quiet", SO_NONE },
698 { OPT_VERSION, "--version", SO_NONE },
699 { OPT_VERSION, "-v", SO_NONE },
700 { OPT_FORCE, "-f", SO_NONE },
701 { OPT_CRASHONERROR, "--crash", SO_NONE },
702 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
703 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
704 { OPT_HELP, "-?", SO_NONE },
705 { OPT_HELP, "-h", SO_NONE },
706 { OPT_HELP, "--help", SO_NONE },
707 { OPT_DEVHELP, "--dev-help", SO_NONE },
708 #ifndef TLS_DISABLED
709 TLS_OPTION_FLAGS
710 #endif
711 SO_END_OF_OPTIONS
712 };
713
714 CSimpleOpt::SOption g_rgDBAbortOptions[] = {
715 #ifdef _WIN32
716 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
717 #endif
718 { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP },
719 { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP },
720 { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP },
721 { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP },
722 { OPT_CLEANUP, "--cleanup", SO_NONE },
723 { OPT_TAGNAME, "-t", SO_REQ_SEP },
724 { OPT_TAGNAME, "--tagname", SO_REQ_SEP },
725 { OPT_TRACE, "--log", SO_NONE },
726 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
727 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
728 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
729 { OPT_QUIET, "-q", SO_NONE },
730 { OPT_QUIET, "--quiet", SO_NONE },
731 { OPT_VERSION, "--version", SO_NONE },
732 { OPT_VERSION, "-v", SO_NONE },
733 { OPT_CRASHONERROR, "--crash", SO_NONE },
734 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
735 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
736 { OPT_HELP, "-?", SO_NONE },
737 { OPT_HELP, "-h", SO_NONE },
738 { OPT_HELP, "--help", SO_NONE },
739 { OPT_DEVHELP, "--dev-help", SO_NONE },
740 #ifndef TLS_DISABLED
741 TLS_OPTION_FLAGS
742 #endif
743 SO_END_OF_OPTIONS
744 };
745
746 CSimpleOpt::SOption g_rgDBPauseOptions[] = {
747 #ifdef _WIN32
748 { OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
749 #endif
750 { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP },
751 { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP },
752 { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP },
753 { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP },
754 { OPT_TRACE, "--log", SO_NONE },
755 { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP },
756 { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
757 { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP },
758 { OPT_QUIET, "-q", SO_NONE },
759 { OPT_QUIET, "--quiet", SO_NONE },
760 { OPT_VERSION, "--version", SO_NONE },
761 { OPT_VERSION, "-v", SO_NONE },
762 { OPT_CRASHONERROR, "--crash", SO_NONE },
763 { OPT_MEMLIMIT, "-m", SO_REQ_SEP },
764 { OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
765 { OPT_HELP, "-?", SO_NONE },
766 { OPT_HELP, "-h", SO_NONE },
767 { OPT_HELP, "--help", SO_NONE },
768 { OPT_DEVHELP, "--dev-help", SO_NONE },
769 #ifndef TLS_DISABLED
770 TLS_OPTION_FLAGS
771 #endif
772 SO_END_OF_OPTIONS
773 };
774
775 const KeyRef exeAgent = LiteralStringRef("backup_agent");
776 const KeyRef exeBackup = LiteralStringRef("fdbbackup");
777 const KeyRef exeRestore = LiteralStringRef("fdbrestore");
778 const KeyRef exeDatabaseAgent = LiteralStringRef("dr_agent");
779 const KeyRef exeDatabaseBackup = LiteralStringRef("fdbdr");
780
781 extern void flushTraceFileVoid();
782 extern const char* getHGVersion();
783
784 #ifdef _WIN32
parentWatcher(void * parentHandle)785 void parentWatcher(void *parentHandle) {
786 HANDLE parent = (HANDLE)parentHandle;
787 int signal = WaitForSingleObject(parent, INFINITE);
788 CloseHandle(parentHandle);
789 if (signal == WAIT_OBJECT_0)
790 criticalError(FDB_EXIT_SUCCESS, "ParentProcessExited", "Parent process exited");
791 TraceEvent(SevError, "ParentProcessWaitFailed").detail("RetCode", signal).GetLastError();
792 }
793
794 #endif
795
printVersion()796 static void printVersion() {
797 printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
798 printf("source version %s\n", getHGVersion());
799 printf("protocol %llx\n", (long long) currentProtocolVersion);
800 }
801
802 const char *BlobCredentialInfo =
803 " BLOB CREDENTIALS\n"
804 " Blob account secret keys can optionally be omitted from blobstore:// URLs, in which case they will be\n"
805 " loaded, if possible, from 1 or more blob credentials definition files.\n\n"
806 " These files can be specified with the --blob_credentials argument described above or via the environment variable\n"
807 " FDB_BLOB_CREDENTIALS, whose value is a colon-separated list of files. The command line takes priority over\n"
808 " over the environment but all files from both sources are used.\n\n"
809 " At connect time, the specified files are read in order and the first matching account specification (user@host)\n"
810 " will be used to obtain the secret key.\n\n"
811 " The JSON schema is:\n"
812 " { \"accounts\" : { \"user@host\" : { \"secret\" : \"SECRETKEY\" }, \"user2@host2\" : { \"secret\" : \"SECRET\" } } }\n";
813
printHelpTeaser(const char * name)814 static void printHelpTeaser( const char *name ) {
815 fprintf(stderr, "Try `%s --help' for more information.\n", name);
816 }
817
printAgentUsage(bool devhelp)818 static void printAgentUsage(bool devhelp) {
819 printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
820 printf("Usage: %s [OPTIONS]\n\n", exeAgent.toString().c_str());
821 printf(" -C CONNFILE The path of a file containing the connection string for the\n"
822 " FoundationDB cluster. The default is first the value of the\n"
823 " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n"
824 " then `%s'.\n", platform::getDefaultClusterFilePath().c_str());
825 printf(" --log Enables trace file logging for the CLI session.\n"
826 " --logdir PATH Specifes the output directory for trace files. If\n"
827 " unspecified, defaults to the current directory. Has\n"
828 " no effect unless --log is specified.\n");
829 printf(" --trace_format FORMAT\n"
830 " Select the format of the trace files. xml (the default) and json are supported.\n"
831 " Has no effect unless --log is specified.\n");
832 printf(" -m SIZE, --memory SIZE\n"
833 " Memory limit. The default value is 8GiB. When specified\n"
834 " without a unit, MiB is assumed.\n");
835 #ifndef TLS_DISABLED
836 printf(TLS_HELP);
837 #endif
838 printf(" -v, --version Print version information and exit.\n");
839 printf(" -h, --help Display this help and exit.\n");
840
841 if (devhelp) {
842 #ifdef _WIN32
843 printf(" -n Create a new console.\n");
844 printf(" -q Disable error dialog on crash.\n");
845 printf(" --parentpid PID\n");
846 printf(" Specify a process after whose termination to exit.\n");
847 #endif
848 }
849
850 printf("\n");
851 puts(BlobCredentialInfo);
852
853 return;
854 }
855
printBackupContainerInfo()856 void printBackupContainerInfo() {
857 printf(" Backup URL forms:\n\n");
858 std::vector<std::string> formats = IBackupContainer::getURLFormats();
859 for(auto &f : formats)
860 printf(" %s\n", f.c_str());
861 printf("\n");
862 }
863
printBackupUsage(bool devhelp)864 static void printBackupUsage(bool devhelp) {
865 printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
866 printf("Usage: %s (start | status | abort | wait | discontinue | pause | resume | expire | delete | describe | list) [OPTIONS]\n\n", exeBackup.toString().c_str());
867 printf(" -C CONNFILE The path of a file containing the connection string for the\n"
868 " FoundationDB cluster. The default is first the value of the\n"
869 " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n"
870 " then `%s'.\n", platform::getDefaultClusterFilePath().c_str());
871 printf(" -d, --destcontainer URL\n"
872 " The Backup container URL for start, modify, describe, expire, and delete operations.\n");
873 printBackupContainerInfo();
874 printf(" -b, --base_url BASEURL\n"
875 " Base backup URL for list operations. This looks like a Backup URL but without a backup name.\n");
876 printf(" --blob_credentials FILE\n"
877 " File containing blob credentials in JSON format. Can be specified multiple times for multiple files. See below for more details.\n");
878 printf(" --expire_before_timestamp DATETIME\n"
879 " Datetime cutoff for expire operations. Requires a cluster file and will use version/timestamp metadata\n"
880 " in the database to obtain a cutoff version very close to the timestamp given in %s.\n", BackupAgentBase::timeFormat().c_str());
881 printf(" --expire_before_version VERSION\n"
882 " Version cutoff for expire operations. Deletes data files containing no data at or after VERSION.\n");
883 printf(" --delete_before_days NUM_DAYS\n"
884 " Another way to specify version cutoff for expire operations. Deletes data files containing no data at or after a\n"
885 " version approximately NUM_DAYS days worth of versions prior to the latest log version in the backup.\n");
886 printf(" --restorable_after_timestamp DATETIME\n"
887 " For expire operations, set minimum acceptable restorability to the version equivalent of DATETIME and later.\n");
888 printf(" --restorable_after_version VERSION\n"
889 " For expire operations, set minimum acceptable restorability to the VERSION and later.\n");
890 printf(" --min_restorable_days NUM_DAYS\n"
891 " For expire operations, set minimum acceptable restorability to approximately NUM_DAYS days worth of versions\n"
892 " prior to the latest log version in the backup.\n");
893 printf(" --version_timestamps\n");
894 printf(" For describe operations, lookup versions in the database to obtain timestamps. A cluster file is required.\n");
895 printf(" -f, --force For expire operations, force expiration even if minimum restorability would be violated.\n");
896 printf(" -s, --snapshot_interval DURATION\n"
897 " For start or modify operations, specifies the backup's default target snapshot interval as DURATION seconds. Defaults to %d for start operations.\n", CLIENT_KNOBS->BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC);
898 printf(" --active_snapshot_interval DURATION\n"
899 " For modify operations, sets the desired interval for the backup's currently active snapshot, relative to the start of the snapshot.\n");
900 printf(" --verify_uid UID\n"
901 " Specifies a UID to verify against the BackupUID of the running backup. If provided, the UID is verified in the same transaction\n"
902 " which sets the new backup parameters (if the UID matches).\n");
903 printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n");
904 printf(" -k KEYS List of key ranges to backup.\n"
905 " If not specified, the entire database will be backed up.\n");
906 printf(" -n, --dryrun For start or restore operations, performs a trial run with no actual changes made.\n");
907 #ifndef TLS_DISABLED
908 printf(TLS_HELP);
909 #endif
910 printf(" -v, --version Print version information and exit.\n");
911 printf(" -w, --wait Wait for the backup to complete (allowed with `start' and `discontinue').\n");
912 printf(" -z, --no-stop-when-done\n"
913 " Do not stop backup when restorable.\n");
914 printf(" -h, --help Display this help and exit.\n");
915
916 if (devhelp) {
917 #ifdef _WIN32
918 printf(" -n Create a new console.\n");
919 printf(" -q Disable error dialog on crash.\n");
920 printf(" --parentpid PID\n");
921 printf(" Specify a process after whose termination to exit.\n");
922 #endif
923 printf(" --deep For describe operations, do not use cached metadata. Warning: Very slow\n");
924
925 }
926 printf("\n"
927 " KEYS FORMAT: \"<BEGINKEY> <ENDKEY>\" [...]\n");
928 printf("\n");
929 puts(BlobCredentialInfo);
930
931 return;
932 }
933
printRestoreUsage(bool devhelp)934 static void printRestoreUsage(bool devhelp ) {
935 printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
936 printf("Usage: %s (start | status | abort | wait) [OPTIONS]\n\n", exeRestore.toString().c_str());
937 //printf(" FOLDERS Paths to folders containing the backup files.\n");
938 printf("Options for all commands:\n\n");
939 printf(" --dest_cluster_file CONNFILE\n");
940 printf(" The cluster file to restore data into.\n");
941 printf(" -t, --tagname TAGNAME\n");
942 printf(" The restore tag to act on. Default is 'default'\n");
943 printf("Options for start:\n\n");
944 printf(" -r URL The Backup URL for the restore to read from.\n");
945 printBackupContainerInfo();
946 printf(" -w, --waitfordone\n");
947 printf(" Wait for the restore to complete before exiting. Prints progress updates.\n");
948 printf(" -k KEYS List of key ranges from the backup to restore.\n");
949 printf(" --remove_prefix PREFIX\n");
950 printf(" Prefix to remove from the restored keys.\n");
951 printf(" --add_prefix PREFIX\n");
952 printf(" Prefix to add to the restored keys\n");
953 printf(" -n, --dryrun Perform a trial run with no changes made.\n");
954 #ifndef TLS_DISABLED
955 printf(TLS_HELP);
956 #endif
957 printf(" -v DBVERSION The version at which the database will be restored.\n");
958 printf(" --timestamp Instead of a numeric version, use this to specify a timestamp in %s\n", BackupAgentBase::timeFormat().c_str());
959 printf(" and it will be converted to a version from that time using metadata in orig_cluster_file.\n");
960 printf(" --orig_cluster_file CONNFILE\n");
961 printf(" The cluster file for the original database from which the backup was created. The original database\n");
962 printf(" is only needed to convert a --timestamp argument to a database version.\n");
963 printf(" -h, --help Display this help and exit.\n");
964
965 if( devhelp ) {
966 #ifdef _WIN32
967 printf(" -q Disable error dialog on crash.\n");
968 printf(" --parentpid PID\n");
969 printf(" Specify a process after whose termination to exit.\n");
970 #endif
971 }
972
973 printf("\n"
974 " KEYS FORMAT: \"<BEGINKEY> <ENDKEY>\" [...]\n");
975 printf("\n");
976 puts(BlobCredentialInfo);
977
978 return;
979 }
980
printDBAgentUsage(bool devhelp)981 static void printDBAgentUsage(bool devhelp) {
982 printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
983 printf("Usage: %s [OPTIONS]\n\n", exeDatabaseAgent.toString().c_str());
984 printf(" -d CONNFILE The path of a file containing the connection string for the\n"
985 " destination FoundationDB cluster.\n");
986 printf(" -s CONNFILE The path of a file containing the connection string for the\n"
987 " source FoundationDB cluster.\n");
988 printf(" --log Enables trace file logging for the CLI session.\n"
989 " --logdir PATH Specifes the output directory for trace files. If\n"
990 " unspecified, defaults to the current directory. Has\n"
991 " no effect unless --log is specified.\n");
992 printf(" --trace_format FORMAT\n"
993 " Select the format of the trace files. xml (the default) and json are supported.\n"
994 " Has no effect unless --log is specified.\n");
995 printf(" -m SIZE, --memory SIZE\n"
996 " Memory limit. The default value is 8GiB. When specified\n"
997 " without a unit, MiB is assumed.\n");
998 #ifndef TLS_DISABLED
999 printf(TLS_HELP);
1000 #endif
1001 printf(" -v, --version Print version information and exit.\n");
1002 printf(" -h, --help Display this help and exit.\n");
1003 if (devhelp) {
1004 #ifdef _WIN32
1005 printf(" -n Create a new console.\n");
1006 printf(" -q Disable error dialog on crash.\n");
1007 printf(" --parentpid PID\n");
1008 printf(" Specify a process after whose termination to exit.\n");
1009 #endif
1010 }
1011
1012 return;
1013 }
1014
printDBBackupUsage(bool devhelp)1015 static void printDBBackupUsage(bool devhelp) {
1016 printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
1017 printf("Usage: %s (start | status | switch | abort | pause | resume) [OPTIONS]\n\n", exeDatabaseBackup.toString().c_str());
1018 printf(" -d, --destination CONNFILE\n"
1019 " The path of a file containing the connection string for the\n");
1020 printf(" destination FoundationDB cluster.\n");
1021 printf(" -s, --source CONNFILE\n"
1022 " The path of a file containing the connection string for the\n"
1023 " source FoundationDB cluster.\n");
1024 printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n");
1025 printf(" -k KEYS List of key ranges to backup.\n"
1026 " If not specified, the entire database will be backed up.\n");
1027 printf(" --cleanup Abort will attempt to stop mutation logging on the source cluster.\n");
1028 #ifndef TLS_DISABLED
1029 printf(TLS_HELP);
1030 #endif
1031 printf(" -v, --version Print version information and exit.\n");
1032 printf(" -h, --help Display this help and exit.\n");
1033 printf("\n"
1034 " KEYS FORMAT: \"<BEGINKEY> <ENDKEY>\" [...]\n");
1035
1036 if (devhelp) {
1037 #ifdef _WIN32
1038 printf(" -n Create a new console.\n");
1039 printf(" -q Disable error dialog on crash.\n");
1040 printf(" --parentpid PID\n");
1041 printf(" Specify a process after whose termination to exit.\n");
1042 #endif
1043 }
1044
1045 return;
1046 }
1047
printUsage(enumProgramExe programExe,bool devhelp)1048 static void printUsage(enumProgramExe programExe, bool devhelp)
1049 {
1050
1051 switch (programExe)
1052 {
1053 case EXE_AGENT:
1054 printAgentUsage(devhelp);
1055 break;
1056 case EXE_BACKUP:
1057 printBackupUsage(devhelp);
1058 break;
1059 case EXE_RESTORE:
1060 printRestoreUsage(devhelp);
1061 break;
1062 case EXE_DR_AGENT:
1063 printDBAgentUsage(devhelp);
1064 break;
1065 case EXE_DB_BACKUP:
1066 printDBBackupUsage(devhelp);
1067 break;
1068 case EXE_UNDEFINED:
1069 default:
1070 break;
1071 }
1072
1073 return;
1074 }
1075
1076 extern bool g_crashOnError;
1077
1078 // Return the type of program executable based on the name of executable file
getProgramType(std::string programExe)1079 enumProgramExe getProgramType(std::string programExe)
1080 {
1081 enumProgramExe enProgramExe = EXE_UNDEFINED;
1082
1083 // lowercase the string
1084 std::transform(programExe.begin(), programExe.end(), programExe.begin(), ::tolower);
1085
1086 // Remove the extension, if Windows
1087 #ifdef _WIN32
1088 size_t lastDot = programExe.find_last_of(".");
1089 if (lastDot != std::string::npos) {
1090 size_t lastSlash = programExe.find_last_of("\\");
1091
1092 // Ensure last dot is after last slash, if present
1093 if ((lastSlash == std::string::npos)||
1094 (lastSlash < lastDot) )
1095 {
1096 programExe = programExe.substr(0, lastDot);
1097 }
1098 }
1099 #endif
1100 // For debugging convenience, remove .debug suffix if present.
1101 if(StringRef(programExe).endsWith(LiteralStringRef(".debug")))
1102 programExe = programExe.substr(0, programExe.size() - 6);
1103
1104 // Check if backup agent
1105 if ((programExe.length() >= exeAgent.size()) &&
1106 (programExe.compare(programExe.length()-exeAgent.size(), exeAgent.size(), (const char*) exeAgent.begin()) == 0) )
1107 {
1108 enProgramExe = EXE_AGENT;
1109 }
1110
1111 // Check if backup
1112 else if ((programExe.length() >= exeBackup.size()) &&
1113 (programExe.compare(programExe.length() - exeBackup.size(), exeBackup.size(), (const char*)exeBackup.begin()) == 0))
1114 {
1115 enProgramExe = EXE_BACKUP;
1116 }
1117
1118 // Check if restore
1119 else if ((programExe.length() >= exeRestore.size()) &&
1120 (programExe.compare(programExe.length() - exeRestore.size(), exeRestore.size(), (const char*)exeRestore.begin()) == 0))
1121 {
1122 enProgramExe = EXE_RESTORE;
1123 }
1124
1125 // Check if db agent
1126 else if ((programExe.length() >= exeDatabaseAgent.size()) &&
1127 (programExe.compare(programExe.length() - exeDatabaseAgent.size(), exeDatabaseAgent.size(), (const char*)exeDatabaseAgent.begin()) == 0))
1128 {
1129 enProgramExe = EXE_DR_AGENT;
1130 }
1131
1132 // Check if db backup
1133 else if ((programExe.length() >= exeDatabaseBackup.size()) &&
1134 (programExe.compare(programExe.length() - exeDatabaseBackup.size(), exeDatabaseBackup.size(), (const char*)exeDatabaseBackup.begin()) == 0))
1135 {
1136 enProgramExe = EXE_DB_BACKUP;
1137 }
1138
1139 return enProgramExe;
1140 }
1141
getBackupType(std::string backupType)1142 enumBackupType getBackupType(std::string backupType)
1143 {
1144 enumBackupType enBackupType = BACKUP_UNDEFINED;
1145
1146 // lowercase the string
1147 std::transform(backupType.begin(), backupType.end(), backupType.begin(), ::tolower);
1148
1149 static std::map<std::string, enumBackupType> values;
1150 if(values.empty()) {
1151 values["start"] = BACKUP_START;
1152 values["status"] = BACKUP_STATUS;
1153 values["abort"] = BACKUP_ABORT;
1154 values["wait"] = BACKUP_WAIT;
1155 values["discontinue"] = BACKUP_DISCONTINUE;
1156 values["pause"] = BACKUP_PAUSE;
1157 values["resume"] = BACKUP_RESUME;
1158 values["expire"] = BACKUP_EXPIRE;
1159 values["delete"] = BACKUP_DELETE;
1160 values["describe"] = BACKUP_DESCRIBE;
1161 values["list"] = BACKUP_LIST;
1162 values["dump"] = BACKUP_DUMP;
1163 values["modify"] = BACKUP_MODIFY;
1164 }
1165
1166 auto i = values.find(backupType);
1167 if(i != values.end())
1168 enBackupType = i->second;
1169
1170 return enBackupType;
1171 }
1172
getRestoreType(std::string name)1173 enumRestoreType getRestoreType(std::string name) {
1174 if(name == "start") return RESTORE_START;
1175 if(name == "abort") return RESTORE_ABORT;
1176 if(name == "status") return RESTORE_STATUS;
1177 if(name == "wait") return RESTORE_WAIT;
1178 return RESTORE_UNKNOWN;
1179 }
1180
getDBType(std::string dbType)1181 enumDBType getDBType(std::string dbType)
1182 {
1183 enumDBType enBackupType = DB_UNDEFINED;
1184
1185 // lowercase the string
1186 std::transform(dbType.begin(), dbType.end(), dbType.begin(), ::tolower);
1187
1188 static std::map<std::string, enumDBType> values;
1189 if(values.empty()) {
1190 values["start"] = DB_START;
1191 values["status"] = DB_STATUS;
1192 values["switch"] = DB_SWITCH;
1193 values["abort"] = DB_ABORT;
1194 values["pause"] = DB_PAUSE;
1195 values["resume"] = DB_RESUME;
1196 }
1197
1198 auto i = values.find(dbType);
1199 if(i != values.end())
1200 enBackupType = i->second;
1201
1202 return enBackupType;
1203 }
1204
getLayerStatus(Reference<ReadYourWritesTransaction> tr,std::string name,std::string id,enumProgramExe exe,Database dest)1205 ACTOR Future<std::string> getLayerStatus(Reference<ReadYourWritesTransaction> tr, std::string name, std::string id, enumProgramExe exe, Database dest) {
1206 // This process will write a document that looks like this:
1207 // { backup : { $expires : {<subdoc>}, version: <version from approximately 30 seconds from now> }
1208 // so that the value under 'backup' will eventually expire to null and thus be ignored by
1209 // readers of status. This is because if all agents die then they can no longer clean up old
1210 // status docs from other dead agents.
1211
1212 state Version readVer = wait(tr->getReadVersion());
1213
1214 state json_spirit::mValue layersRootValue; // Will contain stuff that goes into the doc at the layers status root
1215 JSONDoc layersRoot(layersRootValue); // Convenient mutator / accessor for the layers root
1216 JSONDoc op = layersRoot.subDoc(name); // Operator object for the $expires operation
1217 // Create the $expires key which is where the rest of the status output will go
1218
1219 state JSONDoc layerRoot = op.subDoc("$expires");
1220 // Set the version argument in the $expires operator object.
1221 op.create("version") = readVer + 120 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
1222
1223 layerRoot.create("instances_running.$sum") = 1;
1224 layerRoot.create("last_updated.$max") = now();
1225
1226 state JSONDoc o = layerRoot.subDoc("instances." + id);
1227
1228 o.create("version") = FDB_VT_VERSION;
1229 o.create("id") = id;
1230 o.create("last_updated") = now();
1231 o.create("memory_usage") = (int64_t)getMemoryUsage();
1232 o.create("resident_size") = (int64_t)getResidentMemoryUsage();
1233 o.create("main_thread_cpu_seconds") = getProcessorTimeThread();
1234 o.create("process_cpu_seconds") = getProcessorTimeProcess();
1235 o.create("configured_workers") = CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT;
1236
1237 if(exe == EXE_AGENT) {
1238 static BlobStoreEndpoint::Stats last_stats;
1239 static double last_ts = 0;
1240 BlobStoreEndpoint::Stats current_stats = BlobStoreEndpoint::s_stats;
1241 JSONDoc blobstats = o.create("blob_stats");
1242 blobstats.create("total") = current_stats.getJSON();
1243 BlobStoreEndpoint::Stats diff = current_stats - last_stats;
1244 json_spirit::mObject diffObj = diff.getJSON();
1245 if(last_ts > 0)
1246 diffObj["bytes_per_second"] = double(current_stats.bytes_sent - last_stats.bytes_sent) / (now() - last_ts);
1247 blobstats.create("recent") = diffObj;
1248 last_stats = current_stats;
1249 last_ts = now();
1250
1251 JSONDoc totalBlobStats = layerRoot.subDoc("blob_recent_io");
1252 for(auto &p : diffObj)
1253 totalBlobStats.create(p.first + ".$sum") = p.second;
1254
1255 state FileBackupAgent fba;
1256 state std::vector<KeyBackedTag> backupTags = wait(getAllBackupTags(tr));
1257 state std::vector<Future<Version>> tagLastRestorableVersions;
1258 state std::vector<Future<EBackupState>> tagStates;
1259 state std::vector<Future<Reference<IBackupContainer>>> tagContainers;
1260 state std::vector<Future<int64_t>> tagRangeBytes;
1261 state std::vector<Future<int64_t>> tagLogBytes;
1262 state Future<Optional<Value>> fBackupPaused = tr->get(fba.taskBucket->getPauseKey());
1263 state int i = 0;
1264
1265 tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
1266 tr->setOption(FDBTransactionOptions::LOCK_AWARE);
1267 state std::vector<KeyBackedTag>::iterator tag;
1268 state std::vector<UID> backupTagUids;
1269 for (tag = backupTags.begin(); tag != backupTags.end(); tag++) {
1270 UidAndAbortedFlagT uidAndAbortedFlag = wait(tag->getOrThrow(tr));
1271 BackupConfig config(uidAndAbortedFlag.first);
1272 backupTagUids.push_back(config.getUid());
1273
1274 tagStates.push_back(config.stateEnum().getOrThrow(tr));
1275 tagRangeBytes.push_back(config.rangeBytesWritten().getD(tr, false, 0));
1276 tagLogBytes.push_back(config.logBytesWritten().getD(tr, false, 0));
1277 tagContainers.push_back(config.backupContainer().getOrThrow(tr));
1278 tagLastRestorableVersions.push_back(fba.getLastRestorable(tr, StringRef(tag->tagName)));
1279 }
1280
1281 wait( waitForAll(tagLastRestorableVersions) && waitForAll(tagStates) && waitForAll(tagContainers) && waitForAll(tagRangeBytes) && waitForAll(tagLogBytes) && success(fBackupPaused));
1282
1283 JSONDoc tagsRoot = layerRoot.subDoc("tags.$latest");
1284 layerRoot.create("tags.timestamp") = now();
1285 layerRoot.create("total_workers.$sum") = fBackupPaused.get().present() ? 0 : CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT;
1286 layerRoot.create("paused.$latest") = fBackupPaused.get().present();
1287
1288 int j = 0;
1289 for (KeyBackedTag eachTag : backupTags) {
1290 Version last_restorable_version = tagLastRestorableVersions[j].get();
1291 double last_restorable_seconds_behind = ((double)readVer - last_restorable_version) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
1292 BackupAgentBase::enumState status = (BackupAgentBase::enumState)tagStates[j].get();
1293 const char *statusText = fba.getStateText(status);
1294
1295 // The object for this backup tag inside this instance's subdocument
1296 JSONDoc tagRoot = tagsRoot.subDoc(eachTag.tagName);
1297 tagRoot.create("current_container") = tagContainers[j].get()->getURL();
1298 tagRoot.create("current_status") = statusText;
1299 tagRoot.create("last_restorable_version") = tagLastRestorableVersions[j].get();
1300 tagRoot.create("last_restorable_seconds_behind") = last_restorable_seconds_behind;
1301 tagRoot.create("running_backup") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING);
1302 tagRoot.create("running_backup_is_restorable") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL);
1303 tagRoot.create("range_bytes_written") = tagRangeBytes[j].get();
1304 tagRoot.create("mutation_log_bytes_written") = tagLogBytes[j].get();
1305 tagRoot.create("mutation_stream_id") = backupTagUids[j].toString();
1306
1307 j++;
1308 }
1309 }
1310 else if(exe == EXE_DR_AGENT) {
1311 state DatabaseBackupAgent dba;
1312 state Reference<ReadYourWritesTransaction> tr2(new ReadYourWritesTransaction(dest));
1313 tr2->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
1314 tr2->setOption(FDBTransactionOptions::LOCK_AWARE);
1315 state Standalone<RangeResultRef> tagNames = wait(tr2->getRange(dba.tagNames.range(), 10000));
1316 state std::vector<Future<Optional<Key>>> backupVersion;
1317 state std::vector<Future<int>> backupStatus;
1318 state std::vector<Future<int64_t>> tagRangeBytesDR;
1319 state std::vector<Future<int64_t>> tagLogBytesDR;
1320 state Future<Optional<Value>> fDRPaused = tr->get(dba.taskBucket->getPauseKey());
1321
1322 state std::vector<UID> drTagUids;
1323 for(int i = 0; i < tagNames.size(); i++) {
1324 backupVersion.push_back(tr2->get(tagNames[i].value.withPrefix(applyMutationsBeginRange.begin)));
1325 UID tagUID = BinaryReader::fromStringRef<UID>(tagNames[i].value, Unversioned());
1326 drTagUids.push_back(tagUID);
1327 backupStatus.push_back(dba.getStateValue(tr2, tagUID));
1328 tagRangeBytesDR.push_back(dba.getRangeBytesWritten(tr2, tagUID));
1329 tagLogBytesDR.push_back(dba.getLogBytesWritten(tr2, tagUID));
1330 }
1331
1332 wait(waitForAll(backupStatus) && waitForAll(backupVersion) && waitForAll(tagRangeBytesDR) && waitForAll(tagLogBytesDR) && success(fDRPaused));
1333
1334 JSONDoc tagsRoot = layerRoot.subDoc("tags.$latest");
1335 layerRoot.create("tags.timestamp") = now();
1336 layerRoot.create("total_workers.$sum") = fDRPaused.get().present() ? 0 : CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT;
1337 layerRoot.create("paused.$latest") = fDRPaused.get().present();
1338
1339 for (int i = 0; i < tagNames.size(); i++) {
1340 std::string tagName = dba.sourceTagNames.unpack(tagNames[i].key).getString(0).toString();
1341
1342 BackupAgentBase::enumState status = (BackupAgentBase::enumState)backupStatus[i].get();
1343
1344 JSONDoc tagRoot = tagsRoot.create(tagName);
1345 tagRoot.create("running_backup") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL || status == BackupAgentBase::STATE_RUNNING);
1346 tagRoot.create("running_backup_is_restorable") = (status == BackupAgentBase::STATE_RUNNING_DIFFERENTIAL);
1347 tagRoot.create("range_bytes_written") = tagRangeBytesDR[i].get();
1348 tagRoot.create("mutation_log_bytes_written") = tagLogBytesDR[i].get();
1349 tagRoot.create("mutation_stream_id") = drTagUids[i].toString();
1350
1351 if (backupVersion[i].get().present()) {
1352 double seconds_behind = ((double)readVer - BinaryReader::fromStringRef<Version>(backupVersion[i].get().get(), Unversioned())) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
1353 tagRoot.create("seconds_behind") = seconds_behind;
1354 //TraceEvent("BackupMetrics").detail("SecondsBehind", seconds_behind);
1355 }
1356
1357 tagRoot.create("backup_state") = BackupAgentBase::getStateText(status);
1358 }
1359 }
1360
1361 std::string json = json_spirit::write_string(layersRootValue);
1362 return json;
1363 }
1364
1365 // Check for unparseable or expired statuses and delete them.
1366 // First checks the first doc in the key range, and if it is valid, alive and not "me" then
1367 // returns. Otherwise, checks the rest of the range as well.
cleanupStatus(Reference<ReadYourWritesTransaction> tr,std::string rootKey,std::string name,std::string id,int limit=1)1368 ACTOR Future<Void> cleanupStatus(Reference<ReadYourWritesTransaction> tr, std::string rootKey, std::string name, std::string id, int limit = 1) {
1369 state Standalone<RangeResultRef> docs = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, true));
1370 state bool readMore = false;
1371 state int i;
1372 for(i = 0; i < docs.size(); ++i) {
1373 json_spirit::mValue docValue;
1374 try {
1375 json_spirit::read_string(docs[i].value.toString(), docValue);
1376 JSONDoc doc(docValue);
1377 // Update the reference version for $expires
1378 JSONDoc::expires_reference_version = tr->getReadVersion().get();
1379 // Evaluate the operators in the document, which will reduce to nothing if it is expired.
1380 doc.cleanOps();
1381 if(!doc.has(name + ".last_updated"))
1382 throw Error();
1383
1384 // Alive and valid.
1385 // If limit == 1 and id is present then read more
1386 if(limit == 1 && doc.has(name + ".instances." + id))
1387 readMore = true;
1388 } catch(Error &e) {
1389 // If doc can't be parsed or isn't alive, delete it.
1390 TraceEvent(SevWarn, "RemovedDeadBackupLayerStatus").detail("Key", docs[i].key);
1391 tr->clear(docs[i].key);
1392 // If limit is 1 then read more.
1393 if(limit == 1)
1394 readMore = true;
1395 }
1396 if(readMore) {
1397 limit = 10000;
1398 Standalone<RangeResultRef> docs2 = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, true));
1399 docs = std::move(docs2);
1400 readMore = false;
1401 }
1402 }
1403
1404 return Void();
1405 }
1406
1407 // Get layer status document for just this layer
getLayerStatus(Database src,std::string rootKey)1408 ACTOR Future<json_spirit::mObject> getLayerStatus(Database src, std::string rootKey) {
1409 state Transaction tr(src);
1410
1411 loop {
1412 try {
1413 tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
1414 tr.setOption(FDBTransactionOptions::LOCK_AWARE);
1415 state Standalone<RangeResultRef> kvPairs = wait(tr.getRange(KeyRangeRef(rootKey, strinc(rootKey)), CLIENT_KNOBS->ROW_LIMIT_UNLIMITED));
1416 json_spirit::mObject statusDoc;
1417 JSONDoc modifier(statusDoc);
1418 for(auto &kv : kvPairs) {
1419 json_spirit::mValue docValue;
1420 json_spirit::read_string(kv.value.toString(), docValue);
1421 modifier.absorb(docValue);
1422 }
1423 JSONDoc::expires_reference_version = (uint64_t)tr.getReadVersion().get();
1424 modifier.cleanOps();
1425 return statusDoc;
1426 }
1427 catch (Error& e) {
1428 wait(tr.onError(e));
1429 }
1430 }
1431 }
1432
1433 // Read layer status for this layer and get the total count of agent processes (instances) then adjust the poll delay based on that and BACKUP_AGGREGATE_POLL_RATE
updateAgentPollRate(Database src,std::string rootKey,std::string name,double * pollDelay)1434 ACTOR Future<Void> updateAgentPollRate(Database src, std::string rootKey, std::string name, double *pollDelay) {
1435 loop {
1436 try {
1437 json_spirit::mObject status = wait(getLayerStatus(src, rootKey));
1438 int64_t processes = 0;
1439 // If instances count is present and greater than 0 then update pollDelay
1440 if(JSONDoc(status).tryGet<int64_t>(name + ".instances_running", processes) && processes > 0) {
1441 // The aggregate poll rate is the target poll rate for all agent processes in the cluster
1442 // The poll rate (polls/sec) for a single processes is aggregate poll rate / processes, and pollDelay is the inverse of that
1443 *pollDelay = (double)processes / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE;
1444 }
1445 } catch(Error &e) {
1446 TraceEvent(SevWarn, "BackupAgentPollRateUpdateError").error(e);
1447 }
1448 wait(delay(CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE_UPDATE_INTERVAL));
1449 }
1450 }
1451
statusUpdateActor(Database statusUpdateDest,std::string name,enumProgramExe exe,double * pollDelay,Database taskDest=Database (),std::string id=g_nondeterministic_random->randomUniqueID ().toString ())1452 ACTOR Future<Void> statusUpdateActor(Database statusUpdateDest, std::string name, enumProgramExe exe, double *pollDelay, Database taskDest = Database(),
1453 std::string id = g_nondeterministic_random->randomUniqueID().toString()) {
1454 state std::string metaKey = layerStatusMetaPrefixRange.begin.toString() + "json/" + name;
1455 state std::string rootKey = backupStatusPrefixRange.begin.toString() + name + "/json";
1456 state std::string instanceKey = rootKey + "/" + "agent-" + id;
1457 state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(statusUpdateDest));
1458 state Future<Void> pollRateUpdater;
1459
1460 // Register the existence of this layer in the meta key space
1461 loop {
1462 try {
1463 tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
1464 tr->setOption(FDBTransactionOptions::LOCK_AWARE);
1465 tr->set(metaKey, rootKey);
1466 wait(tr->commit());
1467 break;
1468 }
1469 catch (Error& e) {
1470 wait(tr->onError(e));
1471 }
1472 }
1473
1474 // Write status periodically
1475 loop {
1476 tr->reset();
1477 try {
1478 loop{
1479 try {
1480 tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
1481 tr->setOption(FDBTransactionOptions::LOCK_AWARE);
1482 state Future<std::string> futureStatusDoc = getLayerStatus(tr, name, id, exe, taskDest);
1483 wait(cleanupStatus(tr, rootKey, name, id));
1484 std::string statusdoc = wait(futureStatusDoc);
1485 tr->set(instanceKey, statusdoc);
1486 wait(tr->commit());
1487 break;
1488 }
1489 catch (Error& e) {
1490 wait(tr->onError(e));
1491 }
1492 }
1493
1494 wait(delay(CLIENT_KNOBS->BACKUP_STATUS_DELAY * ( ( 1.0 - CLIENT_KNOBS->BACKUP_STATUS_JITTER ) + 2 * g_random->random01() * CLIENT_KNOBS->BACKUP_STATUS_JITTER )));
1495
1496 // Now that status was written at least once by this process (and hopefully others), start the poll rate control updater if it wasn't started yet
1497 if(!pollRateUpdater.isValid() && pollDelay != nullptr)
1498 pollRateUpdater = updateAgentPollRate(statusUpdateDest, rootKey, name, pollDelay);
1499 }
1500 catch (Error& e) {
1501 TraceEvent(SevWarnAlways, "UnableToWriteStatus").error(e);
1502 wait(delay(10.0));
1503 }
1504 }
1505 }
1506
runDBAgent(Database src,Database dest)1507 ACTOR Future<Void> runDBAgent(Database src, Database dest) {
1508 state double pollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE;
1509 std::string id = g_nondeterministic_random->randomUniqueID().toString();
1510 state Future<Void> status = statusUpdateActor(src, "dr_backup", EXE_DR_AGENT, &pollDelay, dest, id);
1511 state Future<Void> status_other = statusUpdateActor(dest, "dr_backup_dest", EXE_DR_AGENT, &pollDelay, dest, id);
1512
1513 state DatabaseBackupAgent backupAgent(src);
1514
1515 loop {
1516 try {
1517 wait(backupAgent.run(dest, &pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT));
1518 break;
1519 }
1520 catch (Error& e) {
1521 if (e.code() == error_code_operation_cancelled)
1522 throw;
1523
1524 TraceEvent(SevError, "DA_runAgent").error(e);
1525 fprintf(stderr, "ERROR: DR agent encountered fatal error `%s'\n", e.what());
1526
1527 wait( delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY) );
1528 }
1529 }
1530
1531 return Void();
1532 }
1533
runAgent(Database db)1534 ACTOR Future<Void> runAgent(Database db) {
1535 state double pollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE;
1536 state Future<Void> status = statusUpdateActor(db, "backup", EXE_AGENT, &pollDelay);
1537
1538 state FileBackupAgent backupAgent;
1539
1540 loop {
1541 try {
1542 wait(backupAgent.run(db, &pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT));
1543 break;
1544 }
1545 catch (Error& e) {
1546 if (e.code() == error_code_operation_cancelled)
1547 throw;
1548
1549 TraceEvent(SevError, "BA_runAgent").error(e);
1550 fprintf(stderr, "ERROR: backup agent encountered fatal error `%s'\n", e.what());
1551
1552 wait( delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY) );
1553 }
1554 }
1555
1556 return Void();
1557 }
1558
submitDBBackup(Database src,Database dest,Standalone<VectorRef<KeyRangeRef>> backupRanges,std::string tagName)1559 ACTOR Future<Void> submitDBBackup(Database src, Database dest, Standalone<VectorRef<KeyRangeRef>> backupRanges, std::string tagName) {
1560 try
1561 {
1562 state DatabaseBackupAgent backupAgent(src);
1563
1564 // Backup everything, if no ranges were specified
1565 if (backupRanges.size() == 0) {
1566 backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
1567 }
1568
1569
1570 wait(backupAgent.submitBackup(dest, KeyRef(tagName), backupRanges, false, StringRef(), StringRef(), true));
1571
1572 // Check if a backup agent is running
1573 bool agentRunning = wait(backupAgent.checkActive(dest));
1574
1575 if (!agentRunning) {
1576 printf("The DR on tag `%s' was successfully submitted but no DR agents are responding.\n", printable(StringRef(tagName)).c_str());
1577
1578 // Throw an error that will not display any additional information
1579 throw actor_cancelled();
1580 }
1581 else {
1582 printf("The DR on tag `%s' was successfully submitted.\n", printable(StringRef(tagName)).c_str());
1583 }
1584 }
1585
1586 catch (Error& e) {
1587 if(e.code() == error_code_actor_cancelled)
1588 throw;
1589 switch (e.code())
1590 {
1591 case error_code_backup_error:
1592 fprintf(stderr, "ERROR: An error was encountered during submission\n");
1593 break;
1594 case error_code_backup_duplicate:
1595 fprintf(stderr, "ERROR: A DR is already running on tag `%s'\n", printable(StringRef(tagName)).c_str());
1596 break;
1597 default:
1598 fprintf(stderr, "ERROR: %s\n", e.what());
1599 break;
1600 }
1601
1602 throw backup_error();
1603 }
1604
1605 return Void();
1606 }
1607
submitBackup(Database db,std::string url,int snapshotIntervalSeconds,Standalone<VectorRef<KeyRangeRef>> backupRanges,std::string tagName,bool dryRun,bool waitForCompletion,bool stopWhenDone)1608 ACTOR Future<Void> submitBackup(Database db, std::string url, int snapshotIntervalSeconds, Standalone<VectorRef<KeyRangeRef>> backupRanges, std::string tagName, bool dryRun, bool waitForCompletion, bool stopWhenDone) {
1609 try
1610 {
1611 state FileBackupAgent backupAgent;
1612
1613 // Backup everything, if no ranges were specified
1614 if (backupRanges.size() == 0) {
1615 backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
1616 }
1617
1618 if (dryRun) {
1619 state KeyBackedTag tag = makeBackupTag(tagName);
1620 Optional<UidAndAbortedFlagT> uidFlag = wait(tag.get(db));
1621
1622 if (uidFlag.present()) {
1623 BackupConfig config(uidFlag.get().first);
1624 EBackupState backupStatus = wait(config.stateEnum().getOrThrow(db));
1625
1626 // Throw error if a backup is currently running until we support parallel backups
1627 if (BackupAgentBase::isRunnable((BackupAgentBase::enumState)backupStatus)) {
1628 throw backup_duplicate();
1629 }
1630 }
1631
1632 if (waitForCompletion) {
1633 printf("Submitted and now waiting for the backup on tag `%s' to complete. (DRY RUN)\n", printable(StringRef(tagName)).c_str());
1634 }
1635
1636 else {
1637 // Check if a backup agent is running
1638 bool agentRunning = wait(backupAgent.checkActive(db));
1639
1640 if (!agentRunning) {
1641 printf("The backup on tag `%s' was successfully submitted but no backup agents are responding. (DRY RUN)\n", printable(StringRef(tagName)).c_str());
1642
1643 // Throw an error that will not display any additional information
1644 throw actor_cancelled();
1645 }
1646 else {
1647 printf("The backup on tag `%s' was successfully submitted. (DRY RUN)\n", printable(StringRef(tagName)).c_str());
1648 }
1649 }
1650 }
1651
1652 else {
1653 wait(backupAgent.submitBackup(db, KeyRef(url), snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone));
1654
1655 // Wait for the backup to complete, if requested
1656 if (waitForCompletion) {
1657 printf("Submitted and now waiting for the backup on tag `%s' to complete.\n", printable(StringRef(tagName)).c_str());
1658 wait(success(backupAgent.waitBackup(db, tagName)));
1659 }
1660 else {
1661 // Check if a backup agent is running
1662 bool agentRunning = wait(backupAgent.checkActive(db));
1663
1664 if (!agentRunning) {
1665 printf("The backup on tag `%s' was successfully submitted but no backup agents are responding.\n", printable(StringRef(tagName)).c_str());
1666
1667 // Throw an error that will not display any additional information
1668 throw actor_cancelled();
1669 }
1670 else {
1671 printf("The backup on tag `%s' was successfully submitted.\n", printable(StringRef(tagName)).c_str());
1672 }
1673 }
1674 }
1675 }
1676 catch (Error& e) {
1677 if(e.code() == error_code_actor_cancelled)
1678 throw;
1679 switch (e.code())
1680 {
1681 case error_code_backup_error:
1682 fprintf(stderr, "ERROR: An error was encountered during submission\n");
1683 break;
1684 case error_code_backup_duplicate:
1685 fprintf(stderr, "ERROR: A backup is already running on tag `%s'\n", printable(StringRef(tagName)).c_str());
1686 break;
1687 default:
1688 fprintf(stderr, "ERROR: %s\n", e.what());
1689 break;
1690 }
1691
1692 throw backup_error();
1693 }
1694
1695 return Void();
1696 }
1697
switchDBBackup(Database src,Database dest,Standalone<VectorRef<KeyRangeRef>> backupRanges,std::string tagName,bool forceAction)1698 ACTOR Future<Void> switchDBBackup(Database src, Database dest, Standalone<VectorRef<KeyRangeRef>> backupRanges, std::string tagName, bool forceAction) {
1699 try
1700 {
1701 state DatabaseBackupAgent backupAgent(src);
1702
1703 // Backup everything, if no ranges were specified
1704 if (backupRanges.size() == 0) {
1705 backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
1706 }
1707
1708
1709 wait(backupAgent.atomicSwitchover(dest, KeyRef(tagName), backupRanges, StringRef(), StringRef(), forceAction));
1710 printf("The DR on tag `%s' was successfully switched.\n", printable(StringRef(tagName)).c_str());
1711 }
1712
1713 catch (Error& e) {
1714 if(e.code() == error_code_actor_cancelled)
1715 throw;
1716 switch (e.code())
1717 {
1718 case error_code_backup_error:
1719 fprintf(stderr, "ERROR: An error was encountered during submission\n");
1720 break;
1721 case error_code_backup_duplicate:
1722 fprintf(stderr, "ERROR: A DR is already running on tag `%s'\n", printable(StringRef(tagName)).c_str());
1723 break;
1724 default:
1725 fprintf(stderr, "ERROR: %s\n", e.what());
1726 break;
1727 }
1728
1729 throw backup_error();
1730 }
1731
1732 return Void();
1733 }
1734
statusDBBackup(Database src,Database dest,std::string tagName,int errorLimit)1735 ACTOR Future<Void> statusDBBackup(Database src, Database dest, std::string tagName, int errorLimit) {
1736 try
1737 {
1738 state DatabaseBackupAgent backupAgent(src);
1739
1740 std::string statusText = wait(backupAgent.getStatus(dest, errorLimit, StringRef(tagName)));
1741 printf("%s\n", statusText.c_str());
1742 }
1743 catch (Error& e) {
1744 if(e.code() == error_code_actor_cancelled)
1745 throw;
1746 fprintf(stderr, "ERROR: %s\n", e.what());
1747 throw;
1748 }
1749
1750 return Void();
1751 }
1752
statusBackup(Database db,std::string tagName,bool showErrors,bool json)1753 ACTOR Future<Void> statusBackup(Database db, std::string tagName, bool showErrors, bool json) {
1754 try
1755 {
1756 state FileBackupAgent backupAgent;
1757
1758 std::string statusText = wait(json ? backupAgent.getStatusJSON(db, tagName) : backupAgent.getStatus(db, showErrors, tagName));
1759 printf("%s\n", statusText.c_str());
1760 }
1761 catch (Error& e) {
1762 if(e.code() == error_code_actor_cancelled)
1763 throw;
1764 fprintf(stderr, "ERROR: %s\n", e.what());
1765 throw;
1766 }
1767
1768 return Void();
1769 }
1770
abortDBBackup(Database src,Database dest,std::string tagName,bool partial)1771 ACTOR Future<Void> abortDBBackup(Database src, Database dest, std::string tagName, bool partial) {
1772 try
1773 {
1774 state DatabaseBackupAgent backupAgent(src);
1775
1776 wait(backupAgent.abortBackup(dest, Key(tagName), partial));
1777 wait(backupAgent.unlockBackup(dest, Key(tagName)));
1778
1779 printf("The DR on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str());
1780 }
1781 catch (Error& e) {
1782 if(e.code() == error_code_actor_cancelled)
1783 throw;
1784 switch (e.code())
1785 {
1786 case error_code_backup_error:
1787 fprintf(stderr, "ERROR: An error was encountered during submission\n");
1788 break;
1789 case error_code_backup_unneeded:
1790 fprintf(stderr, "ERROR: A DR was not running on tag `%s'\n", printable(StringRef(tagName)).c_str());
1791 break;
1792 default:
1793 fprintf(stderr, "ERROR: %s\n", e.what());
1794 break;
1795 }
1796 throw;
1797 }
1798
1799 return Void();
1800 }
1801
abortBackup(Database db,std::string tagName)1802 ACTOR Future<Void> abortBackup(Database db, std::string tagName) {
1803 try
1804 {
1805 state FileBackupAgent backupAgent;
1806
1807 wait(backupAgent.abortBackup(db, tagName));
1808
1809 printf("The backup on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str());
1810 }
1811 catch (Error& e) {
1812 if(e.code() == error_code_actor_cancelled)
1813 throw;
1814 switch (e.code())
1815 {
1816 case error_code_backup_error:
1817 fprintf(stderr, "ERROR: An error was encountered during submission\n");
1818 break;
1819 case error_code_backup_unneeded:
1820 fprintf(stderr, "ERROR: A backup was not running on tag `%s'\n", printable(StringRef(tagName)).c_str());
1821 break;
1822 default:
1823 fprintf(stderr, "ERROR: %s\n", e.what());
1824 break;
1825 }
1826 throw;
1827 }
1828
1829 return Void();
1830 }
1831
waitBackup(Database db,std::string tagName,bool stopWhenDone)1832 ACTOR Future<Void> waitBackup(Database db, std::string tagName, bool stopWhenDone) {
1833 try
1834 {
1835 state FileBackupAgent backupAgent;
1836
1837 int status = wait(backupAgent.waitBackup(db, tagName, stopWhenDone));
1838
1839 printf("The backup on tag `%s' %s.\n", printable(StringRef(tagName)).c_str(),
1840 BackupAgentBase::getStateText((BackupAgentBase::enumState) status));
1841 }
1842 catch (Error& e) {
1843 if(e.code() == error_code_actor_cancelled)
1844 throw;
1845 fprintf(stderr, "ERROR: %s\n", e.what());
1846 throw;
1847 }
1848
1849 return Void();
1850 }
1851
discontinueBackup(Database db,std::string tagName,bool waitForCompletion)1852 ACTOR Future<Void> discontinueBackup(Database db, std::string tagName, bool waitForCompletion) {
1853 try
1854 {
1855 state FileBackupAgent backupAgent;
1856
1857 wait(backupAgent.discontinueBackup(db, StringRef(tagName)));
1858
1859 // Wait for the backup to complete, if requested
1860 if (waitForCompletion) {
1861 printf("Discontinued and now waiting for the backup on tag `%s' to complete.\n", printable(StringRef(tagName)).c_str());
1862 wait(success(backupAgent.waitBackup(db, tagName)));
1863 }
1864 else {
1865 printf("The backup on tag `%s' was successfully discontinued.\n", printable(StringRef(tagName)).c_str());
1866 }
1867
1868 }
1869 catch (Error& e) {
1870 if(e.code() == error_code_actor_cancelled)
1871 throw;
1872 switch (e.code())
1873 {
1874 case error_code_backup_error:
1875 fprintf(stderr, "ERROR: An encounter was error during submission\n");
1876 break;
1877 case error_code_backup_unneeded:
1878 fprintf(stderr, "ERROR: A backup in not running on tag `%s'\n", printable(StringRef(tagName)).c_str());
1879 break;
1880 case error_code_backup_duplicate:
1881 fprintf(stderr, "ERROR: The backup on tag `%s' is already discontinued\n", printable(StringRef(tagName)).c_str());
1882 break;
1883 default:
1884 fprintf(stderr, "ERROR: %s\n", e.what());
1885 break;
1886 }
1887 throw;
1888 }
1889
1890 return Void();
1891 }
1892
changeBackupResumed(Database db,bool pause)1893 ACTOR Future<Void> changeBackupResumed(Database db, bool pause) {
1894 try {
1895 state FileBackupAgent backupAgent;
1896 wait(backupAgent.taskBucket->changePause(db, pause));
1897 printf("All backup agents have been %s.\n", pause ? "paused" : "resumed");
1898 }
1899 catch (Error& e) {
1900 if(e.code() == error_code_actor_cancelled)
1901 throw;
1902 fprintf(stderr, "ERROR: %s\n", e.what());
1903 throw;
1904 }
1905
1906 return Void();
1907 }
1908
changeDBBackupResumed(Database src,Database dest,bool pause)1909 ACTOR Future<Void> changeDBBackupResumed(Database src, Database dest, bool pause) {
1910 try {
1911 state DatabaseBackupAgent backupAgent(src);
1912 wait(backupAgent.taskBucket->changePause(dest, pause));
1913 printf("All DR agents have been %s.\n", pause ? "paused" : "resumed");
1914 }
1915 catch (Error& e) {
1916 if(e.code() == error_code_actor_cancelled)
1917 throw;
1918 fprintf(stderr, "ERROR: %s\n", e.what());
1919 throw;
1920 }
1921
1922 return Void();
1923 }
1924
openBackupContainer(const char * name,std::string destinationContainer)1925 Reference<IBackupContainer> openBackupContainer(const char *name, std::string destinationContainer) {
1926 // Error, if no dest container was specified
1927 if (destinationContainer.empty()) {
1928 fprintf(stderr, "ERROR: No backup destination was specified.\n");
1929 printHelpTeaser(name);
1930 throw backup_error();
1931 }
1932
1933 Reference<IBackupContainer> c;
1934 try {
1935 c = IBackupContainer::openContainer(destinationContainer);
1936 }
1937 catch (Error& e) {
1938 std::string msg = format("ERROR: '%s' on URL '%s'", e.what(), destinationContainer.c_str());
1939 if(e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) {
1940 msg += format(": %s", IBackupContainer::lastOpenError.c_str());
1941 }
1942 fprintf(stderr, "%s\n", msg.c_str());
1943 printHelpTeaser(name);
1944 throw;
1945 }
1946
1947 return c;
1948 }
1949
runRestore(std::string destClusterFile,std::string originalClusterFile,std::string tagName,std::string container,Standalone<VectorRef<KeyRangeRef>> ranges,Version targetVersion,std::string targetTimestamp,bool performRestore,bool verbose,bool waitForDone,std::string addPrefix,std::string removePrefix)1950 ACTOR Future<Void> runRestore(std::string destClusterFile, std::string originalClusterFile, std::string tagName, std::string container, Standalone<VectorRef<KeyRangeRef>> ranges, Version targetVersion, std::string targetTimestamp, bool performRestore, bool verbose, bool waitForDone, std::string addPrefix, std::string removePrefix) {
1951 if(ranges.empty()) {
1952 ranges.push_back_deep(ranges.arena(), normalKeys);
1953 }
1954
1955 if(targetVersion != invalidVersion && !targetTimestamp.empty()) {
1956 fprintf(stderr, "Restore target version and target timestamp cannot both be specified\n");
1957 throw restore_error();
1958 }
1959
1960 if(destClusterFile.empty()) {
1961 fprintf(stderr, "Restore destination cluster file must be specified explicitly.\n");
1962 throw restore_error();
1963 }
1964
1965 if(!fileExists(destClusterFile)) {
1966 fprintf(stderr, "Restore destination cluster file '%s' does not exist.\n", destClusterFile.c_str());
1967 throw restore_error();
1968 }
1969
1970 state Optional<Database> origDb;
1971
1972 // Resolve targetTimestamp if given
1973 if(!targetTimestamp.empty()) {
1974 if(originalClusterFile.empty()) {
1975 fprintf(stderr, "An original cluster file must be given in order to resolve restore target timestamp '%s'\n", targetTimestamp.c_str());
1976 throw restore_error();
1977 }
1978
1979 if(!fileExists(originalClusterFile)) {
1980 fprintf(stderr, "Original source database cluster file '%s' does not exist.\n", originalClusterFile.c_str());
1981 throw restore_error();
1982 }
1983
1984 origDb = Database::createDatabase(originalClusterFile, Database::API_VERSION_LATEST);
1985 Version v = wait(timeKeeperVersionFromDatetime(targetTimestamp, origDb.get()));
1986 printf("Timestamp '%s' resolves to version %lld\n", targetTimestamp.c_str(), v);
1987 targetVersion = v;
1988 }
1989
1990 try {
1991 state Database db = Database::createDatabase(destClusterFile, Database::API_VERSION_LATEST);
1992 state FileBackupAgent backupAgent;
1993
1994 state Reference<IBackupContainer> bc = openBackupContainer(exeRestore.toString().c_str(), container);
1995
1996 // If targetVersion is unset then use the maximum restorable version from the backup description
1997 if(targetVersion == invalidVersion) {
1998 if(verbose)
1999 printf("No restore target version given, will use maximum restorable version from backup description.\n");
2000
2001 BackupDescription desc = wait(bc->describeBackup());
2002
2003 if(!desc.maxRestorableVersion.present()) {
2004 fprintf(stderr, "The specified backup is not restorable to any version.\n");
2005 throw restore_error();
2006 }
2007
2008 targetVersion = desc.maxRestorableVersion.get();
2009
2010 if(verbose)
2011 printf("Using target restore version %lld\n", targetVersion);
2012 }
2013
2014 if (performRestore) {
2015 Version restoredVersion = wait(backupAgent.restore(db, origDb, KeyRef(tagName), KeyRef(container), ranges, waitForDone, targetVersion, verbose, KeyRef(addPrefix), KeyRef(removePrefix)));
2016
2017 if(waitForDone && verbose) {
2018 // If restore is now complete then report version restored
2019 printf("Restored to version %lld\n", restoredVersion);
2020 }
2021 }
2022 else {
2023 state Optional<RestorableFileSet> rset = wait(bc->getRestoreSet(targetVersion));
2024
2025 if(!rset.present()) {
2026 fprintf(stderr, "Insufficient data to restore to version %lld. Describe backup for more information.\n", targetVersion);
2027 throw restore_invalid_version();
2028 }
2029
2030 printf("Backup can be used to restore to version %lld\n", targetVersion);
2031 }
2032
2033 }
2034 catch (Error& e) {
2035 if(e.code() == error_code_actor_cancelled)
2036 throw;
2037 fprintf(stderr, "ERROR: %s\n", e.what());
2038 throw;
2039 }
2040
2041 return Void();
2042 }
2043
dumpBackupData(const char * name,std::string destinationContainer,Version beginVersion,Version endVersion)2044 ACTOR Future<Void> dumpBackupData(const char *name, std::string destinationContainer, Version beginVersion, Version endVersion) {
2045 state Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer);
2046
2047 if(beginVersion < 0 || endVersion < 0) {
2048 BackupDescription desc = wait(c->describeBackup());
2049
2050 if(!desc.maxLogEnd.present()) {
2051 fprintf(stderr, "ERROR: Backup must have log data in order to use relative begin/end versions.\n");
2052 throw backup_invalid_info();
2053 }
2054
2055 if(beginVersion < 0) {
2056 beginVersion += desc.maxLogEnd.get();
2057 }
2058
2059 if(endVersion < 0) {
2060 endVersion += desc.maxLogEnd.get();
2061 }
2062 }
2063
2064 printf("Scanning version range %lld to %lld\n", beginVersion, endVersion);
2065 BackupFileList files = wait(c->dumpFileList(beginVersion, endVersion));
2066 files.toStream(stdout);
2067
2068 return Void();
2069 }
2070
expireBackupData(const char * name,std::string destinationContainer,Version endVersion,std::string endDatetime,Database db,bool force,Version restorableAfterVersion,std::string restorableAfterDatetime)2071 ACTOR Future<Void> expireBackupData(const char *name, std::string destinationContainer, Version endVersion, std::string endDatetime, Database db, bool force, Version restorableAfterVersion, std::string restorableAfterDatetime) {
2072 if (!endDatetime.empty()) {
2073 Version v = wait( timeKeeperVersionFromDatetime(endDatetime, db) );
2074 endVersion = v;
2075 }
2076
2077 if (!restorableAfterDatetime.empty()) {
2078 Version v = wait( timeKeeperVersionFromDatetime(restorableAfterDatetime, db) );
2079 restorableAfterVersion = v;
2080 }
2081
2082 if (endVersion == invalidVersion) {
2083 fprintf(stderr, "ERROR: No version or date/time is specified.\n");
2084 printHelpTeaser(name);
2085 throw backup_error();;
2086 }
2087
2088 try {
2089 Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer);
2090
2091 state IBackupContainer::ExpireProgress progress;
2092 state std::string lastProgress;
2093 state Future<Void> expire = c->expireData(endVersion, force, &progress, restorableAfterVersion);
2094
2095 loop {
2096 choose {
2097 when(wait(delay(5))) {
2098 std::string p = progress.toString();
2099 if(p != lastProgress) {
2100 int spaces = lastProgress.size() - p.size();
2101 printf("\r%s%s", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "") );
2102 lastProgress = p;
2103 }
2104 }
2105 when(wait(expire)) {
2106 break;
2107 }
2108 }
2109 }
2110
2111 std::string p = progress.toString();
2112 int spaces = lastProgress.size() - p.size();
2113 printf("\r%s%s\n", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "") );
2114
2115 if(endVersion < 0)
2116 printf("All data before %lld versions (%lld days) prior to latest backup log has been deleted.\n", -endVersion, -endVersion / ((int64_t)24 * 3600 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
2117 else
2118 printf("All data before version %lld has been deleted.\n", endVersion);
2119 }
2120 catch (Error& e) {
2121 if(e.code() == error_code_actor_cancelled)
2122 throw;
2123 if(e.code() == error_code_backup_cannot_expire)
2124 fprintf(stderr, "ERROR: Requested expiration would be unsafe. Backup would not meet minimum restorability. Use --force to delete data anyway.\n");
2125 else
2126 fprintf(stderr, "ERROR: %s\n", e.what());
2127 throw;
2128 }
2129
2130 return Void();
2131 }
2132
deleteBackupContainer(const char * name,std::string destinationContainer)2133 ACTOR Future<Void> deleteBackupContainer(const char *name, std::string destinationContainer) {
2134 try {
2135 state Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer);
2136 state int numDeleted = 0;
2137 state Future<Void> done = c->deleteContainer(&numDeleted);
2138
2139 state int lastUpdate = -1;
2140 printf("Deleting %s...\n", destinationContainer.c_str());
2141
2142 loop {
2143 choose {
2144 when ( wait(done) ) {
2145 break;
2146 }
2147 when ( wait(delay(5)) ) {
2148 if(numDeleted != lastUpdate) {
2149 printf("\r%d...", numDeleted);
2150 lastUpdate = numDeleted;
2151 }
2152 }
2153 }
2154 }
2155 printf("\r%d objects deleted\n", numDeleted);
2156 printf("The entire container has been deleted.\n");
2157 }
2158 catch (Error& e) {
2159 if(e.code() == error_code_actor_cancelled)
2160 throw;
2161 fprintf(stderr, "ERROR: %s\n", e.what());
2162 throw;
2163 }
2164
2165 return Void();
2166 }
2167
describeBackup(const char * name,std::string destinationContainer,bool deep,Optional<Database> cx,bool json)2168 ACTOR Future<Void> describeBackup(const char *name, std::string destinationContainer, bool deep, Optional<Database> cx, bool json) {
2169 try {
2170 Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer);
2171 state BackupDescription desc = wait(c->describeBackup(deep));
2172 if(cx.present())
2173 wait(desc.resolveVersionTimes(cx.get()));
2174 printf("%s\n", (json ? desc.toJSON() : desc.toString()).c_str());
2175 }
2176 catch (Error& e) {
2177 if(e.code() == error_code_actor_cancelled)
2178 throw;
2179 fprintf(stderr, "ERROR: %s\n", e.what());
2180 throw;
2181 }
2182
2183 return Void();
2184 }
2185
listBackup(std::string baseUrl)2186 ACTOR Future<Void> listBackup(std::string baseUrl) {
2187 try {
2188 std::vector<std::string> containers = wait(IBackupContainer::listContainers(baseUrl));
2189 for (std::string container : containers) {
2190 printf("%s\n", container.c_str());
2191 }
2192 }
2193 catch (Error& e) {
2194 std::string msg = format("ERROR: %s", e.what());
2195 if(e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) {
2196 msg += format(": %s", IBackupContainer::lastOpenError.c_str());
2197 }
2198 fprintf(stderr, "%s\n", msg.c_str());
2199 throw;
2200 }
2201
2202 return Void();
2203 }
2204
2205 struct BackupModifyOptions {
2206 Optional<std::string> verifyUID;
2207 Optional<std::string> destURL;
2208 Optional<int> snapshotIntervalSeconds;
2209 Optional<int> activeSnapshotIntervalSeconds;
hasChangesBackupModifyOptions2210 bool hasChanges() const {
2211 return destURL.present() || snapshotIntervalSeconds.present() || activeSnapshotIntervalSeconds.present();
2212 }
2213 };
2214
modifyBackup(Database db,std::string tagName,BackupModifyOptions options)2215 ACTOR Future<Void> modifyBackup(Database db, std::string tagName, BackupModifyOptions options) {
2216 if(!options.hasChanges()) {
2217 fprintf(stderr, "No changes were specified, nothing to do!\n");
2218 throw backup_error();
2219 }
2220
2221 state KeyBackedTag tag = makeBackupTag(tagName);
2222
2223 state Reference<IBackupContainer> bc;
2224 if(options.destURL.present()) {
2225 bc = openBackupContainer(exeBackup.toString().c_str(), options.destURL.get());
2226 try {
2227 wait(timeoutError(bc->create(), 30));
2228 } catch(Error &e) {
2229 if(e.code() == error_code_actor_cancelled)
2230 throw;
2231 fprintf(stderr, "ERROR: Could not create backup container at '%s': %s\n", options.destURL.get().c_str(), e.what());
2232 throw backup_error();
2233 }
2234 }
2235
2236 state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(db));
2237 loop {
2238 try {
2239 tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
2240 tr->setOption(FDBTransactionOptions::LOCK_AWARE);
2241
2242 state Optional<UidAndAbortedFlagT> uidFlag = wait(tag.get(db));
2243
2244 if(!uidFlag.present()) {
2245 fprintf(stderr, "No backup exists on tag '%s'\n", tagName.c_str());
2246 throw backup_error();
2247 }
2248
2249 if(uidFlag.get().second) {
2250 fprintf(stderr, "Cannot modify aborted backup on tag '%s'\n", tagName.c_str());
2251 throw backup_error();
2252 }
2253
2254 state BackupConfig config(uidFlag.get().first);
2255 EBackupState s = wait(config.stateEnum().getOrThrow(tr, false, backup_invalid_info()));
2256 if(!FileBackupAgent::isRunnable(s)) {
2257 fprintf(stderr, "Backup on tag '%s' is not runnable.\n", tagName.c_str());
2258 throw backup_error();
2259 }
2260
2261 if(options.verifyUID.present() && options.verifyUID.get() != uidFlag.get().first.toString()) {
2262 fprintf(stderr, "UID verification failed, backup on tag '%s' is '%s' but '%s' was specified.\n", tagName.c_str(), uidFlag.get().first.toString().c_str(), options.verifyUID.get().c_str());
2263 throw backup_error();
2264 }
2265
2266 if(options.snapshotIntervalSeconds.present()) {
2267 config.snapshotIntervalSeconds().set(tr, options.snapshotIntervalSeconds.get());
2268 }
2269
2270 if(options.activeSnapshotIntervalSeconds.present()) {
2271 Version begin = wait(config.snapshotBeginVersion().getOrThrow(tr, false, backup_error()));
2272 config.snapshotTargetEndVersion().set(tr, begin + ((int64_t)options.activeSnapshotIntervalSeconds.get() * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
2273 }
2274
2275 if(options.destURL.present()) {
2276 config.backupContainer().set(tr, bc);
2277 }
2278
2279 wait(tr->commit());
2280 break;
2281 }
2282 catch (Error& e) {
2283 wait(tr->onError(e));
2284 }
2285 }
2286
2287 return Void();
2288 }
2289
parseLine(std::string & line,bool & err,bool & partial)2290 static std::vector<std::vector<StringRef>> parseLine(std::string &line, bool& err, bool& partial)
2291 {
2292 err = false;
2293 partial = false;
2294
2295 bool quoted = false;
2296 std::vector<StringRef> buf;
2297 std::vector<std::vector<StringRef>> ret;
2298
2299 size_t i = line.find_first_not_of(' ');
2300 size_t offset = i;
2301
2302 bool forcetoken = false;
2303
2304 while (i <= line.length()) {
2305 switch (line[i]) {
2306 case ';':
2307 if (!quoted) {
2308 if (i > offset)
2309 buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset));
2310 ret.push_back(std::move(buf));
2311 offset = i = line.find_first_not_of(' ', i + 1);
2312 }
2313 else
2314 i++;
2315 break;
2316 case '"':
2317 quoted = !quoted;
2318 line.erase(i, 1);
2319 if (quoted)
2320 forcetoken = true;
2321 break;
2322 case ' ':
2323 if (!quoted) {
2324 buf.push_back(StringRef((uint8_t *)(line.data() + offset),
2325 i - offset));
2326 offset = i = line.find_first_not_of(' ', i);
2327 forcetoken = false;
2328 }
2329 else
2330 i++;
2331 break;
2332 case '\\':
2333 if (i + 2 > line.length()) {
2334 err = true;
2335 ret.push_back(std::move(buf));
2336 return ret;
2337 }
2338 switch (line[i + 1]) {
2339 char ent, save;
2340 case '"':
2341 case '\\':
2342 case ' ':
2343 case ';':
2344 line.erase(i, 1);
2345 break;
2346 case 'x':
2347 if (i + 4 > line.length()) {
2348 err = true;
2349 ret.push_back(std::move(buf));
2350 return ret;
2351 }
2352 char *pEnd;
2353 save = line[i + 4];
2354 line[i + 4] = 0;
2355 ent = char(strtoul(line.data() + i + 2, &pEnd, 16));
2356 if (*pEnd) {
2357 err = true;
2358 ret.push_back(std::move(buf));
2359 return ret;
2360 }
2361 line[i + 4] = save;
2362 line.replace(i, 4, 1, ent);
2363 break;
2364 default:
2365 err = true;
2366 ret.push_back(std::move(buf));
2367 return ret;
2368 }
2369 default:
2370 i++;
2371 }
2372 }
2373
2374 i -= 1;
2375 if (i > offset || forcetoken)
2376 buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset));
2377
2378 ret.push_back(std::move(buf));
2379
2380 if (quoted)
2381 partial = true;
2382
2383 return ret;
2384 }
2385
addKeyRange(std::string optionValue,Standalone<VectorRef<KeyRangeRef>> & keyRanges)2386 static void addKeyRange(std::string optionValue, Standalone<VectorRef<KeyRangeRef>>& keyRanges)
2387 {
2388 bool err = false, partial = false;
2389 int tokenArray = 0, tokenIndex = 0;
2390
2391 auto parsed = parseLine(optionValue, err, partial);
2392
2393 for (auto tokens : parsed)
2394 {
2395 tokenArray++;
2396 tokenIndex = 0;
2397
2398 /*
2399 for (auto token : tokens)
2400 {
2401 tokenIndex++;
2402
2403 printf("%4d token #%2d: %s\n", tokenArray, tokenIndex, printable(token).c_str());
2404 }
2405 */
2406
2407 // Process the keys
2408 // <begin> [end]
2409 switch (tokens.size())
2410 {
2411 // empty
2412 case 0:
2413 break;
2414
2415 // single key range
2416 case 1:
2417 keyRanges.push_back_deep(keyRanges.arena(), KeyRangeRef(tokens.at(0), strinc(tokens.at(0))));
2418 break;
2419
2420 // full key range
2421 case 2:
2422 try {
2423 keyRanges.push_back_deep(keyRanges.arena(), KeyRangeRef(tokens.at(0), tokens.at(1)));
2424 }
2425 catch (Error& e) {
2426 fprintf(stderr, "ERROR: Invalid key range `%s %s' reported error %s\n",
2427 tokens.at(0).toString().c_str(), tokens.at(1).toString().c_str(), e.what());
2428 throw invalid_option_value();
2429 }
2430 break;
2431
2432 // Too many keys
2433 default:
2434 fprintf(stderr, "ERROR: Invalid key range identified with %ld keys", tokens.size());
2435 throw invalid_option_value();
2436 break;
2437 }
2438 }
2439
2440 return;
2441 }
2442
parseVersion(const char * str)2443 Version parseVersion(const char *str) {
2444 StringRef s((const uint8_t *)str, strlen(str));
2445
2446 if(s.endsWith(LiteralStringRef("days")) || s.endsWith(LiteralStringRef("d"))) {
2447 float days;
2448 if(sscanf(str, "%f", &days) != 1) {
2449 fprintf(stderr, "Could not parse version: %s\n", str);
2450 flushAndExit(FDB_EXIT_ERROR);
2451 }
2452 return (double)CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 3600 * -days;
2453 }
2454
2455 Version ver;
2456 if(sscanf(str, "%lld", &ver) != 1) {
2457 fprintf(stderr, "Could not parse version: %s\n", str);
2458 flushAndExit(FDB_EXIT_ERROR);
2459 }
2460 return ver;
2461 }
2462
2463 #ifdef ALLOC_INSTRUMENTATION
2464 extern uint8_t *g_extra_memory;
2465 #endif
2466
main(int argc,char * argv[])2467 int main(int argc, char* argv[]) {
2468 platformInit();
2469 initSignalSafeUnwind();
2470
2471 int status = FDB_EXIT_SUCCESS;
2472
2473 try {
2474 #ifdef ALLOC_INSTRUMENTATION
2475 g_extra_memory = new uint8_t[1000000];
2476 #endif
2477 registerCrashHandler();
2478
2479 // Set default of line buffering standard out and error
2480 setvbuf(stdout, NULL, _IONBF, 0);
2481 setvbuf(stderr, NULL, _IONBF, 0);
2482
2483 enumProgramExe programExe = getProgramType(argv[0]);
2484 enumBackupType backupType = BACKUP_UNDEFINED;
2485 enumRestoreType restoreType = RESTORE_UNKNOWN;
2486 enumDBType dbType = DB_UNDEFINED;
2487
2488 CSimpleOpt* args = NULL;
2489
2490 switch (programExe)
2491 {
2492 case EXE_AGENT:
2493 args = new CSimpleOpt(argc, argv, g_rgAgentOptions, SO_O_EXACT);
2494 break;
2495 case EXE_DR_AGENT:
2496 args = new CSimpleOpt(argc, argv, g_rgDBAgentOptions, SO_O_EXACT);
2497 break;
2498 case EXE_BACKUP:
2499 // Display backup help, if no arguments
2500 if (argc < 2) {
2501 printBackupUsage(false);
2502 return FDB_EXIT_ERROR;
2503 }
2504 else {
2505 // Get the backup type
2506 backupType = getBackupType(argv[1]);
2507
2508 // Create the appropriate simple opt
2509 switch (backupType)
2510 {
2511 case BACKUP_START:
2512 args = new CSimpleOpt(argc-1, &argv[1], g_rgBackupStartOptions, SO_O_EXACT);
2513 break;
2514 case BACKUP_STATUS:
2515 args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupStatusOptions, SO_O_EXACT);
2516 break;
2517 case BACKUP_ABORT:
2518 args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupAbortOptions, SO_O_EXACT);
2519 break;
2520 case BACKUP_WAIT:
2521 args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupWaitOptions, SO_O_EXACT);
2522 break;
2523 case BACKUP_DISCONTINUE:
2524 args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupDiscontinueOptions, SO_O_EXACT);
2525 break;
2526 case BACKUP_PAUSE:
2527 args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupPauseOptions, SO_O_EXACT);
2528 break;
2529 case BACKUP_RESUME:
2530 args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupPauseOptions, SO_O_EXACT);
2531 break;
2532 case BACKUP_EXPIRE:
2533 args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupExpireOptions, SO_O_EXACT);
2534 break;
2535 case BACKUP_DELETE:
2536 args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupDeleteOptions, SO_O_EXACT);
2537 break;
2538 case BACKUP_DESCRIBE:
2539 args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupDescribeOptions, SO_O_EXACT);
2540 break;
2541 case BACKUP_DUMP:
2542 args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupDumpOptions, SO_O_EXACT);
2543 break;
2544 case BACKUP_LIST:
2545 args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupListOptions, SO_O_EXACT);
2546 break;
2547 case BACKUP_MODIFY:
2548 args = new CSimpleOpt(argc - 1, &argv[1], g_rgBackupModifyOptions, SO_O_EXACT);
2549 break;
2550 case BACKUP_UNDEFINED:
2551 default:
2552 // Display help, if requested
2553 if ((strcmp(argv[1], "-h") == 0) ||
2554 (strcmp(argv[1], "--help") == 0) )
2555 {
2556 printBackupUsage(false);
2557 return FDB_EXIT_ERROR;
2558 }
2559 else {
2560 fprintf(stderr, "ERROR: Unsupported backup action %s\n", argv[1]);
2561 printHelpTeaser(argv[0]);
2562 return FDB_EXIT_ERROR;
2563 }
2564 break;
2565 }
2566 }
2567 break;
2568 case EXE_DB_BACKUP:
2569 // Display backup help, if no arguments
2570 if (argc < 2) {
2571 printDBBackupUsage(false);
2572 return FDB_EXIT_ERROR;
2573 }
2574 else {
2575 // Get the backup type
2576 dbType = getDBType(argv[1]);
2577
2578 // Create the appropriate simple opt
2579 switch (dbType)
2580 {
2581 case DB_START:
2582 args = new CSimpleOpt(argc-1, &argv[1], g_rgDBStartOptions, SO_O_EXACT);
2583 break;
2584 case DB_STATUS:
2585 args = new CSimpleOpt(argc - 1, &argv[1], g_rgDBStatusOptions, SO_O_EXACT);
2586 break;
2587 case DB_SWITCH:
2588 args = new CSimpleOpt(argc - 1, &argv[1], g_rgDBSwitchOptions, SO_O_EXACT);
2589 break;
2590 case DB_ABORT:
2591 args = new CSimpleOpt(argc - 1, &argv[1], g_rgDBAbortOptions, SO_O_EXACT);
2592 break;
2593 case DB_PAUSE:
2594 args = new CSimpleOpt(argc - 1, &argv[1], g_rgDBPauseOptions, SO_O_EXACT);
2595 break;
2596 case DB_RESUME:
2597 args = new CSimpleOpt(argc - 1, &argv[1], g_rgDBPauseOptions, SO_O_EXACT);
2598 break;
2599 case DB_UNDEFINED:
2600 default:
2601 // Display help, if requested
2602 if ((strcmp(argv[1], "-h") == 0) ||
2603 (strcmp(argv[1], "--help") == 0) )
2604 {
2605 printDBBackupUsage(false);
2606 return FDB_EXIT_ERROR;
2607 }
2608 else {
2609 fprintf(stderr, "ERROR: Unsupported dr action %s %d\n", argv[1], dbType);
2610 printHelpTeaser(argv[0]);
2611 return FDB_EXIT_ERROR;
2612 }
2613 break;
2614 }
2615 }
2616 break;
2617 case EXE_RESTORE:
2618 if (argc < 2) {
2619 printRestoreUsage(false);
2620 return FDB_EXIT_ERROR;
2621 }
2622 // Get the restore operation type
2623 restoreType = getRestoreType(argv[1]);
2624 if(restoreType == RESTORE_UNKNOWN) {
2625 // Display help, if requested
2626 if ((strcmp(argv[1], "-h") == 0) ||
2627 (strcmp(argv[1], "--help") == 0) )
2628 {
2629 printRestoreUsage(false);
2630 return FDB_EXIT_ERROR;
2631 }
2632 else {
2633 fprintf(stderr, "ERROR: Unsupported restore command: '%s'\n", argv[1]);
2634 printHelpTeaser(argv[0]);
2635 return FDB_EXIT_ERROR;
2636 }
2637 }
2638 args = new CSimpleOpt(argc - 1, argv + 1, g_rgRestoreOptions, SO_O_EXACT);
2639 break;
2640 case EXE_UNDEFINED:
2641 default:
2642 fprintf(stderr, "FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
2643 fprintf(stderr, "ERROR: Unable to determine program type based on executable `%s'\n", argv[0]);
2644 return FDB_EXIT_ERROR;
2645 break;
2646 }
2647
2648 std::string destinationContainer;
2649 bool describeDeep = false;
2650 bool describeTimestamps = false;
2651 int snapshotIntervalSeconds = CLIENT_KNOBS->BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC;
2652 std::string clusterFile;
2653 std::string sourceClusterFile;
2654 std::string baseUrl;
2655 std::string expireDatetime;
2656 Version expireVersion = invalidVersion;
2657 std::string expireRestorableAfterDatetime;
2658 Version expireRestorableAfterVersion = std::numeric_limits<Version>::max();
2659 std::vector<std::pair<std::string, std::string>> knobs;
2660 std::string tagName = BackupAgentBase::getDefaultTag().toString();
2661 bool tagProvided = false;
2662 std::string restoreContainer;
2663 std::string addPrefix;
2664 std::string removePrefix;
2665 Standalone<VectorRef<KeyRangeRef>> backupKeys;
2666 int maxErrors = 20;
2667 Version restoreVersion = invalidVersion;
2668 std::string restoreTimestamp;
2669 bool waitForDone = false;
2670 bool stopWhenDone = true;
2671 bool forceAction = false;
2672 bool trace = false;
2673 bool quietDisplay = false;
2674 bool dryRun = false;
2675 std::string traceDir = "";
2676 std::string traceFormat = "";
2677 std::string traceLogGroup;
2678 uint64_t traceRollSize = TRACE_DEFAULT_ROLL_SIZE;
2679 uint64_t traceMaxLogsSize = TRACE_DEFAULT_MAX_LOGS_SIZE;
2680 ESOError lastError;
2681 bool partial = true;
2682 LocalityData localities;
2683 uint64_t memLimit = 8LL << 30;
2684 Optional<uint64_t> ti;
2685 std::vector<std::string> blobCredentials;
2686 std::string tlsCertPath, tlsKeyPath, tlsCAPath, tlsPassword, tlsVerifyPeers;
2687 Version dumpBegin = 0;
2688 Version dumpEnd = std::numeric_limits<Version>::max();
2689 std::string restoreClusterFileDest;
2690 std::string restoreClusterFileOrig;
2691 bool jsonOutput = false;
2692
2693 BackupModifyOptions modifyOptions;
2694
2695 if( argc == 1 ) {
2696 printUsage(programExe, false);
2697 return FDB_EXIT_ERROR;
2698 }
2699
2700 #ifdef _WIN32
2701 // Windows needs a gentle nudge to format floats correctly
2702 //_set_output_format(_TWO_DIGIT_EXPONENT);
2703 #endif
2704
2705 while (args->Next()) {
2706 lastError = args->LastError();
2707
2708 switch (lastError)
2709 {
2710 case SO_SUCCESS:
2711 break;
2712
2713 case SO_ARG_INVALID_DATA:
2714 fprintf(stderr, "ERROR: invalid argument to option `%s'\n", args->OptionText());
2715 printHelpTeaser(argv[0]);
2716 return FDB_EXIT_ERROR;
2717 break;
2718
2719 case SO_ARG_INVALID:
2720 fprintf(stderr, "ERROR: argument given for option `%s'\n", args->OptionText());
2721 printHelpTeaser(argv[0]);
2722 return FDB_EXIT_ERROR;
2723 break;
2724
2725 case SO_ARG_MISSING:
2726 fprintf(stderr, "ERROR: missing argument for option `%s'\n", args->OptionText());
2727 printHelpTeaser(argv[0]);
2728 return FDB_EXIT_ERROR;
2729
2730 case SO_OPT_INVALID:
2731 fprintf(stderr, "ERROR: unknown option `%s'\n", args->OptionText());
2732 printHelpTeaser(argv[0]);
2733 return FDB_EXIT_ERROR;
2734 break;
2735
2736 default:
2737 fprintf(stderr, "ERROR: argument given for option `%s'\n", args->OptionText());
2738 printHelpTeaser(argv[0]);
2739 return FDB_EXIT_ERROR;
2740 break;
2741 }
2742
2743 int optId = args->OptionId();
2744 switch (optId) {
2745 case OPT_HELP:
2746 printUsage(programExe, false);
2747 return FDB_EXIT_SUCCESS;
2748 break;
2749 case OPT_DEVHELP:
2750 printUsage(programExe, true);
2751 return FDB_EXIT_SUCCESS;
2752 break;
2753 case OPT_VERSION:
2754 printVersion();
2755 return FDB_EXIT_SUCCESS;
2756 break;
2757 case OPT_NOBUFSTDOUT:
2758 setvbuf(stdout, NULL, _IONBF, 0);
2759 setvbuf(stderr, NULL, _IONBF, 0);
2760 break;
2761 case OPT_BUFSTDOUTERR:
2762 setvbuf(stdout, NULL, _IOFBF, BUFSIZ);
2763 setvbuf(stderr, NULL, _IOFBF, BUFSIZ);
2764 break;
2765 case OPT_QUIET:
2766 quietDisplay = true;
2767 break;
2768 case OPT_DRYRUN:
2769 dryRun = true;
2770 break;
2771 case OPT_FORCE:
2772 forceAction = true;
2773 break;
2774 case OPT_TRACE:
2775 trace = true;
2776 break;
2777 case OPT_TRACE_DIR:
2778 trace = true;
2779 traceDir = args->OptionArg();
2780 break;
2781 case OPT_TRACE_FORMAT:
2782 if (!validateTraceFormat(args->OptionArg())) {
2783 fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args->OptionArg());
2784 }
2785 traceFormat = args->OptionArg();
2786 break;
2787 case OPT_TRACE_LOG_GROUP:
2788 traceLogGroup = args->OptionArg();
2789 break;
2790 case OPT_LOCALITY: {
2791 std::string syn = args->OptionSyntax();
2792 if (!StringRef(syn).startsWith(LiteralStringRef("--locality_"))) {
2793 fprintf(stderr, "ERROR: unable to parse locality key '%s'\n", syn.c_str());
2794 return FDB_EXIT_ERROR;
2795 }
2796 syn = syn.substr(11);
2797 std::transform(syn.begin(), syn.end(), syn.begin(), ::tolower);
2798 localities.set(Standalone<StringRef>(syn), Standalone<StringRef>(std::string(args->OptionArg())));
2799 break;
2800 }
2801 case OPT_EXPIRE_BEFORE_DATETIME:
2802 expireDatetime = args->OptionArg();
2803 break;
2804 case OPT_EXPIRE_RESTORABLE_AFTER_DATETIME:
2805 expireRestorableAfterDatetime = args->OptionArg();
2806 break;
2807 case OPT_EXPIRE_BEFORE_VERSION:
2808 case OPT_EXPIRE_RESTORABLE_AFTER_VERSION:
2809 case OPT_EXPIRE_MIN_RESTORABLE_DAYS:
2810 case OPT_EXPIRE_DELETE_BEFORE_DAYS:
2811 {
2812 const char* a = args->OptionArg();
2813 long long ver = 0;
2814 if (!sscanf(a, "%lld", &ver)) {
2815 fprintf(stderr, "ERROR: Could not parse expiration version `%s'\n", a);
2816 printHelpTeaser(argv[0]);
2817 return FDB_EXIT_ERROR;
2818 }
2819
2820 // Interpret the value as days worth of versions relative to now (negative)
2821 if(optId == OPT_EXPIRE_MIN_RESTORABLE_DAYS || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) {
2822 ver = -ver * 24 * 60 * 60 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND;
2823 }
2824
2825 if(optId == OPT_EXPIRE_BEFORE_VERSION || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS)
2826 expireVersion = ver;
2827 else
2828 expireRestorableAfterVersion = ver;
2829 break;
2830 }
2831 case OPT_RESTORE_TIMESTAMP:
2832 restoreTimestamp = args->OptionArg();
2833 break;
2834 case OPT_BASEURL:
2835 baseUrl = args->OptionArg();
2836 break;
2837 case OPT_RESTORE_CLUSTERFILE_DEST:
2838 restoreClusterFileDest = args->OptionArg();
2839 break;
2840 case OPT_RESTORE_CLUSTERFILE_ORIG:
2841 restoreClusterFileOrig = args->OptionArg();
2842 break;
2843 case OPT_CLUSTERFILE:
2844 clusterFile = args->OptionArg();
2845 break;
2846 case OPT_DEST_CLUSTER:
2847 clusterFile = args->OptionArg();
2848 break;
2849 case OPT_SOURCE_CLUSTER:
2850 sourceClusterFile = args->OptionArg();
2851 break;
2852 case OPT_CLEANUP:
2853 partial = false;
2854 break;
2855 case OPT_KNOB: {
2856 std::string syn = args->OptionSyntax();
2857 if (!StringRef(syn).startsWith(LiteralStringRef("--knob_"))) {
2858 fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str());
2859 return FDB_EXIT_ERROR;
2860 }
2861 syn = syn.substr(7);
2862 knobs.push_back( std::make_pair( syn, args->OptionArg() ) );
2863 break;
2864 }
2865 case OPT_BACKUPKEYS:
2866 try {
2867 addKeyRange(args->OptionArg(), backupKeys);
2868 }
2869 catch (Error &) {
2870 printHelpTeaser(argv[0]);
2871 return FDB_EXIT_ERROR;
2872 }
2873 break;
2874 case OPT_DESTCONTAINER:
2875 destinationContainer = args->OptionArg();
2876 // If the url starts with '/' then prepend "file://" for backwards compatibility
2877 if(StringRef(destinationContainer).startsWith(LiteralStringRef("/")))
2878 destinationContainer = std::string("file://") + destinationContainer;
2879 modifyOptions.destURL = destinationContainer;
2880 break;
2881 case OPT_SNAPSHOTINTERVAL:
2882 case OPT_MOD_ACTIVE_INTERVAL:
2883 {
2884 const char* a = args->OptionArg();
2885 int seconds;
2886 if (!sscanf(a, "%d", &seconds)) {
2887 fprintf(stderr, "ERROR: Could not parse snapshot interval `%s'\n", a);
2888 printHelpTeaser(argv[0]);
2889 return FDB_EXIT_ERROR;
2890 }
2891 if(optId == OPT_SNAPSHOTINTERVAL) {
2892 snapshotIntervalSeconds = seconds;
2893 modifyOptions.snapshotIntervalSeconds = seconds;
2894 }
2895 else if(optId == OPT_MOD_ACTIVE_INTERVAL) {
2896 modifyOptions.activeSnapshotIntervalSeconds = seconds;
2897 }
2898 break;
2899 }
2900 case OPT_MOD_VERIFY_UID:
2901 modifyOptions.verifyUID = args->OptionArg();
2902 break;
2903 case OPT_WAITFORDONE:
2904 waitForDone = true;
2905 break;
2906 case OPT_NOSTOPWHENDONE:
2907 stopWhenDone = false;
2908 break;
2909 case OPT_RESTORECONTAINER:
2910 restoreContainer = args->OptionArg();
2911 // If the url starts with '/' then prepend "file://" for backwards compatibility
2912 if(StringRef(restoreContainer).startsWith(LiteralStringRef("/")))
2913 restoreContainer = std::string("file://") + restoreContainer;
2914 break;
2915 case OPT_DESCRIBE_DEEP:
2916 describeDeep = true;
2917 break;
2918 case OPT_DESCRIBE_TIMESTAMPS:
2919 describeTimestamps = true;
2920 break;
2921 case OPT_PREFIX_ADD:
2922 addPrefix = args->OptionArg();
2923 break;
2924 case OPT_PREFIX_REMOVE:
2925 removePrefix = args->OptionArg();
2926 break;
2927 case OPT_ERRORLIMIT: {
2928 const char* a = args->OptionArg();
2929 if (!sscanf(a, "%d", &maxErrors)) {
2930 fprintf(stderr, "ERROR: Could not parse max number of errors `%s'\n", a);
2931 printHelpTeaser(argv[0]);
2932 return FDB_EXIT_ERROR;
2933 }
2934 break;
2935 }
2936 case OPT_RESTORE_VERSION: {
2937 const char* a = args->OptionArg();
2938 long long ver = 0;
2939 if (!sscanf(a, "%lld", &ver)) {
2940 fprintf(stderr, "ERROR: Could not parse database version `%s'\n", a);
2941 printHelpTeaser(argv[0]);
2942 return FDB_EXIT_ERROR;
2943 }
2944 restoreVersion = ver;
2945 break;
2946 }
2947 #ifdef _WIN32
2948 case OPT_PARENTPID: {
2949 auto pid_str = args->OptionArg();
2950 int parent_pid = atoi(pid_str);
2951 auto pHandle = OpenProcess( SYNCHRONIZE, FALSE, parent_pid );
2952 if( !pHandle ) {
2953 TraceEvent("ParentProcessOpenError").GetLastError();
2954 fprintf(stderr, "Could not open parent process at pid %d (error %d)", parent_pid, GetLastError());
2955 throw platform_error();
2956 }
2957 startThread(&parentWatcher, pHandle);
2958 break;
2959 }
2960 #endif
2961 case OPT_TAGNAME:
2962 tagName = args->OptionArg();
2963 tagProvided = true;
2964 break;
2965 case OPT_CRASHONERROR:
2966 g_crashOnError = true;
2967 break;
2968 case OPT_MEMLIMIT:
2969 ti = parse_with_suffix(args->OptionArg(), "MiB");
2970 if (!ti.present()) {
2971 fprintf(stderr, "ERROR: Could not parse memory limit from `%s'\n", args->OptionArg());
2972 printHelpTeaser(argv[0]);
2973 flushAndExit(FDB_EXIT_ERROR);
2974 }
2975 memLimit = ti.get();
2976 break;
2977 case OPT_BLOB_CREDENTIALS:
2978 blobCredentials.push_back(args->OptionArg());
2979 break;
2980 #ifndef TLS_DISABLED
2981 case TLSOptions::OPT_TLS_PLUGIN:
2982 args->OptionArg();
2983 break;
2984 case TLSOptions::OPT_TLS_CERTIFICATES:
2985 tlsCertPath = args->OptionArg();
2986 break;
2987 case TLSOptions::OPT_TLS_PASSWORD:
2988 tlsPassword = args->OptionArg();
2989 break;
2990 case TLSOptions::OPT_TLS_CA_FILE:
2991 tlsCAPath = args->OptionArg();
2992 break;
2993 case TLSOptions::OPT_TLS_KEY:
2994 tlsKeyPath = args->OptionArg();
2995 break;
2996 case TLSOptions::OPT_TLS_VERIFY_PEERS:
2997 tlsVerifyPeers = args->OptionArg();
2998 break;
2999 #endif
3000 case OPT_DUMP_BEGIN:
3001 dumpBegin = parseVersion(args->OptionArg());
3002 break;
3003 case OPT_DUMP_END:
3004 dumpEnd = parseVersion(args->OptionArg());
3005 break;
3006 case OPT_JSON:
3007 jsonOutput = true;
3008 break;
3009 }
3010 }
3011
3012 // Process the extra arguments
3013 for (int argLoop = 0; argLoop < args->FileCount(); argLoop++)
3014 {
3015 switch (programExe)
3016 {
3017 case EXE_AGENT:
3018 fprintf(stderr, "ERROR: Backup Agent does not support argument value `%s'\n", args->File(argLoop));
3019 printHelpTeaser(argv[0]);
3020 return FDB_EXIT_ERROR;
3021 break;
3022
3023 // Add the backup key range
3024 case EXE_BACKUP:
3025 // Error, if the keys option was not specified
3026 if (backupKeys.size() == 0) {
3027 fprintf(stderr, "ERROR: Unknown backup option value `%s'\n", args->File(argLoop));
3028 printHelpTeaser(argv[0]);
3029 return FDB_EXIT_ERROR;
3030 }
3031 // Otherwise, assume the item is a key range
3032 else {
3033 try {
3034 addKeyRange(args->File(argLoop), backupKeys);
3035 }
3036 catch (Error& ) {
3037 printHelpTeaser(argv[0]);
3038 return FDB_EXIT_ERROR;
3039 }
3040 }
3041 break;
3042
3043 case EXE_RESTORE:
3044 fprintf(stderr, "ERROR: FDB Restore does not support argument value `%s'\n", args->File(argLoop));
3045 printHelpTeaser(argv[0]);
3046 return FDB_EXIT_ERROR;
3047 break;
3048
3049 case EXE_DR_AGENT:
3050 fprintf(stderr, "ERROR: DR Agent does not support argument value `%s'\n", args->File(argLoop));
3051 printHelpTeaser(argv[0]);
3052 return FDB_EXIT_ERROR;
3053 break;
3054
3055 case EXE_DB_BACKUP:
3056 // Error, if the keys option was not specified
3057 if (backupKeys.size() == 0) {
3058 fprintf(stderr, "ERROR: Unknown DR option value `%s'\n", args->File(argLoop));
3059 printHelpTeaser(argv[0]);
3060 return FDB_EXIT_ERROR;
3061 }
3062 // Otherwise, assume the item is a key range
3063 else {
3064 try {
3065 addKeyRange(args->File(argLoop), backupKeys);
3066 }
3067 catch (Error& ) {
3068 printHelpTeaser(argv[0]);
3069 return FDB_EXIT_ERROR;
3070 }
3071 }
3072 break;
3073
3074 case EXE_UNDEFINED:
3075 default:
3076 return FDB_EXIT_ERROR;
3077 }
3078 }
3079
3080 // Delete the simple option object, if defined
3081 if (args)
3082 {
3083 delete args;
3084 args = NULL;
3085 }
3086
3087 std::string commandLine;
3088 for(int a=0; a<argc; a++) {
3089 if (a) commandLine += ' ';
3090 commandLine += argv[a];
3091 }
3092
3093 delete FLOW_KNOBS;
3094 FlowKnobs* flowKnobs = new FlowKnobs(true);
3095 FLOW_KNOBS = flowKnobs;
3096
3097 delete CLIENT_KNOBS;
3098 ClientKnobs* clientKnobs = new ClientKnobs(true);
3099 CLIENT_KNOBS = clientKnobs;
3100
3101 for(auto k=knobs.begin(); k!=knobs.end(); ++k) {
3102 try {
3103 if (!flowKnobs->setKnob( k->first, k->second ) &&
3104 !clientKnobs->setKnob( k->first, k->second ))
3105 {
3106 fprintf(stderr, "Unrecognized knob option '%s'\n", k->first.c_str());
3107 return FDB_EXIT_ERROR;
3108 }
3109 } catch (Error& e) {
3110 if (e.code() == error_code_invalid_option_value) {
3111 fprintf(stderr, "Invalid value '%s' for option '%s'\n", k->second.c_str(), k->first.c_str());
3112 return FDB_EXIT_ERROR;
3113 }
3114 throw;
3115 }
3116 }
3117
3118 if (trace) {
3119 if(!traceLogGroup.empty())
3120 setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(traceLogGroup));
3121
3122 if (traceDir.empty())
3123 setNetworkOption(FDBNetworkOptions::TRACE_ENABLE);
3124 else
3125 setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(traceDir));
3126 if (!traceFormat.empty()) {
3127 setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(traceFormat));
3128 }
3129
3130 setNetworkOption(FDBNetworkOptions::ENABLE_SLOW_TASK_PROFILING);
3131 }
3132 setNetworkOption(FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING);
3133
3134 // deferred TLS options
3135 if (tlsCertPath.size()) {
3136 try {
3137 setNetworkOption(FDBNetworkOptions::TLS_CERT_PATH, tlsCertPath);
3138 }
3139 catch (Error& e) {
3140 fprintf(stderr, "ERROR: cannot set TLS certificate path to `%s' (%s)\n", tlsCertPath.c_str(), e.what());
3141 return 1;
3142 }
3143 }
3144
3145 if (tlsCAPath.size()) {
3146 try {
3147 setNetworkOption(FDBNetworkOptions::TLS_CA_PATH, tlsCAPath);
3148 }
3149 catch (Error& e) {
3150 fprintf(stderr, "ERROR: cannot set TLS CA path to `%s' (%s)\n", tlsCAPath.c_str(), e.what());
3151 return 1;
3152 }
3153 }
3154 if (tlsKeyPath.size()) {
3155 try {
3156 if (tlsPassword.size())
3157 setNetworkOption(FDBNetworkOptions::TLS_PASSWORD, tlsPassword);
3158
3159 setNetworkOption(FDBNetworkOptions::TLS_KEY_PATH, tlsKeyPath);
3160 }
3161 catch (Error& e) {
3162 fprintf(stderr, "ERROR: cannot set TLS key path to `%s' (%s)\n", tlsKeyPath.c_str(), e.what());
3163 return 1;
3164 }
3165 }
3166 if (tlsVerifyPeers.size()) {
3167 try {
3168 setNetworkOption(FDBNetworkOptions::TLS_VERIFY_PEERS, tlsVerifyPeers);
3169 }
3170 catch (Error& e) {
3171 fprintf(stderr, "ERROR: cannot set TLS peer verification to `%s' (%s)\n", tlsVerifyPeers.c_str(), e.what());
3172 return 1;
3173 }
3174 }
3175
3176 Error::init();
3177 std::set_new_handler( &platform::outOfMemory );
3178 setMemoryQuota( memLimit );
3179
3180 int total = 0;
3181 for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i)
3182 total += i->second;
3183 if (total)
3184 printf("%d errors:\n", total);
3185 for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i)
3186 if (i->second > 0)
3187 printf(" %d: %d %s\n", i->second, i->first, Error::fromCode(i->first).what());
3188
3189
3190 Reference<ClusterConnectionFile> ccf;
3191 Database db;
3192 Reference<ClusterConnectionFile> sourceCcf;
3193 Database sourceDb;
3194 FileBackupAgent ba;
3195 Key tag;
3196 Future<Optional<Void>> f;
3197 Future<Optional<int>> fstatus;
3198 Reference<IBackupContainer> c;
3199
3200 try {
3201 setupNetwork(0, true);
3202 }
3203 catch (Error& e) {
3204 fprintf(stderr, "ERROR: %s\n", e.what());
3205 return FDB_EXIT_ERROR;
3206 }
3207
3208 TraceEvent("ProgramStart")
3209 .detail("SourceVersion", getHGVersion())
3210 .detail("Version", FDB_VT_VERSION )
3211 .detail("PackageName", FDB_VT_PACKAGE_NAME)
3212 .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL))
3213 .detail("CommandLine", commandLine)
3214 .detail("MemoryLimit", memLimit)
3215 .trackLatest("ProgramStart");
3216
3217 // Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents are logged. This thread will eventually run the network, so call it now.
3218 TraceEvent::setNetworkThread();
3219
3220 // Add blob credentials files from the environment to the list collected from the command line.
3221 const char *blobCredsFromENV = getenv("FDB_BLOB_CREDENTIALS");
3222 if(blobCredsFromENV != nullptr) {
3223 StringRef t((uint8_t*)blobCredsFromENV, strlen(blobCredsFromENV));
3224 do {
3225 StringRef file = t.eat(":");
3226 if(file.size() != 0)
3227 blobCredentials.push_back(file.toString());
3228 } while(t.size() != 0);
3229 }
3230
3231 // Update the global blob credential files list
3232 std::vector<std::string> *pFiles = (std::vector<std::string> *)g_network->global(INetwork::enBlobCredentialFiles);
3233 if(pFiles != nullptr) {
3234 for(auto &f : blobCredentials) {
3235 pFiles->push_back(f);
3236 }
3237 }
3238
3239 // Opens a trace file if trace is set (and if a trace file isn't already open)
3240 // For most modes, initCluster() will open a trace file, but some fdbbackup operations do not require
3241 // a cluster so they should use this instead.
3242 auto initTraceFile = [&]() {
3243 if(trace)
3244 openTraceFile(NetworkAddress(), traceRollSize, traceMaxLogsSize, traceDir, "trace", traceLogGroup);
3245 };
3246
3247 auto initCluster = [&](bool quiet = false) {
3248 auto resolvedClusterFile = ClusterConnectionFile::lookupClusterFileName(clusterFile);
3249 try {
3250 ccf = Reference<ClusterConnectionFile>(new ClusterConnectionFile(resolvedClusterFile.first));
3251 }
3252 catch (Error& e) {
3253 if(!quiet)
3254 fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedClusterFile, e).c_str());
3255 return false;
3256 }
3257
3258 try {
3259 db = Database::createDatabase(ccf, -1, localities);
3260 }
3261 catch (Error& e) {
3262 fprintf(stderr, "ERROR: %s\n", e.what());
3263 fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", ccf->getFilename().c_str());
3264 return false;
3265 }
3266
3267 return true;
3268 };
3269
3270 if(sourceClusterFile.size()) {
3271 auto resolvedSourceClusterFile = ClusterConnectionFile::lookupClusterFileName(sourceClusterFile);
3272 try {
3273 sourceCcf = Reference<ClusterConnectionFile>(new ClusterConnectionFile(resolvedSourceClusterFile.first));
3274 }
3275 catch (Error& e) {
3276 fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedSourceClusterFile, e).c_str());
3277 return FDB_EXIT_ERROR;
3278 }
3279
3280 try {
3281 sourceDb = Database::createDatabase(sourceCcf, -1, localities);
3282 }
3283 catch (Error& e) {
3284 fprintf(stderr, "ERROR: %s\n", e.what());
3285 fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", sourceCcf->getFilename().c_str());
3286 return FDB_EXIT_ERROR;
3287 }
3288 }
3289
3290 switch (programExe)
3291 {
3292 case EXE_AGENT:
3293 if(!initCluster())
3294 return FDB_EXIT_ERROR;
3295 f = stopAfter(runAgent(db));
3296 break;
3297 case EXE_BACKUP:
3298 switch (backupType)
3299 {
3300 case BACKUP_START:
3301 {
3302 if(!initCluster())
3303 return FDB_EXIT_ERROR;
3304 // Test out the backup url to make sure it parses. Doesn't test to make sure it's actually writeable.
3305 openBackupContainer(argv[0], destinationContainer);
3306 f = stopAfter( submitBackup(db, destinationContainer, snapshotIntervalSeconds, backupKeys, tagName, dryRun, waitForDone, stopWhenDone) );
3307 break;
3308 }
3309
3310 case BACKUP_MODIFY:
3311 {
3312 if(!initCluster())
3313 return FDB_EXIT_ERROR;
3314
3315 f = stopAfter( modifyBackup(db, tagName, modifyOptions) );
3316 break;
3317 }
3318
3319 case BACKUP_STATUS:
3320 if(!initCluster())
3321 return FDB_EXIT_ERROR;
3322 f = stopAfter( statusBackup(db, tagName, true, jsonOutput) );
3323 break;
3324
3325 case BACKUP_ABORT:
3326 if(!initCluster())
3327 return FDB_EXIT_ERROR;
3328 f = stopAfter( abortBackup(db, tagName) );
3329 break;
3330
3331 case BACKUP_WAIT:
3332 if(!initCluster())
3333 return FDB_EXIT_ERROR;
3334 f = stopAfter( waitBackup(db, tagName, stopWhenDone) );
3335 break;
3336
3337 case BACKUP_DISCONTINUE:
3338 if(!initCluster())
3339 return FDB_EXIT_ERROR;
3340 f = stopAfter( discontinueBackup(db, tagName, waitForDone) );
3341 break;
3342
3343 case BACKUP_PAUSE:
3344 if(!initCluster())
3345 return FDB_EXIT_ERROR;
3346 f = stopAfter( changeBackupResumed(db, true) );
3347 break;
3348
3349 case BACKUP_RESUME:
3350 if(!initCluster())
3351 return FDB_EXIT_ERROR;
3352 f = stopAfter( changeBackupResumed(db, false) );
3353 break;
3354
3355 case BACKUP_EXPIRE:
3356 initTraceFile();
3357 // Must have a usable cluster if either expire DateTime options were used
3358 if(!expireDatetime.empty() || !expireRestorableAfterDatetime.empty()) {
3359 if(!initCluster())
3360 return FDB_EXIT_ERROR;
3361 }
3362 f = stopAfter( expireBackupData(argv[0], destinationContainer, expireVersion, expireDatetime, db, forceAction, expireRestorableAfterVersion, expireRestorableAfterDatetime) );
3363 break;
3364
3365 case BACKUP_DELETE:
3366 initTraceFile();
3367 f = stopAfter( deleteBackupContainer(argv[0], destinationContainer) );
3368 break;
3369
3370 case BACKUP_DESCRIBE:
3371 initTraceFile();
3372 // If timestamp lookups are desired, require a cluster file
3373 if(describeTimestamps && !initCluster())
3374 return FDB_EXIT_ERROR;
3375
3376 // Only pass database optionDatabase Describe will lookup version timestamps if a cluster file was given, but quietly skip them if not.
3377 f = stopAfter( describeBackup(argv[0], destinationContainer, describeDeep, describeTimestamps ? Optional<Database>(db) : Optional<Database>(), jsonOutput) );
3378 break;
3379
3380 case BACKUP_LIST:
3381 initTraceFile();
3382 f = stopAfter( listBackup(baseUrl) );
3383 break;
3384
3385 case BACKUP_DUMP:
3386 initTraceFile();
3387 f = stopAfter( dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd) );
3388 break;
3389
3390 case BACKUP_UNDEFINED:
3391 default:
3392 fprintf(stderr, "ERROR: Unsupported backup action %s\n", argv[1]);
3393 printHelpTeaser(argv[0]);
3394 return FDB_EXIT_ERROR;
3395 break;
3396 }
3397
3398 break;
3399 case EXE_RESTORE:
3400 if(dryRun) {
3401 initTraceFile();
3402 }
3403 else if(restoreType != RESTORE_START && !initCluster()) {
3404 return FDB_EXIT_ERROR;
3405 }
3406
3407 switch(restoreType) {
3408 case RESTORE_START:
3409 f = stopAfter( runRestore(restoreClusterFileDest, restoreClusterFileOrig, tagName, restoreContainer, backupKeys, restoreVersion, restoreTimestamp, !dryRun, !quietDisplay, waitForDone, addPrefix, removePrefix) );
3410 break;
3411 case RESTORE_WAIT:
3412 f = stopAfter( success(ba.waitRestore(db, KeyRef(tagName), true)) );
3413 break;
3414 case RESTORE_ABORT:
3415 f = stopAfter( map(ba.abortRestore(db, KeyRef(tagName)), [tagName](FileBackupAgent::ERestoreState s) -> Void {
3416 printf("Tag: %s State: %s\n", tagName.c_str(), FileBackupAgent::restoreStateText(s).toString().c_str());
3417 return Void();
3418 }) );
3419 break;
3420 case RESTORE_STATUS:
3421 // If no tag is specifically provided then print all tag status, don't just use "default"
3422 if(tagProvided)
3423 tag = tagName;
3424 f = stopAfter( map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void {
3425 printf("%s\n", s.c_str());
3426 return Void();
3427 }) );
3428 break;
3429 default:
3430 throw restore_error();
3431 }
3432 break;
3433 case EXE_DR_AGENT:
3434 if(!initCluster())
3435 return FDB_EXIT_ERROR;
3436 f = stopAfter( runDBAgent(sourceDb, db) );
3437 break;
3438 case EXE_DB_BACKUP:
3439 if(!initCluster())
3440 return FDB_EXIT_ERROR;
3441 switch (dbType)
3442 {
3443 case DB_START:
3444 f = stopAfter( submitDBBackup(sourceDb, db, backupKeys, tagName) );
3445 break;
3446 case DB_STATUS:
3447 f = stopAfter( statusDBBackup(sourceDb, db, tagName, maxErrors) );
3448 break;
3449 case DB_SWITCH:
3450 f = stopAfter( switchDBBackup(sourceDb, db, backupKeys, tagName, forceAction) );
3451 break;
3452 case DB_ABORT:
3453 f = stopAfter( abortDBBackup(sourceDb, db, tagName, partial) );
3454 break;
3455 case DB_PAUSE:
3456 f = stopAfter( changeDBBackupResumed(sourceDb, db, true) );
3457 break;
3458 case DB_RESUME:
3459 f = stopAfter( changeDBBackupResumed(sourceDb, db, false) );
3460 break;
3461 case DB_UNDEFINED:
3462 default:
3463 fprintf(stderr, "ERROR: Unsupported DR action %s\n", argv[1]);
3464 printHelpTeaser(argv[0]);
3465 return FDB_EXIT_ERROR;
3466 break;
3467 }
3468 break;
3469 case EXE_UNDEFINED:
3470 default:
3471 return FDB_EXIT_ERROR;
3472 }
3473
3474 runNetwork();
3475
3476 if(f.isValid() && f.isReady() && !f.isError() && !f.get().present()) {
3477 status = FDB_EXIT_ERROR;
3478 }
3479
3480 if(fstatus.isValid() && fstatus.isReady() && !fstatus.isError() && fstatus.get().present()) {
3481 status = fstatus.get().get();
3482 }
3483
3484 #ifdef ALLOC_INSTRUMENTATION
3485 {
3486 cout << "Page Counts: "
3487 << FastAllocator<16>::pageCount << " "
3488 << FastAllocator<32>::pageCount << " "
3489 << FastAllocator<64>::pageCount << " "
3490 << FastAllocator<128>::pageCount << " "
3491 << FastAllocator<256>::pageCount << " "
3492 << FastAllocator<512>::pageCount << " "
3493 << FastAllocator<1024>::pageCount << " "
3494 << FastAllocator<2048>::pageCount << " "
3495 << FastAllocator<4096>::pageCount << " "
3496 << FastAllocator<8192>::pageCount << endl;
3497
3498 vector< std::pair<std::string, const char*> > typeNames;
3499 for( auto i = allocInstr.begin(); i != allocInstr.end(); ++i ) {
3500 std::string s;
3501
3502 #ifdef __linux__
3503 char *demangled = abi::__cxa_demangle(i->first, NULL, NULL, NULL);
3504 if (demangled) {
3505 s = demangled;
3506 if (StringRef(s).startsWith(LiteralStringRef("(anonymous namespace)::")))
3507 s = s.substr(LiteralStringRef("(anonymous namespace)::").size());
3508 free(demangled);
3509 } else
3510 s = i->first;
3511 #else
3512 s = i->first;
3513 if (StringRef(s).startsWith(LiteralStringRef("class `anonymous namespace'::")))
3514 s = s.substr(LiteralStringRef("class `anonymous namespace'::").size());
3515 else if (StringRef(s).startsWith(LiteralStringRef("class ")))
3516 s = s.substr(LiteralStringRef("class ").size());
3517 else if (StringRef(s).startsWith(LiteralStringRef("struct ")))
3518 s = s.substr(LiteralStringRef("struct ").size());
3519 #endif
3520
3521 typeNames.push_back( std::make_pair(s, i->first) );
3522 }
3523 std::sort(typeNames.begin(), typeNames.end());
3524 for(int i=0; i<typeNames.size(); i++) {
3525 const char* n = typeNames[i].second;
3526 auto& f = allocInstr[n];
3527 printf("%+d\t%+d\t%d\t%d\t%s\n", f.allocCount, -f.deallocCount, f.allocCount-f.deallocCount, f.maxAllocated, typeNames[i].first.c_str());
3528 }
3529
3530 // We're about to exit and clean up data structures, this will wreak havoc on allocation recording
3531 memSample_entered = true;
3532 }
3533 #endif
3534 } catch (Error& e) {
3535 TraceEvent(SevError, "MainError").error(e);
3536 status = FDB_EXIT_MAIN_ERROR;
3537 } catch (std::exception& e) {
3538 TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
3539 status = FDB_EXIT_MAIN_EXCEPTION;
3540 }
3541
3542 flushAndExit(status);
3543 }
3544