1#!/usr/local/bin/bash 2# Copyright (C) 2013 Percona Inc 3# 4# This program is free software; you can redistribute it and/or modify 5# it under the terms of the GNU General Public License as published by 6# the Free Software Foundation; version 2 of the License. 7# 8# This program is distributed in the hope that it will be useful, 9# but WITHOUT ANY WARRANTY; without even the implied warranty of 10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11# GNU General Public License for more details. 12# 13# You should have received a copy of the GNU General Public License 14# along with this program; see the file COPYING. If not, write to the 15# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston 16# MA 02110-1301 USA. 17 18# Documentation: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html 19# Make sure to read that before proceeding! 20 21set -o nounset -o errexit 22 23. $(dirname $0)/wsrep_sst_common 24 25OS=$(uname) 26 27ealgo="" 28ekey="" 29ekeyfile="" 30encrypt=0 31nproc=1 32ecode=0 33ssyslog="" 34ssystag="" 35XTRABACKUP_PID="" 36SST_PORT="" 37REMOTEIP="" 38tca="" 39tcert="" 40tkey="" 41sockopt="" 42progress="" 43ttime=0 44totime=0 45lsn="" 46ecmd="" 47rlimit="" 48# Initially 49stagemsg="${WSREP_SST_OPT_ROLE}" 50cpat="" 51ib_home_dir="" 52ib_log_dir="" 53ib_undo_dir="" 54 55sfmt="tar" 56strmcmd="" 57tfmt="" 58tcmd="" 59rebuild=0 60rebuildcmd="" 61payload=0 62pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' " 63pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE " 64STATDIR="" 65uextra=0 66disver="" 67 68tmpopts="" 69itmpdir="" 70xtmpdir="" 71 72scomp="" 73sdecomp="" 74ssl_dhparams="" 75 76ssl_cert="" 77ssl_ca="" 78ssl_key="" 79 80# Required for backup locks 81# For backup locks it is 1 sent by joiner 82# 5.6.21 PXC and later can't donate to an older joiner 83sst_ver=1 84 85if which pv &>/dev/null && pv --help | grep -q FORMAT;then 86 pvopts+=$pvformat 87fi 88pcmd="pv $pvopts" 89declare -a RC 90 91INNOBACKUPEX_BIN=innobackupex 92DATA="${WSREP_SST_OPT_DATA}" 93INFO_FILE="xtrabackup_galera_info" 94IST_FILE="xtrabackup_ist" 95MAGIC_FILE="${DATA}/${INFO_FILE}" 96 97# Setting the path for ss and ip 98export PATH="/usr/sbin:/sbin:$PATH" 99 100timeit(){ 101 local stage=$1 102 shift 103 local cmd="$@" 104 local x1 x2 took extcode 105 106 if [[ $ttime -eq 1 ]];then 107 x1=$(date +%s) 108 wsrep_log_info "Evaluating $cmd" 109 eval "$cmd" 110 extcode=$? 111 x2=$(date +%s) 112 took=$(( x2-x1 )) 113 wsrep_log_info "NOTE: $stage took $took seconds" 114 totime=$(( totime+took )) 115 else 116 wsrep_log_info "Evaluating $cmd" 117 eval "$cmd" 118 extcode=$? 119 fi 120 return $extcode 121} 122 123get_keys() 124{ 125 # $encrypt -eq 1 is for internal purposes only 126 if [[ $encrypt -ge 2 || $encrypt -eq -1 ]];then 127 return 128 fi 129 130 if [[ $encrypt -eq 0 ]];then 131 if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF xtrabackup | grep -q encrypt;then 132 wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html " 133 fi 134 return 135 fi 136 137 if [[ $sfmt == 'tar' ]];then 138 wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format" 139 encrypt=-1 140 return 141 fi 142 143 wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4" 144 145 if [[ -z $ealgo ]];then 146 wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out" 147 exit 3 148 fi 149 150 if [[ -z $ekey && ! -r $ekeyfile ]];then 151 wsrep_log_error "FATAL: Either key or keyfile must be readable" 152 exit 3 153 fi 154 155 if [[ -z $ekey ]];then 156 ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile" 157 else 158 wsrep_log_warning "Using the 'encrypt-key' option causes the encryption key" 159 wsrep_log_warning "to be set via the command-line and is considered insecure." 160 wsrep_log_warning "It is recommended to use the 'encrypt-key-file' option instead." 161 162 ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey" 163 fi 164 165 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then 166 ecmd+=" -d" 167 fi 168 169 stagemsg+="-XB-Encrypted" 170} 171 172# 173# If the ssl_dhparams variable is already set, uses that as a source 174# of dh parameters for OpenSSL. Otherwise, looks for dhparams.pem in the 175# datadir, and creates it there if it can't find the file. 176# No input parameters 177# 178check_for_dhparams() 179{ 180 if [[ -z "$ssl_dhparams" ]]; then 181 if ! [[ -r "$DATA/dhparams.pem" ]]; then 182 wsrep_check_programs openssl 183 wsrep_log_info "Could not find dhparams file, creating $DATA/dhparams.pem" 184 185 if ! openssl dhparam -out "$DATA/dhparams.pem" 2048 >/dev/null 2>&1 186 then 187 wsrep_log_error "******** FATAL ERROR ********************************* " 188 wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. " 189 wsrep_log_error "****************************************************** " 190 exit 22 191 fi 192 fi 193 ssl_dhparams="$DATA/dhparams.pem" 194 fi 195} 196 197# 198# verifies that the certificate matches the private key 199# doing this will save us having to wait for a timeout that would 200# otherwise occur. 201# 202# 1st param: path to the cert 203# 2nd param: path to the private key 204# 205verify_cert_matches_key() 206{ 207 local cert_path=$1 208 local key_path=$2 209 210 wsrep_check_programs openssl diff 211 212 # generate the public key from the cert and the key 213 # they should match (otherwise we can't create an SSL connection) 214 if ! diff <(openssl x509 -in "$cert_path" -pubkey -noout) <(openssl rsa -in "$key_path" -pubout 2>/dev/null) >/dev/null 2>&1 215 then 216 wsrep_log_error "******** FATAL ERROR ************************* " 217 wsrep_log_error "* The certifcate and private key do not match. " 218 wsrep_log_error "* Please check your certificate and key files. " 219 wsrep_log_error "********************************************** " 220 exit 22 221 fi 222} 223 224# Checks to see if the file exists 225# If the file does not exist (or cannot be read), issues an error 226# and exits 227# 228# 1st param: file name to be checked (for read access) 229# 2nd param: 1st error message (header) 230# 3rd param: 2nd error message (footer, optional) 231# 232verify_file_exists() 233{ 234 local file_path=$1 235 local error_message1=$2 236 local error_message2=$3 237 238 if ! [[ -r "$file_path" ]]; then 239 wsrep_log_error "******** FATAL ERROR ************************* " 240 wsrep_log_error "* $error_message1 " 241 wsrep_log_error "* Could not find/access : $file_path " 242 243 if ! [[ -z "$error_message2" ]]; then 244 wsrep_log_error "* $error_message2 " 245 fi 246 247 wsrep_log_error "********************************************** " 248 exit 22 249 fi 250} 251 252get_transfer() 253{ 254 if [[ -z $SST_PORT ]];then 255 TSST_PORT=4444 256 else 257 TSST_PORT=$SST_PORT 258 fi 259 260 if [[ $tfmt == 'nc' ]];then 261 if [[ ! -x `which nc` ]];then 262 wsrep_log_error "nc(netcat) not found in path: $PATH" 263 exit 2 264 fi 265 266 if [[ $encrypt -eq 2 || $encrypt -eq 3 || $encrypt -eq 4 ]]; then 267 wsrep_log_error "******** FATAL ERROR *********************** " 268 wsrep_log_error "* Using SSL encryption (encrypt= 2, 3, or 4) " 269 wsrep_log_error "* is not supported when using nc(netcat). " 270 wsrep_log_error "******************************************** " 271 exit 22 272 fi 273 274 wsrep_log_info "Using netcat as streamer" 275 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then 276 if nc -h 2>&1 | grep -q ncat; then 277 tcmd="nc $sockopt -l ${TSST_PORT}" 278 else 279 tcmd="nc $sockopt -dl ${TSST_PORT}" 280 fi 281 else 282 # netcat doesn't understand [] around IPv6 address 283 tcmd="nc ${REMOTEIP//[\[\]]/} ${TSST_PORT}" 284 fi 285 else 286 tfmt='socat' 287 wsrep_log_info "Using socat as streamer" 288 if [[ ! -x `which socat` ]];then 289 wsrep_log_error "socat not found in path: $PATH" 290 exit 2 291 fi 292 293 donor_extra="" 294 joiner_extra="" 295 if [[ $encrypt -eq 2 || $encrypt -eq 3 || $encrypt -eq 4 ]]; then 296 if ! socat -V | grep -q WITH_OPENSSL; then 297 wsrep_log_error "******** FATAL ERROR ****************** " 298 wsrep_log_error "* socat is not openssl enabled. " 299 wsrep_log_error "* Unable to encrypt SST communications. " 300 wsrep_log_error "*************************************** " 301 exit 2 302 fi 303 304 # Determine the socat version 305 SOCAT_VERSION=`socat -V 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1` 306 if [[ -z "$SOCAT_VERSION" ]]; then 307 wsrep_log_error "******** FATAL ERROR **************** " 308 wsrep_log_error "* Cannot determine the socat version. " 309 wsrep_log_error "************************************* " 310 exit 2 311 fi 312 313 # socat versions < 1.7.3 will have 512-bit dhparams (too small) 314 # so create 2048-bit dhparams and send that as a parameter 315 # socat version >= 1.7.3, checks to see if the peername matches the hostname 316 # set commonname="" to disable the peername checks 317 # 318 if ! check_for_version "$SOCAT_VERSION" "1.7.3"; then 319 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]]; then 320 # dhparams check (will create ssl_dhparams if needed) 321 check_for_dhparams 322 joiner_extra=",dhparam=$ssl_dhparams" 323 fi 324 fi 325 if check_for_version "$SOCAT_VERSION" "1.7.3"; then 326 donor_extra=',commonname=""' 327 fi 328 fi 329 330 if [[ $encrypt -eq 2 ]]; then 331 wsrep_log_warning "**** WARNING **** encrypt=2 is deprecated and will be removed in a future release" 332 wsrep_log_info "Using openssl based encryption with socat: with crt and ca" 333 334 verify_file_exists "$tcert" "Both certificate and CA files are required." \ 335 "Please check the 'tcert' option. " 336 verify_file_exists "$tca" "Both certificate and CA files are required." \ 337 "Please check the 'tca' option. " 338 339 stagemsg+="-OpenSSL-Encrypted-2" 340 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then 341 wsrep_log_info "Decrypting with CERT: $tcert, CA: $tca" 342 tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tcert},cafile=${tca}${joiner_extra}${sockopt} stdio" 343 else 344 wsrep_log_info "Encrypting with CERT: $tcert, CA: $tca" 345 tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tcert},cafile=${tca}${donor_extra}${sockopt}" 346 fi 347 elif [[ $encrypt -eq 3 ]];then 348 wsrep_log_warning "**** WARNING **** encrypt=3 is deprecated and will be removed in a future release" 349 wsrep_log_info "Using openssl based encryption with socat: with key and crt" 350 351 verify_file_exists "$tcert" "Both certificate and key files are required." \ 352 "Please check the 'tcert' option. " 353 verify_file_exists "$tkey" "Both certificate and key files are required." \ 354 "Please check the 'tkey' option. " 355 356 stagemsg+="-OpenSSL-Encrypted-3" 357 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then 358 wsrep_log_info "Decrypting with CERT: $tcert, KEY: $tkey" 359 tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tcert},key=${tkey},verify=0${joiner_extra}${sockopt} stdio" 360 else 361 wsrep_log_info "Encrypting with CERT: $tcert, KEY: $tkey" 362 tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tcert},key=${tkey},verify=0${sockopt}" 363 fi 364 elif [[ $encrypt -eq 4 ]]; then 365 wsrep_log_info "Using openssl based encryption with socat: with key, crt, and ca" 366 367 verify_file_exists "$ssl_ca" "CA, certificate, and key files are required." \ 368 "Please check the 'ssl-ca' option. " 369 verify_file_exists "$ssl_cert" "CA, certificate, and key files are required." \ 370 "Please check the 'ssl-cert' option. " 371 verify_file_exists "$ssl_key" "CA, certificate, and key files are required." \ 372 "Please check the 'ssl-key' option. " 373 374 # Check to see that the key matches the cert 375 verify_cert_matches_key $ssl_cert $ssl_key 376 377 stagemsg+="-OpenSSL-Encrypted-4" 378 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]]; then 379 wsrep_log_info "Decrypting with CERT: $ssl_cert, KEY: $ssl_key, CA: $ssl_ca" 380 tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${ssl_cert},key=${ssl_key},cafile=${ssl_ca},verify=1${joiner_extra}${sockopt} stdio" 381 else 382 wsrep_log_info "Encrypting with CERT: $ssl_cert, KEY: $ssl_key, CA: $ssl_ca" 383 tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${ssl_cert},key=${ssl_key},cafile=${ssl_ca},verify=1${donor_extra}${sockopt}" 384 fi 385 386 else 387 if [[ $encrypt -eq 1 ]]; then 388 wsrep_log_warning "**** WARNING **** encrypt=1 is deprecated and will be removed in a future release" 389 fi 390 391 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]]; then 392 tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio" 393 else 394 tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}" 395 fi 396 fi 397 fi 398} 399 400get_footprint() 401{ 402 pushd $WSREP_SST_OPT_DATA 1>/dev/null 403 payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | xargs -0 du --block-size=1 -c | awk 'END { print $1 }') 404 if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then 405 # QuickLZ has around 50% compression ratio 406 # When compression/compaction used, the progress is only an approximate. 407 payload=$(( payload*1/2 )) 408 fi 409 popd 1>/dev/null 410 pcmd+=" -s $payload" 411 adjust_progress 412} 413 414adjust_progress() 415{ 416 417 if [[ ! -x `which pv` ]];then 418 wsrep_log_error "pv not found in path: $PATH" 419 wsrep_log_error "Disabling all progress/rate-limiting" 420 pcmd="" 421 rlimit="" 422 progress="" 423 return 424 fi 425 426 if [[ -n $progress && $progress != '1' ]];then 427 if [[ -e $progress ]];then 428 pcmd+=" 2>>$progress" 429 else 430 pcmd+=" 2>$progress" 431 fi 432 elif [[ -z $progress && -n $rlimit ]];then 433 # When rlimit is non-zero 434 pcmd="pv -q" 435 fi 436 437 if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE" == "donor" ]];then 438 wsrep_log_info "Rate-limiting SST to $rlimit" 439 pcmd+=" -L \$rlimit" 440 fi 441} 442 443read_cnf() 444{ 445 sfmt=$(parse_cnf sst streamfmt "xbstream") 446 tfmt=$(parse_cnf sst transferfmt "socat") 447 tca=$(parse_cnf sst tca "") 448 tcert=$(parse_cnf sst tcert "") 449 tkey=$(parse_cnf sst tkey "") 450 encrypt=$(parse_cnf sst encrypt 0) 451 sockopt=$(parse_cnf sst sockopt "") 452 progress=$(parse_cnf sst progress "") 453 rebuild=$(parse_cnf sst rebuild 0) 454 ttime=$(parse_cnf sst time 0) 455 if [ "$OS" = "FreeBSD" ] ; then 456 cpat=$(parse_cnf sst cpat '.*\.pem$|.*init\.ok$|.*galera\.cache$|.*sst_in_progress$|.*\.sst$|.*gvwstate\.dat$|.*grastate\.dat$|.*\.err$|.*\.log$|.*RPM_UPGRADE_MARKER$|.*RPM_UPGRADE_HISTORY$') 457 else 458 cpat=$(parse_cnf sst cpat '.*\.pem$\|.*init\.ok$\|.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$') 459 fi 460 ealgo=$(parse_cnf xtrabackup encrypt "") 461 ekey=$(parse_cnf xtrabackup encrypt-key "") 462 ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") 463 scomp=$(parse_cnf sst compressor "") 464 sdecomp=$(parse_cnf sst decompressor "") 465 466 467 # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html 468 if [[ -z $ealgo ]];then 469 ealgo=$(parse_cnf sst encrypt-algo "") 470 ekey=$(parse_cnf sst encrypt-key "") 471 ekeyfile=$(parse_cnf sst encrypt-key-file "") 472 fi 473 474 # Pull the parameters needed for encrypt=4 475 ssl_ca=$(parse_cnf sst ssl-ca "") 476 if [[ -z "$ssl_ca" ]]; then 477 ssl_ca=$(parse_cnf mysqld ssl-ca "") 478 fi 479 ssl_cert=$(parse_cnf sst ssl-cert "") 480 if [[ -z "$ssl_cert" ]]; then 481 ssl_cert=$(parse_cnf mysqld ssl-cert "") 482 fi 483 ssl_key=$(parse_cnf sst ssl-key "") 484 if [[ -z "$ssl_key" ]]; then 485 ssl_key=$(parse_cnf mysqld ssl-key "") 486 fi 487 488 rlimit=$(parse_cnf sst rlimit "") 489 uextra=$(parse_cnf sst use-extra 0) 490 iopts=$(parse_cnf sst inno-backup-opts "") 491 iapts=$(parse_cnf sst inno-apply-opts "") 492 impts=$(parse_cnf sst inno-move-opts "") 493 stimeout=$(parse_cnf sst sst-initial-timeout 100) 494 ssyslog=$(parse_cnf sst sst-syslog 0) 495 ssystag=$(parse_cnf mysqld_safe syslog-tag "${SST_SYSLOG_TAG:-}") 496 ssystag+="-" 497 498 if [[ $ssyslog -ne -1 ]];then 499 if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld_safe | tr '_' '-' | grep -q -- "--syslog";then 500 ssyslog=1 501 fi 502 fi 503} 504 505get_stream() 506{ 507 if [[ $sfmt == 'xbstream' ]];then 508 wsrep_log_info "Streaming with xbstream" 509 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then 510 strmcmd="xbstream -x" 511 else 512 strmcmd="xbstream -c \${INFO_FILE}" 513 fi 514 else 515 sfmt="tar" 516 wsrep_log_info "Streaming with tar" 517 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then 518 strmcmd="tar xfi - " 519 else 520 strmcmd="tar cf - \${INFO_FILE} " 521 fi 522 523 fi 524} 525 526get_proc() 527{ 528 set +e 529 nproc=1 530 [ "$OS" = "Linux" ] && nproc=$(grep -c processor /proc/cpuinfo) 531 [ "$OS" = "Darwin" -o "$OS" = "FreeBSD" ] && nproc=$(sysctl -n hw.ncpu) 532 [[ -z $nproc || $nproc -eq 0 ]] && nproc=1 533 set -e 534} 535 536sig_joiner_cleanup() 537{ 538 wsrep_log_error "Removing $MAGIC_FILE file due to signal" 539 rm -f "$MAGIC_FILE" 540} 541 542cleanup_joiner() 543{ 544 # Since this is invoked just after exit NNN 545 local estatus=$? 546 if [[ $estatus -ne 0 ]];then 547 wsrep_log_error "Cleanup after exit with status:$estatus" 548 elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then 549 wsrep_log_info "Removing the sst_in_progress file" 550 wsrep_cleanup_progress_file 551 fi 552 if [[ -n $progress && -p $progress ]];then 553 wsrep_log_info "Cleaning up fifo file $progress" 554 rm $progress 555 fi 556 if [[ -n ${STATDIR:-} ]];then 557 [[ -d $STATDIR ]] && rm -rf $STATDIR 558 fi 559 560 # Final cleanup 561 pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') 562 563 # This means no setsid done in mysqld. 564 # We don't want to kill mysqld here otherwise. 565 if [[ $$ -eq $pgid ]];then 566 567 # This means a signal was delivered to the process. 568 # So, more cleanup. 569 if [[ $estatus -ge 128 ]];then 570 kill -KILL -$$ || true 571 fi 572 573 fi 574 575 exit $estatus 576} 577 578check_pid() 579{ 580 local pid_file="$1" 581 [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1 582} 583 584cleanup_donor() 585{ 586 # Since this is invoked just after exit NNN 587 local estatus=$? 588 if [[ $estatus -ne 0 ]];then 589 wsrep_log_error "Cleanup after exit with status:$estatus" 590 fi 591 592 if [[ -n ${XTRABACKUP_PID:-} ]];then 593 if check_pid $XTRABACKUP_PID 594 then 595 wsrep_log_error "xtrabackup process is still running. Killing... " 596 kill_xtrabackup 597 fi 598 599 fi 600 rm -f ${DATA}/${IST_FILE} || true 601 602 if [[ -n $progress && -p $progress ]];then 603 wsrep_log_info "Cleaning up fifo file $progress" 604 rm -f $progress || true 605 fi 606 607 wsrep_log_info "Cleaning up temporary directories" 608 609 if [[ -n $xtmpdir ]];then 610 [[ -d $xtmpdir ]] && rm -rf $xtmpdir || true 611 fi 612 613 if [[ -n $itmpdir ]];then 614 [[ -d $itmpdir ]] && rm -rf $itmpdir || true 615 fi 616 617 # Final cleanup 618 pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') 619 620 # This means no setsid done in mysqld. 621 # We don't want to kill mysqld here otherwise. 622 if [[ $$ -eq $pgid ]];then 623 624 # This means a signal was delivered to the process. 625 # So, more cleanup. 626 if [[ $estatus -ge 128 ]];then 627 kill -KILL -$$ || true 628 fi 629 630 fi 631 632 exit $estatus 633 634} 635 636kill_xtrabackup() 637{ 638 local PID=$(cat $XTRABACKUP_PID) 639 [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : 640 wsrep_log_info "Removing xtrabackup pid file $XTRABACKUP_PID" 641 rm -f "$XTRABACKUP_PID" || true 642} 643 644setup_ports() 645{ 646 if [[ "$WSREP_SST_OPT_ROLE" == "donor" ]];then 647 SST_PORT=$WSREP_SST_OPT_PORT 648 REMOTEIP=$WSREP_SST_OPT_HOST 649 lsn=$(echo $WSREP_SST_OPT_PATH | awk -F '[/]' '{ print $2 }') 650 sst_ver=$(echo $WSREP_SST_OPT_PATH | awk -F '[/]' '{ print $3 }') 651 else 652 SST_PORT=$WSREP_SST_OPT_PORT 653 fi 654} 655 656# waits ~1 minute for nc/socat to open the port and then reports ready 657# (regardless of timeout) 658wait_for_listen() 659{ 660 local HOST=$1 661 local PORT=$2 662 local MODULE=$3 663 664 for i in {1..300} 665 do 666 if [ "`uname`" = "FreeBSD" ] ; then 667 get_listening_on_port_cmd="sockstat -l -P tcp -p $PORT" 668 else 669 get_listening_on_port_cmd="ss -p state listening ( sport = :$PORT )" 670 fi 671 $get_listening_on_port_cmd | grep -qE 'socat|nc' && break 672 sleep 0.2 673 done 674 675 echo "ready ${HOST}:${PORT}/${MODULE}//$sst_ver" 676} 677 678check_extra() 679{ 680 local use_socket=1 681 if [[ $uextra -eq 1 ]];then 682 if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then 683 local eport=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) 684 if [[ -n $eport ]];then 685 # Xtrabackup works only locally. 686 # Hence, setting host to 127.0.0.1 unconditionally. 687 wsrep_log_info "SST through extra_port $eport" 688 INNOEXTRA+=" --host=127.0.0.1 --port=$eport " 689 use_socket=0 690 else 691 wsrep_log_error "Extra port $eport null, failing" 692 exit 1 693 fi 694 else 695 wsrep_log_info "Thread pool not set, ignore the option use_extra" 696 fi 697 fi 698 if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then 699 INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}" 700 fi 701} 702 703recv_joiner() 704{ 705 local dir=$1 706 local msg=$2 707 local tmt=$3 708 local checkf=$4 709 local ltcmd 710 711 if [[ ! -d ${dir} ]];then 712 # This indicates that IST is in progress 713 return 714 fi 715 716 pushd ${dir} 1>/dev/null 717 set +e 718 719 if [[ $tmt -gt 0 && -x `which timeout` ]];then 720 if timeout --help 2>&1 | grep -q -- '-k';then 721 ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" 722 else 723 ltcmd="timeout -s9 $tmt $tcmd" 724 fi 725 timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" 726 else 727 timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" 728 fi 729 730 set -e 731 popd 1>/dev/null 732 733 if [[ ${RC[0]} -eq 124 ]];then 734 wsrep_log_error "Possible timeout in receving first data from donor in gtid stage" 735 exit 32 736 fi 737 738 for ecode in "${RC[@]}";do 739 if [[ $ecode -ne 0 ]];then 740 wsrep_log_error "Error while getting data from donor node: " \ 741 "exit codes: ${RC[@]}" 742 exit 32 743 fi 744 done 745 746 if [[ $checkf -eq 1 && ! -r "${MAGIC_FILE}" ]];then 747 # this message should cause joiner to abort 748 wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'" 749 wsrep_log_info "Contents of datadir" 750 wsrep_log_info "$(ls -l ${dir}/*)" 751 exit 32 752 fi 753} 754 755 756send_donor() 757{ 758 local dir=$1 759 local msg=$2 760 761 pushd ${dir} 1>/dev/null 762 set +e 763 timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )" 764 set -e 765 popd 1>/dev/null 766 767 768 for ecode in "${RC[@]}";do 769 if [[ $ecode -ne 0 ]];then 770 wsrep_log_error "Error while getting data from donor node: " \ 771 "exit codes: ${RC[@]}" 772 exit 32 773 fi 774 done 775 776} 777 778# Returns the version string in a standardized format 779# Input "1.2.3" => echoes "010203" 780# Wrongly formatted values => echoes "000000" 781normalize_version() 782{ 783 local major=0 784 local minor=0 785 local patch=0 786 787 # Only parses purely numeric version numbers, 1.2.3 788 # Everything after the first three values are ignored 789 if [[ $1 =~ ^([0-9]+)\.([0-9]+)\.?([0-9]*)([\.0-9])*$ ]]; then 790 major=${BASH_REMATCH[1]} 791 minor=${BASH_REMATCH[2]} 792 patch=${BASH_REMATCH[3]} 793 fi 794 795 printf %02d%02d%02d $major $minor $patch 796} 797 798# Compares two version strings 799# The first parameter is the version to be checked 800# The second parameter is the minimum version required 801# Returns 1 (failure) if $1 >= $2, 0 (success) otherwise 802check_for_version() 803{ 804 local local_version_str="$( normalize_version $1 )" 805 local required_version_str="$( normalize_version $2 )" 806 807 if [[ "$local_version_str" < "$required_version_str" ]]; then 808 return 1 809 else 810 return 0 811 fi 812} 813 814 815if [[ ! -x `which $INNOBACKUPEX_BIN` ]];then 816 wsrep_log_error "innobackupex not in path: $PATH" 817 exit 2 818fi 819 820# check the version, we require XB-2.4 to ensure that we can pass the 821# datadir via the command-line option 822XB_REQUIRED_VERSION="2.3.5" 823 824XB_VERSION=`$INNOBACKUPEX_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1` 825if [[ -z $XB_VERSION ]]; then 826 wsrep_log_error "FATAL: Cannot determine the $INNOBACKUPEX_BIN version. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST" 827 exit 2 828fi 829 830if ! check_for_version $XB_VERSION $XB_REQUIRED_VERSION; then 831 wsrep_log_error "FATAL: The $INNOBACKUPEX_BIN version is $XB_VERSION. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST" 832 exit 2 833fi 834 835 836rm -f "${MAGIC_FILE}" 837 838if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then 839 wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}" 840 exit 22 841fi 842 843read_cnf 844setup_ports 845 846if ${INNOBACKUPEX_BIN} /tmp --help 2>/dev/null | grep -q -- '--version-check'; then 847 disver="--no-version-check" 848fi 849 850if [[ ${FORCE_FTWRL:-0} -eq 1 ]];then 851 wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL" 852 iopts+=" --no-backup-locks " 853fi 854 855 856INNOEXTRA="" 857 858if [[ $ssyslog -eq 1 ]];then 859 860 if [[ ! -x `which logger` ]];then 861 wsrep_log_error "logger not in path: $PATH. Ignoring" 862 else 863 864 wsrep_log_info "Logging all stderr of SST/Innobackupex to syslog" 865 866 exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE) 867 868 wsrep_log_error() 869 { 870 logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" 871 } 872 873 wsrep_log_info() 874 { 875 logger -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" 876 } 877 878 INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply " 879 INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move " 880 INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" 881 fi 882 883else 884 INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log" 885 INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --defaults-group=mysqld${WSREP_SST_OPT_CONF_SUFFIX} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log" 886 INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --defaults-group=mysqld${WSREP_SST_OPT_CONF_SUFFIX} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log" 887fi 888 889get_stream 890get_transfer 891 892if [ "$WSREP_SST_OPT_ROLE" = "donor" ] 893then 894 trap cleanup_donor EXIT 895 896 if [ $WSREP_SST_OPT_BYPASS -eq 0 ] 897 then 898 usrst=0 899 if [[ -z $sst_ver ]];then 900 wsrep_log_error "Upgrade joiner to 5.6.21 or higher for backup locks support" 901 wsrep_log_error "The joiner is not supported for this version of donor" 902 exit 93 903 fi 904 905 if [[ -z $(parse_cnf mysqld tmpdir "") && -z $(parse_cnf xtrabackup tmpdir "") ]];then 906 xtmpdir=$(mktemp -d) 907 tmpopts=" --tmpdir=$xtmpdir " 908 wsrep_log_info "Using $xtmpdir as xtrabackup temporary directory" 909 fi 910 911 itmpdir=$(mktemp -d) 912 wsrep_log_info "Using $itmpdir as innobackupex temporary directory" 913 914 if [[ -n "${WSREP_SST_OPT_USER:-}" && "$WSREP_SST_OPT_USER" != "(null)" ]]; then 915 INNOEXTRA+=" --user=$WSREP_SST_OPT_USER" 916 usrst=1 917 fi 918 919 if [ -n "${WSREP_SST_OPT_PSWD:-}" ]; then 920 INNOEXTRA+=" --password=$WSREP_SST_OPT_PSWD" 921 elif [[ $usrst -eq 1 ]];then 922 # Empty password, used for testing, debugging etc. 923 INNOEXTRA+=" --password=" 924 fi 925 926 get_keys 927 check_extra 928 929 wsrep_log_info "Streaming GTID file before SST" 930 931 echo "${WSREP_SST_OPT_GTID}" > "${MAGIC_FILE}" 932 933 ttcmd="$tcmd" 934 935 if [[ $encrypt -eq 1 ]];then 936 if [[ -n $scomp ]];then 937 tcmd=" \$ecmd | $scomp | $tcmd " 938 else 939 tcmd=" \$ecmd | $tcmd " 940 fi 941 elif [[ -n $scomp ]];then 942 tcmd=" $scomp | $tcmd " 943 fi 944 945 send_donor $DATA "${stagemsg}-gtid" 946 947 # Restore the transport commmand to its original state 948 tcmd="$ttcmd" 949 if [[ -n $progress ]];then 950 get_footprint 951 tcmd="$pcmd | $tcmd" 952 elif [[ -n $rlimit ]];then 953 adjust_progress 954 tcmd="$pcmd | $tcmd" 955 fi 956 957 wsrep_log_info "Sleeping before data transfer for SST" 958 sleep 10 959 960 wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT:-4444}" 961 962 # Add compression to the head of the stream (if specified) 963 if [[ -n $scomp ]]; then 964 tcmd="$scomp | $tcmd" 965 fi 966 967 # Add encryption to the head of the stream (if specified) 968 if [[ $encrypt -eq 1 ]]; then 969 tcmd=" \$ecmd | $tcmd " 970 fi 971 972 set +e 973 timeit "${stagemsg}-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" 974 set -e 975 976 if [ ${RC[0]} -ne 0 ]; then 977 wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ 978 "Check ${DATA}/innobackup.backup.log" 979 exit 22 980 elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then 981 wsrep_log_error "$tcmd finished with error: ${RC[1]}" 982 exit 22 983 fi 984 985 # innobackupex implicitly writes PID to fixed location in $xtmpdir 986 XTRABACKUP_PID="$xtmpdir/xtrabackup_pid" 987 988 989 else # BYPASS FOR IST 990 991 wsrep_log_info "Bypassing the SST for IST" 992 echo "continue" # now server can resume updating data 993 echo "${WSREP_SST_OPT_GTID}" > "${MAGIC_FILE}" 994 echo "1" > "${DATA}/${IST_FILE}" 995 get_keys 996 if [[ $encrypt -eq 1 ]];then 997 if [[ -n $scomp ]];then 998 tcmd=" \$ecmd | $scomp | $tcmd " 999 else 1000 tcmd=" \$ecmd | $tcmd " 1001 fi 1002 elif [[ -n $scomp ]];then 1003 tcmd=" $scomp | $tcmd " 1004 fi 1005 strmcmd+=" \${IST_FILE}" 1006 1007 send_donor $DATA "${stagemsg}-IST" 1008 1009 fi 1010 1011 echo "done ${WSREP_SST_OPT_GTID}" 1012 wsrep_log_info "Total time on donor: $totime seconds" 1013 1014elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ] 1015then 1016 [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" 1017 [[ -n $SST_PROGRESS_FILE ]] && touch $SST_PROGRESS_FILE 1018 1019 ib_home_dir=$(parse_cnf mysqld innodb-data-home-dir "") 1020 ib_log_dir=$(parse_cnf mysqld innodb-log-group-home-dir "") 1021 ib_undo_dir=$(parse_cnf mysqld innodb-undo-directory "") 1022 1023 stagemsg="Joiner-Recv" 1024 1025 sencrypted=1 1026 nthreads=1 1027 1028 MODULE="xtrabackup_sst" 1029 1030 rm -f "${DATA}/${IST_FILE}" 1031 1032 # May need xtrabackup_checkpoints later on 1033 rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile 1034 1035 wait_for_listen ${WSREP_SST_OPT_HOST} ${WSREP_SST_OPT_PORT:-4444} ${MODULE} & 1036 1037 trap sig_joiner_cleanup HUP PIPE INT TERM 1038 trap cleanup_joiner EXIT 1039 1040 if [[ -n $progress ]];then 1041 adjust_progress 1042 tcmd+=" | $pcmd" 1043 fi 1044 1045 get_keys 1046 if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then 1047 if [[ -n $sdecomp ]];then 1048 strmcmd=" $sdecomp | \$ecmd | $strmcmd" 1049 else 1050 strmcmd=" \$ecmd | $strmcmd" 1051 fi 1052 elif [[ -n $sdecomp ]];then 1053 strmcmd=" $sdecomp | $strmcmd" 1054 fi 1055 1056 STATDIR=$(mktemp -d) 1057 MAGIC_FILE="${STATDIR}/${INFO_FILE}" 1058 recv_joiner $STATDIR "${stagemsg}-gtid" $stimeout 1 1059 1060 1061 if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null 1062 then 1063 wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." 1064 exit 32 1065 fi 1066 1067 if [ ! -r "${STATDIR}/${IST_FILE}" ] 1068 then 1069 1070 if [[ -d ${DATA}/.sst ]];then 1071 wsrep_log_info "WARNING: Stale temporary SST directory: ${DATA}/.sst from previous state transfer. Removing" 1072 rm -rf ${DATA}/.sst 1073 fi 1074 mkdir -p ${DATA}/.sst 1075 (recv_joiner $DATA/.sst "${stagemsg}-SST" 0 0) & 1076 jpid=$! 1077 wsrep_log_info "Proceeding with SST" 1078 1079 1080 wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" 1081 if [ "$OS" = "FreeBSD" ] ; then 1082 find -E $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ 1083 else 1084 find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ 1085 fi 1086 tempdir=$(parse_cnf mysqld log-bin "") 1087 if [[ -n ${tempdir:-} ]];then 1088 binlog_dir=$(dirname $tempdir) 1089 binlog_file=$(basename $tempdir) 1090 if [[ -n ${binlog_dir:-} && $binlog_dir != '.' && $binlog_dir != $DATA ]];then 1091 pattern="$binlog_dir/$binlog_file\.[0-9]+$" 1092 wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" 1093 find $binlog_dir -maxdepth 1 -type f -regex $pattern -exec rm -fv {} 1>&2 \+ || true 1094 rm $binlog_dir/*.index || true 1095 fi 1096 fi 1097 1098 1099 1100 TDATA=${DATA} 1101 DATA="${DATA}/.sst" 1102 1103 1104 MAGIC_FILE="${DATA}/${INFO_FILE}" 1105 wsrep_log_info "Waiting for SST streaming to complete!" 1106 wait $jpid 1107 1108 get_proc 1109 1110 if [[ ! -s ${DATA}/xtrabackup_checkpoints ]];then 1111 wsrep_log_error "xtrabackup_checkpoints missing, failed innobackupex/SST on donor" 1112 exit 2 1113 fi 1114 1115 # Rebuild indexes for compact backups 1116 if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then 1117 wsrep_log_info "Index compaction detected" 1118 rebuild=1 1119 fi 1120 1121 if [[ $rebuild -eq 1 ]];then 1122 nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc) 1123 wsrep_log_info "Rebuilding during prepare with $nthreads threads" 1124 rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads" 1125 fi 1126 1127 if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then 1128 1129 wsrep_log_info "Compressed qpress files found" 1130 1131 if [[ ! -x `which qpress` ]];then 1132 wsrep_log_error "qpress not found in path: $PATH" 1133 exit 22 1134 fi 1135 1136 if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then 1137 count=$(find ${DATA} -type f -name '*.qp' | wc -l) 1138 count=$(( count*2 )) 1139 if pv --help | grep -q FORMAT;then 1140 pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" 1141 else 1142 pvopts="-f -s $count -l -N Decompression" 1143 fi 1144 pcmd="pv $pvopts" 1145 adjust_progress 1146 dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d" 1147 else 1148 dcmd="xargs -n 2 qpress -T${nproc}d" 1149 fi 1150 1151 1152 # Decompress the qpress files 1153 wsrep_log_info "Decompression with $nproc threads" 1154 timeit "Joiner-Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" 1155 extcode=$? 1156 1157 if [[ $extcode -eq 0 ]];then 1158 wsrep_log_info "Removing qpress files after decompression" 1159 find ${DATA} -type f -name '*.qp' -delete 1160 if [[ $? -ne 0 ]];then 1161 wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" 1162 fi 1163 else 1164 wsrep_log_error "Decompression failed. Exit code: $extcode" 1165 exit 22 1166 fi 1167 fi 1168 1169 1170 if [[ ! -z $WSREP_SST_OPT_BINLOG ]];then 1171 1172 BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) 1173 BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) 1174 1175 # To avoid comparing data directory and BINLOG_DIRNAME 1176 mv $DATA/${BINLOG_FILENAME}.* $BINLOG_DIRNAME/ 2>/dev/null || true 1177 1178 pushd $BINLOG_DIRNAME &>/dev/null 1179 for bfiles in $(ls -1 ${BINLOG_FILENAME}.*);do 1180 echo ${BINLOG_DIRNAME}/${bfiles} >> ${BINLOG_FILENAME}.index 1181 done 1182 popd &> /dev/null 1183 1184 fi 1185 1186 1187 wsrep_log_info "Preparing the backup at ${DATA}" 1188 timeit "Xtrabackup prepare stage" "$INNOAPPLY" 1189 1190 if [ $? -ne 0 ]; 1191 then 1192 wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log" 1193 exit 22 1194 fi 1195 1196 MAGIC_FILE="${TDATA}/${INFO_FILE}" 1197 set +e 1198 rm $TDATA/innobackup.prepare.log $TDATA/innobackup.move.log 1199 set -e 1200 wsrep_log_info "Moving the backup to ${TDATA}" 1201 timeit "Xtrabackup move stage" "$INNOMOVE" 1202 if [[ $? -eq 0 ]];then 1203 wsrep_log_info "Move successful, removing ${DATA}" 1204 rm -rf $DATA 1205 DATA=${TDATA} 1206 else 1207 wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis" 1208 wsrep_log_error "Check ${DATA}/innobackup.move.log for details" 1209 exit 22 1210 fi 1211 1212 1213 else 1214 wsrep_log_info "${IST_FILE} received from donor: Running IST" 1215 fi 1216 1217 if [[ ! -r ${MAGIC_FILE} ]];then 1218 wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable" 1219 exit 2 1220 fi 1221 wsrep_log_info "Galera co-ords from recovery: $(cat ${MAGIC_FILE})" 1222 cat "${MAGIC_FILE}" # output UUID:seqno 1223 wsrep_log_info "Total time on joiner: $totime seconds" 1224fi 1225 1226exit 0 1227