1#!/usr/local/bin/bash 2# Copyright (C) 2013 Percona Inc 3# 4# This program is free software; you can redistribute it and/or modify 5# it under the terms of the GNU General Public License as published by 6# the Free Software Foundation; version 2 of the License. 7# 8# This program is distributed in the hope that it will be useful, 9# but WITHOUT ANY WARRANTY; without even the implied warranty of 10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11# GNU General Public License for more details. 12# 13# You should have received a copy of the GNU General Public License 14# along with this program; see the file COPYING. If not, write to the 15# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston 16# MA 02110-1301 USA. 17 18# Documentation: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html 19# Make sure to read that before proceeding! 20 21set -o nounset -o errexit 22 23. $(dirname $0)/wsrep_sst_common 24 25OS=$(uname) 26 27ealgo="" 28ekey="" 29ekeyfile="" 30encrypt=0 31nproc=1 32ecode=0 33ssyslog="" 34ssystag="" 35XTRABACKUP_PID="" 36SST_PORT="" 37REMOTEIP="" 38tca="" 39tcert="" 40tkey="" 41sockopt="" 42progress="" 43ttime=0 44totime=0 45lsn="" 46ecmd="" 47rlimit="" 48# Initially 49stagemsg="${WSREP_SST_OPT_ROLE}" 50cpat="" 51ib_home_dir="" 52ib_log_dir="" 53ib_undo_dir="" 54 55sfmt="tar" 56strmcmd="" 57tfmt="" 58tcmd="" 59rebuild=0 60rebuildcmd="" 61payload=0 62pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' " 63pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE " 64STATDIR="" 65uextra=0 66disver="" 67 68tmpopts="" 69itmpdir="" 70xtmpdir="" 71 72scomp="" 73sdecomp="" 74ssl_dhparams="" 75 76ssl_cert="" 77ssl_ca="" 78ssl_key="" 79ssl_mode="DISABLED" 80 81readonly SECRET_TAG="secret" 82JOINER_PID_FILE="" 83 84# Required for backup locks 85# For backup locks it is 1 sent by joiner 86# 5.6.21 PXC and later can't donate to an older joiner 87sst_ver=1 88 89if which pv &>/dev/null && pv --help | grep -q FORMAT;then 90 pvopts+=$pvformat 91fi 92pcmd="pv $pvopts" 93declare -a RC 94 95INNOBACKUPEX_BIN=innobackupex 96DATA="${WSREP_SST_OPT_DATA}" 97INFO_FILE="xtrabackup_galera_info" 98IST_FILE="xtrabackup_ist" 99MAGIC_FILE="${DATA}/${INFO_FILE}" 100 101# Setting the path for ss and ip 102export PATH="/usr/sbin:/sbin:$PATH" 103 104timeit(){ 105 local stage=$1 106 shift 107 local cmd="$@" 108 local x1 x2 took extcode 109 110 if [[ $ttime -eq 1 ]];then 111 x1=$(date +%s) 112 wsrep_log_info "Evaluating $cmd" 113 eval "$cmd" 114 extcode=$? 115 x2=$(date +%s) 116 took=$(( x2-x1 )) 117 wsrep_log_info "NOTE: $stage took $took seconds" 118 totime=$(( totime+took )) 119 else 120 wsrep_log_info "Evaluating $cmd" 121 eval "$cmd" 122 extcode=$? 123 fi 124 return $extcode 125} 126 127get_keys() 128{ 129 # $encrypt -eq 1 is for internal purposes only 130 if [[ $encrypt -ge 2 || $encrypt -eq -1 ]];then 131 return 132 fi 133 134 if [[ $encrypt -eq 0 ]];then 135 if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF xtrabackup | grep -q encrypt;then 136 wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html " 137 fi 138 return 139 fi 140 141 if [[ $sfmt == 'tar' ]];then 142 wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format" 143 encrypt=-1 144 return 145 fi 146 147 wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4" 148 149 if [[ -z $ealgo ]];then 150 wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out" 151 exit 3 152 fi 153 154 if [[ -z $ekey && ! -r $ekeyfile ]];then 155 wsrep_log_error "FATAL: Either key or keyfile must be readable" 156 exit 3 157 fi 158 159 if [[ -z $ekey ]];then 160 ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile" 161 else 162 wsrep_log_warning "Using the 'encrypt-key' option causes the encryption key" 163 wsrep_log_warning "to be set via the command-line and is considered insecure." 164 wsrep_log_warning "It is recommended to use the 'encrypt-key-file' option instead." 165 166 ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey" 167 fi 168 169 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then 170 ecmd+=" -d" 171 fi 172 173 stagemsg+="-XB-Encrypted" 174} 175 176# 177# If the ssl_dhparams variable is already set, uses that as a source 178# of dh parameters for OpenSSL. Otherwise, looks for dhparams.pem in the 179# datadir, and creates it there if it can't find the file. 180# No input parameters 181# 182check_for_dhparams() 183{ 184 if [[ -z "$ssl_dhparams" ]]; then 185 if ! [[ -r "$DATA/dhparams.pem" ]]; then 186 wsrep_check_programs openssl 187 wsrep_log_info "Could not find dhparams file, creating $DATA/dhparams.pem" 188 189 if ! openssl dhparam -out "$DATA/dhparams.pem" 2048 >/dev/null 2>&1 190 then 191 wsrep_log_error "******** FATAL ERROR ********************************* " 192 wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. " 193 wsrep_log_error "****************************************************** " 194 exit 22 195 fi 196 fi 197 ssl_dhparams="$DATA/dhparams.pem" 198 fi 199} 200 201# 202# verifies that the certificate matches the private key 203# doing this will save us having to wait for a timeout that would 204# otherwise occur. 205# 206# 1st param: path to the cert 207# 2nd param: path to the private key 208# 209verify_cert_matches_key() 210{ 211 local cert_path=$1 212 local key_path=$2 213 214 wsrep_check_programs openssl diff 215 216 # generate the public key from the cert and the key 217 # they should match (otherwise we can't create an SSL connection) 218 if ! diff <(openssl x509 -in "$cert_path" -pubkey -noout) <(openssl rsa -in "$key_path" -pubout 2>/dev/null) >/dev/null 2>&1 219 then 220 wsrep_log_error "******** FATAL ERROR ************************* " 221 wsrep_log_error "* The certifcate and private key do not match. " 222 wsrep_log_error "* Please check your certificate and key files. " 223 wsrep_log_error "********************************************** " 224 exit 22 225 fi 226} 227 228# Checks to see if the file exists 229# If the file does not exist (or cannot be read), issues an error 230# and exits 231# 232# 1st param: file name to be checked (for read access) 233# 2nd param: 1st error message (header) 234# 3rd param: 2nd error message (footer, optional) 235# 236verify_file_exists() 237{ 238 local file_path=$1 239 local error_message1=$2 240 local error_message2=$3 241 242 if ! [[ -r "$file_path" ]]; then 243 wsrep_log_error "******** FATAL ERROR ************************* " 244 wsrep_log_error "* $error_message1 " 245 wsrep_log_error "* Could not find/access : $file_path " 246 247 if ! [[ -z "$error_message2" ]]; then 248 wsrep_log_error "* $error_message2 " 249 fi 250 251 wsrep_log_error "********************************************** " 252 exit 22 253 fi 254} 255 256get_transfer() 257{ 258 if [[ -z $SST_PORT ]];then 259 TSST_PORT=4444 260 else 261 TSST_PORT=$SST_PORT 262 fi 263 264 if [[ $tfmt == 'nc' ]];then 265 if [[ ! -x `which nc` ]];then 266 wsrep_log_error "nc(netcat) not found in path: $PATH" 267 exit 2 268 fi 269 270 if [[ $encrypt -eq 2 || $encrypt -eq 3 || $encrypt -eq 4 ]]; then 271 wsrep_log_error "******** FATAL ERROR *********************** " 272 wsrep_log_error "* Using SSL encryption (encrypt= 2, 3, or 4) " 273 wsrep_log_error "* is not supported when using nc(netcat). " 274 wsrep_log_error "******************************************** " 275 exit 22 276 fi 277 278 wsrep_log_info "Using netcat as streamer" 279 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then 280 if nc -h 2>&1 | grep -q ncat; then 281 tcmd="nc $sockopt -l ${TSST_PORT}" 282 else 283 tcmd="nc $sockopt -dl ${TSST_PORT}" 284 fi 285 else 286 # netcat doesn't understand [] around IPv6 address 287 tcmd="nc ${REMOTEIP//[\[\]]/} ${TSST_PORT}" 288 fi 289 else 290 tfmt='socat' 291 wsrep_log_info "Using socat as streamer" 292 if [[ ! -x `which socat` ]];then 293 wsrep_log_error "socat not found in path: $PATH" 294 exit 2 295 fi 296 297 donor_extra=",connect-timeout=$WSREP_SST_DONOR_TIMEOUT" 298 joiner_extra="" 299 if [[ $encrypt -eq 2 || $encrypt -eq 3 || $encrypt -eq 4 ]]; then 300 if ! socat -V | grep -q WITH_OPENSSL; then 301 wsrep_log_error "******** FATAL ERROR ****************** " 302 wsrep_log_error "* socat is not openssl enabled. " 303 wsrep_log_error "* Unable to encrypt SST communications. " 304 wsrep_log_error "*************************************** " 305 exit 2 306 fi 307 308 # Determine the socat version 309 SOCAT_VERSION=`socat -V 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1` 310 if [[ -z "$SOCAT_VERSION" ]]; then 311 wsrep_log_error "******** FATAL ERROR **************** " 312 wsrep_log_error "* Cannot determine the socat version. " 313 wsrep_log_error "************************************* " 314 exit 2 315 fi 316 317 # socat versions < 1.7.3 will have 512-bit dhparams (too small) 318 # so create 2048-bit dhparams and send that as a parameter 319 # socat version >= 1.7.3, checks to see if the peername matches the hostname 320 # set commonname="" to disable the peername checks 321 # 322 if ! check_for_version "$SOCAT_VERSION" "1.7.3"; then 323 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]]; then 324 # dhparams check (will create ssl_dhparams if needed) 325 check_for_dhparams 326 joiner_extra+=",dhparam=$ssl_dhparams" 327 fi 328 fi 329 if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then 330 donor_extra+=",commonname='$WSREP_SST_OPT_REMOTE_USER'" 331 else 332 if check_for_version "$SOCAT_VERSION" "1.7.3"; then 333 donor_extra+=',commonname=""' 334 fi 335 fi 336 fi 337 338 if [[ $encrypt -eq 2 ]]; then 339 wsrep_log_warning "**** WARNING **** encrypt=2 is deprecated and will be removed in a future release" 340 wsrep_log_info "Using openssl based encryption with socat: with crt and ca" 341 342 verify_file_exists "$tcert" "Both certificate and CA files are required." \ 343 "Please check the 'tcert' option. " 344 verify_file_exists "$tca" "Both certificate and CA files are required." \ 345 "Please check the 'tca' option. " 346 347 stagemsg+="-OpenSSL-Encrypted-2" 348 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then 349 wsrep_log_info "Decrypting with CERT: $tcert, CA: $tca" 350 tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tcert},cafile=${tca}${joiner_extra}${sockopt} stdio" 351 else 352 wsrep_log_info "Encrypting with CERT: $tcert, CA: $tca" 353 tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tcert},cafile=${tca}${donor_extra}${sockopt}" 354 fi 355 elif [[ $encrypt -eq 3 ]];then 356 wsrep_log_warning "**** WARNING **** encrypt=3 is deprecated and will be removed in a future release" 357 wsrep_log_info "Using openssl based encryption with socat: with key and crt" 358 359 verify_file_exists "$tcert" "Both certificate and key files are required." \ 360 "Please check the 'tcert' option. " 361 verify_file_exists "$tkey" "Both certificate and key files are required." \ 362 "Please check the 'tkey' option. " 363 364 stagemsg+="-OpenSSL-Encrypted-3" 365 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then 366 wsrep_log_info "Decrypting with CERT: $tcert, KEY: $tkey" 367 tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tcert},key=${tkey},verify=0${joiner_extra}${sockopt} stdio" 368 else 369 wsrep_log_info "Encrypting with CERT: $tcert, KEY: $tkey" 370 tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tcert},key=${tkey},verify=0${sockopt}" 371 fi 372 elif [[ $encrypt -eq 4 ]]; then 373 wsrep_log_info "Using openssl based encryption with socat: with key, crt, and ca" 374 375 verify_file_exists "$ssl_ca" "CA, certificate, and key files are required." \ 376 "Please check the 'ssl-ca' option. " 377 verify_file_exists "$ssl_cert" "CA, certificate, and key files are required." \ 378 "Please check the 'ssl-cert' option. " 379 verify_file_exists "$ssl_key" "CA, certificate, and key files are required." \ 380 "Please check the 'ssl-key' option. " 381 382 # Check to see that the key matches the cert 383 verify_cert_matches_key $ssl_cert $ssl_key 384 385 stagemsg+="-OpenSSL-Encrypted-4" 386 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]]; then 387 wsrep_log_info "Decrypting with CERT: $ssl_cert, KEY: $ssl_key, CA: $ssl_ca" 388 tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${ssl_cert},key=${ssl_key},cafile=${ssl_ca},verify=1${joiner_extra}${sockopt} stdio" 389 else 390 wsrep_log_info "Encrypting with CERT: $ssl_cert, KEY: $ssl_key, CA: $ssl_ca" 391 tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${ssl_cert},key=${ssl_key},cafile=${ssl_ca},verify=1${donor_extra}${sockopt}" 392 fi 393 394 else 395 if [[ $encrypt -eq 1 ]]; then 396 wsrep_log_warning "**** WARNING **** encrypt=1 is deprecated and will be removed in a future release" 397 fi 398 399 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]]; then 400 tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio" 401 else 402 tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}" 403 fi 404 fi 405 fi 406} 407 408get_footprint() 409{ 410 pushd $WSREP_SST_OPT_DATA 1>/dev/null 411 payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | xargs -0 du --block-size=1 -c | awk 'END { print $1 }') 412 if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then 413 # QuickLZ has around 50% compression ratio 414 # When compression/compaction used, the progress is only an approximate. 415 payload=$(( payload*1/2 )) 416 fi 417 popd 1>/dev/null 418 pcmd+=" -s $payload" 419 adjust_progress 420} 421 422adjust_progress() 423{ 424 425 if [[ ! -x `which pv` ]];then 426 wsrep_log_error "pv not found in path: $PATH" 427 wsrep_log_error "Disabling all progress/rate-limiting" 428 pcmd="" 429 rlimit="" 430 progress="" 431 return 432 fi 433 434 if [[ -n $progress && $progress != '1' ]];then 435 if [[ -e $progress ]];then 436 pcmd+=" 2>>$progress" 437 else 438 pcmd+=" 2>$progress" 439 fi 440 elif [[ -z $progress && -n $rlimit ]];then 441 # When rlimit is non-zero 442 pcmd="pv -q" 443 fi 444 445 if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE" == "donor" ]];then 446 wsrep_log_info "Rate-limiting SST to $rlimit" 447 pcmd+=" -L \$rlimit" 448 fi 449} 450 451check_server_ssl_config() 452{ 453 local section=$1 454 local sst_mode=$2 455 ssl_ca=$(parse_cnf $section ssl-ca "") 456 ssl_cert=$(parse_cnf $section ssl-cert "") 457 ssl_key=$(parse_cnf $section ssl-key "") 458 if [ 0 -eq $encrypt -a -n "$ssl_cert" -a -n "$ssl_key" ] 459 then 460 # Enable SSL encryption 461 462 # BACKWARD COMPATIBILITY: avoid CA verification if either ssl-ca or 463 # ssl-mode not set explicitly in [SST] section of config: 464 # nodes may happen to have different CA if self-generated 465 # so zero-up ssl_ca in such case 466 if [[ ${sst_mode} != *VERIFY* ]] && [[ ${section} != "sst" ]] 467 then 468 ssl_ca= 469 fi 470 471 if [ -n "$ssl_ca" ] 472 then 473 encrypt=4 474 else 475 encrypt=3 476 tcert=$ssl_cert 477 tkey=$ssl_key 478 fi 479 fi 480} 481 482read_cnf() 483{ 484 sfmt=$(parse_cnf sst streamfmt "xbstream") 485 tfmt=$(parse_cnf sst transferfmt "socat") 486 tca=$(parse_cnf sst tca "") 487 tcert=$(parse_cnf sst tcert "") 488 tkey=$(parse_cnf sst tkey "") 489 encrypt=$(parse_cnf sst encrypt 0) 490 sockopt=$(parse_cnf sst sockopt "") 491 progress=$(parse_cnf sst progress "") 492 rebuild=$(parse_cnf sst rebuild 0) 493 ttime=$(parse_cnf sst time 0) 494 if [ "$OS" = "FreeBSD" ] ; then 495 cpat=$(parse_cnf sst cpat '.*\.pem$|.*init\.ok$|.*galera\.cache$|.*sst_in_progress$|.*\.sst$|.*gvwstate\.dat$|.*grastate\.dat$|.*\.err$|.*\.log$|.*RPM_UPGRADE_MARKER$|.*RPM_UPGRADE_HISTORY$') 496 else 497 cpat=$(parse_cnf sst cpat '.*\.pem$\|.*init\.ok$\|.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$') 498 fi 499 ealgo=$(parse_cnf xtrabackup encrypt "") 500 ekey=$(parse_cnf xtrabackup encrypt-key "") 501 ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") 502 scomp=$(parse_cnf sst compressor "") 503 sdecomp=$(parse_cnf sst decompressor "") 504 505 506 # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html 507 if [[ -z $ealgo ]];then 508 ealgo=$(parse_cnf sst encrypt-algo "") 509 ekey=$(parse_cnf sst encrypt-key "") 510 ekeyfile=$(parse_cnf sst encrypt-key-file "") 511 fi 512 513 # Pull the parameters needed for encrypt=4 514 if [ -z "$tca" -a -z "$tkey" -a -z "$tcert" ] 515 then # check for new configuration 516 ssl_mode=$(parse_cnf sst ssl-mode "DISABLED" | tr [:lower:] [:upper:]) 517 check_server_ssl_config "sst" $ssl_mode 518 if [ -z "$ssl_ca" -a -z "$ssl_key" -a -z "$ssl_cert" ] 519 then # no new-style SSL config in [sst], try server-wide SSL config 520 check_server_ssl_config "mysqld.$WSREP_SST_OPT_CONF_SUFFIX" "$ssl_mode" 521 if [ -z "$ssl_ca" -a -z "$ssl_key" -a -z "$ssl_cert" ] 522 then 523 check_server_ssl_config "mysqld" "$ssl_mode" 524 fi 525 fi 526 fi 527 528 rlimit=$(parse_cnf sst rlimit "") 529 uextra=$(parse_cnf sst use-extra 0) 530 iopts=$(parse_cnf sst inno-backup-opts "") 531 iapts=$(parse_cnf sst inno-apply-opts "") 532 impts=$(parse_cnf sst inno-move-opts "") 533 stimeout=$WSREP_SST_JOINER_TIMEOUT 534 ssyslog=$(parse_cnf sst sst-syslog 0) 535 ssystag=$(parse_cnf mysqld_safe syslog-tag "${SST_SYSLOG_TAG:-}") 536 ssystag+="-" 537 538 if [[ $ssyslog -ne -1 ]];then 539 if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld_safe | tr '_' '-' | grep -q -- "--syslog";then 540 ssyslog=1 541 fi 542 fi 543} 544 545get_stream() 546{ 547 if [[ $sfmt == 'xbstream' ]];then 548 wsrep_log_info "Streaming with xbstream" 549 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then 550 strmcmd="xbstream -x" 551 else 552 strmcmd="xbstream -c \${INFO_FILE}" 553 fi 554 else 555 sfmt="tar" 556 wsrep_log_info "Streaming with tar" 557 if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then 558 strmcmd="tar xfi - " 559 else 560 strmcmd="tar cf - \${INFO_FILE} " 561 fi 562 563 fi 564} 565 566get_proc() 567{ 568 set +e 569 nproc=1 570 [ "$OS" = "Linux" ] && nproc=$(grep -c processor /proc/cpuinfo) 571 [ "$OS" = "Darwin" -o "$OS" = "FreeBSD" ] && nproc=$(sysctl -n hw.ncpu) 572 [[ -z $nproc || $nproc -eq 0 ]] && nproc=1 573 set -e 574} 575 576sig_joiner_cleanup() 577{ 578 wsrep_log_error "Removing $MAGIC_FILE file due to signal" 579 rm -f "$MAGIC_FILE" 580} 581 582cleanup_joiner() 583{ 584 # Since this is invoked just after exit NNN 585 local estatus=$? 586 if [[ $estatus -ne 0 ]];then 587 wsrep_log_error "Cleanup after exit with status:$estatus" 588 elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then 589 wsrep_log_info "Removing the sst_in_progress file" 590 wsrep_cleanup_progress_file 591 fi 592 if [[ -n $progress && -p $progress ]];then 593 wsrep_log_info "Cleaning up fifo file $progress" 594 rm $progress 595 fi 596 if [[ -n ${STATDIR:-} ]];then 597 [[ -d $STATDIR ]] && rm -rf $STATDIR 598 fi 599 600 if [ -n "$JOINER_PID_FILE" -a -r "$JOINER_PID_FILE" ]; then 601 local joiner_pid=$(<$JOINER_PID_FILE) 602 kill -KILL $joiner_pid || : 603 rm -f $JOINER_PID_FILE 604 fi 605 606 # Final cleanup 607 pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') 608 609 # This means no setsid done in mysqld. 610 # We don't want to kill mysqld here otherwise. 611 if [[ $$ -eq $pgid ]];then 612 613 # This means a signal was delivered to the process. 614 # So, more cleanup. 615 if [[ $estatus -ge 128 ]];then 616 kill -KILL -$$ || true 617 fi 618 619 fi 620 621 exit $estatus 622} 623 624check_pid() 625{ 626 local pid_file="$1" 627 [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1 628} 629 630cleanup_donor() 631{ 632 # Since this is invoked just after exit NNN 633 local estatus=$? 634 if [[ $estatus -ne 0 ]];then 635 wsrep_log_error "Cleanup after exit with status:$estatus" 636 fi 637 638 if [[ -n ${XTRABACKUP_PID:-} ]];then 639 if check_pid $XTRABACKUP_PID 640 then 641 wsrep_log_error "xtrabackup process is still running. Killing... " 642 kill_xtrabackup 643 fi 644 645 fi 646 rm -f ${DATA}/${IST_FILE} || true 647 648 if [[ -n $progress && -p $progress ]];then 649 wsrep_log_info "Cleaning up fifo file $progress" 650 rm -f $progress || true 651 fi 652 653 wsrep_log_info "Cleaning up temporary directories" 654 655 if [[ -n $xtmpdir ]];then 656 [[ -d $xtmpdir ]] && rm -rf $xtmpdir || true 657 fi 658 659 if [[ -n $itmpdir ]];then 660 [[ -d $itmpdir ]] && rm -rf $itmpdir || true 661 fi 662 663 # Final cleanup 664 pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') 665 666 # This means no setsid done in mysqld. 667 # We don't want to kill mysqld here otherwise. 668 if [[ $$ -eq $pgid ]];then 669 670 # This means a signal was delivered to the process. 671 # So, more cleanup. 672 if [[ $estatus -ge 128 ]];then 673 kill -KILL -$$ || true 674 fi 675 676 fi 677 678 exit $estatus 679 680} 681 682kill_xtrabackup() 683{ 684 local PID=$(cat $XTRABACKUP_PID) 685 [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : 686 wsrep_log_info "Removing xtrabackup pid file $XTRABACKUP_PID" 687 rm -f "$XTRABACKUP_PID" || true 688} 689 690setup_ports() 691{ 692 if [[ "$WSREP_SST_OPT_ROLE" == "donor" ]];then 693 SST_PORT=$WSREP_SST_OPT_PORT 694 REMOTEIP=$WSREP_SST_OPT_HOST 695 lsn=$(echo $WSREP_SST_OPT_PATH | awk -F '[/]' '{ print $2 }') 696 sst_ver=$(echo $WSREP_SST_OPT_PATH | awk -F '[/]' '{ print $3 }') 697 else 698 SST_PORT=$WSREP_SST_OPT_PORT 699 fi 700} 701 702# waits ~1 minute for nc/socat to open the port and then reports ready 703# (regardless of timeout) 704wait_for_listen() 705{ 706 local HOST=$1 707 local PORT=$2 708 local MODULE=$3 709 710 for i in {1..300} 711 do 712 if [ "`uname`" = "FreeBSD" ] ; then 713 get_listening_on_port_cmd="sockstat -l -P tcp -p $PORT" 714 else 715 get_listening_on_port_cmd="ss -p state listening ( sport = :$PORT )" 716 fi 717 $get_listening_on_port_cmd | grep -qE 'socat|nc' && break 718 sleep 0.2 719 done 720 721 echo "ready ${HOST}:${PORT}/${MODULE}//$sst_ver" 722} 723 724check_extra() 725{ 726 local use_socket=1 727 if [[ $uextra -eq 1 ]];then 728 if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then 729 local eport=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) 730 if [[ -n $eport ]];then 731 # Xtrabackup works only locally. 732 # Hence, setting host to 127.0.0.1 unconditionally. 733 wsrep_log_info "SST through extra_port $eport" 734 INNOEXTRA+=" --host=127.0.0.1 --port=$eport " 735 use_socket=0 736 else 737 wsrep_log_error "Extra port $eport null, failing" 738 exit 1 739 fi 740 else 741 wsrep_log_info "Thread pool not set, ignore the option use_extra" 742 fi 743 fi 744 if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then 745 INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}" 746 fi 747} 748 749recv_joiner() 750{ 751 local dir=$1 752 local msg=$2 753 local tmt=$3 754 local checkf=$4 755 local ltcmd=$tcmd 756 757 if [[ ! -d ${dir} ]];then 758 # This indicates that IST is in progress 759 return 760 fi 761 762 pushd ${dir} 1>/dev/null 763 set +e 764 765 if [[ $tmt -gt 0 && -x `which timeout` ]];then 766 if timeout --help 2>&1 | grep -q -- '-k';then 767 ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" 768 else 769 ltcmd="timeout -s9 $tmt $tcmd" 770 fi 771 fi 772 773 JOINER_PID_FILE=`mktemp` 774 timeit "$msg" "($tcmd & echo \$!>$JOINER_PID_FILE) | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" 775 776 set -e 777 popd 1>/dev/null 778 779 if [[ ${RC[0]} -eq 124 ]];then 780 wsrep_log_error "Possible timeout in receiving first data from donor "\ 781 "in gtid stage" 782 exit 32 783 fi 784 785 for ecode in "${RC[@]}";do 786 if [[ $ecode -ne 0 ]];then 787 wsrep_log_error "Error while getting data from donor node: "\ 788 "exit codes: ${RC[@]}" 789 exit 32 790 fi 791 done 792 793 if [[ $checkf -eq 1 ]]; then 794 if [[ ! -r "${MAGIC_FILE}" ]];then 795 # this message should cause joiner to abort 796 wsrep_log_error "receiving process ended without creating "\ 797 "'${MAGIC_FILE}'" 798 wsrep_log_info "Contents of datadir" 799 wsrep_log_info "$(ls -l ${dir}/*)" 800 exit 32 801 fi 802 803 # check donor supplied secret 804 SECRET=$(grep "$SECRET_TAG " ${MAGIC_FILE} 2>/dev/null | cut -d ' ' -f 2) 805 if [[ $SECRET != $MY_SECRET ]]; then 806 wsrep_log_error "Donor does not know my secret!" 807 wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'" 808 exit 32 809 fi 810 811 # remove secret from magic file 812 grep -v "$SECRET_TAG " ${MAGIC_FILE} > ${MAGIC_FILE}.new 813 mv ${MAGIC_FILE}.new ${MAGIC_FILE} 814 fi 815} 816 817send_donor() 818{ 819 local dir=$1 820 local msg=$2 821 822 pushd ${dir} 1>/dev/null 823 set +e 824 timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )" 825 set -e 826 popd 1>/dev/null 827 828 829 for ecode in "${RC[@]}";do 830 if [[ $ecode -ne 0 ]];then 831 wsrep_log_error "Error while sending data to joiner node: "\ 832 "exit codes: ${RC[@]}" 833 exit 32 834 fi 835 done 836 837} 838 839# Returns the version string in a standardized format 840# Input "1.2.3" => echoes "010203" 841# Wrongly formatted values => echoes "000000" 842normalize_version() 843{ 844 local major=0 845 local minor=0 846 local patch=0 847 848 # Only parses purely numeric version numbers, 1.2.3 849 # Everything after the first three values are ignored 850 if [[ $1 =~ ^([0-9]+)\.([0-9]+)\.?([0-9]*)([\.0-9])*$ ]]; then 851 major=${BASH_REMATCH[1]} 852 minor=${BASH_REMATCH[2]} 853 patch=${BASH_REMATCH[3]} 854 fi 855 856 printf %02d%02d%02d $major $minor $patch 857} 858 859# Compares two version strings 860# The first parameter is the version to be checked 861# The second parameter is the minimum version required 862# Returns 1 (failure) if $1 >= $2, 0 (success) otherwise 863check_for_version() 864{ 865 local local_version_str="$( normalize_version $1 )" 866 local required_version_str="$( normalize_version $2 )" 867 868 if [[ "$local_version_str" < "$required_version_str" ]]; then 869 return 1 870 else 871 return 0 872 fi 873} 874 875 876if [[ ! -x `which $INNOBACKUPEX_BIN` ]];then 877 wsrep_log_error "innobackupex not in path: $PATH" 878 exit 2 879fi 880 881# check the version, we require XB-2.4 to ensure that we can pass the 882# datadir via the command-line option 883XB_REQUIRED_VERSION="2.3.5" 884 885XB_VERSION=`$INNOBACKUPEX_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1` 886if [[ -z $XB_VERSION ]]; then 887 wsrep_log_error "FATAL: Cannot determine the $INNOBACKUPEX_BIN version. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST" 888 exit 2 889fi 890 891if ! check_for_version $XB_VERSION $XB_REQUIRED_VERSION; then 892 wsrep_log_error "FATAL: The $INNOBACKUPEX_BIN version is $XB_VERSION. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST" 893 exit 2 894fi 895 896 897rm -f "${MAGIC_FILE}" 898 899if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then 900 wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}" 901 exit 22 902fi 903 904read_cnf 905setup_ports 906 907if ${INNOBACKUPEX_BIN} /tmp --help 2>/dev/null | grep -q -- '--version-check'; then 908 disver="--no-version-check" 909fi 910 911if [[ ${FORCE_FTWRL:-0} -eq 1 ]];then 912 wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL" 913 iopts+=" --no-backup-locks " 914fi 915 916 917INNOEXTRA="" 918 919if [[ $ssyslog -eq 1 ]];then 920 921 if [[ ! -x `which logger` ]];then 922 wsrep_log_error "logger not in path: $PATH. Ignoring" 923 else 924 925 wsrep_log_info "Logging all stderr of SST/Innobackupex to syslog" 926 927 exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE) 928 929 wsrep_log_error() 930 { 931 logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" 932 } 933 934 wsrep_log_info() 935 { 936 logger -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" 937 } 938 939 INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply " 940 INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move " 941 INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" 942 fi 943 944else 945 INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log" 946 INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --defaults-group=mysqld${WSREP_SST_OPT_CONF_SUFFIX} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log" 947 INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --defaults-group=mysqld${WSREP_SST_OPT_CONF_SUFFIX} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log" 948fi 949 950get_stream 951get_transfer 952get_keys 953 954if [ "$WSREP_SST_OPT_ROLE" = "donor" ] 955then 956 trap cleanup_donor EXIT 957 958 wsrep_log_info "Streaming GTID file before SST" 959 960 echo "${WSREP_SST_OPT_GTID}" > "${MAGIC_FILE}" 961 962 if [[ -n ${WSREP_SST_OPT_REMOTE_PSWD} ]]; then 963 # Let joiner know that we know its secret 964 echo "$SECRET_TAG ${WSREP_SST_OPT_REMOTE_PSWD}" >> ${MAGIC_FILE} 965 fi 966 967 ttcmd="$tcmd" 968 969 if [[ $encrypt -eq 1 ]]; then 970 if [[ -n $scomp ]]; then 971 tcmd=" \$ecmd | $scomp | $tcmd " 972 else 973 tcmd=" \$ecmd | $tcmd " 974 fi 975 elif [[ -n $scomp ]]; then 976 tcmd=" $scomp | $tcmd " 977 fi 978 979 if [ $WSREP_SST_OPT_BYPASS -eq 0 ] 980 then 981 usrst=0 982 if [[ -z $sst_ver ]];then 983 wsrep_log_error "Upgrade joiner to 5.6.21 or higher for backup locks support" 984 wsrep_log_error "The joiner is not supported for this version of donor" 985 exit 93 986 fi 987 988 if [[ -z $(parse_cnf mysqld tmpdir "") && -z $(parse_cnf xtrabackup tmpdir "") ]];then 989 xtmpdir=$(mktemp -d) 990 tmpopts=" --tmpdir=$xtmpdir " 991 wsrep_log_info "Using $xtmpdir as xtrabackup temporary directory" 992 fi 993 994 itmpdir=$(mktemp -d) 995 wsrep_log_info "Using $itmpdir as innobackupex temporary directory" 996 997 if [[ -n "${WSREP_SST_OPT_USER:-}" && "$WSREP_SST_OPT_USER" != "(null)" ]]; then 998 INNOEXTRA+=" --user=$WSREP_SST_OPT_USER" 999 usrst=1 1000 fi 1001 1002 if [ -n "${WSREP_SST_OPT_PSWD:-}" ]; then 1003 export MYSQL_PWD="$WSREP_SST_OPT_PSWD" 1004 elif [[ $usrst -eq 1 ]];then 1005 # Empty password, used for testing, debugging etc. 1006 unset MYSQL_PWD 1007 fi 1008 1009 check_extra 1010 1011 send_donor $DATA "${stagemsg}-gtid" 1012 1013 # Restore the transport commmand to its original state 1014 tcmd="$ttcmd" 1015 if [[ -n $progress ]];then 1016 get_footprint 1017 tcmd="$pcmd | $tcmd" 1018 elif [[ -n $rlimit ]];then 1019 adjust_progress 1020 tcmd="$pcmd | $tcmd" 1021 fi 1022 1023 wsrep_log_info "Sleeping before data transfer for SST" 1024 sleep 10 1025 1026 wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT:-4444}" 1027 1028 # Add compression to the head of the stream (if specified) 1029 if [[ -n $scomp ]]; then 1030 tcmd="$scomp | $tcmd" 1031 fi 1032 1033 # Add encryption to the head of the stream (if specified) 1034 if [[ $encrypt -eq 1 ]]; then 1035 tcmd=" \$ecmd | $tcmd " 1036 fi 1037 1038 set +e 1039 timeit "${stagemsg}-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" 1040 set -e 1041 1042 if [ ${RC[0]} -ne 0 ]; then 1043 wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ 1044 "Check ${DATA}/innobackup.backup.log" 1045 exit 22 1046 elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then 1047 wsrep_log_error "$tcmd finished with error: ${RC[1]}" 1048 exit 22 1049 fi 1050 1051 # innobackupex implicitly writes PID to fixed location in $xtmpdir 1052 XTRABACKUP_PID="$xtmpdir/xtrabackup_pid" 1053 1054 1055 else # BYPASS FOR IST 1056 1057 wsrep_log_info "Bypassing SST for IST" 1058 echo "continue" # now server can resume updating data 1059 echo "1" > "${DATA}/${IST_FILE}" 1060 strmcmd+=" \${IST_FILE}" 1061 1062 send_donor $DATA "${stagemsg}-IST" 1063 1064 fi 1065 1066 echo "done ${WSREP_SST_OPT_GTID}" 1067 wsrep_log_info "Total time on donor: $totime seconds" 1068 1069elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ] 1070then 1071 [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" 1072 [[ -n $SST_PROGRESS_FILE ]] && touch $SST_PROGRESS_FILE 1073 1074 ib_home_dir=$(parse_cnf mysqld innodb-data-home-dir "") 1075 ib_log_dir=$(parse_cnf mysqld innodb-log-group-home-dir "") 1076 ib_undo_dir=$(parse_cnf mysqld innodb-undo-directory "") 1077 1078 stagemsg="Joiner-Recv" 1079 1080 sencrypted=1 1081 nthreads=1 1082 1083 MODULE="xtrabackup_sst" 1084 1085 rm -f "${DATA}/${IST_FILE}" 1086 1087 # May need xtrabackup_checkpoints later on 1088 rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile 1089 1090 if [[ "$ssl_mode" = *"VERIFY"* ]] 1091 then # backward-incompatible behavior 1092 if [ -n "$ssl_cert" ] 1093 then 1094 # find out my Common Name 1095 wsrep_check_programs openssl 1096 CN=$(openssl x509 -noout -subject -in $ssl_cert | \ 1097 tr "," "\n" | grep "CN =" | cut -d= -f2 | sed s/^\ // | \ 1098 sed s/\ %//) 1099 else 1100 CN="" 1101 fi 1102 MY_SECRET=$(wsrep_gen_secret) 1103 # Add authentication data to address 1104 AUTH="$CN:$MY_SECRET@" 1105 else 1106 MY_SECRET="" # for check down in recv_joiner() 1107 AUTH="" 1108 fi # tmode == *VERIFY* 1109 1110 wait_for_listen "${AUTH}${WSREP_SST_OPT_HOST}" ${WSREP_SST_OPT_PORT:-4444} \ 1111 ${MODULE} & 1112 1113 trap sig_joiner_cleanup HUP PIPE INT TERM 1114 trap cleanup_joiner EXIT 1115 1116 if [[ -n $progress ]];then 1117 adjust_progress 1118 tcmd+=" | $pcmd" 1119 fi 1120 1121 if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then 1122 if [[ -n $sdecomp ]];then 1123 strmcmd=" $sdecomp | \$ecmd | $strmcmd" 1124 else 1125 strmcmd=" \$ecmd | $strmcmd" 1126 fi 1127 elif [[ -n $sdecomp ]];then 1128 strmcmd=" $sdecomp | $strmcmd" 1129 fi 1130 1131 STATDIR=$(mktemp -d) 1132 MAGIC_FILE="${STATDIR}/${INFO_FILE}" 1133 recv_joiner $STATDIR "${stagemsg}-gtid" $stimeout 1 1134 1135 1136 if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null 1137 then 1138 wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." 1139 exit 32 1140 fi 1141 1142 if [ ! -r "${STATDIR}/${IST_FILE}" ] 1143 then 1144 1145 if [[ -d ${DATA}/.sst ]];then 1146 wsrep_log_info "WARNING: Stale temporary SST directory: ${DATA}/.sst from previous state transfer. Removing" 1147 rm -rf ${DATA}/.sst 1148 fi 1149 mkdir -p ${DATA}/.sst 1150 (recv_joiner $DATA/.sst "${stagemsg}-SST" 0 0) & 1151 jpid=$! 1152 wsrep_log_info "Proceeding with SST" 1153 1154 1155 wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" 1156 if [ "$OS" = "FreeBSD" ] ; then 1157 find -E $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ 1158 else 1159 find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ 1160 fi 1161 tempdir=$(parse_cnf mysqld log-bin "") 1162 if [[ -n ${tempdir:-} ]];then 1163 binlog_dir=$(dirname $tempdir) 1164 binlog_file=$(basename $tempdir) 1165 if [[ -n ${binlog_dir:-} && $binlog_dir != '.' && $binlog_dir != $DATA ]];then 1166 pattern="$binlog_dir/$binlog_file\.[0-9]+$" 1167 wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" 1168 find $binlog_dir -maxdepth 1 -type f -regex $pattern -exec rm -fv {} 1>&2 \+ || true 1169 rm $binlog_dir/*.index || true 1170 fi 1171 fi 1172 1173 1174 1175 TDATA=${DATA} 1176 DATA="${DATA}/.sst" 1177 1178 1179 MAGIC_FILE="${DATA}/${INFO_FILE}" 1180 wsrep_log_info "Waiting for SST streaming to complete!" 1181 wait $jpid 1182 1183 get_proc 1184 1185 if [[ ! -s ${DATA}/xtrabackup_checkpoints ]];then 1186 wsrep_log_error "xtrabackup_checkpoints missing, failed innobackupex/SST on donor" 1187 exit 2 1188 fi 1189 1190 # Rebuild indexes for compact backups 1191 if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then 1192 wsrep_log_info "Index compaction detected" 1193 rebuild=1 1194 fi 1195 1196 if [[ $rebuild -eq 1 ]];then 1197 nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc) 1198 wsrep_log_info "Rebuilding during prepare with $nthreads threads" 1199 rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads" 1200 fi 1201 1202 if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then 1203 1204 wsrep_log_info "Compressed qpress files found" 1205 1206 if [[ ! -x `which qpress` ]];then 1207 wsrep_log_error "qpress not found in path: $PATH" 1208 exit 22 1209 fi 1210 1211 if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then 1212 count=$(find ${DATA} -type f -name '*.qp' | wc -l) 1213 count=$(( count*2 )) 1214 if pv --help | grep -q FORMAT;then 1215 pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" 1216 else 1217 pvopts="-f -s $count -l -N Decompression" 1218 fi 1219 pcmd="pv $pvopts" 1220 adjust_progress 1221 dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d" 1222 else 1223 dcmd="xargs -n 2 qpress -T${nproc}d" 1224 fi 1225 1226 1227 # Decompress the qpress files 1228 wsrep_log_info "Decompression with $nproc threads" 1229 timeit "Joiner-Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" 1230 extcode=$? 1231 1232 if [[ $extcode -eq 0 ]];then 1233 wsrep_log_info "Removing qpress files after decompression" 1234 find ${DATA} -type f -name '*.qp' -delete 1235 if [[ $? -ne 0 ]];then 1236 wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" 1237 fi 1238 else 1239 wsrep_log_error "Decompression failed. Exit code: $extcode" 1240 exit 22 1241 fi 1242 fi 1243 1244 1245 if [[ ! -z $WSREP_SST_OPT_BINLOG ]];then 1246 1247 BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) 1248 BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) 1249 1250 # To avoid comparing data directory and BINLOG_DIRNAME 1251 mv $DATA/${BINLOG_FILENAME}.* $BINLOG_DIRNAME/ 2>/dev/null || true 1252 1253 pushd $BINLOG_DIRNAME &>/dev/null 1254 for bfiles in $(ls -1 ${BINLOG_FILENAME}.*);do 1255 echo ${BINLOG_DIRNAME}/${bfiles} >> ${BINLOG_FILENAME}.index 1256 done 1257 popd &> /dev/null 1258 1259 fi 1260 1261 1262 wsrep_log_info "Preparing the backup at ${DATA}" 1263 timeit "Xtrabackup prepare stage" "$INNOAPPLY" 1264 1265 if [ $? -ne 0 ]; 1266 then 1267 wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log" 1268 exit 22 1269 fi 1270 1271 MAGIC_FILE="${TDATA}/${INFO_FILE}" 1272 set +e 1273 rm $TDATA/innobackup.prepare.log $TDATA/innobackup.move.log 1274 set -e 1275 wsrep_log_info "Moving the backup to ${TDATA}" 1276 timeit "Xtrabackup move stage" "$INNOMOVE" 1277 if [[ $? -eq 0 ]];then 1278 wsrep_log_info "Move successful, removing ${DATA}" 1279 rm -rf $DATA 1280 DATA=${TDATA} 1281 else 1282 wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis" 1283 wsrep_log_error "Check ${DATA}/innobackup.move.log for details" 1284 exit 22 1285 fi 1286 1287 1288 else 1289 wsrep_log_info "${IST_FILE} received from donor: Running IST" 1290 fi 1291 1292 if [[ ! -r ${MAGIC_FILE} ]];then 1293 wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable" 1294 exit 2 1295 fi 1296 wsrep_log_info "Galera co-ords from recovery: $(cat ${MAGIC_FILE})" 1297 cat "${MAGIC_FILE}" # output UUID:seqno 1298 wsrep_log_info "Total time on joiner: $totime seconds" 1299fi 1300 1301exit 0 1302