1#!/usr/local/bin/bash
2
3set -ue
4
5# Copyright (C) 2017-2021 MariaDB
6# Copyright (C) 2013 Percona Inc
7#
8# This program is free software; you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation; version 2 of the License.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with this program; see the file COPYING. If not, write to the
19# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston
20# MA  02110-1335  USA.
21
22# Documentation:
23# https://mariadb.com/kb/en/mariabackup-overview/
24# Make sure to read that before proceeding!
25
26OS="$(uname)"
27
28. $(dirname "$0")/wsrep_sst_common
29wsrep_check_datadir
30
31ealgo=""
32eformat=""
33ekey=""
34ekeyfile=""
35encrypt=0
36ssyslog=""
37ssystag=""
38BACKUP_PID=""
39tcert=""
40tcap=""
41tpem=""
42tkey=""
43tmode="DISABLED"
44sockopt=""
45progress=""
46ttime=0
47totime=0
48lsn=""
49ecmd=""
50rlimit=""
51# Initially
52stagemsg="$WSREP_SST_OPT_ROLE"
53cpat=""
54speciald=1
55ib_home_dir=""
56ib_log_dir=""
57ib_undo_dir=""
58
59sfmt=""
60strmcmd=""
61tfmt=""
62tcmd=""
63payload=0
64pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p'"
65pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE"
66STATDIR=""
67uextra=0
68disver=""
69
70tmpopts=""
71itmpdir=""
72xtmpdir=""
73
74scomp=""
75sdecomp=""
76
77ssl_dhparams=""
78
79compress='none'
80compress_chunk=""
81compress_threads=""
82
83backup_threads=""
84
85encrypt_threads=""
86encrypt_chunk=""
87
88readonly SECRET_TAG="secret"
89
90# Required for backup locks
91# For backup locks it is 1 sent by joiner
92sst_ver=1
93
94if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then
95    pvopts="$pvopts $pvformat"
96fi
97pcmd="pv $pvopts"
98declare -a RC
99
100BACKUP_BIN=$(commandex 'mariabackup')
101if [ -z "$BACKUP_BIN" ]; then
102    wsrep_log_error 'mariabackup binary not found in path'
103    exit 42
104fi
105
106DATA="$WSREP_SST_OPT_DATA"
107INFO_FILE="xtrabackup_galera_info"
108IST_FILE="xtrabackup_ist"
109MAGIC_FILE="$DATA/$INFO_FILE"
110
111INNOAPPLYLOG="$DATA/mariabackup.prepare.log"
112INNOMOVELOG="$DATA/mariabackup.move.log"
113INNOBACKUPLOG="$DATA/mariabackup.backup.log"
114
115# Setting the path for ss and ip
116export PATH="/usr/sbin:/sbin:$PATH"
117
118timeit()
119{
120    local stage="$1"
121    shift
122    local cmd="$@"
123    local x1 x2 took extcode
124
125    if [ $ttime -eq 1 ]; then
126        x1=$(date +%s)
127        wsrep_log_info "Evaluating $cmd"
128        eval "$cmd"
129        extcode=$?
130        x2=$(date +%s)
131        took=$(( x2-x1 ))
132        wsrep_log_info "NOTE: $stage took $took seconds"
133        totime=$(( totime+took ))
134    else
135        wsrep_log_info "Evaluating $cmd"
136        eval "$cmd"
137        extcode=$?
138    fi
139    return $extcode
140}
141
142get_keys()
143{
144    # $encrypt -eq 1 is for internal purposes only
145    if [ $encrypt -ge 2 -o $encrypt -eq -1 ]; then
146        return
147    fi
148
149    if [ $encrypt -eq 0 ]; then
150        if [ -n "$ealgo" -o -n "$ekey" -o -n "$ekeyfile" ]; then
151            wsrep_log_error "Options for encryption are specified," \
152                            "but encryption itself is disabled. SST may fail."
153        fi
154        return
155    fi
156
157    if [ $sfmt = 'tar' ]; then
158        wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \
159                       "cannot be enabled with tar format"
160        encrypt=-1
161        return
162    fi
163
164    wsrep_log_info "Key based encryption enabled in my.cnf"
165
166    if [ -z "$ealgo" ]; then
167        wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out"
168        exit 3
169    fi
170
171    if [ -z "$ekey" ]; then
172        if [ ! -r "$ekeyfile" ]; then
173            wsrep_log_error "FATAL: Either key must be specified" \
174                            "or keyfile must be readable"
175            exit 3
176        fi
177    fi
178
179    if [ "$eformat" = 'openssl' ]; then
180        get_openssl
181        if [ -z "$OPENSSL_BINARY" ]; then
182            wsrep_log_error "If encryption using the openssl is enabled," \
183                            "then you need to install openssl"
184            exit 2
185        fi
186        ecmd="'$OPENSSL_BINARY' enc -$ealgo"
187        if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then
188            ecmd="$ecmd -pbkdf2"
189        elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then
190            ecmd="$ecmd -iter 1"
191        elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then
192            ecmd="$ecmd -md sha256"
193        fi
194        if [ -z "$ekey" ]; then
195            ecmd="$ecmd -kfile '$ekeyfile'"
196        else
197            ecmd="$ecmd -k '$ekey'"
198        fi
199    elif [ "$eformat" = 'xbcrypt' ]; then
200        if [ -z "$(commandex xbcrypt)" ]; then
201            wsrep_log_error "If encryption using the xbcrypt is enabled," \
202                            "then you need to install xbcrypt"
203            exit 2
204        fi
205        wsrep_log_info "NOTE: xbcrypt-based encryption," \
206                       "supported only from Xtrabackup 2.1.4"
207        if [ -z "$ekey" ]; then
208            ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key-file='$ekeyfile'"
209        else
210            ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key='$ekey'"
211        fi
212        if [ -n "$encrypt_threads" ]; then
213            ecmd="$ecmd --encrypt-threads=$encrypt_threads"
214        fi
215        if [ -n "$encrypt_chunk" ]; then
216            ecmd="$ecmd --encrypt-chunk-size=$encrypt_chunk"
217        fi
218    else
219        wsrep_log_error "Unknown encryption format='$eformat'"
220        exit 2
221    fi
222
223    if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
224        ecmd="$ecmd -d"
225    fi
226
227    stagemsg="$stagemsg-XB-Encrypted"
228}
229
230get_transfer()
231{
232    if [ $tfmt = 'nc' ]; then
233        wsrep_log_info "Using netcat as streamer"
234        wsrep_check_programs nc
235        tcmd="nc"
236        if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
237            if nc -h 2>&1 | grep -q 'ncat'; then
238                wsrep_log_info "Using Ncat as streamer"
239                tcmd="$tcmd -l"
240            elif nc -h 2>&1 | grep -qw -- '-d'; then
241                wsrep_log_info "Using Debian netcat as streamer"
242                tcmd="$tcmd -dl"
243                if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then
244                    # When host is not explicitly specified (when only the port
245                    # is specified) netcat can only bind to an IPv4 address if
246                    # the "-6" option is not explicitly specified:
247                    tcmd="$tcmd -6"
248                fi
249            else
250                wsrep_log_info "Using traditional netcat as streamer"
251                tcmd="$tcmd -l -p"
252            fi
253            tcmd="$tcmd $SST_PORT"
254        else
255            # Check to see if netcat supports the '-N' flag.
256            # -N Shutdown the network socket after EOF on stdin
257            # If it supports the '-N' flag, then we need to use the '-N'
258            # flag, otherwise the transfer will stay open after the file
259            # transfer and cause the command to timeout.
260            # Older versions of netcat did not need this flag and will
261            # return an error if the flag is used.
262            if nc -h 2>&1 | grep -qw -- '-N'; then
263                tcmd="$tcmd -N"
264                wsrep_log_info "Using nc -N"
265            fi
266            # netcat doesn't understand [] around IPv6 address
267            if nc -h 2>&1 | grep -q ncat; then
268                wsrep_log_info "Using Ncat as streamer"
269            elif nc -h 2>&1 | grep -qw -- '-d'; then
270                wsrep_log_info "Using Debian netcat as streamer"
271            else
272                wsrep_log_info "Using traditional netcat as streamer"
273                tcmd="$tcmd -q0"
274            fi
275            tcmd="$tcmd $WSREP_SST_OPT_HOST_UNESCAPED $SST_PORT"
276        fi
277    else
278        tfmt='socat'
279
280        wsrep_log_info "Using socat as streamer"
281        wsrep_check_programs socat
282
283        if [ -n "$sockopt" ]; then
284            sockopt=$(trim_string "$sockopt" ',')
285            if [ -n "$sockopt" ]; then
286                sockopt=",$sockopt"
287            fi
288        fi
289
290        # Add an option for ipv6 if needed:
291        if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then
292            # If sockopt contains 'pf=ip6' somewhere in the middle,
293            # this will not interfere with socat, but exclude the trivial
294            # cases when sockopt contains 'pf=ip6' as prefix or suffix:
295            if [ "$sockopt" = "${sockopt#,pf=ip6}" -a \
296                 "$sockopt" = "${sockopt%,pf=ip6}" ]
297            then
298                sockopt=",pf=ip6$sockopt"
299            fi
300        fi
301
302        if [ $encrypt -lt 2 ]; then
303            if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
304                tcmd="socat -u TCP-LISTEN:$SST_PORT,reuseaddr$sockopt stdio"
305            else
306                tcmd="socat -u stdio TCP:$REMOTEIP:$SST_PORT$sockopt"
307            fi
308            return
309        fi
310
311        if ! socat -V | grep -q -F 'WITH_OPENSSL 1'; then
312            wsrep_log_error "******** FATAL ERROR ************************************************ "
313            wsrep_log_error "* Encryption requested, but socat is not OpenSSL enabled (encrypt=$encrypt) *"
314            wsrep_log_error "********************************************************************* "
315            exit 2
316        fi
317
318        local action='Decrypting'
319        if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
320            tcmd="socat -u openssl-listen:$SST_PORT,reuseaddr"
321        else
322            tcmd="socat -u stdio openssl-connect:$REMOTEIP:$SST_PORT"
323            action='Encrypting'
324        fi
325
326        if [ "${sockopt#*,dhparam=}" != "$sockopt" ]; then
327            if [ -z "$ssl_dhparams" ]; then
328                # Determine the socat version
329                SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*')
330                if [ -z "$SOCAT_VERSION" ]; then
331                    wsrep_log_error "******** FATAL ERROR ******************"
332                    wsrep_log_error "* Cannot determine the socat version. *"
333                    wsrep_log_error "***************************************"
334                    exit 2
335                fi
336                if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then
337                    # socat versions < 1.7.3 will have 512-bit dhparams (too small)
338                    # so create 2048-bit dhparams and send that as a parameter:
339                    check_for_dhparams
340                fi
341            fi
342            if [ -n "$ssl_dhparams" ]; then
343                tcmd="$tcmd,dhparam='$ssl_dhparams'"
344            fi
345        fi
346
347        CN_option=",commonname=''"
348
349        if [ $encrypt -eq 2 ]; then
350            wsrep_log_info \
351                "Using openssl based encryption with socat: with crt and pem"
352            if [ -z "$tpem" -o -z "$tcert$tcap" ]; then
353                wsrep_log_error \
354                    "Both PEM file and CRT file (or path) are required"
355                exit 22
356            fi
357            verify_ca_matches_cert "$tpem" "$tcert" "$tcap"
358            tcmd="$tcmd,cert='$tpem'"
359            if [ -n "$tcert" ]; then
360                tcmd="$tcmd,cafile='$tcert'"
361            fi
362            if [ -n "$tcap" ]; then
363                tcmd="$tcmd,capath='$tcap'"
364            fi
365            stagemsg="$stagemsg-OpenSSL-Encrypted-2"
366            wsrep_log_info "$action with cert='$tpem', ca='$tcert', capath='$tcap'"
367        elif [ $encrypt -eq 3 -o $encrypt -eq 4 ]; then
368            wsrep_log_info \
369                "Using openssl based encryption with socat: with key and crt"
370            if [ -z "$tpem" -o -z "$tkey" ]; then
371                wsrep_log_error "Both the certificate file (or path) and" \
372                                "the key file are required"
373                exit 22
374            fi
375            verify_cert_matches_key "$tpem" "$tkey"
376            stagemsg="$stagemsg-OpenSSL-Encrypted-3"
377            if [ -z "$tcert$tcap" ]; then
378                if [ $encrypt -eq 4 ]; then
379                    wsrep_log_error \
380                        "Peer certificate file (or path) required if encrypt=4"
381                    exit 22
382                fi
383                # no verification
384                CN_option=""
385                tcmd="$tcmd,cert='$tpem',key='$tkey',verify=0"
386                wsrep_log_info \
387                    "$action with cert='$tpem', key='$tkey', verify=0"
388            else
389                # CA verification
390                verify_ca_matches_cert "$tpem" "$tcert" "$tcap"
391                if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then
392                    CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'"
393                elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' -o $encrypt -eq 4 ]
394                then
395                    CN_option=",commonname=''"
396                elif is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then
397                    CN_option=',commonname=localhost'
398                else
399                    CN_option=",commonname='$WSREP_SST_OPT_HOST_UNESCAPED'"
400                fi
401                tcmd="$tcmd,cert='$tpem',key='$tkey'"
402                if [ -n "$tcert" ]; then
403                    tcmd="$tcmd,cafile='$tcert'"
404                fi
405                if [ -n "$tcap" ]; then
406                    tcmd="$tcmd,capath='$tcap'"
407                fi
408                wsrep_log_info "$action with cert='$tpem', key='$tkey'," \
409                               "ca='$tcert', capath='$tcap'"
410            fi
411        else
412            wsrep_log_info "Unknown encryption mode: encrypt=$encrypt"
413            exit 22
414        fi
415
416        tcmd="$tcmd$CN_option$sockopt"
417
418        if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
419            tcmd="$tcmd stdio"
420        fi
421    fi
422}
423
424get_footprint()
425{
426    pushd "$WSREP_SST_OPT_DATA" 1>/dev/null
427    payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' \
428              -type f -print0 | du --files0-from=- --block-size=1 -c -s | \
429              awk 'END { print $1 }')
430    if [ "$compress" != 'none' ]; then
431        # QuickLZ has around 50% compression ratio
432        # When compression/compaction used, the progress is only an approximate.
433        payload=$(( payload*1/2 ))
434    fi
435    popd 1>/dev/null
436    pcmd="$pcmd -s $payload"
437    adjust_progress
438}
439
440adjust_progress()
441{
442    if [ -z "$(commandex pv)" ]; then
443        wsrep_log_error "pv not found in path: $PATH"
444        wsrep_log_error "Disabling all progress/rate-limiting"
445        pcmd=""
446        rlimit=""
447        progress=""
448        return
449    fi
450
451    if [ -n "$progress" -a "$progress" != '1' ]; then
452        if [ -e "$progress" ]; then
453            pcmd="$pcmd 2>>'$progress'"
454        else
455            pcmd="$pcmd 2>'$progress'"
456        fi
457    elif [ -z "$progress" -a -n "$rlimit" ]; then
458            # When rlimit is non-zero
459            pcmd="pv -q"
460    fi
461
462    if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
463        wsrep_log_info "Rate-limiting SST to $rlimit"
464        pcmd="$pcmd -L \$rlimit"
465    fi
466}
467
468encgroups='--mysqld|sst|xtrabackup'
469
470read_cnf()
471{
472    sfmt=$(parse_cnf sst streamfmt 'mbstream')
473    tfmt=$(parse_cnf sst transferfmt 'socat')
474
475    encrypt=$(parse_cnf "$encgroups" 'encrypt' 0)
476    tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | \
477            tr [:lower:] [:upper:])
478
479    if [ $encrypt -eq 0 -o $encrypt -ge 2 ]; then
480        if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ]; then
481            check_server_ssl_config
482        fi
483        if [ "$tmode" != 'DISABLED' ]; then
484            if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ]
485            then
486                encrypt=3 # enable cert/key SSL encyption
487                # avoid CA verification if not set explicitly:
488                # nodes may happen to have different CA if self-generated,
489                # zeroing up tcert and tcap does the trick:
490                if [ "${tmode#VERIFY}" = "$tmode" ]; then
491                    tcert=""
492                    tcap=""
493                fi
494            fi
495        fi
496    elif [ $encrypt -eq 1 ]; then
497        ealgo=$(parse_cnf "$encgroups" 'encrypt-algo')
498        eformat=$(parse_cnf "$encgroups" 'encrypt-format' 'openssl')
499        ekey=$(parse_cnf "$encgroups" 'encrypt-key')
500        # The keyfile should be read only when the key
501        # is not specified or empty:
502        if [ -z "$ekey" ]; then
503            ekeyfile=$(parse_cnf "$encgroups" 'encrypt-key-file')
504        fi
505    fi
506
507    wsrep_log_info "SSL configuration: CA='$tcert', CAPATH='$tcap'," \
508                   "CERT='$tpem', KEY='$tkey', MODE='$tmode'," \
509                   "encrypt='$encrypt'"
510
511    sockopt=$(parse_cnf sst sockopt "")
512    progress=$(parse_cnf sst progress "")
513    ttime=$(parse_cnf sst time 0)
514    cpat='.*\.pem$\|.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$'
515    [ "$OS" = 'FreeBSD' ] && cpat=$(echo "$cpat" | sed 's/\\|/|/g')
516    cpat=$(parse_cnf sst cpat "$cpat")
517    scomp=$(parse_cnf sst compressor "")
518    sdecomp=$(parse_cnf sst decompressor "")
519
520    rlimit=$(parse_cnf sst rlimit "")
521    uextra=$(parse_cnf sst use-extra 0)
522    speciald=$(parse_cnf sst sst-special-dirs 1)
523    iopts=$(parse_cnf sst inno-backup-opts "")
524    iapts=$(parse_cnf sst inno-apply-opts "")
525    impts=$(parse_cnf sst inno-move-opts "")
526    stimeout=$(parse_cnf sst sst-initial-timeout 300)
527    ssyslog=$(parse_cnf sst sst-syslog 0)
528    ssystag=$(parse_cnf mysqld_safe syslog-tag "${SST_SYSLOG_TAG:-}")
529    ssystag="$ssystag-"
530    sstlogarchive=$(parse_cnf sst sst-log-archive 1)
531    sstlogarchivedir=$(parse_cnf sst sst-log-archive-dir '/tmp/sst_log_archive')
532
533    if [ $speciald -eq 0 ]; then
534        wsrep_log_error \
535            "sst-special-dirs equal to 0 is not supported, falling back to 1"
536        speciald=1
537    fi
538
539    if [ $ssyslog -ne -1 ]; then
540        ssyslog=$(in_config 'mysqld_safe' 'syslog')
541    fi
542
543    if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
544        compress=$(parse_cnf "$encgroups" 'compress' 'none')
545        if [ "$compress" != 'none' ]; then
546            compress_chunk=$(parse_cnf "$encgroups" 'compress-chunk-size')
547            compress_threads=$(parse_cnf "$encgroups" 'compress-threads')
548        fi
549    fi
550
551    backup_threads=$(parse_cnf "$encgroups" 'backup-threads')
552
553    if [ "$eformat" = 'xbcrypt' ]; then
554        encrypt_threads=$(parse_cnf "$encgroups" 'encrypt-threads')
555        encrypt_chunk=$(parse_cnf "$encgroups" 'encrypt-chunk-size')
556    fi
557}
558
559get_stream()
560{
561    if [ "$sfmt" = 'mbstream' -o "$sfmt" = 'xbstream' ]; then
562        sfmt='mbstream'
563        local STREAM_BIN=$(commandex "$sfmt")
564        if [ -z "$STREAM_BIN" ]; then
565            wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path"
566            exit 42
567        fi
568        if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
569            strmcmd="'$STREAM_BIN' -x"
570        else
571            strmcmd="'$STREAM_BIN' -c '$INFO_FILE'"
572        fi
573    else
574        sfmt='tar'
575        if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
576            strmcmd='tar xfi -'
577        else
578            strmcmd="tar cf - '$INFO_FILE'"
579        fi
580    fi
581    wsrep_log_info "Streaming with $sfmt"
582}
583
584sig_joiner_cleanup()
585{
586    wsrep_log_error "Removing $MAGIC_FILE file due to signal"
587    [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
588}
589
590cleanup_at_exit()
591{
592    # Since this is invoked just after exit NNN
593    local estatus=$?
594    if [ $estatus -ne 0 ]; then
595        wsrep_log_error "Cleanup after exit with status: $estatus"
596    fi
597
598    if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
599        wsrep_log_info "Removing the sst_in_progress file"
600        wsrep_cleanup_progress_file
601    else
602        if [ -n "$BACKUP_PID" ]; then
603            if check_pid "$BACKUP_PID" 1; then
604                wsrep_log_error \
605                    "mariabackup process is still running. Killing..."
606                cleanup_pid $CHECK_PID "$BACKUP_PID"
607            fi
608        fi
609        [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" || :
610    fi
611
612    if [ -n "$progress" -a -p "$progress" ]; then
613        wsrep_log_info "Cleaning up fifo file: $progress"
614        rm -f "$progress" || :
615    fi
616
617    wsrep_log_info "Cleaning up temporary directories"
618
619    if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
620        [ -n "$STATDIR" -a -d "$STATDIR" ] && rm -rf "$STATDIR" || :
621    else
622        [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || :
623        [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || :
624    fi
625
626    # Final cleanup
627    pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :)
628
629    # This means no setsid done in mysqld.
630    # We don't want to kill mysqld here otherwise.
631    if [ -n "$pgid" ]; then
632        if [ $$ -eq $pgid ]; then
633            # This means a signal was delivered to the process.
634            # So, more cleanup.
635            if [ $estatus -ge 128 ]; then
636                kill -KILL -- -$$ || :
637            fi
638        fi
639    fi
640
641    if [ -n "${SST_PID:-}" ]; then
642        [ -f "$SST_PID" ] && rm -f "$SST_PID" || :
643    fi
644
645    exit $estatus
646}
647
648setup_ports()
649{
650    SST_PORT="$WSREP_SST_OPT_PORT"
651    if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
652        REMOTEIP="$WSREP_SST_OPT_HOST"
653        lsn="$WSREP_SST_OPT_LSN"
654        sst_ver="$WSREP_SST_OPT_SST_VER"
655    fi
656}
657
658#
659# Waits ~30 seconds for socat or nc to open the port and
660# then reports ready, regardless of timeout.
661#
662wait_for_listen()
663{
664    local PORT="$1"
665    local ADDR="$2"
666    local MODULE="$3"
667    for i in {1..150}; do
668        if check_port "" "$PORT" 'socat|nc'; then
669            break
670        fi
671        sleep 0.2
672    done
673    echo "ready $ADDR/$MODULE//$sst_ver"
674}
675
676check_extra()
677{
678    local use_socket=1
679    if [ $uextra -eq 1 ]; then
680        local thread_handling=$(parse_cnf '--mysqld' 'thread-handling')
681        if [ "$thread_handling" = 'pool-of-threads' ]; then
682            local eport=$(parse_cnf '--mysqld' 'extra-port')
683            if [ -n "$eport" ]; then
684                # mariabackup works only locally.
685                # Hence, setting host to 127.0.0.1 unconditionally:
686                wsrep_log_info "SST through extra_port $eport"
687                INNOEXTRA="$INNOEXTRA --host=127.0.0.1 --port=$eport"
688                use_socket=0
689            else
690                wsrep_log_error "Extra port $eport null, failing"
691                exit 1
692            fi
693        else
694            wsrep_log_info "Thread pool not set, ignore the option use_extra"
695        fi
696    fi
697    if [ $use_socket -eq 1 -a -n "$WSREP_SST_OPT_SOCKET" ]; then
698        INNOEXTRA="$INNOEXTRA --socket='$WSREP_SST_OPT_SOCKET'"
699    fi
700}
701
702recv_joiner()
703{
704    local dir="$1"
705    local msg="$2"
706    local tmt=$3
707    local checkf=$4
708    local wait=$5
709
710    if [ ! -d "$dir" ]; then
711        # This indicates that IST is in progress
712        return
713    fi
714
715    local ltcmd="$tcmd"
716    if [ $tmt -gt 0 ]; then
717        if [ -n "$(commandex timeout)" ]; then
718            if timeout --help 2>&1 | grep -qw -- '-k'; then
719                ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd"
720            else
721                ltcmd="timeout -s9 $tmt $tcmd"
722            fi
723        fi
724    fi
725
726    pushd "$dir" 1>/dev/null
727    set +e
728
729    if [ $wait -ne 0 ]; then
730        wait_for_listen "$SST_PORT" "$ADDR" "$MODULE" &
731    fi
732
733    timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
734
735    set -e
736    popd 1>/dev/null
737
738    if [ ${RC[0]} -eq 124 ]; then
739        wsrep_log_error "Possible timeout in receiving first data from" \
740                        "donor in gtid stage: exit codes: ${RC[@]}"
741        exit 32
742    fi
743
744    for ecode in "${RC[@]}"; do
745        if [ $ecode -ne 0 ]; then
746            wsrep_log_error "Error while getting data from donor node:" \
747                            "exit codes: ${RC[@]}"
748            exit 32
749        fi
750    done
751
752    if [ $checkf -eq 1 ]; then
753        if [ ! -r "$MAGIC_FILE" ]; then
754            # this message should cause joiner to abort
755            wsrep_log_error "receiving process ended without creating" \
756                            "'$MAGIC_FILE'"
757            wsrep_log_info "Contents of datadir"
758            wsrep_log_info $(ls -l "$dir/"*)
759            exit 32
760        fi
761
762        # check donor supplied secret
763        SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \
764                 cut -d ' ' -f 2)
765        if [ "$SECRET" != "$MY_SECRET" ]; then
766            wsrep_log_error "Donor does not know my secret!"
767            wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'"
768            exit 32
769        fi
770
771        # remove secret from the magic file
772        grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" > "$MAGIC_FILE.new"
773        mv "$MAGIC_FILE.new" "$MAGIC_FILE"
774    fi
775}
776
777send_donor()
778{
779    local dir="$1"
780    local msg="$2"
781
782    pushd "$dir" 1>/dev/null
783    set +e
784    timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
785    set -e
786    popd 1>/dev/null
787
788    for ecode in "${RC[@]}"; do
789        if [ $ecode -ne 0 ]; then
790            wsrep_log_error "Error while sending data to joiner node:" \
791                            "exit codes: ${RC[@]}"
792            exit 32
793        fi
794    done
795}
796
797monitor_process()
798{
799    local sst_stream_pid=$1
800
801    while true ; do
802        if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then
803            wsrep_log_error \
804                "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \
805                "terminated unexpectedly."
806            kill -- -"$WSREP_SST_OPT_PARENT"
807            exit 32
808        fi
809        if ! ps -p "$sst_stream_pid" >/dev/null 2>&1; then
810            break
811        fi
812        sleep 0.1
813    done
814}
815
816[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
817
818if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then
819    wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'"
820    exit 22
821fi
822
823read_cnf
824setup_ports
825
826if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then
827    disver=' --no-version-check'
828fi
829
830# if no command line argument and INNODB_DATA_HOME_DIR environment variable
831# is not set, try to get it from my.cnf:
832if [ -z "$INNODB_DATA_HOME_DIR" ]; then
833    INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir')
834fi
835
836OLD_PWD="$(pwd)"
837
838cd "$WSREP_SST_OPT_DATA"
839if [ -n "$INNODB_DATA_HOME_DIR" ]; then
840    # handle both relative and absolute paths
841    [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
842    cd "$INNODB_DATA_HOME_DIR"
843fi
844INNODB_DATA_HOME_DIR=$(pwd -P)
845
846cd "$OLD_PWD"
847
848if [ $ssyslog -eq 1 ]; then
849    if [ -n "$(commandex logger)" ]; then
850        wsrep_log_info "Logging all stderr of SST/mariabackup to syslog"
851
852        exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE)
853
854        wsrep_log_error()
855        {
856            logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@"
857        }
858
859        wsrep_log_info()
860        {
861            logger -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@"
862        }
863    else
864        wsrep_log_error "logger not in path: $PATH. Ignoring"
865    fi
866    INNOAPPLY="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply"
867    INNOMOVE="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move"
868    INNOBACKUP="2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)"
869else
870    if [ $sstlogarchive -eq 1 ]
871    then
872        ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N")
873
874        if [ -n "$sstlogarchivedir" ]; then
875            if [ ! -d "$sstlogarchivedir" ]; then
876                mkdir -p "$sstlogarchivedir"
877            fi
878        fi
879
880        if [ -e "$INNOAPPLYLOG" ]; then
881            if [ -n "$sstlogarchivedir" ]; then
882                newfile=$(basename "$INNOAPPLYLOG")
883                newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP"
884            else
885                newfile="$INNOAPPLYLOG.$ARCHIVETIMESTAMP"
886            fi
887            wsrep_log_info "Moving '$INNOAPPLYLOG' to '$newfile'"
888            mv "$INNOAPPLYLOG" "$newfile"
889            gzip "$newfile"
890        fi
891
892        if [ -e "$INNOMOVELOG" ]; then
893            if [ -n "$sstlogarchivedir" ]; then
894                newfile=$(basename "$INNOMOVELOG")
895                newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP"
896            else
897                newfile="$INNOMOVELOG.$ARCHIVETIMESTAMP"
898            fi
899            wsrep_log_info "Moving '$INNOMOVELOG' to '$newfile'"
900            mv "$INNOMOVELOG" "$newfile"
901            gzip "$newfile"
902        fi
903
904        if [ -e "$INNOBACKUPLOG" ]; then
905            if [ -n "$sstlogarchivedir" ]; then
906                newfile=$(basename "$INNOBACKUPLOG")
907                newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP"
908            else
909                newfile="$INNOBACKUPLOG.$ARCHIVETIMESTAMP"
910            fi
911            wsrep_log_info "Moving '$INNOBACKUPLOG' to '$newfile'"
912            mv "$INNOBACKUPLOG" "$newfile"
913            gzip "$newfile"
914        fi
915    fi
916    INNOAPPLY="> '$INNOAPPLYLOG' 2>&1"
917    INNOMOVE="> '$INNOMOVELOG' 2>&1"
918    INNOBACKUP="2> '$INNOBACKUPLOG'"
919fi
920
921setup_commands()
922{
923    local mysqld_args=""
924    if [ -n "$WSREP_SST_OPT_MYSQLD" ]; then
925        mysqld_args=" --mysqld-args $WSREP_SST_OPT_MYSQLD"
926    fi
927    local recovery=""
928    if [ -n "$INNODB_FORCE_RECOVERY" ]; then
929        recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY"
930    fi
931    INNOAPPLY="$BACKUP_BIN --prepare$disver$recovery${iapts:+ }$iapts$INNOEXTRA --target-dir='$DATA' --datadir='$DATA'$mysqld_args $INNOAPPLY"
932    INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE"
933    INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF --backup$disver${iopts:+ }$iopts$tmpopts$INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA'$mysqld_args $INNOBACKUP"
934}
935
936get_stream
937get_transfer
938
939if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]
940then
941    trap cleanup_at_exit EXIT
942
943    if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
944    then
945        if [ -z "$sst_ver" ]; then
946            wsrep_log_error "Upgrade joiner to 5.6.21 or higher for backup locks support"
947            wsrep_log_error "The joiner is not supported for this version of donor"
948            exit 93
949        fi
950
951        tmpdir=$(parse_cnf "$encgroups" 'tmpdir')
952        if [ -z "$tmpdir" ]; then
953            xtmpdir="$(mktemp -d)"
954        elif [ "$OS" = 'Linux' ]; then
955            xtmpdir=$(mktemp '-d' "--tmpdir=$tmpdir")
956        else
957            xtmpdir=$(TMPDIR="$tmpdir"; mktemp '-d')
958        fi
959
960        wsrep_log_info "Using '$xtmpdir' as mariabackup temporary directory"
961        tmpopts=" --tmpdir='$xtmpdir'"
962
963        itmpdir="$(mktemp -d)"
964        wsrep_log_info "Using '$itmpdir' as mariabackup working directory"
965
966        usrst=0
967        if [ -n "$WSREP_SST_OPT_USER" ]; then
968           INNOEXTRA="$INNOEXTRA --user='$WSREP_SST_OPT_USER'"
969           usrst=1
970        fi
971
972        if [ -n "$WSREP_SST_OPT_PSWD" ]; then
973            export MYSQL_PWD="$WSREP_SST_OPT_PSWD"
974        elif [ $usrst -eq 1 ]; then
975            # Empty password, used for testing, debugging etc.
976            unset MYSQL_PWD
977        fi
978
979        check_extra
980
981        wsrep_log_info "Streaming GTID file before SST"
982
983        # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id
984        # (separated by a space).
985        echo "$WSREP_SST_OPT_GTID $WSREP_SST_OPT_GTID_DOMAIN_ID" > "$MAGIC_FILE"
986
987        if [ -n "$WSREP_SST_OPT_REMOTE_PSWD" ]; then
988            # Let joiner know that we know its secret
989            echo "$SECRET_TAG $WSREP_SST_OPT_REMOTE_PSWD" >> "$MAGIC_FILE"
990        fi
991
992        ttcmd="$tcmd"
993
994        if [ -n "$scomp" ]; then
995            tcmd="$scomp | $tcmd"
996        fi
997
998        get_keys
999        if [ $encrypt -eq 1 ]; then
1000            tcmd="$ecmd | $tcmd"
1001        fi
1002
1003        send_donor "$DATA" "$stagemsg-gtid"
1004
1005        # Restore the transport commmand to its original state
1006        tcmd="$ttcmd"
1007
1008        if [ -n "$progress" ]; then
1009            get_footprint
1010            tcmd="$pcmd | $tcmd"
1011        elif [ -n "$rlimit" ]; then
1012            adjust_progress
1013            tcmd="$pcmd | $tcmd"
1014        fi
1015
1016        wsrep_log_info "Sleeping before data transfer for SST"
1017        sleep 10
1018
1019        wsrep_log_info "Streaming the backup to joiner at $REMOTEIP:$SST_PORT"
1020
1021        # Add compression to the head of the stream (if specified)
1022        if [ -n "$scomp" ]; then
1023            tcmd="$scomp | $tcmd"
1024        fi
1025
1026        # Add encryption to the head of the stream (if specified)
1027        if [ $encrypt -eq 1 ]; then
1028            tcmd="$ecmd | $tcmd"
1029        fi
1030
1031        iopts="--databases-exclude='lost+found'${iopts:+ }$iopts"
1032
1033        if [ ${FORCE_FTWRL:-0} -eq 1 ]; then
1034            wsrep_log_info "Forcing FTWRL due to environment variable" \
1035                           "FORCE_FTWRL equal to $FORCE_FTWRL"
1036            iopts="--no-backup-locks${iopts:+ }$iopts"
1037        fi
1038
1039        # if compression is enabled for backup files, then add the
1040        # appropriate options to the mariabackup command line:
1041        if [ "$compress" != 'none' ]; then
1042            iopts="--compress${compress:+=$compress}${iopts:+ }$iopts"
1043            if [ -n "$compress_threads" ]; then
1044                iopts="--compress-threads=$compress_threads${iopts:+ }$iopts"
1045            fi
1046            if [ -n "$compress_chunk" ]; then
1047                iopts="--compress-chunk-size=$compress_chunk${iopts:+ }$iopts"
1048            fi
1049        fi
1050
1051        if [ -n "$backup_threads" ]; then
1052            iopts="--parallel=$backup_threads${iopts:+ }$iopts"
1053        fi
1054
1055        setup_commands
1056        set +e
1057        timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
1058        set -e
1059
1060        if [ ${RC[0]} -ne 0 ]; then
1061            wsrep_log_error "mariabackup finished with error: ${RC[0]}." \
1062                            "Check syslog or '$INNOBACKUPLOG' for details"
1063            exit 22
1064        elif [ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]; then
1065            wsrep_log_error "$tcmd finished with error: ${RC[1]}"
1066            exit 22
1067        fi
1068
1069        # mariabackup implicitly writes PID to fixed location in $xtmpdir
1070        BACKUP_PID="$xtmpdir/xtrabackup_pid"
1071
1072    else # BYPASS FOR IST
1073
1074        wsrep_log_info "Bypassing the SST for IST"
1075        echo "continue" # now server can resume updating data
1076
1077        # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id
1078        # (separated by a space).
1079        echo "$WSREP_SST_OPT_GTID $WSREP_SST_OPT_GTID_DOMAIN_ID" > "$MAGIC_FILE"
1080        echo "1" > "$DATA/$IST_FILE"
1081
1082        if [ -n "$scomp" ]; then
1083            tcmd="$scomp | $tcmd"
1084        fi
1085
1086        get_keys
1087        if [ $encrypt -eq 1 ]; then
1088            tcmd="$ecmd | $tcmd"
1089        fi
1090
1091        strmcmd="$strmcmd '$IST_FILE'"
1092
1093        send_donor "$DATA" "$stagemsg-IST"
1094
1095    fi
1096
1097    echo "done $WSREP_SST_OPT_GTID"
1098    wsrep_log_info "Total time on donor: $totime seconds"
1099
1100elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]
1101then
1102    [ -e "$SST_PROGRESS_FILE" ] && \
1103        wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE"
1104    [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE"
1105
1106    ib_home_dir="$INNODB_DATA_HOME_DIR"
1107
1108    # if no command line argument and INNODB_LOG_GROUP_HOME is not set,
1109    # try to get it from my.cnf:
1110    if [ -z "$INNODB_LOG_GROUP_HOME" ]; then
1111        INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir')
1112    fi
1113
1114    ib_log_dir="$INNODB_LOG_GROUP_HOME"
1115
1116    # if no command line argument then try to get it from my.cnf:
1117    if [ -z "$INNODB_UNDO_DIR" ]; then
1118        INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory')
1119    fi
1120
1121    ib_undo_dir="$INNODB_UNDO_DIR"
1122
1123    if [ -n "$backup_threads" ]; then
1124        impts="--parallel=$backup_threads${impts:+ }$impts"
1125    fi
1126
1127    SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid"
1128
1129    # give some time for previous SST to complete:
1130    check_round=0
1131    while check_pid "$SST_PID" 0; do
1132        wsrep_log_info "previous SST is not completed, waiting for it to exit"
1133        check_round=$(( check_round + 1 ))
1134        if [ $check_round -eq 10 ]; then
1135            wsrep_log_error "previous SST script still running."
1136            exit 114 # EALREADY
1137        fi
1138        sleep 1
1139    done
1140
1141    echo $$ > "$SST_PID"
1142
1143    stagemsg='Joiner-Recv'
1144
1145    MODULE="xtrabackup_sst"
1146
1147    [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE"
1148
1149    # May need xtrabackup_checkpoints later on
1150    [ -f "$DATA/xtrabackup_binary"      ] && rm -f "$DATA/xtrabackup_binary"
1151    [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info"
1152    [ -f "$DATA/ib_logfile0"            ] && rm -f "$DATA/ib_logfile0"
1153
1154    ADDR="$WSREP_SST_OPT_ADDR"
1155
1156    if [ "${tmode#VERIFY}" != "$tmode" ]; then
1157        # backward-incompatible behavior:
1158        CN=""
1159        if [ -n "$tpem" ]; then
1160            # find out my Common Name
1161            get_openssl
1162            if [ -z "$OPENSSL_BINARY" ]; then
1163                wsrep_log_error \
1164                    'openssl not found but it is required for authentication'
1165                exit 42
1166            fi
1167            CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \
1168                 tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \
1169                 sed s/\ %//)
1170        fi
1171        MY_SECRET="$(wsrep_gen_secret)"
1172        # Add authentication data to address
1173        ADDR="$CN:$MY_SECRET@$ADDR"
1174    else
1175        MY_SECRET="" # for check down in recv_joiner()
1176    fi
1177
1178    trap sig_joiner_cleanup HUP PIPE INT TERM
1179    trap cleanup_at_exit EXIT
1180
1181    if [ -n "$progress" ]; then
1182        adjust_progress
1183        tcmd="$tcmd | $pcmd"
1184    fi
1185
1186    get_keys
1187    if [ $encrypt -eq 1 ]; then
1188        strmcmd="$ecmd | $strmcmd"
1189    fi
1190
1191    if [ -n "$sdecomp" ]; then
1192        strmcmd="$sdecomp | $strmcmd"
1193    fi
1194
1195    check_sockets_utils
1196
1197    STATDIR="$(mktemp -d)"
1198    MAGIC_FILE="$STATDIR/$INFO_FILE"
1199
1200    recv_joiner "$STATDIR" "$stagemsg-gtid" $stimeout 1 1
1201
1202    if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1
1203    then
1204        wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \
1205                        "terminated unexpectedly."
1206        exit 32
1207    fi
1208
1209    if [ ! -r "$STATDIR/$IST_FILE" ]; then
1210
1211        if [ -d "$DATA/.sst" ]; then
1212            wsrep_log_info \
1213                "WARNING: Stale temporary SST directory:" \
1214                "'$DATA/.sst' from previous state transfer, removing..."
1215            rm -rf "$DATA/.sst"
1216        fi
1217        mkdir -p "$DATA/.sst"
1218        (recv_joiner "$DATA/.sst" "$stagemsg-SST" 0 0 0) &
1219        jpid=$!
1220        wsrep_log_info "Proceeding with SST"
1221
1222        wsrep_log_info \
1223            "Cleaning the existing datadir and innodb-data/log directories"
1224        if [ "$OS" = 'FreeBSD' ]; then
1225            find -E ${ib_home_dir:+"$ib_home_dir"} \
1226                    ${ib_undo_dir:+"$ib_undo_dir"} \
1227                    ${ib_log_dir:+"$ib_log_dir"} \
1228                    "$DATA" -mindepth 1 -prune -regex "$cpat" \
1229                    -o -exec rm -rfv {} 1>&2 \+
1230        else
1231            find ${ib_home_dir:+"$ib_home_dir"} \
1232                 ${ib_undo_dir:+"$ib_undo_dir"} \
1233                 ${ib_log_dir:+"$ib_log_dir"} \
1234                 "$DATA" -mindepth 1 -prune -regex "$cpat" \
1235                 -o -exec rm -rfv {} 1>&2 \+
1236        fi
1237
1238        get_binlog
1239
1240        if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
1241            binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG")
1242            if [ -d "$binlog_dir" ]; then
1243                cd "$binlog_dir"
1244                wsrep_log_info "Cleaning the binlog directory $binlog_dir as well"
1245                rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || :
1246                [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \
1247                    rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+
1248                cd "$OLD_PWD"
1249            fi
1250        fi
1251
1252        TDATA="$DATA"
1253        DATA="$DATA/.sst"
1254
1255        MAGIC_FILE="$DATA/$INFO_FILE"
1256        wsrep_log_info "Waiting for SST streaming to complete!"
1257        monitor_process $jpid
1258
1259        if [ ! -s "$DATA/xtrabackup_checkpoints" ]; then
1260            wsrep_log_error "xtrabackup_checkpoints missing," \
1261                            "failed mariabackup/SST on donor"
1262            exit 2
1263        fi
1264
1265        # Compact backups are not supported by mariabackup
1266        if grep -qw -F 'compact = 1' "$DATA/xtrabackup_checkpoints"; then
1267            wsrep_log_info "Index compaction detected"
1268            wsrel_log_error "Compact backups are not supported by mariabackup"
1269            exit 2
1270        fi
1271
1272        qpfiles=$(find "$DATA" -maxdepth 1 -type f -name '*.qp' -print -quit)
1273        if [ -n "$qpfiles" ]; then
1274            wsrep_log_info "Compressed qpress files found"
1275
1276            if [ -z "$(commandex qpress)" ]; then
1277                wsrep_log_error "qpress utility not found in the path"
1278                exit 22
1279            fi
1280
1281            get_proc
1282
1283            dcmd="xargs -n 2 qpress -dT$nproc"
1284
1285            if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then
1286                count=$(find "$DATA" -type f -name '*.qp' | wc -l)
1287                count=$(( count*2 ))
1288                pvopts="-f -s $count -l -N Decompression"
1289                if pv --help | grep -qw -- '-F'; then
1290                    pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'"
1291                fi
1292                pcmd="pv $pvopts"
1293                adjust_progress
1294                dcmd="$pcmd | $dcmd"
1295            fi
1296
1297            # Decompress the qpress files
1298            wsrep_log_info "Decompression with $nproc threads"
1299            timeit "Joiner-Decompression" \
1300                   "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd"
1301            extcode=$?
1302
1303            if [ $extcode -eq 0 ]; then
1304                wsrep_log_info "Removing qpress files after decompression"
1305                find "$DATA" -type f -name '*.qp' -delete
1306                if [ $? -ne 0 ]; then
1307                    wsrep_log_error \
1308                        "Something went wrong with deletion of qpress files." \
1309                        "Investigate"
1310                fi
1311            else
1312                wsrep_log_error "Decompression failed. Exit code: $extcode"
1313                exit 22
1314            fi
1315        fi
1316
1317        if  [ -n "$WSREP_SST_OPT_BINLOG" ]; then
1318
1319            BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG")
1320            BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG")
1321
1322            # To avoid comparing data directory and BINLOG_DIRNAME
1323            mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || :
1324
1325            cd "$BINLOG_DIRNAME"
1326            for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do
1327                echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX"
1328            done
1329            cd "$OLD_PWD"
1330
1331        fi
1332
1333        wsrep_log_info "Preparing the backup at $DATA"
1334        setup_commands
1335        timeit "mariabackup prepare stage" "$INNOAPPLY"
1336
1337        if [ $? -ne 0 ]; then
1338            wsrep_log_error "mariabackup apply finished with errors." \
1339                            "Check syslog or '$INNOAPPLYLOG' for details."
1340            exit 22
1341        fi
1342
1343        MAGIC_FILE="$TDATA/$INFO_FILE"
1344
1345        wsrep_log_info "Moving the backup to $TDATA"
1346        timeit "mariabackup move stage" "$INNOMOVE"
1347        if [ $? -eq 0 ]; then
1348            wsrep_log_info "Move successful, removing $DATA"
1349            rm -rf "$DATA"
1350            DATA="$TDATA"
1351        else
1352            wsrep_log_error "Move failed, keeping '$DATA' for further diagnosis"
1353            wsrep_log_error "Check syslog or '$INNOMOVELOG' for details"
1354            exit 22
1355        fi
1356
1357    else
1358
1359        wsrep_log_info "'$IST_FILE' received from donor: Running IST"
1360
1361    fi
1362
1363    if [ ! -r "$MAGIC_FILE" ]; then
1364        wsrep_log_error "SST magic file '$MAGIC_FILE' not found/readable"
1365        exit 2
1366    fi
1367
1368    coords=$(cat "$MAGIC_FILE")
1369    wsrep_log_info "Galera co-ords from recovery: $coords"
1370    cat "$MAGIC_FILE" # Output : UUID:seqno wsrep_gtid_domain_id
1371
1372    wsrep_log_info "Total time on joiner: $totime seconds"
1373fi
1374
1375exit 0
1376