1# Copyright (c) 2015, 2020 Oracle and/or its affiliates. All rights reserved. 2# 3# See the file LICENSE for license information. 4# 5# $Id$ 6# 7# TEST repmgr046 8# TEST repmgr singleton write forwarding test. 9# TEST 10# TEST The initial basic test has several purposes: test general use 11# TEST of repmgr write forwarding API options (config, timeout, stats), 12# TEST test basic forwarded put and del operations, test that expected 13# TEST errors are returned from most simple error scenarios. 14# TEST 15# TEST The 3-site test makes sure that write forwarding continues to 16# TEST operate as expected during replication group changes such as 17# TEST a change of master and a former master rejoining as a client. 18# TEST 19# TEST The other tests include cases for blobs, subdatabases, 20# TEST various duplicate options, and additional error cases. 21# TEST Run for btree only because access method shouldn't matter. 22# TEST 23proc repmgr046 { { niter 100 } { tnum "046" } args } { 24 25 source ./include.tcl 26 global databases_in_memory 27 28 if { $is_freebsd_test == 1 } { 29 puts "Skipping replication manager test on FreeBSD platform." 30 return 31 } 32 33 set method "btree" 34 set args [convert_args $method $args] 35 36 puts "Repmgr$tnum ($method): repmgr write forwarding basic test." 37 repmgr046_basic $method $niter $tnum $args 38 39 puts "Repmgr$tnum ($method): repmgr write forwarding 3-site test." 40 repmgr046_3site $method $niter $tnum $args 41 42 # Blobs are not expected to work with in-memory databases. 43 if {!$databases_in_memory} { 44 puts "Repmgr$tnum ($method): repmgr write forwarding blob test." 45 repmgr046_blob $method $niter $tnum $args 46 } 47 48 puts "Repmgr$tnum ($method): repmgr write forwarding subdatabase\ 49 and duplicate test." 50 repmgr046_subdb $method $niter $tnum $args 51 52 puts "Repmgr$tnum ($method): repmgr write forwarding dead handle test." 53 repmgr046_hdldead $method $niter $tnum $args 54} 55 56proc repmgr046_basic { method niter tnum largs } { 57 58 source ./include.tcl 59 global rep_verbose 60 global verbose_type 61 global databases_in_memory 62 global ipversion 63 set nsites 2 64 set omethod [convert_method $method] 65 66 set verbargs "" 67 if { $rep_verbose == 1 } { 68 set verbargs " -verbose {$verbose_type on} " 69 } 70 71 set sslargs [setup_repmgr_sslargs] 72 73 env_cleanup $testdir 74 set hoststr [get_hoststr $ipversion] 75 set ports [available_ports $nsites] 76 set dmlsleep 1 77 78 set masterdir $testdir/MASTERDIR 79 set clientdir $testdir/CLIENTDIR 80 81 file mkdir $masterdir 82 file mkdir $clientdir 83 84 puts "\tRepmgr$tnum.a: Start a master and a client." 85 set ma_envcmd "berkdb_env_noerr -create $verbargs $sslargs \ 86 -errpfx MASTER -home $masterdir -txn -rep -thread" 87 set masterenv [eval $ma_envcmd] 88 $masterenv rep_config {mgrforwardwrites on} 89 $masterenv repmgr -ack all \ 90 -local [list $hoststr [lindex $ports 0]] \ 91 -start master 92 93 set cl_envcmd "berkdb_env_noerr -create $verbargs $sslargs \ 94 -errpfx CLIENT -home $clientdir -txn -rep -thread" 95 set clientenv [eval $cl_envcmd] 96 $clientenv rep_config {mgrforwardwrites on} 97 $clientenv repmgr -ack all \ 98 -local [list $hoststr [lindex $ports 1]] \ 99 -remote [list $hoststr [lindex $ports 0]] \ 100 -timeout [list write_forward 3000000] \ 101 -start client 102 await_startup_done $clientenv 103 104 puts "\tRepmgr$tnum.b: Check expected config, timeout and stat values." 105 # Check rep_set_config flags. 106 error_check_good wfmas [$masterenv rep_get_config mgrforwardwrites] 1 107 error_check_good wfcli [$clientenv rep_get_config mgrforwardwrites] 1 108 # Check write forwarding timeout from rep_set_timeout. Master has 109 # default value of 5 seconds. 110 error_check_good mdefwftimeout \ 111 [$masterenv rep_get_timeout write_forward] 5000000 112 error_check_good csetwftimeout \ 113 [$clientenv rep_get_timeout write_forward] 3000000 114 # Check write forwarding repmgr stats, 0 values expected. 115 error_check_good mrcvstat [stat_field $masterenv repmgr_stat \ 116 "Forwarded write operations received"] 0 117 error_check_good cforstat [stat_field $clientenv repmgr_stat \ 118 "Write operations forwarded"] 0 119 120 # 121 # Use of -ack all guarantees that replication is complete before the 122 # repmgr send function returns and rep_test finishes. 123 # 124 puts "\tRepmgr$tnum.c: Run transactions at master." 125 set start 0 126 eval rep_test $method $masterenv NULL $niter $start 0 0 $largs 127 incr start $niter 128 129 # 130 # Basic write forwarding DML test. Perform some inserts and an update 131 # and a delete, then verify that the expected data is replicated back 132 # to the client. 133 # 134 puts "\tRepmgr$tnum.d: Perform simple DMLs to be forwarded on client." 135 if {$databases_in_memory} { 136 set dbname { "" "test.db" } 137 } else { 138 set dbname "test.db" 139 } 140 141 set mdmldb [eval "berkdb_open_noerr -create $omethod -auto_commit \ 142 -env $masterenv $largs $dbname"] 143 set cdmldb [eval "berkdb_open_noerr -create $omethod -auto_commit \ 144 -env $clientenv $largs $dbname"] 145 146 set key1 1 147 set key2 2 148 set data2 222 149 set key3 3 150 set data3 3333.33 151 set key4 4 152 # Insert key1 and key2. 153 error_check_good cdmldb_put1 \ 154 [eval $cdmldb put $key1 [chop_data $method data$key1]] 0 155 error_check_good cdmldb_put2 [eval $cdmldb put $key2 $data3] 0 156 # Now delete key1. 157 error_check_good cdmldb_del1 [eval $cdmldb del $key1] 0 158 # Update key2. 159 error_check_good cdmldb_putupd2 [eval $cdmldb put $key2 $data2] 0 160 # Insert key3. 161 error_check_good cdmldb_put3 [eval $cdmldb put $key3 $data3] 0 162 # Now delete a key that's not there, which should just succeed. 163 error_check_good cdmldb_del4 [eval $cdmldb del $key4] 0 164 # Allow time for DML to be replicated back to client. 165 tclsleep $dmlsleep 166 # Verify key1 is gone. 167 set ret [lindex [$cdmldb get $key1] 0] 168 error_check_good cdmldb_get1 $ret "" 169 # Verify key2 has updated value. 170 set ret [lindex [$cdmldb get $key2] 0] 171 error_check_good cdmldb_get2 $ret [list $key2 $data2] 172 # Verify key3 is there. 173 set ret [lindex [$cdmldb get $key3] 0] 174 error_check_good cdmldb_get3 $ret [list $key3 $data3] 175 176 # Make sure stats reflect the 6 write operations above. 177 error_check_good mrcvstat [stat_field $masterenv repmgr_stat \ 178 "Forwarded write operations received"] 6 179 error_check_good cforstat [stat_field $clientenv repmgr_stat \ 180 "Write operations forwarded"] 6 181 182 puts "\tRepmgr$tnum.e: EACCES if write forwarding disabled on master." 183 set key 4 184 $masterenv rep_config {mgrforwardwrites off} 185 catch { $cdmldb put $key [chop_data $method data$key] } res 186 error_check_good mwfoffp [is_substr $res "permission denied"] 1 187 catch { $cdmldb del $key } res 188 error_check_good mwfoffd [is_substr $res "permission denied"] 1 189 # Reenable write forwarding on master and make sure it works. 190 $masterenv rep_config {mgrforwardwrites on} 191 error_check_good cdmldb_put$key \ 192 [eval $cdmldb put $key [chop_data $method data$key]] 0 193 # Allow time for DML to be replicated back to client. 194 tclsleep $dmlsleep 195 set ret [lindex [$cdmldb get $key] 0] 196 error_check_good cdmldb_get$key $ret \ 197 [list $key [pad_data $method data$key]] 198 199 puts "\tRepmgr$tnum.f: EACCES if write forwarding disabled on client." 200 set key 5 201 $clientenv rep_config {mgrforwardwrites off} 202 catch { $cdmldb put $key [chop_data $method data$key] } res 203 error_check_good cwfoffp [is_substr $res "permission denied"] 1 204 catch { $cdmldb del $key } res 205 error_check_good cwfoffd [is_substr $res "permission denied"] 1 206 # Reenable write forwarding on client and make sure it works. 207 $clientenv rep_config {mgrforwardwrites on} 208 error_check_good cdmldb_put$key \ 209 [eval $cdmldb put $key [chop_data $method data$key]] 0 210 # Allow time for DML to be replicated back to client. 211 tclsleep $dmlsleep 212 set ret [lindex [$cdmldb get $key] 0] 213 error_check_good cdmldb_get$key $ret \ 214 [list $key [pad_data $method data$key]] 215 216 puts "\tRepmgr$tnum.g: EACCES if non-NULL transaction." 217 set key 6 218 set t [$clientenv txn] 219 catch { $cdmldb put -txn $t $key [chop_data $method data$key] } res 220 error_check_good ctxnp [is_substr $res "permission denied"] 1 221 catch { $cdmldb del -txn $t $key } res 222 error_check_good ctxnd [is_substr $res "permission denied"] 1 223 error_check_good txn_abort [$t abort] 0 224 225 puts "\tRepmgr$tnum.h: EACCES if cursor put or del." 226 set key 7 227 set c [$cdmldb cursor] 228 catch { $c put $key [chop_data $method data$key] } res 229 error_check_good cursput [is_substr $res "permission denied"] 1 230 error_check_good cdmldb_put$key \ 231 [eval $cdmldb put $key [chop_data $method data$key]] 0 232 catch { $c del $key } res 233 error_check_good cursdel [is_substr $res "permission denied"] 1 234 $c close 235 236 # The granularity of timeouts on Windows is too large for thie 237 # test to be reliable. 238 if { $is_windows_test } { 239 puts "\tRepmgr$tnum.i: Skipping for Windows platform." 240 } else { 241 puts "\tRepmgr$tnum.i: DB_TIMEOUT if operation takes too long." 242 set key 8 243 # Set a tiny write forwarding timeout. 244 $clientenv repmgr -timeout {write_forward 3} 245 catch { $cdmldb put $key [chop_data $method data$key] } res 246 error_check_good ctimeoutp [is_substr $res "timed out"] 1 247 catch { $cdmldb del $key } res 248 error_check_good ctimeoutd [is_substr $res "timed out"] 1 249 # Restore reasonable timeout and verify write forwarding works. 250 $clientenv repmgr -timeout {write_forward 3000000} 251 error_check_good cdmldb_put$key \ 252 [eval $cdmldb put $key [chop_data $method data$key]] 0 253 # Allow time for DML to be replicated back to client. 254 tclsleep $dmlsleep 255 set ret [lindex [$cdmldb get $key] 0] 256 error_check_good cdmldb_get$key $ret \ 257 [list $key [pad_data $method data$key]] 258 } 259 260 error_check_good cdmldb_close [$cdmldb close] 0 261 error_check_good mdmldb_close [$mdmldb close] 0 262 263 puts "\tRepmgr$tnum.j: EACCES if no open master database handle." 264 set cdmldb [eval "berkdb_open_noerr $omethod -auto_commit \ 265 -env $clientenv $largs $dbname"] 266 set key 9 267 catch { $cdmldb put $key [chop_data $method data$key] } res 268 error_check_good cnomashdlp [is_substr $res "permission denied"] 1 269 catch { $cdmldb del $key } res 270 error_check_good cnomashdld [is_substr $res "permission denied"] 1 271 # Open master database handle and make sure write forwarding works. 272 set mdmldb [eval "berkdb_open_noerr $omethod -auto_commit \ 273 -env $masterenv $largs $dbname"] 274 error_check_good cdmldb_put$key \ 275 [eval $cdmldb put $key [chop_data $method data$key]] 0 276 # Allow time for DML to be replicated back to client. 277 tclsleep $dmlsleep 278 set ret [lindex [$cdmldb get $key] 0] 279 error_check_good cdmldb_get$key $ret \ 280 [list $key [pad_data $method data$key]] 281 282 puts "\tRepmgr$tnum.k: EACCES for unsupported bulk put or del." 283 set key 10 284 catch { $cdmldb put -multiple $key [chop_data $method data$key] } res 285 error_check_good cnobulkp [is_substr $res "permission denied"] 1 286 catch { $cdmldb del -multiple_key $key } res 287 error_check_good cnobulkd [is_substr $res "permission denied"] 1 288 289 error_check_good mdmldb_close [$mdmldb close] 0 290 error_check_good cdmldb_close [$cdmldb close] 0 291 292 puts "\tRepmgr$tnum.l: Verify master and client database contents." 293 rep_verify $masterdir $masterenv $clientdir $clientenv 1 1 1 294 295 error_check_good client_close [$clientenv close] 0 296 error_check_good masterenv_close [$masterenv close] 0 297 298 puts "\tRepmgr$tnum.m: EACCES if no master site in repgroup." 299 # Use of default 2SITE_STRICT ensures client won't elect itself master. 300 set key 11 301 set clientenv [eval $cl_envcmd] 302 $clientenv rep_config {mgrforwardwrites on} 303 $clientenv repmgr -ack all -pri 0 \ 304 -local [list $hoststr [lindex $ports 1]] \ 305 -remote [list $hoststr [lindex $ports 0]] \ 306 -timeout [list write_forward 3000000] \ 307 -start client 308 set cdmldb [eval "berkdb_open_noerr $omethod -auto_commit \ 309 -env $clientenv $largs $dbname"] 310 catch { $cdmldb put $key [chop_data $method data$key] } res 311 error_check_good cnomasp [is_substr $res "permission denied"] 1 312 catch { $cdmldb del $key } res 313 error_check_good cnomasd [is_substr $res "permission denied"] 1 314 error_check_good cdmldb_close [$cdmldb close] 0 315 error_check_good client_close [$clientenv close] 0 316} 317 318# 319# Ensure that write forwarding continues working in a 3-site replication 320# group that goes through a change of master. Also ensure that the 321# original master restarted as a client can forward writes to the new 322# master. 323# 324proc repmgr046_3site { method niter tnum largs } { 325 global testdir 326 global rep_verbose 327 global verbose_type 328 global databases_in_memory 329 global ipversion 330 set nsites 3 331 set omethod [convert_method $method] 332 333 set verbargs "" 334 if { $rep_verbose == 1 } { 335 set verbargs " -verbose {$verbose_type on} " 336 } 337 338 set sslargs [setup_repmgr_sslargs] 339 340 env_cleanup $testdir 341 set hoststr [get_hoststr $ipversion] 342 set ports [available_ports $nsites] 343 set dmlsleep 1 344 345 set masterdir $testdir/MASTERDIR 346 set clientdir $testdir/CLIENTDIR 347 set clientdir2 $testdir/CLIENTDIR2 348 349 file mkdir $masterdir 350 file mkdir $clientdir 351 file mkdir $clientdir2 352 353 puts "\tRepmgr$tnum.3s.a: Start a master and two clients." 354 set ma_envcmd "berkdb_env_noerr -create $verbargs $sslargs \ 355 -errpfx MASTER -home $masterdir -txn -rep -thread" 356 set masterenv [eval $ma_envcmd] 357 $masterenv rep_config {mgrforwardwrites on} 358 $masterenv repmgr -ack all \ 359 -local [list $hoststr [lindex $ports 0]] \ 360 -timeout [list write_forward 4000000] \ 361 -start master 362 363 set cl_envcmd "berkdb_env_noerr -create $verbargs $sslargs \ 364 -errpfx CLIENT -home $clientdir -txn -rep -thread" 365 set clientenv [eval $cl_envcmd] 366 $clientenv rep_config {mgrforwardwrites on} 367 $clientenv repmgr -ack all -pri 50 \ 368 -local [list $hoststr [lindex $ports 1]] \ 369 -remote [list $hoststr [lindex $ports 0]] \ 370 -start client 371 await_startup_done $clientenv 372 373 set cl2_envcmd "berkdb_env_noerr -create $verbargs $sslargs \ 374 -errpfx CLIENT2 -home $clientdir2 -txn -rep -thread" 375 set clientenv2 [eval $cl2_envcmd] 376 $clientenv2 rep_config {mgrforwardwrites on} 377 $clientenv2 repmgr -ack all -pri 30 \ 378 -local [list $hoststr [lindex $ports 2]] \ 379 -remote [list $hoststr [lindex $ports 0]] \ 380 -start client 381 await_startup_done $clientenv2 382 383 if {$databases_in_memory} { 384 set dbname { "" "test.db" } 385 } else { 386 set dbname "test.db" 387 } 388 389 set mdb3s [eval "berkdb_open_noerr -create $omethod -auto_commit \ 390 -env $masterenv $largs $dbname"] 391 set cdb3s [eval "berkdb_open_noerr -create $omethod -auto_commit \ 392 -env $clientenv $largs $dbname"] 393 set c2db3s [eval "berkdb_open_noerr -create $omethod -auto_commit \ 394 -env $clientenv2 $largs $dbname"] 395 396 puts "\tRepmgr$tnum.3s.b: Do a DML on each client, verify replication." 397 set key1 1 398 set key2 2 399 error_check_good cdb3s_put1 \ 400 [eval $cdb3s put $key1 [chop_data $method data$key1]] 0 401 error_check_good c2db3s_put2 \ 402 [eval $c2db3s put $key2 [chop_data $method data$key2]] 0 403 # Allow time for DML to be replicated back to clients. 404 tclsleep $dmlsleep 405 # Verify key1 and key2 are present on all sites. 406 set ret [lindex [$mdb3s get $key1] 0] 407 error_check_good mdb3s_get1 $ret \ 408 [list $key1 [pad_data $method data$key1]] 409 set ret [lindex [$mdb3s get $key2] 0] 410 error_check_good mdb3s_get2 $ret \ 411 [list $key2 [pad_data $method data$key2]] 412 set ret [lindex [$cdb3s get $key1] 0] 413 error_check_good cdb3s_get1 $ret \ 414 [list $key1 [pad_data $method data$key1]] 415 set ret [lindex [$cdb3s get $key2] 0] 416 error_check_good cdb3s_get2 $ret \ 417 [list $key2 [pad_data $method data$key2]] 418 set ret [lindex [$c2db3s get $key1] 0] 419 error_check_good c2db3s_get1 $ret \ 420 [list $key1 [pad_data $method data$key1]] 421 set ret [lindex [$c2db3s get $key2] 0] 422 error_check_good c2db3s_get2 $ret \ 423 [list $key2 [pad_data $method data$key2]] 424 425 # Do not close client handles - this test needs to make sure 426 # they are still usable after the change of master because an 427 # application exclusively using write forwarding shouldn't need 428 # to track changes of master. 429 error_check_good mdb3s_close [$mdb3s close] 0 430 431 puts "\tRepmgr$tnum.3s.c: Shut down master, client takes over." 432 error_check_good master_close [$masterenv close] 0 433 await_expected_master $clientenv 434 await_startup_done $clientenv2 435 436 puts "\tRepmgr$tnum.3s.d: Verify client2 can do DML with new master." 437 set key3 3 438 error_check_good c2db3s_put3 \ 439 [eval $c2db3s put $key3 [chop_data $method data$key3]] 0 440 # Allow time for DML to be replicated back to client. 441 tclsleep $dmlsleep 442 # Verify key3 is present on both sites. 443 set ret [lindex [$cdb3s get $key3] 0] 444 error_check_good cdb3s_get3 $ret \ 445 [list $key3 [pad_data $method data$key3]] 446 set ret [lindex [$c2db3s get $key3] 0] 447 error_check_good c2db3s_get3 $ret \ 448 [list $key3 [pad_data $method data$key3]] 449 450 puts "\tRepmgr$tnum.3s.e: Restart original master to rejoin as client." 451 set masterenv [eval $ma_envcmd] 452 $masterenv rep_config {mgrforwardwrites on} 453 $masterenv repmgr -ack all \ 454 -local [list $hoststr [lindex $ports 0]] \ 455 -timeout [list write_forward 4000000] \ 456 -start client 457 await_startup_done $masterenv 458 set mdb3s [eval "berkdb_open_noerr -create $omethod -auto_commit \ 459 -env $masterenv $largs $dbname"] 460 461 puts "\tRepmgr$tnum.3s.f: Verify master rejoining as client can do DML." 462 set key4 4 463 error_check_good mdb3s_put4 \ 464 [eval $mdb3s put $key4 [chop_data $method data$key4]] 0 465 # Allow time for DML to be replicated back to clients. 466 tclsleep $dmlsleep 467 # Verify key4 is present on all sites. 468 set ret [lindex [$mdb3s get $key4] 0] 469 error_check_good mdb3s_get4 $ret \ 470 [list $key4 [pad_data $method data$key4]] 471 set ret [lindex [$cdb3s get $key4] 0] 472 error_check_good cdb3s_get4 $ret \ 473 [list $key4 [pad_data $method data$key4]] 474 set ret [lindex [$c2db3s get $key4] 0] 475 error_check_good c2db3s_get4 $ret \ 476 [list $key4 [pad_data $method data$key4]] 477 478 error_check_good c2db3s_close [$c2db3s close] 0 479 error_check_good cdb3s_close [$cdb3s close] 0 480 error_check_good mdb3s_close [$mdb3s close] 0 481 482 puts "\tRepmgr$tnum.3s.g: Verify master and client database contents." 483 rep_verify $clientdir $clientenv $masterdir $masterenv 1 1 1 484 rep_verify $clientdir $clientenv $clientdir2 $clientenv2 1 1 1 485 486 error_check_good masterenv_close [$masterenv close] 0 487 error_check_good client2_close [$clientenv2 close] 0 488 error_check_good client_close [$clientenv close] 0 489} 490 491# 492# Note that blobs are not supported with in-memory databases, so this 493# test case should only be run with on-disk database files. 494# 495proc repmgr046_blob { method niter tnum largs } { 496 global testdir 497 global rep_verbose 498 global verbose_type 499 global ipversion 500 set nsites 2 501 set omethod [convert_method $method] 502 503 set verbargs "" 504 if { $rep_verbose == 1 } { 505 set verbargs " -verbose {$verbose_type on} " 506 } 507 508 set sslargs [setup_repmgr_sslargs] 509 510 env_cleanup $testdir 511 set hoststr [get_hoststr $ipversion] 512 set ports [available_ports $nsites] 513 set dmlsleep 1 514 515 set masterdir $testdir/MASTERDIR 516 set clientdir $testdir/CLIENTDIR 517 518 file mkdir $masterdir 519 file mkdir $clientdir 520 521 puts "\tRepmgr$tnum.bl.a: Start a master and a client." 522 set ma_envcmd "berkdb_env_noerr -create $verbargs $sslargs \ 523 -errpfx MASTER -home $masterdir -txn -rep -thread" 524 set masterenv [eval $ma_envcmd] 525 $masterenv rep_config {mgrforwardwrites on} 526 $masterenv repmgr -ack all \ 527 -local [list $hoststr [lindex $ports 0]] \ 528 -timeout [list write_forward 4000000] \ 529 -start master 530 531 set cl_envcmd "berkdb_env_noerr -create $verbargs $sslargs \ 532 -errpfx CLIENT -home $clientdir -txn -rep -thread" 533 set clientenv [eval $cl_envcmd] 534 $clientenv rep_config {mgrforwardwrites on} 535 $clientenv repmgr -ack all -pri 50 \ 536 -local [list $hoststr [lindex $ports 1]] \ 537 -remote [list $hoststr [lindex $ports 0]] \ 538 -start client 539 await_startup_done $clientenv 540 541 set dbname "test.db" 542 543 puts "\tRepmgr$tnum.3s.b: Test write forwarding blobs." 544 set mdbblob [eval "berkdb_open_noerr -create $omethod -auto_commit \ 545 -env $masterenv -blob_threshold 10 $largs $dbname"] 546 set cdbblob [eval "berkdb_open_noerr -create $omethod -auto_commit \ 547 -env $clientenv -blob_threshold 10 $largs $dbname"] 548 549 set keyb1 1 550 set keyb2 2 551 set keyb3 3 552 set b2_data [string repeat "a" 100] 553 set b3_data [string repeat "b" 1000] 554 555 error_check_good cdbblob_putb1 \ 556 [eval $cdbblob put $keyb1 "under"] 0 557 error_check_good cdbblob_putb2 [eval $cdbblob put $keyb2 $b2_data] 0 558 error_check_good cdbblob_putb3 [eval $cdbblob put $keyb3 $b3_data] 0 559 560 # Allow time for DML to be replicated back to clients. 561 tclsleep $dmlsleep 562 set ret [lindex [$cdbblob get $keyb1] 0] 563 error_check_good cdbblob_getb1 $ret [list $keyb1 "under"] 564 set ret [lindex [$cdbblob get $keyb2] 0] 565 error_check_good cdbblob_getb2 $ret [list $keyb2 $b2_data] 566 set ret [lindex [$cdbblob get $keyb3] 0] 567 error_check_good cdbblob_getb3 $ret [list $keyb3 $b3_data] 568 set ret [lindex [$mdbblob get $keyb1] 0] 569 error_check_good mdbblob_getb1 $ret [list $keyb1 "under"] 570 set ret [lindex [$mdbblob get $keyb2] 0] 571 error_check_good mdbblob_getb2 $ret [list $keyb2 $b2_data] 572 set ret [lindex [$mdbblob get $keyb3] 0] 573 error_check_good mdbblob_getb3 $ret [list $keyb3 $b3_data] 574 575 error_check_good cdbblob_close [$cdbblob close] 0 576 error_check_good mdbblob_close [$mdbblob close] 0 577 578 puts "\tRepmgr$tnum.bl.c: Verify master and client database contents." 579 rep_verify $masterdir $masterenv $clientdir $clientenv 1 1 1 580 581 error_check_good client_close [$clientenv close] 0 582 error_check_good masterenv_close [$masterenv close] 0 583} 584 585# Test write forwarding to subdatabases and some cases using duplicate values. 586proc repmgr046_subdb { method niter tnum largs } { 587 global testdir 588 global rep_verbose 589 global verbose_type 590 global databases_in_memory 591 global ipversion 592 set nsites 2 593 set omethod [convert_method $method] 594 595 set verbargs "" 596 if { $rep_verbose == 1 } { 597 set verbargs " -verbose {$verbose_type on} " 598 } 599 600 set sslargs [setup_repmgr_sslargs] 601 602 env_cleanup $testdir 603 set hoststr [get_hoststr $ipversion] 604 set ports [available_ports $nsites] 605 set dmlsleep 1 606 607 set masterdir $testdir/MASTERDIR 608 set clientdir $testdir/CLIENTDIR 609 610 file mkdir $masterdir 611 file mkdir $clientdir 612 613 puts "\tRepmgr$tnum.sdb.a: Start a master and a client." 614 set ma_envcmd "berkdb_env_noerr -create $verbargs $sslargs \ 615 -errpfx MASTER -home $masterdir -txn -rep -thread" 616 set masterenv [eval $ma_envcmd] 617 $masterenv rep_config {mgrforwardwrites on} 618 $masterenv repmgr -ack all \ 619 -local [list $hoststr [lindex $ports 0]] \ 620 -start master 621 622 set cl_envcmd "berkdb_env_noerr -create $verbargs $sslargs \ 623 -errpfx CLIENT -home $clientdir -txn -rep -thread" 624 set clientenv [eval $cl_envcmd] 625 $clientenv rep_config {mgrforwardwrites on} 626 $clientenv repmgr -ack all -pri 50 \ 627 -local [list $hoststr [lindex $ports 1]] \ 628 -remote [list $hoststr [lindex $ports 0]] \ 629 -start client 630 await_startup_done $clientenv 631 632 if {$databases_in_memory} { 633 # 634 # Create separate in-memory databases oddtest.db and 635 # eventest.db because there can't be in-memory subdatabases. 636 # 637 set odbname { "" "oddtest.db" } 638 set edbname { "" "eventest.db" } 639 set modb [eval "berkdb_open_noerr -create -mode 0644 $omethod \ 640 -auto_commit -env $masterenv $largs $odbname"] 641 set medb [eval "berkdb_open_noerr -create -mode 0644 $omethod \ 642 -auto_commit -env $masterenv $largs $edbname"] 643 set codb [eval "berkdb_open_noerr -create -mode 0644 $omethod \ 644 -auto_commit -env $clientenv $largs $odbname"] 645 set cedb [eval "berkdb_open_noerr -create -mode 0644 $omethod \ 646 -auto_commit -env $clientenv $largs $edbname"] 647 } else { 648 # 649 # Create a single database file test.db that contains the 650 # two subdatabases oddsubdb and evensubdb. 651 # 652 set dbname "test.db" 653 set osub "oddsubdb" 654 set esub "evensubdb" 655 set modb [eval "berkdb_open_noerr -create $omethod \ 656 -auto_commit -env $masterenv $largs $dbname $osub"] 657 set medb [eval "berkdb_open_noerr -create $omethod \ 658 -auto_commit -env $masterenv $largs $dbname $esub"] 659 set codb [eval "berkdb_open_noerr -create $omethod \ 660 -auto_commit -env $clientenv $largs $dbname $osub"] 661 set cedb [eval "berkdb_open_noerr -create $omethod \ 662 -auto_commit -env $clientenv $largs $dbname $esub"] 663 } 664 error_check_good modb_open [is_valid_db $modb] TRUE 665 error_check_good medb_open [is_valid_db $medb] TRUE 666 error_check_good codb_open [is_valid_db $codb] TRUE 667 error_check_good cedb_open [is_valid_db $cedb] TRUE 668 669 puts "\tRepmgr$tnum.sdb.b: Test client DMLs to subdatabases." 670 set key1 1 671 set key2 2 672 set key3 3 673 set key4 4 674 error_check_good codb_put1 \ 675 [eval $codb put $key1 [chop_data $method data$key1]] 0 676 error_check_good cedb_put2 \ 677 [eval $cedb put $key2 [chop_data $method data$key2]] 0 678 # Put key3 in wrong subdb, give key4 wrong data. 679 error_check_good cedb_put3 \ 680 [eval $cedb put $key3 [chop_data $method data$key3]] 0 681 error_check_good cedb_put4wrongdata \ 682 [eval $cedb put $key4 [chop_data $method data$key3]] 0 683 # Correct key3 and key4. 684 error_check_good cedb_del1 [eval $cedb del $key3] 0 685 error_check_good codb_put3 \ 686 [eval $codb put $key3 [chop_data $method data$key3]] 0 687 error_check_good cedb_put4 \ 688 [eval $cedb put $key4 [chop_data $method data$key4]] 0 689 # Allow time for DML to be replicated back to clients. 690 tclsleep $dmlsleep 691 # Verify keys are present in the correct database. 692 set ret [lindex [$codb get $key1] 0] 693 error_check_good codb_get1 $ret \ 694 [list $key1 [pad_data $method data$key1]] 695 set ret [lindex [$cedb get $key2] 0] 696 error_check_good cedb_get2 $ret \ 697 [list $key2 [pad_data $method data$key2]] 698 set ret [lindex [$codb get $key3] 0] 699 error_check_good codb_get3 $ret \ 700 [list $key3 [pad_data $method data$key3]] 701 set ret [lindex [$cedb get $key4] 0] 702 error_check_good cedb_get4 $ret \ 703 [list $key4 [pad_data $method data$key4]] 704 # Verify keys are not present in the other database. 705 set ret [lindex [$cedb get $key1] 0] 706 error_check_good cedb_badget1 $ret "" 707 set ret [lindex [$codb get $key2] 0] 708 error_check_good codb_badget2 $ret "" 709 set ret [lindex [$cedb get $key3] 0] 710 error_check_good cedb_badget3 $ret "" 711 set ret [lindex [$codb get $key4] 0] 712 error_check_good codb_badget4 $ret "" 713 714 error_check_good cedb_close [$cedb close] 0 715 error_check_good codb_close [$codb close] 0 716 error_check_good medb_close [$medb close] 0 717 error_check_good modb_close [$modb close] 0 718 719 puts "\tRepmgr$tnum.sdb.c: Test client DMLs with duplicates." 720 if {$databases_in_memory} { 721 set dupdbname { "" "duptest.db" } 722 } else { 723 set dupdbname "duptest.db" 724 } 725 726 # Need to sort duplicates for a put -nodupdata to work. 727 set mdupdb [eval "berkdb_open_noerr -create $omethod -auto_commit \ 728 -env $masterenv -dup -dupsort $largs $dupdbname"] 729 set cdupdb [eval "berkdb_open_noerr -create $omethod -auto_commit \ 730 -env $clientenv -dup -dupsort $largs $dupdbname"] 731 732 # Insert multiple values for key1 and key2. 733 error_check_good cdupdb_put1 \ 734 [eval $cdupdb put $key1 [chop_data $method data$key1]] 0 735 error_check_good cdupdb_put1dup1 \ 736 [eval $cdupdb put $key1 [chop_data $method datadup1$key1]] 0 737 error_check_good cdupdb_put1dup2 \ 738 [eval $cdupdb put $key1 [chop_data $method datadup2$key1]] 0 739 error_check_good cdupdb_put2 \ 740 [eval $cdupdb put $key2 [chop_data $method data$key2]] 0 741 error_check_good cdupdb_put1dup2 \ 742 [eval $cdupdb put $key2 [chop_data $method datadup1$key2]] 0 743 # Make sure putting truly duplicate value returns KEYEXIST. 744 catch { $cdupdb put -nodupdata $key2 \ 745 [chop_data $method datadup1$key2] } res 746 error_check_good cdupdb_dupdat [is_substr $res "pair already exists"] 1 747 # Allow time for DML to be replicated back to client. 748 tclsleep $dmlsleep 749 # Verify all dup values are there. 750 set ret [lindex [$cdupdb get -get_both $key1 data$key1] 0] 751 error_check_good cdupdb_get1 $ret [list $key1 data$key1] 752 set ret [lindex [$cdupdb get -get_both $key1 datadup1$key1] 0] 753 error_check_good cdupdb_get1dup1 $ret [list $key1 datadup1$key1] 754 set ret [lindex [$cdupdb get -get_both $key1 datadup2$key1] 0] 755 error_check_good cdupdb_get1dup2 $ret [list $key1 datadup2$key1] 756 set ret [lindex [$cdupdb get -get_both $key2 data$key2] 0] 757 error_check_good cdupdb_get2 $ret [list $key2 data$key2] 758 set ret [lindex [$cdupdb get -get_both $key2 datadup1$key2] 0] 759 error_check_good cdupdb_get2dup1 $ret [list $key2 datadup1$key2] 760 761 # Now delete key1, which deletes all of its duplicate values. 762 error_check_good cdupdb_del1 [eval $cdupdb del $key1] 0 763 # Allow time for DML to be replicated back to client. 764 tclsleep $dmlsleep 765 # Verify key1 is gone. 766 set ret [lindex [$cdupdb get $key1] 0] 767 error_check_good cdupdb_get1 $ret "" 768 769 error_check_good cdupdb_close [$cdupdb close] 0 770 error_check_good mdupdb_close [$mdupdb close] 0 771 772 puts "\tRepmgr$tnum.sdb.d: Verify master and client database contents." 773 if {$databases_in_memory} { 774 rep_verify $clientdir $clientenv $masterdir $masterenv 1 1 1 \ 775 oddtest.db 776 rep_verify $clientdir $clientenv $masterdir $masterenv 1 1 1 \ 777 eventest.db 778 } else { 779 rep_verify $clientdir $clientenv $masterdir $masterenv 1 1 1 \ 780 duptest.db 781 rep_verify $clientdir $clientenv $masterdir $masterenv 1 1 1 782 } 783 784 error_check_good client_close [$clientenv close] 0 785 error_check_good masterenv_close [$masterenv close] 0 786} 787 788# 789# Test write forwarding in a scenario that generates a HANDLE_DEAD 790# error because some client transactions were rolled back. 791# 792proc repmgr046_hdldead { method niter tnum largs } { 793 global testdir 794 global rep_verbose 795 global verbose_type 796 global databases_in_memory 797 global ipversion 798 set nsites 3 799 set omethod [convert_method $method] 800 801 set verbargs "" 802 if { $rep_verbose == 1 } { 803 set verbargs " -verbose {$verbose_type on} " 804 } 805 806 set sslargs [setup_repmgr_sslargs] 807 808 env_cleanup $testdir 809 set hoststr [get_hoststr $ipversion] 810 set ports [available_ports $nsites] 811 set dmlsleep 1 812 813 set masterdir $testdir/MASTERDIR 814 set clientdir $testdir/CLIENTDIR 815 set clientdir2 $testdir/CLIENTDIR2 816 817 file mkdir $masterdir 818 file mkdir $clientdir 819 file mkdir $clientdir2 820 821 # 822 # Using noelections mode because we need tight control over which 823 # site gets appointed master later in the test, otherwise a 824 # different site might be elected. 825 # 826 puts "\tRepmgr$tnum.hd.a: Start a master and two clients." 827 set ma_envcmd "berkdb_env_noerr -create $verbargs $sslargs \ 828 -errpfx MASTER -home $masterdir -txn -rep -thread" 829 set masterenv [eval $ma_envcmd] 830 $masterenv rep_config {mgrelections off} 831 $masterenv rep_config {mgrforwardwrites on} 832 $masterenv repmgr -ack all \ 833 -local [list $hoststr [lindex $ports 0]] \ 834 -start master 835 836 set cl_envcmd "berkdb_env_noerr -create $verbargs $sslargs \ 837 -errpfx CLIENT -home $clientdir -txn -rep -thread" 838 set clientenv [eval $cl_envcmd] 839 $clientenv rep_config {mgrelections off} 840 $clientenv rep_config {mgrforwardwrites on} 841 $clientenv repmgr -ack all \ 842 -local [list $hoststr [lindex $ports 1]] \ 843 -remote [list $hoststr [lindex $ports 0]] \ 844 -start client 845 await_startup_done $clientenv 846 847 set cl2_envcmd "berkdb_env_noerr -create $verbargs $sslargs \ 848 -errpfx CLIENT2 -home $clientdir2 -txn -rep -thread" 849 set clientenv2 [eval $cl2_envcmd] 850 $clientenv2 rep_config {mgrelections off} 851 $clientenv2 rep_config {mgrforwardwrites on} 852 $clientenv2 repmgr -ack all \ 853 -local [list $hoststr [lindex $ports 2]] \ 854 -remote [list $hoststr [lindex $ports 0]] \ 855 -start client 856 await_startup_done $clientenv2 857 858 if {$databases_in_memory} { 859 set dbname { "" "test.db" } 860 } else { 861 set dbname "test.db" 862 } 863 864 set mdbhd [eval "berkdb_open_noerr -create $omethod -auto_commit \ 865 -env $masterenv $largs $dbname"] 866 set c2dbhd [eval "berkdb_open_noerr -create $omethod -auto_commit \ 867 -env $clientenv2 $largs $dbname"] 868 869 puts "\tRepmgr$tnum.hd.b: Do first DML on client2." 870 set key1 1 871 set key2 2 872 error_check_good c2dbhd_put1 \ 873 [eval $c2dbhd put $key1 [chop_data $method data$key1]] 0 874 # Allow time for DML to be replicated back to clients. 875 tclsleep $dmlsleep 876 877 puts "\tRepmgr$tnum.hd.c: Close client and do more master DMLs." 878 set start 0 879 error_check_good client_close [$clientenv close] 0 880 eval rep_test $method $masterenv NULL $niter $start 0 0 $largs 881 incr start $niter 882 883 puts "\tRepmgr$tnum.hd.d: Close master, restart client as master." 884 error_check_good mdbhd_close [$mdbhd close] 0 885 error_check_good master_close [$masterenv close] 0 886 set clientenv [eval $cl_envcmd] 887 $clientenv rep_config {mgrelections off} 888 $clientenv rep_config {mgrforwardwrites on} 889 $clientenv repmgr -ack all \ 890 -local [list $hoststr [lindex $ports 1]] \ 891 -remote [list $hoststr [lindex $ports 0]] \ 892 -start master 893 await_expected_master $clientenv 894 # 895 # This client sync makes client2 roll back its most recent master 896 # transactions. 897 # 898 await_startup_done $clientenv2 899 set cdbhd [eval "berkdb_open_noerr -create $omethod -auto_commit \ 900 -env $clientenv $largs $dbname"] 901 902 puts "\tRepmgr$tnum.hd.e: HANDLE_DEAD for forwarded put using old\ 903 handle." 904 # Allow time for repmgr connections to get reestablished. 905 tclsleep 2 906 catch { $c2dbhd put $key2 [chop_data $method data$key2] } res 907 error_check_good c2dbhd_hdead [is_substr $res "no longer valid"] 1 908 909 puts "\tRepmgr$tnum.hd.f: Close/reopen db handle, put succeeds." 910 error_check_good c2dbhd_close [$c2dbhd close] 0 911 set c2dbhd [eval "berkdb_open_noerr -create $omethod -auto_commit \ 912 -env $clientenv2 $largs $dbname"] 913 error_check_good c2dbhd_put2 \ 914 [eval $c2dbhd put $key2 [chop_data $method data$key2]] 0 915 # Allow time for DML to be replicated back to client. 916 tclsleep $dmlsleep 917 918 puts "\tRepmgr$tnum.hd.g: Verify both forwarded DMLs." 919 set ret [lindex [$c2dbhd get $key1] 0] 920 error_check_good c2dbhd_get1 $ret \ 921 [list $key1 [pad_data $method data$key1]] 922 set ret [lindex [$c2dbhd get $key2] 0] 923 error_check_good c2dbhd_get2 $ret \ 924 [list $key2 [pad_data $method data$key2]] 925 926 error_check_good c2dbhd_close [$c2dbhd close] 0 927 error_check_good cdbhd_close [$cdbhd close] 0 928 929 puts "\tRepmgr$tnum.hd.h: Verify database contents." 930 rep_verify $clientdir $clientenv $clientdir2 $clientenv2 1 1 1 931 932 error_check_good client2_close [$clientenv2 close] 0 933 error_check_good client_close [$clientenv close] 0 934} 935