1#!/usr/bin/env python 2# 3# Public Domain 2014-2018 MongoDB, Inc. 4# Public Domain 2008-2014 WiredTiger, Inc. 5# 6# This is free and unencumbered software released into the public domain. 7# 8# Anyone is free to copy, modify, publish, use, compile, sell, or 9# distribute this software, either in source code form or as a compiled 10# binary, for any purpose, commercial or non-commercial, and by any 11# means. 12# 13# In jurisdictions that recognize copyright laws, the author or authors 14# of this software dedicate any and all copyright interest in the 15# software to the public domain. We make this dedication for the benefit 16# of the public at large and to the detriment of our heirs and 17# successors. We intend this dedication to be an overt act of 18# relinquishment in perpetuity of all present and future rights to this 19# software under copyright law. 20# 21# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 23# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 24# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 25# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 26# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 27# OTHER DEALINGS IN THE SOFTWARE. 28 29import wiredtiger, wttest 30from wtdataset import SimpleDataSet, ComplexLSMDataSet 31from wtscenario import make_scenarios 32 33# test_checkpoint01.py 34# Checkpoint tests 35# General checkpoint test: create an object containing sets of data associated 36# with a set of checkpoints, then confirm the checkpoint's values are correct, 37# including after other checkpoints are dropped. 38class test_checkpoint(wttest.WiredTigerTestCase): 39 scenarios = make_scenarios([ 40 ('file', dict(uri='file:checkpoint',fmt='S')), 41 ('table', dict(uri='table:checkpoint',fmt='S')) 42 ]) 43 44 # Each checkpoint has a key range and a "is dropped" flag. 45 checkpoints = { 46 "checkpoint-1": ((100, 200), 0), 47 "checkpoint-2": ((200, 220), 0), 48 "checkpoint-3": ((300, 320), 0), 49 "checkpoint-4": ((400, 420), 0), 50 "checkpoint-5": ((500, 520), 0), 51 "checkpoint-6": ((100, 620), 0), 52 "checkpoint-7": ((200, 250), 0), 53 "checkpoint-8": ((300, 820), 0), 54 "checkpoint-9": ((400, 920), 0) 55 } 56 57 # Add a set of records for a checkpoint. 58 def add_records(self, name): 59 cursor = self.session.open_cursor(self.uri, None, "overwrite") 60 start, stop = self.checkpoints[name][0] 61 for i in range(start, stop+1): 62 cursor["%010d KEY------" % i] = ("%010d VALUE " % i) + name 63 cursor.close() 64 self.checkpoints[name] = (self.checkpoints[name][0], 1) 65 66 # For each checkpoint entry, add/overwrite the specified records, then 67 # checkpoint the object, and verify it (which verifies all underlying 68 # checkpoints individually). 69 def build_file_with_checkpoints(self): 70 for checkpoint_name, entry in self.checkpoints.iteritems(): 71 self.add_records(checkpoint_name) 72 self.session.checkpoint("name=" + checkpoint_name) 73 74 # Create a dictionary of sorted records a checkpoint should include. 75 def list_expected(self, name): 76 records = {} 77 for checkpoint_name, entry in self.checkpoints.iteritems(): 78 start, stop = entry[0] 79 for i in range(start, stop+1): 80 records['%010d KEY------' % i] =\ 81 '%010d VALUE ' % i + checkpoint_name 82 if name == checkpoint_name: 83 break 84 return records 85 86 # Create a dictionary of sorted records a checkpoint does include. 87 def list_checkpoint(self, name): 88 records = {} 89 cursor = self.session.open_cursor(self.uri, None, 'checkpoint=' + name) 90 while cursor.next() == 0: 91 records[cursor.get_key()] = cursor.get_value() 92 cursor.close() 93 return records 94 95 # For each existing checkpoint entry, verify it contains the records it 96 # should, and no other checkpoints exist. 97 def check(self): 98 # Physically verify the file, including the individual checkpoints. 99 self.session.verify(self.uri, None) 100 101 for checkpoint_name, entry in self.checkpoints.iteritems(): 102 if entry[1] == 0: 103 self.assertRaises(wiredtiger.WiredTigerError, 104 lambda: self.session.open_cursor( 105 self.uri, None, "checkpoint=" + checkpoint_name)) 106 else: 107 list_expected = self.list_expected(checkpoint_name) 108 list_checkpoint = self.list_checkpoint(checkpoint_name) 109 self.assertEqual(list_expected, list_checkpoint) 110 111 # Main checkpoint test driver. 112 def test_checkpoint(self): 113 # Build a file with a set of checkpoints, and confirm they all have 114 # the correct key/value pairs. 115 self.session.create(self.uri, 116 "key_format=" + self.fmt +\ 117 ",value_format=S,allocation_size=512,leaf_page_max=512") 118 self.build_file_with_checkpoints() 119 self.check() 120 121 # Drop a set of checkpoints sequentially, and each time confirm the 122 # contents of remaining checkpoints, and that dropped checkpoints 123 # don't appear. 124 for i in [1,3,7,9]: 125 checkpoint_name = 'checkpoint-' + str(i) 126 self.session.checkpoint('drop=(' + checkpoint_name + ')') 127 self.checkpoints[checkpoint_name] =\ 128 (self.checkpoints[checkpoint_name][0], 0) 129 self.check() 130 131 # Drop remaining checkpoints, all subsequent checkpoint opens should 132 # fail. 133 self.session.checkpoint("drop=(from=all)") 134 for checkpoint_name, entry in self.checkpoints.iteritems(): 135 self.checkpoints[checkpoint_name] =\ 136 (self.checkpoints[checkpoint_name][0], 0) 137 self.check() 138 139# Check some specific cursor checkpoint combinations. 140class test_checkpoint_cursor(wttest.WiredTigerTestCase): 141 scenarios = make_scenarios([ 142 ('file', dict(uri='file:checkpoint',fmt='S')), 143 ('table', dict(uri='table:checkpoint',fmt='S')) 144 ]) 145 146 # Check that you cannot open a checkpoint that doesn't exist. 147 def test_checkpoint_dne(self): 148 SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate() 149 self.assertRaises(wiredtiger.WiredTigerError, 150 lambda: self.session.open_cursor( 151 self.uri, None, "checkpoint=checkpoint-1")) 152 self.assertRaises(wiredtiger.WiredTigerError, 153 lambda: self.session.open_cursor( 154 self.uri, None, "checkpoint=WiredTigerCheckpoint")) 155 156 # Check that you can open checkpoints more than once. 157 def test_checkpoint_multiple_open(self): 158 SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate() 159 self.session.checkpoint("name=checkpoint-1") 160 c1 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1") 161 c2 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1") 162 c3 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1") 163 c4 = self.session.open_cursor(self.uri, None, None) 164 c4.close, c3.close, c2.close, c1.close 165 166 self.session.checkpoint("name=checkpoint-2") 167 c1 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1") 168 c2 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-2") 169 c3 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-2") 170 c4 = self.session.open_cursor(self.uri, None, None) 171 c4.close, c3.close, c2.close, c1.close 172 173 # Check that you cannot drop a checkpoint if it's in use. 174 def test_checkpoint_inuse(self): 175 SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate() 176 self.session.checkpoint("name=checkpoint-1") 177 self.session.checkpoint("name=checkpoint-2") 178 self.session.checkpoint("name=checkpoint-3") 179 cursor = self.session.open_cursor( 180 self.uri, None, "checkpoint=checkpoint-2") 181 182 # Check creating an identically named checkpoint fails. */ 183 # Check dropping the specific checkpoint fails. 184 # Check dropping all checkpoints fails. 185 msg = '/checkpoints cannot be dropped/' 186 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, 187 lambda: self.session.checkpoint("force,name=checkpoint-2"), msg) 188 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, 189 lambda: self.session.checkpoint("drop=(checkpoint-2)"), msg) 190 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, 191 lambda: self.session.checkpoint("drop=(from=all)"), msg) 192 193 # Check dropping other checkpoints succeeds (which also tests that you 194 # can create new checkpoints while other checkpoints are in-use). 195 self.session.checkpoint("drop=(checkpoint-1,checkpoint-3)") 196 197 # Close the cursor and repeat the failing commands, they should now 198 # succeed. 199 cursor.close() 200 self.session.checkpoint("name=checkpoint-2") 201 self.session.checkpoint("drop=(checkpoint-2)") 202 self.session.checkpoint("drop=(from=all)") 203 204# Check that you can checkpoint targets. 205class test_checkpoint_target(wttest.WiredTigerTestCase): 206 scenarios = make_scenarios([ 207 ('file', dict(uri='file:checkpoint',fmt='S')), 208 ('table', dict(uri='table:checkpoint',fmt='S')) 209 ]) 210 211 def update(self, uri, ds, value): 212 cursor = self.session.open_cursor(uri, None, "overwrite") 213 cursor[ds.key(10)] = value 214 cursor.close() 215 216 def check(self, uri, ds, value): 217 cursor = self.session.open_cursor(uri, None, "checkpoint=checkpoint-1") 218 self.assertEquals(cursor[ds.key(10)], value) 219 cursor.close() 220 221 def test_checkpoint_target(self): 222 # Create 3 objects, change one record to an easily recognizable string. 223 uri = self.uri + '1' 224 ds1 = SimpleDataSet(self, uri, 100, key_format=self.fmt) 225 ds1.populate() 226 self.update(uri, ds1, 'ORIGINAL') 227 228 uri = self.uri + '2' 229 ds2 = SimpleDataSet(self, uri, 100, key_format=self.fmt) 230 ds2.populate() 231 self.update(uri, ds2, 'ORIGINAL') 232 233 uri = self.uri + '3' 234 ds3 = SimpleDataSet(self, uri, 100, key_format=self.fmt) 235 ds3.populate() 236 self.update(uri, ds3, 'ORIGINAL') 237 238 # Checkpoint all three objects. 239 self.session.checkpoint("name=checkpoint-1") 240 241 # Update all 3 objects, then checkpoint two of the objects with the 242 # same checkpoint name. 243 self.update(self.uri + '1', ds1, 'UPDATE') 244 self.update(self.uri + '2', ds2, 'UPDATE') 245 self.update(self.uri + '3', ds3, 'UPDATE') 246 target = 'target=("' + self.uri + '1"' + ',"' + self.uri + '2")' 247 self.session.checkpoint("name=checkpoint-1," + target) 248 249 # Confirm the checkpoint has the old value in objects that weren't 250 # checkpointed, and the new value in objects that were checkpointed. 251 self.check(self.uri + '1', ds1, 'UPDATE') 252 self.check(self.uri + '2', ds2, 'UPDATE') 253 self.check(self.uri + '3', ds3, 'ORIGINAL') 254 255# Check that you can't write checkpoint cursors. 256class test_checkpoint_cursor_update(wttest.WiredTigerTestCase): 257 scenarios = make_scenarios([ 258 ('file-r', dict(uri='file:checkpoint',fmt='r')), 259 ('file-S', dict(uri='file:checkpoint',fmt='S')), 260 ('table-r', dict(uri='table:checkpoint',fmt='r')), 261 ('table-S', dict(uri='table:checkpoint',fmt='S')) 262 ]) 263 264 def test_checkpoint_cursor_update(self): 265 ds = SimpleDataSet(self, self.uri, 100, key_format=self.fmt) 266 ds.populate() 267 self.session.checkpoint("name=ckpt") 268 cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt") 269 cursor.set_key(ds.key(10)) 270 cursor.set_value("XXX") 271 msg = "/Unsupported cursor/" 272 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, 273 lambda: cursor.insert(), msg) 274 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, 275 lambda: cursor.remove(), msg) 276 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, 277 lambda: cursor.update(), msg) 278 cursor.close() 279 280# Check that WiredTigerCheckpoint works as a checkpoint specifier. 281class test_checkpoint_last(wttest.WiredTigerTestCase): 282 scenarios = make_scenarios([ 283 ('file', dict(uri='file:checkpoint',fmt='S')), 284 ('table', dict(uri='table:checkpoint',fmt='S')) 285 ]) 286 287 def test_checkpoint_last(self): 288 # Create an object, change one record to an easily recognizable string, 289 # then checkpoint it and open a cursor, confirming we see the correct 290 # value. Repeat this action, we want to be sure the engine gets the 291 # latest checkpoint information each time. 292 uri = self.uri 293 ds = SimpleDataSet(self, uri, 100, key_format=self.fmt) 294 ds.populate() 295 296 for value in ('FIRST', 'SECOND', 'THIRD', 'FOURTH', 'FIFTH'): 297 # Update the object. 298 cursor = self.session.open_cursor(uri, None, "overwrite") 299 cursor[ds.key(10)] = value 300 cursor.close() 301 302 # Checkpoint the object. 303 self.session.checkpoint() 304 305 # Verify the "last" checkpoint sees the correct value. 306 cursor = self.session.open_cursor( 307 uri, None, "checkpoint=WiredTigerCheckpoint") 308 self.assertEquals(cursor[ds.key(10)], value) 309 # Don't close the checkpoint cursor, we want it to remain open until 310 # the test completes. 311 312# Check we can't use the reserved name as an application checkpoint name. 313class test_checkpoint_illegal_name(wttest.WiredTigerTestCase): 314 def test_checkpoint_illegal_name(self): 315 ds = SimpleDataSet(self, "file:checkpoint", 100, key_format='S') 316 ds.populate() 317 msg = '/the checkpoint name.*is reserved/' 318 for conf in ( 319 'name=WiredTigerCheckpoint', 320 'name=WiredTigerCheckpoint.', 321 'name=WiredTigerCheckpointX', 322 'drop=(from=WiredTigerCheckpoint)', 323 'drop=(from=WiredTigerCheckpoint.)', 324 'drop=(from=WiredTigerCheckpointX)', 325 'drop=(to=WiredTigerCheckpoint)', 326 'drop=(to=WiredTigerCheckpoint.)', 327 'drop=(to=WiredTigerCheckpointX)'): 328 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, 329 lambda: self.session.checkpoint(conf), msg) 330 msg = '/WiredTiger objects should not include grouping/' 331 for conf in ( 332 'name=check{point', 333 'name=check\\point'): 334 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, 335 lambda: self.session.checkpoint(conf), msg) 336 337# Check we can't name checkpoints that include LSM tables. 338class test_checkpoint_lsm_name(wttest.WiredTigerTestCase): 339 def test_checkpoint_lsm_name(self): 340 ds = ComplexLSMDataSet(self, "table:checkpoint", 1000) 341 ds.populate() 342 msg = '/object does not support named checkpoints/' 343 self.assertRaisesWithMessage(wiredtiger.WiredTigerError, 344 lambda: self.session.checkpoint("name=ckpt"), msg) 345 346class test_checkpoint_empty(wttest.WiredTigerTestCase): 347 scenarios = make_scenarios([ 348 ('file', dict(uri='file:checkpoint')), 349 ('table', dict(uri='table:checkpoint')), 350 ]) 351 352 # Create an empty file, do one of 4 cases of checkpoint, then verify the 353 # checkpoints exist. The reason for the 4 cases is we must create all 354 # checkpoints an application can explicitly reference using a cursor, and 355 # I want to test when other types of checkpoints are created before the 356 # checkpoint we really need. 357 # Case 1: a named checkpoint 358 def test_checkpoint_empty_one(self): 359 self.session.create(self.uri, "key_format=S,value_format=S") 360 self.session.checkpoint('name=ckpt') 361 cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt") 362 363 # Case 2: an internal checkpoint 364 def test_checkpoint_empty_two(self): 365 self.session.create(self.uri, "key_format=S,value_format=S") 366 self.session.checkpoint() 367 cursor = self.session.open_cursor( 368 self.uri, None, "checkpoint=WiredTigerCheckpoint") 369 370 # Case 3: a named checkpoint, then an internal checkpoint 371 def test_checkpoint_empty_three(self): 372 self.session.create(self.uri, "key_format=S,value_format=S") 373 self.session.checkpoint('name=ckpt') 374 self.session.checkpoint() 375 cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt") 376 cursor = self.session.open_cursor( 377 self.uri, None, "checkpoint=WiredTigerCheckpoint") 378 379 # Case 4: an internal checkpoint, then a named checkpoint 380 def test_checkpoint_empty_four(self): 381 self.session.create(self.uri, "key_format=S,value_format=S") 382 self.session.checkpoint() 383 self.session.checkpoint('name=ckpt') 384 cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt") 385 cursor = self.session.open_cursor( 386 self.uri, None, "checkpoint=WiredTigerCheckpoint") 387 388 # Check that we can create an empty checkpoint, change the underlying 389 # object, checkpoint again, and still see the original empty tree, for 390 # both named and unnamed checkpoints. 391 def test_checkpoint_empty_five(self): 392 self.session.create(self.uri, "key_format=S,value_format=S") 393 self.session.checkpoint('name=ckpt') 394 cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt") 395 self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND) 396 cursor.close() 397 398 cursor = self.session.open_cursor(self.uri, None) 399 cursor["key"] = "value" 400 self.session.checkpoint() 401 402 cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt") 403 self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND) 404 405 def test_checkpoint_empty_six(self): 406 self.session.create(self.uri, "key_format=S,value_format=S") 407 self.session.checkpoint() 408 cursor = self.session.open_cursor( 409 self.uri, None, "checkpoint=WiredTigerCheckpoint") 410 self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND) 411 cursor.close() 412 413 cursor = self.session.open_cursor(self.uri, None) 414 cursor["key"] = "value" 415 self.session.checkpoint('name=ckpt') 416 417 cursor = self.session.open_cursor( 418 self.uri, None, "checkpoint=WiredTigerCheckpoint") 419 self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND) 420 421if __name__ == '__main__': 422 wttest.run() 423