1#!/usr/bin/env python
2#
3# Public Domain 2014-2018 MongoDB, Inc.
4# Public Domain 2008-2014 WiredTiger, Inc.
5#
6# This is free and unencumbered software released into the public domain.
7#
8# Anyone is free to copy, modify, publish, use, compile, sell, or
9# distribute this software, either in source code form or as a compiled
10# binary, for any purpose, commercial or non-commercial, and by any
11# means.
12#
13# In jurisdictions that recognize copyright laws, the author or authors
14# of this software dedicate any and all copyright interest in the
15# software to the public domain. We make this dedication for the benefit
16# of the public at large and to the detriment of our heirs and
17# successors. We intend this dedication to be an overt act of
18# relinquishment in perpetuity of all present and future rights to this
19# software under copyright law.
20#
21# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
23# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
24# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
25# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
26# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
27# OTHER DEALINGS IN THE SOFTWARE.
28
29import wiredtiger, wttest
30from wtdataset import SimpleDataSet, ComplexLSMDataSet
31from wtscenario import make_scenarios
32
33# test_checkpoint01.py
34#    Checkpoint tests
35# General checkpoint test: create an object containing sets of data associated
36# with a set of checkpoints, then confirm the checkpoint's values are correct,
37# including after other checkpoints are dropped.
38class test_checkpoint(wttest.WiredTigerTestCase):
39    scenarios = make_scenarios([
40        ('file', dict(uri='file:checkpoint',fmt='S')),
41        ('table', dict(uri='table:checkpoint',fmt='S'))
42    ])
43
44    # Each checkpoint has a key range and a "is dropped" flag.
45    checkpoints = {
46        "checkpoint-1": ((100, 200), 0),
47        "checkpoint-2": ((200, 220), 0),
48        "checkpoint-3": ((300, 320), 0),
49        "checkpoint-4": ((400, 420), 0),
50        "checkpoint-5": ((500, 520), 0),
51        "checkpoint-6": ((100, 620), 0),
52        "checkpoint-7": ((200, 250), 0),
53        "checkpoint-8": ((300, 820), 0),
54        "checkpoint-9": ((400, 920), 0)
55        }
56
57    # Add a set of records for a checkpoint.
58    def add_records(self, name):
59        cursor = self.session.open_cursor(self.uri, None, "overwrite")
60        start, stop = self.checkpoints[name][0]
61        for i in range(start, stop+1):
62            cursor["%010d KEY------" % i] = ("%010d VALUE " % i) + name
63        cursor.close()
64        self.checkpoints[name] = (self.checkpoints[name][0], 1)
65
66    # For each checkpoint entry, add/overwrite the specified records, then
67    # checkpoint the object, and verify it (which verifies all underlying
68    # checkpoints individually).
69    def build_file_with_checkpoints(self):
70        for checkpoint_name, entry in self.checkpoints.iteritems():
71            self.add_records(checkpoint_name)
72            self.session.checkpoint("name=" + checkpoint_name)
73
74    # Create a dictionary of sorted records a checkpoint should include.
75    def list_expected(self, name):
76        records = {}
77        for checkpoint_name, entry in self.checkpoints.iteritems():
78            start, stop = entry[0]
79            for i in range(start, stop+1):
80                records['%010d KEY------' % i] =\
81                    '%010d VALUE ' % i + checkpoint_name
82            if name == checkpoint_name:
83                break
84        return records
85
86    # Create a dictionary of sorted records a checkpoint does include.
87    def list_checkpoint(self, name):
88        records = {}
89        cursor = self.session.open_cursor(self.uri, None, 'checkpoint=' + name)
90        while cursor.next() == 0:
91            records[cursor.get_key()] = cursor.get_value()
92        cursor.close()
93        return records
94
95    # For each existing checkpoint entry, verify it contains the records it
96    # should, and no other checkpoints exist.
97    def check(self):
98        # Physically verify the file, including the individual checkpoints.
99        self.session.verify(self.uri, None)
100
101        for checkpoint_name, entry in self.checkpoints.iteritems():
102            if entry[1] == 0:
103                self.assertRaises(wiredtiger.WiredTigerError,
104                    lambda: self.session.open_cursor(
105                    self.uri, None, "checkpoint=" + checkpoint_name))
106            else:
107                list_expected = self.list_expected(checkpoint_name)
108                list_checkpoint = self.list_checkpoint(checkpoint_name)
109                self.assertEqual(list_expected, list_checkpoint)
110
111    # Main checkpoint test driver.
112    def test_checkpoint(self):
113        # Build a file with a set of checkpoints, and confirm they all have
114        # the correct key/value pairs.
115        self.session.create(self.uri,
116            "key_format=" + self.fmt +\
117                ",value_format=S,allocation_size=512,leaf_page_max=512")
118        self.build_file_with_checkpoints()
119        self.check()
120
121        # Drop a set of checkpoints sequentially, and each time confirm the
122        # contents of remaining checkpoints, and that dropped checkpoints
123        # don't appear.
124        for i in [1,3,7,9]:
125            checkpoint_name = 'checkpoint-' + str(i)
126            self.session.checkpoint('drop=(' + checkpoint_name + ')')
127            self.checkpoints[checkpoint_name] =\
128                (self.checkpoints[checkpoint_name][0], 0)
129            self.check()
130
131        # Drop remaining checkpoints, all subsequent checkpoint opens should
132        # fail.
133        self.session.checkpoint("drop=(from=all)")
134        for checkpoint_name, entry in self.checkpoints.iteritems():
135            self.checkpoints[checkpoint_name] =\
136                (self.checkpoints[checkpoint_name][0], 0)
137        self.check()
138
139# Check some specific cursor checkpoint combinations.
140class test_checkpoint_cursor(wttest.WiredTigerTestCase):
141    scenarios = make_scenarios([
142        ('file', dict(uri='file:checkpoint',fmt='S')),
143        ('table', dict(uri='table:checkpoint',fmt='S'))
144    ])
145
146    # Check that you cannot open a checkpoint that doesn't exist.
147    def test_checkpoint_dne(self):
148        SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate()
149        self.assertRaises(wiredtiger.WiredTigerError,
150            lambda: self.session.open_cursor(
151            self.uri, None, "checkpoint=checkpoint-1"))
152        self.assertRaises(wiredtiger.WiredTigerError,
153            lambda: self.session.open_cursor(
154            self.uri, None, "checkpoint=WiredTigerCheckpoint"))
155
156    # Check that you can open checkpoints more than once.
157    def test_checkpoint_multiple_open(self):
158        SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate()
159        self.session.checkpoint("name=checkpoint-1")
160        c1 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1")
161        c2 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1")
162        c3 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1")
163        c4 = self.session.open_cursor(self.uri, None, None)
164        c4.close, c3.close, c2.close, c1.close
165
166        self.session.checkpoint("name=checkpoint-2")
167        c1 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-1")
168        c2 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-2")
169        c3 = self.session.open_cursor(self.uri, None, "checkpoint=checkpoint-2")
170        c4 = self.session.open_cursor(self.uri, None, None)
171        c4.close, c3.close, c2.close, c1.close
172
173    # Check that you cannot drop a checkpoint if it's in use.
174    def test_checkpoint_inuse(self):
175        SimpleDataSet(self, self.uri, 100, key_format=self.fmt).populate()
176        self.session.checkpoint("name=checkpoint-1")
177        self.session.checkpoint("name=checkpoint-2")
178        self.session.checkpoint("name=checkpoint-3")
179        cursor = self.session.open_cursor(
180            self.uri, None, "checkpoint=checkpoint-2")
181
182        # Check creating an identically named checkpoint fails. */
183        # Check dropping the specific checkpoint fails.
184        # Check dropping all checkpoints fails.
185        msg = '/checkpoints cannot be dropped/'
186        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
187            lambda: self.session.checkpoint("force,name=checkpoint-2"), msg)
188        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
189            lambda: self.session.checkpoint("drop=(checkpoint-2)"), msg)
190        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
191            lambda: self.session.checkpoint("drop=(from=all)"), msg)
192
193        # Check dropping other checkpoints succeeds (which also tests that you
194        # can create new checkpoints while other checkpoints are in-use).
195        self.session.checkpoint("drop=(checkpoint-1,checkpoint-3)")
196
197        # Close the cursor and repeat the failing commands, they should now
198        # succeed.
199        cursor.close()
200        self.session.checkpoint("name=checkpoint-2")
201        self.session.checkpoint("drop=(checkpoint-2)")
202        self.session.checkpoint("drop=(from=all)")
203
204# Check that you can checkpoint targets.
205class test_checkpoint_target(wttest.WiredTigerTestCase):
206    scenarios = make_scenarios([
207        ('file', dict(uri='file:checkpoint',fmt='S')),
208        ('table', dict(uri='table:checkpoint',fmt='S'))
209    ])
210
211    def update(self, uri, ds, value):
212        cursor = self.session.open_cursor(uri, None, "overwrite")
213        cursor[ds.key(10)] = value
214        cursor.close()
215
216    def check(self, uri, ds, value):
217        cursor = self.session.open_cursor(uri, None, "checkpoint=checkpoint-1")
218        self.assertEquals(cursor[ds.key(10)], value)
219        cursor.close()
220
221    def test_checkpoint_target(self):
222        # Create 3 objects, change one record to an easily recognizable string.
223        uri = self.uri + '1'
224        ds1 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
225        ds1.populate()
226        self.update(uri, ds1, 'ORIGINAL')
227
228        uri = self.uri + '2'
229        ds2 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
230        ds2.populate()
231        self.update(uri, ds2, 'ORIGINAL')
232
233        uri = self.uri + '3'
234        ds3 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
235        ds3.populate()
236        self.update(uri, ds3, 'ORIGINAL')
237
238        # Checkpoint all three objects.
239        self.session.checkpoint("name=checkpoint-1")
240
241        # Update all 3 objects, then checkpoint two of the objects with the
242        # same checkpoint name.
243        self.update(self.uri + '1', ds1, 'UPDATE')
244        self.update(self.uri + '2', ds2, 'UPDATE')
245        self.update(self.uri + '3', ds3, 'UPDATE')
246        target = 'target=("' + self.uri + '1"' + ',"' + self.uri + '2")'
247        self.session.checkpoint("name=checkpoint-1," + target)
248
249        # Confirm the checkpoint has the old value in objects that weren't
250        # checkpointed, and the new value in objects that were checkpointed.
251        self.check(self.uri + '1', ds1, 'UPDATE')
252        self.check(self.uri + '2', ds2, 'UPDATE')
253        self.check(self.uri + '3', ds3, 'ORIGINAL')
254
255# Check that you can't write checkpoint cursors.
256class test_checkpoint_cursor_update(wttest.WiredTigerTestCase):
257    scenarios = make_scenarios([
258        ('file-r', dict(uri='file:checkpoint',fmt='r')),
259        ('file-S', dict(uri='file:checkpoint',fmt='S')),
260        ('table-r', dict(uri='table:checkpoint',fmt='r')),
261        ('table-S', dict(uri='table:checkpoint',fmt='S'))
262    ])
263
264    def test_checkpoint_cursor_update(self):
265        ds = SimpleDataSet(self, self.uri, 100, key_format=self.fmt)
266        ds.populate()
267        self.session.checkpoint("name=ckpt")
268        cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
269        cursor.set_key(ds.key(10))
270        cursor.set_value("XXX")
271        msg = "/Unsupported cursor/"
272        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
273            lambda: cursor.insert(), msg)
274        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
275            lambda: cursor.remove(), msg)
276        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
277            lambda: cursor.update(), msg)
278        cursor.close()
279
280# Check that WiredTigerCheckpoint works as a checkpoint specifier.
281class test_checkpoint_last(wttest.WiredTigerTestCase):
282    scenarios = make_scenarios([
283        ('file', dict(uri='file:checkpoint',fmt='S')),
284        ('table', dict(uri='table:checkpoint',fmt='S'))
285    ])
286
287    def test_checkpoint_last(self):
288        # Create an object, change one record to an easily recognizable string,
289        # then checkpoint it and open a cursor, confirming we see the correct
290        # value.   Repeat this action, we want to be sure the engine gets the
291        # latest checkpoint information each time.
292        uri = self.uri
293        ds = SimpleDataSet(self, uri, 100, key_format=self.fmt)
294        ds.populate()
295
296        for value in ('FIRST', 'SECOND', 'THIRD', 'FOURTH', 'FIFTH'):
297            # Update the object.
298            cursor = self.session.open_cursor(uri, None, "overwrite")
299            cursor[ds.key(10)] = value
300            cursor.close()
301
302            # Checkpoint the object.
303            self.session.checkpoint()
304
305            # Verify the "last" checkpoint sees the correct value.
306            cursor = self.session.open_cursor(
307                uri, None, "checkpoint=WiredTigerCheckpoint")
308            self.assertEquals(cursor[ds.key(10)], value)
309            # Don't close the checkpoint cursor, we want it to remain open until
310            # the test completes.
311
312# Check we can't use the reserved name as an application checkpoint name.
313class test_checkpoint_illegal_name(wttest.WiredTigerTestCase):
314    def test_checkpoint_illegal_name(self):
315        ds = SimpleDataSet(self, "file:checkpoint", 100, key_format='S')
316        ds.populate()
317        msg = '/the checkpoint name.*is reserved/'
318        for conf in (
319            'name=WiredTigerCheckpoint',
320            'name=WiredTigerCheckpoint.',
321            'name=WiredTigerCheckpointX',
322            'drop=(from=WiredTigerCheckpoint)',
323            'drop=(from=WiredTigerCheckpoint.)',
324            'drop=(from=WiredTigerCheckpointX)',
325            'drop=(to=WiredTigerCheckpoint)',
326            'drop=(to=WiredTigerCheckpoint.)',
327            'drop=(to=WiredTigerCheckpointX)'):
328                self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
329                    lambda: self.session.checkpoint(conf), msg)
330        msg = '/WiredTiger objects should not include grouping/'
331        for conf in (
332            'name=check{point',
333            'name=check\\point'):
334                self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
335                    lambda: self.session.checkpoint(conf), msg)
336
337# Check we can't name checkpoints that include LSM tables.
338class test_checkpoint_lsm_name(wttest.WiredTigerTestCase):
339    def test_checkpoint_lsm_name(self):
340        ds = ComplexLSMDataSet(self, "table:checkpoint", 1000)
341        ds.populate()
342        msg = '/object does not support named checkpoints/'
343        self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
344            lambda: self.session.checkpoint("name=ckpt"), msg)
345
346class test_checkpoint_empty(wttest.WiredTigerTestCase):
347    scenarios = make_scenarios([
348        ('file', dict(uri='file:checkpoint')),
349        ('table', dict(uri='table:checkpoint')),
350    ])
351
352    # Create an empty file, do one of 4 cases of checkpoint, then verify the
353    # checkpoints exist.   The reason for the 4 cases is we must create all
354    # checkpoints an application can explicitly reference using a cursor, and
355    # I want to test when other types of checkpoints are created before the
356    # checkpoint we really need.
357    # Case 1: a named checkpoint
358    def test_checkpoint_empty_one(self):
359        self.session.create(self.uri, "key_format=S,value_format=S")
360        self.session.checkpoint('name=ckpt')
361        cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
362
363    # Case 2: an internal checkpoint
364    def test_checkpoint_empty_two(self):
365        self.session.create(self.uri, "key_format=S,value_format=S")
366        self.session.checkpoint()
367        cursor = self.session.open_cursor(
368            self.uri, None, "checkpoint=WiredTigerCheckpoint")
369
370    # Case 3: a named checkpoint, then an internal checkpoint
371    def test_checkpoint_empty_three(self):
372        self.session.create(self.uri, "key_format=S,value_format=S")
373        self.session.checkpoint('name=ckpt')
374        self.session.checkpoint()
375        cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
376        cursor = self.session.open_cursor(
377            self.uri, None, "checkpoint=WiredTigerCheckpoint")
378
379    # Case 4: an internal checkpoint, then a named checkpoint
380    def test_checkpoint_empty_four(self):
381        self.session.create(self.uri, "key_format=S,value_format=S")
382        self.session.checkpoint()
383        self.session.checkpoint('name=ckpt')
384        cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
385        cursor = self.session.open_cursor(
386            self.uri, None, "checkpoint=WiredTigerCheckpoint")
387
388    # Check that we can create an empty checkpoint, change the underlying
389    # object, checkpoint again, and still see the original empty tree, for
390    # both named and unnamed checkpoints.
391    def test_checkpoint_empty_five(self):
392        self.session.create(self.uri, "key_format=S,value_format=S")
393        self.session.checkpoint('name=ckpt')
394        cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
395        self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND)
396        cursor.close()
397
398        cursor = self.session.open_cursor(self.uri, None)
399        cursor["key"] = "value"
400        self.session.checkpoint()
401
402        cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
403        self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND)
404
405    def test_checkpoint_empty_six(self):
406        self.session.create(self.uri, "key_format=S,value_format=S")
407        self.session.checkpoint()
408        cursor = self.session.open_cursor(
409            self.uri, None, "checkpoint=WiredTigerCheckpoint")
410        self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND)
411        cursor.close()
412
413        cursor = self.session.open_cursor(self.uri, None)
414        cursor["key"] = "value"
415        self.session.checkpoint('name=ckpt')
416
417        cursor = self.session.open_cursor(
418            self.uri, None, "checkpoint=WiredTigerCheckpoint")
419        self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND)
420
421if __name__ == '__main__':
422    wttest.run()
423