1#!/usr/bin/env python
2#
3# Public Domain 2014-2018 MongoDB, Inc.
4# Public Domain 2008-2014 WiredTiger, Inc.
5#
6# This is free and unencumbered software released into the public domain.
7#
8# Anyone is free to copy, modify, publish, use, compile, sell, or
9# distribute this software, either in source code form or as a compiled
10# binary, for any purpose, commercial or non-commercial, and by any
11# means.
12#
13# In jurisdictions that recognize copyright laws, the author or authors
14# of this software dedicate any and all copyright interest in the
15# software to the public domain. We make this dedication for the benefit
16# of the public at large and to the detriment of our heirs and
17# successors. We intend this dedication to be an overt act of
18# relinquishment in perpetuity of all present and future rights to this
19# software under copyright law.
20#
21# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
23# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
24# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
25# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
26# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
27# OTHER DEALINGS IN THE SOFTWARE.
28#
29# test_timestamp10.py
30#   Timestamps: Saving and querying the last checkpoint and recovery timestamps.
31#
32
33from suite_subprocess import suite_subprocess
34import wiredtiger, wttest
35from wtscenario import make_scenarios
36
37def timestamp_str(t):
38    return '%x' % t
39
40class test_timestamp10(wttest.WiredTigerTestCase, suite_subprocess):
41    conn_config = 'config_base=false,create,log=(enabled)'
42    coll1_uri = 'table:collection10.1'
43    coll2_uri = 'table:collection10.2'
44    coll3_uri = 'table:collection10.3'
45    oplog_uri = 'table:oplog10'
46
47    nentries = 10
48    table_cnt = 3
49
50    types = [
51        ('all', dict(use_stable='false', run_wt=0)),
52        ('all+wt', dict(use_stable='false', run_wt=1)),
53        ('all+wt2', dict(use_stable='false', run_wt=2)),
54        ('default', dict(use_stable='default', run_wt=0)),
55        ('default+wt', dict(use_stable='default', run_wt=1)),
56        ('default+wt2', dict(use_stable='default', run_wt=2)),
57        ('stable', dict(use_stable='true', run_wt=0)),
58        ('stable+wt', dict(use_stable='true', run_wt=1)),
59        ('stable+wt2', dict(use_stable='true', run_wt=2)),
60    ]
61    scenarios = make_scenarios(types)
62
63    def data_and_checkpoint(self):
64        #
65        # Create several collection-like tables that are checkpoint durability.
66        # Add data to each of them separately and checkpoint so that each one
67        # has a different stable timestamp.
68        #
69        self.session.create(self.oplog_uri, 'key_format=i,value_format=i')
70        self.session.create(self.coll1_uri, 'key_format=i,value_format=i,log=(enabled=false)')
71        self.session.create(self.coll2_uri, 'key_format=i,value_format=i,log=(enabled=false)')
72        self.session.create(self.coll3_uri, 'key_format=i,value_format=i,log=(enabled=false)')
73        c_op = self.session.open_cursor(self.oplog_uri)
74        c = []
75        c.append(self.session.open_cursor(self.coll1_uri))
76        c.append(self.session.open_cursor(self.coll2_uri))
77        c.append(self.session.open_cursor(self.coll3_uri))
78
79        # Begin by adding some data.
80        for table in range(1,self.table_cnt+1):
81            curs = c[table - 1]
82            start = self.nentries * table
83            end = start + self.nentries
84            ts = (end - 3)
85            self.pr("table: " + str(table))
86            for i in range(start,end):
87                self.session.begin_transaction()
88                c_op[i] = i
89                curs[i] = i
90                self.pr("i: " + str(i))
91                self.session.commit_transaction(
92                  'commit_timestamp=' + timestamp_str(i))
93            # Set the oldest and stable timestamp a bit earlier than the data
94            # we inserted. Take a checkpoint to the stable timestamp.
95            self.pr("stable ts: " + str(ts))
96            self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(ts) +
97                ',stable_timestamp=' + timestamp_str(ts))
98            # This forces a different checkpoint timestamp for each table.
99            self.session.checkpoint()
100            q = self.conn.query_timestamp('get=last_checkpoint')
101            self.assertTimestampsEqual(q, timestamp_str(ts))
102        return ts
103
104    def close_and_recover(self, expected_rec_ts):
105        #
106        # Close with the close configuration string and optionally run
107        # the 'wt' command. Then open the connection again and query the
108        # recovery timestamp verifying it is the expected value.
109        #
110        if self.use_stable == 'true':
111            close_cfg = 'use_timestamp=true'
112        elif self.use_stable == 'false':
113            close_cfg = 'use_timestamp=false'
114        else:
115            close_cfg = ''
116        self.close_conn(close_cfg)
117
118        # Run the wt command some number of times to get some runs in that do
119        # not use timestamps. Make sure the recovery checkpoint is maintained.
120        for i in range(0, self.run_wt):
121            self.runWt(['-h', '.', '-R', 'list', '-v'], outfilename="list.out")
122
123        self.open_conn()
124        q = self.conn.query_timestamp('get=recovery')
125        self.pr("query recovery ts: " + q)
126        self.assertTimestampsEqual(q, timestamp_str(expected_rec_ts))
127
128    def test_timestamp_recovery(self):
129        if not wiredtiger.timestamp_build():
130            self.skipTest('requires a timestamp build')
131
132        # Add some data and checkpoint at a stable timestamp.
133        last_stable = self.data_and_checkpoint()
134
135        expected = 0
136        # Note: assumes default is true.
137        if self.use_stable != 'false':
138            expected = last_stable
139        # Close and run recovery checking the stable timestamp.
140        self.close_and_recover(expected)
141
142        # Verify the data in the recovered database.
143        c_op = self.session.open_cursor(self.oplog_uri)
144        c = []
145        c.append(self.session.open_cursor(self.coll1_uri))
146        c.append(self.session.open_cursor(self.coll2_uri))
147        c.append(self.session.open_cursor(self.coll3_uri))
148        for table in range(1,self.table_cnt+1):
149            curs = c[table - 1]
150            start = self.nentries * table
151            end = start + self.nentries
152            ts = (end - 3)
153            for i in range(start,end):
154                # The oplog-like table is logged so it always has all the data.
155                self.assertEquals(c_op[i], i)
156                curs.set_key(i)
157                # Earlier tables have all the data because later checkpoints
158                # will save the last bit of data. Only the last table will
159                # be missing some.
160                if self.use_stable == 'false' or i <= ts or table != self.table_cnt:
161                    self.assertEquals(curs[i], i)
162                else:
163                    self.assertEqual(curs.search(), wiredtiger.WT_NOTFOUND)
164
165if __name__ == '__main__':
166    wttest.run()
167