1# Tests of Python-specific parts of the xapian bindings.
2#
3# Copyright (C) 2007,2008 Lemur Consulting Ltd
4# Copyright (C) 2008,2009 Olly Betts
5#
6# This program is free software; you can redistribute it and/or
7# modify it under the terms of the GNU General Public License as
8# published by the Free Software Foundation; either version 2 of the
9# License, or (at your option) any later version.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
19# USA
20
21import os
22import shutil
23import subprocess
24import sys
25import time
26import xapian
27
28from testsuite import *
29
30def set_master(masterpath, srcpath):
31    # Take a copy of the source, to make modifications to.
32    if os.path.exists(masterpath + "_"):
33        shutil.rmtree(masterpath + "_")
34    shutil.copytree(srcpath, masterpath + "_")
35
36    # Set a new uuid on the copy.
37    xapian.WritableDatabase(masterpath + "__", xapian.DB_CREATE_OR_OVERWRITE)
38    os.unlink(os.path.join(masterpath + "_", "iamchert"))
39    os.rename(os.path.join(masterpath + "__", "iamchert"),
40              os.path.join(masterpath + "_", "iamchert"))
41    shutil.rmtree(masterpath + "__")
42
43    # Replace the current master with the copy of the source.
44    # Note that this isn't an atomic replace, so we'll sometimes get errors
45    # such as "NetworkError: Unable to fully synchronise: Can't open database:
46    # Cannot open tables at consistent revisions" - the replication protocol
47    # should recover happily from this, though.
48    if os.path.exists(masterpath):
49        os.rename(masterpath, masterpath + "__")
50    os.rename(masterpath + '_', masterpath)
51    if os.path.exists(masterpath + "__"):
52        shutil.rmtree(masterpath + "__")
53
54def test_replication_concurrency():
55    """Test concurrent replication and modification
56
57    """
58
59    builddir = os.environ['abs_builddir']
60    dbsdir = os.path.join(builddir, 'dbs_replication')
61    if not os.path.isdir(dbsdir):
62        os.makedirs(dbsdir)
63
64    masterpath = os.path.join(dbsdir, 'master')
65    firstpath = os.path.join(dbsdir, 'first')
66    secondpath = os.path.join(dbsdir, 'second')
67    slavepath = os.path.join(dbsdir, 'slave')
68    if os.path.isdir(masterpath):
69        shutil.rmtree(masterpath)
70    if os.path.isdir(slavepath):
71        shutil.rmtree(slavepath)
72    port = 7876
73
74    expect_exception(xapian.DatabaseOpeningError,
75                     "Couldn't stat '" + dbsdir + "/slave' (No such file or directory)",
76                     xapian.Database, slavepath)
77
78    clientp = None
79    serverp = subprocess.Popen(('../../xapian-core/bin/xapian-replicate-server',
80                                dbsdir,
81                                '--port=7876',
82                               ),
83                              )
84
85    doccount1 = 10000
86    doccount2 = 1000
87
88    starttime = time.time()
89    if not os.path.isdir(firstpath):
90        firstdb = xapian.WritableDatabase(firstpath, xapian.DB_CREATE_OR_OVERWRITE)
91        # Make an initial, large database
92        print
93        print "Building initial database ..."
94        for num in xrange(1, doccount1):
95            doc=xapian.Document()
96            val = 'val%d' % num
97            doc.add_value(1, val)
98            firstdb.add_document(doc)
99            if num % 100000 == 0:
100                print "%d documents..." % num
101        firstdb.set_metadata('dbname', '1')
102        firstdb.commit()
103        print "built"
104
105    # The secondary database gets modified during the test, so needs to be
106    # cleared now.
107    shutil.rmtree(secondpath)
108    if not os.path.isdir(secondpath):
109        seconddb = xapian.WritableDatabase(secondpath, xapian.DB_CREATE_OR_OVERWRITE)
110        # Make second, small database
111        print
112        print "Building secondary database ..."
113        for num in xrange(1, doccount2):
114            doc=xapian.Document()
115            val = 'val%d' % num
116            doc.add_value(1, val)
117            seconddb.add_document(doc)
118            if num % 100000 == 0:
119                print "%d documents..." % num
120        seconddb.set_metadata('dbname', '2')
121        seconddb.commit()
122        print "built"
123
124    if time.time() - starttime < 1:
125        time.sleep(1) # Give server time to start
126
127    try:
128        set_master(masterpath, firstpath)
129        clientp = subprocess.Popen(('../../xapian-core/bin/xapian-replicate',
130                                    '--host=127.0.0.1',
131                                    '--master=master',
132                                    os.path.join(dbsdir, 'slave'),
133                                    '--interval=0',
134                                    '--port=7876',
135                                    '-r 0',
136                                   ),
137                                  )
138        time.sleep(1) # Give client time to start
139        expect(xapian.Database(slavepath).get_metadata('dbname'), '1')
140
141        for count in xrange(10):
142            # Test that swapping between databases doesn't confuse replication.
143            for count2 in xrange(2):
144                set_master(masterpath, secondpath)
145                time.sleep(0.1)
146                set_master(masterpath, firstpath)
147                time.sleep(0.1)
148
149            # Test making changes to the database.
150            set_master(masterpath, secondpath)
151            masterdb = xapian.WritableDatabase(masterpath, xapian.DB_OPEN)
152            print "making 100 changes"
153            for num in xrange(100):
154                masterdb.set_metadata('num%d' % num, str(num + count))
155                masterdb.commit()
156            print "changes done"
157            masterdb.close()
158
159            # Allow time for the replication client to catch up with the
160            # changes.
161            time.sleep(2)
162            expect(xapian.Database(slavepath).get_metadata('dbname'), '2')
163            expect(xapian.Database(slavepath).get_metadata('num99'), str(99 + count))
164
165    finally:
166        if clientp is not None:
167            os.kill(clientp.pid, 9)
168            clientp.wait()
169        os.kill(serverp.pid, 9)
170        serverp.wait()
171        #shutil.rmtree(dbsdir)
172
173# Run all tests (ie, callables with names starting "test_").
174if not runtests(globals(), sys.argv[1:]):
175    sys.exit(1)
176
177# vim:syntax=python:set expandtab:
178