1 /*
2 ** 2014 August 30
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 *************************************************************************
12 **
13 **
14 ** OVERVIEW
15 **
16 ** The RBU extension requires that the RBU update be packaged as an
17 ** SQLite database. The tables it expects to find are described in
18 ** sqlite3rbu.h. Essentially, for each table xyz in the target database
19 ** that the user wishes to write to, a corresponding data_xyz table is
20 ** created in the RBU database and populated with one row for each row to
21 ** update, insert or delete from the target table.
22 **
23 ** The update proceeds in three stages:
24 **
25 ** 1) The database is updated. The modified database pages are written
26 ** to a *-oal file. A *-oal file is just like a *-wal file, except
27 ** that it is named "<database>-oal" instead of "<database>-wal".
28 ** Because regular SQLite clients do not look for file named
29 ** "<database>-oal", they go on using the original database in
30 ** rollback mode while the *-oal file is being generated.
31 **
32 ** During this stage RBU does not update the database by writing
33 ** directly to the target tables. Instead it creates "imposter"
34 ** tables using the SQLITE_TESTCTRL_IMPOSTER interface that it uses
35 ** to update each b-tree individually. All updates required by each
36 ** b-tree are completed before moving on to the next, and all
37 ** updates are done in sorted key order.
38 **
39 ** 2) The "<database>-oal" file is moved to the equivalent "<database>-wal"
40 ** location using a call to rename(2). Before doing this the RBU
41 ** module takes an EXCLUSIVE lock on the database file, ensuring
42 ** that there are no other active readers.
43 **
44 ** Once the EXCLUSIVE lock is released, any other database readers
45 ** detect the new *-wal file and read the database in wal mode. At
46 ** this point they see the new version of the database - including
47 ** the updates made as part of the RBU update.
48 **
49 ** 3) The new *-wal file is checkpointed. This proceeds in the same way
50 ** as a regular database checkpoint, except that a single frame is
51 ** checkpointed each time sqlite3rbu_step() is called. If the RBU
52 ** handle is closed before the entire *-wal file is checkpointed,
53 ** the checkpoint progress is saved in the RBU database and the
54 ** checkpoint can be resumed by another RBU client at some point in
55 ** the future.
56 **
57 ** POTENTIAL PROBLEMS
58 **
59 ** The rename() call might not be portable. And RBU is not currently
60 ** syncing the directory after renaming the file.
61 **
62 ** When state is saved, any commit to the *-oal file and the commit to
63 ** the RBU update database are not atomic. So if the power fails at the
64 ** wrong moment they might get out of sync. As the main database will be
65 ** committed before the RBU update database this will likely either just
66 ** pass unnoticed, or result in SQLITE_CONSTRAINT errors (due to UNIQUE
67 ** constraint violations).
68 **
69 ** If some client does modify the target database mid RBU update, or some
70 ** other error occurs, the RBU extension will keep throwing errors. It's
71 ** not really clear how to get out of this state. The system could just
72 ** by delete the RBU update database and *-oal file and have the device
73 ** download the update again and start over.
74 **
75 ** At present, for an UPDATE, both the new.* and old.* records are
76 ** collected in the rbu_xyz table. And for both UPDATEs and DELETEs all
77 ** fields are collected. This means we're probably writing a lot more
78 ** data to disk when saving the state of an ongoing update to the RBU
79 ** update database than is strictly necessary.
80 **
81 */
82
83 #include <assert.h>
84 #include <string.h>
85 #include <stdio.h>
86
87 #include "sqlite3.h"
88
89 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU)
90 #include "sqlite3rbu.h"
91
92 #if defined(_WIN32_WCE)
93 #include "windows.h"
94 #endif
95
96 /* Maximum number of prepared UPDATE statements held by this module */
97 #define SQLITE_RBU_UPDATE_CACHESIZE 16
98
99 /* Delta checksums disabled by default. Compile with -DRBU_ENABLE_DELTA_CKSUM
100 ** to enable checksum verification.
101 */
102 #ifndef RBU_ENABLE_DELTA_CKSUM
103 # define RBU_ENABLE_DELTA_CKSUM 0
104 #endif
105
106 /*
107 ** Swap two objects of type TYPE.
108 */
109 #if !defined(SQLITE_AMALGAMATION)
110 # define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;}
111 #endif
112
113 /*
114 ** The rbu_state table is used to save the state of a partially applied
115 ** update so that it can be resumed later. The table consists of integer
116 ** keys mapped to values as follows:
117 **
118 ** RBU_STATE_STAGE:
119 ** May be set to integer values 1, 2, 4 or 5. As follows:
120 ** 1: the *-rbu file is currently under construction.
121 ** 2: the *-rbu file has been constructed, but not yet moved
122 ** to the *-wal path.
123 ** 4: the checkpoint is underway.
124 ** 5: the rbu update has been checkpointed.
125 **
126 ** RBU_STATE_TBL:
127 ** Only valid if STAGE==1. The target database name of the table
128 ** currently being written.
129 **
130 ** RBU_STATE_IDX:
131 ** Only valid if STAGE==1. The target database name of the index
132 ** currently being written, or NULL if the main table is currently being
133 ** updated.
134 **
135 ** RBU_STATE_ROW:
136 ** Only valid if STAGE==1. Number of rows already processed for the current
137 ** table/index.
138 **
139 ** RBU_STATE_PROGRESS:
140 ** Trbul number of sqlite3rbu_step() calls made so far as part of this
141 ** rbu update.
142 **
143 ** RBU_STATE_CKPT:
144 ** Valid if STAGE==4. The 64-bit checksum associated with the wal-index
145 ** header created by recovering the *-wal file. This is used to detect
146 ** cases when another client appends frames to the *-wal file in the
147 ** middle of an incremental checkpoint (an incremental checkpoint cannot
148 ** be continued if this happens).
149 **
150 ** RBU_STATE_COOKIE:
151 ** Valid if STAGE==1. The current change-counter cookie value in the
152 ** target db file.
153 **
154 ** RBU_STATE_OALSZ:
155 ** Valid if STAGE==1. The size in bytes of the *-oal file.
156 **
157 ** RBU_STATE_DATATBL:
158 ** Only valid if STAGE==1. The RBU database name of the table
159 ** currently being read.
160 */
161 #define RBU_STATE_STAGE 1
162 #define RBU_STATE_TBL 2
163 #define RBU_STATE_IDX 3
164 #define RBU_STATE_ROW 4
165 #define RBU_STATE_PROGRESS 5
166 #define RBU_STATE_CKPT 6
167 #define RBU_STATE_COOKIE 7
168 #define RBU_STATE_OALSZ 8
169 #define RBU_STATE_PHASEONESTEP 9
170 #define RBU_STATE_DATATBL 10
171
172 #define RBU_STAGE_OAL 1
173 #define RBU_STAGE_MOVE 2
174 #define RBU_STAGE_CAPTURE 3
175 #define RBU_STAGE_CKPT 4
176 #define RBU_STAGE_DONE 5
177
178
179 #define RBU_CREATE_STATE \
180 "CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)"
181
182 typedef struct RbuFrame RbuFrame;
183 typedef struct RbuObjIter RbuObjIter;
184 typedef struct RbuState RbuState;
185 typedef struct RbuSpan RbuSpan;
186 typedef struct rbu_vfs rbu_vfs;
187 typedef struct rbu_file rbu_file;
188 typedef struct RbuUpdateStmt RbuUpdateStmt;
189
190 #if !defined(SQLITE_AMALGAMATION)
191 typedef unsigned int u32;
192 typedef unsigned short u16;
193 typedef unsigned char u8;
194 typedef sqlite3_int64 i64;
195 #endif
196
197 /*
198 ** These values must match the values defined in wal.c for the equivalent
199 ** locks. These are not magic numbers as they are part of the SQLite file
200 ** format.
201 */
202 #define WAL_LOCK_WRITE 0
203 #define WAL_LOCK_CKPT 1
204 #define WAL_LOCK_READ0 3
205
206 #define SQLITE_FCNTL_RBUCNT 5149216
207
208 /*
209 ** A structure to store values read from the rbu_state table in memory.
210 */
211 struct RbuState {
212 int eStage;
213 char *zTbl;
214 char *zDataTbl;
215 char *zIdx;
216 i64 iWalCksum;
217 int nRow;
218 i64 nProgress;
219 u32 iCookie;
220 i64 iOalSz;
221 i64 nPhaseOneStep;
222 };
223
224 struct RbuUpdateStmt {
225 char *zMask; /* Copy of update mask used with pUpdate */
226 sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */
227 RbuUpdateStmt *pNext;
228 };
229
230 struct RbuSpan {
231 const char *zSpan;
232 int nSpan;
233 };
234
235 /*
236 ** An iterator of this type is used to iterate through all objects in
237 ** the target database that require updating. For each such table, the
238 ** iterator visits, in order:
239 **
240 ** * the table itself,
241 ** * each index of the table (zero or more points to visit), and
242 ** * a special "cleanup table" state.
243 **
244 ** abIndexed:
245 ** If the table has no indexes on it, abIndexed is set to NULL. Otherwise,
246 ** it points to an array of flags nTblCol elements in size. The flag is
247 ** set for each column that is either a part of the PK or a part of an
248 ** index. Or clear otherwise.
249 **
250 ** If there are one or more partial indexes on the table, all fields of
251 ** this array set set to 1. This is because in that case, the module has
252 ** no way to tell which fields will be required to add and remove entries
253 ** from the partial indexes.
254 **
255 */
256 struct RbuObjIter {
257 sqlite3_stmt *pTblIter; /* Iterate through tables */
258 sqlite3_stmt *pIdxIter; /* Index iterator */
259 int nTblCol; /* Size of azTblCol[] array */
260 char **azTblCol; /* Array of unquoted target column names */
261 char **azTblType; /* Array of target column types */
262 int *aiSrcOrder; /* src table col -> target table col */
263 u8 *abTblPk; /* Array of flags, set on target PK columns */
264 u8 *abNotNull; /* Array of flags, set on NOT NULL columns */
265 u8 *abIndexed; /* Array of flags, set on indexed & PK cols */
266 int eType; /* Table type - an RBU_PK_XXX value */
267
268 /* Output variables. zTbl==0 implies EOF. */
269 int bCleanup; /* True in "cleanup" state */
270 const char *zTbl; /* Name of target db table */
271 const char *zDataTbl; /* Name of rbu db table (or null) */
272 const char *zIdx; /* Name of target db index (or null) */
273 int iTnum; /* Root page of current object */
274 int iPkTnum; /* If eType==EXTERNAL, root of PK index */
275 int bUnique; /* Current index is unique */
276 int nIndex; /* Number of aux. indexes on table zTbl */
277
278 /* Statements created by rbuObjIterPrepareAll() */
279 int nCol; /* Number of columns in current object */
280 sqlite3_stmt *pSelect; /* Source data */
281 sqlite3_stmt *pInsert; /* Statement for INSERT operations */
282 sqlite3_stmt *pDelete; /* Statement for DELETE ops */
283 sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */
284 int nIdxCol;
285 RbuSpan *aIdxCol;
286 char *zIdxSql;
287
288 /* Last UPDATE used (for PK b-tree updates only), or NULL. */
289 RbuUpdateStmt *pRbuUpdate;
290 };
291
292 /*
293 ** Values for RbuObjIter.eType
294 **
295 ** 0: Table does not exist (error)
296 ** 1: Table has an implicit rowid.
297 ** 2: Table has an explicit IPK column.
298 ** 3: Table has an external PK index.
299 ** 4: Table is WITHOUT ROWID.
300 ** 5: Table is a virtual table.
301 */
302 #define RBU_PK_NOTABLE 0
303 #define RBU_PK_NONE 1
304 #define RBU_PK_IPK 2
305 #define RBU_PK_EXTERNAL 3
306 #define RBU_PK_WITHOUT_ROWID 4
307 #define RBU_PK_VTAB 5
308
309
310 /*
311 ** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs
312 ** one of the following operations.
313 */
314 #define RBU_INSERT 1 /* Insert on a main table b-tree */
315 #define RBU_DELETE 2 /* Delete a row from a main table b-tree */
316 #define RBU_REPLACE 3 /* Delete and then insert a row */
317 #define RBU_IDX_DELETE 4 /* Delete a row from an aux. index b-tree */
318 #define RBU_IDX_INSERT 5 /* Insert on an aux. index b-tree */
319
320 #define RBU_UPDATE 6 /* Update a row in a main table b-tree */
321
322 /*
323 ** A single step of an incremental checkpoint - frame iWalFrame of the wal
324 ** file should be copied to page iDbPage of the database file.
325 */
326 struct RbuFrame {
327 u32 iDbPage;
328 u32 iWalFrame;
329 };
330
331 /*
332 ** RBU handle.
333 **
334 ** nPhaseOneStep:
335 ** If the RBU database contains an rbu_count table, this value is set to
336 ** a running estimate of the number of b-tree operations required to
337 ** finish populating the *-oal file. This allows the sqlite3_bp_progress()
338 ** API to calculate the permyriadage progress of populating the *-oal file
339 ** using the formula:
340 **
341 ** permyriadage = (10000 * nProgress) / nPhaseOneStep
342 **
343 ** nPhaseOneStep is initialized to the sum of:
344 **
345 ** nRow * (nIndex + 1)
346 **
347 ** for all source tables in the RBU database, where nRow is the number
348 ** of rows in the source table and nIndex the number of indexes on the
349 ** corresponding target database table.
350 **
351 ** This estimate is accurate if the RBU update consists entirely of
352 ** INSERT operations. However, it is inaccurate if:
353 **
354 ** * the RBU update contains any UPDATE operations. If the PK specified
355 ** for an UPDATE operation does not exist in the target table, then
356 ** no b-tree operations are required on index b-trees. Or if the
357 ** specified PK does exist, then (nIndex*2) such operations are
358 ** required (one delete and one insert on each index b-tree).
359 **
360 ** * the RBU update contains any DELETE operations for which the specified
361 ** PK does not exist. In this case no operations are required on index
362 ** b-trees.
363 **
364 ** * the RBU update contains REPLACE operations. These are similar to
365 ** UPDATE operations.
366 **
367 ** nPhaseOneStep is updated to account for the conditions above during the
368 ** first pass of each source table. The updated nPhaseOneStep value is
369 ** stored in the rbu_state table if the RBU update is suspended.
370 */
371 struct sqlite3rbu {
372 int eStage; /* Value of RBU_STATE_STAGE field */
373 sqlite3 *dbMain; /* target database handle */
374 sqlite3 *dbRbu; /* rbu database handle */
375 char *zTarget; /* Path to target db */
376 char *zRbu; /* Path to rbu db */
377 char *zState; /* Path to state db (or NULL if zRbu) */
378 char zStateDb[5]; /* Db name for state ("stat" or "main") */
379 int rc; /* Value returned by last rbu_step() call */
380 char *zErrmsg; /* Error message if rc!=SQLITE_OK */
381 int nStep; /* Rows processed for current object */
382 int nProgress; /* Rows processed for all objects */
383 RbuObjIter objiter; /* Iterator for skipping through tbl/idx */
384 const char *zVfsName; /* Name of automatically created rbu vfs */
385 rbu_file *pTargetFd; /* File handle open on target db */
386 int nPagePerSector; /* Pages per sector for pTargetFd */
387 i64 iOalSz;
388 i64 nPhaseOneStep;
389
390 /* The following state variables are used as part of the incremental
391 ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding
392 ** function rbuSetupCheckpoint() for details. */
393 u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */
394 u32 mLock;
395 int nFrame; /* Entries in aFrame[] array */
396 int nFrameAlloc; /* Allocated size of aFrame[] array */
397 RbuFrame *aFrame;
398 int pgsz;
399 u8 *aBuf;
400 i64 iWalCksum;
401 i64 szTemp; /* Current size of all temp files in use */
402 i64 szTempLimit; /* Total size limit for temp files */
403
404 /* Used in RBU vacuum mode only */
405 int nRbu; /* Number of RBU VFS in the stack */
406 rbu_file *pRbuFd; /* Fd for main db of dbRbu */
407 };
408
409 /*
410 ** An rbu VFS is implemented using an instance of this structure.
411 **
412 ** Variable pRbu is only non-NULL for automatically created RBU VFS objects.
413 ** It is NULL for RBU VFS objects created explicitly using
414 ** sqlite3rbu_create_vfs(). It is used to track the total amount of temp
415 ** space used by the RBU handle.
416 */
417 struct rbu_vfs {
418 sqlite3_vfs base; /* rbu VFS shim methods */
419 sqlite3_vfs *pRealVfs; /* Underlying VFS */
420 sqlite3_mutex *mutex; /* Mutex to protect pMain */
421 sqlite3rbu *pRbu; /* Owner RBU object */
422 rbu_file *pMain; /* List of main db files */
423 rbu_file *pMainRbu; /* List of main db files with pRbu!=0 */
424 };
425
426 /*
427 ** Each file opened by an rbu VFS is represented by an instance of
428 ** the following structure.
429 **
430 ** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable
431 ** "sz" is set to the current size of the database file.
432 */
433 struct rbu_file {
434 sqlite3_file base; /* sqlite3_file methods */
435 sqlite3_file *pReal; /* Underlying file handle */
436 rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */
437 sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */
438 i64 sz; /* Size of file in bytes (temp only) */
439
440 int openFlags; /* Flags this file was opened with */
441 u32 iCookie; /* Cookie value for main db files */
442 u8 iWriteVer; /* "write-version" value for main db files */
443 u8 bNolock; /* True to fail EXCLUSIVE locks */
444
445 int nShm; /* Number of entries in apShm[] array */
446 char **apShm; /* Array of mmap'd *-shm regions */
447 char *zDel; /* Delete this when closing file */
448
449 const char *zWal; /* Wal filename for this main db file */
450 rbu_file *pWalFd; /* Wal file descriptor for this main db */
451 rbu_file *pMainNext; /* Next MAIN_DB file */
452 rbu_file *pMainRbuNext; /* Next MAIN_DB file with pRbu!=0 */
453 };
454
455 /*
456 ** True for an RBU vacuum handle, or false otherwise.
457 */
458 #define rbuIsVacuum(p) ((p)->zTarget==0)
459
460
461 /*************************************************************************
462 ** The following three functions, found below:
463 **
464 ** rbuDeltaGetInt()
465 ** rbuDeltaChecksum()
466 ** rbuDeltaApply()
467 **
468 ** are lifted from the fossil source code (http://fossil-scm.org). They
469 ** are used to implement the scalar SQL function rbu_fossil_delta().
470 */
471
472 /*
473 ** Read bytes from *pz and convert them into a positive integer. When
474 ** finished, leave *pz pointing to the first character past the end of
475 ** the integer. The *pLen parameter holds the length of the string
476 ** in *pz and is decremented once for each character in the integer.
477 */
rbuDeltaGetInt(const char ** pz,int * pLen)478 static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){
479 static const signed char zValue[] = {
480 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
481 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
482 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
483 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1,
484 -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
485 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36,
486 -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
487 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1,
488 };
489 unsigned int v = 0;
490 int c;
491 unsigned char *z = (unsigned char*)*pz;
492 unsigned char *zStart = z;
493 while( (c = zValue[0x7f&*(z++)])>=0 ){
494 v = (v<<6) + c;
495 }
496 z--;
497 *pLen -= z - zStart;
498 *pz = (char*)z;
499 return v;
500 }
501
502 #if RBU_ENABLE_DELTA_CKSUM
503 /*
504 ** Compute a 32-bit checksum on the N-byte buffer. Return the result.
505 */
rbuDeltaChecksum(const char * zIn,size_t N)506 static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){
507 const unsigned char *z = (const unsigned char *)zIn;
508 unsigned sum0 = 0;
509 unsigned sum1 = 0;
510 unsigned sum2 = 0;
511 unsigned sum3 = 0;
512 while(N >= 16){
513 sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]);
514 sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]);
515 sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]);
516 sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]);
517 z += 16;
518 N -= 16;
519 }
520 while(N >= 4){
521 sum0 += z[0];
522 sum1 += z[1];
523 sum2 += z[2];
524 sum3 += z[3];
525 z += 4;
526 N -= 4;
527 }
528 sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24);
529 switch(N){
530 case 3: sum3 += (z[2] << 8);
531 case 2: sum3 += (z[1] << 16);
532 case 1: sum3 += (z[0] << 24);
533 default: ;
534 }
535 return sum3;
536 }
537 #endif
538
539 /*
540 ** Apply a delta.
541 **
542 ** The output buffer should be big enough to hold the whole output
543 ** file and a NUL terminator at the end. The delta_output_size()
544 ** routine will determine this size for you.
545 **
546 ** The delta string should be null-terminated. But the delta string
547 ** may contain embedded NUL characters (if the input and output are
548 ** binary files) so we also have to pass in the length of the delta in
549 ** the lenDelta parameter.
550 **
551 ** This function returns the size of the output file in bytes (excluding
552 ** the final NUL terminator character). Except, if the delta string is
553 ** malformed or intended for use with a source file other than zSrc,
554 ** then this routine returns -1.
555 **
556 ** Refer to the delta_create() documentation above for a description
557 ** of the delta file format.
558 */
rbuDeltaApply(const char * zSrc,int lenSrc,const char * zDelta,int lenDelta,char * zOut)559 static int rbuDeltaApply(
560 const char *zSrc, /* The source or pattern file */
561 int lenSrc, /* Length of the source file */
562 const char *zDelta, /* Delta to apply to the pattern */
563 int lenDelta, /* Length of the delta */
564 char *zOut /* Write the output into this preallocated buffer */
565 ){
566 unsigned int limit;
567 unsigned int total = 0;
568 #if RBU_ENABLE_DELTA_CKSUM
569 char *zOrigOut = zOut;
570 #endif
571
572 limit = rbuDeltaGetInt(&zDelta, &lenDelta);
573 if( *zDelta!='\n' ){
574 /* ERROR: size integer not terminated by "\n" */
575 return -1;
576 }
577 zDelta++; lenDelta--;
578 while( *zDelta && lenDelta>0 ){
579 unsigned int cnt, ofst;
580 cnt = rbuDeltaGetInt(&zDelta, &lenDelta);
581 switch( zDelta[0] ){
582 case '@': {
583 zDelta++; lenDelta--;
584 ofst = rbuDeltaGetInt(&zDelta, &lenDelta);
585 if( lenDelta>0 && zDelta[0]!=',' ){
586 /* ERROR: copy command not terminated by ',' */
587 return -1;
588 }
589 zDelta++; lenDelta--;
590 total += cnt;
591 if( total>limit ){
592 /* ERROR: copy exceeds output file size */
593 return -1;
594 }
595 if( (int)(ofst+cnt) > lenSrc ){
596 /* ERROR: copy extends past end of input */
597 return -1;
598 }
599 memcpy(zOut, &zSrc[ofst], cnt);
600 zOut += cnt;
601 break;
602 }
603 case ':': {
604 zDelta++; lenDelta--;
605 total += cnt;
606 if( total>limit ){
607 /* ERROR: insert command gives an output larger than predicted */
608 return -1;
609 }
610 if( (int)cnt>lenDelta ){
611 /* ERROR: insert count exceeds size of delta */
612 return -1;
613 }
614 memcpy(zOut, zDelta, cnt);
615 zOut += cnt;
616 zDelta += cnt;
617 lenDelta -= cnt;
618 break;
619 }
620 case ';': {
621 zDelta++; lenDelta--;
622 zOut[0] = 0;
623 #if RBU_ENABLE_DELTA_CKSUM
624 if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){
625 /* ERROR: bad checksum */
626 return -1;
627 }
628 #endif
629 if( total!=limit ){
630 /* ERROR: generated size does not match predicted size */
631 return -1;
632 }
633 return total;
634 }
635 default: {
636 /* ERROR: unknown delta operator */
637 return -1;
638 }
639 }
640 }
641 /* ERROR: unterminated delta */
642 return -1;
643 }
644
rbuDeltaOutputSize(const char * zDelta,int lenDelta)645 static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){
646 int size;
647 size = rbuDeltaGetInt(&zDelta, &lenDelta);
648 if( *zDelta!='\n' ){
649 /* ERROR: size integer not terminated by "\n" */
650 return -1;
651 }
652 return size;
653 }
654
655 /*
656 ** End of code taken from fossil.
657 *************************************************************************/
658
659 /*
660 ** Implementation of SQL scalar function rbu_fossil_delta().
661 **
662 ** This function applies a fossil delta patch to a blob. Exactly two
663 ** arguments must be passed to this function. The first is the blob to
664 ** patch and the second the patch to apply. If no error occurs, this
665 ** function returns the patched blob.
666 */
rbuFossilDeltaFunc(sqlite3_context * context,int argc,sqlite3_value ** argv)667 static void rbuFossilDeltaFunc(
668 sqlite3_context *context,
669 int argc,
670 sqlite3_value **argv
671 ){
672 const char *aDelta;
673 int nDelta;
674 const char *aOrig;
675 int nOrig;
676
677 int nOut;
678 int nOut2;
679 char *aOut;
680
681 assert( argc==2 );
682
683 nOrig = sqlite3_value_bytes(argv[0]);
684 aOrig = (const char*)sqlite3_value_blob(argv[0]);
685 nDelta = sqlite3_value_bytes(argv[1]);
686 aDelta = (const char*)sqlite3_value_blob(argv[1]);
687
688 /* Figure out the size of the output */
689 nOut = rbuDeltaOutputSize(aDelta, nDelta);
690 if( nOut<0 ){
691 sqlite3_result_error(context, "corrupt fossil delta", -1);
692 return;
693 }
694
695 aOut = sqlite3_malloc(nOut+1);
696 if( aOut==0 ){
697 sqlite3_result_error_nomem(context);
698 }else{
699 nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut);
700 if( nOut2!=nOut ){
701 sqlite3_free(aOut);
702 sqlite3_result_error(context, "corrupt fossil delta", -1);
703 }else{
704 sqlite3_result_blob(context, aOut, nOut, sqlite3_free);
705 }
706 }
707 }
708
709
710 /*
711 ** Prepare the SQL statement in buffer zSql against database handle db.
712 ** If successful, set *ppStmt to point to the new statement and return
713 ** SQLITE_OK.
714 **
715 ** Otherwise, if an error does occur, set *ppStmt to NULL and return
716 ** an SQLite error code. Additionally, set output variable *pzErrmsg to
717 ** point to a buffer containing an error message. It is the responsibility
718 ** of the caller to (eventually) free this buffer using sqlite3_free().
719 */
prepareAndCollectError(sqlite3 * db,sqlite3_stmt ** ppStmt,char ** pzErrmsg,const char * zSql)720 static int prepareAndCollectError(
721 sqlite3 *db,
722 sqlite3_stmt **ppStmt,
723 char **pzErrmsg,
724 const char *zSql
725 ){
726 int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
727 if( rc!=SQLITE_OK ){
728 *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
729 *ppStmt = 0;
730 }
731 return rc;
732 }
733
734 /*
735 ** Reset the SQL statement passed as the first argument. Return a copy
736 ** of the value returned by sqlite3_reset().
737 **
738 ** If an error has occurred, then set *pzErrmsg to point to a buffer
739 ** containing an error message. It is the responsibility of the caller
740 ** to eventually free this buffer using sqlite3_free().
741 */
resetAndCollectError(sqlite3_stmt * pStmt,char ** pzErrmsg)742 static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){
743 int rc = sqlite3_reset(pStmt);
744 if( rc!=SQLITE_OK ){
745 *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt)));
746 }
747 return rc;
748 }
749
750 /*
751 ** Unless it is NULL, argument zSql points to a buffer allocated using
752 ** sqlite3_malloc containing an SQL statement. This function prepares the SQL
753 ** statement against database db and frees the buffer. If statement
754 ** compilation is successful, *ppStmt is set to point to the new statement
755 ** handle and SQLITE_OK is returned.
756 **
757 ** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code
758 ** returned. In this case, *pzErrmsg may also be set to point to an error
759 ** message. It is the responsibility of the caller to free this error message
760 ** buffer using sqlite3_free().
761 **
762 ** If argument zSql is NULL, this function assumes that an OOM has occurred.
763 ** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL.
764 */
prepareFreeAndCollectError(sqlite3 * db,sqlite3_stmt ** ppStmt,char ** pzErrmsg,char * zSql)765 static int prepareFreeAndCollectError(
766 sqlite3 *db,
767 sqlite3_stmt **ppStmt,
768 char **pzErrmsg,
769 char *zSql
770 ){
771 int rc;
772 assert( *pzErrmsg==0 );
773 if( zSql==0 ){
774 rc = SQLITE_NOMEM;
775 *ppStmt = 0;
776 }else{
777 rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql);
778 sqlite3_free(zSql);
779 }
780 return rc;
781 }
782
783 /*
784 ** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated
785 ** by an earlier call to rbuObjIterCacheTableInfo().
786 */
rbuObjIterFreeCols(RbuObjIter * pIter)787 static void rbuObjIterFreeCols(RbuObjIter *pIter){
788 int i;
789 for(i=0; i<pIter->nTblCol; i++){
790 sqlite3_free(pIter->azTblCol[i]);
791 sqlite3_free(pIter->azTblType[i]);
792 }
793 sqlite3_free(pIter->azTblCol);
794 pIter->azTblCol = 0;
795 pIter->azTblType = 0;
796 pIter->aiSrcOrder = 0;
797 pIter->abTblPk = 0;
798 pIter->abNotNull = 0;
799 pIter->nTblCol = 0;
800 pIter->eType = 0; /* Invalid value */
801 }
802
803 /*
804 ** Finalize all statements and free all allocations that are specific to
805 ** the current object (table/index pair).
806 */
rbuObjIterClearStatements(RbuObjIter * pIter)807 static void rbuObjIterClearStatements(RbuObjIter *pIter){
808 RbuUpdateStmt *pUp;
809
810 sqlite3_finalize(pIter->pSelect);
811 sqlite3_finalize(pIter->pInsert);
812 sqlite3_finalize(pIter->pDelete);
813 sqlite3_finalize(pIter->pTmpInsert);
814 pUp = pIter->pRbuUpdate;
815 while( pUp ){
816 RbuUpdateStmt *pTmp = pUp->pNext;
817 sqlite3_finalize(pUp->pUpdate);
818 sqlite3_free(pUp);
819 pUp = pTmp;
820 }
821 sqlite3_free(pIter->aIdxCol);
822 sqlite3_free(pIter->zIdxSql);
823
824 pIter->pSelect = 0;
825 pIter->pInsert = 0;
826 pIter->pDelete = 0;
827 pIter->pRbuUpdate = 0;
828 pIter->pTmpInsert = 0;
829 pIter->nCol = 0;
830 pIter->nIdxCol = 0;
831 pIter->aIdxCol = 0;
832 pIter->zIdxSql = 0;
833 }
834
835 /*
836 ** Clean up any resources allocated as part of the iterator object passed
837 ** as the only argument.
838 */
rbuObjIterFinalize(RbuObjIter * pIter)839 static void rbuObjIterFinalize(RbuObjIter *pIter){
840 rbuObjIterClearStatements(pIter);
841 sqlite3_finalize(pIter->pTblIter);
842 sqlite3_finalize(pIter->pIdxIter);
843 rbuObjIterFreeCols(pIter);
844 memset(pIter, 0, sizeof(RbuObjIter));
845 }
846
847 /*
848 ** Advance the iterator to the next position.
849 **
850 ** If no error occurs, SQLITE_OK is returned and the iterator is left
851 ** pointing to the next entry. Otherwise, an error code and message is
852 ** left in the RBU handle passed as the first argument. A copy of the
853 ** error code is returned.
854 */
rbuObjIterNext(sqlite3rbu * p,RbuObjIter * pIter)855 static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){
856 int rc = p->rc;
857 if( rc==SQLITE_OK ){
858
859 /* Free any SQLite statements used while processing the previous object */
860 rbuObjIterClearStatements(pIter);
861 if( pIter->zIdx==0 ){
862 rc = sqlite3_exec(p->dbMain,
863 "DROP TRIGGER IF EXISTS temp.rbu_insert_tr;"
864 "DROP TRIGGER IF EXISTS temp.rbu_update1_tr;"
865 "DROP TRIGGER IF EXISTS temp.rbu_update2_tr;"
866 "DROP TRIGGER IF EXISTS temp.rbu_delete_tr;"
867 , 0, 0, &p->zErrmsg
868 );
869 }
870
871 if( rc==SQLITE_OK ){
872 if( pIter->bCleanup ){
873 rbuObjIterFreeCols(pIter);
874 pIter->bCleanup = 0;
875 rc = sqlite3_step(pIter->pTblIter);
876 if( rc!=SQLITE_ROW ){
877 rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg);
878 pIter->zTbl = 0;
879 }else{
880 pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0);
881 pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1);
882 rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM;
883 }
884 }else{
885 if( pIter->zIdx==0 ){
886 sqlite3_stmt *pIdx = pIter->pIdxIter;
887 rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC);
888 }
889 if( rc==SQLITE_OK ){
890 rc = sqlite3_step(pIter->pIdxIter);
891 if( rc!=SQLITE_ROW ){
892 rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg);
893 pIter->bCleanup = 1;
894 pIter->zIdx = 0;
895 }else{
896 pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0);
897 pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1);
898 pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2);
899 rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM;
900 }
901 }
902 }
903 }
904 }
905
906 if( rc!=SQLITE_OK ){
907 rbuObjIterFinalize(pIter);
908 p->rc = rc;
909 }
910 return rc;
911 }
912
913
914 /*
915 ** The implementation of the rbu_target_name() SQL function. This function
916 ** accepts one or two arguments. The first argument is the name of a table -
917 ** the name of a table in the RBU database. The second, if it is present, is 1
918 ** for a view or 0 for a table.
919 **
920 ** For a non-vacuum RBU handle, if the table name matches the pattern:
921 **
922 ** data[0-9]_<name>
923 **
924 ** where <name> is any sequence of 1 or more characters, <name> is returned.
925 ** Otherwise, if the only argument does not match the above pattern, an SQL
926 ** NULL is returned.
927 **
928 ** "data_t1" -> "t1"
929 ** "data0123_t2" -> "t2"
930 ** "dataAB_t3" -> NULL
931 **
932 ** For an rbu vacuum handle, a copy of the first argument is returned if
933 ** the second argument is either missing or 0 (not a view).
934 */
rbuTargetNameFunc(sqlite3_context * pCtx,int argc,sqlite3_value ** argv)935 static void rbuTargetNameFunc(
936 sqlite3_context *pCtx,
937 int argc,
938 sqlite3_value **argv
939 ){
940 sqlite3rbu *p = sqlite3_user_data(pCtx);
941 const char *zIn;
942 assert( argc==1 || argc==2 );
943
944 zIn = (const char*)sqlite3_value_text(argv[0]);
945 if( zIn ){
946 if( rbuIsVacuum(p) ){
947 assert( argc==2 || argc==1 );
948 if( argc==1 || 0==sqlite3_value_int(argv[1]) ){
949 sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC);
950 }
951 }else{
952 if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){
953 int i;
954 for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++);
955 if( zIn[i]=='_' && zIn[i+1] ){
956 sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC);
957 }
958 }
959 }
960 }
961 }
962
963 /*
964 ** Initialize the iterator structure passed as the second argument.
965 **
966 ** If no error occurs, SQLITE_OK is returned and the iterator is left
967 ** pointing to the first entry. Otherwise, an error code and message is
968 ** left in the RBU handle passed as the first argument. A copy of the
969 ** error code is returned.
970 */
rbuObjIterFirst(sqlite3rbu * p,RbuObjIter * pIter)971 static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){
972 int rc;
973 memset(pIter, 0, sizeof(RbuObjIter));
974
975 rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg,
976 sqlite3_mprintf(
977 "SELECT rbu_target_name(name, type='view') AS target, name "
978 "FROM sqlite_schema "
979 "WHERE type IN ('table', 'view') AND target IS NOT NULL "
980 " %s "
981 "ORDER BY name"
982 , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : ""));
983
984 if( rc==SQLITE_OK ){
985 rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg,
986 "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' "
987 " FROM main.sqlite_schema "
988 " WHERE type='index' AND tbl_name = ?"
989 );
990 }
991
992 pIter->bCleanup = 1;
993 p->rc = rc;
994 return rbuObjIterNext(p, pIter);
995 }
996
997 /*
998 ** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs,
999 ** an error code is stored in the RBU handle passed as the first argument.
1000 **
1001 ** If an error has already occurred (p->rc is already set to something other
1002 ** than SQLITE_OK), then this function returns NULL without modifying the
1003 ** stored error code. In this case it still calls sqlite3_free() on any
1004 ** printf() parameters associated with %z conversions.
1005 */
rbuMPrintf(sqlite3rbu * p,const char * zFmt,...)1006 static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){
1007 char *zSql = 0;
1008 va_list ap;
1009 va_start(ap, zFmt);
1010 zSql = sqlite3_vmprintf(zFmt, ap);
1011 if( p->rc==SQLITE_OK ){
1012 if( zSql==0 ) p->rc = SQLITE_NOMEM;
1013 }else{
1014 sqlite3_free(zSql);
1015 zSql = 0;
1016 }
1017 va_end(ap);
1018 return zSql;
1019 }
1020
1021 /*
1022 ** Argument zFmt is a sqlite3_mprintf() style format string. The trailing
1023 ** arguments are the usual subsitution values. This function performs
1024 ** the printf() style substitutions and executes the result as an SQL
1025 ** statement on the RBU handles database.
1026 **
1027 ** If an error occurs, an error code and error message is stored in the
1028 ** RBU handle. If an error has already occurred when this function is
1029 ** called, it is a no-op.
1030 */
rbuMPrintfExec(sqlite3rbu * p,sqlite3 * db,const char * zFmt,...)1031 static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){
1032 va_list ap;
1033 char *zSql;
1034 va_start(ap, zFmt);
1035 zSql = sqlite3_vmprintf(zFmt, ap);
1036 if( p->rc==SQLITE_OK ){
1037 if( zSql==0 ){
1038 p->rc = SQLITE_NOMEM;
1039 }else{
1040 p->rc = sqlite3_exec(db, zSql, 0, 0, &p->zErrmsg);
1041 }
1042 }
1043 sqlite3_free(zSql);
1044 va_end(ap);
1045 return p->rc;
1046 }
1047
1048 /*
1049 ** Attempt to allocate and return a pointer to a zeroed block of nByte
1050 ** bytes.
1051 **
1052 ** If an error (i.e. an OOM condition) occurs, return NULL and leave an
1053 ** error code in the rbu handle passed as the first argument. Or, if an
1054 ** error has already occurred when this function is called, return NULL
1055 ** immediately without attempting the allocation or modifying the stored
1056 ** error code.
1057 */
rbuMalloc(sqlite3rbu * p,sqlite3_int64 nByte)1058 static void *rbuMalloc(sqlite3rbu *p, sqlite3_int64 nByte){
1059 void *pRet = 0;
1060 if( p->rc==SQLITE_OK ){
1061 assert( nByte>0 );
1062 pRet = sqlite3_malloc64(nByte);
1063 if( pRet==0 ){
1064 p->rc = SQLITE_NOMEM;
1065 }else{
1066 memset(pRet, 0, nByte);
1067 }
1068 }
1069 return pRet;
1070 }
1071
1072
1073 /*
1074 ** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that
1075 ** there is room for at least nCol elements. If an OOM occurs, store an
1076 ** error code in the RBU handle passed as the first argument.
1077 */
rbuAllocateIterArrays(sqlite3rbu * p,RbuObjIter * pIter,int nCol)1078 static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){
1079 sqlite3_int64 nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol;
1080 char **azNew;
1081
1082 azNew = (char**)rbuMalloc(p, nByte);
1083 if( azNew ){
1084 pIter->azTblCol = azNew;
1085 pIter->azTblType = &azNew[nCol];
1086 pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol];
1087 pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol];
1088 pIter->abNotNull = (u8*)&pIter->abTblPk[nCol];
1089 pIter->abIndexed = (u8*)&pIter->abNotNull[nCol];
1090 }
1091 }
1092
1093 /*
1094 ** The first argument must be a nul-terminated string. This function
1095 ** returns a copy of the string in memory obtained from sqlite3_malloc().
1096 ** It is the responsibility of the caller to eventually free this memory
1097 ** using sqlite3_free().
1098 **
1099 ** If an OOM condition is encountered when attempting to allocate memory,
1100 ** output variable (*pRc) is set to SQLITE_NOMEM before returning. Otherwise,
1101 ** if the allocation succeeds, (*pRc) is left unchanged.
1102 */
rbuStrndup(const char * zStr,int * pRc)1103 static char *rbuStrndup(const char *zStr, int *pRc){
1104 char *zRet = 0;
1105
1106 if( *pRc==SQLITE_OK ){
1107 if( zStr ){
1108 size_t nCopy = strlen(zStr) + 1;
1109 zRet = (char*)sqlite3_malloc64(nCopy);
1110 if( zRet ){
1111 memcpy(zRet, zStr, nCopy);
1112 }else{
1113 *pRc = SQLITE_NOMEM;
1114 }
1115 }
1116 }
1117
1118 return zRet;
1119 }
1120
1121 /*
1122 ** Finalize the statement passed as the second argument.
1123 **
1124 ** If the sqlite3_finalize() call indicates that an error occurs, and the
1125 ** rbu handle error code is not already set, set the error code and error
1126 ** message accordingly.
1127 */
rbuFinalize(sqlite3rbu * p,sqlite3_stmt * pStmt)1128 static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){
1129 sqlite3 *db = sqlite3_db_handle(pStmt);
1130 int rc = sqlite3_finalize(pStmt);
1131 if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){
1132 p->rc = rc;
1133 p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
1134 }
1135 }
1136
1137 /* Determine the type of a table.
1138 **
1139 ** peType is of type (int*), a pointer to an output parameter of type
1140 ** (int). This call sets the output parameter as follows, depending
1141 ** on the type of the table specified by parameters dbName and zTbl.
1142 **
1143 ** RBU_PK_NOTABLE: No such table.
1144 ** RBU_PK_NONE: Table has an implicit rowid.
1145 ** RBU_PK_IPK: Table has an explicit IPK column.
1146 ** RBU_PK_EXTERNAL: Table has an external PK index.
1147 ** RBU_PK_WITHOUT_ROWID: Table is WITHOUT ROWID.
1148 ** RBU_PK_VTAB: Table is a virtual table.
1149 **
1150 ** Argument *piPk is also of type (int*), and also points to an output
1151 ** parameter. Unless the table has an external primary key index
1152 ** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or,
1153 ** if the table does have an external primary key index, then *piPk
1154 ** is set to the root page number of the primary key index before
1155 ** returning.
1156 **
1157 ** ALGORITHM:
1158 **
1159 ** if( no entry exists in sqlite_schema ){
1160 ** return RBU_PK_NOTABLE
1161 ** }else if( sql for the entry starts with "CREATE VIRTUAL" ){
1162 ** return RBU_PK_VTAB
1163 ** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){
1164 ** if( the index that is the pk exists in sqlite_schema ){
1165 ** *piPK = rootpage of that index.
1166 ** return RBU_PK_EXTERNAL
1167 ** }else{
1168 ** return RBU_PK_WITHOUT_ROWID
1169 ** }
1170 ** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){
1171 ** return RBU_PK_IPK
1172 ** }else{
1173 ** return RBU_PK_NONE
1174 ** }
1175 */
rbuTableType(sqlite3rbu * p,const char * zTab,int * peType,int * piTnum,int * piPk)1176 static void rbuTableType(
1177 sqlite3rbu *p,
1178 const char *zTab,
1179 int *peType,
1180 int *piTnum,
1181 int *piPk
1182 ){
1183 /*
1184 ** 0) SELECT count(*) FROM sqlite_schema where name=%Q AND IsVirtual(%Q)
1185 ** 1) PRAGMA index_list = ?
1186 ** 2) SELECT count(*) FROM sqlite_schema where name=%Q
1187 ** 3) PRAGMA table_info = ?
1188 */
1189 sqlite3_stmt *aStmt[4] = {0, 0, 0, 0};
1190
1191 *peType = RBU_PK_NOTABLE;
1192 *piPk = 0;
1193
1194 assert( p->rc==SQLITE_OK );
1195 p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg,
1196 sqlite3_mprintf(
1197 "SELECT (sql LIKE 'create virtual%%'), rootpage"
1198 " FROM sqlite_schema"
1199 " WHERE name=%Q", zTab
1200 ));
1201 if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){
1202 /* Either an error, or no such table. */
1203 goto rbuTableType_end;
1204 }
1205 if( sqlite3_column_int(aStmt[0], 0) ){
1206 *peType = RBU_PK_VTAB; /* virtual table */
1207 goto rbuTableType_end;
1208 }
1209 *piTnum = sqlite3_column_int(aStmt[0], 1);
1210
1211 p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg,
1212 sqlite3_mprintf("PRAGMA index_list=%Q",zTab)
1213 );
1214 if( p->rc ) goto rbuTableType_end;
1215 while( sqlite3_step(aStmt[1])==SQLITE_ROW ){
1216 const u8 *zOrig = sqlite3_column_text(aStmt[1], 3);
1217 const u8 *zIdx = sqlite3_column_text(aStmt[1], 1);
1218 if( zOrig && zIdx && zOrig[0]=='p' ){
1219 p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg,
1220 sqlite3_mprintf(
1221 "SELECT rootpage FROM sqlite_schema WHERE name = %Q", zIdx
1222 ));
1223 if( p->rc==SQLITE_OK ){
1224 if( sqlite3_step(aStmt[2])==SQLITE_ROW ){
1225 *piPk = sqlite3_column_int(aStmt[2], 0);
1226 *peType = RBU_PK_EXTERNAL;
1227 }else{
1228 *peType = RBU_PK_WITHOUT_ROWID;
1229 }
1230 }
1231 goto rbuTableType_end;
1232 }
1233 }
1234
1235 p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg,
1236 sqlite3_mprintf("PRAGMA table_info=%Q",zTab)
1237 );
1238 if( p->rc==SQLITE_OK ){
1239 while( sqlite3_step(aStmt[3])==SQLITE_ROW ){
1240 if( sqlite3_column_int(aStmt[3],5)>0 ){
1241 *peType = RBU_PK_IPK; /* explicit IPK column */
1242 goto rbuTableType_end;
1243 }
1244 }
1245 *peType = RBU_PK_NONE;
1246 }
1247
1248 rbuTableType_end: {
1249 unsigned int i;
1250 for(i=0; i<sizeof(aStmt)/sizeof(aStmt[0]); i++){
1251 rbuFinalize(p, aStmt[i]);
1252 }
1253 }
1254 }
1255
1256 /*
1257 ** This is a helper function for rbuObjIterCacheTableInfo(). It populates
1258 ** the pIter->abIndexed[] array.
1259 */
rbuObjIterCacheIndexedCols(sqlite3rbu * p,RbuObjIter * pIter)1260 static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){
1261 sqlite3_stmt *pList = 0;
1262 int bIndex = 0;
1263
1264 if( p->rc==SQLITE_OK ){
1265 memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol);
1266 p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg,
1267 sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl)
1268 );
1269 }
1270
1271 pIter->nIndex = 0;
1272 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){
1273 const char *zIdx = (const char*)sqlite3_column_text(pList, 1);
1274 int bPartial = sqlite3_column_int(pList, 4);
1275 sqlite3_stmt *pXInfo = 0;
1276 if( zIdx==0 ) break;
1277 if( bPartial ){
1278 memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol);
1279 }
1280 p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
1281 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
1282 );
1283 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
1284 int iCid = sqlite3_column_int(pXInfo, 1);
1285 if( iCid>=0 ) pIter->abIndexed[iCid] = 1;
1286 if( iCid==-2 ){
1287 memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol);
1288 }
1289 }
1290 rbuFinalize(p, pXInfo);
1291 bIndex = 1;
1292 pIter->nIndex++;
1293 }
1294
1295 if( pIter->eType==RBU_PK_WITHOUT_ROWID ){
1296 /* "PRAGMA index_list" includes the main PK b-tree */
1297 pIter->nIndex--;
1298 }
1299
1300 rbuFinalize(p, pList);
1301 if( bIndex==0 ) pIter->abIndexed = 0;
1302 }
1303
1304
1305 /*
1306 ** If they are not already populated, populate the pIter->azTblCol[],
1307 ** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to
1308 ** the table (not index) that the iterator currently points to.
1309 **
1310 ** Return SQLITE_OK if successful, or an SQLite error code otherwise. If
1311 ** an error does occur, an error code and error message are also left in
1312 ** the RBU handle.
1313 */
rbuObjIterCacheTableInfo(sqlite3rbu * p,RbuObjIter * pIter)1314 static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){
1315 if( pIter->azTblCol==0 ){
1316 sqlite3_stmt *pStmt = 0;
1317 int nCol = 0;
1318 int i; /* for() loop iterator variable */
1319 int bRbuRowid = 0; /* If input table has column "rbu_rowid" */
1320 int iOrder = 0;
1321 int iTnum = 0;
1322
1323 /* Figure out the type of table this step will deal with. */
1324 assert( pIter->eType==0 );
1325 rbuTableType(p, pIter->zTbl, &pIter->eType, &iTnum, &pIter->iPkTnum);
1326 if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_NOTABLE ){
1327 p->rc = SQLITE_ERROR;
1328 p->zErrmsg = sqlite3_mprintf("no such table: %s", pIter->zTbl);
1329 }
1330 if( p->rc ) return p->rc;
1331 if( pIter->zIdx==0 ) pIter->iTnum = iTnum;
1332
1333 assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK
1334 || pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_WITHOUT_ROWID
1335 || pIter->eType==RBU_PK_VTAB
1336 );
1337
1338 /* Populate the azTblCol[] and nTblCol variables based on the columns
1339 ** of the input table. Ignore any input table columns that begin with
1340 ** "rbu_". */
1341 p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
1342 sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl)
1343 );
1344 if( p->rc==SQLITE_OK ){
1345 nCol = sqlite3_column_count(pStmt);
1346 rbuAllocateIterArrays(p, pIter, nCol);
1347 }
1348 for(i=0; p->rc==SQLITE_OK && i<nCol; i++){
1349 const char *zName = (const char*)sqlite3_column_name(pStmt, i);
1350 if( sqlite3_strnicmp("rbu_", zName, 4) ){
1351 char *zCopy = rbuStrndup(zName, &p->rc);
1352 pIter->aiSrcOrder[pIter->nTblCol] = pIter->nTblCol;
1353 pIter->azTblCol[pIter->nTblCol++] = zCopy;
1354 }
1355 else if( 0==sqlite3_stricmp("rbu_rowid", zName) ){
1356 bRbuRowid = 1;
1357 }
1358 }
1359 sqlite3_finalize(pStmt);
1360 pStmt = 0;
1361
1362 if( p->rc==SQLITE_OK
1363 && rbuIsVacuum(p)==0
1364 && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE)
1365 ){
1366 p->rc = SQLITE_ERROR;
1367 p->zErrmsg = sqlite3_mprintf(
1368 "table %q %s rbu_rowid column", pIter->zDataTbl,
1369 (bRbuRowid ? "may not have" : "requires")
1370 );
1371 }
1372
1373 /* Check that all non-HIDDEN columns in the destination table are also
1374 ** present in the input table. Populate the abTblPk[], azTblType[] and
1375 ** aiTblOrder[] arrays at the same time. */
1376 if( p->rc==SQLITE_OK ){
1377 p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg,
1378 sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl)
1379 );
1380 }
1381 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
1382 const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
1383 if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */
1384 for(i=iOrder; i<pIter->nTblCol; i++){
1385 if( 0==strcmp(zName, pIter->azTblCol[i]) ) break;
1386 }
1387 if( i==pIter->nTblCol ){
1388 p->rc = SQLITE_ERROR;
1389 p->zErrmsg = sqlite3_mprintf("column missing from %q: %s",
1390 pIter->zDataTbl, zName
1391 );
1392 }else{
1393 int iPk = sqlite3_column_int(pStmt, 5);
1394 int bNotNull = sqlite3_column_int(pStmt, 3);
1395 const char *zType = (const char*)sqlite3_column_text(pStmt, 2);
1396
1397 if( i!=iOrder ){
1398 SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]);
1399 SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]);
1400 }
1401
1402 pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc);
1403 assert( iPk>=0 );
1404 pIter->abTblPk[iOrder] = (u8)iPk;
1405 pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0);
1406 iOrder++;
1407 }
1408 }
1409
1410 rbuFinalize(p, pStmt);
1411 rbuObjIterCacheIndexedCols(p, pIter);
1412 assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 );
1413 assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 );
1414 }
1415
1416 return p->rc;
1417 }
1418
1419 /*
1420 ** This function constructs and returns a pointer to a nul-terminated
1421 ** string containing some SQL clause or list based on one or more of the
1422 ** column names currently stored in the pIter->azTblCol[] array.
1423 */
rbuObjIterGetCollist(sqlite3rbu * p,RbuObjIter * pIter)1424 static char *rbuObjIterGetCollist(
1425 sqlite3rbu *p, /* RBU object */
1426 RbuObjIter *pIter /* Object iterator for column names */
1427 ){
1428 char *zList = 0;
1429 const char *zSep = "";
1430 int i;
1431 for(i=0; i<pIter->nTblCol; i++){
1432 const char *z = pIter->azTblCol[i];
1433 zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z);
1434 zSep = ", ";
1435 }
1436 return zList;
1437 }
1438
1439 /*
1440 ** Return a comma separated list of the quoted PRIMARY KEY column names,
1441 ** in order, for the current table. Before each column name, add the text
1442 ** zPre. After each column name, add the zPost text. Use zSeparator as
1443 ** the separator text (usually ", ").
1444 */
rbuObjIterGetPkList(sqlite3rbu * p,RbuObjIter * pIter,const char * zPre,const char * zSeparator,const char * zPost)1445 static char *rbuObjIterGetPkList(
1446 sqlite3rbu *p, /* RBU object */
1447 RbuObjIter *pIter, /* Object iterator for column names */
1448 const char *zPre, /* Before each quoted column name */
1449 const char *zSeparator, /* Separator to use between columns */
1450 const char *zPost /* After each quoted column name */
1451 ){
1452 int iPk = 1;
1453 char *zRet = 0;
1454 const char *zSep = "";
1455 while( 1 ){
1456 int i;
1457 for(i=0; i<pIter->nTblCol; i++){
1458 if( (int)pIter->abTblPk[i]==iPk ){
1459 const char *zCol = pIter->azTblCol[i];
1460 zRet = rbuMPrintf(p, "%z%s%s\"%w\"%s", zRet, zSep, zPre, zCol, zPost);
1461 zSep = zSeparator;
1462 break;
1463 }
1464 }
1465 if( i==pIter->nTblCol ) break;
1466 iPk++;
1467 }
1468 return zRet;
1469 }
1470
1471 /*
1472 ** This function is called as part of restarting an RBU vacuum within
1473 ** stage 1 of the process (while the *-oal file is being built) while
1474 ** updating a table (not an index). The table may be a rowid table or
1475 ** a WITHOUT ROWID table. It queries the target database to find the
1476 ** largest key that has already been written to the target table and
1477 ** constructs a WHERE clause that can be used to extract the remaining
1478 ** rows from the source table. For a rowid table, the WHERE clause
1479 ** is of the form:
1480 **
1481 ** "WHERE _rowid_ > ?"
1482 **
1483 ** and for WITHOUT ROWID tables:
1484 **
1485 ** "WHERE (key1, key2) > (?, ?)"
1486 **
1487 ** Instead of "?" placeholders, the actual WHERE clauses created by
1488 ** this function contain literal SQL values.
1489 */
rbuVacuumTableStart(sqlite3rbu * p,RbuObjIter * pIter,int bRowid,const char * zWrite)1490 static char *rbuVacuumTableStart(
1491 sqlite3rbu *p, /* RBU handle */
1492 RbuObjIter *pIter, /* RBU iterator object */
1493 int bRowid, /* True for a rowid table */
1494 const char *zWrite /* Target table name prefix */
1495 ){
1496 sqlite3_stmt *pMax = 0;
1497 char *zRet = 0;
1498 if( bRowid ){
1499 p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg,
1500 sqlite3_mprintf(
1501 "SELECT max(_rowid_) FROM \"%s%w\"", zWrite, pIter->zTbl
1502 )
1503 );
1504 if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
1505 sqlite3_int64 iMax = sqlite3_column_int64(pMax, 0);
1506 zRet = rbuMPrintf(p, " WHERE _rowid_ > %lld ", iMax);
1507 }
1508 rbuFinalize(p, pMax);
1509 }else{
1510 char *zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", " DESC");
1511 char *zSelect = rbuObjIterGetPkList(p, pIter, "quote(", "||','||", ")");
1512 char *zList = rbuObjIterGetPkList(p, pIter, "", ", ", "");
1513
1514 if( p->rc==SQLITE_OK ){
1515 p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg,
1516 sqlite3_mprintf(
1517 "SELECT %s FROM \"%s%w\" ORDER BY %s LIMIT 1",
1518 zSelect, zWrite, pIter->zTbl, zOrder
1519 )
1520 );
1521 if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
1522 const char *zVal = (const char*)sqlite3_column_text(pMax, 0);
1523 zRet = rbuMPrintf(p, " WHERE (%s) > (%s) ", zList, zVal);
1524 }
1525 rbuFinalize(p, pMax);
1526 }
1527
1528 sqlite3_free(zOrder);
1529 sqlite3_free(zSelect);
1530 sqlite3_free(zList);
1531 }
1532 return zRet;
1533 }
1534
1535 /*
1536 ** This function is called as part of restating an RBU vacuum when the
1537 ** current operation is writing content to an index. If possible, it
1538 ** queries the target index b-tree for the largest key already written to
1539 ** it, then composes and returns an expression that can be used in a WHERE
1540 ** clause to select the remaining required rows from the source table.
1541 ** It is only possible to return such an expression if:
1542 **
1543 ** * The index contains no DESC columns, and
1544 ** * The last key written to the index before the operation was
1545 ** suspended does not contain any NULL values.
1546 **
1547 ** The expression is of the form:
1548 **
1549 ** (index-field1, index-field2, ...) > (?, ?, ...)
1550 **
1551 ** except that the "?" placeholders are replaced with literal values.
1552 **
1553 ** If the expression cannot be created, NULL is returned. In this case,
1554 ** the caller has to use an OFFSET clause to extract only the required
1555 ** rows from the sourct table, just as it does for an RBU update operation.
1556 */
rbuVacuumIndexStart(sqlite3rbu * p,RbuObjIter * pIter)1557 char *rbuVacuumIndexStart(
1558 sqlite3rbu *p, /* RBU handle */
1559 RbuObjIter *pIter /* RBU iterator object */
1560 ){
1561 char *zOrder = 0;
1562 char *zLhs = 0;
1563 char *zSelect = 0;
1564 char *zVector = 0;
1565 char *zRet = 0;
1566 int bFailed = 0;
1567 const char *zSep = "";
1568 int iCol = 0;
1569 sqlite3_stmt *pXInfo = 0;
1570
1571 p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
1572 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx)
1573 );
1574 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
1575 int iCid = sqlite3_column_int(pXInfo, 1);
1576 const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
1577 const char *zCol;
1578 if( sqlite3_column_int(pXInfo, 3) ){
1579 bFailed = 1;
1580 break;
1581 }
1582
1583 if( iCid<0 ){
1584 if( pIter->eType==RBU_PK_IPK ){
1585 int i;
1586 for(i=0; pIter->abTblPk[i]==0; i++);
1587 assert( i<pIter->nTblCol );
1588 zCol = pIter->azTblCol[i];
1589 }else{
1590 zCol = "_rowid_";
1591 }
1592 }else{
1593 zCol = pIter->azTblCol[iCid];
1594 }
1595
1596 zLhs = rbuMPrintf(p, "%z%s \"%w\" COLLATE %Q",
1597 zLhs, zSep, zCol, zCollate
1598 );
1599 zOrder = rbuMPrintf(p, "%z%s \"rbu_imp_%d%w\" COLLATE %Q DESC",
1600 zOrder, zSep, iCol, zCol, zCollate
1601 );
1602 zSelect = rbuMPrintf(p, "%z%s quote(\"rbu_imp_%d%w\")",
1603 zSelect, zSep, iCol, zCol
1604 );
1605 zSep = ", ";
1606 iCol++;
1607 }
1608 rbuFinalize(p, pXInfo);
1609 if( bFailed ) goto index_start_out;
1610
1611 if( p->rc==SQLITE_OK ){
1612 sqlite3_stmt *pSel = 0;
1613
1614 p->rc = prepareFreeAndCollectError(p->dbMain, &pSel, &p->zErrmsg,
1615 sqlite3_mprintf("SELECT %s FROM \"rbu_imp_%w\" ORDER BY %s LIMIT 1",
1616 zSelect, pIter->zTbl, zOrder
1617 )
1618 );
1619 if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSel) ){
1620 zSep = "";
1621 for(iCol=0; iCol<pIter->nCol; iCol++){
1622 const char *zQuoted = (const char*)sqlite3_column_text(pSel, iCol);
1623 if( zQuoted[0]=='N' ){
1624 bFailed = 1;
1625 break;
1626 }
1627 zVector = rbuMPrintf(p, "%z%s%s", zVector, zSep, zQuoted);
1628 zSep = ", ";
1629 }
1630
1631 if( !bFailed ){
1632 zRet = rbuMPrintf(p, "(%s) > (%s)", zLhs, zVector);
1633 }
1634 }
1635 rbuFinalize(p, pSel);
1636 }
1637
1638 index_start_out:
1639 sqlite3_free(zOrder);
1640 sqlite3_free(zSelect);
1641 sqlite3_free(zVector);
1642 sqlite3_free(zLhs);
1643 return zRet;
1644 }
1645
1646 /*
1647 ** This function is used to create a SELECT list (the list of SQL
1648 ** expressions that follows a SELECT keyword) for a SELECT statement
1649 ** used to read from an data_xxx or rbu_tmp_xxx table while updating the
1650 ** index object currently indicated by the iterator object passed as the
1651 ** second argument. A "PRAGMA index_xinfo = <idxname>" statement is used
1652 ** to obtain the required information.
1653 **
1654 ** If the index is of the following form:
1655 **
1656 ** CREATE INDEX i1 ON t1(c, b COLLATE nocase);
1657 **
1658 ** and "t1" is a table with an explicit INTEGER PRIMARY KEY column
1659 ** "ipk", the returned string is:
1660 **
1661 ** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'"
1662 **
1663 ** As well as the returned string, three other malloc'd strings are
1664 ** returned via output parameters. As follows:
1665 **
1666 ** pzImposterCols: ...
1667 ** pzImposterPk: ...
1668 ** pzWhere: ...
1669 */
rbuObjIterGetIndexCols(sqlite3rbu * p,RbuObjIter * pIter,char ** pzImposterCols,char ** pzImposterPk,char ** pzWhere,int * pnBind)1670 static char *rbuObjIterGetIndexCols(
1671 sqlite3rbu *p, /* RBU object */
1672 RbuObjIter *pIter, /* Object iterator for column names */
1673 char **pzImposterCols, /* OUT: Columns for imposter table */
1674 char **pzImposterPk, /* OUT: Imposter PK clause */
1675 char **pzWhere, /* OUT: WHERE clause */
1676 int *pnBind /* OUT: Trbul number of columns */
1677 ){
1678 int rc = p->rc; /* Error code */
1679 int rc2; /* sqlite3_finalize() return code */
1680 char *zRet = 0; /* String to return */
1681 char *zImpCols = 0; /* String to return via *pzImposterCols */
1682 char *zImpPK = 0; /* String to return via *pzImposterPK */
1683 char *zWhere = 0; /* String to return via *pzWhere */
1684 int nBind = 0; /* Value to return via *pnBind */
1685 const char *zCom = ""; /* Set to ", " later on */
1686 const char *zAnd = ""; /* Set to " AND " later on */
1687 sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */
1688
1689 if( rc==SQLITE_OK ){
1690 assert( p->zErrmsg==0 );
1691 rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
1692 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx)
1693 );
1694 }
1695
1696 while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
1697 int iCid = sqlite3_column_int(pXInfo, 1);
1698 int bDesc = sqlite3_column_int(pXInfo, 3);
1699 const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
1700 const char *zCol = 0;
1701 const char *zType;
1702
1703 if( iCid==-2 ){
1704 int iSeq = sqlite3_column_int(pXInfo, 0);
1705 zRet = sqlite3_mprintf("%z%s(%.*s) COLLATE %Q", zRet, zCom,
1706 pIter->aIdxCol[iSeq].nSpan, pIter->aIdxCol[iSeq].zSpan, zCollate
1707 );
1708 zType = "";
1709 }else {
1710 if( iCid<0 ){
1711 /* An integer primary key. If the table has an explicit IPK, use
1712 ** its name. Otherwise, use "rbu_rowid". */
1713 if( pIter->eType==RBU_PK_IPK ){
1714 int i;
1715 for(i=0; pIter->abTblPk[i]==0; i++);
1716 assert( i<pIter->nTblCol );
1717 zCol = pIter->azTblCol[i];
1718 }else if( rbuIsVacuum(p) ){
1719 zCol = "_rowid_";
1720 }else{
1721 zCol = "rbu_rowid";
1722 }
1723 zType = "INTEGER";
1724 }else{
1725 zCol = pIter->azTblCol[iCid];
1726 zType = pIter->azTblType[iCid];
1727 }
1728 zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom,zCol,zCollate);
1729 }
1730
1731 if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){
1732 const char *zOrder = (bDesc ? " DESC" : "");
1733 zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s",
1734 zImpPK, zCom, nBind, zCol, zOrder
1735 );
1736 }
1737 zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q",
1738 zImpCols, zCom, nBind, zCol, zType, zCollate
1739 );
1740 zWhere = sqlite3_mprintf(
1741 "%z%s\"rbu_imp_%d%w\" IS ?", zWhere, zAnd, nBind, zCol
1742 );
1743 if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM;
1744 zCom = ", ";
1745 zAnd = " AND ";
1746 nBind++;
1747 }
1748
1749 rc2 = sqlite3_finalize(pXInfo);
1750 if( rc==SQLITE_OK ) rc = rc2;
1751
1752 if( rc!=SQLITE_OK ){
1753 sqlite3_free(zRet);
1754 sqlite3_free(zImpCols);
1755 sqlite3_free(zImpPK);
1756 sqlite3_free(zWhere);
1757 zRet = 0;
1758 zImpCols = 0;
1759 zImpPK = 0;
1760 zWhere = 0;
1761 p->rc = rc;
1762 }
1763
1764 *pzImposterCols = zImpCols;
1765 *pzImposterPk = zImpPK;
1766 *pzWhere = zWhere;
1767 *pnBind = nBind;
1768 return zRet;
1769 }
1770
1771 /*
1772 ** Assuming the current table columns are "a", "b" and "c", and the zObj
1773 ** paramter is passed "old", return a string of the form:
1774 **
1775 ** "old.a, old.b, old.b"
1776 **
1777 ** With the column names escaped.
1778 **
1779 ** For tables with implicit rowids - RBU_PK_EXTERNAL and RBU_PK_NONE, append
1780 ** the text ", old._rowid_" to the returned value.
1781 */
rbuObjIterGetOldlist(sqlite3rbu * p,RbuObjIter * pIter,const char * zObj)1782 static char *rbuObjIterGetOldlist(
1783 sqlite3rbu *p,
1784 RbuObjIter *pIter,
1785 const char *zObj
1786 ){
1787 char *zList = 0;
1788 if( p->rc==SQLITE_OK && pIter->abIndexed ){
1789 const char *zS = "";
1790 int i;
1791 for(i=0; i<pIter->nTblCol; i++){
1792 if( pIter->abIndexed[i] ){
1793 const char *zCol = pIter->azTblCol[i];
1794 zList = sqlite3_mprintf("%z%s%s.\"%w\"", zList, zS, zObj, zCol);
1795 }else{
1796 zList = sqlite3_mprintf("%z%sNULL", zList, zS);
1797 }
1798 zS = ", ";
1799 if( zList==0 ){
1800 p->rc = SQLITE_NOMEM;
1801 break;
1802 }
1803 }
1804
1805 /* For a table with implicit rowids, append "old._rowid_" to the list. */
1806 if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
1807 zList = rbuMPrintf(p, "%z, %s._rowid_", zList, zObj);
1808 }
1809 }
1810 return zList;
1811 }
1812
1813 /*
1814 ** Return an expression that can be used in a WHERE clause to match the
1815 ** primary key of the current table. For example, if the table is:
1816 **
1817 ** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c));
1818 **
1819 ** Return the string:
1820 **
1821 ** "b = ?1 AND c = ?2"
1822 */
rbuObjIterGetWhere(sqlite3rbu * p,RbuObjIter * pIter)1823 static char *rbuObjIterGetWhere(
1824 sqlite3rbu *p,
1825 RbuObjIter *pIter
1826 ){
1827 char *zList = 0;
1828 if( pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE ){
1829 zList = rbuMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1);
1830 }else if( pIter->eType==RBU_PK_EXTERNAL ){
1831 const char *zSep = "";
1832 int i;
1833 for(i=0; i<pIter->nTblCol; i++){
1834 if( pIter->abTblPk[i] ){
1835 zList = rbuMPrintf(p, "%z%sc%d=?%d", zList, zSep, i, i+1);
1836 zSep = " AND ";
1837 }
1838 }
1839 zList = rbuMPrintf(p,
1840 "_rowid_ = (SELECT id FROM rbu_imposter2 WHERE %z)", zList
1841 );
1842
1843 }else{
1844 const char *zSep = "";
1845 int i;
1846 for(i=0; i<pIter->nTblCol; i++){
1847 if( pIter->abTblPk[i] ){
1848 const char *zCol = pIter->azTblCol[i];
1849 zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1);
1850 zSep = " AND ";
1851 }
1852 }
1853 }
1854 return zList;
1855 }
1856
1857 /*
1858 ** The SELECT statement iterating through the keys for the current object
1859 ** (p->objiter.pSelect) currently points to a valid row. However, there
1860 ** is something wrong with the rbu_control value in the rbu_control value
1861 ** stored in the (p->nCol+1)'th column. Set the error code and error message
1862 ** of the RBU handle to something reflecting this.
1863 */
rbuBadControlError(sqlite3rbu * p)1864 static void rbuBadControlError(sqlite3rbu *p){
1865 p->rc = SQLITE_ERROR;
1866 p->zErrmsg = sqlite3_mprintf("invalid rbu_control value");
1867 }
1868
1869
1870 /*
1871 ** Return a nul-terminated string containing the comma separated list of
1872 ** assignments that should be included following the "SET" keyword of
1873 ** an UPDATE statement used to update the table object that the iterator
1874 ** passed as the second argument currently points to if the rbu_control
1875 ** column of the data_xxx table entry is set to zMask.
1876 **
1877 ** The memory for the returned string is obtained from sqlite3_malloc().
1878 ** It is the responsibility of the caller to eventually free it using
1879 ** sqlite3_free().
1880 **
1881 ** If an OOM error is encountered when allocating space for the new
1882 ** string, an error code is left in the rbu handle passed as the first
1883 ** argument and NULL is returned. Or, if an error has already occurred
1884 ** when this function is called, NULL is returned immediately, without
1885 ** attempting the allocation or modifying the stored error code.
1886 */
rbuObjIterGetSetlist(sqlite3rbu * p,RbuObjIter * pIter,const char * zMask)1887 static char *rbuObjIterGetSetlist(
1888 sqlite3rbu *p,
1889 RbuObjIter *pIter,
1890 const char *zMask
1891 ){
1892 char *zList = 0;
1893 if( p->rc==SQLITE_OK ){
1894 int i;
1895
1896 if( (int)strlen(zMask)!=pIter->nTblCol ){
1897 rbuBadControlError(p);
1898 }else{
1899 const char *zSep = "";
1900 for(i=0; i<pIter->nTblCol; i++){
1901 char c = zMask[pIter->aiSrcOrder[i]];
1902 if( c=='x' ){
1903 zList = rbuMPrintf(p, "%z%s\"%w\"=?%d",
1904 zList, zSep, pIter->azTblCol[i], i+1
1905 );
1906 zSep = ", ";
1907 }
1908 else if( c=='d' ){
1909 zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)",
1910 zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1
1911 );
1912 zSep = ", ";
1913 }
1914 else if( c=='f' ){
1915 zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)",
1916 zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1
1917 );
1918 zSep = ", ";
1919 }
1920 }
1921 }
1922 }
1923 return zList;
1924 }
1925
1926 /*
1927 ** Return a nul-terminated string consisting of nByte comma separated
1928 ** "?" expressions. For example, if nByte is 3, return a pointer to
1929 ** a buffer containing the string "?,?,?".
1930 **
1931 ** The memory for the returned string is obtained from sqlite3_malloc().
1932 ** It is the responsibility of the caller to eventually free it using
1933 ** sqlite3_free().
1934 **
1935 ** If an OOM error is encountered when allocating space for the new
1936 ** string, an error code is left in the rbu handle passed as the first
1937 ** argument and NULL is returned. Or, if an error has already occurred
1938 ** when this function is called, NULL is returned immediately, without
1939 ** attempting the allocation or modifying the stored error code.
1940 */
rbuObjIterGetBindlist(sqlite3rbu * p,int nBind)1941 static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){
1942 char *zRet = 0;
1943 sqlite3_int64 nByte = 2*(sqlite3_int64)nBind + 1;
1944
1945 zRet = (char*)rbuMalloc(p, nByte);
1946 if( zRet ){
1947 int i;
1948 for(i=0; i<nBind; i++){
1949 zRet[i*2] = '?';
1950 zRet[i*2+1] = (i+1==nBind) ? '\0' : ',';
1951 }
1952 }
1953 return zRet;
1954 }
1955
1956 /*
1957 ** The iterator currently points to a table (not index) of type
1958 ** RBU_PK_WITHOUT_ROWID. This function creates the PRIMARY KEY
1959 ** declaration for the corresponding imposter table. For example,
1960 ** if the iterator points to a table created as:
1961 **
1962 ** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, a DESC)) WITHOUT ROWID
1963 **
1964 ** this function returns:
1965 **
1966 ** PRIMARY KEY("b", "a" DESC)
1967 */
rbuWithoutRowidPK(sqlite3rbu * p,RbuObjIter * pIter)1968 static char *rbuWithoutRowidPK(sqlite3rbu *p, RbuObjIter *pIter){
1969 char *z = 0;
1970 assert( pIter->zIdx==0 );
1971 if( p->rc==SQLITE_OK ){
1972 const char *zSep = "PRIMARY KEY(";
1973 sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */
1974 sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = <pk-index> */
1975
1976 p->rc = prepareFreeAndCollectError(p->dbMain, &pXList, &p->zErrmsg,
1977 sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl)
1978 );
1979 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXList) ){
1980 const char *zOrig = (const char*)sqlite3_column_text(pXList,3);
1981 if( zOrig && strcmp(zOrig, "pk")==0 ){
1982 const char *zIdx = (const char*)sqlite3_column_text(pXList,1);
1983 if( zIdx ){
1984 p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
1985 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
1986 );
1987 }
1988 break;
1989 }
1990 }
1991 rbuFinalize(p, pXList);
1992
1993 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
1994 if( sqlite3_column_int(pXInfo, 5) ){
1995 /* int iCid = sqlite3_column_int(pXInfo, 0); */
1996 const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2);
1997 const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : "";
1998 z = rbuMPrintf(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc);
1999 zSep = ", ";
2000 }
2001 }
2002 z = rbuMPrintf(p, "%z)", z);
2003 rbuFinalize(p, pXInfo);
2004 }
2005 return z;
2006 }
2007
2008 /*
2009 ** This function creates the second imposter table used when writing to
2010 ** a table b-tree where the table has an external primary key. If the
2011 ** iterator passed as the second argument does not currently point to
2012 ** a table (not index) with an external primary key, this function is a
2013 ** no-op.
2014 **
2015 ** Assuming the iterator does point to a table with an external PK, this
2016 ** function creates a WITHOUT ROWID imposter table named "rbu_imposter2"
2017 ** used to access that PK index. For example, if the target table is
2018 ** declared as follows:
2019 **
2020 ** CREATE TABLE t1(a, b TEXT, c REAL, PRIMARY KEY(b, c));
2021 **
2022 ** then the imposter table schema is:
2023 **
2024 ** CREATE TABLE rbu_imposter2(c1 TEXT, c2 REAL, id INTEGER) WITHOUT ROWID;
2025 **
2026 */
rbuCreateImposterTable2(sqlite3rbu * p,RbuObjIter * pIter)2027 static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){
2028 if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_EXTERNAL ){
2029 int tnum = pIter->iPkTnum; /* Root page of PK index */
2030 sqlite3_stmt *pQuery = 0; /* SELECT name ... WHERE rootpage = $tnum */
2031 const char *zIdx = 0; /* Name of PK index */
2032 sqlite3_stmt *pXInfo = 0; /* PRAGMA main.index_xinfo = $zIdx */
2033 const char *zComma = "";
2034 char *zCols = 0; /* Used to build up list of table cols */
2035 char *zPk = 0; /* Used to build up table PK declaration */
2036
2037 /* Figure out the name of the primary key index for the current table.
2038 ** This is needed for the argument to "PRAGMA index_xinfo". Set
2039 ** zIdx to point to a nul-terminated string containing this name. */
2040 p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg,
2041 "SELECT name FROM sqlite_schema WHERE rootpage = ?"
2042 );
2043 if( p->rc==SQLITE_OK ){
2044 sqlite3_bind_int(pQuery, 1, tnum);
2045 if( SQLITE_ROW==sqlite3_step(pQuery) ){
2046 zIdx = (const char*)sqlite3_column_text(pQuery, 0);
2047 }
2048 }
2049 if( zIdx ){
2050 p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
2051 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
2052 );
2053 }
2054 rbuFinalize(p, pQuery);
2055
2056 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
2057 int bKey = sqlite3_column_int(pXInfo, 5);
2058 if( bKey ){
2059 int iCid = sqlite3_column_int(pXInfo, 1);
2060 int bDesc = sqlite3_column_int(pXInfo, 3);
2061 const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
2062 zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %Q", zCols, zComma,
2063 iCid, pIter->azTblType[iCid], zCollate
2064 );
2065 zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":"");
2066 zComma = ", ";
2067 }
2068 }
2069 zCols = rbuMPrintf(p, "%z, id INTEGER", zCols);
2070 rbuFinalize(p, pXInfo);
2071
2072 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum);
2073 rbuMPrintfExec(p, p->dbMain,
2074 "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID",
2075 zCols, zPk
2076 );
2077 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0);
2078 }
2079 }
2080
2081 /*
2082 ** If an error has already occurred when this function is called, it
2083 ** immediately returns zero (without doing any work). Or, if an error
2084 ** occurs during the execution of this function, it sets the error code
2085 ** in the sqlite3rbu object indicated by the first argument and returns
2086 ** zero.
2087 **
2088 ** The iterator passed as the second argument is guaranteed to point to
2089 ** a table (not an index) when this function is called. This function
2090 ** attempts to create any imposter table required to write to the main
2091 ** table b-tree of the table before returning. Non-zero is returned if
2092 ** an imposter table are created, or zero otherwise.
2093 **
2094 ** An imposter table is required in all cases except RBU_PK_VTAB. Only
2095 ** virtual tables are written to directly. The imposter table has the
2096 ** same schema as the actual target table (less any UNIQUE constraints).
2097 ** More precisely, the "same schema" means the same columns, types,
2098 ** collation sequences. For tables that do not have an external PRIMARY
2099 ** KEY, it also means the same PRIMARY KEY declaration.
2100 */
rbuCreateImposterTable(sqlite3rbu * p,RbuObjIter * pIter)2101 static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){
2102 if( p->rc==SQLITE_OK && pIter->eType!=RBU_PK_VTAB ){
2103 int tnum = pIter->iTnum;
2104 const char *zComma = "";
2105 char *zSql = 0;
2106 int iCol;
2107 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1);
2108
2109 for(iCol=0; p->rc==SQLITE_OK && iCol<pIter->nTblCol; iCol++){
2110 const char *zPk = "";
2111 const char *zCol = pIter->azTblCol[iCol];
2112 const char *zColl = 0;
2113
2114 p->rc = sqlite3_table_column_metadata(
2115 p->dbMain, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0
2116 );
2117
2118 if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){
2119 /* If the target table column is an "INTEGER PRIMARY KEY", add
2120 ** "PRIMARY KEY" to the imposter table column declaration. */
2121 zPk = "PRIMARY KEY ";
2122 }
2123 zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %Q%s",
2124 zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl,
2125 (pIter->abNotNull[iCol] ? " NOT NULL" : "")
2126 );
2127 zComma = ", ";
2128 }
2129
2130 if( pIter->eType==RBU_PK_WITHOUT_ROWID ){
2131 char *zPk = rbuWithoutRowidPK(p, pIter);
2132 if( zPk ){
2133 zSql = rbuMPrintf(p, "%z, %z", zSql, zPk);
2134 }
2135 }
2136
2137 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum);
2138 rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s",
2139 pIter->zTbl, zSql,
2140 (pIter->eType==RBU_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "")
2141 );
2142 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0);
2143 }
2144 }
2145
2146 /*
2147 ** Prepare a statement used to insert rows into the "rbu_tmp_xxx" table.
2148 ** Specifically a statement of the form:
2149 **
2150 ** INSERT INTO rbu_tmp_xxx VALUES(?, ?, ? ...);
2151 **
2152 ** The number of bound variables is equal to the number of columns in
2153 ** the target table, plus one (for the rbu_control column), plus one more
2154 ** (for the rbu_rowid column) if the target table is an implicit IPK or
2155 ** virtual table.
2156 */
rbuObjIterPrepareTmpInsert(sqlite3rbu * p,RbuObjIter * pIter,const char * zCollist,const char * zRbuRowid)2157 static void rbuObjIterPrepareTmpInsert(
2158 sqlite3rbu *p,
2159 RbuObjIter *pIter,
2160 const char *zCollist,
2161 const char *zRbuRowid
2162 ){
2163 int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE);
2164 char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid);
2165 if( zBind ){
2166 assert( pIter->pTmpInsert==0 );
2167 p->rc = prepareFreeAndCollectError(
2168 p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf(
2169 "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)",
2170 p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind
2171 ));
2172 }
2173 }
2174
rbuTmpInsertFunc(sqlite3_context * pCtx,int nVal,sqlite3_value ** apVal)2175 static void rbuTmpInsertFunc(
2176 sqlite3_context *pCtx,
2177 int nVal,
2178 sqlite3_value **apVal
2179 ){
2180 sqlite3rbu *p = sqlite3_user_data(pCtx);
2181 int rc = SQLITE_OK;
2182 int i;
2183
2184 assert( sqlite3_value_int(apVal[0])!=0
2185 || p->objiter.eType==RBU_PK_EXTERNAL
2186 || p->objiter.eType==RBU_PK_NONE
2187 );
2188 if( sqlite3_value_int(apVal[0])!=0 ){
2189 p->nPhaseOneStep += p->objiter.nIndex;
2190 }
2191
2192 for(i=0; rc==SQLITE_OK && i<nVal; i++){
2193 rc = sqlite3_bind_value(p->objiter.pTmpInsert, i+1, apVal[i]);
2194 }
2195 if( rc==SQLITE_OK ){
2196 sqlite3_step(p->objiter.pTmpInsert);
2197 rc = sqlite3_reset(p->objiter.pTmpInsert);
2198 }
2199
2200 if( rc!=SQLITE_OK ){
2201 sqlite3_result_error_code(pCtx, rc);
2202 }
2203 }
2204
rbuObjIterGetIndexWhere(sqlite3rbu * p,RbuObjIter * pIter)2205 static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){
2206 sqlite3_stmt *pStmt = 0;
2207 int rc = p->rc;
2208 char *zRet = 0;
2209
2210 assert( pIter->zIdxSql==0 && pIter->nIdxCol==0 && pIter->aIdxCol==0 );
2211
2212 if( rc==SQLITE_OK ){
2213 rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg,
2214 "SELECT trim(sql) FROM sqlite_schema WHERE type='index' AND name=?"
2215 );
2216 }
2217 if( rc==SQLITE_OK ){
2218 int rc2;
2219 rc = sqlite3_bind_text(pStmt, 1, pIter->zIdx, -1, SQLITE_STATIC);
2220 if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
2221 char *zSql = (char*)sqlite3_column_text(pStmt, 0);
2222 if( zSql ){
2223 pIter->zIdxSql = zSql = rbuStrndup(zSql, &rc);
2224 }
2225 if( zSql ){
2226 int nParen = 0; /* Number of open parenthesis */
2227 int i;
2228 int iIdxCol = 0;
2229 int nIdxAlloc = 0;
2230 for(i=0; zSql[i]; i++){
2231 char c = zSql[i];
2232
2233 /* If necessary, grow the pIter->aIdxCol[] array */
2234 if( iIdxCol==nIdxAlloc ){
2235 RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc(
2236 pIter->aIdxCol, (nIdxAlloc+16)*sizeof(RbuSpan)
2237 );
2238 if( aIdxCol==0 ){
2239 rc = SQLITE_NOMEM;
2240 break;
2241 }
2242 pIter->aIdxCol = aIdxCol;
2243 nIdxAlloc += 16;
2244 }
2245
2246 if( c=='(' ){
2247 if( nParen==0 ){
2248 assert( iIdxCol==0 );
2249 pIter->aIdxCol[0].zSpan = &zSql[i+1];
2250 }
2251 nParen++;
2252 }
2253 else if( c==')' ){
2254 nParen--;
2255 if( nParen==0 ){
2256 int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan;
2257 pIter->aIdxCol[iIdxCol++].nSpan = nSpan;
2258 i++;
2259 break;
2260 }
2261 }else if( c==',' && nParen==1 ){
2262 int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan;
2263 pIter->aIdxCol[iIdxCol++].nSpan = nSpan;
2264 pIter->aIdxCol[iIdxCol].zSpan = &zSql[i+1];
2265 }else if( c=='"' || c=='\'' || c=='`' ){
2266 for(i++; 1; i++){
2267 if( zSql[i]==c ){
2268 if( zSql[i+1]!=c ) break;
2269 i++;
2270 }
2271 }
2272 }else if( c=='[' ){
2273 for(i++; 1; i++){
2274 if( zSql[i]==']' ) break;
2275 }
2276 }else if( c=='-' && zSql[i+1]=='-' ){
2277 for(i=i+2; zSql[i] && zSql[i]!='\n'; i++);
2278 if( zSql[i]=='\0' ) break;
2279 }else if( c=='/' && zSql[i+1]=='*' ){
2280 for(i=i+2; zSql[i] && (zSql[i]!='*' || zSql[i+1]!='/'); i++);
2281 if( zSql[i]=='\0' ) break;
2282 i++;
2283 }
2284 }
2285 if( zSql[i] ){
2286 zRet = rbuStrndup(&zSql[i], &rc);
2287 }
2288 pIter->nIdxCol = iIdxCol;
2289 }
2290 }
2291
2292 rc2 = sqlite3_finalize(pStmt);
2293 if( rc==SQLITE_OK ) rc = rc2;
2294 }
2295
2296 p->rc = rc;
2297 return zRet;
2298 }
2299
2300 /*
2301 ** Ensure that the SQLite statement handles required to update the
2302 ** target database object currently indicated by the iterator passed
2303 ** as the second argument are available.
2304 */
rbuObjIterPrepareAll(sqlite3rbu * p,RbuObjIter * pIter,int nOffset)2305 static int rbuObjIterPrepareAll(
2306 sqlite3rbu *p,
2307 RbuObjIter *pIter,
2308 int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */
2309 ){
2310 assert( pIter->bCleanup==0 );
2311 if( pIter->pSelect==0 && rbuObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){
2312 const int tnum = pIter->iTnum;
2313 char *zCollist = 0; /* List of indexed columns */
2314 char **pz = &p->zErrmsg;
2315 const char *zIdx = pIter->zIdx;
2316 char *zLimit = 0;
2317
2318 if( nOffset ){
2319 zLimit = sqlite3_mprintf(" LIMIT -1 OFFSET %d", nOffset);
2320 if( !zLimit ) p->rc = SQLITE_NOMEM;
2321 }
2322
2323 if( zIdx ){
2324 const char *zTbl = pIter->zTbl;
2325 char *zImposterCols = 0; /* Columns for imposter table */
2326 char *zImposterPK = 0; /* Primary key declaration for imposter */
2327 char *zWhere = 0; /* WHERE clause on PK columns */
2328 char *zBind = 0;
2329 char *zPart = 0;
2330 int nBind = 0;
2331
2332 assert( pIter->eType!=RBU_PK_VTAB );
2333 zPart = rbuObjIterGetIndexWhere(p, pIter);
2334 zCollist = rbuObjIterGetIndexCols(
2335 p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind
2336 );
2337 zBind = rbuObjIterGetBindlist(p, nBind);
2338
2339 /* Create the imposter table used to write to this index. */
2340 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1);
2341 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1,tnum);
2342 rbuMPrintfExec(p, p->dbMain,
2343 "CREATE TABLE \"rbu_imp_%w\"( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID",
2344 zTbl, zImposterCols, zImposterPK
2345 );
2346 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0);
2347
2348 /* Create the statement to insert index entries */
2349 pIter->nCol = nBind;
2350 if( p->rc==SQLITE_OK ){
2351 p->rc = prepareFreeAndCollectError(
2352 p->dbMain, &pIter->pInsert, &p->zErrmsg,
2353 sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind)
2354 );
2355 }
2356
2357 /* And to delete index entries */
2358 if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){
2359 p->rc = prepareFreeAndCollectError(
2360 p->dbMain, &pIter->pDelete, &p->zErrmsg,
2361 sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere)
2362 );
2363 }
2364
2365 /* Create the SELECT statement to read keys in sorted order */
2366 if( p->rc==SQLITE_OK ){
2367 char *zSql;
2368 if( rbuIsVacuum(p) ){
2369 char *zStart = 0;
2370 if( nOffset ){
2371 zStart = rbuVacuumIndexStart(p, pIter);
2372 if( zStart ){
2373 sqlite3_free(zLimit);
2374 zLimit = 0;
2375 }
2376 }
2377
2378 zSql = sqlite3_mprintf(
2379 "SELECT %s, 0 AS rbu_control FROM '%q' %s %s %s ORDER BY %s%s",
2380 zCollist,
2381 pIter->zDataTbl,
2382 zPart,
2383 (zStart ? (zPart ? "AND" : "WHERE") : ""), zStart,
2384 zCollist, zLimit
2385 );
2386 sqlite3_free(zStart);
2387 }else
2388
2389 if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
2390 zSql = sqlite3_mprintf(
2391 "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s ORDER BY %s%s",
2392 zCollist, p->zStateDb, pIter->zDataTbl,
2393 zPart, zCollist, zLimit
2394 );
2395 }else{
2396 zSql = sqlite3_mprintf(
2397 "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s "
2398 "UNION ALL "
2399 "SELECT %s, rbu_control FROM '%q' "
2400 "%s %s typeof(rbu_control)='integer' AND rbu_control!=1 "
2401 "ORDER BY %s%s",
2402 zCollist, p->zStateDb, pIter->zDataTbl, zPart,
2403 zCollist, pIter->zDataTbl,
2404 zPart,
2405 (zPart ? "AND" : "WHERE"),
2406 zCollist, zLimit
2407 );
2408 }
2409 if( p->rc==SQLITE_OK ){
2410 p->rc = prepareFreeAndCollectError(p->dbRbu,&pIter->pSelect,pz,zSql);
2411 }else{
2412 sqlite3_free(zSql);
2413 }
2414 }
2415
2416 sqlite3_free(zImposterCols);
2417 sqlite3_free(zImposterPK);
2418 sqlite3_free(zWhere);
2419 sqlite3_free(zBind);
2420 sqlite3_free(zPart);
2421 }else{
2422 int bRbuRowid = (pIter->eType==RBU_PK_VTAB)
2423 ||(pIter->eType==RBU_PK_NONE)
2424 ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p));
2425 const char *zTbl = pIter->zTbl; /* Table this step applies to */
2426 const char *zWrite; /* Imposter table name */
2427
2428 char *zBindings = rbuObjIterGetBindlist(p, pIter->nTblCol + bRbuRowid);
2429 char *zWhere = rbuObjIterGetWhere(p, pIter);
2430 char *zOldlist = rbuObjIterGetOldlist(p, pIter, "old");
2431 char *zNewlist = rbuObjIterGetOldlist(p, pIter, "new");
2432
2433 zCollist = rbuObjIterGetCollist(p, pIter);
2434 pIter->nCol = pIter->nTblCol;
2435
2436 /* Create the imposter table or tables (if required). */
2437 rbuCreateImposterTable(p, pIter);
2438 rbuCreateImposterTable2(p, pIter);
2439 zWrite = (pIter->eType==RBU_PK_VTAB ? "" : "rbu_imp_");
2440
2441 /* Create the INSERT statement to write to the target PK b-tree */
2442 if( p->rc==SQLITE_OK ){
2443 p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz,
2444 sqlite3_mprintf(
2445 "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)",
2446 zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings
2447 )
2448 );
2449 }
2450
2451 /* Create the DELETE statement to write to the target PK b-tree.
2452 ** Because it only performs INSERT operations, this is not required for
2453 ** an rbu vacuum handle. */
2454 if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){
2455 p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz,
2456 sqlite3_mprintf(
2457 "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere
2458 )
2459 );
2460 }
2461
2462 if( rbuIsVacuum(p)==0 && pIter->abIndexed ){
2463 const char *zRbuRowid = "";
2464 if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
2465 zRbuRowid = ", rbu_rowid";
2466 }
2467
2468 /* Create the rbu_tmp_xxx table and the triggers to populate it. */
2469 rbuMPrintfExec(p, p->dbRbu,
2470 "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS "
2471 "SELECT *%s FROM '%q' WHERE 0;"
2472 , p->zStateDb, pIter->zDataTbl
2473 , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "")
2474 , pIter->zDataTbl
2475 );
2476
2477 rbuMPrintfExec(p, p->dbMain,
2478 "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" "
2479 "BEGIN "
2480 " SELECT rbu_tmp_insert(3, %s);"
2481 "END;"
2482
2483 "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" "
2484 "BEGIN "
2485 " SELECT rbu_tmp_insert(3, %s);"
2486 "END;"
2487
2488 "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" "
2489 "BEGIN "
2490 " SELECT rbu_tmp_insert(4, %s);"
2491 "END;",
2492 zWrite, zTbl, zOldlist,
2493 zWrite, zTbl, zOldlist,
2494 zWrite, zTbl, zNewlist
2495 );
2496
2497 if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
2498 rbuMPrintfExec(p, p->dbMain,
2499 "CREATE TEMP TRIGGER rbu_insert_tr AFTER INSERT ON \"%s%w\" "
2500 "BEGIN "
2501 " SELECT rbu_tmp_insert(0, %s);"
2502 "END;",
2503 zWrite, zTbl, zNewlist
2504 );
2505 }
2506
2507 rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid);
2508 }
2509
2510 /* Create the SELECT statement to read keys from data_xxx */
2511 if( p->rc==SQLITE_OK ){
2512 const char *zRbuRowid = "";
2513 char *zStart = 0;
2514 char *zOrder = 0;
2515 if( bRbuRowid ){
2516 zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid";
2517 }
2518
2519 if( rbuIsVacuum(p) ){
2520 if( nOffset ){
2521 zStart = rbuVacuumTableStart(p, pIter, bRbuRowid, zWrite);
2522 if( zStart ){
2523 sqlite3_free(zLimit);
2524 zLimit = 0;
2525 }
2526 }
2527 if( bRbuRowid ){
2528 zOrder = rbuMPrintf(p, "_rowid_");
2529 }else{
2530 zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", "");
2531 }
2532 }
2533
2534 if( p->rc==SQLITE_OK ){
2535 p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz,
2536 sqlite3_mprintf(
2537 "SELECT %s,%s rbu_control%s FROM '%q'%s %s %s %s",
2538 zCollist,
2539 (rbuIsVacuum(p) ? "0 AS " : ""),
2540 zRbuRowid,
2541 pIter->zDataTbl, (zStart ? zStart : ""),
2542 (zOrder ? "ORDER BY" : ""), zOrder,
2543 zLimit
2544 )
2545 );
2546 }
2547 sqlite3_free(zStart);
2548 sqlite3_free(zOrder);
2549 }
2550
2551 sqlite3_free(zWhere);
2552 sqlite3_free(zOldlist);
2553 sqlite3_free(zNewlist);
2554 sqlite3_free(zBindings);
2555 }
2556 sqlite3_free(zCollist);
2557 sqlite3_free(zLimit);
2558 }
2559
2560 return p->rc;
2561 }
2562
2563 /*
2564 ** Set output variable *ppStmt to point to an UPDATE statement that may
2565 ** be used to update the imposter table for the main table b-tree of the
2566 ** table object that pIter currently points to, assuming that the
2567 ** rbu_control column of the data_xyz table contains zMask.
2568 **
2569 ** If the zMask string does not specify any columns to update, then this
2570 ** is not an error. Output variable *ppStmt is set to NULL in this case.
2571 */
rbuGetUpdateStmt(sqlite3rbu * p,RbuObjIter * pIter,const char * zMask,sqlite3_stmt ** ppStmt)2572 static int rbuGetUpdateStmt(
2573 sqlite3rbu *p, /* RBU handle */
2574 RbuObjIter *pIter, /* Object iterator */
2575 const char *zMask, /* rbu_control value ('x.x.') */
2576 sqlite3_stmt **ppStmt /* OUT: UPDATE statement handle */
2577 ){
2578 RbuUpdateStmt **pp;
2579 RbuUpdateStmt *pUp = 0;
2580 int nUp = 0;
2581
2582 /* In case an error occurs */
2583 *ppStmt = 0;
2584
2585 /* Search for an existing statement. If one is found, shift it to the front
2586 ** of the LRU queue and return immediately. Otherwise, leave nUp pointing
2587 ** to the number of statements currently in the cache and pUp to the
2588 ** last object in the list. */
2589 for(pp=&pIter->pRbuUpdate; *pp; pp=&((*pp)->pNext)){
2590 pUp = *pp;
2591 if( strcmp(pUp->zMask, zMask)==0 ){
2592 *pp = pUp->pNext;
2593 pUp->pNext = pIter->pRbuUpdate;
2594 pIter->pRbuUpdate = pUp;
2595 *ppStmt = pUp->pUpdate;
2596 return SQLITE_OK;
2597 }
2598 nUp++;
2599 }
2600 assert( pUp==0 || pUp->pNext==0 );
2601
2602 if( nUp>=SQLITE_RBU_UPDATE_CACHESIZE ){
2603 for(pp=&pIter->pRbuUpdate; *pp!=pUp; pp=&((*pp)->pNext));
2604 *pp = 0;
2605 sqlite3_finalize(pUp->pUpdate);
2606 pUp->pUpdate = 0;
2607 }else{
2608 pUp = (RbuUpdateStmt*)rbuMalloc(p, sizeof(RbuUpdateStmt)+pIter->nTblCol+1);
2609 }
2610
2611 if( pUp ){
2612 char *zWhere = rbuObjIterGetWhere(p, pIter);
2613 char *zSet = rbuObjIterGetSetlist(p, pIter, zMask);
2614 char *zUpdate = 0;
2615
2616 pUp->zMask = (char*)&pUp[1];
2617 memcpy(pUp->zMask, zMask, pIter->nTblCol);
2618 pUp->pNext = pIter->pRbuUpdate;
2619 pIter->pRbuUpdate = pUp;
2620
2621 if( zSet ){
2622 const char *zPrefix = "";
2623
2624 if( pIter->eType!=RBU_PK_VTAB ) zPrefix = "rbu_imp_";
2625 zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s",
2626 zPrefix, pIter->zTbl, zSet, zWhere
2627 );
2628 p->rc = prepareFreeAndCollectError(
2629 p->dbMain, &pUp->pUpdate, &p->zErrmsg, zUpdate
2630 );
2631 *ppStmt = pUp->pUpdate;
2632 }
2633 sqlite3_free(zWhere);
2634 sqlite3_free(zSet);
2635 }
2636
2637 return p->rc;
2638 }
2639
rbuOpenDbhandle(sqlite3rbu * p,const char * zName,int bUseVfs)2640 static sqlite3 *rbuOpenDbhandle(
2641 sqlite3rbu *p,
2642 const char *zName,
2643 int bUseVfs
2644 ){
2645 sqlite3 *db = 0;
2646 if( p->rc==SQLITE_OK ){
2647 const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI;
2648 p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0);
2649 if( p->rc ){
2650 p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
2651 sqlite3_close(db);
2652 db = 0;
2653 }
2654 }
2655 return db;
2656 }
2657
2658 /*
2659 ** Free an RbuState object allocated by rbuLoadState().
2660 */
rbuFreeState(RbuState * p)2661 static void rbuFreeState(RbuState *p){
2662 if( p ){
2663 sqlite3_free(p->zTbl);
2664 sqlite3_free(p->zDataTbl);
2665 sqlite3_free(p->zIdx);
2666 sqlite3_free(p);
2667 }
2668 }
2669
2670 /*
2671 ** Allocate an RbuState object and load the contents of the rbu_state
2672 ** table into it. Return a pointer to the new object. It is the
2673 ** responsibility of the caller to eventually free the object using
2674 ** sqlite3_free().
2675 **
2676 ** If an error occurs, leave an error code and message in the rbu handle
2677 ** and return NULL.
2678 */
rbuLoadState(sqlite3rbu * p)2679 static RbuState *rbuLoadState(sqlite3rbu *p){
2680 RbuState *pRet = 0;
2681 sqlite3_stmt *pStmt = 0;
2682 int rc;
2683 int rc2;
2684
2685 pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState));
2686 if( pRet==0 ) return 0;
2687
2688 rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
2689 sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb)
2690 );
2691 while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
2692 switch( sqlite3_column_int(pStmt, 0) ){
2693 case RBU_STATE_STAGE:
2694 pRet->eStage = sqlite3_column_int(pStmt, 1);
2695 if( pRet->eStage!=RBU_STAGE_OAL
2696 && pRet->eStage!=RBU_STAGE_MOVE
2697 && pRet->eStage!=RBU_STAGE_CKPT
2698 ){
2699 p->rc = SQLITE_CORRUPT;
2700 }
2701 break;
2702
2703 case RBU_STATE_TBL:
2704 pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
2705 break;
2706
2707 case RBU_STATE_IDX:
2708 pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
2709 break;
2710
2711 case RBU_STATE_ROW:
2712 pRet->nRow = sqlite3_column_int(pStmt, 1);
2713 break;
2714
2715 case RBU_STATE_PROGRESS:
2716 pRet->nProgress = sqlite3_column_int64(pStmt, 1);
2717 break;
2718
2719 case RBU_STATE_CKPT:
2720 pRet->iWalCksum = sqlite3_column_int64(pStmt, 1);
2721 break;
2722
2723 case RBU_STATE_COOKIE:
2724 pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1);
2725 break;
2726
2727 case RBU_STATE_OALSZ:
2728 pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1);
2729 break;
2730
2731 case RBU_STATE_PHASEONESTEP:
2732 pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1);
2733 break;
2734
2735 case RBU_STATE_DATATBL:
2736 pRet->zDataTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
2737 break;
2738
2739 default:
2740 rc = SQLITE_CORRUPT;
2741 break;
2742 }
2743 }
2744 rc2 = sqlite3_finalize(pStmt);
2745 if( rc==SQLITE_OK ) rc = rc2;
2746
2747 p->rc = rc;
2748 return pRet;
2749 }
2750
2751
2752 /*
2753 ** Open the database handle and attach the RBU database as "rbu". If an
2754 ** error occurs, leave an error code and message in the RBU handle.
2755 */
rbuOpenDatabase(sqlite3rbu * p,int * pbRetry)2756 static void rbuOpenDatabase(sqlite3rbu *p, int *pbRetry){
2757 assert( p->rc || (p->dbMain==0 && p->dbRbu==0) );
2758 assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 );
2759
2760 /* Open the RBU database */
2761 p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1);
2762
2763 if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
2764 sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p);
2765 if( p->zState==0 ){
2766 const char *zFile = sqlite3_db_filename(p->dbRbu, "main");
2767 p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile);
2768 }
2769 }
2770
2771 /* If using separate RBU and state databases, attach the state database to
2772 ** the RBU db handle now. */
2773 if( p->zState ){
2774 rbuMPrintfExec(p, p->dbRbu, "ATTACH %Q AS stat", p->zState);
2775 memcpy(p->zStateDb, "stat", 4);
2776 }else{
2777 memcpy(p->zStateDb, "main", 4);
2778 }
2779
2780 #if 0
2781 if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
2782 p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0);
2783 }
2784 #endif
2785
2786 /* If it has not already been created, create the rbu_state table */
2787 rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb);
2788
2789 #if 0
2790 if( rbuIsVacuum(p) ){
2791 if( p->rc==SQLITE_OK ){
2792 int rc2;
2793 int bOk = 0;
2794 sqlite3_stmt *pCnt = 0;
2795 p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg,
2796 "SELECT count(*) FROM stat.sqlite_schema"
2797 );
2798 if( p->rc==SQLITE_OK
2799 && sqlite3_step(pCnt)==SQLITE_ROW
2800 && 1==sqlite3_column_int(pCnt, 0)
2801 ){
2802 bOk = 1;
2803 }
2804 rc2 = sqlite3_finalize(pCnt);
2805 if( p->rc==SQLITE_OK ) p->rc = rc2;
2806
2807 if( p->rc==SQLITE_OK && bOk==0 ){
2808 p->rc = SQLITE_ERROR;
2809 p->zErrmsg = sqlite3_mprintf("invalid state database");
2810 }
2811
2812 if( p->rc==SQLITE_OK ){
2813 p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0);
2814 }
2815 }
2816 }
2817 #endif
2818
2819 if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
2820 int bOpen = 0;
2821 int rc;
2822 p->nRbu = 0;
2823 p->pRbuFd = 0;
2824 rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p);
2825 if( rc!=SQLITE_NOTFOUND ) p->rc = rc;
2826 if( p->eStage>=RBU_STAGE_MOVE ){
2827 bOpen = 1;
2828 }else{
2829 RbuState *pState = rbuLoadState(p);
2830 if( pState ){
2831 bOpen = (pState->eStage>=RBU_STAGE_MOVE);
2832 rbuFreeState(pState);
2833 }
2834 }
2835 if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1);
2836 }
2837
2838 p->eStage = 0;
2839 if( p->rc==SQLITE_OK && p->dbMain==0 ){
2840 if( !rbuIsVacuum(p) ){
2841 p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1);
2842 }else if( p->pRbuFd->pWalFd ){
2843 if( pbRetry ){
2844 p->pRbuFd->bNolock = 0;
2845 sqlite3_close(p->dbRbu);
2846 sqlite3_close(p->dbMain);
2847 p->dbMain = 0;
2848 p->dbRbu = 0;
2849 *pbRetry = 1;
2850 return;
2851 }
2852 p->rc = SQLITE_ERROR;
2853 p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database");
2854 }else{
2855 char *zTarget;
2856 char *zExtra = 0;
2857 if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){
2858 zExtra = &p->zRbu[5];
2859 while( *zExtra ){
2860 if( *zExtra++=='?' ) break;
2861 }
2862 if( *zExtra=='\0' ) zExtra = 0;
2863 }
2864
2865 zTarget = sqlite3_mprintf("file:%s-vactmp?rbu_memory=1%s%s",
2866 sqlite3_db_filename(p->dbRbu, "main"),
2867 (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra)
2868 );
2869
2870 if( zTarget==0 ){
2871 p->rc = SQLITE_NOMEM;
2872 return;
2873 }
2874 p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1);
2875 sqlite3_free(zTarget);
2876 }
2877 }
2878
2879 if( p->rc==SQLITE_OK ){
2880 p->rc = sqlite3_create_function(p->dbMain,
2881 "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0
2882 );
2883 }
2884
2885 if( p->rc==SQLITE_OK ){
2886 p->rc = sqlite3_create_function(p->dbMain,
2887 "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0
2888 );
2889 }
2890
2891 if( p->rc==SQLITE_OK ){
2892 p->rc = sqlite3_create_function(p->dbRbu,
2893 "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0
2894 );
2895 }
2896
2897 if( p->rc==SQLITE_OK ){
2898 p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p);
2899 }
2900 rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_schema");
2901
2902 /* Mark the database file just opened as an RBU target database. If
2903 ** this call returns SQLITE_NOTFOUND, then the RBU vfs is not in use.
2904 ** This is an error. */
2905 if( p->rc==SQLITE_OK ){
2906 p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p);
2907 }
2908
2909 if( p->rc==SQLITE_NOTFOUND ){
2910 p->rc = SQLITE_ERROR;
2911 p->zErrmsg = sqlite3_mprintf("rbu vfs not found");
2912 }
2913 }
2914
2915 /*
2916 ** This routine is a copy of the sqlite3FileSuffix3() routine from the core.
2917 ** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined.
2918 **
2919 ** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database
2920 ** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and
2921 ** if filename in z[] has a suffix (a.k.a. "extension") that is longer than
2922 ** three characters, then shorten the suffix on z[] to be the last three
2923 ** characters of the original suffix.
2924 **
2925 ** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always
2926 ** do the suffix shortening regardless of URI parameter.
2927 **
2928 ** Examples:
2929 **
2930 ** test.db-journal => test.nal
2931 ** test.db-wal => test.wal
2932 ** test.db-shm => test.shm
2933 ** test.db-mj7f3319fa => test.9fa
2934 */
rbuFileSuffix3(const char * zBase,char * z)2935 static void rbuFileSuffix3(const char *zBase, char *z){
2936 #ifdef SQLITE_ENABLE_8_3_NAMES
2937 #if SQLITE_ENABLE_8_3_NAMES<2
2938 if( sqlite3_uri_boolean(zBase, "8_3_names", 0) )
2939 #endif
2940 {
2941 int i, sz;
2942 sz = (int)strlen(z)&0xffffff;
2943 for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){}
2944 if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4);
2945 }
2946 #endif
2947 }
2948
2949 /*
2950 ** Return the current wal-index header checksum for the target database
2951 ** as a 64-bit integer.
2952 **
2953 ** The checksum is store in the first page of xShmMap memory as an 8-byte
2954 ** blob starting at byte offset 40.
2955 */
rbuShmChecksum(sqlite3rbu * p)2956 static i64 rbuShmChecksum(sqlite3rbu *p){
2957 i64 iRet = 0;
2958 if( p->rc==SQLITE_OK ){
2959 sqlite3_file *pDb = p->pTargetFd->pReal;
2960 u32 volatile *ptr;
2961 p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr);
2962 if( p->rc==SQLITE_OK ){
2963 iRet = ((i64)ptr[10] << 32) + ptr[11];
2964 }
2965 }
2966 return iRet;
2967 }
2968
2969 /*
2970 ** This function is called as part of initializing or reinitializing an
2971 ** incremental checkpoint.
2972 **
2973 ** It populates the sqlite3rbu.aFrame[] array with the set of
2974 ** (wal frame -> db page) copy operations required to checkpoint the
2975 ** current wal file, and obtains the set of shm locks required to safely
2976 ** perform the copy operations directly on the file-system.
2977 **
2978 ** If argument pState is not NULL, then the incremental checkpoint is
2979 ** being resumed. In this case, if the checksum of the wal-index-header
2980 ** following recovery is not the same as the checksum saved in the RbuState
2981 ** object, then the rbu handle is set to DONE state. This occurs if some
2982 ** other client appends a transaction to the wal file in the middle of
2983 ** an incremental checkpoint.
2984 */
rbuSetupCheckpoint(sqlite3rbu * p,RbuState * pState)2985 static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){
2986
2987 /* If pState is NULL, then the wal file may not have been opened and
2988 ** recovered. Running a read-statement here to ensure that doing so
2989 ** does not interfere with the "capture" process below. */
2990 if( pState==0 ){
2991 p->eStage = 0;
2992 if( p->rc==SQLITE_OK ){
2993 p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_schema", 0, 0, 0);
2994 }
2995 }
2996
2997 /* Assuming no error has occurred, run a "restart" checkpoint with the
2998 ** sqlite3rbu.eStage variable set to CAPTURE. This turns on the following
2999 ** special behaviour in the rbu VFS:
3000 **
3001 ** * If the exclusive shm WRITER or READ0 lock cannot be obtained,
3002 ** the checkpoint fails with SQLITE_BUSY (normally SQLite would
3003 ** proceed with running a passive checkpoint instead of failing).
3004 **
3005 ** * Attempts to read from the *-wal file or write to the database file
3006 ** do not perform any IO. Instead, the frame/page combinations that
3007 ** would be read/written are recorded in the sqlite3rbu.aFrame[]
3008 ** array.
3009 **
3010 ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER,
3011 ** READ0 and CHECKPOINT locks taken as part of the checkpoint are
3012 ** no-ops. These locks will not be released until the connection
3013 ** is closed.
3014 **
3015 ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL
3016 ** error.
3017 **
3018 ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the
3019 ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[]
3020 ** array populated with a set of (frame -> page) mappings. Because the
3021 ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy
3022 ** data from the wal file into the database file according to the
3023 ** contents of aFrame[].
3024 */
3025 if( p->rc==SQLITE_OK ){
3026 int rc2;
3027 p->eStage = RBU_STAGE_CAPTURE;
3028 rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0);
3029 if( rc2!=SQLITE_INTERNAL ) p->rc = rc2;
3030 }
3031
3032 if( p->rc==SQLITE_OK && p->nFrame>0 ){
3033 p->eStage = RBU_STAGE_CKPT;
3034 p->nStep = (pState ? pState->nRow : 0);
3035 p->aBuf = rbuMalloc(p, p->pgsz);
3036 p->iWalCksum = rbuShmChecksum(p);
3037 }
3038
3039 if( p->rc==SQLITE_OK ){
3040 if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){
3041 p->rc = SQLITE_DONE;
3042 p->eStage = RBU_STAGE_DONE;
3043 }else{
3044 int nSectorSize;
3045 sqlite3_file *pDb = p->pTargetFd->pReal;
3046 sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal;
3047 assert( p->nPagePerSector==0 );
3048 nSectorSize = pDb->pMethods->xSectorSize(pDb);
3049 if( nSectorSize>p->pgsz ){
3050 p->nPagePerSector = nSectorSize / p->pgsz;
3051 }else{
3052 p->nPagePerSector = 1;
3053 }
3054
3055 /* Call xSync() on the wal file. This causes SQLite to sync the
3056 ** directory in which the target database and the wal file reside, in
3057 ** case it has not been synced since the rename() call in
3058 ** rbuMoveOalFile(). */
3059 p->rc = pWal->pMethods->xSync(pWal, SQLITE_SYNC_NORMAL);
3060 }
3061 }
3062 }
3063
3064 /*
3065 ** Called when iAmt bytes are read from offset iOff of the wal file while
3066 ** the rbu object is in capture mode. Record the frame number of the frame
3067 ** being read in the aFrame[] array.
3068 */
rbuCaptureWalRead(sqlite3rbu * pRbu,i64 iOff,int iAmt)3069 static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){
3070 const u32 mReq = (1<<WAL_LOCK_WRITE)|(1<<WAL_LOCK_CKPT)|(1<<WAL_LOCK_READ0);
3071 u32 iFrame;
3072
3073 if( pRbu->mLock!=mReq ){
3074 pRbu->rc = SQLITE_BUSY;
3075 return SQLITE_INTERNAL;
3076 }
3077
3078 pRbu->pgsz = iAmt;
3079 if( pRbu->nFrame==pRbu->nFrameAlloc ){
3080 int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2;
3081 RbuFrame *aNew;
3082 aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame));
3083 if( aNew==0 ) return SQLITE_NOMEM;
3084 pRbu->aFrame = aNew;
3085 pRbu->nFrameAlloc = nNew;
3086 }
3087
3088 iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1;
3089 if( pRbu->iMaxFrame<iFrame ) pRbu->iMaxFrame = iFrame;
3090 pRbu->aFrame[pRbu->nFrame].iWalFrame = iFrame;
3091 pRbu->aFrame[pRbu->nFrame].iDbPage = 0;
3092 pRbu->nFrame++;
3093 return SQLITE_OK;
3094 }
3095
3096 /*
3097 ** Called when a page of data is written to offset iOff of the database
3098 ** file while the rbu handle is in capture mode. Record the page number
3099 ** of the page being written in the aFrame[] array.
3100 */
rbuCaptureDbWrite(sqlite3rbu * pRbu,i64 iOff)3101 static int rbuCaptureDbWrite(sqlite3rbu *pRbu, i64 iOff){
3102 pRbu->aFrame[pRbu->nFrame-1].iDbPage = (u32)(iOff / pRbu->pgsz) + 1;
3103 return SQLITE_OK;
3104 }
3105
3106 /*
3107 ** This is called as part of an incremental checkpoint operation. Copy
3108 ** a single frame of data from the wal file into the database file, as
3109 ** indicated by the RbuFrame object.
3110 */
rbuCheckpointFrame(sqlite3rbu * p,RbuFrame * pFrame)3111 static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){
3112 sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal;
3113 sqlite3_file *pDb = p->pTargetFd->pReal;
3114 i64 iOff;
3115
3116 assert( p->rc==SQLITE_OK );
3117 iOff = (i64)(pFrame->iWalFrame-1) * (p->pgsz + 24) + 32 + 24;
3118 p->rc = pWal->pMethods->xRead(pWal, p->aBuf, p->pgsz, iOff);
3119 if( p->rc ) return;
3120
3121 iOff = (i64)(pFrame->iDbPage-1) * p->pgsz;
3122 p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff);
3123 }
3124
3125
3126 /*
3127 ** Take an EXCLUSIVE lock on the database file.
3128 */
rbuLockDatabase(sqlite3rbu * p)3129 static void rbuLockDatabase(sqlite3rbu *p){
3130 sqlite3_file *pReal = p->pTargetFd->pReal;
3131 assert( p->rc==SQLITE_OK );
3132 p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_SHARED);
3133 if( p->rc==SQLITE_OK ){
3134 p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_EXCLUSIVE);
3135 }
3136 }
3137
3138 #if defined(_WIN32_WCE)
rbuWinUtf8ToUnicode(const char * zFilename)3139 static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){
3140 int nChar;
3141 LPWSTR zWideFilename;
3142
3143 nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0);
3144 if( nChar==0 ){
3145 return 0;
3146 }
3147 zWideFilename = sqlite3_malloc64( nChar*sizeof(zWideFilename[0]) );
3148 if( zWideFilename==0 ){
3149 return 0;
3150 }
3151 memset(zWideFilename, 0, nChar*sizeof(zWideFilename[0]));
3152 nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename,
3153 nChar);
3154 if( nChar==0 ){
3155 sqlite3_free(zWideFilename);
3156 zWideFilename = 0;
3157 }
3158 return zWideFilename;
3159 }
3160 #endif
3161
3162 /*
3163 ** The RBU handle is currently in RBU_STAGE_OAL state, with a SHARED lock
3164 ** on the database file. This proc moves the *-oal file to the *-wal path,
3165 ** then reopens the database file (this time in vanilla, non-oal, WAL mode).
3166 ** If an error occurs, leave an error code and error message in the rbu
3167 ** handle.
3168 */
rbuMoveOalFile(sqlite3rbu * p)3169 static void rbuMoveOalFile(sqlite3rbu *p){
3170 const char *zBase = sqlite3_db_filename(p->dbMain, "main");
3171 const char *zMove = zBase;
3172 char *zOal;
3173 char *zWal;
3174
3175 if( rbuIsVacuum(p) ){
3176 zMove = sqlite3_db_filename(p->dbRbu, "main");
3177 }
3178 zOal = sqlite3_mprintf("%s-oal", zMove);
3179 zWal = sqlite3_mprintf("%s-wal", zMove);
3180
3181 assert( p->eStage==RBU_STAGE_MOVE );
3182 assert( p->rc==SQLITE_OK && p->zErrmsg==0 );
3183 if( zWal==0 || zOal==0 ){
3184 p->rc = SQLITE_NOMEM;
3185 }else{
3186 /* Move the *-oal file to *-wal. At this point connection p->db is
3187 ** holding a SHARED lock on the target database file (because it is
3188 ** in WAL mode). So no other connection may be writing the db.
3189 **
3190 ** In order to ensure that there are no database readers, an EXCLUSIVE
3191 ** lock is obtained here before the *-oal is moved to *-wal.
3192 */
3193 rbuLockDatabase(p);
3194 if( p->rc==SQLITE_OK ){
3195 rbuFileSuffix3(zBase, zWal);
3196 rbuFileSuffix3(zBase, zOal);
3197
3198 /* Re-open the databases. */
3199 rbuObjIterFinalize(&p->objiter);
3200 sqlite3_close(p->dbRbu);
3201 sqlite3_close(p->dbMain);
3202 p->dbMain = 0;
3203 p->dbRbu = 0;
3204
3205 #if defined(_WIN32_WCE)
3206 {
3207 LPWSTR zWideOal;
3208 LPWSTR zWideWal;
3209
3210 zWideOal = rbuWinUtf8ToUnicode(zOal);
3211 if( zWideOal ){
3212 zWideWal = rbuWinUtf8ToUnicode(zWal);
3213 if( zWideWal ){
3214 if( MoveFileW(zWideOal, zWideWal) ){
3215 p->rc = SQLITE_OK;
3216 }else{
3217 p->rc = SQLITE_IOERR;
3218 }
3219 sqlite3_free(zWideWal);
3220 }else{
3221 p->rc = SQLITE_IOERR_NOMEM;
3222 }
3223 sqlite3_free(zWideOal);
3224 }else{
3225 p->rc = SQLITE_IOERR_NOMEM;
3226 }
3227 }
3228 #else
3229 p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK;
3230 #endif
3231
3232 if( p->rc==SQLITE_OK ){
3233 rbuOpenDatabase(p, 0);
3234 rbuSetupCheckpoint(p, 0);
3235 }
3236 }
3237 }
3238
3239 sqlite3_free(zWal);
3240 sqlite3_free(zOal);
3241 }
3242
3243 /*
3244 ** The SELECT statement iterating through the keys for the current object
3245 ** (p->objiter.pSelect) currently points to a valid row. This function
3246 ** determines the type of operation requested by this row and returns
3247 ** one of the following values to indicate the result:
3248 **
3249 ** * RBU_INSERT
3250 ** * RBU_DELETE
3251 ** * RBU_IDX_DELETE
3252 ** * RBU_UPDATE
3253 **
3254 ** If RBU_UPDATE is returned, then output variable *pzMask is set to
3255 ** point to the text value indicating the columns to update.
3256 **
3257 ** If the rbu_control field contains an invalid value, an error code and
3258 ** message are left in the RBU handle and zero returned.
3259 */
rbuStepType(sqlite3rbu * p,const char ** pzMask)3260 static int rbuStepType(sqlite3rbu *p, const char **pzMask){
3261 int iCol = p->objiter.nCol; /* Index of rbu_control column */
3262 int res = 0; /* Return value */
3263
3264 switch( sqlite3_column_type(p->objiter.pSelect, iCol) ){
3265 case SQLITE_INTEGER: {
3266 int iVal = sqlite3_column_int(p->objiter.pSelect, iCol);
3267 switch( iVal ){
3268 case 0: res = RBU_INSERT; break;
3269 case 1: res = RBU_DELETE; break;
3270 case 2: res = RBU_REPLACE; break;
3271 case 3: res = RBU_IDX_DELETE; break;
3272 case 4: res = RBU_IDX_INSERT; break;
3273 }
3274 break;
3275 }
3276
3277 case SQLITE_TEXT: {
3278 const unsigned char *z = sqlite3_column_text(p->objiter.pSelect, iCol);
3279 if( z==0 ){
3280 p->rc = SQLITE_NOMEM;
3281 }else{
3282 *pzMask = (const char*)z;
3283 }
3284 res = RBU_UPDATE;
3285
3286 break;
3287 }
3288
3289 default:
3290 break;
3291 }
3292
3293 if( res==0 ){
3294 rbuBadControlError(p);
3295 }
3296 return res;
3297 }
3298
3299 #ifdef SQLITE_DEBUG
3300 /*
3301 ** Assert that column iCol of statement pStmt is named zName.
3302 */
assertColumnName(sqlite3_stmt * pStmt,int iCol,const char * zName)3303 static void assertColumnName(sqlite3_stmt *pStmt, int iCol, const char *zName){
3304 const char *zCol = sqlite3_column_name(pStmt, iCol);
3305 assert( 0==sqlite3_stricmp(zName, zCol) );
3306 }
3307 #else
3308 # define assertColumnName(x,y,z)
3309 #endif
3310
3311 /*
3312 ** Argument eType must be one of RBU_INSERT, RBU_DELETE, RBU_IDX_INSERT or
3313 ** RBU_IDX_DELETE. This function performs the work of a single
3314 ** sqlite3rbu_step() call for the type of operation specified by eType.
3315 */
rbuStepOneOp(sqlite3rbu * p,int eType)3316 static void rbuStepOneOp(sqlite3rbu *p, int eType){
3317 RbuObjIter *pIter = &p->objiter;
3318 sqlite3_value *pVal;
3319 sqlite3_stmt *pWriter;
3320 int i;
3321
3322 assert( p->rc==SQLITE_OK );
3323 assert( eType!=RBU_DELETE || pIter->zIdx==0 );
3324 assert( eType==RBU_DELETE || eType==RBU_IDX_DELETE
3325 || eType==RBU_INSERT || eType==RBU_IDX_INSERT
3326 );
3327
3328 /* If this is a delete, decrement nPhaseOneStep by nIndex. If the DELETE
3329 ** statement below does actually delete a row, nPhaseOneStep will be
3330 ** incremented by the same amount when SQL function rbu_tmp_insert()
3331 ** is invoked by the trigger. */
3332 if( eType==RBU_DELETE ){
3333 p->nPhaseOneStep -= p->objiter.nIndex;
3334 }
3335
3336 if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){
3337 pWriter = pIter->pDelete;
3338 }else{
3339 pWriter = pIter->pInsert;
3340 }
3341
3342 for(i=0; i<pIter->nCol; i++){
3343 /* If this is an INSERT into a table b-tree and the table has an
3344 ** explicit INTEGER PRIMARY KEY, check that this is not an attempt
3345 ** to write a NULL into the IPK column. That is not permitted. */
3346 if( eType==RBU_INSERT
3347 && pIter->zIdx==0 && pIter->eType==RBU_PK_IPK && pIter->abTblPk[i]
3348 && sqlite3_column_type(pIter->pSelect, i)==SQLITE_NULL
3349 ){
3350 p->rc = SQLITE_MISMATCH;
3351 p->zErrmsg = sqlite3_mprintf("datatype mismatch");
3352 return;
3353 }
3354
3355 if( eType==RBU_DELETE && pIter->abTblPk[i]==0 ){
3356 continue;
3357 }
3358
3359 pVal = sqlite3_column_value(pIter->pSelect, i);
3360 p->rc = sqlite3_bind_value(pWriter, i+1, pVal);
3361 if( p->rc ) return;
3362 }
3363 if( pIter->zIdx==0 ){
3364 if( pIter->eType==RBU_PK_VTAB
3365 || pIter->eType==RBU_PK_NONE
3366 || (pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p))
3367 ){
3368 /* For a virtual table, or a table with no primary key, the
3369 ** SELECT statement is:
3370 **
3371 ** SELECT <cols>, rbu_control, rbu_rowid FROM ....
3372 **
3373 ** Hence column_value(pIter->nCol+1).
3374 */
3375 assertColumnName(pIter->pSelect, pIter->nCol+1,
3376 rbuIsVacuum(p) ? "rowid" : "rbu_rowid"
3377 );
3378 pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1);
3379 p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal);
3380 }
3381 }
3382 if( p->rc==SQLITE_OK ){
3383 sqlite3_step(pWriter);
3384 p->rc = resetAndCollectError(pWriter, &p->zErrmsg);
3385 }
3386 }
3387
3388 /*
3389 ** This function does the work for an sqlite3rbu_step() call.
3390 **
3391 ** The object-iterator (p->objiter) currently points to a valid object,
3392 ** and the input cursor (p->objiter.pSelect) currently points to a valid
3393 ** input row. Perform whatever processing is required and return.
3394 **
3395 ** If no error occurs, SQLITE_OK is returned. Otherwise, an error code
3396 ** and message is left in the RBU handle and a copy of the error code
3397 ** returned.
3398 */
rbuStep(sqlite3rbu * p)3399 static int rbuStep(sqlite3rbu *p){
3400 RbuObjIter *pIter = &p->objiter;
3401 const char *zMask = 0;
3402 int eType = rbuStepType(p, &zMask);
3403
3404 if( eType ){
3405 assert( eType==RBU_INSERT || eType==RBU_DELETE
3406 || eType==RBU_REPLACE || eType==RBU_IDX_DELETE
3407 || eType==RBU_IDX_INSERT || eType==RBU_UPDATE
3408 );
3409 assert( eType!=RBU_UPDATE || pIter->zIdx==0 );
3410
3411 if( pIter->zIdx==0 && (eType==RBU_IDX_DELETE || eType==RBU_IDX_INSERT) ){
3412 rbuBadControlError(p);
3413 }
3414 else if( eType==RBU_REPLACE ){
3415 if( pIter->zIdx==0 ){
3416 p->nPhaseOneStep += p->objiter.nIndex;
3417 rbuStepOneOp(p, RBU_DELETE);
3418 }
3419 if( p->rc==SQLITE_OK ) rbuStepOneOp(p, RBU_INSERT);
3420 }
3421 else if( eType!=RBU_UPDATE ){
3422 rbuStepOneOp(p, eType);
3423 }
3424 else{
3425 sqlite3_value *pVal;
3426 sqlite3_stmt *pUpdate = 0;
3427 assert( eType==RBU_UPDATE );
3428 p->nPhaseOneStep -= p->objiter.nIndex;
3429 rbuGetUpdateStmt(p, pIter, zMask, &pUpdate);
3430 if( pUpdate ){
3431 int i;
3432 for(i=0; p->rc==SQLITE_OK && i<pIter->nCol; i++){
3433 char c = zMask[pIter->aiSrcOrder[i]];
3434 pVal = sqlite3_column_value(pIter->pSelect, i);
3435 if( pIter->abTblPk[i] || c!='.' ){
3436 p->rc = sqlite3_bind_value(pUpdate, i+1, pVal);
3437 }
3438 }
3439 if( p->rc==SQLITE_OK
3440 && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE)
3441 ){
3442 /* Bind the rbu_rowid value to column _rowid_ */
3443 assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid");
3444 pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1);
3445 p->rc = sqlite3_bind_value(pUpdate, pIter->nCol+1, pVal);
3446 }
3447 if( p->rc==SQLITE_OK ){
3448 sqlite3_step(pUpdate);
3449 p->rc = resetAndCollectError(pUpdate, &p->zErrmsg);
3450 }
3451 }
3452 }
3453 }
3454 return p->rc;
3455 }
3456
3457 /*
3458 ** Increment the schema cookie of the main database opened by p->dbMain.
3459 **
3460 ** Or, if this is an RBU vacuum, set the schema cookie of the main db
3461 ** opened by p->dbMain to one more than the schema cookie of the main
3462 ** db opened by p->dbRbu.
3463 */
rbuIncrSchemaCookie(sqlite3rbu * p)3464 static void rbuIncrSchemaCookie(sqlite3rbu *p){
3465 if( p->rc==SQLITE_OK ){
3466 sqlite3 *dbread = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain);
3467 int iCookie = 1000000;
3468 sqlite3_stmt *pStmt;
3469
3470 p->rc = prepareAndCollectError(dbread, &pStmt, &p->zErrmsg,
3471 "PRAGMA schema_version"
3472 );
3473 if( p->rc==SQLITE_OK ){
3474 /* Coverage: it may be that this sqlite3_step() cannot fail. There
3475 ** is already a transaction open, so the prepared statement cannot
3476 ** throw an SQLITE_SCHEMA exception. The only database page the
3477 ** statement reads is page 1, which is guaranteed to be in the cache.
3478 ** And no memory allocations are required. */
3479 if( SQLITE_ROW==sqlite3_step(pStmt) ){
3480 iCookie = sqlite3_column_int(pStmt, 0);
3481 }
3482 rbuFinalize(p, pStmt);
3483 }
3484 if( p->rc==SQLITE_OK ){
3485 rbuMPrintfExec(p, p->dbMain, "PRAGMA schema_version = %d", iCookie+1);
3486 }
3487 }
3488 }
3489
3490 /*
3491 ** Update the contents of the rbu_state table within the rbu database. The
3492 ** value stored in the RBU_STATE_STAGE column is eStage. All other values
3493 ** are determined by inspecting the rbu handle passed as the first argument.
3494 */
rbuSaveState(sqlite3rbu * p,int eStage)3495 static void rbuSaveState(sqlite3rbu *p, int eStage){
3496 if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){
3497 sqlite3_stmt *pInsert = 0;
3498 rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd);
3499 int rc;
3500
3501 assert( p->zErrmsg==0 );
3502 rc = prepareFreeAndCollectError(p->dbRbu, &pInsert, &p->zErrmsg,
3503 sqlite3_mprintf(
3504 "INSERT OR REPLACE INTO %s.rbu_state(k, v) VALUES "
3505 "(%d, %d), "
3506 "(%d, %Q), "
3507 "(%d, %Q), "
3508 "(%d, %d), "
3509 "(%d, %d), "
3510 "(%d, %lld), "
3511 "(%d, %lld), "
3512 "(%d, %lld), "
3513 "(%d, %lld), "
3514 "(%d, %Q) ",
3515 p->zStateDb,
3516 RBU_STATE_STAGE, eStage,
3517 RBU_STATE_TBL, p->objiter.zTbl,
3518 RBU_STATE_IDX, p->objiter.zIdx,
3519 RBU_STATE_ROW, p->nStep,
3520 RBU_STATE_PROGRESS, p->nProgress,
3521 RBU_STATE_CKPT, p->iWalCksum,
3522 RBU_STATE_COOKIE, (i64)pFd->iCookie,
3523 RBU_STATE_OALSZ, p->iOalSz,
3524 RBU_STATE_PHASEONESTEP, p->nPhaseOneStep,
3525 RBU_STATE_DATATBL, p->objiter.zDataTbl
3526 )
3527 );
3528 assert( pInsert==0 || rc==SQLITE_OK );
3529
3530 if( rc==SQLITE_OK ){
3531 sqlite3_step(pInsert);
3532 rc = sqlite3_finalize(pInsert);
3533 }
3534 if( rc!=SQLITE_OK ) p->rc = rc;
3535 }
3536 }
3537
3538
3539 /*
3540 ** The second argument passed to this function is the name of a PRAGMA
3541 ** setting - "page_size", "auto_vacuum", "user_version" or "application_id".
3542 ** This function executes the following on sqlite3rbu.dbRbu:
3543 **
3544 ** "PRAGMA main.$zPragma"
3545 **
3546 ** where $zPragma is the string passed as the second argument, then
3547 ** on sqlite3rbu.dbMain:
3548 **
3549 ** "PRAGMA main.$zPragma = $val"
3550 **
3551 ** where $val is the value returned by the first PRAGMA invocation.
3552 **
3553 ** In short, it copies the value of the specified PRAGMA setting from
3554 ** dbRbu to dbMain.
3555 */
rbuCopyPragma(sqlite3rbu * p,const char * zPragma)3556 static void rbuCopyPragma(sqlite3rbu *p, const char *zPragma){
3557 if( p->rc==SQLITE_OK ){
3558 sqlite3_stmt *pPragma = 0;
3559 p->rc = prepareFreeAndCollectError(p->dbRbu, &pPragma, &p->zErrmsg,
3560 sqlite3_mprintf("PRAGMA main.%s", zPragma)
3561 );
3562 if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPragma) ){
3563 p->rc = rbuMPrintfExec(p, p->dbMain, "PRAGMA main.%s = %d",
3564 zPragma, sqlite3_column_int(pPragma, 0)
3565 );
3566 }
3567 rbuFinalize(p, pPragma);
3568 }
3569 }
3570
3571 /*
3572 ** The RBU handle passed as the only argument has just been opened and
3573 ** the state database is empty. If this RBU handle was opened for an
3574 ** RBU vacuum operation, create the schema in the target db.
3575 */
rbuCreateTargetSchema(sqlite3rbu * p)3576 static void rbuCreateTargetSchema(sqlite3rbu *p){
3577 sqlite3_stmt *pSql = 0;
3578 sqlite3_stmt *pInsert = 0;
3579
3580 assert( rbuIsVacuum(p) );
3581 p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg);
3582 if( p->rc==SQLITE_OK ){
3583 p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg,
3584 "SELECT sql FROM sqlite_schema WHERE sql!='' AND rootpage!=0"
3585 " AND name!='sqlite_sequence' "
3586 " ORDER BY type DESC"
3587 );
3588 }
3589
3590 while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){
3591 const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
3592 p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg);
3593 }
3594 rbuFinalize(p, pSql);
3595 if( p->rc!=SQLITE_OK ) return;
3596
3597 if( p->rc==SQLITE_OK ){
3598 p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg,
3599 "SELECT * FROM sqlite_schema WHERE rootpage=0 OR rootpage IS NULL"
3600 );
3601 }
3602
3603 if( p->rc==SQLITE_OK ){
3604 p->rc = prepareAndCollectError(p->dbMain, &pInsert, &p->zErrmsg,
3605 "INSERT INTO sqlite_schema VALUES(?,?,?,?,?)"
3606 );
3607 }
3608
3609 while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){
3610 int i;
3611 for(i=0; i<5; i++){
3612 sqlite3_bind_value(pInsert, i+1, sqlite3_column_value(pSql, i));
3613 }
3614 sqlite3_step(pInsert);
3615 p->rc = sqlite3_reset(pInsert);
3616 }
3617 if( p->rc==SQLITE_OK ){
3618 p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=0",0,0,&p->zErrmsg);
3619 }
3620
3621 rbuFinalize(p, pSql);
3622 rbuFinalize(p, pInsert);
3623 }
3624
3625 /*
3626 ** Step the RBU object.
3627 */
sqlite3rbu_step(sqlite3rbu * p)3628 int sqlite3rbu_step(sqlite3rbu *p){
3629 if( p ){
3630 switch( p->eStage ){
3631 case RBU_STAGE_OAL: {
3632 RbuObjIter *pIter = &p->objiter;
3633
3634 /* If this is an RBU vacuum operation and the state table was empty
3635 ** when this handle was opened, create the target database schema. */
3636 if( rbuIsVacuum(p) && p->nProgress==0 && p->rc==SQLITE_OK ){
3637 rbuCreateTargetSchema(p);
3638 rbuCopyPragma(p, "user_version");
3639 rbuCopyPragma(p, "application_id");
3640 }
3641
3642 while( p->rc==SQLITE_OK && pIter->zTbl ){
3643
3644 if( pIter->bCleanup ){
3645 /* Clean up the rbu_tmp_xxx table for the previous table. It
3646 ** cannot be dropped as there are currently active SQL statements.
3647 ** But the contents can be deleted. */
3648 if( rbuIsVacuum(p)==0 && pIter->abIndexed ){
3649 rbuMPrintfExec(p, p->dbRbu,
3650 "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl
3651 );
3652 }
3653 }else{
3654 rbuObjIterPrepareAll(p, pIter, 0);
3655
3656 /* Advance to the next row to process. */
3657 if( p->rc==SQLITE_OK ){
3658 int rc = sqlite3_step(pIter->pSelect);
3659 if( rc==SQLITE_ROW ){
3660 p->nProgress++;
3661 p->nStep++;
3662 return rbuStep(p);
3663 }
3664 p->rc = sqlite3_reset(pIter->pSelect);
3665 p->nStep = 0;
3666 }
3667 }
3668
3669 rbuObjIterNext(p, pIter);
3670 }
3671
3672 if( p->rc==SQLITE_OK ){
3673 assert( pIter->zTbl==0 );
3674 rbuSaveState(p, RBU_STAGE_MOVE);
3675 rbuIncrSchemaCookie(p);
3676 if( p->rc==SQLITE_OK ){
3677 p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg);
3678 }
3679 if( p->rc==SQLITE_OK ){
3680 p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg);
3681 }
3682 p->eStage = RBU_STAGE_MOVE;
3683 }
3684 break;
3685 }
3686
3687 case RBU_STAGE_MOVE: {
3688 if( p->rc==SQLITE_OK ){
3689 rbuMoveOalFile(p);
3690 p->nProgress++;
3691 }
3692 break;
3693 }
3694
3695 case RBU_STAGE_CKPT: {
3696 if( p->rc==SQLITE_OK ){
3697 if( p->nStep>=p->nFrame ){
3698 sqlite3_file *pDb = p->pTargetFd->pReal;
3699
3700 /* Sync the db file */
3701 p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
3702
3703 /* Update nBackfill */
3704 if( p->rc==SQLITE_OK ){
3705 void volatile *ptr;
3706 p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, &ptr);
3707 if( p->rc==SQLITE_OK ){
3708 ((u32 volatile*)ptr)[24] = p->iMaxFrame;
3709 }
3710 }
3711
3712 if( p->rc==SQLITE_OK ){
3713 p->eStage = RBU_STAGE_DONE;
3714 p->rc = SQLITE_DONE;
3715 }
3716 }else{
3717 /* At one point the following block copied a single frame from the
3718 ** wal file to the database file. So that one call to sqlite3rbu_step()
3719 ** checkpointed a single frame.
3720 **
3721 ** However, if the sector-size is larger than the page-size, and the
3722 ** application calls sqlite3rbu_savestate() or close() immediately
3723 ** after this step, then rbu_step() again, then a power failure occurs,
3724 ** then the database page written here may be damaged. Work around
3725 ** this by checkpointing frames until the next page in the aFrame[]
3726 ** lies on a different disk sector to the current one. */
3727 u32 iSector;
3728 do{
3729 RbuFrame *pFrame = &p->aFrame[p->nStep];
3730 iSector = (pFrame->iDbPage-1) / p->nPagePerSector;
3731 rbuCheckpointFrame(p, pFrame);
3732 p->nStep++;
3733 }while( p->nStep<p->nFrame
3734 && iSector==((p->aFrame[p->nStep].iDbPage-1) / p->nPagePerSector)
3735 && p->rc==SQLITE_OK
3736 );
3737 }
3738 p->nProgress++;
3739 }
3740 break;
3741 }
3742
3743 default:
3744 break;
3745 }
3746 return p->rc;
3747 }else{
3748 return SQLITE_NOMEM;
3749 }
3750 }
3751
3752 /*
3753 ** Compare strings z1 and z2, returning 0 if they are identical, or non-zero
3754 ** otherwise. Either or both argument may be NULL. Two NULL values are
3755 ** considered equal, and NULL is considered distinct from all other values.
3756 */
rbuStrCompare(const char * z1,const char * z2)3757 static int rbuStrCompare(const char *z1, const char *z2){
3758 if( z1==0 && z2==0 ) return 0;
3759 if( z1==0 || z2==0 ) return 1;
3760 return (sqlite3_stricmp(z1, z2)!=0);
3761 }
3762
3763 /*
3764 ** This function is called as part of sqlite3rbu_open() when initializing
3765 ** an rbu handle in OAL stage. If the rbu update has not started (i.e.
3766 ** the rbu_state table was empty) it is a no-op. Otherwise, it arranges
3767 ** things so that the next call to sqlite3rbu_step() continues on from
3768 ** where the previous rbu handle left off.
3769 **
3770 ** If an error occurs, an error code and error message are left in the
3771 ** rbu handle passed as the first argument.
3772 */
rbuSetupOal(sqlite3rbu * p,RbuState * pState)3773 static void rbuSetupOal(sqlite3rbu *p, RbuState *pState){
3774 assert( p->rc==SQLITE_OK );
3775 if( pState->zTbl ){
3776 RbuObjIter *pIter = &p->objiter;
3777 int rc = SQLITE_OK;
3778
3779 while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup
3780 || rbuStrCompare(pIter->zIdx, pState->zIdx)
3781 || (pState->zDataTbl==0 && rbuStrCompare(pIter->zTbl, pState->zTbl))
3782 || (pState->zDataTbl && rbuStrCompare(pIter->zDataTbl, pState->zDataTbl))
3783 )){
3784 rc = rbuObjIterNext(p, pIter);
3785 }
3786
3787 if( rc==SQLITE_OK && !pIter->zTbl ){
3788 rc = SQLITE_ERROR;
3789 p->zErrmsg = sqlite3_mprintf("rbu_state mismatch error");
3790 }
3791
3792 if( rc==SQLITE_OK ){
3793 p->nStep = pState->nRow;
3794 rc = rbuObjIterPrepareAll(p, &p->objiter, p->nStep);
3795 }
3796
3797 p->rc = rc;
3798 }
3799 }
3800
3801 /*
3802 ** If there is a "*-oal" file in the file-system corresponding to the
3803 ** target database in the file-system, delete it. If an error occurs,
3804 ** leave an error code and error message in the rbu handle.
3805 */
rbuDeleteOalFile(sqlite3rbu * p)3806 static void rbuDeleteOalFile(sqlite3rbu *p){
3807 char *zOal = rbuMPrintf(p, "%s-oal", p->zTarget);
3808 if( zOal ){
3809 sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
3810 assert( pVfs && p->rc==SQLITE_OK && p->zErrmsg==0 );
3811 pVfs->xDelete(pVfs, zOal, 0);
3812 sqlite3_free(zOal);
3813 }
3814 }
3815
3816 /*
3817 ** Allocate a private rbu VFS for the rbu handle passed as the only
3818 ** argument. This VFS will be used unless the call to sqlite3rbu_open()
3819 ** specified a URI with a vfs=? option in place of a target database
3820 ** file name.
3821 */
rbuCreateVfs(sqlite3rbu * p)3822 static void rbuCreateVfs(sqlite3rbu *p){
3823 int rnd;
3824 char zRnd[64];
3825
3826 assert( p->rc==SQLITE_OK );
3827 sqlite3_randomness(sizeof(int), (void*)&rnd);
3828 sqlite3_snprintf(sizeof(zRnd), zRnd, "rbu_vfs_%d", rnd);
3829 p->rc = sqlite3rbu_create_vfs(zRnd, 0);
3830 if( p->rc==SQLITE_OK ){
3831 sqlite3_vfs *pVfs = sqlite3_vfs_find(zRnd);
3832 assert( pVfs );
3833 p->zVfsName = pVfs->zName;
3834 ((rbu_vfs*)pVfs)->pRbu = p;
3835 }
3836 }
3837
3838 /*
3839 ** Destroy the private VFS created for the rbu handle passed as the only
3840 ** argument by an earlier call to rbuCreateVfs().
3841 */
rbuDeleteVfs(sqlite3rbu * p)3842 static void rbuDeleteVfs(sqlite3rbu *p){
3843 if( p->zVfsName ){
3844 sqlite3rbu_destroy_vfs(p->zVfsName);
3845 p->zVfsName = 0;
3846 }
3847 }
3848
3849 /*
3850 ** This user-defined SQL function is invoked with a single argument - the
3851 ** name of a table expected to appear in the target database. It returns
3852 ** the number of auxilliary indexes on the table.
3853 */
rbuIndexCntFunc(sqlite3_context * pCtx,int nVal,sqlite3_value ** apVal)3854 static void rbuIndexCntFunc(
3855 sqlite3_context *pCtx,
3856 int nVal,
3857 sqlite3_value **apVal
3858 ){
3859 sqlite3rbu *p = (sqlite3rbu*)sqlite3_user_data(pCtx);
3860 sqlite3_stmt *pStmt = 0;
3861 char *zErrmsg = 0;
3862 int rc;
3863 sqlite3 *db = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain);
3864
3865 assert( nVal==1 );
3866
3867 rc = prepareFreeAndCollectError(db, &pStmt, &zErrmsg,
3868 sqlite3_mprintf("SELECT count(*) FROM sqlite_schema "
3869 "WHERE type='index' AND tbl_name = %Q", sqlite3_value_text(apVal[0]))
3870 );
3871 if( rc!=SQLITE_OK ){
3872 sqlite3_result_error(pCtx, zErrmsg, -1);
3873 }else{
3874 int nIndex = 0;
3875 if( SQLITE_ROW==sqlite3_step(pStmt) ){
3876 nIndex = sqlite3_column_int(pStmt, 0);
3877 }
3878 rc = sqlite3_finalize(pStmt);
3879 if( rc==SQLITE_OK ){
3880 sqlite3_result_int(pCtx, nIndex);
3881 }else{
3882 sqlite3_result_error(pCtx, sqlite3_errmsg(db), -1);
3883 }
3884 }
3885
3886 sqlite3_free(zErrmsg);
3887 }
3888
3889 /*
3890 ** If the RBU database contains the rbu_count table, use it to initialize
3891 ** the sqlite3rbu.nPhaseOneStep variable. The schema of the rbu_count table
3892 ** is assumed to contain the same columns as:
3893 **
3894 ** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID;
3895 **
3896 ** There should be one row in the table for each data_xxx table in the
3897 ** database. The 'tbl' column should contain the name of a data_xxx table,
3898 ** and the cnt column the number of rows it contains.
3899 **
3900 ** sqlite3rbu.nPhaseOneStep is initialized to the sum of (1 + nIndex) * cnt
3901 ** for all rows in the rbu_count table, where nIndex is the number of
3902 ** indexes on the corresponding target database table.
3903 */
rbuInitPhaseOneSteps(sqlite3rbu * p)3904 static void rbuInitPhaseOneSteps(sqlite3rbu *p){
3905 if( p->rc==SQLITE_OK ){
3906 sqlite3_stmt *pStmt = 0;
3907 int bExists = 0; /* True if rbu_count exists */
3908
3909 p->nPhaseOneStep = -1;
3910
3911 p->rc = sqlite3_create_function(p->dbRbu,
3912 "rbu_index_cnt", 1, SQLITE_UTF8, (void*)p, rbuIndexCntFunc, 0, 0
3913 );
3914
3915 /* Check for the rbu_count table. If it does not exist, or if an error
3916 ** occurs, nPhaseOneStep will be left set to -1. */
3917 if( p->rc==SQLITE_OK ){
3918 p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
3919 "SELECT 1 FROM sqlite_schema WHERE tbl_name = 'rbu_count'"
3920 );
3921 }
3922 if( p->rc==SQLITE_OK ){
3923 if( SQLITE_ROW==sqlite3_step(pStmt) ){
3924 bExists = 1;
3925 }
3926 p->rc = sqlite3_finalize(pStmt);
3927 }
3928
3929 if( p->rc==SQLITE_OK && bExists ){
3930 p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
3931 "SELECT sum(cnt * (1 + rbu_index_cnt(rbu_target_name(tbl))))"
3932 "FROM rbu_count"
3933 );
3934 if( p->rc==SQLITE_OK ){
3935 if( SQLITE_ROW==sqlite3_step(pStmt) ){
3936 p->nPhaseOneStep = sqlite3_column_int64(pStmt, 0);
3937 }
3938 p->rc = sqlite3_finalize(pStmt);
3939 }
3940 }
3941 }
3942 }
3943
3944
openRbuHandle(const char * zTarget,const char * zRbu,const char * zState)3945 static sqlite3rbu *openRbuHandle(
3946 const char *zTarget,
3947 const char *zRbu,
3948 const char *zState
3949 ){
3950 sqlite3rbu *p;
3951 size_t nTarget = zTarget ? strlen(zTarget) : 0;
3952 size_t nRbu = strlen(zRbu);
3953 size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1;
3954
3955 p = (sqlite3rbu*)sqlite3_malloc64(nByte);
3956 if( p ){
3957 RbuState *pState = 0;
3958
3959 /* Create the custom VFS. */
3960 memset(p, 0, sizeof(sqlite3rbu));
3961 rbuCreateVfs(p);
3962
3963 /* Open the target, RBU and state databases */
3964 if( p->rc==SQLITE_OK ){
3965 char *pCsr = (char*)&p[1];
3966 int bRetry = 0;
3967 if( zTarget ){
3968 p->zTarget = pCsr;
3969 memcpy(p->zTarget, zTarget, nTarget+1);
3970 pCsr += nTarget+1;
3971 }
3972 p->zRbu = pCsr;
3973 memcpy(p->zRbu, zRbu, nRbu+1);
3974 pCsr += nRbu+1;
3975 if( zState ){
3976 p->zState = rbuMPrintf(p, "%s", zState);
3977 }
3978
3979 /* If the first attempt to open the database file fails and the bRetry
3980 ** flag it set, this means that the db was not opened because it seemed
3981 ** to be a wal-mode db. But, this may have happened due to an earlier
3982 ** RBU vacuum operation leaving an old wal file in the directory.
3983 ** If this is the case, it will have been checkpointed and deleted
3984 ** when the handle was closed and a second attempt to open the
3985 ** database may succeed. */
3986 rbuOpenDatabase(p, &bRetry);
3987 if( bRetry ){
3988 rbuOpenDatabase(p, 0);
3989 }
3990 }
3991
3992 if( p->rc==SQLITE_OK ){
3993 pState = rbuLoadState(p);
3994 assert( pState || p->rc!=SQLITE_OK );
3995 if( p->rc==SQLITE_OK ){
3996
3997 if( pState->eStage==0 ){
3998 rbuDeleteOalFile(p);
3999 rbuInitPhaseOneSteps(p);
4000 p->eStage = RBU_STAGE_OAL;
4001 }else{
4002 p->eStage = pState->eStage;
4003 p->nPhaseOneStep = pState->nPhaseOneStep;
4004 }
4005 p->nProgress = pState->nProgress;
4006 p->iOalSz = pState->iOalSz;
4007 }
4008 }
4009 assert( p->rc!=SQLITE_OK || p->eStage!=0 );
4010
4011 if( p->rc==SQLITE_OK && p->pTargetFd->pWalFd ){
4012 if( p->eStage==RBU_STAGE_OAL ){
4013 p->rc = SQLITE_ERROR;
4014 p->zErrmsg = sqlite3_mprintf("cannot update wal mode database");
4015 }else if( p->eStage==RBU_STAGE_MOVE ){
4016 p->eStage = RBU_STAGE_CKPT;
4017 p->nStep = 0;
4018 }
4019 }
4020
4021 if( p->rc==SQLITE_OK
4022 && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE)
4023 && pState->eStage!=0
4024 ){
4025 rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd);
4026 if( pFd->iCookie!=pState->iCookie ){
4027 /* At this point (pTargetFd->iCookie) contains the value of the
4028 ** change-counter cookie (the thing that gets incremented when a
4029 ** transaction is committed in rollback mode) currently stored on
4030 ** page 1 of the database file. */
4031 p->rc = SQLITE_BUSY;
4032 p->zErrmsg = sqlite3_mprintf("database modified during rbu %s",
4033 (rbuIsVacuum(p) ? "vacuum" : "update")
4034 );
4035 }
4036 }
4037
4038 if( p->rc==SQLITE_OK ){
4039 if( p->eStage==RBU_STAGE_OAL ){
4040 sqlite3 *db = p->dbMain;
4041 p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg);
4042
4043 /* Point the object iterator at the first object */
4044 if( p->rc==SQLITE_OK ){
4045 p->rc = rbuObjIterFirst(p, &p->objiter);
4046 }
4047
4048 /* If the RBU database contains no data_xxx tables, declare the RBU
4049 ** update finished. */
4050 if( p->rc==SQLITE_OK && p->objiter.zTbl==0 ){
4051 p->rc = SQLITE_DONE;
4052 p->eStage = RBU_STAGE_DONE;
4053 }else{
4054 if( p->rc==SQLITE_OK && pState->eStage==0 && rbuIsVacuum(p) ){
4055 rbuCopyPragma(p, "page_size");
4056 rbuCopyPragma(p, "auto_vacuum");
4057 }
4058
4059 /* Open transactions both databases. The *-oal file is opened or
4060 ** created at this point. */
4061 if( p->rc==SQLITE_OK ){
4062 p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
4063 }
4064
4065 /* Check if the main database is a zipvfs db. If it is, set the upper
4066 ** level pager to use "journal_mode=off". This prevents it from
4067 ** generating a large journal using a temp file. */
4068 if( p->rc==SQLITE_OK ){
4069 int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0);
4070 if( frc==SQLITE_OK ){
4071 p->rc = sqlite3_exec(
4072 db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg);
4073 }
4074 }
4075
4076 if( p->rc==SQLITE_OK ){
4077 rbuSetupOal(p, pState);
4078 }
4079 }
4080 }else if( p->eStage==RBU_STAGE_MOVE ){
4081 /* no-op */
4082 }else if( p->eStage==RBU_STAGE_CKPT ){
4083 rbuSetupCheckpoint(p, pState);
4084 }else if( p->eStage==RBU_STAGE_DONE ){
4085 p->rc = SQLITE_DONE;
4086 }else{
4087 p->rc = SQLITE_CORRUPT;
4088 }
4089 }
4090
4091 rbuFreeState(pState);
4092 }
4093
4094 return p;
4095 }
4096
4097 /*
4098 ** Allocate and return an RBU handle with all fields zeroed except for the
4099 ** error code, which is set to SQLITE_MISUSE.
4100 */
rbuMisuseError(void)4101 static sqlite3rbu *rbuMisuseError(void){
4102 sqlite3rbu *pRet;
4103 pRet = sqlite3_malloc64(sizeof(sqlite3rbu));
4104 if( pRet ){
4105 memset(pRet, 0, sizeof(sqlite3rbu));
4106 pRet->rc = SQLITE_MISUSE;
4107 }
4108 return pRet;
4109 }
4110
4111 /*
4112 ** Open and return a new RBU handle.
4113 */
sqlite3rbu_open(const char * zTarget,const char * zRbu,const char * zState)4114 sqlite3rbu *sqlite3rbu_open(
4115 const char *zTarget,
4116 const char *zRbu,
4117 const char *zState
4118 ){
4119 if( zTarget==0 || zRbu==0 ){ return rbuMisuseError(); }
4120 /* TODO: Check that zTarget and zRbu are non-NULL */
4121 return openRbuHandle(zTarget, zRbu, zState);
4122 }
4123
4124 /*
4125 ** Open a handle to begin or resume an RBU VACUUM operation.
4126 */
sqlite3rbu_vacuum(const char * zTarget,const char * zState)4127 sqlite3rbu *sqlite3rbu_vacuum(
4128 const char *zTarget,
4129 const char *zState
4130 ){
4131 if( zTarget==0 ){ return rbuMisuseError(); }
4132 if( zState ){
4133 int n = strlen(zState);
4134 if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){
4135 return rbuMisuseError();
4136 }
4137 }
4138 /* TODO: Check that both arguments are non-NULL */
4139 return openRbuHandle(0, zTarget, zState);
4140 }
4141
4142 /*
4143 ** Return the database handle used by pRbu.
4144 */
sqlite3rbu_db(sqlite3rbu * pRbu,int bRbu)4145 sqlite3 *sqlite3rbu_db(sqlite3rbu *pRbu, int bRbu){
4146 sqlite3 *db = 0;
4147 if( pRbu ){
4148 db = (bRbu ? pRbu->dbRbu : pRbu->dbMain);
4149 }
4150 return db;
4151 }
4152
4153
4154 /*
4155 ** If the error code currently stored in the RBU handle is SQLITE_CONSTRAINT,
4156 ** then edit any error message string so as to remove all occurrences of
4157 ** the pattern "rbu_imp_[0-9]*".
4158 */
rbuEditErrmsg(sqlite3rbu * p)4159 static void rbuEditErrmsg(sqlite3rbu *p){
4160 if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){
4161 unsigned int i;
4162 size_t nErrmsg = strlen(p->zErrmsg);
4163 for(i=0; i<(nErrmsg-8); i++){
4164 if( memcmp(&p->zErrmsg[i], "rbu_imp_", 8)==0 ){
4165 int nDel = 8;
4166 while( p->zErrmsg[i+nDel]>='0' && p->zErrmsg[i+nDel]<='9' ) nDel++;
4167 memmove(&p->zErrmsg[i], &p->zErrmsg[i+nDel], nErrmsg + 1 - i - nDel);
4168 nErrmsg -= nDel;
4169 }
4170 }
4171 }
4172 }
4173
4174 /*
4175 ** Close the RBU handle.
4176 */
sqlite3rbu_close(sqlite3rbu * p,char ** pzErrmsg)4177 int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){
4178 int rc;
4179 if( p ){
4180
4181 /* Commit the transaction to the *-oal file. */
4182 if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
4183 p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg);
4184 }
4185
4186 /* Sync the db file if currently doing an incremental checkpoint */
4187 if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){
4188 sqlite3_file *pDb = p->pTargetFd->pReal;
4189 p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
4190 }
4191
4192 rbuSaveState(p, p->eStage);
4193
4194 if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
4195 p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg);
4196 }
4197
4198 /* Close any open statement handles. */
4199 rbuObjIterFinalize(&p->objiter);
4200
4201 /* If this is an RBU vacuum handle and the vacuum has either finished
4202 ** successfully or encountered an error, delete the contents of the
4203 ** state table. This causes the next call to sqlite3rbu_vacuum()
4204 ** specifying the current target and state databases to start a new
4205 ** vacuum from scratch. */
4206 if( rbuIsVacuum(p) && p->rc!=SQLITE_OK && p->dbRbu ){
4207 int rc2 = sqlite3_exec(p->dbRbu, "DELETE FROM stat.rbu_state", 0, 0, 0);
4208 if( p->rc==SQLITE_DONE && rc2!=SQLITE_OK ) p->rc = rc2;
4209 }
4210
4211 /* Close the open database handle and VFS object. */
4212 sqlite3_close(p->dbRbu);
4213 sqlite3_close(p->dbMain);
4214 assert( p->szTemp==0 );
4215 rbuDeleteVfs(p);
4216 sqlite3_free(p->aBuf);
4217 sqlite3_free(p->aFrame);
4218
4219 rbuEditErrmsg(p);
4220 rc = p->rc;
4221 if( pzErrmsg ){
4222 *pzErrmsg = p->zErrmsg;
4223 }else{
4224 sqlite3_free(p->zErrmsg);
4225 }
4226 sqlite3_free(p->zState);
4227 sqlite3_free(p);
4228 }else{
4229 rc = SQLITE_NOMEM;
4230 *pzErrmsg = 0;
4231 }
4232 return rc;
4233 }
4234
4235 /*
4236 ** Return the total number of key-value operations (inserts, deletes or
4237 ** updates) that have been performed on the target database since the
4238 ** current RBU update was started.
4239 */
sqlite3rbu_progress(sqlite3rbu * pRbu)4240 sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){
4241 return pRbu->nProgress;
4242 }
4243
4244 /*
4245 ** Return permyriadage progress indications for the two main stages of
4246 ** an RBU update.
4247 */
sqlite3rbu_bp_progress(sqlite3rbu * p,int * pnOne,int * pnTwo)4248 void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){
4249 const int MAX_PROGRESS = 10000;
4250 switch( p->eStage ){
4251 case RBU_STAGE_OAL:
4252 if( p->nPhaseOneStep>0 ){
4253 *pnOne = (int)(MAX_PROGRESS * (i64)p->nProgress/(i64)p->nPhaseOneStep);
4254 }else{
4255 *pnOne = -1;
4256 }
4257 *pnTwo = 0;
4258 break;
4259
4260 case RBU_STAGE_MOVE:
4261 *pnOne = MAX_PROGRESS;
4262 *pnTwo = 0;
4263 break;
4264
4265 case RBU_STAGE_CKPT:
4266 *pnOne = MAX_PROGRESS;
4267 *pnTwo = (int)(MAX_PROGRESS * (i64)p->nStep / (i64)p->nFrame);
4268 break;
4269
4270 case RBU_STAGE_DONE:
4271 *pnOne = MAX_PROGRESS;
4272 *pnTwo = MAX_PROGRESS;
4273 break;
4274
4275 default:
4276 assert( 0 );
4277 }
4278 }
4279
4280 /*
4281 ** Return the current state of the RBU vacuum or update operation.
4282 */
sqlite3rbu_state(sqlite3rbu * p)4283 int sqlite3rbu_state(sqlite3rbu *p){
4284 int aRes[] = {
4285 0, SQLITE_RBU_STATE_OAL, SQLITE_RBU_STATE_MOVE,
4286 0, SQLITE_RBU_STATE_CHECKPOINT, SQLITE_RBU_STATE_DONE
4287 };
4288
4289 assert( RBU_STAGE_OAL==1 );
4290 assert( RBU_STAGE_MOVE==2 );
4291 assert( RBU_STAGE_CKPT==4 );
4292 assert( RBU_STAGE_DONE==5 );
4293 assert( aRes[RBU_STAGE_OAL]==SQLITE_RBU_STATE_OAL );
4294 assert( aRes[RBU_STAGE_MOVE]==SQLITE_RBU_STATE_MOVE );
4295 assert( aRes[RBU_STAGE_CKPT]==SQLITE_RBU_STATE_CHECKPOINT );
4296 assert( aRes[RBU_STAGE_DONE]==SQLITE_RBU_STATE_DONE );
4297
4298 if( p->rc!=SQLITE_OK && p->rc!=SQLITE_DONE ){
4299 return SQLITE_RBU_STATE_ERROR;
4300 }else{
4301 assert( p->rc!=SQLITE_DONE || p->eStage==RBU_STAGE_DONE );
4302 assert( p->eStage==RBU_STAGE_OAL
4303 || p->eStage==RBU_STAGE_MOVE
4304 || p->eStage==RBU_STAGE_CKPT
4305 || p->eStage==RBU_STAGE_DONE
4306 );
4307 return aRes[p->eStage];
4308 }
4309 }
4310
sqlite3rbu_savestate(sqlite3rbu * p)4311 int sqlite3rbu_savestate(sqlite3rbu *p){
4312 int rc = p->rc;
4313 if( rc==SQLITE_DONE ) return SQLITE_OK;
4314
4315 assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE );
4316 if( p->eStage==RBU_STAGE_OAL ){
4317 assert( rc!=SQLITE_DONE );
4318 if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0);
4319 }
4320
4321 /* Sync the db file */
4322 if( rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){
4323 sqlite3_file *pDb = p->pTargetFd->pReal;
4324 rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
4325 }
4326
4327 p->rc = rc;
4328 rbuSaveState(p, p->eStage);
4329 rc = p->rc;
4330
4331 if( p->eStage==RBU_STAGE_OAL ){
4332 assert( rc!=SQLITE_DONE );
4333 if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0);
4334 if( rc==SQLITE_OK ){
4335 const char *zBegin = rbuIsVacuum(p) ? "BEGIN" : "BEGIN IMMEDIATE";
4336 rc = sqlite3_exec(p->dbRbu, zBegin, 0, 0, 0);
4337 }
4338 if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "BEGIN IMMEDIATE", 0, 0,0);
4339 }
4340
4341 p->rc = rc;
4342 return rc;
4343 }
4344
4345 /**************************************************************************
4346 ** Beginning of RBU VFS shim methods. The VFS shim modifies the behaviour
4347 ** of a standard VFS in the following ways:
4348 **
4349 ** 1. Whenever the first page of a main database file is read or
4350 ** written, the value of the change-counter cookie is stored in
4351 ** rbu_file.iCookie. Similarly, the value of the "write-version"
4352 ** database header field is stored in rbu_file.iWriteVer. This ensures
4353 ** that the values are always trustworthy within an open transaction.
4354 **
4355 ** 2. Whenever an SQLITE_OPEN_WAL file is opened, the (rbu_file.pWalFd)
4356 ** member variable of the associated database file descriptor is set
4357 ** to point to the new file. A mutex protected linked list of all main
4358 ** db fds opened using a particular RBU VFS is maintained at
4359 ** rbu_vfs.pMain to facilitate this.
4360 **
4361 ** 3. Using a new file-control "SQLITE_FCNTL_RBU", a main db rbu_file
4362 ** object can be marked as the target database of an RBU update. This
4363 ** turns on the following extra special behaviour:
4364 **
4365 ** 3a. If xAccess() is called to check if there exists a *-wal file
4366 ** associated with an RBU target database currently in RBU_STAGE_OAL
4367 ** stage (preparing the *-oal file), the following special handling
4368 ** applies:
4369 **
4370 ** * if the *-wal file does exist, return SQLITE_CANTOPEN. An RBU
4371 ** target database may not be in wal mode already.
4372 **
4373 ** * if the *-wal file does not exist, set the output parameter to
4374 ** non-zero (to tell SQLite that it does exist) anyway.
4375 **
4376 ** Then, when xOpen() is called to open the *-wal file associated with
4377 ** the RBU target in RBU_STAGE_OAL stage, instead of opening the *-wal
4378 ** file, the rbu vfs opens the corresponding *-oal file instead.
4379 **
4380 ** 3b. The *-shm pages returned by xShmMap() for a target db file in
4381 ** RBU_STAGE_OAL mode are actually stored in heap memory. This is to
4382 ** avoid creating a *-shm file on disk. Additionally, xShmLock() calls
4383 ** are no-ops on target database files in RBU_STAGE_OAL mode. This is
4384 ** because assert() statements in some VFS implementations fail if
4385 ** xShmLock() is called before xShmMap().
4386 **
4387 ** 3c. If an EXCLUSIVE lock is attempted on a target database file in any
4388 ** mode except RBU_STAGE_DONE (all work completed and checkpointed), it
4389 ** fails with an SQLITE_BUSY error. This is to stop RBU connections
4390 ** from automatically checkpointing a *-wal (or *-oal) file from within
4391 ** sqlite3_close().
4392 **
4393 ** 3d. In RBU_STAGE_CAPTURE mode, all xRead() calls on the wal file, and
4394 ** all xWrite() calls on the target database file perform no IO.
4395 ** Instead the frame and page numbers that would be read and written
4396 ** are recorded. Additionally, successful attempts to obtain exclusive
4397 ** xShmLock() WRITER, CHECKPOINTER and READ0 locks on the target
4398 ** database file are recorded. xShmLock() calls to unlock the same
4399 ** locks are no-ops (so that once obtained, these locks are never
4400 ** relinquished). Finally, calls to xSync() on the target database
4401 ** file fail with SQLITE_INTERNAL errors.
4402 */
4403
rbuUnlockShm(rbu_file * p)4404 static void rbuUnlockShm(rbu_file *p){
4405 assert( p->openFlags & SQLITE_OPEN_MAIN_DB );
4406 if( p->pRbu ){
4407 int (*xShmLock)(sqlite3_file*,int,int,int) = p->pReal->pMethods->xShmLock;
4408 int i;
4409 for(i=0; i<SQLITE_SHM_NLOCK;i++){
4410 if( (1<<i) & p->pRbu->mLock ){
4411 xShmLock(p->pReal, i, 1, SQLITE_SHM_UNLOCK|SQLITE_SHM_EXCLUSIVE);
4412 }
4413 }
4414 p->pRbu->mLock = 0;
4415 }
4416 }
4417
4418 /*
4419 */
rbuUpdateTempSize(rbu_file * pFd,sqlite3_int64 nNew)4420 static int rbuUpdateTempSize(rbu_file *pFd, sqlite3_int64 nNew){
4421 sqlite3rbu *pRbu = pFd->pRbu;
4422 i64 nDiff = nNew - pFd->sz;
4423 pRbu->szTemp += nDiff;
4424 pFd->sz = nNew;
4425 assert( pRbu->szTemp>=0 );
4426 if( pRbu->szTempLimit && pRbu->szTemp>pRbu->szTempLimit ) return SQLITE_FULL;
4427 return SQLITE_OK;
4428 }
4429
4430 /*
4431 ** Add an item to the main-db lists, if it is not already present.
4432 **
4433 ** There are two main-db lists. One for all file descriptors, and one
4434 ** for all file descriptors with rbu_file.pDb!=0. If the argument has
4435 ** rbu_file.pDb!=0, then it is assumed to already be present on the
4436 ** main list and is only added to the pDb!=0 list.
4437 */
rbuMainlistAdd(rbu_file * p)4438 static void rbuMainlistAdd(rbu_file *p){
4439 rbu_vfs *pRbuVfs = p->pRbuVfs;
4440 rbu_file *pIter;
4441 assert( (p->openFlags & SQLITE_OPEN_MAIN_DB) );
4442 sqlite3_mutex_enter(pRbuVfs->mutex);
4443 if( p->pRbu==0 ){
4444 for(pIter=pRbuVfs->pMain; pIter; pIter=pIter->pMainNext);
4445 p->pMainNext = pRbuVfs->pMain;
4446 pRbuVfs->pMain = p;
4447 }else{
4448 for(pIter=pRbuVfs->pMainRbu; pIter && pIter!=p; pIter=pIter->pMainRbuNext){}
4449 if( pIter==0 ){
4450 p->pMainRbuNext = pRbuVfs->pMainRbu;
4451 pRbuVfs->pMainRbu = p;
4452 }
4453 }
4454 sqlite3_mutex_leave(pRbuVfs->mutex);
4455 }
4456
4457 /*
4458 ** Remove an item from the main-db lists.
4459 */
rbuMainlistRemove(rbu_file * p)4460 static void rbuMainlistRemove(rbu_file *p){
4461 rbu_file **pp;
4462 sqlite3_mutex_enter(p->pRbuVfs->mutex);
4463 for(pp=&p->pRbuVfs->pMain; *pp && *pp!=p; pp=&((*pp)->pMainNext)){}
4464 if( *pp ) *pp = p->pMainNext;
4465 p->pMainNext = 0;
4466 for(pp=&p->pRbuVfs->pMainRbu; *pp && *pp!=p; pp=&((*pp)->pMainRbuNext)){}
4467 if( *pp ) *pp = p->pMainRbuNext;
4468 p->pMainRbuNext = 0;
4469 sqlite3_mutex_leave(p->pRbuVfs->mutex);
4470 }
4471
4472 /*
4473 ** Given that zWal points to a buffer containing a wal file name passed to
4474 ** either the xOpen() or xAccess() VFS method, search the main-db list for
4475 ** a file-handle opened by the same database connection on the corresponding
4476 ** database file.
4477 **
4478 ** If parameter bRbu is true, only search for file-descriptors with
4479 ** rbu_file.pDb!=0.
4480 */
rbuFindMaindb(rbu_vfs * pRbuVfs,const char * zWal,int bRbu)4481 static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal, int bRbu){
4482 rbu_file *pDb;
4483 sqlite3_mutex_enter(pRbuVfs->mutex);
4484 if( bRbu ){
4485 for(pDb=pRbuVfs->pMainRbu; pDb && pDb->zWal!=zWal; pDb=pDb->pMainRbuNext){}
4486 }else{
4487 for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){}
4488 }
4489 sqlite3_mutex_leave(pRbuVfs->mutex);
4490 return pDb;
4491 }
4492
4493 /*
4494 ** Close an rbu file.
4495 */
rbuVfsClose(sqlite3_file * pFile)4496 static int rbuVfsClose(sqlite3_file *pFile){
4497 rbu_file *p = (rbu_file*)pFile;
4498 int rc;
4499 int i;
4500
4501 /* Free the contents of the apShm[] array. And the array itself. */
4502 for(i=0; i<p->nShm; i++){
4503 sqlite3_free(p->apShm[i]);
4504 }
4505 sqlite3_free(p->apShm);
4506 p->apShm = 0;
4507 sqlite3_free(p->zDel);
4508
4509 if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
4510 rbuMainlistRemove(p);
4511 rbuUnlockShm(p);
4512 p->pReal->pMethods->xShmUnmap(p->pReal, 0);
4513 }
4514 else if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){
4515 rbuUpdateTempSize(p, 0);
4516 }
4517 assert( p->pMainNext==0 && p->pRbuVfs->pMain!=p );
4518
4519 /* Close the underlying file handle */
4520 rc = p->pReal->pMethods->xClose(p->pReal);
4521 return rc;
4522 }
4523
4524
4525 /*
4526 ** Read and return an unsigned 32-bit big-endian integer from the buffer
4527 ** passed as the only argument.
4528 */
rbuGetU32(u8 * aBuf)4529 static u32 rbuGetU32(u8 *aBuf){
4530 return ((u32)aBuf[0] << 24)
4531 + ((u32)aBuf[1] << 16)
4532 + ((u32)aBuf[2] << 8)
4533 + ((u32)aBuf[3]);
4534 }
4535
4536 /*
4537 ** Write an unsigned 32-bit value in big-endian format to the supplied
4538 ** buffer.
4539 */
rbuPutU32(u8 * aBuf,u32 iVal)4540 static void rbuPutU32(u8 *aBuf, u32 iVal){
4541 aBuf[0] = (iVal >> 24) & 0xFF;
4542 aBuf[1] = (iVal >> 16) & 0xFF;
4543 aBuf[2] = (iVal >> 8) & 0xFF;
4544 aBuf[3] = (iVal >> 0) & 0xFF;
4545 }
4546
rbuPutU16(u8 * aBuf,u16 iVal)4547 static void rbuPutU16(u8 *aBuf, u16 iVal){
4548 aBuf[0] = (iVal >> 8) & 0xFF;
4549 aBuf[1] = (iVal >> 0) & 0xFF;
4550 }
4551
4552 /*
4553 ** Read data from an rbuVfs-file.
4554 */
rbuVfsRead(sqlite3_file * pFile,void * zBuf,int iAmt,sqlite_int64 iOfst)4555 static int rbuVfsRead(
4556 sqlite3_file *pFile,
4557 void *zBuf,
4558 int iAmt,
4559 sqlite_int64 iOfst
4560 ){
4561 rbu_file *p = (rbu_file*)pFile;
4562 sqlite3rbu *pRbu = p->pRbu;
4563 int rc;
4564
4565 if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){
4566 assert( p->openFlags & SQLITE_OPEN_WAL );
4567 rc = rbuCaptureWalRead(p->pRbu, iOfst, iAmt);
4568 }else{
4569 if( pRbu && pRbu->eStage==RBU_STAGE_OAL
4570 && (p->openFlags & SQLITE_OPEN_WAL)
4571 && iOfst>=pRbu->iOalSz
4572 ){
4573 rc = SQLITE_OK;
4574 memset(zBuf, 0, iAmt);
4575 }else{
4576 rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst);
4577 #if 1
4578 /* If this is being called to read the first page of the target
4579 ** database as part of an rbu vacuum operation, synthesize the
4580 ** contents of the first page if it does not yet exist. Otherwise,
4581 ** SQLite will not check for a *-wal file. */
4582 if( pRbu && rbuIsVacuum(pRbu)
4583 && rc==SQLITE_IOERR_SHORT_READ && iOfst==0
4584 && (p->openFlags & SQLITE_OPEN_MAIN_DB)
4585 && pRbu->rc==SQLITE_OK
4586 ){
4587 sqlite3_file *pFd = (sqlite3_file*)pRbu->pRbuFd;
4588 rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst);
4589 if( rc==SQLITE_OK ){
4590 u8 *aBuf = (u8*)zBuf;
4591 u32 iRoot = rbuGetU32(&aBuf[52]) ? 1 : 0;
4592 rbuPutU32(&aBuf[52], iRoot); /* largest root page number */
4593 rbuPutU32(&aBuf[36], 0); /* number of free pages */
4594 rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */
4595 rbuPutU32(&aBuf[28], 1); /* size of db file in pages */
4596 rbuPutU32(&aBuf[24], pRbu->pRbuFd->iCookie+1); /* Change counter */
4597
4598 if( iAmt>100 ){
4599 memset(&aBuf[100], 0, iAmt-100);
4600 rbuPutU16(&aBuf[105], iAmt & 0xFFFF);
4601 aBuf[100] = 0x0D;
4602 }
4603 }
4604 }
4605 #endif
4606 }
4607 if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
4608 /* These look like magic numbers. But they are stable, as they are part
4609 ** of the definition of the SQLite file format, which may not change. */
4610 u8 *pBuf = (u8*)zBuf;
4611 p->iCookie = rbuGetU32(&pBuf[24]);
4612 p->iWriteVer = pBuf[19];
4613 }
4614 }
4615 return rc;
4616 }
4617
4618 /*
4619 ** Write data to an rbuVfs-file.
4620 */
rbuVfsWrite(sqlite3_file * pFile,const void * zBuf,int iAmt,sqlite_int64 iOfst)4621 static int rbuVfsWrite(
4622 sqlite3_file *pFile,
4623 const void *zBuf,
4624 int iAmt,
4625 sqlite_int64 iOfst
4626 ){
4627 rbu_file *p = (rbu_file*)pFile;
4628 sqlite3rbu *pRbu = p->pRbu;
4629 int rc;
4630
4631 if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){
4632 assert( p->openFlags & SQLITE_OPEN_MAIN_DB );
4633 rc = rbuCaptureDbWrite(p->pRbu, iOfst);
4634 }else{
4635 if( pRbu ){
4636 if( pRbu->eStage==RBU_STAGE_OAL
4637 && (p->openFlags & SQLITE_OPEN_WAL)
4638 && iOfst>=pRbu->iOalSz
4639 ){
4640 pRbu->iOalSz = iAmt + iOfst;
4641 }else if( p->openFlags & SQLITE_OPEN_DELETEONCLOSE ){
4642 i64 szNew = iAmt+iOfst;
4643 if( szNew>p->sz ){
4644 rc = rbuUpdateTempSize(p, szNew);
4645 if( rc!=SQLITE_OK ) return rc;
4646 }
4647 }
4648 }
4649 rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst);
4650 if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
4651 /* These look like magic numbers. But they are stable, as they are part
4652 ** of the definition of the SQLite file format, which may not change. */
4653 u8 *pBuf = (u8*)zBuf;
4654 p->iCookie = rbuGetU32(&pBuf[24]);
4655 p->iWriteVer = pBuf[19];
4656 }
4657 }
4658 return rc;
4659 }
4660
4661 /*
4662 ** Truncate an rbuVfs-file.
4663 */
rbuVfsTruncate(sqlite3_file * pFile,sqlite_int64 size)4664 static int rbuVfsTruncate(sqlite3_file *pFile, sqlite_int64 size){
4665 rbu_file *p = (rbu_file*)pFile;
4666 if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){
4667 int rc = rbuUpdateTempSize(p, size);
4668 if( rc!=SQLITE_OK ) return rc;
4669 }
4670 return p->pReal->pMethods->xTruncate(p->pReal, size);
4671 }
4672
4673 /*
4674 ** Sync an rbuVfs-file.
4675 */
rbuVfsSync(sqlite3_file * pFile,int flags)4676 static int rbuVfsSync(sqlite3_file *pFile, int flags){
4677 rbu_file *p = (rbu_file *)pFile;
4678 if( p->pRbu && p->pRbu->eStage==RBU_STAGE_CAPTURE ){
4679 if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
4680 return SQLITE_INTERNAL;
4681 }
4682 return SQLITE_OK;
4683 }
4684 return p->pReal->pMethods->xSync(p->pReal, flags);
4685 }
4686
4687 /*
4688 ** Return the current file-size of an rbuVfs-file.
4689 */
rbuVfsFileSize(sqlite3_file * pFile,sqlite_int64 * pSize)4690 static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
4691 rbu_file *p = (rbu_file *)pFile;
4692 int rc;
4693 rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
4694
4695 /* If this is an RBU vacuum operation and this is the target database,
4696 ** pretend that it has at least one page. Otherwise, SQLite will not
4697 ** check for the existance of a *-wal file. rbuVfsRead() contains
4698 ** similar logic. */
4699 if( rc==SQLITE_OK && *pSize==0
4700 && p->pRbu && rbuIsVacuum(p->pRbu)
4701 && (p->openFlags & SQLITE_OPEN_MAIN_DB)
4702 ){
4703 *pSize = 1024;
4704 }
4705 return rc;
4706 }
4707
4708 /*
4709 ** Lock an rbuVfs-file.
4710 */
rbuVfsLock(sqlite3_file * pFile,int eLock)4711 static int rbuVfsLock(sqlite3_file *pFile, int eLock){
4712 rbu_file *p = (rbu_file*)pFile;
4713 sqlite3rbu *pRbu = p->pRbu;
4714 int rc = SQLITE_OK;
4715
4716 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
4717 if( eLock==SQLITE_LOCK_EXCLUSIVE
4718 && (p->bNolock || (pRbu && pRbu->eStage!=RBU_STAGE_DONE))
4719 ){
4720 /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this
4721 ** prevents it from checkpointing the database from sqlite3_close(). */
4722 rc = SQLITE_BUSY;
4723 }else{
4724 rc = p->pReal->pMethods->xLock(p->pReal, eLock);
4725 }
4726
4727 return rc;
4728 }
4729
4730 /*
4731 ** Unlock an rbuVfs-file.
4732 */
rbuVfsUnlock(sqlite3_file * pFile,int eLock)4733 static int rbuVfsUnlock(sqlite3_file *pFile, int eLock){
4734 rbu_file *p = (rbu_file *)pFile;
4735 return p->pReal->pMethods->xUnlock(p->pReal, eLock);
4736 }
4737
4738 /*
4739 ** Check if another file-handle holds a RESERVED lock on an rbuVfs-file.
4740 */
rbuVfsCheckReservedLock(sqlite3_file * pFile,int * pResOut)4741 static int rbuVfsCheckReservedLock(sqlite3_file *pFile, int *pResOut){
4742 rbu_file *p = (rbu_file *)pFile;
4743 return p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut);
4744 }
4745
4746 /*
4747 ** File control method. For custom operations on an rbuVfs-file.
4748 */
rbuVfsFileControl(sqlite3_file * pFile,int op,void * pArg)4749 static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){
4750 rbu_file *p = (rbu_file *)pFile;
4751 int (*xControl)(sqlite3_file*,int,void*) = p->pReal->pMethods->xFileControl;
4752 int rc;
4753
4754 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB)
4755 || p->openFlags & (SQLITE_OPEN_TRANSIENT_DB|SQLITE_OPEN_TEMP_JOURNAL)
4756 );
4757 if( op==SQLITE_FCNTL_RBU ){
4758 sqlite3rbu *pRbu = (sqlite3rbu*)pArg;
4759
4760 /* First try to find another RBU vfs lower down in the vfs stack. If
4761 ** one is found, this vfs will operate in pass-through mode. The lower
4762 ** level vfs will do the special RBU handling. */
4763 rc = xControl(p->pReal, op, pArg);
4764
4765 if( rc==SQLITE_NOTFOUND ){
4766 /* Now search for a zipvfs instance lower down in the VFS stack. If
4767 ** one is found, this is an error. */
4768 void *dummy = 0;
4769 rc = xControl(p->pReal, SQLITE_FCNTL_ZIPVFS, &dummy);
4770 if( rc==SQLITE_OK ){
4771 rc = SQLITE_ERROR;
4772 pRbu->zErrmsg = sqlite3_mprintf("rbu/zipvfs setup error");
4773 }else if( rc==SQLITE_NOTFOUND ){
4774 pRbu->pTargetFd = p;
4775 p->pRbu = pRbu;
4776 rbuMainlistAdd(p);
4777 if( p->pWalFd ) p->pWalFd->pRbu = pRbu;
4778 rc = SQLITE_OK;
4779 }
4780 }
4781 return rc;
4782 }
4783 else if( op==SQLITE_FCNTL_RBUCNT ){
4784 sqlite3rbu *pRbu = (sqlite3rbu*)pArg;
4785 pRbu->nRbu++;
4786 pRbu->pRbuFd = p;
4787 p->bNolock = 1;
4788 }
4789
4790 rc = xControl(p->pReal, op, pArg);
4791 if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
4792 rbu_vfs *pRbuVfs = p->pRbuVfs;
4793 char *zIn = *(char**)pArg;
4794 char *zOut = sqlite3_mprintf("rbu(%s)/%z", pRbuVfs->base.zName, zIn);
4795 *(char**)pArg = zOut;
4796 if( zOut==0 ) rc = SQLITE_NOMEM;
4797 }
4798
4799 return rc;
4800 }
4801
4802 /*
4803 ** Return the sector-size in bytes for an rbuVfs-file.
4804 */
rbuVfsSectorSize(sqlite3_file * pFile)4805 static int rbuVfsSectorSize(sqlite3_file *pFile){
4806 rbu_file *p = (rbu_file *)pFile;
4807 return p->pReal->pMethods->xSectorSize(p->pReal);
4808 }
4809
4810 /*
4811 ** Return the device characteristic flags supported by an rbuVfs-file.
4812 */
rbuVfsDeviceCharacteristics(sqlite3_file * pFile)4813 static int rbuVfsDeviceCharacteristics(sqlite3_file *pFile){
4814 rbu_file *p = (rbu_file *)pFile;
4815 return p->pReal->pMethods->xDeviceCharacteristics(p->pReal);
4816 }
4817
4818 /*
4819 ** Take or release a shared-memory lock.
4820 */
rbuVfsShmLock(sqlite3_file * pFile,int ofst,int n,int flags)4821 static int rbuVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){
4822 rbu_file *p = (rbu_file*)pFile;
4823 sqlite3rbu *pRbu = p->pRbu;
4824 int rc = SQLITE_OK;
4825
4826 #ifdef SQLITE_AMALGAMATION
4827 assert( WAL_CKPT_LOCK==1 );
4828 #endif
4829
4830 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
4831 if( pRbu && (
4832 pRbu->eStage==RBU_STAGE_OAL
4833 || pRbu->eStage==RBU_STAGE_MOVE
4834 || pRbu->eStage==RBU_STAGE_DONE
4835 )){
4836 /* Prevent SQLite from taking a shm-lock on the target file when it
4837 ** is supplying heap memory to the upper layer in place of *-shm
4838 ** segments. */
4839 if( ofst==WAL_LOCK_CKPT && n==1 ) rc = SQLITE_BUSY;
4840 }else{
4841 int bCapture = 0;
4842 if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){
4843 bCapture = 1;
4844 }
4845 if( bCapture==0 || 0==(flags & SQLITE_SHM_UNLOCK) ){
4846 rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags);
4847 if( bCapture && rc==SQLITE_OK ){
4848 pRbu->mLock |= ((1<<n) - 1) << ofst;
4849 }
4850 }
4851 }
4852
4853 return rc;
4854 }
4855
4856 /*
4857 ** Obtain a pointer to a mapping of a single 32KiB page of the *-shm file.
4858 */
rbuVfsShmMap(sqlite3_file * pFile,int iRegion,int szRegion,int isWrite,void volatile ** pp)4859 static int rbuVfsShmMap(
4860 sqlite3_file *pFile,
4861 int iRegion,
4862 int szRegion,
4863 int isWrite,
4864 void volatile **pp
4865 ){
4866 rbu_file *p = (rbu_file*)pFile;
4867 int rc = SQLITE_OK;
4868 int eStage = (p->pRbu ? p->pRbu->eStage : 0);
4869
4870 /* If not in RBU_STAGE_OAL, allow this call to pass through. Or, if this
4871 ** rbu is in the RBU_STAGE_OAL state, use heap memory for *-shm space
4872 ** instead of a file on disk. */
4873 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
4874 if( eStage==RBU_STAGE_OAL ){
4875 sqlite3_int64 nByte = (iRegion+1) * sizeof(char*);
4876 char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte);
4877
4878 /* This is an RBU connection that uses its own heap memory for the
4879 ** pages of the *-shm file. Since no other process can have run
4880 ** recovery, the connection must request *-shm pages in order
4881 ** from start to finish. */
4882 assert( iRegion==p->nShm );
4883 if( apNew==0 ){
4884 rc = SQLITE_NOMEM;
4885 }else{
4886 memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm));
4887 p->apShm = apNew;
4888 p->nShm = iRegion+1;
4889 }
4890
4891 if( rc==SQLITE_OK ){
4892 char *pNew = (char*)sqlite3_malloc64(szRegion);
4893 if( pNew==0 ){
4894 rc = SQLITE_NOMEM;
4895 }else{
4896 memset(pNew, 0, szRegion);
4897 p->apShm[iRegion] = pNew;
4898 }
4899 }
4900
4901 if( rc==SQLITE_OK ){
4902 *pp = p->apShm[iRegion];
4903 }else{
4904 *pp = 0;
4905 }
4906 }else{
4907 assert( p->apShm==0 );
4908 rc = p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp);
4909 }
4910
4911 return rc;
4912 }
4913
4914 /*
4915 ** Memory barrier.
4916 */
rbuVfsShmBarrier(sqlite3_file * pFile)4917 static void rbuVfsShmBarrier(sqlite3_file *pFile){
4918 rbu_file *p = (rbu_file *)pFile;
4919 p->pReal->pMethods->xShmBarrier(p->pReal);
4920 }
4921
4922 /*
4923 ** The xShmUnmap method.
4924 */
rbuVfsShmUnmap(sqlite3_file * pFile,int delFlag)4925 static int rbuVfsShmUnmap(sqlite3_file *pFile, int delFlag){
4926 rbu_file *p = (rbu_file*)pFile;
4927 int rc = SQLITE_OK;
4928 int eStage = (p->pRbu ? p->pRbu->eStage : 0);
4929
4930 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
4931 if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){
4932 /* no-op */
4933 }else{
4934 /* Release the checkpointer and writer locks */
4935 rbuUnlockShm(p);
4936 rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag);
4937 }
4938 return rc;
4939 }
4940
4941 /*
4942 ** Open an rbu file handle.
4943 */
rbuVfsOpen(sqlite3_vfs * pVfs,const char * zName,sqlite3_file * pFile,int flags,int * pOutFlags)4944 static int rbuVfsOpen(
4945 sqlite3_vfs *pVfs,
4946 const char *zName,
4947 sqlite3_file *pFile,
4948 int flags,
4949 int *pOutFlags
4950 ){
4951 static sqlite3_io_methods rbuvfs_io_methods = {
4952 2, /* iVersion */
4953 rbuVfsClose, /* xClose */
4954 rbuVfsRead, /* xRead */
4955 rbuVfsWrite, /* xWrite */
4956 rbuVfsTruncate, /* xTruncate */
4957 rbuVfsSync, /* xSync */
4958 rbuVfsFileSize, /* xFileSize */
4959 rbuVfsLock, /* xLock */
4960 rbuVfsUnlock, /* xUnlock */
4961 rbuVfsCheckReservedLock, /* xCheckReservedLock */
4962 rbuVfsFileControl, /* xFileControl */
4963 rbuVfsSectorSize, /* xSectorSize */
4964 rbuVfsDeviceCharacteristics, /* xDeviceCharacteristics */
4965 rbuVfsShmMap, /* xShmMap */
4966 rbuVfsShmLock, /* xShmLock */
4967 rbuVfsShmBarrier, /* xShmBarrier */
4968 rbuVfsShmUnmap, /* xShmUnmap */
4969 0, 0 /* xFetch, xUnfetch */
4970 };
4971 rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs;
4972 sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs;
4973 rbu_file *pFd = (rbu_file *)pFile;
4974 int rc = SQLITE_OK;
4975 const char *zOpen = zName;
4976 int oflags = flags;
4977
4978 memset(pFd, 0, sizeof(rbu_file));
4979 pFd->pReal = (sqlite3_file*)&pFd[1];
4980 pFd->pRbuVfs = pRbuVfs;
4981 pFd->openFlags = flags;
4982 if( zName ){
4983 if( flags & SQLITE_OPEN_MAIN_DB ){
4984 /* A main database has just been opened. The following block sets
4985 ** (pFd->zWal) to point to a buffer owned by SQLite that contains
4986 ** the name of the *-wal file this db connection will use. SQLite
4987 ** happens to pass a pointer to this buffer when using xAccess()
4988 ** or xOpen() to operate on the *-wal file. */
4989 pFd->zWal = sqlite3_filename_wal(zName);
4990 }
4991 else if( flags & SQLITE_OPEN_WAL ){
4992 rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName, 0);
4993 if( pDb ){
4994 if( pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
4995 /* This call is to open a *-wal file. Intead, open the *-oal. This
4996 ** code ensures that the string passed to xOpen() is terminated by a
4997 ** pair of '\0' bytes in case the VFS attempts to extract a URI
4998 ** parameter from it. */
4999 const char *zBase = zName;
5000 size_t nCopy;
5001 char *zCopy;
5002 if( rbuIsVacuum(pDb->pRbu) ){
5003 zBase = sqlite3_db_filename(pDb->pRbu->dbRbu, "main");
5004 zBase = sqlite3_filename_wal(zBase);
5005 }
5006 nCopy = strlen(zBase);
5007 zCopy = sqlite3_malloc64(nCopy+2);
5008 if( zCopy ){
5009 memcpy(zCopy, zBase, nCopy);
5010 zCopy[nCopy-3] = 'o';
5011 zCopy[nCopy] = '\0';
5012 zCopy[nCopy+1] = '\0';
5013 zOpen = (const char*)(pFd->zDel = zCopy);
5014 }else{
5015 rc = SQLITE_NOMEM;
5016 }
5017 pFd->pRbu = pDb->pRbu;
5018 }
5019 pDb->pWalFd = pFd;
5020 }
5021 }
5022 }else{
5023 pFd->pRbu = pRbuVfs->pRbu;
5024 }
5025
5026 if( oflags & SQLITE_OPEN_MAIN_DB
5027 && sqlite3_uri_boolean(zName, "rbu_memory", 0)
5028 ){
5029 assert( oflags & SQLITE_OPEN_MAIN_DB );
5030 oflags = SQLITE_OPEN_TEMP_DB | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
5031 SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE;
5032 zOpen = 0;
5033 }
5034
5035 if( rc==SQLITE_OK ){
5036 rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, oflags, pOutFlags);
5037 }
5038 if( pFd->pReal->pMethods ){
5039 /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods
5040 ** pointer and, if the file is a main database file, link it into the
5041 ** mutex protected linked list of all such files. */
5042 pFile->pMethods = &rbuvfs_io_methods;
5043 if( flags & SQLITE_OPEN_MAIN_DB ){
5044 rbuMainlistAdd(pFd);
5045 }
5046 }else{
5047 sqlite3_free(pFd->zDel);
5048 }
5049
5050 return rc;
5051 }
5052
5053 /*
5054 ** Delete the file located at zPath.
5055 */
rbuVfsDelete(sqlite3_vfs * pVfs,const char * zPath,int dirSync)5056 static int rbuVfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
5057 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5058 return pRealVfs->xDelete(pRealVfs, zPath, dirSync);
5059 }
5060
5061 /*
5062 ** Test for access permissions. Return true if the requested permission
5063 ** is available, or false otherwise.
5064 */
rbuVfsAccess(sqlite3_vfs * pVfs,const char * zPath,int flags,int * pResOut)5065 static int rbuVfsAccess(
5066 sqlite3_vfs *pVfs,
5067 const char *zPath,
5068 int flags,
5069 int *pResOut
5070 ){
5071 rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs;
5072 sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs;
5073 int rc;
5074
5075 rc = pRealVfs->xAccess(pRealVfs, zPath, flags, pResOut);
5076
5077 /* If this call is to check if a *-wal file associated with an RBU target
5078 ** database connection exists, and the RBU update is in RBU_STAGE_OAL,
5079 ** the following special handling is activated:
5080 **
5081 ** a) if the *-wal file does exist, return SQLITE_CANTOPEN. This
5082 ** ensures that the RBU extension never tries to update a database
5083 ** in wal mode, even if the first page of the database file has
5084 ** been damaged.
5085 **
5086 ** b) if the *-wal file does not exist, claim that it does anyway,
5087 ** causing SQLite to call xOpen() to open it. This call will also
5088 ** be intercepted (see the rbuVfsOpen() function) and the *-oal
5089 ** file opened instead.
5090 */
5091 if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS ){
5092 rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath, 1);
5093 if( pDb && pDb->pRbu->eStage==RBU_STAGE_OAL ){
5094 assert( pDb->pRbu );
5095 if( *pResOut ){
5096 rc = SQLITE_CANTOPEN;
5097 }else{
5098 sqlite3_int64 sz = 0;
5099 rc = rbuVfsFileSize(&pDb->base, &sz);
5100 *pResOut = (sz>0);
5101 }
5102 }
5103 }
5104
5105 return rc;
5106 }
5107
5108 /*
5109 ** Populate buffer zOut with the full canonical pathname corresponding
5110 ** to the pathname in zPath. zOut is guaranteed to point to a buffer
5111 ** of at least (DEVSYM_MAX_PATHNAME+1) bytes.
5112 */
rbuVfsFullPathname(sqlite3_vfs * pVfs,const char * zPath,int nOut,char * zOut)5113 static int rbuVfsFullPathname(
5114 sqlite3_vfs *pVfs,
5115 const char *zPath,
5116 int nOut,
5117 char *zOut
5118 ){
5119 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5120 return pRealVfs->xFullPathname(pRealVfs, zPath, nOut, zOut);
5121 }
5122
5123 #ifndef SQLITE_OMIT_LOAD_EXTENSION
5124 /*
5125 ** Open the dynamic library located at zPath and return a handle.
5126 */
rbuVfsDlOpen(sqlite3_vfs * pVfs,const char * zPath)5127 static void *rbuVfsDlOpen(sqlite3_vfs *pVfs, const char *zPath){
5128 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5129 return pRealVfs->xDlOpen(pRealVfs, zPath);
5130 }
5131
5132 /*
5133 ** Populate the buffer zErrMsg (size nByte bytes) with a human readable
5134 ** utf-8 string describing the most recent error encountered associated
5135 ** with dynamic libraries.
5136 */
rbuVfsDlError(sqlite3_vfs * pVfs,int nByte,char * zErrMsg)5137 static void rbuVfsDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
5138 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5139 pRealVfs->xDlError(pRealVfs, nByte, zErrMsg);
5140 }
5141
5142 /*
5143 ** Return a pointer to the symbol zSymbol in the dynamic library pHandle.
5144 */
rbuVfsDlSym(sqlite3_vfs * pVfs,void * pArg,const char * zSym)5145 static void (*rbuVfsDlSym(
5146 sqlite3_vfs *pVfs,
5147 void *pArg,
5148 const char *zSym
5149 ))(void){
5150 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5151 return pRealVfs->xDlSym(pRealVfs, pArg, zSym);
5152 }
5153
5154 /*
5155 ** Close the dynamic library handle pHandle.
5156 */
rbuVfsDlClose(sqlite3_vfs * pVfs,void * pHandle)5157 static void rbuVfsDlClose(sqlite3_vfs *pVfs, void *pHandle){
5158 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5159 pRealVfs->xDlClose(pRealVfs, pHandle);
5160 }
5161 #endif /* SQLITE_OMIT_LOAD_EXTENSION */
5162
5163 /*
5164 ** Populate the buffer pointed to by zBufOut with nByte bytes of
5165 ** random data.
5166 */
rbuVfsRandomness(sqlite3_vfs * pVfs,int nByte,char * zBufOut)5167 static int rbuVfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
5168 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5169 return pRealVfs->xRandomness(pRealVfs, nByte, zBufOut);
5170 }
5171
5172 /*
5173 ** Sleep for nMicro microseconds. Return the number of microseconds
5174 ** actually slept.
5175 */
rbuVfsSleep(sqlite3_vfs * pVfs,int nMicro)5176 static int rbuVfsSleep(sqlite3_vfs *pVfs, int nMicro){
5177 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5178 return pRealVfs->xSleep(pRealVfs, nMicro);
5179 }
5180
5181 /*
5182 ** Return the current time as a Julian Day number in *pTimeOut.
5183 */
rbuVfsCurrentTime(sqlite3_vfs * pVfs,double * pTimeOut)5184 static int rbuVfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
5185 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
5186 return pRealVfs->xCurrentTime(pRealVfs, pTimeOut);
5187 }
5188
5189 /*
5190 ** No-op.
5191 */
rbuVfsGetLastError(sqlite3_vfs * pVfs,int a,char * b)5192 static int rbuVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){
5193 return 0;
5194 }
5195
5196 /*
5197 ** Deregister and destroy an RBU vfs created by an earlier call to
5198 ** sqlite3rbu_create_vfs().
5199 */
sqlite3rbu_destroy_vfs(const char * zName)5200 void sqlite3rbu_destroy_vfs(const char *zName){
5201 sqlite3_vfs *pVfs = sqlite3_vfs_find(zName);
5202 if( pVfs && pVfs->xOpen==rbuVfsOpen ){
5203 sqlite3_mutex_free(((rbu_vfs*)pVfs)->mutex);
5204 sqlite3_vfs_unregister(pVfs);
5205 sqlite3_free(pVfs);
5206 }
5207 }
5208
5209 /*
5210 ** Create an RBU VFS named zName that accesses the underlying file-system
5211 ** via existing VFS zParent. The new object is registered as a non-default
5212 ** VFS with SQLite before returning.
5213 */
sqlite3rbu_create_vfs(const char * zName,const char * zParent)5214 int sqlite3rbu_create_vfs(const char *zName, const char *zParent){
5215
5216 /* Template for VFS */
5217 static sqlite3_vfs vfs_template = {
5218 1, /* iVersion */
5219 0, /* szOsFile */
5220 0, /* mxPathname */
5221 0, /* pNext */
5222 0, /* zName */
5223 0, /* pAppData */
5224 rbuVfsOpen, /* xOpen */
5225 rbuVfsDelete, /* xDelete */
5226 rbuVfsAccess, /* xAccess */
5227 rbuVfsFullPathname, /* xFullPathname */
5228
5229 #ifndef SQLITE_OMIT_LOAD_EXTENSION
5230 rbuVfsDlOpen, /* xDlOpen */
5231 rbuVfsDlError, /* xDlError */
5232 rbuVfsDlSym, /* xDlSym */
5233 rbuVfsDlClose, /* xDlClose */
5234 #else
5235 0, 0, 0, 0,
5236 #endif
5237
5238 rbuVfsRandomness, /* xRandomness */
5239 rbuVfsSleep, /* xSleep */
5240 rbuVfsCurrentTime, /* xCurrentTime */
5241 rbuVfsGetLastError, /* xGetLastError */
5242 0, /* xCurrentTimeInt64 (version 2) */
5243 0, 0, 0 /* Unimplemented version 3 methods */
5244 };
5245
5246 rbu_vfs *pNew = 0; /* Newly allocated VFS */
5247 int rc = SQLITE_OK;
5248 size_t nName;
5249 size_t nByte;
5250
5251 nName = strlen(zName);
5252 nByte = sizeof(rbu_vfs) + nName + 1;
5253 pNew = (rbu_vfs*)sqlite3_malloc64(nByte);
5254 if( pNew==0 ){
5255 rc = SQLITE_NOMEM;
5256 }else{
5257 sqlite3_vfs *pParent; /* Parent VFS */
5258 memset(pNew, 0, nByte);
5259 pParent = sqlite3_vfs_find(zParent);
5260 if( pParent==0 ){
5261 rc = SQLITE_NOTFOUND;
5262 }else{
5263 char *zSpace;
5264 memcpy(&pNew->base, &vfs_template, sizeof(sqlite3_vfs));
5265 pNew->base.mxPathname = pParent->mxPathname;
5266 pNew->base.szOsFile = sizeof(rbu_file) + pParent->szOsFile;
5267 pNew->pRealVfs = pParent;
5268 pNew->base.zName = (const char*)(zSpace = (char*)&pNew[1]);
5269 memcpy(zSpace, zName, nName);
5270
5271 /* Allocate the mutex and register the new VFS (not as the default) */
5272 pNew->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE);
5273 if( pNew->mutex==0 ){
5274 rc = SQLITE_NOMEM;
5275 }else{
5276 rc = sqlite3_vfs_register(&pNew->base, 0);
5277 }
5278 }
5279
5280 if( rc!=SQLITE_OK ){
5281 sqlite3_mutex_free(pNew->mutex);
5282 sqlite3_free(pNew);
5283 }
5284 }
5285
5286 return rc;
5287 }
5288
5289 /*
5290 ** Configure the aggregate temp file size limit for this RBU handle.
5291 */
sqlite3rbu_temp_size_limit(sqlite3rbu * pRbu,sqlite3_int64 n)5292 sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu *pRbu, sqlite3_int64 n){
5293 if( n>=0 ){
5294 pRbu->szTempLimit = n;
5295 }
5296 return pRbu->szTempLimit;
5297 }
5298
sqlite3rbu_temp_size(sqlite3rbu * pRbu)5299 sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu *pRbu){
5300 return pRbu->szTemp;
5301 }
5302
5303
5304 /**************************************************************************/
5305
5306 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) */
5307