1 /*
2 ** 2014 August 30
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 *************************************************************************
12 **
13 **
14 ** OVERVIEW
15 **
16 ** The RBU extension requires that the RBU update be packaged as an
17 ** SQLite database. The tables it expects to find are described in
18 ** sqlite3rbu.h. Essentially, for each table xyz in the target database
19 ** that the user wishes to write to, a corresponding data_xyz table is
20 ** created in the RBU database and populated with one row for each row to
21 ** update, insert or delete from the target table.
22 **
23 ** The update proceeds in three stages:
24 **
25 ** 1) The database is updated. The modified database pages are written
26 ** to a *-oal file. A *-oal file is just like a *-wal file, except
27 ** that it is named "<database>-oal" instead of "<database>-wal".
28 ** Because regular SQLite clients do not look for file named
29 ** "<database>-oal", they go on using the original database in
30 ** rollback mode while the *-oal file is being generated.
31 **
32 ** During this stage RBU does not update the database by writing
33 ** directly to the target tables. Instead it creates "imposter"
34 ** tables using the SQLITE_TESTCTRL_IMPOSTER interface that it uses
35 ** to update each b-tree individually. All updates required by each
36 ** b-tree are completed before moving on to the next, and all
37 ** updates are done in sorted key order.
38 **
39 ** 2) The "<database>-oal" file is moved to the equivalent "<database>-wal"
40 ** location using a call to rename(2). Before doing this the RBU
41 ** module takes an EXCLUSIVE lock on the database file, ensuring
42 ** that there are no other active readers.
43 **
44 ** Once the EXCLUSIVE lock is released, any other database readers
45 ** detect the new *-wal file and read the database in wal mode. At
46 ** this point they see the new version of the database - including
47 ** the updates made as part of the RBU update.
48 **
49 ** 3) The new *-wal file is checkpointed. This proceeds in the same way
50 ** as a regular database checkpoint, except that a single frame is
51 ** checkpointed each time sqlite3rbu_step() is called. If the RBU
52 ** handle is closed before the entire *-wal file is checkpointed,
53 ** the checkpoint progress is saved in the RBU database and the
54 ** checkpoint can be resumed by another RBU client at some point in
55 ** the future.
56 **
57 ** POTENTIAL PROBLEMS
58 **
59 ** The rename() call might not be portable. And RBU is not currently
60 ** syncing the directory after renaming the file.
61 **
62 ** When state is saved, any commit to the *-oal file and the commit to
63 ** the RBU update database are not atomic. So if the power fails at the
64 ** wrong moment they might get out of sync. As the main database will be
65 ** committed before the RBU update database this will likely either just
66 ** pass unnoticed, or result in SQLITE_CONSTRAINT errors (due to UNIQUE
67 ** constraint violations).
68 **
69 ** If some client does modify the target database mid RBU update, or some
70 ** other error occurs, the RBU extension will keep throwing errors. It's
71 ** not really clear how to get out of this state. The system could just
72 ** by delete the RBU update database and *-oal file and have the device
73 ** download the update again and start over.
74 **
75 ** At present, for an UPDATE, both the new.* and old.* records are
76 ** collected in the rbu_xyz table. And for both UPDATEs and DELETEs all
77 ** fields are collected. This means we're probably writing a lot more
78 ** data to disk when saving the state of an ongoing update to the RBU
79 ** update database than is strictly necessary.
80 **
81 */
82
83 #include <assert.h>
84 #include <string.h>
85 #include <stdio.h>
86
87 #include "sqlite3.h"
88
89 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU)
90 #include "sqlite3rbu.h"
91
92 #if defined(_WIN32_WCE)
93 #include "windows.h"
94 #endif
95
96 /* Maximum number of prepared UPDATE statements held by this module */
97 #define SQLITE_RBU_UPDATE_CACHESIZE 16
98
99 /*
100 ** Swap two objects of type TYPE.
101 */
102 #if !defined(SQLITE_AMALGAMATION)
103 # define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;}
104 #endif
105
106 /*
107 ** The rbu_state table is used to save the state of a partially applied
108 ** update so that it can be resumed later. The table consists of integer
109 ** keys mapped to values as follows:
110 **
111 ** RBU_STATE_STAGE:
112 ** May be set to integer values 1, 2, 4 or 5. As follows:
113 ** 1: the *-rbu file is currently under construction.
114 ** 2: the *-rbu file has been constructed, but not yet moved
115 ** to the *-wal path.
116 ** 4: the checkpoint is underway.
117 ** 5: the rbu update has been checkpointed.
118 **
119 ** RBU_STATE_TBL:
120 ** Only valid if STAGE==1. The target database name of the table
121 ** currently being written.
122 **
123 ** RBU_STATE_IDX:
124 ** Only valid if STAGE==1. The target database name of the index
125 ** currently being written, or NULL if the main table is currently being
126 ** updated.
127 **
128 ** RBU_STATE_ROW:
129 ** Only valid if STAGE==1. Number of rows already processed for the current
130 ** table/index.
131 **
132 ** RBU_STATE_PROGRESS:
133 ** Trbul number of sqlite3rbu_step() calls made so far as part of this
134 ** rbu update.
135 **
136 ** RBU_STATE_CKPT:
137 ** Valid if STAGE==4. The 64-bit checksum associated with the wal-index
138 ** header created by recovering the *-wal file. This is used to detect
139 ** cases when another client appends frames to the *-wal file in the
140 ** middle of an incremental checkpoint (an incremental checkpoint cannot
141 ** be continued if this happens).
142 **
143 ** RBU_STATE_COOKIE:
144 ** Valid if STAGE==1. The current change-counter cookie value in the
145 ** target db file.
146 **
147 ** RBU_STATE_OALSZ:
148 ** Valid if STAGE==1. The size in bytes of the *-oal file.
149 */
150 #define RBU_STATE_STAGE 1
151 #define RBU_STATE_TBL 2
152 #define RBU_STATE_IDX 3
153 #define RBU_STATE_ROW 4
154 #define RBU_STATE_PROGRESS 5
155 #define RBU_STATE_CKPT 6
156 #define RBU_STATE_COOKIE 7
157 #define RBU_STATE_OALSZ 8
158 #define RBU_STATE_PHASEONESTEP 9
159
160 #define RBU_STAGE_OAL 1
161 #define RBU_STAGE_MOVE 2
162 #define RBU_STAGE_CAPTURE 3
163 #define RBU_STAGE_CKPT 4
164 #define RBU_STAGE_DONE 5
165
166
167 #define RBU_CREATE_STATE \
168 "CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)"
169
170 typedef struct RbuFrame RbuFrame;
171 typedef struct RbuObjIter RbuObjIter;
172 typedef struct RbuState RbuState;
173 typedef struct rbu_vfs rbu_vfs;
174 typedef struct rbu_file rbu_file;
175 typedef struct RbuUpdateStmt RbuUpdateStmt;
176
177 #if !defined(SQLITE_AMALGAMATION)
178 typedef unsigned int u32;
179 typedef unsigned short u16;
180 typedef unsigned char u8;
181 typedef sqlite3_int64 i64;
182 #endif
183
184 /*
185 ** These values must match the values defined in wal.c for the equivalent
186 ** locks. These are not magic numbers as they are part of the SQLite file
187 ** format.
188 */
189 #define WAL_LOCK_WRITE 0
190 #define WAL_LOCK_CKPT 1
191 #define WAL_LOCK_READ0 3
192
193 #define SQLITE_FCNTL_RBUCNT 5149216
194
195 /*
196 ** A structure to store values read from the rbu_state table in memory.
197 */
198 struct RbuState {
199 int eStage;
200 char *zTbl;
201 char *zIdx;
202 i64 iWalCksum;
203 int nRow;
204 i64 nProgress;
205 u32 iCookie;
206 i64 iOalSz;
207 i64 nPhaseOneStep;
208 };
209
210 struct RbuUpdateStmt {
211 char *zMask; /* Copy of update mask used with pUpdate */
212 sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */
213 RbuUpdateStmt *pNext;
214 };
215
216 /*
217 ** An iterator of this type is used to iterate through all objects in
218 ** the target database that require updating. For each such table, the
219 ** iterator visits, in order:
220 **
221 ** * the table itself,
222 ** * each index of the table (zero or more points to visit), and
223 ** * a special "cleanup table" state.
224 **
225 ** abIndexed:
226 ** If the table has no indexes on it, abIndexed is set to NULL. Otherwise,
227 ** it points to an array of flags nTblCol elements in size. The flag is
228 ** set for each column that is either a part of the PK or a part of an
229 ** index. Or clear otherwise.
230 **
231 */
232 struct RbuObjIter {
233 sqlite3_stmt *pTblIter; /* Iterate through tables */
234 sqlite3_stmt *pIdxIter; /* Index iterator */
235 int nTblCol; /* Size of azTblCol[] array */
236 char **azTblCol; /* Array of unquoted target column names */
237 char **azTblType; /* Array of target column types */
238 int *aiSrcOrder; /* src table col -> target table col */
239 u8 *abTblPk; /* Array of flags, set on target PK columns */
240 u8 *abNotNull; /* Array of flags, set on NOT NULL columns */
241 u8 *abIndexed; /* Array of flags, set on indexed & PK cols */
242 int eType; /* Table type - an RBU_PK_XXX value */
243
244 /* Output variables. zTbl==0 implies EOF. */
245 int bCleanup; /* True in "cleanup" state */
246 const char *zTbl; /* Name of target db table */
247 const char *zDataTbl; /* Name of rbu db table (or null) */
248 const char *zIdx; /* Name of target db index (or null) */
249 int iTnum; /* Root page of current object */
250 int iPkTnum; /* If eType==EXTERNAL, root of PK index */
251 int bUnique; /* Current index is unique */
252 int nIndex; /* Number of aux. indexes on table zTbl */
253
254 /* Statements created by rbuObjIterPrepareAll() */
255 int nCol; /* Number of columns in current object */
256 sqlite3_stmt *pSelect; /* Source data */
257 sqlite3_stmt *pInsert; /* Statement for INSERT operations */
258 sqlite3_stmt *pDelete; /* Statement for DELETE ops */
259 sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */
260
261 /* Last UPDATE used (for PK b-tree updates only), or NULL. */
262 RbuUpdateStmt *pRbuUpdate;
263 };
264
265 /*
266 ** Values for RbuObjIter.eType
267 **
268 ** 0: Table does not exist (error)
269 ** 1: Table has an implicit rowid.
270 ** 2: Table has an explicit IPK column.
271 ** 3: Table has an external PK index.
272 ** 4: Table is WITHOUT ROWID.
273 ** 5: Table is a virtual table.
274 */
275 #define RBU_PK_NOTABLE 0
276 #define RBU_PK_NONE 1
277 #define RBU_PK_IPK 2
278 #define RBU_PK_EXTERNAL 3
279 #define RBU_PK_WITHOUT_ROWID 4
280 #define RBU_PK_VTAB 5
281
282
283 /*
284 ** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs
285 ** one of the following operations.
286 */
287 #define RBU_INSERT 1 /* Insert on a main table b-tree */
288 #define RBU_DELETE 2 /* Delete a row from a main table b-tree */
289 #define RBU_REPLACE 3 /* Delete and then insert a row */
290 #define RBU_IDX_DELETE 4 /* Delete a row from an aux. index b-tree */
291 #define RBU_IDX_INSERT 5 /* Insert on an aux. index b-tree */
292
293 #define RBU_UPDATE 6 /* Update a row in a main table b-tree */
294
295 /*
296 ** A single step of an incremental checkpoint - frame iWalFrame of the wal
297 ** file should be copied to page iDbPage of the database file.
298 */
299 struct RbuFrame {
300 u32 iDbPage;
301 u32 iWalFrame;
302 };
303
304 /*
305 ** RBU handle.
306 **
307 ** nPhaseOneStep:
308 ** If the RBU database contains an rbu_count table, this value is set to
309 ** a running estimate of the number of b-tree operations required to
310 ** finish populating the *-oal file. This allows the sqlite3_bp_progress()
311 ** API to calculate the permyriadage progress of populating the *-oal file
312 ** using the formula:
313 **
314 ** permyriadage = (10000 * nProgress) / nPhaseOneStep
315 **
316 ** nPhaseOneStep is initialized to the sum of:
317 **
318 ** nRow * (nIndex + 1)
319 **
320 ** for all source tables in the RBU database, where nRow is the number
321 ** of rows in the source table and nIndex the number of indexes on the
322 ** corresponding target database table.
323 **
324 ** This estimate is accurate if the RBU update consists entirely of
325 ** INSERT operations. However, it is inaccurate if:
326 **
327 ** * the RBU update contains any UPDATE operations. If the PK specified
328 ** for an UPDATE operation does not exist in the target table, then
329 ** no b-tree operations are required on index b-trees. Or if the
330 ** specified PK does exist, then (nIndex*2) such operations are
331 ** required (one delete and one insert on each index b-tree).
332 **
333 ** * the RBU update contains any DELETE operations for which the specified
334 ** PK does not exist. In this case no operations are required on index
335 ** b-trees.
336 **
337 ** * the RBU update contains REPLACE operations. These are similar to
338 ** UPDATE operations.
339 **
340 ** nPhaseOneStep is updated to account for the conditions above during the
341 ** first pass of each source table. The updated nPhaseOneStep value is
342 ** stored in the rbu_state table if the RBU update is suspended.
343 */
344 struct sqlite3rbu {
345 int eStage; /* Value of RBU_STATE_STAGE field */
346 sqlite3 *dbMain; /* target database handle */
347 sqlite3 *dbRbu; /* rbu database handle */
348 char *zTarget; /* Path to target db */
349 char *zRbu; /* Path to rbu db */
350 char *zState; /* Path to state db (or NULL if zRbu) */
351 char zStateDb[5]; /* Db name for state ("stat" or "main") */
352 int rc; /* Value returned by last rbu_step() call */
353 char *zErrmsg; /* Error message if rc!=SQLITE_OK */
354 int nStep; /* Rows processed for current object */
355 int nProgress; /* Rows processed for all objects */
356 RbuObjIter objiter; /* Iterator for skipping through tbl/idx */
357 const char *zVfsName; /* Name of automatically created rbu vfs */
358 rbu_file *pTargetFd; /* File handle open on target db */
359 int nPagePerSector; /* Pages per sector for pTargetFd */
360 i64 iOalSz;
361 i64 nPhaseOneStep;
362
363 /* The following state variables are used as part of the incremental
364 ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding
365 ** function rbuSetupCheckpoint() for details. */
366 u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */
367 u32 mLock;
368 int nFrame; /* Entries in aFrame[] array */
369 int nFrameAlloc; /* Allocated size of aFrame[] array */
370 RbuFrame *aFrame;
371 int pgsz;
372 u8 *aBuf;
373 i64 iWalCksum;
374
375 /* Used in RBU vacuum mode only */
376 int nRbu; /* Number of RBU VFS in the stack */
377 rbu_file *pRbuFd; /* Fd for main db of dbRbu */
378 };
379
380 /*
381 ** An rbu VFS is implemented using an instance of this structure.
382 */
383 struct rbu_vfs {
384 sqlite3_vfs base; /* rbu VFS shim methods */
385 sqlite3_vfs *pRealVfs; /* Underlying VFS */
386 sqlite3_mutex *mutex; /* Mutex to protect pMain */
387 rbu_file *pMain; /* Linked list of main db files */
388 };
389
390 /*
391 ** Each file opened by an rbu VFS is represented by an instance of
392 ** the following structure.
393 */
394 struct rbu_file {
395 sqlite3_file base; /* sqlite3_file methods */
396 sqlite3_file *pReal; /* Underlying file handle */
397 rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */
398 sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */
399
400 int openFlags; /* Flags this file was opened with */
401 u32 iCookie; /* Cookie value for main db files */
402 u8 iWriteVer; /* "write-version" value for main db files */
403 u8 bNolock; /* True to fail EXCLUSIVE locks */
404
405 int nShm; /* Number of entries in apShm[] array */
406 char **apShm; /* Array of mmap'd *-shm regions */
407 char *zDel; /* Delete this when closing file */
408
409 const char *zWal; /* Wal filename for this main db file */
410 rbu_file *pWalFd; /* Wal file descriptor for this main db */
411 rbu_file *pMainNext; /* Next MAIN_DB file */
412 };
413
414 /*
415 ** True for an RBU vacuum handle, or false otherwise.
416 */
417 #define rbuIsVacuum(p) ((p)->zTarget==0)
418
419
420 /*************************************************************************
421 ** The following three functions, found below:
422 **
423 ** rbuDeltaGetInt()
424 ** rbuDeltaChecksum()
425 ** rbuDeltaApply()
426 **
427 ** are lifted from the fossil source code (http://fossil-scm.org). They
428 ** are used to implement the scalar SQL function rbu_fossil_delta().
429 */
430
431 /*
432 ** Read bytes from *pz and convert them into a positive integer. When
433 ** finished, leave *pz pointing to the first character past the end of
434 ** the integer. The *pLen parameter holds the length of the string
435 ** in *pz and is decremented once for each character in the integer.
436 */
rbuDeltaGetInt(const char ** pz,int * pLen)437 static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){
438 static const signed char zValue[] = {
439 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
440 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
441 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
442 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1,
443 -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
444 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36,
445 -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
446 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1,
447 };
448 unsigned int v = 0;
449 int c;
450 unsigned char *z = (unsigned char*)*pz;
451 unsigned char *zStart = z;
452 while( (c = zValue[0x7f&*(z++)])>=0 ){
453 v = (v<<6) + c;
454 }
455 z--;
456 *pLen -= z - zStart;
457 *pz = (char*)z;
458 return v;
459 }
460
461 /*
462 ** Compute a 32-bit checksum on the N-byte buffer. Return the result.
463 */
rbuDeltaChecksum(const char * zIn,size_t N)464 static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){
465 const unsigned char *z = (const unsigned char *)zIn;
466 unsigned sum0 = 0;
467 unsigned sum1 = 0;
468 unsigned sum2 = 0;
469 unsigned sum3 = 0;
470 while(N >= 16){
471 sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]);
472 sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]);
473 sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]);
474 sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]);
475 z += 16;
476 N -= 16;
477 }
478 while(N >= 4){
479 sum0 += z[0];
480 sum1 += z[1];
481 sum2 += z[2];
482 sum3 += z[3];
483 z += 4;
484 N -= 4;
485 }
486 sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24);
487 switch(N){
488 case 3: sum3 += (z[2] << 8);
489 case 2: sum3 += (z[1] << 16);
490 case 1: sum3 += (z[0] << 24);
491 default: ;
492 }
493 return sum3;
494 }
495
496 /*
497 ** Apply a delta.
498 **
499 ** The output buffer should be big enough to hold the whole output
500 ** file and a NUL terminator at the end. The delta_output_size()
501 ** routine will determine this size for you.
502 **
503 ** The delta string should be null-terminated. But the delta string
504 ** may contain embedded NUL characters (if the input and output are
505 ** binary files) so we also have to pass in the length of the delta in
506 ** the lenDelta parameter.
507 **
508 ** This function returns the size of the output file in bytes (excluding
509 ** the final NUL terminator character). Except, if the delta string is
510 ** malformed or intended for use with a source file other than zSrc,
511 ** then this routine returns -1.
512 **
513 ** Refer to the delta_create() documentation above for a description
514 ** of the delta file format.
515 */
rbuDeltaApply(const char * zSrc,int lenSrc,const char * zDelta,int lenDelta,char * zOut)516 static int rbuDeltaApply(
517 const char *zSrc, /* The source or pattern file */
518 int lenSrc, /* Length of the source file */
519 const char *zDelta, /* Delta to apply to the pattern */
520 int lenDelta, /* Length of the delta */
521 char *zOut /* Write the output into this preallocated buffer */
522 ){
523 unsigned int limit;
524 unsigned int total = 0;
525 #ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST
526 char *zOrigOut = zOut;
527 #endif
528
529 limit = rbuDeltaGetInt(&zDelta, &lenDelta);
530 if( *zDelta!='\n' ){
531 /* ERROR: size integer not terminated by "\n" */
532 return -1;
533 }
534 zDelta++; lenDelta--;
535 while( *zDelta && lenDelta>0 ){
536 unsigned int cnt, ofst;
537 cnt = rbuDeltaGetInt(&zDelta, &lenDelta);
538 switch( zDelta[0] ){
539 case '@': {
540 zDelta++; lenDelta--;
541 ofst = rbuDeltaGetInt(&zDelta, &lenDelta);
542 if( lenDelta>0 && zDelta[0]!=',' ){
543 /* ERROR: copy command not terminated by ',' */
544 return -1;
545 }
546 zDelta++; lenDelta--;
547 total += cnt;
548 if( total>limit ){
549 /* ERROR: copy exceeds output file size */
550 return -1;
551 }
552 if( (int)(ofst+cnt) > lenSrc ){
553 /* ERROR: copy extends past end of input */
554 return -1;
555 }
556 memcpy(zOut, &zSrc[ofst], cnt);
557 zOut += cnt;
558 break;
559 }
560 case ':': {
561 zDelta++; lenDelta--;
562 total += cnt;
563 if( total>limit ){
564 /* ERROR: insert command gives an output larger than predicted */
565 return -1;
566 }
567 if( (int)cnt>lenDelta ){
568 /* ERROR: insert count exceeds size of delta */
569 return -1;
570 }
571 memcpy(zOut, zDelta, cnt);
572 zOut += cnt;
573 zDelta += cnt;
574 lenDelta -= cnt;
575 break;
576 }
577 case ';': {
578 zDelta++; lenDelta--;
579 zOut[0] = 0;
580 #ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST
581 if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){
582 /* ERROR: bad checksum */
583 return -1;
584 }
585 #endif
586 if( total!=limit ){
587 /* ERROR: generated size does not match predicted size */
588 return -1;
589 }
590 return total;
591 }
592 default: {
593 /* ERROR: unknown delta operator */
594 return -1;
595 }
596 }
597 }
598 /* ERROR: unterminated delta */
599 return -1;
600 }
601
rbuDeltaOutputSize(const char * zDelta,int lenDelta)602 static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){
603 int size;
604 size = rbuDeltaGetInt(&zDelta, &lenDelta);
605 if( *zDelta!='\n' ){
606 /* ERROR: size integer not terminated by "\n" */
607 return -1;
608 }
609 return size;
610 }
611
612 /*
613 ** End of code taken from fossil.
614 *************************************************************************/
615
616 /*
617 ** Implementation of SQL scalar function rbu_fossil_delta().
618 **
619 ** This function applies a fossil delta patch to a blob. Exactly two
620 ** arguments must be passed to this function. The first is the blob to
621 ** patch and the second the patch to apply. If no error occurs, this
622 ** function returns the patched blob.
623 */
rbuFossilDeltaFunc(sqlite3_context * context,int argc,sqlite3_value ** argv)624 static void rbuFossilDeltaFunc(
625 sqlite3_context *context,
626 int argc,
627 sqlite3_value **argv
628 ){
629 const char *aDelta;
630 int nDelta;
631 const char *aOrig;
632 int nOrig;
633
634 int nOut;
635 int nOut2;
636 char *aOut;
637
638 assert( argc==2 );
639
640 nOrig = sqlite3_value_bytes(argv[0]);
641 aOrig = (const char*)sqlite3_value_blob(argv[0]);
642 nDelta = sqlite3_value_bytes(argv[1]);
643 aDelta = (const char*)sqlite3_value_blob(argv[1]);
644
645 /* Figure out the size of the output */
646 nOut = rbuDeltaOutputSize(aDelta, nDelta);
647 if( nOut<0 ){
648 sqlite3_result_error(context, "corrupt fossil delta", -1);
649 return;
650 }
651
652 aOut = sqlite3_malloc(nOut+1);
653 if( aOut==0 ){
654 sqlite3_result_error_nomem(context);
655 }else{
656 nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut);
657 if( nOut2!=nOut ){
658 sqlite3_result_error(context, "corrupt fossil delta", -1);
659 }else{
660 sqlite3_result_blob(context, aOut, nOut, sqlite3_free);
661 }
662 }
663 }
664
665
666 /*
667 ** Prepare the SQL statement in buffer zSql against database handle db.
668 ** If successful, set *ppStmt to point to the new statement and return
669 ** SQLITE_OK.
670 **
671 ** Otherwise, if an error does occur, set *ppStmt to NULL and return
672 ** an SQLite error code. Additionally, set output variable *pzErrmsg to
673 ** point to a buffer containing an error message. It is the responsibility
674 ** of the caller to (eventually) free this buffer using sqlite3_free().
675 */
prepareAndCollectError(sqlite3 * db,sqlite3_stmt ** ppStmt,char ** pzErrmsg,const char * zSql)676 static int prepareAndCollectError(
677 sqlite3 *db,
678 sqlite3_stmt **ppStmt,
679 char **pzErrmsg,
680 const char *zSql
681 ){
682 int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
683 if( rc!=SQLITE_OK ){
684 *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
685 *ppStmt = 0;
686 }
687 return rc;
688 }
689
690 /*
691 ** Reset the SQL statement passed as the first argument. Return a copy
692 ** of the value returned by sqlite3_reset().
693 **
694 ** If an error has occurred, then set *pzErrmsg to point to a buffer
695 ** containing an error message. It is the responsibility of the caller
696 ** to eventually free this buffer using sqlite3_free().
697 */
resetAndCollectError(sqlite3_stmt * pStmt,char ** pzErrmsg)698 static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){
699 int rc = sqlite3_reset(pStmt);
700 if( rc!=SQLITE_OK ){
701 *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt)));
702 }
703 return rc;
704 }
705
706 /*
707 ** Unless it is NULL, argument zSql points to a buffer allocated using
708 ** sqlite3_malloc containing an SQL statement. This function prepares the SQL
709 ** statement against database db and frees the buffer. If statement
710 ** compilation is successful, *ppStmt is set to point to the new statement
711 ** handle and SQLITE_OK is returned.
712 **
713 ** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code
714 ** returned. In this case, *pzErrmsg may also be set to point to an error
715 ** message. It is the responsibility of the caller to free this error message
716 ** buffer using sqlite3_free().
717 **
718 ** If argument zSql is NULL, this function assumes that an OOM has occurred.
719 ** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL.
720 */
prepareFreeAndCollectError(sqlite3 * db,sqlite3_stmt ** ppStmt,char ** pzErrmsg,char * zSql)721 static int prepareFreeAndCollectError(
722 sqlite3 *db,
723 sqlite3_stmt **ppStmt,
724 char **pzErrmsg,
725 char *zSql
726 ){
727 int rc;
728 assert( *pzErrmsg==0 );
729 if( zSql==0 ){
730 rc = SQLITE_NOMEM;
731 *ppStmt = 0;
732 }else{
733 rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql);
734 sqlite3_free(zSql);
735 }
736 return rc;
737 }
738
739 /*
740 ** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated
741 ** by an earlier call to rbuObjIterCacheTableInfo().
742 */
rbuObjIterFreeCols(RbuObjIter * pIter)743 static void rbuObjIterFreeCols(RbuObjIter *pIter){
744 int i;
745 for(i=0; i<pIter->nTblCol; i++){
746 sqlite3_free(pIter->azTblCol[i]);
747 sqlite3_free(pIter->azTblType[i]);
748 }
749 sqlite3_free(pIter->azTblCol);
750 pIter->azTblCol = 0;
751 pIter->azTblType = 0;
752 pIter->aiSrcOrder = 0;
753 pIter->abTblPk = 0;
754 pIter->abNotNull = 0;
755 pIter->nTblCol = 0;
756 pIter->eType = 0; /* Invalid value */
757 }
758
759 /*
760 ** Finalize all statements and free all allocations that are specific to
761 ** the current object (table/index pair).
762 */
rbuObjIterClearStatements(RbuObjIter * pIter)763 static void rbuObjIterClearStatements(RbuObjIter *pIter){
764 RbuUpdateStmt *pUp;
765
766 sqlite3_finalize(pIter->pSelect);
767 sqlite3_finalize(pIter->pInsert);
768 sqlite3_finalize(pIter->pDelete);
769 sqlite3_finalize(pIter->pTmpInsert);
770 pUp = pIter->pRbuUpdate;
771 while( pUp ){
772 RbuUpdateStmt *pTmp = pUp->pNext;
773 sqlite3_finalize(pUp->pUpdate);
774 sqlite3_free(pUp);
775 pUp = pTmp;
776 }
777
778 pIter->pSelect = 0;
779 pIter->pInsert = 0;
780 pIter->pDelete = 0;
781 pIter->pRbuUpdate = 0;
782 pIter->pTmpInsert = 0;
783 pIter->nCol = 0;
784 }
785
786 /*
787 ** Clean up any resources allocated as part of the iterator object passed
788 ** as the only argument.
789 */
rbuObjIterFinalize(RbuObjIter * pIter)790 static void rbuObjIterFinalize(RbuObjIter *pIter){
791 rbuObjIterClearStatements(pIter);
792 sqlite3_finalize(pIter->pTblIter);
793 sqlite3_finalize(pIter->pIdxIter);
794 rbuObjIterFreeCols(pIter);
795 memset(pIter, 0, sizeof(RbuObjIter));
796 }
797
798 /*
799 ** Advance the iterator to the next position.
800 **
801 ** If no error occurs, SQLITE_OK is returned and the iterator is left
802 ** pointing to the next entry. Otherwise, an error code and message is
803 ** left in the RBU handle passed as the first argument. A copy of the
804 ** error code is returned.
805 */
rbuObjIterNext(sqlite3rbu * p,RbuObjIter * pIter)806 static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){
807 int rc = p->rc;
808 if( rc==SQLITE_OK ){
809
810 /* Free any SQLite statements used while processing the previous object */
811 rbuObjIterClearStatements(pIter);
812 if( pIter->zIdx==0 ){
813 rc = sqlite3_exec(p->dbMain,
814 "DROP TRIGGER IF EXISTS temp.rbu_insert_tr;"
815 "DROP TRIGGER IF EXISTS temp.rbu_update1_tr;"
816 "DROP TRIGGER IF EXISTS temp.rbu_update2_tr;"
817 "DROP TRIGGER IF EXISTS temp.rbu_delete_tr;"
818 , 0, 0, &p->zErrmsg
819 );
820 }
821
822 if( rc==SQLITE_OK ){
823 if( pIter->bCleanup ){
824 rbuObjIterFreeCols(pIter);
825 pIter->bCleanup = 0;
826 rc = sqlite3_step(pIter->pTblIter);
827 if( rc!=SQLITE_ROW ){
828 rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg);
829 pIter->zTbl = 0;
830 }else{
831 pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0);
832 pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1);
833 rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM;
834 }
835 }else{
836 if( pIter->zIdx==0 ){
837 sqlite3_stmt *pIdx = pIter->pIdxIter;
838 rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC);
839 }
840 if( rc==SQLITE_OK ){
841 rc = sqlite3_step(pIter->pIdxIter);
842 if( rc!=SQLITE_ROW ){
843 rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg);
844 pIter->bCleanup = 1;
845 pIter->zIdx = 0;
846 }else{
847 pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0);
848 pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1);
849 pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2);
850 rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM;
851 }
852 }
853 }
854 }
855 }
856
857 if( rc!=SQLITE_OK ){
858 rbuObjIterFinalize(pIter);
859 p->rc = rc;
860 }
861 return rc;
862 }
863
864
865 /*
866 ** The implementation of the rbu_target_name() SQL function. This function
867 ** accepts one or two arguments. The first argument is the name of a table -
868 ** the name of a table in the RBU database. The second, if it is present, is 1
869 ** for a view or 0 for a table.
870 **
871 ** For a non-vacuum RBU handle, if the table name matches the pattern:
872 **
873 ** data[0-9]_<name>
874 **
875 ** where <name> is any sequence of 1 or more characters, <name> is returned.
876 ** Otherwise, if the only argument does not match the above pattern, an SQL
877 ** NULL is returned.
878 **
879 ** "data_t1" -> "t1"
880 ** "data0123_t2" -> "t2"
881 ** "dataAB_t3" -> NULL
882 **
883 ** For an rbu vacuum handle, a copy of the first argument is returned if
884 ** the second argument is either missing or 0 (not a view).
885 */
rbuTargetNameFunc(sqlite3_context * pCtx,int argc,sqlite3_value ** argv)886 static void rbuTargetNameFunc(
887 sqlite3_context *pCtx,
888 int argc,
889 sqlite3_value **argv
890 ){
891 sqlite3rbu *p = sqlite3_user_data(pCtx);
892 const char *zIn;
893 assert( argc==1 || argc==2 );
894
895 zIn = (const char*)sqlite3_value_text(argv[0]);
896 if( zIn ){
897 if( rbuIsVacuum(p) ){
898 if( argc==1 || 0==sqlite3_value_int(argv[1]) ){
899 sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC);
900 }
901 }else{
902 if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){
903 int i;
904 for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++);
905 if( zIn[i]=='_' && zIn[i+1] ){
906 sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC);
907 }
908 }
909 }
910 }
911 }
912
913 /*
914 ** Initialize the iterator structure passed as the second argument.
915 **
916 ** If no error occurs, SQLITE_OK is returned and the iterator is left
917 ** pointing to the first entry. Otherwise, an error code and message is
918 ** left in the RBU handle passed as the first argument. A copy of the
919 ** error code is returned.
920 */
rbuObjIterFirst(sqlite3rbu * p,RbuObjIter * pIter)921 static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){
922 int rc;
923 memset(pIter, 0, sizeof(RbuObjIter));
924
925 rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg,
926 sqlite3_mprintf(
927 "SELECT rbu_target_name(name, type='view') AS target, name "
928 "FROM sqlite_master "
929 "WHERE type IN ('table', 'view') AND target IS NOT NULL "
930 " %s "
931 "ORDER BY name"
932 , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : ""));
933
934 if( rc==SQLITE_OK ){
935 rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg,
936 "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' "
937 " FROM main.sqlite_master "
938 " WHERE type='index' AND tbl_name = ?"
939 );
940 }
941
942 pIter->bCleanup = 1;
943 p->rc = rc;
944 return rbuObjIterNext(p, pIter);
945 }
946
947 /*
948 ** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs,
949 ** an error code is stored in the RBU handle passed as the first argument.
950 **
951 ** If an error has already occurred (p->rc is already set to something other
952 ** than SQLITE_OK), then this function returns NULL without modifying the
953 ** stored error code. In this case it still calls sqlite3_free() on any
954 ** printf() parameters associated with %z conversions.
955 */
rbuMPrintf(sqlite3rbu * p,const char * zFmt,...)956 static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){
957 char *zSql = 0;
958 va_list ap;
959 va_start(ap, zFmt);
960 zSql = sqlite3_vmprintf(zFmt, ap);
961 if( p->rc==SQLITE_OK ){
962 if( zSql==0 ) p->rc = SQLITE_NOMEM;
963 }else{
964 sqlite3_free(zSql);
965 zSql = 0;
966 }
967 va_end(ap);
968 return zSql;
969 }
970
971 /*
972 ** Argument zFmt is a sqlite3_mprintf() style format string. The trailing
973 ** arguments are the usual subsitution values. This function performs
974 ** the printf() style substitutions and executes the result as an SQL
975 ** statement on the RBU handles database.
976 **
977 ** If an error occurs, an error code and error message is stored in the
978 ** RBU handle. If an error has already occurred when this function is
979 ** called, it is a no-op.
980 */
rbuMPrintfExec(sqlite3rbu * p,sqlite3 * db,const char * zFmt,...)981 static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){
982 va_list ap;
983 char *zSql;
984 va_start(ap, zFmt);
985 zSql = sqlite3_vmprintf(zFmt, ap);
986 if( p->rc==SQLITE_OK ){
987 if( zSql==0 ){
988 p->rc = SQLITE_NOMEM;
989 }else{
990 p->rc = sqlite3_exec(db, zSql, 0, 0, &p->zErrmsg);
991 }
992 }
993 sqlite3_free(zSql);
994 va_end(ap);
995 return p->rc;
996 }
997
998 /*
999 ** Attempt to allocate and return a pointer to a zeroed block of nByte
1000 ** bytes.
1001 **
1002 ** If an error (i.e. an OOM condition) occurs, return NULL and leave an
1003 ** error code in the rbu handle passed as the first argument. Or, if an
1004 ** error has already occurred when this function is called, return NULL
1005 ** immediately without attempting the allocation or modifying the stored
1006 ** error code.
1007 */
rbuMalloc(sqlite3rbu * p,int nByte)1008 static void *rbuMalloc(sqlite3rbu *p, int nByte){
1009 void *pRet = 0;
1010 if( p->rc==SQLITE_OK ){
1011 assert( nByte>0 );
1012 pRet = sqlite3_malloc64(nByte);
1013 if( pRet==0 ){
1014 p->rc = SQLITE_NOMEM;
1015 }else{
1016 memset(pRet, 0, nByte);
1017 }
1018 }
1019 return pRet;
1020 }
1021
1022
1023 /*
1024 ** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that
1025 ** there is room for at least nCol elements. If an OOM occurs, store an
1026 ** error code in the RBU handle passed as the first argument.
1027 */
rbuAllocateIterArrays(sqlite3rbu * p,RbuObjIter * pIter,int nCol)1028 static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){
1029 int nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol;
1030 char **azNew;
1031
1032 azNew = (char**)rbuMalloc(p, nByte);
1033 if( azNew ){
1034 pIter->azTblCol = azNew;
1035 pIter->azTblType = &azNew[nCol];
1036 pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol];
1037 pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol];
1038 pIter->abNotNull = (u8*)&pIter->abTblPk[nCol];
1039 pIter->abIndexed = (u8*)&pIter->abNotNull[nCol];
1040 }
1041 }
1042
1043 /*
1044 ** The first argument must be a nul-terminated string. This function
1045 ** returns a copy of the string in memory obtained from sqlite3_malloc().
1046 ** It is the responsibility of the caller to eventually free this memory
1047 ** using sqlite3_free().
1048 **
1049 ** If an OOM condition is encountered when attempting to allocate memory,
1050 ** output variable (*pRc) is set to SQLITE_NOMEM before returning. Otherwise,
1051 ** if the allocation succeeds, (*pRc) is left unchanged.
1052 */
rbuStrndup(const char * zStr,int * pRc)1053 static char *rbuStrndup(const char *zStr, int *pRc){
1054 char *zRet = 0;
1055
1056 assert( *pRc==SQLITE_OK );
1057 if( zStr ){
1058 size_t nCopy = strlen(zStr) + 1;
1059 zRet = (char*)sqlite3_malloc64(nCopy);
1060 if( zRet ){
1061 memcpy(zRet, zStr, nCopy);
1062 }else{
1063 *pRc = SQLITE_NOMEM;
1064 }
1065 }
1066
1067 return zRet;
1068 }
1069
1070 /*
1071 ** Finalize the statement passed as the second argument.
1072 **
1073 ** If the sqlite3_finalize() call indicates that an error occurs, and the
1074 ** rbu handle error code is not already set, set the error code and error
1075 ** message accordingly.
1076 */
rbuFinalize(sqlite3rbu * p,sqlite3_stmt * pStmt)1077 static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){
1078 sqlite3 *db = sqlite3_db_handle(pStmt);
1079 int rc = sqlite3_finalize(pStmt);
1080 if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){
1081 p->rc = rc;
1082 p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
1083 }
1084 }
1085
1086 /* Determine the type of a table.
1087 **
1088 ** peType is of type (int*), a pointer to an output parameter of type
1089 ** (int). This call sets the output parameter as follows, depending
1090 ** on the type of the table specified by parameters dbName and zTbl.
1091 **
1092 ** RBU_PK_NOTABLE: No such table.
1093 ** RBU_PK_NONE: Table has an implicit rowid.
1094 ** RBU_PK_IPK: Table has an explicit IPK column.
1095 ** RBU_PK_EXTERNAL: Table has an external PK index.
1096 ** RBU_PK_WITHOUT_ROWID: Table is WITHOUT ROWID.
1097 ** RBU_PK_VTAB: Table is a virtual table.
1098 **
1099 ** Argument *piPk is also of type (int*), and also points to an output
1100 ** parameter. Unless the table has an external primary key index
1101 ** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or,
1102 ** if the table does have an external primary key index, then *piPk
1103 ** is set to the root page number of the primary key index before
1104 ** returning.
1105 **
1106 ** ALGORITHM:
1107 **
1108 ** if( no entry exists in sqlite_master ){
1109 ** return RBU_PK_NOTABLE
1110 ** }else if( sql for the entry starts with "CREATE VIRTUAL" ){
1111 ** return RBU_PK_VTAB
1112 ** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){
1113 ** if( the index that is the pk exists in sqlite_master ){
1114 ** *piPK = rootpage of that index.
1115 ** return RBU_PK_EXTERNAL
1116 ** }else{
1117 ** return RBU_PK_WITHOUT_ROWID
1118 ** }
1119 ** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){
1120 ** return RBU_PK_IPK
1121 ** }else{
1122 ** return RBU_PK_NONE
1123 ** }
1124 */
rbuTableType(sqlite3rbu * p,const char * zTab,int * peType,int * piTnum,int * piPk)1125 static void rbuTableType(
1126 sqlite3rbu *p,
1127 const char *zTab,
1128 int *peType,
1129 int *piTnum,
1130 int *piPk
1131 ){
1132 /*
1133 ** 0) SELECT count(*) FROM sqlite_master where name=%Q AND IsVirtual(%Q)
1134 ** 1) PRAGMA index_list = ?
1135 ** 2) SELECT count(*) FROM sqlite_master where name=%Q
1136 ** 3) PRAGMA table_info = ?
1137 */
1138 sqlite3_stmt *aStmt[4] = {0, 0, 0, 0};
1139
1140 *peType = RBU_PK_NOTABLE;
1141 *piPk = 0;
1142
1143 assert( p->rc==SQLITE_OK );
1144 p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg,
1145 sqlite3_mprintf(
1146 "SELECT (sql LIKE 'create virtual%%'), rootpage"
1147 " FROM sqlite_master"
1148 " WHERE name=%Q", zTab
1149 ));
1150 if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){
1151 /* Either an error, or no such table. */
1152 goto rbuTableType_end;
1153 }
1154 if( sqlite3_column_int(aStmt[0], 0) ){
1155 *peType = RBU_PK_VTAB; /* virtual table */
1156 goto rbuTableType_end;
1157 }
1158 *piTnum = sqlite3_column_int(aStmt[0], 1);
1159
1160 p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg,
1161 sqlite3_mprintf("PRAGMA index_list=%Q",zTab)
1162 );
1163 if( p->rc ) goto rbuTableType_end;
1164 while( sqlite3_step(aStmt[1])==SQLITE_ROW ){
1165 const u8 *zOrig = sqlite3_column_text(aStmt[1], 3);
1166 const u8 *zIdx = sqlite3_column_text(aStmt[1], 1);
1167 if( zOrig && zIdx && zOrig[0]=='p' ){
1168 p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg,
1169 sqlite3_mprintf(
1170 "SELECT rootpage FROM sqlite_master WHERE name = %Q", zIdx
1171 ));
1172 if( p->rc==SQLITE_OK ){
1173 if( sqlite3_step(aStmt[2])==SQLITE_ROW ){
1174 *piPk = sqlite3_column_int(aStmt[2], 0);
1175 *peType = RBU_PK_EXTERNAL;
1176 }else{
1177 *peType = RBU_PK_WITHOUT_ROWID;
1178 }
1179 }
1180 goto rbuTableType_end;
1181 }
1182 }
1183
1184 p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg,
1185 sqlite3_mprintf("PRAGMA table_info=%Q",zTab)
1186 );
1187 if( p->rc==SQLITE_OK ){
1188 while( sqlite3_step(aStmt[3])==SQLITE_ROW ){
1189 if( sqlite3_column_int(aStmt[3],5)>0 ){
1190 *peType = RBU_PK_IPK; /* explicit IPK column */
1191 goto rbuTableType_end;
1192 }
1193 }
1194 *peType = RBU_PK_NONE;
1195 }
1196
1197 rbuTableType_end: {
1198 unsigned int i;
1199 for(i=0; i<sizeof(aStmt)/sizeof(aStmt[0]); i++){
1200 rbuFinalize(p, aStmt[i]);
1201 }
1202 }
1203 }
1204
1205 /*
1206 ** This is a helper function for rbuObjIterCacheTableInfo(). It populates
1207 ** the pIter->abIndexed[] array.
1208 */
rbuObjIterCacheIndexedCols(sqlite3rbu * p,RbuObjIter * pIter)1209 static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){
1210 sqlite3_stmt *pList = 0;
1211 int bIndex = 0;
1212
1213 if( p->rc==SQLITE_OK ){
1214 memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol);
1215 p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg,
1216 sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl)
1217 );
1218 }
1219
1220 pIter->nIndex = 0;
1221 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){
1222 const char *zIdx = (const char*)sqlite3_column_text(pList, 1);
1223 sqlite3_stmt *pXInfo = 0;
1224 if( zIdx==0 ) break;
1225 p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
1226 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
1227 );
1228 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
1229 int iCid = sqlite3_column_int(pXInfo, 1);
1230 if( iCid>=0 ) pIter->abIndexed[iCid] = 1;
1231 }
1232 rbuFinalize(p, pXInfo);
1233 bIndex = 1;
1234 pIter->nIndex++;
1235 }
1236
1237 if( pIter->eType==RBU_PK_WITHOUT_ROWID ){
1238 /* "PRAGMA index_list" includes the main PK b-tree */
1239 pIter->nIndex--;
1240 }
1241
1242 rbuFinalize(p, pList);
1243 if( bIndex==0 ) pIter->abIndexed = 0;
1244 }
1245
1246
1247 /*
1248 ** If they are not already populated, populate the pIter->azTblCol[],
1249 ** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to
1250 ** the table (not index) that the iterator currently points to.
1251 **
1252 ** Return SQLITE_OK if successful, or an SQLite error code otherwise. If
1253 ** an error does occur, an error code and error message are also left in
1254 ** the RBU handle.
1255 */
rbuObjIterCacheTableInfo(sqlite3rbu * p,RbuObjIter * pIter)1256 static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){
1257 if( pIter->azTblCol==0 ){
1258 sqlite3_stmt *pStmt = 0;
1259 int nCol = 0;
1260 int i; /* for() loop iterator variable */
1261 int bRbuRowid = 0; /* If input table has column "rbu_rowid" */
1262 int iOrder = 0;
1263 int iTnum = 0;
1264
1265 /* Figure out the type of table this step will deal with. */
1266 assert( pIter->eType==0 );
1267 rbuTableType(p, pIter->zTbl, &pIter->eType, &iTnum, &pIter->iPkTnum);
1268 if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_NOTABLE ){
1269 p->rc = SQLITE_ERROR;
1270 p->zErrmsg = sqlite3_mprintf("no such table: %s", pIter->zTbl);
1271 }
1272 if( p->rc ) return p->rc;
1273 if( pIter->zIdx==0 ) pIter->iTnum = iTnum;
1274
1275 assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK
1276 || pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_WITHOUT_ROWID
1277 || pIter->eType==RBU_PK_VTAB
1278 );
1279
1280 /* Populate the azTblCol[] and nTblCol variables based on the columns
1281 ** of the input table. Ignore any input table columns that begin with
1282 ** "rbu_". */
1283 p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
1284 sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl)
1285 );
1286 if( p->rc==SQLITE_OK ){
1287 nCol = sqlite3_column_count(pStmt);
1288 rbuAllocateIterArrays(p, pIter, nCol);
1289 }
1290 for(i=0; p->rc==SQLITE_OK && i<nCol; i++){
1291 const char *zName = (const char*)sqlite3_column_name(pStmt, i);
1292 if( sqlite3_strnicmp("rbu_", zName, 4) ){
1293 char *zCopy = rbuStrndup(zName, &p->rc);
1294 pIter->aiSrcOrder[pIter->nTblCol] = pIter->nTblCol;
1295 pIter->azTblCol[pIter->nTblCol++] = zCopy;
1296 }
1297 else if( 0==sqlite3_stricmp("rbu_rowid", zName) ){
1298 bRbuRowid = 1;
1299 }
1300 }
1301 sqlite3_finalize(pStmt);
1302 pStmt = 0;
1303
1304 if( p->rc==SQLITE_OK
1305 && rbuIsVacuum(p)==0
1306 && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE)
1307 ){
1308 p->rc = SQLITE_ERROR;
1309 p->zErrmsg = sqlite3_mprintf(
1310 "table %q %s rbu_rowid column", pIter->zDataTbl,
1311 (bRbuRowid ? "may not have" : "requires")
1312 );
1313 }
1314
1315 /* Check that all non-HIDDEN columns in the destination table are also
1316 ** present in the input table. Populate the abTblPk[], azTblType[] and
1317 ** aiTblOrder[] arrays at the same time. */
1318 if( p->rc==SQLITE_OK ){
1319 p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg,
1320 sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl)
1321 );
1322 }
1323 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
1324 const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
1325 if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */
1326 for(i=iOrder; i<pIter->nTblCol; i++){
1327 if( 0==strcmp(zName, pIter->azTblCol[i]) ) break;
1328 }
1329 if( i==pIter->nTblCol ){
1330 p->rc = SQLITE_ERROR;
1331 p->zErrmsg = sqlite3_mprintf("column missing from %q: %s",
1332 pIter->zDataTbl, zName
1333 );
1334 }else{
1335 int iPk = sqlite3_column_int(pStmt, 5);
1336 int bNotNull = sqlite3_column_int(pStmt, 3);
1337 const char *zType = (const char*)sqlite3_column_text(pStmt, 2);
1338
1339 if( i!=iOrder ){
1340 SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]);
1341 SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]);
1342 }
1343
1344 pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc);
1345 pIter->abTblPk[iOrder] = (iPk!=0);
1346 pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0);
1347 iOrder++;
1348 }
1349 }
1350
1351 rbuFinalize(p, pStmt);
1352 rbuObjIterCacheIndexedCols(p, pIter);
1353 assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 );
1354 assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 );
1355 }
1356
1357 return p->rc;
1358 }
1359
1360 /*
1361 ** This function constructs and returns a pointer to a nul-terminated
1362 ** string containing some SQL clause or list based on one or more of the
1363 ** column names currently stored in the pIter->azTblCol[] array.
1364 */
rbuObjIterGetCollist(sqlite3rbu * p,RbuObjIter * pIter)1365 static char *rbuObjIterGetCollist(
1366 sqlite3rbu *p, /* RBU object */
1367 RbuObjIter *pIter /* Object iterator for column names */
1368 ){
1369 char *zList = 0;
1370 const char *zSep = "";
1371 int i;
1372 for(i=0; i<pIter->nTblCol; i++){
1373 const char *z = pIter->azTblCol[i];
1374 zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z);
1375 zSep = ", ";
1376 }
1377 return zList;
1378 }
1379
1380 /*
1381 ** This function is used to create a SELECT list (the list of SQL
1382 ** expressions that follows a SELECT keyword) for a SELECT statement
1383 ** used to read from an data_xxx or rbu_tmp_xxx table while updating the
1384 ** index object currently indicated by the iterator object passed as the
1385 ** second argument. A "PRAGMA index_xinfo = <idxname>" statement is used
1386 ** to obtain the required information.
1387 **
1388 ** If the index is of the following form:
1389 **
1390 ** CREATE INDEX i1 ON t1(c, b COLLATE nocase);
1391 **
1392 ** and "t1" is a table with an explicit INTEGER PRIMARY KEY column
1393 ** "ipk", the returned string is:
1394 **
1395 ** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'"
1396 **
1397 ** As well as the returned string, three other malloc'd strings are
1398 ** returned via output parameters. As follows:
1399 **
1400 ** pzImposterCols: ...
1401 ** pzImposterPk: ...
1402 ** pzWhere: ...
1403 */
rbuObjIterGetIndexCols(sqlite3rbu * p,RbuObjIter * pIter,char ** pzImposterCols,char ** pzImposterPk,char ** pzWhere,int * pnBind)1404 static char *rbuObjIterGetIndexCols(
1405 sqlite3rbu *p, /* RBU object */
1406 RbuObjIter *pIter, /* Object iterator for column names */
1407 char **pzImposterCols, /* OUT: Columns for imposter table */
1408 char **pzImposterPk, /* OUT: Imposter PK clause */
1409 char **pzWhere, /* OUT: WHERE clause */
1410 int *pnBind /* OUT: Trbul number of columns */
1411 ){
1412 int rc = p->rc; /* Error code */
1413 int rc2; /* sqlite3_finalize() return code */
1414 char *zRet = 0; /* String to return */
1415 char *zImpCols = 0; /* String to return via *pzImposterCols */
1416 char *zImpPK = 0; /* String to return via *pzImposterPK */
1417 char *zWhere = 0; /* String to return via *pzWhere */
1418 int nBind = 0; /* Value to return via *pnBind */
1419 const char *zCom = ""; /* Set to ", " later on */
1420 const char *zAnd = ""; /* Set to " AND " later on */
1421 sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */
1422
1423 if( rc==SQLITE_OK ){
1424 assert( p->zErrmsg==0 );
1425 rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
1426 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx)
1427 );
1428 }
1429
1430 while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
1431 int iCid = sqlite3_column_int(pXInfo, 1);
1432 int bDesc = sqlite3_column_int(pXInfo, 3);
1433 const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
1434 const char *zCol;
1435 const char *zType;
1436
1437 if( iCid<0 ){
1438 /* An integer primary key. If the table has an explicit IPK, use
1439 ** its name. Otherwise, use "rbu_rowid". */
1440 if( pIter->eType==RBU_PK_IPK ){
1441 int i;
1442 for(i=0; pIter->abTblPk[i]==0; i++);
1443 assert( i<pIter->nTblCol );
1444 zCol = pIter->azTblCol[i];
1445 }else if( rbuIsVacuum(p) ){
1446 zCol = "_rowid_";
1447 }else{
1448 zCol = "rbu_rowid";
1449 }
1450 zType = "INTEGER";
1451 }else{
1452 zCol = pIter->azTblCol[iCid];
1453 zType = pIter->azTblType[iCid];
1454 }
1455
1456 zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom, zCol, zCollate);
1457 if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){
1458 const char *zOrder = (bDesc ? " DESC" : "");
1459 zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s",
1460 zImpPK, zCom, nBind, zCol, zOrder
1461 );
1462 }
1463 zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q",
1464 zImpCols, zCom, nBind, zCol, zType, zCollate
1465 );
1466 zWhere = sqlite3_mprintf(
1467 "%z%s\"rbu_imp_%d%w\" IS ?", zWhere, zAnd, nBind, zCol
1468 );
1469 if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM;
1470 zCom = ", ";
1471 zAnd = " AND ";
1472 nBind++;
1473 }
1474
1475 rc2 = sqlite3_finalize(pXInfo);
1476 if( rc==SQLITE_OK ) rc = rc2;
1477
1478 if( rc!=SQLITE_OK ){
1479 sqlite3_free(zRet);
1480 sqlite3_free(zImpCols);
1481 sqlite3_free(zImpPK);
1482 sqlite3_free(zWhere);
1483 zRet = 0;
1484 zImpCols = 0;
1485 zImpPK = 0;
1486 zWhere = 0;
1487 p->rc = rc;
1488 }
1489
1490 *pzImposterCols = zImpCols;
1491 *pzImposterPk = zImpPK;
1492 *pzWhere = zWhere;
1493 *pnBind = nBind;
1494 return zRet;
1495 }
1496
1497 /*
1498 ** Assuming the current table columns are "a", "b" and "c", and the zObj
1499 ** paramter is passed "old", return a string of the form:
1500 **
1501 ** "old.a, old.b, old.b"
1502 **
1503 ** With the column names escaped.
1504 **
1505 ** For tables with implicit rowids - RBU_PK_EXTERNAL and RBU_PK_NONE, append
1506 ** the text ", old._rowid_" to the returned value.
1507 */
rbuObjIterGetOldlist(sqlite3rbu * p,RbuObjIter * pIter,const char * zObj)1508 static char *rbuObjIterGetOldlist(
1509 sqlite3rbu *p,
1510 RbuObjIter *pIter,
1511 const char *zObj
1512 ){
1513 char *zList = 0;
1514 if( p->rc==SQLITE_OK && pIter->abIndexed ){
1515 const char *zS = "";
1516 int i;
1517 for(i=0; i<pIter->nTblCol; i++){
1518 if( pIter->abIndexed[i] ){
1519 const char *zCol = pIter->azTblCol[i];
1520 zList = sqlite3_mprintf("%z%s%s.\"%w\"", zList, zS, zObj, zCol);
1521 }else{
1522 zList = sqlite3_mprintf("%z%sNULL", zList, zS);
1523 }
1524 zS = ", ";
1525 if( zList==0 ){
1526 p->rc = SQLITE_NOMEM;
1527 break;
1528 }
1529 }
1530
1531 /* For a table with implicit rowids, append "old._rowid_" to the list. */
1532 if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
1533 zList = rbuMPrintf(p, "%z, %s._rowid_", zList, zObj);
1534 }
1535 }
1536 return zList;
1537 }
1538
1539 /*
1540 ** Return an expression that can be used in a WHERE clause to match the
1541 ** primary key of the current table. For example, if the table is:
1542 **
1543 ** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c));
1544 **
1545 ** Return the string:
1546 **
1547 ** "b = ?1 AND c = ?2"
1548 */
rbuObjIterGetWhere(sqlite3rbu * p,RbuObjIter * pIter)1549 static char *rbuObjIterGetWhere(
1550 sqlite3rbu *p,
1551 RbuObjIter *pIter
1552 ){
1553 char *zList = 0;
1554 if( pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE ){
1555 zList = rbuMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1);
1556 }else if( pIter->eType==RBU_PK_EXTERNAL ){
1557 const char *zSep = "";
1558 int i;
1559 for(i=0; i<pIter->nTblCol; i++){
1560 if( pIter->abTblPk[i] ){
1561 zList = rbuMPrintf(p, "%z%sc%d=?%d", zList, zSep, i, i+1);
1562 zSep = " AND ";
1563 }
1564 }
1565 zList = rbuMPrintf(p,
1566 "_rowid_ = (SELECT id FROM rbu_imposter2 WHERE %z)", zList
1567 );
1568
1569 }else{
1570 const char *zSep = "";
1571 int i;
1572 for(i=0; i<pIter->nTblCol; i++){
1573 if( pIter->abTblPk[i] ){
1574 const char *zCol = pIter->azTblCol[i];
1575 zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1);
1576 zSep = " AND ";
1577 }
1578 }
1579 }
1580 return zList;
1581 }
1582
1583 /*
1584 ** The SELECT statement iterating through the keys for the current object
1585 ** (p->objiter.pSelect) currently points to a valid row. However, there
1586 ** is something wrong with the rbu_control value in the rbu_control value
1587 ** stored in the (p->nCol+1)'th column. Set the error code and error message
1588 ** of the RBU handle to something reflecting this.
1589 */
rbuBadControlError(sqlite3rbu * p)1590 static void rbuBadControlError(sqlite3rbu *p){
1591 p->rc = SQLITE_ERROR;
1592 p->zErrmsg = sqlite3_mprintf("invalid rbu_control value");
1593 }
1594
1595
1596 /*
1597 ** Return a nul-terminated string containing the comma separated list of
1598 ** assignments that should be included following the "SET" keyword of
1599 ** an UPDATE statement used to update the table object that the iterator
1600 ** passed as the second argument currently points to if the rbu_control
1601 ** column of the data_xxx table entry is set to zMask.
1602 **
1603 ** The memory for the returned string is obtained from sqlite3_malloc().
1604 ** It is the responsibility of the caller to eventually free it using
1605 ** sqlite3_free().
1606 **
1607 ** If an OOM error is encountered when allocating space for the new
1608 ** string, an error code is left in the rbu handle passed as the first
1609 ** argument and NULL is returned. Or, if an error has already occurred
1610 ** when this function is called, NULL is returned immediately, without
1611 ** attempting the allocation or modifying the stored error code.
1612 */
rbuObjIterGetSetlist(sqlite3rbu * p,RbuObjIter * pIter,const char * zMask)1613 static char *rbuObjIterGetSetlist(
1614 sqlite3rbu *p,
1615 RbuObjIter *pIter,
1616 const char *zMask
1617 ){
1618 char *zList = 0;
1619 if( p->rc==SQLITE_OK ){
1620 int i;
1621
1622 if( (int)strlen(zMask)!=pIter->nTblCol ){
1623 rbuBadControlError(p);
1624 }else{
1625 const char *zSep = "";
1626 for(i=0; i<pIter->nTblCol; i++){
1627 char c = zMask[pIter->aiSrcOrder[i]];
1628 if( c=='x' ){
1629 zList = rbuMPrintf(p, "%z%s\"%w\"=?%d",
1630 zList, zSep, pIter->azTblCol[i], i+1
1631 );
1632 zSep = ", ";
1633 }
1634 else if( c=='d' ){
1635 zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)",
1636 zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1
1637 );
1638 zSep = ", ";
1639 }
1640 else if( c=='f' ){
1641 zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)",
1642 zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1
1643 );
1644 zSep = ", ";
1645 }
1646 }
1647 }
1648 }
1649 return zList;
1650 }
1651
1652 /*
1653 ** Return a nul-terminated string consisting of nByte comma separated
1654 ** "?" expressions. For example, if nByte is 3, return a pointer to
1655 ** a buffer containing the string "?,?,?".
1656 **
1657 ** The memory for the returned string is obtained from sqlite3_malloc().
1658 ** It is the responsibility of the caller to eventually free it using
1659 ** sqlite3_free().
1660 **
1661 ** If an OOM error is encountered when allocating space for the new
1662 ** string, an error code is left in the rbu handle passed as the first
1663 ** argument and NULL is returned. Or, if an error has already occurred
1664 ** when this function is called, NULL is returned immediately, without
1665 ** attempting the allocation or modifying the stored error code.
1666 */
rbuObjIterGetBindlist(sqlite3rbu * p,int nBind)1667 static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){
1668 char *zRet = 0;
1669 int nByte = nBind*2 + 1;
1670
1671 zRet = (char*)rbuMalloc(p, nByte);
1672 if( zRet ){
1673 int i;
1674 for(i=0; i<nBind; i++){
1675 zRet[i*2] = '?';
1676 zRet[i*2+1] = (i+1==nBind) ? '\0' : ',';
1677 }
1678 }
1679 return zRet;
1680 }
1681
1682 /*
1683 ** The iterator currently points to a table (not index) of type
1684 ** RBU_PK_WITHOUT_ROWID. This function creates the PRIMARY KEY
1685 ** declaration for the corresponding imposter table. For example,
1686 ** if the iterator points to a table created as:
1687 **
1688 ** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, a DESC)) WITHOUT ROWID
1689 **
1690 ** this function returns:
1691 **
1692 ** PRIMARY KEY("b", "a" DESC)
1693 */
rbuWithoutRowidPK(sqlite3rbu * p,RbuObjIter * pIter)1694 static char *rbuWithoutRowidPK(sqlite3rbu *p, RbuObjIter *pIter){
1695 char *z = 0;
1696 assert( pIter->zIdx==0 );
1697 if( p->rc==SQLITE_OK ){
1698 const char *zSep = "PRIMARY KEY(";
1699 sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */
1700 sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = <pk-index> */
1701
1702 p->rc = prepareFreeAndCollectError(p->dbMain, &pXList, &p->zErrmsg,
1703 sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl)
1704 );
1705 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXList) ){
1706 const char *zOrig = (const char*)sqlite3_column_text(pXList,3);
1707 if( zOrig && strcmp(zOrig, "pk")==0 ){
1708 const char *zIdx = (const char*)sqlite3_column_text(pXList,1);
1709 if( zIdx ){
1710 p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
1711 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
1712 );
1713 }
1714 break;
1715 }
1716 }
1717 rbuFinalize(p, pXList);
1718
1719 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
1720 if( sqlite3_column_int(pXInfo, 5) ){
1721 /* int iCid = sqlite3_column_int(pXInfo, 0); */
1722 const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2);
1723 const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : "";
1724 z = rbuMPrintf(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc);
1725 zSep = ", ";
1726 }
1727 }
1728 z = rbuMPrintf(p, "%z)", z);
1729 rbuFinalize(p, pXInfo);
1730 }
1731 return z;
1732 }
1733
1734 /*
1735 ** This function creates the second imposter table used when writing to
1736 ** a table b-tree where the table has an external primary key. If the
1737 ** iterator passed as the second argument does not currently point to
1738 ** a table (not index) with an external primary key, this function is a
1739 ** no-op.
1740 **
1741 ** Assuming the iterator does point to a table with an external PK, this
1742 ** function creates a WITHOUT ROWID imposter table named "rbu_imposter2"
1743 ** used to access that PK index. For example, if the target table is
1744 ** declared as follows:
1745 **
1746 ** CREATE TABLE t1(a, b TEXT, c REAL, PRIMARY KEY(b, c));
1747 **
1748 ** then the imposter table schema is:
1749 **
1750 ** CREATE TABLE rbu_imposter2(c1 TEXT, c2 REAL, id INTEGER) WITHOUT ROWID;
1751 **
1752 */
rbuCreateImposterTable2(sqlite3rbu * p,RbuObjIter * pIter)1753 static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){
1754 if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_EXTERNAL ){
1755 int tnum = pIter->iPkTnum; /* Root page of PK index */
1756 sqlite3_stmt *pQuery = 0; /* SELECT name ... WHERE rootpage = $tnum */
1757 const char *zIdx = 0; /* Name of PK index */
1758 sqlite3_stmt *pXInfo = 0; /* PRAGMA main.index_xinfo = $zIdx */
1759 const char *zComma = "";
1760 char *zCols = 0; /* Used to build up list of table cols */
1761 char *zPk = 0; /* Used to build up table PK declaration */
1762
1763 /* Figure out the name of the primary key index for the current table.
1764 ** This is needed for the argument to "PRAGMA index_xinfo". Set
1765 ** zIdx to point to a nul-terminated string containing this name. */
1766 p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg,
1767 "SELECT name FROM sqlite_master WHERE rootpage = ?"
1768 );
1769 if( p->rc==SQLITE_OK ){
1770 sqlite3_bind_int(pQuery, 1, tnum);
1771 if( SQLITE_ROW==sqlite3_step(pQuery) ){
1772 zIdx = (const char*)sqlite3_column_text(pQuery, 0);
1773 }
1774 }
1775 if( zIdx ){
1776 p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
1777 sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
1778 );
1779 }
1780 rbuFinalize(p, pQuery);
1781
1782 while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
1783 int bKey = sqlite3_column_int(pXInfo, 5);
1784 if( bKey ){
1785 int iCid = sqlite3_column_int(pXInfo, 1);
1786 int bDesc = sqlite3_column_int(pXInfo, 3);
1787 const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
1788 zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %s", zCols, zComma,
1789 iCid, pIter->azTblType[iCid], zCollate
1790 );
1791 zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":"");
1792 zComma = ", ";
1793 }
1794 }
1795 zCols = rbuMPrintf(p, "%z, id INTEGER", zCols);
1796 rbuFinalize(p, pXInfo);
1797
1798 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum);
1799 rbuMPrintfExec(p, p->dbMain,
1800 "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID",
1801 zCols, zPk
1802 );
1803 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0);
1804 }
1805 }
1806
1807 /*
1808 ** If an error has already occurred when this function is called, it
1809 ** immediately returns zero (without doing any work). Or, if an error
1810 ** occurs during the execution of this function, it sets the error code
1811 ** in the sqlite3rbu object indicated by the first argument and returns
1812 ** zero.
1813 **
1814 ** The iterator passed as the second argument is guaranteed to point to
1815 ** a table (not an index) when this function is called. This function
1816 ** attempts to create any imposter table required to write to the main
1817 ** table b-tree of the table before returning. Non-zero is returned if
1818 ** an imposter table are created, or zero otherwise.
1819 **
1820 ** An imposter table is required in all cases except RBU_PK_VTAB. Only
1821 ** virtual tables are written to directly. The imposter table has the
1822 ** same schema as the actual target table (less any UNIQUE constraints).
1823 ** More precisely, the "same schema" means the same columns, types,
1824 ** collation sequences. For tables that do not have an external PRIMARY
1825 ** KEY, it also means the same PRIMARY KEY declaration.
1826 */
rbuCreateImposterTable(sqlite3rbu * p,RbuObjIter * pIter)1827 static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){
1828 if( p->rc==SQLITE_OK && pIter->eType!=RBU_PK_VTAB ){
1829 int tnum = pIter->iTnum;
1830 const char *zComma = "";
1831 char *zSql = 0;
1832 int iCol;
1833 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1);
1834
1835 for(iCol=0; p->rc==SQLITE_OK && iCol<pIter->nTblCol; iCol++){
1836 const char *zPk = "";
1837 const char *zCol = pIter->azTblCol[iCol];
1838 const char *zColl = 0;
1839
1840 p->rc = sqlite3_table_column_metadata(
1841 p->dbMain, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0
1842 );
1843
1844 if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){
1845 /* If the target table column is an "INTEGER PRIMARY KEY", add
1846 ** "PRIMARY KEY" to the imposter table column declaration. */
1847 zPk = "PRIMARY KEY ";
1848 }
1849 zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %s%s",
1850 zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl,
1851 (pIter->abNotNull[iCol] ? " NOT NULL" : "")
1852 );
1853 zComma = ", ";
1854 }
1855
1856 if( pIter->eType==RBU_PK_WITHOUT_ROWID ){
1857 char *zPk = rbuWithoutRowidPK(p, pIter);
1858 if( zPk ){
1859 zSql = rbuMPrintf(p, "%z, %z", zSql, zPk);
1860 }
1861 }
1862
1863 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum);
1864 rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s",
1865 pIter->zTbl, zSql,
1866 (pIter->eType==RBU_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "")
1867 );
1868 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0);
1869 }
1870 }
1871
1872 /*
1873 ** Prepare a statement used to insert rows into the "rbu_tmp_xxx" table.
1874 ** Specifically a statement of the form:
1875 **
1876 ** INSERT INTO rbu_tmp_xxx VALUES(?, ?, ? ...);
1877 **
1878 ** The number of bound variables is equal to the number of columns in
1879 ** the target table, plus one (for the rbu_control column), plus one more
1880 ** (for the rbu_rowid column) if the target table is an implicit IPK or
1881 ** virtual table.
1882 */
rbuObjIterPrepareTmpInsert(sqlite3rbu * p,RbuObjIter * pIter,const char * zCollist,const char * zRbuRowid)1883 static void rbuObjIterPrepareTmpInsert(
1884 sqlite3rbu *p,
1885 RbuObjIter *pIter,
1886 const char *zCollist,
1887 const char *zRbuRowid
1888 ){
1889 int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE);
1890 char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid);
1891 if( zBind ){
1892 assert( pIter->pTmpInsert==0 );
1893 p->rc = prepareFreeAndCollectError(
1894 p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf(
1895 "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)",
1896 p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind
1897 ));
1898 }
1899 }
1900
rbuTmpInsertFunc(sqlite3_context * pCtx,int nVal,sqlite3_value ** apVal)1901 static void rbuTmpInsertFunc(
1902 sqlite3_context *pCtx,
1903 int nVal,
1904 sqlite3_value **apVal
1905 ){
1906 sqlite3rbu *p = sqlite3_user_data(pCtx);
1907 int rc = SQLITE_OK;
1908 int i;
1909
1910 assert( sqlite3_value_int(apVal[0])!=0
1911 || p->objiter.eType==RBU_PK_EXTERNAL
1912 || p->objiter.eType==RBU_PK_NONE
1913 );
1914 if( sqlite3_value_int(apVal[0])!=0 ){
1915 p->nPhaseOneStep += p->objiter.nIndex;
1916 }
1917
1918 for(i=0; rc==SQLITE_OK && i<nVal; i++){
1919 rc = sqlite3_bind_value(p->objiter.pTmpInsert, i+1, apVal[i]);
1920 }
1921 if( rc==SQLITE_OK ){
1922 sqlite3_step(p->objiter.pTmpInsert);
1923 rc = sqlite3_reset(p->objiter.pTmpInsert);
1924 }
1925
1926 if( rc!=SQLITE_OK ){
1927 sqlite3_result_error_code(pCtx, rc);
1928 }
1929 }
1930
1931 /*
1932 ** Ensure that the SQLite statement handles required to update the
1933 ** target database object currently indicated by the iterator passed
1934 ** as the second argument are available.
1935 */
rbuObjIterPrepareAll(sqlite3rbu * p,RbuObjIter * pIter,int nOffset)1936 static int rbuObjIterPrepareAll(
1937 sqlite3rbu *p,
1938 RbuObjIter *pIter,
1939 int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */
1940 ){
1941 assert( pIter->bCleanup==0 );
1942 if( pIter->pSelect==0 && rbuObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){
1943 const int tnum = pIter->iTnum;
1944 char *zCollist = 0; /* List of indexed columns */
1945 char **pz = &p->zErrmsg;
1946 const char *zIdx = pIter->zIdx;
1947 char *zLimit = 0;
1948
1949 if( nOffset ){
1950 zLimit = sqlite3_mprintf(" LIMIT -1 OFFSET %d", nOffset);
1951 if( !zLimit ) p->rc = SQLITE_NOMEM;
1952 }
1953
1954 if( zIdx ){
1955 const char *zTbl = pIter->zTbl;
1956 char *zImposterCols = 0; /* Columns for imposter table */
1957 char *zImposterPK = 0; /* Primary key declaration for imposter */
1958 char *zWhere = 0; /* WHERE clause on PK columns */
1959 char *zBind = 0;
1960 int nBind = 0;
1961
1962 assert( pIter->eType!=RBU_PK_VTAB );
1963 zCollist = rbuObjIterGetIndexCols(
1964 p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind
1965 );
1966 zBind = rbuObjIterGetBindlist(p, nBind);
1967
1968 /* Create the imposter table used to write to this index. */
1969 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1);
1970 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1,tnum);
1971 rbuMPrintfExec(p, p->dbMain,
1972 "CREATE TABLE \"rbu_imp_%w\"( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID",
1973 zTbl, zImposterCols, zImposterPK
1974 );
1975 sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0);
1976
1977 /* Create the statement to insert index entries */
1978 pIter->nCol = nBind;
1979 if( p->rc==SQLITE_OK ){
1980 p->rc = prepareFreeAndCollectError(
1981 p->dbMain, &pIter->pInsert, &p->zErrmsg,
1982 sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind)
1983 );
1984 }
1985
1986 /* And to delete index entries */
1987 if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){
1988 p->rc = prepareFreeAndCollectError(
1989 p->dbMain, &pIter->pDelete, &p->zErrmsg,
1990 sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere)
1991 );
1992 }
1993
1994 /* Create the SELECT statement to read keys in sorted order */
1995 if( p->rc==SQLITE_OK ){
1996 char *zSql;
1997 if( rbuIsVacuum(p) ){
1998 zSql = sqlite3_mprintf(
1999 "SELECT %s, 0 AS rbu_control FROM '%q' ORDER BY %s%s",
2000 zCollist,
2001 pIter->zDataTbl,
2002 zCollist, zLimit
2003 );
2004 }else
2005
2006 if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
2007 zSql = sqlite3_mprintf(
2008 "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s",
2009 zCollist, p->zStateDb, pIter->zDataTbl,
2010 zCollist, zLimit
2011 );
2012 }else{
2013 zSql = sqlite3_mprintf(
2014 "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' "
2015 "UNION ALL "
2016 "SELECT %s, rbu_control FROM '%q' "
2017 "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 "
2018 "ORDER BY %s%s",
2019 zCollist, p->zStateDb, pIter->zDataTbl,
2020 zCollist, pIter->zDataTbl,
2021 zCollist, zLimit
2022 );
2023 }
2024 p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, zSql);
2025 }
2026
2027 sqlite3_free(zImposterCols);
2028 sqlite3_free(zImposterPK);
2029 sqlite3_free(zWhere);
2030 sqlite3_free(zBind);
2031 }else{
2032 int bRbuRowid = (pIter->eType==RBU_PK_VTAB)
2033 ||(pIter->eType==RBU_PK_NONE)
2034 ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p));
2035 const char *zTbl = pIter->zTbl; /* Table this step applies to */
2036 const char *zWrite; /* Imposter table name */
2037
2038 char *zBindings = rbuObjIterGetBindlist(p, pIter->nTblCol + bRbuRowid);
2039 char *zWhere = rbuObjIterGetWhere(p, pIter);
2040 char *zOldlist = rbuObjIterGetOldlist(p, pIter, "old");
2041 char *zNewlist = rbuObjIterGetOldlist(p, pIter, "new");
2042
2043 zCollist = rbuObjIterGetCollist(p, pIter);
2044 pIter->nCol = pIter->nTblCol;
2045
2046 /* Create the imposter table or tables (if required). */
2047 rbuCreateImposterTable(p, pIter);
2048 rbuCreateImposterTable2(p, pIter);
2049 zWrite = (pIter->eType==RBU_PK_VTAB ? "" : "rbu_imp_");
2050
2051 /* Create the INSERT statement to write to the target PK b-tree */
2052 if( p->rc==SQLITE_OK ){
2053 p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz,
2054 sqlite3_mprintf(
2055 "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)",
2056 zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings
2057 )
2058 );
2059 }
2060
2061 /* Create the DELETE statement to write to the target PK b-tree.
2062 ** Because it only performs INSERT operations, this is not required for
2063 ** an rbu vacuum handle. */
2064 if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){
2065 p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz,
2066 sqlite3_mprintf(
2067 "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere
2068 )
2069 );
2070 }
2071
2072 if( rbuIsVacuum(p)==0 && pIter->abIndexed ){
2073 const char *zRbuRowid = "";
2074 if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
2075 zRbuRowid = ", rbu_rowid";
2076 }
2077
2078 /* Create the rbu_tmp_xxx table and the triggers to populate it. */
2079 rbuMPrintfExec(p, p->dbRbu,
2080 "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS "
2081 "SELECT *%s FROM '%q' WHERE 0;"
2082 , p->zStateDb, pIter->zDataTbl
2083 , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "")
2084 , pIter->zDataTbl
2085 );
2086
2087 rbuMPrintfExec(p, p->dbMain,
2088 "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" "
2089 "BEGIN "
2090 " SELECT rbu_tmp_insert(3, %s);"
2091 "END;"
2092
2093 "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" "
2094 "BEGIN "
2095 " SELECT rbu_tmp_insert(3, %s);"
2096 "END;"
2097
2098 "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" "
2099 "BEGIN "
2100 " SELECT rbu_tmp_insert(4, %s);"
2101 "END;",
2102 zWrite, zTbl, zOldlist,
2103 zWrite, zTbl, zOldlist,
2104 zWrite, zTbl, zNewlist
2105 );
2106
2107 if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
2108 rbuMPrintfExec(p, p->dbMain,
2109 "CREATE TEMP TRIGGER rbu_insert_tr AFTER INSERT ON \"%s%w\" "
2110 "BEGIN "
2111 " SELECT rbu_tmp_insert(0, %s);"
2112 "END;",
2113 zWrite, zTbl, zNewlist
2114 );
2115 }
2116
2117 rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid);
2118 }
2119
2120 /* Create the SELECT statement to read keys from data_xxx */
2121 if( p->rc==SQLITE_OK ){
2122 const char *zRbuRowid = "";
2123 if( bRbuRowid ){
2124 zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid";
2125 }
2126 p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz,
2127 sqlite3_mprintf(
2128 "SELECT %s,%s rbu_control%s FROM '%q'%s",
2129 zCollist,
2130 (rbuIsVacuum(p) ? "0 AS " : ""),
2131 zRbuRowid,
2132 pIter->zDataTbl, zLimit
2133 )
2134 );
2135 }
2136
2137 sqlite3_free(zWhere);
2138 sqlite3_free(zOldlist);
2139 sqlite3_free(zNewlist);
2140 sqlite3_free(zBindings);
2141 }
2142 sqlite3_free(zCollist);
2143 sqlite3_free(zLimit);
2144 }
2145
2146 return p->rc;
2147 }
2148
2149 /*
2150 ** Set output variable *ppStmt to point to an UPDATE statement that may
2151 ** be used to update the imposter table for the main table b-tree of the
2152 ** table object that pIter currently points to, assuming that the
2153 ** rbu_control column of the data_xyz table contains zMask.
2154 **
2155 ** If the zMask string does not specify any columns to update, then this
2156 ** is not an error. Output variable *ppStmt is set to NULL in this case.
2157 */
rbuGetUpdateStmt(sqlite3rbu * p,RbuObjIter * pIter,const char * zMask,sqlite3_stmt ** ppStmt)2158 static int rbuGetUpdateStmt(
2159 sqlite3rbu *p, /* RBU handle */
2160 RbuObjIter *pIter, /* Object iterator */
2161 const char *zMask, /* rbu_control value ('x.x.') */
2162 sqlite3_stmt **ppStmt /* OUT: UPDATE statement handle */
2163 ){
2164 RbuUpdateStmt **pp;
2165 RbuUpdateStmt *pUp = 0;
2166 int nUp = 0;
2167
2168 /* In case an error occurs */
2169 *ppStmt = 0;
2170
2171 /* Search for an existing statement. If one is found, shift it to the front
2172 ** of the LRU queue and return immediately. Otherwise, leave nUp pointing
2173 ** to the number of statements currently in the cache and pUp to the
2174 ** last object in the list. */
2175 for(pp=&pIter->pRbuUpdate; *pp; pp=&((*pp)->pNext)){
2176 pUp = *pp;
2177 if( strcmp(pUp->zMask, zMask)==0 ){
2178 *pp = pUp->pNext;
2179 pUp->pNext = pIter->pRbuUpdate;
2180 pIter->pRbuUpdate = pUp;
2181 *ppStmt = pUp->pUpdate;
2182 return SQLITE_OK;
2183 }
2184 nUp++;
2185 }
2186 assert( pUp==0 || pUp->pNext==0 );
2187
2188 if( nUp>=SQLITE_RBU_UPDATE_CACHESIZE ){
2189 for(pp=&pIter->pRbuUpdate; *pp!=pUp; pp=&((*pp)->pNext));
2190 *pp = 0;
2191 sqlite3_finalize(pUp->pUpdate);
2192 pUp->pUpdate = 0;
2193 }else{
2194 pUp = (RbuUpdateStmt*)rbuMalloc(p, sizeof(RbuUpdateStmt)+pIter->nTblCol+1);
2195 }
2196
2197 if( pUp ){
2198 char *zWhere = rbuObjIterGetWhere(p, pIter);
2199 char *zSet = rbuObjIterGetSetlist(p, pIter, zMask);
2200 char *zUpdate = 0;
2201
2202 pUp->zMask = (char*)&pUp[1];
2203 memcpy(pUp->zMask, zMask, pIter->nTblCol);
2204 pUp->pNext = pIter->pRbuUpdate;
2205 pIter->pRbuUpdate = pUp;
2206
2207 if( zSet ){
2208 const char *zPrefix = "";
2209
2210 if( pIter->eType!=RBU_PK_VTAB ) zPrefix = "rbu_imp_";
2211 zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s",
2212 zPrefix, pIter->zTbl, zSet, zWhere
2213 );
2214 p->rc = prepareFreeAndCollectError(
2215 p->dbMain, &pUp->pUpdate, &p->zErrmsg, zUpdate
2216 );
2217 *ppStmt = pUp->pUpdate;
2218 }
2219 sqlite3_free(zWhere);
2220 sqlite3_free(zSet);
2221 }
2222
2223 return p->rc;
2224 }
2225
rbuOpenDbhandle(sqlite3rbu * p,const char * zName,int bUseVfs)2226 static sqlite3 *rbuOpenDbhandle(
2227 sqlite3rbu *p,
2228 const char *zName,
2229 int bUseVfs
2230 ){
2231 sqlite3 *db = 0;
2232 if( p->rc==SQLITE_OK ){
2233 const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI;
2234 p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0);
2235 if( p->rc ){
2236 p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
2237 sqlite3_close(db);
2238 db = 0;
2239 }
2240 }
2241 return db;
2242 }
2243
2244 /*
2245 ** Free an RbuState object allocated by rbuLoadState().
2246 */
rbuFreeState(RbuState * p)2247 static void rbuFreeState(RbuState *p){
2248 if( p ){
2249 sqlite3_free(p->zTbl);
2250 sqlite3_free(p->zIdx);
2251 sqlite3_free(p);
2252 }
2253 }
2254
2255 /*
2256 ** Allocate an RbuState object and load the contents of the rbu_state
2257 ** table into it. Return a pointer to the new object. It is the
2258 ** responsibility of the caller to eventually free the object using
2259 ** sqlite3_free().
2260 **
2261 ** If an error occurs, leave an error code and message in the rbu handle
2262 ** and return NULL.
2263 */
rbuLoadState(sqlite3rbu * p)2264 static RbuState *rbuLoadState(sqlite3rbu *p){
2265 RbuState *pRet = 0;
2266 sqlite3_stmt *pStmt = 0;
2267 int rc;
2268 int rc2;
2269
2270 pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState));
2271 if( pRet==0 ) return 0;
2272
2273 rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
2274 sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb)
2275 );
2276 while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
2277 switch( sqlite3_column_int(pStmt, 0) ){
2278 case RBU_STATE_STAGE:
2279 pRet->eStage = sqlite3_column_int(pStmt, 1);
2280 if( pRet->eStage!=RBU_STAGE_OAL
2281 && pRet->eStage!=RBU_STAGE_MOVE
2282 && pRet->eStage!=RBU_STAGE_CKPT
2283 ){
2284 p->rc = SQLITE_CORRUPT;
2285 }
2286 break;
2287
2288 case RBU_STATE_TBL:
2289 pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
2290 break;
2291
2292 case RBU_STATE_IDX:
2293 pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
2294 break;
2295
2296 case RBU_STATE_ROW:
2297 pRet->nRow = sqlite3_column_int(pStmt, 1);
2298 break;
2299
2300 case RBU_STATE_PROGRESS:
2301 pRet->nProgress = sqlite3_column_int64(pStmt, 1);
2302 break;
2303
2304 case RBU_STATE_CKPT:
2305 pRet->iWalCksum = sqlite3_column_int64(pStmt, 1);
2306 break;
2307
2308 case RBU_STATE_COOKIE:
2309 pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1);
2310 break;
2311
2312 case RBU_STATE_OALSZ:
2313 pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1);
2314 break;
2315
2316 case RBU_STATE_PHASEONESTEP:
2317 pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1);
2318 break;
2319
2320 default:
2321 rc = SQLITE_CORRUPT;
2322 break;
2323 }
2324 }
2325 rc2 = sqlite3_finalize(pStmt);
2326 if( rc==SQLITE_OK ) rc = rc2;
2327
2328 p->rc = rc;
2329 return pRet;
2330 }
2331
2332
2333 /*
2334 ** Open the database handle and attach the RBU database as "rbu". If an
2335 ** error occurs, leave an error code and message in the RBU handle.
2336 */
rbuOpenDatabase(sqlite3rbu * p,int * pbRetry)2337 static void rbuOpenDatabase(sqlite3rbu *p, int *pbRetry){
2338 assert( p->rc || (p->dbMain==0 && p->dbRbu==0) );
2339 assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 );
2340
2341 /* Open the RBU database */
2342 p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1);
2343
2344 if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
2345 sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p);
2346 if( p->zState==0 ){
2347 const char *zFile = sqlite3_db_filename(p->dbRbu, "main");
2348 p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile);
2349 }
2350 }
2351
2352 /* If using separate RBU and state databases, attach the state database to
2353 ** the RBU db handle now. */
2354 if( p->zState ){
2355 rbuMPrintfExec(p, p->dbRbu, "ATTACH %Q AS stat", p->zState);
2356 memcpy(p->zStateDb, "stat", 4);
2357 }else{
2358 memcpy(p->zStateDb, "main", 4);
2359 }
2360
2361 #if 0
2362 if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
2363 p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0);
2364 }
2365 #endif
2366
2367 /* If it has not already been created, create the rbu_state table */
2368 rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb);
2369
2370 #if 0
2371 if( rbuIsVacuum(p) ){
2372 if( p->rc==SQLITE_OK ){
2373 int rc2;
2374 int bOk = 0;
2375 sqlite3_stmt *pCnt = 0;
2376 p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg,
2377 "SELECT count(*) FROM stat.sqlite_master"
2378 );
2379 if( p->rc==SQLITE_OK
2380 && sqlite3_step(pCnt)==SQLITE_ROW
2381 && 1==sqlite3_column_int(pCnt, 0)
2382 ){
2383 bOk = 1;
2384 }
2385 rc2 = sqlite3_finalize(pCnt);
2386 if( p->rc==SQLITE_OK ) p->rc = rc2;
2387
2388 if( p->rc==SQLITE_OK && bOk==0 ){
2389 p->rc = SQLITE_ERROR;
2390 p->zErrmsg = sqlite3_mprintf("invalid state database");
2391 }
2392
2393 if( p->rc==SQLITE_OK ){
2394 p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0);
2395 }
2396 }
2397 }
2398 #endif
2399
2400 if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
2401 int bOpen = 0;
2402 int rc;
2403 p->nRbu = 0;
2404 p->pRbuFd = 0;
2405 rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p);
2406 if( rc!=SQLITE_NOTFOUND ) p->rc = rc;
2407 if( p->eStage>=RBU_STAGE_MOVE ){
2408 bOpen = 1;
2409 }else{
2410 RbuState *pState = rbuLoadState(p);
2411 if( pState ){
2412 bOpen = (pState->eStage>=RBU_STAGE_MOVE);
2413 rbuFreeState(pState);
2414 }
2415 }
2416 if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1);
2417 }
2418
2419 p->eStage = 0;
2420 if( p->rc==SQLITE_OK && p->dbMain==0 ){
2421 if( !rbuIsVacuum(p) ){
2422 p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1);
2423 }else if( p->pRbuFd->pWalFd ){
2424 if( pbRetry ){
2425 p->pRbuFd->bNolock = 0;
2426 sqlite3_close(p->dbRbu);
2427 sqlite3_close(p->dbMain);
2428 p->dbMain = 0;
2429 p->dbRbu = 0;
2430 *pbRetry = 1;
2431 return;
2432 }
2433 p->rc = SQLITE_ERROR;
2434 p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database");
2435 }else{
2436 char *zTarget;
2437 char *zExtra = 0;
2438 if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){
2439 zExtra = &p->zRbu[5];
2440 while( *zExtra ){
2441 if( *zExtra++=='?' ) break;
2442 }
2443 if( *zExtra=='\0' ) zExtra = 0;
2444 }
2445
2446 zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s",
2447 sqlite3_db_filename(p->dbRbu, "main"),
2448 (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra)
2449 );
2450
2451 if( zTarget==0 ){
2452 p->rc = SQLITE_NOMEM;
2453 return;
2454 }
2455 p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1);
2456 sqlite3_free(zTarget);
2457 }
2458 }
2459
2460 if( p->rc==SQLITE_OK ){
2461 p->rc = sqlite3_create_function(p->dbMain,
2462 "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0
2463 );
2464 }
2465
2466 if( p->rc==SQLITE_OK ){
2467 p->rc = sqlite3_create_function(p->dbMain,
2468 "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0
2469 );
2470 }
2471
2472 if( p->rc==SQLITE_OK ){
2473 p->rc = sqlite3_create_function(p->dbRbu,
2474 "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0
2475 );
2476 }
2477
2478 if( p->rc==SQLITE_OK ){
2479 p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p);
2480 }
2481 rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_master");
2482
2483 /* Mark the database file just opened as an RBU target database. If
2484 ** this call returns SQLITE_NOTFOUND, then the RBU vfs is not in use.
2485 ** This is an error. */
2486 if( p->rc==SQLITE_OK ){
2487 p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p);
2488 }
2489
2490 if( p->rc==SQLITE_NOTFOUND ){
2491 p->rc = SQLITE_ERROR;
2492 p->zErrmsg = sqlite3_mprintf("rbu vfs not found");
2493 }
2494 }
2495
2496 /*
2497 ** This routine is a copy of the sqlite3FileSuffix3() routine from the core.
2498 ** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined.
2499 **
2500 ** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database
2501 ** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and
2502 ** if filename in z[] has a suffix (a.k.a. "extension") that is longer than
2503 ** three characters, then shorten the suffix on z[] to be the last three
2504 ** characters of the original suffix.
2505 **
2506 ** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always
2507 ** do the suffix shortening regardless of URI parameter.
2508 **
2509 ** Examples:
2510 **
2511 ** test.db-journal => test.nal
2512 ** test.db-wal => test.wal
2513 ** test.db-shm => test.shm
2514 ** test.db-mj7f3319fa => test.9fa
2515 */
rbuFileSuffix3(const char * zBase,char * z)2516 static void rbuFileSuffix3(const char *zBase, char *z){
2517 #ifdef SQLITE_ENABLE_8_3_NAMES
2518 #if SQLITE_ENABLE_8_3_NAMES<2
2519 if( sqlite3_uri_boolean(zBase, "8_3_names", 0) )
2520 #endif
2521 {
2522 int i, sz;
2523 sz = (int)strlen(z)&0xffffff;
2524 for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){}
2525 if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4);
2526 }
2527 #endif
2528 }
2529
2530 /*
2531 ** Return the current wal-index header checksum for the target database
2532 ** as a 64-bit integer.
2533 **
2534 ** The checksum is store in the first page of xShmMap memory as an 8-byte
2535 ** blob starting at byte offset 40.
2536 */
rbuShmChecksum(sqlite3rbu * p)2537 static i64 rbuShmChecksum(sqlite3rbu *p){
2538 i64 iRet = 0;
2539 if( p->rc==SQLITE_OK ){
2540 sqlite3_file *pDb = p->pTargetFd->pReal;
2541 u32 volatile *ptr;
2542 p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr);
2543 if( p->rc==SQLITE_OK ){
2544 iRet = ((i64)ptr[10] << 32) + ptr[11];
2545 }
2546 }
2547 return iRet;
2548 }
2549
2550 /*
2551 ** This function is called as part of initializing or reinitializing an
2552 ** incremental checkpoint.
2553 **
2554 ** It populates the sqlite3rbu.aFrame[] array with the set of
2555 ** (wal frame -> db page) copy operations required to checkpoint the
2556 ** current wal file, and obtains the set of shm locks required to safely
2557 ** perform the copy operations directly on the file-system.
2558 **
2559 ** If argument pState is not NULL, then the incremental checkpoint is
2560 ** being resumed. In this case, if the checksum of the wal-index-header
2561 ** following recovery is not the same as the checksum saved in the RbuState
2562 ** object, then the rbu handle is set to DONE state. This occurs if some
2563 ** other client appends a transaction to the wal file in the middle of
2564 ** an incremental checkpoint.
2565 */
rbuSetupCheckpoint(sqlite3rbu * p,RbuState * pState)2566 static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){
2567
2568 /* If pState is NULL, then the wal file may not have been opened and
2569 ** recovered. Running a read-statement here to ensure that doing so
2570 ** does not interfere with the "capture" process below. */
2571 if( pState==0 ){
2572 p->eStage = 0;
2573 if( p->rc==SQLITE_OK ){
2574 p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_master", 0, 0, 0);
2575 }
2576 }
2577
2578 /* Assuming no error has occurred, run a "restart" checkpoint with the
2579 ** sqlite3rbu.eStage variable set to CAPTURE. This turns on the following
2580 ** special behaviour in the rbu VFS:
2581 **
2582 ** * If the exclusive shm WRITER or READ0 lock cannot be obtained,
2583 ** the checkpoint fails with SQLITE_BUSY (normally SQLite would
2584 ** proceed with running a passive checkpoint instead of failing).
2585 **
2586 ** * Attempts to read from the *-wal file or write to the database file
2587 ** do not perform any IO. Instead, the frame/page combinations that
2588 ** would be read/written are recorded in the sqlite3rbu.aFrame[]
2589 ** array.
2590 **
2591 ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER,
2592 ** READ0 and CHECKPOINT locks taken as part of the checkpoint are
2593 ** no-ops. These locks will not be released until the connection
2594 ** is closed.
2595 **
2596 ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL
2597 ** error.
2598 **
2599 ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the
2600 ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[]
2601 ** array populated with a set of (frame -> page) mappings. Because the
2602 ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy
2603 ** data from the wal file into the database file according to the
2604 ** contents of aFrame[].
2605 */
2606 if( p->rc==SQLITE_OK ){
2607 int rc2;
2608 p->eStage = RBU_STAGE_CAPTURE;
2609 rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0);
2610 if( rc2!=SQLITE_INTERNAL ) p->rc = rc2;
2611 }
2612
2613 if( p->rc==SQLITE_OK && p->nFrame>0 ){
2614 p->eStage = RBU_STAGE_CKPT;
2615 p->nStep = (pState ? pState->nRow : 0);
2616 p->aBuf = rbuMalloc(p, p->pgsz);
2617 p->iWalCksum = rbuShmChecksum(p);
2618 }
2619
2620 if( p->rc==SQLITE_OK ){
2621 if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){
2622 p->rc = SQLITE_DONE;
2623 p->eStage = RBU_STAGE_DONE;
2624 }else{
2625 int nSectorSize;
2626 sqlite3_file *pDb = p->pTargetFd->pReal;
2627 sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal;
2628 assert( p->nPagePerSector==0 );
2629 nSectorSize = pDb->pMethods->xSectorSize(pDb);
2630 if( nSectorSize>p->pgsz ){
2631 p->nPagePerSector = nSectorSize / p->pgsz;
2632 }else{
2633 p->nPagePerSector = 1;
2634 }
2635
2636 /* Call xSync() on the wal file. This causes SQLite to sync the
2637 ** directory in which the target database and the wal file reside, in
2638 ** case it has not been synced since the rename() call in
2639 ** rbuMoveOalFile(). */
2640 p->rc = pWal->pMethods->xSync(pWal, SQLITE_SYNC_NORMAL);
2641 }
2642 }
2643 }
2644
2645 /*
2646 ** Called when iAmt bytes are read from offset iOff of the wal file while
2647 ** the rbu object is in capture mode. Record the frame number of the frame
2648 ** being read in the aFrame[] array.
2649 */
rbuCaptureWalRead(sqlite3rbu * pRbu,i64 iOff,int iAmt)2650 static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){
2651 const u32 mReq = (1<<WAL_LOCK_WRITE)|(1<<WAL_LOCK_CKPT)|(1<<WAL_LOCK_READ0);
2652 u32 iFrame;
2653
2654 if( pRbu->mLock!=mReq ){
2655 pRbu->rc = SQLITE_BUSY;
2656 return SQLITE_INTERNAL;
2657 }
2658
2659 pRbu->pgsz = iAmt;
2660 if( pRbu->nFrame==pRbu->nFrameAlloc ){
2661 int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2;
2662 RbuFrame *aNew;
2663 aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame));
2664 if( aNew==0 ) return SQLITE_NOMEM;
2665 pRbu->aFrame = aNew;
2666 pRbu->nFrameAlloc = nNew;
2667 }
2668
2669 iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1;
2670 if( pRbu->iMaxFrame<iFrame ) pRbu->iMaxFrame = iFrame;
2671 pRbu->aFrame[pRbu->nFrame].iWalFrame = iFrame;
2672 pRbu->aFrame[pRbu->nFrame].iDbPage = 0;
2673 pRbu->nFrame++;
2674 return SQLITE_OK;
2675 }
2676
2677 /*
2678 ** Called when a page of data is written to offset iOff of the database
2679 ** file while the rbu handle is in capture mode. Record the page number
2680 ** of the page being written in the aFrame[] array.
2681 */
rbuCaptureDbWrite(sqlite3rbu * pRbu,i64 iOff)2682 static int rbuCaptureDbWrite(sqlite3rbu *pRbu, i64 iOff){
2683 pRbu->aFrame[pRbu->nFrame-1].iDbPage = (u32)(iOff / pRbu->pgsz) + 1;
2684 return SQLITE_OK;
2685 }
2686
2687 /*
2688 ** This is called as part of an incremental checkpoint operation. Copy
2689 ** a single frame of data from the wal file into the database file, as
2690 ** indicated by the RbuFrame object.
2691 */
rbuCheckpointFrame(sqlite3rbu * p,RbuFrame * pFrame)2692 static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){
2693 sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal;
2694 sqlite3_file *pDb = p->pTargetFd->pReal;
2695 i64 iOff;
2696
2697 assert( p->rc==SQLITE_OK );
2698 iOff = (i64)(pFrame->iWalFrame-1) * (p->pgsz + 24) + 32 + 24;
2699 p->rc = pWal->pMethods->xRead(pWal, p->aBuf, p->pgsz, iOff);
2700 if( p->rc ) return;
2701
2702 iOff = (i64)(pFrame->iDbPage-1) * p->pgsz;
2703 p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff);
2704 }
2705
2706
2707 /*
2708 ** Take an EXCLUSIVE lock on the database file.
2709 */
rbuLockDatabase(sqlite3rbu * p)2710 static void rbuLockDatabase(sqlite3rbu *p){
2711 sqlite3_file *pReal = p->pTargetFd->pReal;
2712 assert( p->rc==SQLITE_OK );
2713 p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_SHARED);
2714 if( p->rc==SQLITE_OK ){
2715 p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_EXCLUSIVE);
2716 }
2717 }
2718
2719 #if defined(_WIN32_WCE)
rbuWinUtf8ToUnicode(const char * zFilename)2720 static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){
2721 int nChar;
2722 LPWSTR zWideFilename;
2723
2724 nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0);
2725 if( nChar==0 ){
2726 return 0;
2727 }
2728 zWideFilename = sqlite3_malloc64( nChar*sizeof(zWideFilename[0]) );
2729 if( zWideFilename==0 ){
2730 return 0;
2731 }
2732 memset(zWideFilename, 0, nChar*sizeof(zWideFilename[0]));
2733 nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename,
2734 nChar);
2735 if( nChar==0 ){
2736 sqlite3_free(zWideFilename);
2737 zWideFilename = 0;
2738 }
2739 return zWideFilename;
2740 }
2741 #endif
2742
2743 /*
2744 ** The RBU handle is currently in RBU_STAGE_OAL state, with a SHARED lock
2745 ** on the database file. This proc moves the *-oal file to the *-wal path,
2746 ** then reopens the database file (this time in vanilla, non-oal, WAL mode).
2747 ** If an error occurs, leave an error code and error message in the rbu
2748 ** handle.
2749 */
rbuMoveOalFile(sqlite3rbu * p)2750 static void rbuMoveOalFile(sqlite3rbu *p){
2751 const char *zBase = sqlite3_db_filename(p->dbMain, "main");
2752 const char *zMove = zBase;
2753 char *zOal;
2754 char *zWal;
2755
2756 if( rbuIsVacuum(p) ){
2757 zMove = sqlite3_db_filename(p->dbRbu, "main");
2758 }
2759 zOal = sqlite3_mprintf("%s-oal", zMove);
2760 zWal = sqlite3_mprintf("%s-wal", zMove);
2761
2762 assert( p->eStage==RBU_STAGE_MOVE );
2763 assert( p->rc==SQLITE_OK && p->zErrmsg==0 );
2764 if( zWal==0 || zOal==0 ){
2765 p->rc = SQLITE_NOMEM;
2766 }else{
2767 /* Move the *-oal file to *-wal. At this point connection p->db is
2768 ** holding a SHARED lock on the target database file (because it is
2769 ** in WAL mode). So no other connection may be writing the db.
2770 **
2771 ** In order to ensure that there are no database readers, an EXCLUSIVE
2772 ** lock is obtained here before the *-oal is moved to *-wal.
2773 */
2774 rbuLockDatabase(p);
2775 if( p->rc==SQLITE_OK ){
2776 rbuFileSuffix3(zBase, zWal);
2777 rbuFileSuffix3(zBase, zOal);
2778
2779 /* Re-open the databases. */
2780 rbuObjIterFinalize(&p->objiter);
2781 sqlite3_close(p->dbRbu);
2782 sqlite3_close(p->dbMain);
2783 p->dbMain = 0;
2784 p->dbRbu = 0;
2785
2786 #if defined(_WIN32_WCE)
2787 {
2788 LPWSTR zWideOal;
2789 LPWSTR zWideWal;
2790
2791 zWideOal = rbuWinUtf8ToUnicode(zOal);
2792 if( zWideOal ){
2793 zWideWal = rbuWinUtf8ToUnicode(zWal);
2794 if( zWideWal ){
2795 if( MoveFileW(zWideOal, zWideWal) ){
2796 p->rc = SQLITE_OK;
2797 }else{
2798 p->rc = SQLITE_IOERR;
2799 }
2800 sqlite3_free(zWideWal);
2801 }else{
2802 p->rc = SQLITE_IOERR_NOMEM;
2803 }
2804 sqlite3_free(zWideOal);
2805 }else{
2806 p->rc = SQLITE_IOERR_NOMEM;
2807 }
2808 }
2809 #else
2810 p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK;
2811 #endif
2812
2813 if( p->rc==SQLITE_OK ){
2814 rbuOpenDatabase(p, 0);
2815 rbuSetupCheckpoint(p, 0);
2816 }
2817 }
2818 }
2819
2820 sqlite3_free(zWal);
2821 sqlite3_free(zOal);
2822 }
2823
2824 /*
2825 ** The SELECT statement iterating through the keys for the current object
2826 ** (p->objiter.pSelect) currently points to a valid row. This function
2827 ** determines the type of operation requested by this row and returns
2828 ** one of the following values to indicate the result:
2829 **
2830 ** * RBU_INSERT
2831 ** * RBU_DELETE
2832 ** * RBU_IDX_DELETE
2833 ** * RBU_UPDATE
2834 **
2835 ** If RBU_UPDATE is returned, then output variable *pzMask is set to
2836 ** point to the text value indicating the columns to update.
2837 **
2838 ** If the rbu_control field contains an invalid value, an error code and
2839 ** message are left in the RBU handle and zero returned.
2840 */
rbuStepType(sqlite3rbu * p,const char ** pzMask)2841 static int rbuStepType(sqlite3rbu *p, const char **pzMask){
2842 int iCol = p->objiter.nCol; /* Index of rbu_control column */
2843 int res = 0; /* Return value */
2844
2845 switch( sqlite3_column_type(p->objiter.pSelect, iCol) ){
2846 case SQLITE_INTEGER: {
2847 int iVal = sqlite3_column_int(p->objiter.pSelect, iCol);
2848 switch( iVal ){
2849 case 0: res = RBU_INSERT; break;
2850 case 1: res = RBU_DELETE; break;
2851 case 2: res = RBU_REPLACE; break;
2852 case 3: res = RBU_IDX_DELETE; break;
2853 case 4: res = RBU_IDX_INSERT; break;
2854 }
2855 break;
2856 }
2857
2858 case SQLITE_TEXT: {
2859 const unsigned char *z = sqlite3_column_text(p->objiter.pSelect, iCol);
2860 if( z==0 ){
2861 p->rc = SQLITE_NOMEM;
2862 }else{
2863 *pzMask = (const char*)z;
2864 }
2865 res = RBU_UPDATE;
2866
2867 break;
2868 }
2869
2870 default:
2871 break;
2872 }
2873
2874 if( res==0 ){
2875 rbuBadControlError(p);
2876 }
2877 return res;
2878 }
2879
2880 #ifdef SQLITE_DEBUG
2881 /*
2882 ** Assert that column iCol of statement pStmt is named zName.
2883 */
assertColumnName(sqlite3_stmt * pStmt,int iCol,const char * zName)2884 static void assertColumnName(sqlite3_stmt *pStmt, int iCol, const char *zName){
2885 const char *zCol = sqlite3_column_name(pStmt, iCol);
2886 assert( 0==sqlite3_stricmp(zName, zCol) );
2887 }
2888 #else
2889 # define assertColumnName(x,y,z)
2890 #endif
2891
2892 /*
2893 ** Argument eType must be one of RBU_INSERT, RBU_DELETE, RBU_IDX_INSERT or
2894 ** RBU_IDX_DELETE. This function performs the work of a single
2895 ** sqlite3rbu_step() call for the type of operation specified by eType.
2896 */
rbuStepOneOp(sqlite3rbu * p,int eType)2897 static void rbuStepOneOp(sqlite3rbu *p, int eType){
2898 RbuObjIter *pIter = &p->objiter;
2899 sqlite3_value *pVal;
2900 sqlite3_stmt *pWriter;
2901 int i;
2902
2903 assert( p->rc==SQLITE_OK );
2904 assert( eType!=RBU_DELETE || pIter->zIdx==0 );
2905 assert( eType==RBU_DELETE || eType==RBU_IDX_DELETE
2906 || eType==RBU_INSERT || eType==RBU_IDX_INSERT
2907 );
2908
2909 /* If this is a delete, decrement nPhaseOneStep by nIndex. If the DELETE
2910 ** statement below does actually delete a row, nPhaseOneStep will be
2911 ** incremented by the same amount when SQL function rbu_tmp_insert()
2912 ** is invoked by the trigger. */
2913 if( eType==RBU_DELETE ){
2914 p->nPhaseOneStep -= p->objiter.nIndex;
2915 }
2916
2917 if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){
2918 pWriter = pIter->pDelete;
2919 }else{
2920 pWriter = pIter->pInsert;
2921 }
2922
2923 for(i=0; i<pIter->nCol; i++){
2924 /* If this is an INSERT into a table b-tree and the table has an
2925 ** explicit INTEGER PRIMARY KEY, check that this is not an attempt
2926 ** to write a NULL into the IPK column. That is not permitted. */
2927 if( eType==RBU_INSERT
2928 && pIter->zIdx==0 && pIter->eType==RBU_PK_IPK && pIter->abTblPk[i]
2929 && sqlite3_column_type(pIter->pSelect, i)==SQLITE_NULL
2930 ){
2931 p->rc = SQLITE_MISMATCH;
2932 p->zErrmsg = sqlite3_mprintf("datatype mismatch");
2933 return;
2934 }
2935
2936 if( eType==RBU_DELETE && pIter->abTblPk[i]==0 ){
2937 continue;
2938 }
2939
2940 pVal = sqlite3_column_value(pIter->pSelect, i);
2941 p->rc = sqlite3_bind_value(pWriter, i+1, pVal);
2942 if( p->rc ) return;
2943 }
2944 if( pIter->zIdx==0 ){
2945 if( pIter->eType==RBU_PK_VTAB
2946 || pIter->eType==RBU_PK_NONE
2947 || (pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p))
2948 ){
2949 /* For a virtual table, or a table with no primary key, the
2950 ** SELECT statement is:
2951 **
2952 ** SELECT <cols>, rbu_control, rbu_rowid FROM ....
2953 **
2954 ** Hence column_value(pIter->nCol+1).
2955 */
2956 assertColumnName(pIter->pSelect, pIter->nCol+1,
2957 rbuIsVacuum(p) ? "rowid" : "rbu_rowid"
2958 );
2959 pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1);
2960 p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal);
2961 }
2962 }
2963 if( p->rc==SQLITE_OK ){
2964 sqlite3_step(pWriter);
2965 p->rc = resetAndCollectError(pWriter, &p->zErrmsg);
2966 }
2967 }
2968
2969 /*
2970 ** This function does the work for an sqlite3rbu_step() call.
2971 **
2972 ** The object-iterator (p->objiter) currently points to a valid object,
2973 ** and the input cursor (p->objiter.pSelect) currently points to a valid
2974 ** input row. Perform whatever processing is required and return.
2975 **
2976 ** If no error occurs, SQLITE_OK is returned. Otherwise, an error code
2977 ** and message is left in the RBU handle and a copy of the error code
2978 ** returned.
2979 */
rbuStep(sqlite3rbu * p)2980 static int rbuStep(sqlite3rbu *p){
2981 RbuObjIter *pIter = &p->objiter;
2982 const char *zMask = 0;
2983 int eType = rbuStepType(p, &zMask);
2984
2985 if( eType ){
2986 assert( eType==RBU_INSERT || eType==RBU_DELETE
2987 || eType==RBU_REPLACE || eType==RBU_IDX_DELETE
2988 || eType==RBU_IDX_INSERT || eType==RBU_UPDATE
2989 );
2990 assert( eType!=RBU_UPDATE || pIter->zIdx==0 );
2991
2992 if( pIter->zIdx==0 && (eType==RBU_IDX_DELETE || eType==RBU_IDX_INSERT) ){
2993 rbuBadControlError(p);
2994 }
2995 else if( eType==RBU_REPLACE ){
2996 if( pIter->zIdx==0 ){
2997 p->nPhaseOneStep += p->objiter.nIndex;
2998 rbuStepOneOp(p, RBU_DELETE);
2999 }
3000 if( p->rc==SQLITE_OK ) rbuStepOneOp(p, RBU_INSERT);
3001 }
3002 else if( eType!=RBU_UPDATE ){
3003 rbuStepOneOp(p, eType);
3004 }
3005 else{
3006 sqlite3_value *pVal;
3007 sqlite3_stmt *pUpdate = 0;
3008 assert( eType==RBU_UPDATE );
3009 p->nPhaseOneStep -= p->objiter.nIndex;
3010 rbuGetUpdateStmt(p, pIter, zMask, &pUpdate);
3011 if( pUpdate ){
3012 int i;
3013 for(i=0; p->rc==SQLITE_OK && i<pIter->nCol; i++){
3014 char c = zMask[pIter->aiSrcOrder[i]];
3015 pVal = sqlite3_column_value(pIter->pSelect, i);
3016 if( pIter->abTblPk[i] || c!='.' ){
3017 p->rc = sqlite3_bind_value(pUpdate, i+1, pVal);
3018 }
3019 }
3020 if( p->rc==SQLITE_OK
3021 && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE)
3022 ){
3023 /* Bind the rbu_rowid value to column _rowid_ */
3024 assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid");
3025 pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1);
3026 p->rc = sqlite3_bind_value(pUpdate, pIter->nCol+1, pVal);
3027 }
3028 if( p->rc==SQLITE_OK ){
3029 sqlite3_step(pUpdate);
3030 p->rc = resetAndCollectError(pUpdate, &p->zErrmsg);
3031 }
3032 }
3033 }
3034 }
3035 return p->rc;
3036 }
3037
3038 /*
3039 ** Increment the schema cookie of the main database opened by p->dbMain.
3040 **
3041 ** Or, if this is an RBU vacuum, set the schema cookie of the main db
3042 ** opened by p->dbMain to one more than the schema cookie of the main
3043 ** db opened by p->dbRbu.
3044 */
rbuIncrSchemaCookie(sqlite3rbu * p)3045 static void rbuIncrSchemaCookie(sqlite3rbu *p){
3046 if( p->rc==SQLITE_OK ){
3047 sqlite3 *dbread = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain);
3048 int iCookie = 1000000;
3049 sqlite3_stmt *pStmt;
3050
3051 p->rc = prepareAndCollectError(dbread, &pStmt, &p->zErrmsg,
3052 "PRAGMA schema_version"
3053 );
3054 if( p->rc==SQLITE_OK ){
3055 /* Coverage: it may be that this sqlite3_step() cannot fail. There
3056 ** is already a transaction open, so the prepared statement cannot
3057 ** throw an SQLITE_SCHEMA exception. The only database page the
3058 ** statement reads is page 1, which is guaranteed to be in the cache.
3059 ** And no memory allocations are required. */
3060 if( SQLITE_ROW==sqlite3_step(pStmt) ){
3061 iCookie = sqlite3_column_int(pStmt, 0);
3062 }
3063 rbuFinalize(p, pStmt);
3064 }
3065 if( p->rc==SQLITE_OK ){
3066 rbuMPrintfExec(p, p->dbMain, "PRAGMA schema_version = %d", iCookie+1);
3067 }
3068 }
3069 }
3070
3071 /*
3072 ** Update the contents of the rbu_state table within the rbu database. The
3073 ** value stored in the RBU_STATE_STAGE column is eStage. All other values
3074 ** are determined by inspecting the rbu handle passed as the first argument.
3075 */
rbuSaveState(sqlite3rbu * p,int eStage)3076 static void rbuSaveState(sqlite3rbu *p, int eStage){
3077 if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){
3078 sqlite3_stmt *pInsert = 0;
3079 rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd);
3080 int rc;
3081
3082 assert( p->zErrmsg==0 );
3083 rc = prepareFreeAndCollectError(p->dbRbu, &pInsert, &p->zErrmsg,
3084 sqlite3_mprintf(
3085 "INSERT OR REPLACE INTO %s.rbu_state(k, v) VALUES "
3086 "(%d, %d), "
3087 "(%d, %Q), "
3088 "(%d, %Q), "
3089 "(%d, %d), "
3090 "(%d, %d), "
3091 "(%d, %lld), "
3092 "(%d, %lld), "
3093 "(%d, %lld), "
3094 "(%d, %lld) ",
3095 p->zStateDb,
3096 RBU_STATE_STAGE, eStage,
3097 RBU_STATE_TBL, p->objiter.zTbl,
3098 RBU_STATE_IDX, p->objiter.zIdx,
3099 RBU_STATE_ROW, p->nStep,
3100 RBU_STATE_PROGRESS, p->nProgress,
3101 RBU_STATE_CKPT, p->iWalCksum,
3102 RBU_STATE_COOKIE, (i64)pFd->iCookie,
3103 RBU_STATE_OALSZ, p->iOalSz,
3104 RBU_STATE_PHASEONESTEP, p->nPhaseOneStep
3105 )
3106 );
3107 assert( pInsert==0 || rc==SQLITE_OK );
3108
3109 if( rc==SQLITE_OK ){
3110 sqlite3_step(pInsert);
3111 rc = sqlite3_finalize(pInsert);
3112 }
3113 if( rc!=SQLITE_OK ) p->rc = rc;
3114 }
3115 }
3116
3117
3118 /*
3119 ** The second argument passed to this function is the name of a PRAGMA
3120 ** setting - "page_size", "auto_vacuum", "user_version" or "application_id".
3121 ** This function executes the following on sqlite3rbu.dbRbu:
3122 **
3123 ** "PRAGMA main.$zPragma"
3124 **
3125 ** where $zPragma is the string passed as the second argument, then
3126 ** on sqlite3rbu.dbMain:
3127 **
3128 ** "PRAGMA main.$zPragma = $val"
3129 **
3130 ** where $val is the value returned by the first PRAGMA invocation.
3131 **
3132 ** In short, it copies the value of the specified PRAGMA setting from
3133 ** dbRbu to dbMain.
3134 */
rbuCopyPragma(sqlite3rbu * p,const char * zPragma)3135 static void rbuCopyPragma(sqlite3rbu *p, const char *zPragma){
3136 if( p->rc==SQLITE_OK ){
3137 sqlite3_stmt *pPragma = 0;
3138 p->rc = prepareFreeAndCollectError(p->dbRbu, &pPragma, &p->zErrmsg,
3139 sqlite3_mprintf("PRAGMA main.%s", zPragma)
3140 );
3141 if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPragma) ){
3142 p->rc = rbuMPrintfExec(p, p->dbMain, "PRAGMA main.%s = %d",
3143 zPragma, sqlite3_column_int(pPragma, 0)
3144 );
3145 }
3146 rbuFinalize(p, pPragma);
3147 }
3148 }
3149
3150 /*
3151 ** The RBU handle passed as the only argument has just been opened and
3152 ** the state database is empty. If this RBU handle was opened for an
3153 ** RBU vacuum operation, create the schema in the target db.
3154 */
rbuCreateTargetSchema(sqlite3rbu * p)3155 static void rbuCreateTargetSchema(sqlite3rbu *p){
3156 sqlite3_stmt *pSql = 0;
3157 sqlite3_stmt *pInsert = 0;
3158
3159 assert( rbuIsVacuum(p) );
3160 p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg);
3161 if( p->rc==SQLITE_OK ){
3162 p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg,
3163 "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0"
3164 " AND name!='sqlite_sequence' "
3165 " ORDER BY type DESC"
3166 );
3167 }
3168
3169 while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){
3170 const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
3171 p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg);
3172 }
3173 rbuFinalize(p, pSql);
3174 if( p->rc!=SQLITE_OK ) return;
3175
3176 if( p->rc==SQLITE_OK ){
3177 p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg,
3178 "SELECT * FROM sqlite_master WHERE rootpage=0 OR rootpage IS NULL"
3179 );
3180 }
3181
3182 if( p->rc==SQLITE_OK ){
3183 p->rc = prepareAndCollectError(p->dbMain, &pInsert, &p->zErrmsg,
3184 "INSERT INTO sqlite_master VALUES(?,?,?,?,?)"
3185 );
3186 }
3187
3188 while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){
3189 int i;
3190 for(i=0; i<5; i++){
3191 sqlite3_bind_value(pInsert, i+1, sqlite3_column_value(pSql, i));
3192 }
3193 sqlite3_step(pInsert);
3194 p->rc = sqlite3_reset(pInsert);
3195 }
3196 if( p->rc==SQLITE_OK ){
3197 p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=0",0,0,&p->zErrmsg);
3198 }
3199
3200 rbuFinalize(p, pSql);
3201 rbuFinalize(p, pInsert);
3202 }
3203
3204 /*
3205 ** Step the RBU object.
3206 */
sqlite3rbu_step(sqlite3rbu * p)3207 int sqlite3rbu_step(sqlite3rbu *p){
3208 if( p ){
3209 switch( p->eStage ){
3210 case RBU_STAGE_OAL: {
3211 RbuObjIter *pIter = &p->objiter;
3212
3213 /* If this is an RBU vacuum operation and the state table was empty
3214 ** when this handle was opened, create the target database schema. */
3215 if( rbuIsVacuum(p) && p->nProgress==0 && p->rc==SQLITE_OK ){
3216 rbuCreateTargetSchema(p);
3217 rbuCopyPragma(p, "user_version");
3218 rbuCopyPragma(p, "application_id");
3219 }
3220
3221 while( p->rc==SQLITE_OK && pIter->zTbl ){
3222
3223 if( pIter->bCleanup ){
3224 /* Clean up the rbu_tmp_xxx table for the previous table. It
3225 ** cannot be dropped as there are currently active SQL statements.
3226 ** But the contents can be deleted. */
3227 if( rbuIsVacuum(p)==0 && pIter->abIndexed ){
3228 rbuMPrintfExec(p, p->dbRbu,
3229 "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl
3230 );
3231 }
3232 }else{
3233 rbuObjIterPrepareAll(p, pIter, 0);
3234
3235 /* Advance to the next row to process. */
3236 if( p->rc==SQLITE_OK ){
3237 int rc = sqlite3_step(pIter->pSelect);
3238 if( rc==SQLITE_ROW ){
3239 p->nProgress++;
3240 p->nStep++;
3241 return rbuStep(p);
3242 }
3243 p->rc = sqlite3_reset(pIter->pSelect);
3244 p->nStep = 0;
3245 }
3246 }
3247
3248 rbuObjIterNext(p, pIter);
3249 }
3250
3251 if( p->rc==SQLITE_OK ){
3252 assert( pIter->zTbl==0 );
3253 rbuSaveState(p, RBU_STAGE_MOVE);
3254 rbuIncrSchemaCookie(p);
3255 if( p->rc==SQLITE_OK ){
3256 p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg);
3257 }
3258 if( p->rc==SQLITE_OK ){
3259 p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg);
3260 }
3261 p->eStage = RBU_STAGE_MOVE;
3262 }
3263 break;
3264 }
3265
3266 case RBU_STAGE_MOVE: {
3267 if( p->rc==SQLITE_OK ){
3268 rbuMoveOalFile(p);
3269 p->nProgress++;
3270 }
3271 break;
3272 }
3273
3274 case RBU_STAGE_CKPT: {
3275 if( p->rc==SQLITE_OK ){
3276 if( p->nStep>=p->nFrame ){
3277 sqlite3_file *pDb = p->pTargetFd->pReal;
3278
3279 /* Sync the db file */
3280 p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
3281
3282 /* Update nBackfill */
3283 if( p->rc==SQLITE_OK ){
3284 void volatile *ptr;
3285 p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, &ptr);
3286 if( p->rc==SQLITE_OK ){
3287 ((u32 volatile*)ptr)[24] = p->iMaxFrame;
3288 }
3289 }
3290
3291 if( p->rc==SQLITE_OK ){
3292 p->eStage = RBU_STAGE_DONE;
3293 p->rc = SQLITE_DONE;
3294 }
3295 }else{
3296 /* At one point the following block copied a single frame from the
3297 ** wal file to the database file. So that one call to sqlite3rbu_step()
3298 ** checkpointed a single frame.
3299 **
3300 ** However, if the sector-size is larger than the page-size, and the
3301 ** application calls sqlite3rbu_savestate() or close() immediately
3302 ** after this step, then rbu_step() again, then a power failure occurs,
3303 ** then the database page written here may be damaged. Work around
3304 ** this by checkpointing frames until the next page in the aFrame[]
3305 ** lies on a different disk sector to the current one. */
3306 u32 iSector;
3307 do{
3308 RbuFrame *pFrame = &p->aFrame[p->nStep];
3309 iSector = (pFrame->iDbPage-1) / p->nPagePerSector;
3310 rbuCheckpointFrame(p, pFrame);
3311 p->nStep++;
3312 }while( p->nStep<p->nFrame
3313 && iSector==((p->aFrame[p->nStep].iDbPage-1) / p->nPagePerSector)
3314 && p->rc==SQLITE_OK
3315 );
3316 }
3317 p->nProgress++;
3318 }
3319 break;
3320 }
3321
3322 default:
3323 break;
3324 }
3325 return p->rc;
3326 }else{
3327 return SQLITE_NOMEM;
3328 }
3329 }
3330
3331 /*
3332 ** Compare strings z1 and z2, returning 0 if they are identical, or non-zero
3333 ** otherwise. Either or both argument may be NULL. Two NULL values are
3334 ** considered equal, and NULL is considered distinct from all other values.
3335 */
rbuStrCompare(const char * z1,const char * z2)3336 static int rbuStrCompare(const char *z1, const char *z2){
3337 if( z1==0 && z2==0 ) return 0;
3338 if( z1==0 || z2==0 ) return 1;
3339 return (sqlite3_stricmp(z1, z2)!=0);
3340 }
3341
3342 /*
3343 ** This function is called as part of sqlite3rbu_open() when initializing
3344 ** an rbu handle in OAL stage. If the rbu update has not started (i.e.
3345 ** the rbu_state table was empty) it is a no-op. Otherwise, it arranges
3346 ** things so that the next call to sqlite3rbu_step() continues on from
3347 ** where the previous rbu handle left off.
3348 **
3349 ** If an error occurs, an error code and error message are left in the
3350 ** rbu handle passed as the first argument.
3351 */
rbuSetupOal(sqlite3rbu * p,RbuState * pState)3352 static void rbuSetupOal(sqlite3rbu *p, RbuState *pState){
3353 assert( p->rc==SQLITE_OK );
3354 if( pState->zTbl ){
3355 RbuObjIter *pIter = &p->objiter;
3356 int rc = SQLITE_OK;
3357
3358 while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup
3359 || rbuStrCompare(pIter->zIdx, pState->zIdx)
3360 || rbuStrCompare(pIter->zTbl, pState->zTbl)
3361 )){
3362 rc = rbuObjIterNext(p, pIter);
3363 }
3364
3365 if( rc==SQLITE_OK && !pIter->zTbl ){
3366 rc = SQLITE_ERROR;
3367 p->zErrmsg = sqlite3_mprintf("rbu_state mismatch error");
3368 }
3369
3370 if( rc==SQLITE_OK ){
3371 p->nStep = pState->nRow;
3372 rc = rbuObjIterPrepareAll(p, &p->objiter, p->nStep);
3373 }
3374
3375 p->rc = rc;
3376 }
3377 }
3378
3379 /*
3380 ** If there is a "*-oal" file in the file-system corresponding to the
3381 ** target database in the file-system, delete it. If an error occurs,
3382 ** leave an error code and error message in the rbu handle.
3383 */
rbuDeleteOalFile(sqlite3rbu * p)3384 static void rbuDeleteOalFile(sqlite3rbu *p){
3385 char *zOal = rbuMPrintf(p, "%s-oal", p->zTarget);
3386 if( zOal ){
3387 sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
3388 assert( pVfs && p->rc==SQLITE_OK && p->zErrmsg==0 );
3389 pVfs->xDelete(pVfs, zOal, 0);
3390 sqlite3_free(zOal);
3391 }
3392 }
3393
3394 /*
3395 ** Allocate a private rbu VFS for the rbu handle passed as the only
3396 ** argument. This VFS will be used unless the call to sqlite3rbu_open()
3397 ** specified a URI with a vfs=? option in place of a target database
3398 ** file name.
3399 */
rbuCreateVfs(sqlite3rbu * p)3400 static void rbuCreateVfs(sqlite3rbu *p){
3401 int rnd;
3402 char zRnd[64];
3403
3404 assert( p->rc==SQLITE_OK );
3405 sqlite3_randomness(sizeof(int), (void*)&rnd);
3406 sqlite3_snprintf(sizeof(zRnd), zRnd, "rbu_vfs_%d", rnd);
3407 p->rc = sqlite3rbu_create_vfs(zRnd, 0);
3408 if( p->rc==SQLITE_OK ){
3409 sqlite3_vfs *pVfs = sqlite3_vfs_find(zRnd);
3410 assert( pVfs );
3411 p->zVfsName = pVfs->zName;
3412 }
3413 }
3414
3415 /*
3416 ** Destroy the private VFS created for the rbu handle passed as the only
3417 ** argument by an earlier call to rbuCreateVfs().
3418 */
rbuDeleteVfs(sqlite3rbu * p)3419 static void rbuDeleteVfs(sqlite3rbu *p){
3420 if( p->zVfsName ){
3421 sqlite3rbu_destroy_vfs(p->zVfsName);
3422 p->zVfsName = 0;
3423 }
3424 }
3425
3426 /*
3427 ** This user-defined SQL function is invoked with a single argument - the
3428 ** name of a table expected to appear in the target database. It returns
3429 ** the number of auxilliary indexes on the table.
3430 */
rbuIndexCntFunc(sqlite3_context * pCtx,int nVal,sqlite3_value ** apVal)3431 static void rbuIndexCntFunc(
3432 sqlite3_context *pCtx,
3433 int nVal,
3434 sqlite3_value **apVal
3435 ){
3436 sqlite3rbu *p = (sqlite3rbu*)sqlite3_user_data(pCtx);
3437 sqlite3_stmt *pStmt = 0;
3438 char *zErrmsg = 0;
3439 int rc;
3440
3441 assert( nVal==1 );
3442
3443 rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &zErrmsg,
3444 sqlite3_mprintf("SELECT count(*) FROM sqlite_master "
3445 "WHERE type='index' AND tbl_name = %Q", sqlite3_value_text(apVal[0]))
3446 );
3447 if( rc!=SQLITE_OK ){
3448 sqlite3_result_error(pCtx, zErrmsg, -1);
3449 }else{
3450 int nIndex = 0;
3451 if( SQLITE_ROW==sqlite3_step(pStmt) ){
3452 nIndex = sqlite3_column_int(pStmt, 0);
3453 }
3454 rc = sqlite3_finalize(pStmt);
3455 if( rc==SQLITE_OK ){
3456 sqlite3_result_int(pCtx, nIndex);
3457 }else{
3458 sqlite3_result_error(pCtx, sqlite3_errmsg(p->dbMain), -1);
3459 }
3460 }
3461
3462 sqlite3_free(zErrmsg);
3463 }
3464
3465 /*
3466 ** If the RBU database contains the rbu_count table, use it to initialize
3467 ** the sqlite3rbu.nPhaseOneStep variable. The schema of the rbu_count table
3468 ** is assumed to contain the same columns as:
3469 **
3470 ** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID;
3471 **
3472 ** There should be one row in the table for each data_xxx table in the
3473 ** database. The 'tbl' column should contain the name of a data_xxx table,
3474 ** and the cnt column the number of rows it contains.
3475 **
3476 ** sqlite3rbu.nPhaseOneStep is initialized to the sum of (1 + nIndex) * cnt
3477 ** for all rows in the rbu_count table, where nIndex is the number of
3478 ** indexes on the corresponding target database table.
3479 */
rbuInitPhaseOneSteps(sqlite3rbu * p)3480 static void rbuInitPhaseOneSteps(sqlite3rbu *p){
3481 if( p->rc==SQLITE_OK ){
3482 sqlite3_stmt *pStmt = 0;
3483 int bExists = 0; /* True if rbu_count exists */
3484
3485 p->nPhaseOneStep = -1;
3486
3487 p->rc = sqlite3_create_function(p->dbRbu,
3488 "rbu_index_cnt", 1, SQLITE_UTF8, (void*)p, rbuIndexCntFunc, 0, 0
3489 );
3490
3491 /* Check for the rbu_count table. If it does not exist, or if an error
3492 ** occurs, nPhaseOneStep will be left set to -1. */
3493 if( p->rc==SQLITE_OK ){
3494 p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
3495 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'rbu_count'"
3496 );
3497 }
3498 if( p->rc==SQLITE_OK ){
3499 if( SQLITE_ROW==sqlite3_step(pStmt) ){
3500 bExists = 1;
3501 }
3502 p->rc = sqlite3_finalize(pStmt);
3503 }
3504
3505 if( p->rc==SQLITE_OK && bExists ){
3506 p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
3507 "SELECT sum(cnt * (1 + rbu_index_cnt(rbu_target_name(tbl))))"
3508 "FROM rbu_count"
3509 );
3510 if( p->rc==SQLITE_OK ){
3511 if( SQLITE_ROW==sqlite3_step(pStmt) ){
3512 p->nPhaseOneStep = sqlite3_column_int64(pStmt, 0);
3513 }
3514 p->rc = sqlite3_finalize(pStmt);
3515 }
3516 }
3517 }
3518 }
3519
3520
openRbuHandle(const char * zTarget,const char * zRbu,const char * zState)3521 static sqlite3rbu *openRbuHandle(
3522 const char *zTarget,
3523 const char *zRbu,
3524 const char *zState
3525 ){
3526 sqlite3rbu *p;
3527 size_t nTarget = zTarget ? strlen(zTarget) : 0;
3528 size_t nRbu = strlen(zRbu);
3529 size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1;
3530
3531 p = (sqlite3rbu*)sqlite3_malloc64(nByte);
3532 if( p ){
3533 RbuState *pState = 0;
3534
3535 /* Create the custom VFS. */
3536 memset(p, 0, sizeof(sqlite3rbu));
3537 rbuCreateVfs(p);
3538
3539 /* Open the target, RBU and state databases */
3540 if( p->rc==SQLITE_OK ){
3541 char *pCsr = (char*)&p[1];
3542 int bRetry = 0;
3543 if( zTarget ){
3544 p->zTarget = pCsr;
3545 memcpy(p->zTarget, zTarget, nTarget+1);
3546 pCsr += nTarget+1;
3547 }
3548 p->zRbu = pCsr;
3549 memcpy(p->zRbu, zRbu, nRbu+1);
3550 pCsr += nRbu+1;
3551 if( zState ){
3552 p->zState = rbuMPrintf(p, "%s", zState);
3553 }
3554
3555 /* If the first attempt to open the database file fails and the bRetry
3556 ** flag it set, this means that the db was not opened because it seemed
3557 ** to be a wal-mode db. But, this may have happened due to an earlier
3558 ** RBU vacuum operation leaving an old wal file in the directory.
3559 ** If this is the case, it will have been checkpointed and deleted
3560 ** when the handle was closed and a second attempt to open the
3561 ** database may succeed. */
3562 rbuOpenDatabase(p, &bRetry);
3563 if( bRetry ){
3564 rbuOpenDatabase(p, 0);
3565 }
3566 }
3567
3568 if( p->rc==SQLITE_OK ){
3569 pState = rbuLoadState(p);
3570 assert( pState || p->rc!=SQLITE_OK );
3571 if( p->rc==SQLITE_OK ){
3572
3573 if( pState->eStage==0 ){
3574 rbuDeleteOalFile(p);
3575 rbuInitPhaseOneSteps(p);
3576 p->eStage = RBU_STAGE_OAL;
3577 }else{
3578 p->eStage = pState->eStage;
3579 p->nPhaseOneStep = pState->nPhaseOneStep;
3580 }
3581 p->nProgress = pState->nProgress;
3582 p->iOalSz = pState->iOalSz;
3583 }
3584 }
3585 assert( p->rc!=SQLITE_OK || p->eStage!=0 );
3586
3587 if( p->rc==SQLITE_OK && p->pTargetFd->pWalFd ){
3588 if( p->eStage==RBU_STAGE_OAL ){
3589 p->rc = SQLITE_ERROR;
3590 p->zErrmsg = sqlite3_mprintf("cannot update wal mode database");
3591 }else if( p->eStage==RBU_STAGE_MOVE ){
3592 p->eStage = RBU_STAGE_CKPT;
3593 p->nStep = 0;
3594 }
3595 }
3596
3597 if( p->rc==SQLITE_OK
3598 && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE)
3599 && pState->eStage!=0
3600 ){
3601 rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd);
3602 if( pFd->iCookie!=pState->iCookie ){
3603 /* At this point (pTargetFd->iCookie) contains the value of the
3604 ** change-counter cookie (the thing that gets incremented when a
3605 ** transaction is committed in rollback mode) currently stored on
3606 ** page 1 of the database file. */
3607 p->rc = SQLITE_BUSY;
3608 p->zErrmsg = sqlite3_mprintf("database modified during rbu %s",
3609 (rbuIsVacuum(p) ? "vacuum" : "update")
3610 );
3611 }
3612 }
3613
3614 if( p->rc==SQLITE_OK ){
3615 if( p->eStage==RBU_STAGE_OAL ){
3616 sqlite3 *db = p->dbMain;
3617 p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg);
3618
3619 /* Point the object iterator at the first object */
3620 if( p->rc==SQLITE_OK ){
3621 p->rc = rbuObjIterFirst(p, &p->objiter);
3622 }
3623
3624 /* If the RBU database contains no data_xxx tables, declare the RBU
3625 ** update finished. */
3626 if( p->rc==SQLITE_OK && p->objiter.zTbl==0 ){
3627 p->rc = SQLITE_DONE;
3628 p->eStage = RBU_STAGE_DONE;
3629 }else{
3630 if( p->rc==SQLITE_OK && pState->eStage==0 && rbuIsVacuum(p) ){
3631 rbuCopyPragma(p, "page_size");
3632 rbuCopyPragma(p, "auto_vacuum");
3633 }
3634
3635 /* Open transactions both databases. The *-oal file is opened or
3636 ** created at this point. */
3637 if( p->rc==SQLITE_OK ){
3638 p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
3639 }
3640
3641 /* Check if the main database is a zipvfs db. If it is, set the upper
3642 ** level pager to use "journal_mode=off". This prevents it from
3643 ** generating a large journal using a temp file. */
3644 if( p->rc==SQLITE_OK ){
3645 int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0);
3646 if( frc==SQLITE_OK ){
3647 p->rc = sqlite3_exec(
3648 db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg);
3649 }
3650 }
3651
3652 if( p->rc==SQLITE_OK ){
3653 rbuSetupOal(p, pState);
3654 }
3655 }
3656 }else if( p->eStage==RBU_STAGE_MOVE ){
3657 /* no-op */
3658 }else if( p->eStage==RBU_STAGE_CKPT ){
3659 rbuSetupCheckpoint(p, pState);
3660 }else if( p->eStage==RBU_STAGE_DONE ){
3661 p->rc = SQLITE_DONE;
3662 }else{
3663 p->rc = SQLITE_CORRUPT;
3664 }
3665 }
3666
3667 rbuFreeState(pState);
3668 }
3669
3670 return p;
3671 }
3672
3673 /*
3674 ** Allocate and return an RBU handle with all fields zeroed except for the
3675 ** error code, which is set to SQLITE_MISUSE.
3676 */
rbuMisuseError(void)3677 static sqlite3rbu *rbuMisuseError(void){
3678 sqlite3rbu *pRet;
3679 pRet = sqlite3_malloc64(sizeof(sqlite3rbu));
3680 if( pRet ){
3681 memset(pRet, 0, sizeof(sqlite3rbu));
3682 pRet->rc = SQLITE_MISUSE;
3683 }
3684 return pRet;
3685 }
3686
3687 /*
3688 ** Open and return a new RBU handle.
3689 */
sqlite3rbu_open(const char * zTarget,const char * zRbu,const char * zState)3690 sqlite3rbu *sqlite3rbu_open(
3691 const char *zTarget,
3692 const char *zRbu,
3693 const char *zState
3694 ){
3695 if( zTarget==0 || zRbu==0 ){ return rbuMisuseError(); }
3696 /* TODO: Check that zTarget and zRbu are non-NULL */
3697 return openRbuHandle(zTarget, zRbu, zState);
3698 }
3699
3700 /*
3701 ** Open a handle to begin or resume an RBU VACUUM operation.
3702 */
sqlite3rbu_vacuum(const char * zTarget,const char * zState)3703 sqlite3rbu *sqlite3rbu_vacuum(
3704 const char *zTarget,
3705 const char *zState
3706 ){
3707 if( zTarget==0 ){ return rbuMisuseError(); }
3708 /* TODO: Check that both arguments are non-NULL */
3709 return openRbuHandle(0, zTarget, zState);
3710 }
3711
3712 /*
3713 ** Return the database handle used by pRbu.
3714 */
sqlite3rbu_db(sqlite3rbu * pRbu,int bRbu)3715 sqlite3 *sqlite3rbu_db(sqlite3rbu *pRbu, int bRbu){
3716 sqlite3 *db = 0;
3717 if( pRbu ){
3718 db = (bRbu ? pRbu->dbRbu : pRbu->dbMain);
3719 }
3720 return db;
3721 }
3722
3723
3724 /*
3725 ** If the error code currently stored in the RBU handle is SQLITE_CONSTRAINT,
3726 ** then edit any error message string so as to remove all occurrences of
3727 ** the pattern "rbu_imp_[0-9]*".
3728 */
rbuEditErrmsg(sqlite3rbu * p)3729 static void rbuEditErrmsg(sqlite3rbu *p){
3730 if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){
3731 unsigned int i;
3732 size_t nErrmsg = strlen(p->zErrmsg);
3733 for(i=0; i<(nErrmsg-8); i++){
3734 if( memcmp(&p->zErrmsg[i], "rbu_imp_", 8)==0 ){
3735 int nDel = 8;
3736 while( p->zErrmsg[i+nDel]>='0' && p->zErrmsg[i+nDel]<='9' ) nDel++;
3737 memmove(&p->zErrmsg[i], &p->zErrmsg[i+nDel], nErrmsg + 1 - i - nDel);
3738 nErrmsg -= nDel;
3739 }
3740 }
3741 }
3742 }
3743
3744 /*
3745 ** Close the RBU handle.
3746 */
sqlite3rbu_close(sqlite3rbu * p,char ** pzErrmsg)3747 int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){
3748 int rc;
3749 if( p ){
3750
3751 /* Commit the transaction to the *-oal file. */
3752 if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
3753 p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg);
3754 }
3755
3756 /* Sync the db file if currently doing an incremental checkpoint */
3757 if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){
3758 sqlite3_file *pDb = p->pTargetFd->pReal;
3759 p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
3760 }
3761
3762 rbuSaveState(p, p->eStage);
3763
3764 if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
3765 p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg);
3766 }
3767
3768 /* Close any open statement handles. */
3769 rbuObjIterFinalize(&p->objiter);
3770
3771 /* If this is an RBU vacuum handle and the vacuum has either finished
3772 ** successfully or encountered an error, delete the contents of the
3773 ** state table. This causes the next call to sqlite3rbu_vacuum()
3774 ** specifying the current target and state databases to start a new
3775 ** vacuum from scratch. */
3776 if( rbuIsVacuum(p) && p->rc!=SQLITE_OK && p->dbRbu ){
3777 int rc2 = sqlite3_exec(p->dbRbu, "DELETE FROM stat.rbu_state", 0, 0, 0);
3778 if( p->rc==SQLITE_DONE && rc2!=SQLITE_OK ) p->rc = rc2;
3779 }
3780
3781 /* Close the open database handle and VFS object. */
3782 sqlite3_close(p->dbRbu);
3783 sqlite3_close(p->dbMain);
3784 rbuDeleteVfs(p);
3785 sqlite3_free(p->aBuf);
3786 sqlite3_free(p->aFrame);
3787
3788 rbuEditErrmsg(p);
3789 rc = p->rc;
3790 if( pzErrmsg ){
3791 *pzErrmsg = p->zErrmsg;
3792 }else{
3793 sqlite3_free(p->zErrmsg);
3794 }
3795 sqlite3_free(p->zState);
3796 sqlite3_free(p);
3797 }else{
3798 rc = SQLITE_NOMEM;
3799 *pzErrmsg = 0;
3800 }
3801 return rc;
3802 }
3803
3804 /*
3805 ** Return the total number of key-value operations (inserts, deletes or
3806 ** updates) that have been performed on the target database since the
3807 ** current RBU update was started.
3808 */
sqlite3rbu_progress(sqlite3rbu * pRbu)3809 sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){
3810 return pRbu->nProgress;
3811 }
3812
3813 /*
3814 ** Return permyriadage progress indications for the two main stages of
3815 ** an RBU update.
3816 */
sqlite3rbu_bp_progress(sqlite3rbu * p,int * pnOne,int * pnTwo)3817 void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){
3818 const int MAX_PROGRESS = 10000;
3819 switch( p->eStage ){
3820 case RBU_STAGE_OAL:
3821 if( p->nPhaseOneStep>0 ){
3822 *pnOne = (int)(MAX_PROGRESS * (i64)p->nProgress/(i64)p->nPhaseOneStep);
3823 }else{
3824 *pnOne = -1;
3825 }
3826 *pnTwo = 0;
3827 break;
3828
3829 case RBU_STAGE_MOVE:
3830 *pnOne = MAX_PROGRESS;
3831 *pnTwo = 0;
3832 break;
3833
3834 case RBU_STAGE_CKPT:
3835 *pnOne = MAX_PROGRESS;
3836 *pnTwo = (int)(MAX_PROGRESS * (i64)p->nStep / (i64)p->nFrame);
3837 break;
3838
3839 case RBU_STAGE_DONE:
3840 *pnOne = MAX_PROGRESS;
3841 *pnTwo = MAX_PROGRESS;
3842 break;
3843
3844 default:
3845 assert( 0 );
3846 }
3847 }
3848
3849 /*
3850 ** Return the current state of the RBU vacuum or update operation.
3851 */
sqlite3rbu_state(sqlite3rbu * p)3852 int sqlite3rbu_state(sqlite3rbu *p){
3853 int aRes[] = {
3854 0, SQLITE_RBU_STATE_OAL, SQLITE_RBU_STATE_MOVE,
3855 0, SQLITE_RBU_STATE_CHECKPOINT, SQLITE_RBU_STATE_DONE
3856 };
3857
3858 assert( RBU_STAGE_OAL==1 );
3859 assert( RBU_STAGE_MOVE==2 );
3860 assert( RBU_STAGE_CKPT==4 );
3861 assert( RBU_STAGE_DONE==5 );
3862 assert( aRes[RBU_STAGE_OAL]==SQLITE_RBU_STATE_OAL );
3863 assert( aRes[RBU_STAGE_MOVE]==SQLITE_RBU_STATE_MOVE );
3864 assert( aRes[RBU_STAGE_CKPT]==SQLITE_RBU_STATE_CHECKPOINT );
3865 assert( aRes[RBU_STAGE_DONE]==SQLITE_RBU_STATE_DONE );
3866
3867 if( p->rc!=SQLITE_OK && p->rc!=SQLITE_DONE ){
3868 return SQLITE_RBU_STATE_ERROR;
3869 }else{
3870 assert( p->rc!=SQLITE_DONE || p->eStage==RBU_STAGE_DONE );
3871 assert( p->eStage==RBU_STAGE_OAL
3872 || p->eStage==RBU_STAGE_MOVE
3873 || p->eStage==RBU_STAGE_CKPT
3874 || p->eStage==RBU_STAGE_DONE
3875 );
3876 return aRes[p->eStage];
3877 }
3878 }
3879
sqlite3rbu_savestate(sqlite3rbu * p)3880 int sqlite3rbu_savestate(sqlite3rbu *p){
3881 int rc = p->rc;
3882 if( rc==SQLITE_DONE ) return SQLITE_OK;
3883
3884 assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE );
3885 if( p->eStage==RBU_STAGE_OAL ){
3886 assert( rc!=SQLITE_DONE );
3887 if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0);
3888 }
3889
3890 /* Sync the db file */
3891 if( rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){
3892 sqlite3_file *pDb = p->pTargetFd->pReal;
3893 rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL);
3894 }
3895
3896 p->rc = rc;
3897 rbuSaveState(p, p->eStage);
3898 rc = p->rc;
3899
3900 if( p->eStage==RBU_STAGE_OAL ){
3901 assert( rc!=SQLITE_DONE );
3902 if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0);
3903 if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, 0);
3904 if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "BEGIN IMMEDIATE", 0, 0,0);
3905 }
3906
3907 p->rc = rc;
3908 return rc;
3909 }
3910
3911 /**************************************************************************
3912 ** Beginning of RBU VFS shim methods. The VFS shim modifies the behaviour
3913 ** of a standard VFS in the following ways:
3914 **
3915 ** 1. Whenever the first page of a main database file is read or
3916 ** written, the value of the change-counter cookie is stored in
3917 ** rbu_file.iCookie. Similarly, the value of the "write-version"
3918 ** database header field is stored in rbu_file.iWriteVer. This ensures
3919 ** that the values are always trustworthy within an open transaction.
3920 **
3921 ** 2. Whenever an SQLITE_OPEN_WAL file is opened, the (rbu_file.pWalFd)
3922 ** member variable of the associated database file descriptor is set
3923 ** to point to the new file. A mutex protected linked list of all main
3924 ** db fds opened using a particular RBU VFS is maintained at
3925 ** rbu_vfs.pMain to facilitate this.
3926 **
3927 ** 3. Using a new file-control "SQLITE_FCNTL_RBU", a main db rbu_file
3928 ** object can be marked as the target database of an RBU update. This
3929 ** turns on the following extra special behaviour:
3930 **
3931 ** 3a. If xAccess() is called to check if there exists a *-wal file
3932 ** associated with an RBU target database currently in RBU_STAGE_OAL
3933 ** stage (preparing the *-oal file), the following special handling
3934 ** applies:
3935 **
3936 ** * if the *-wal file does exist, return SQLITE_CANTOPEN. An RBU
3937 ** target database may not be in wal mode already.
3938 **
3939 ** * if the *-wal file does not exist, set the output parameter to
3940 ** non-zero (to tell SQLite that it does exist) anyway.
3941 **
3942 ** Then, when xOpen() is called to open the *-wal file associated with
3943 ** the RBU target in RBU_STAGE_OAL stage, instead of opening the *-wal
3944 ** file, the rbu vfs opens the corresponding *-oal file instead.
3945 **
3946 ** 3b. The *-shm pages returned by xShmMap() for a target db file in
3947 ** RBU_STAGE_OAL mode are actually stored in heap memory. This is to
3948 ** avoid creating a *-shm file on disk. Additionally, xShmLock() calls
3949 ** are no-ops on target database files in RBU_STAGE_OAL mode. This is
3950 ** because assert() statements in some VFS implementations fail if
3951 ** xShmLock() is called before xShmMap().
3952 **
3953 ** 3c. If an EXCLUSIVE lock is attempted on a target database file in any
3954 ** mode except RBU_STAGE_DONE (all work completed and checkpointed), it
3955 ** fails with an SQLITE_BUSY error. This is to stop RBU connections
3956 ** from automatically checkpointing a *-wal (or *-oal) file from within
3957 ** sqlite3_close().
3958 **
3959 ** 3d. In RBU_STAGE_CAPTURE mode, all xRead() calls on the wal file, and
3960 ** all xWrite() calls on the target database file perform no IO.
3961 ** Instead the frame and page numbers that would be read and written
3962 ** are recorded. Additionally, successful attempts to obtain exclusive
3963 ** xShmLock() WRITER, CHECKPOINTER and READ0 locks on the target
3964 ** database file are recorded. xShmLock() calls to unlock the same
3965 ** locks are no-ops (so that once obtained, these locks are never
3966 ** relinquished). Finally, calls to xSync() on the target database
3967 ** file fail with SQLITE_INTERNAL errors.
3968 */
3969
rbuUnlockShm(rbu_file * p)3970 static void rbuUnlockShm(rbu_file *p){
3971 if( p->pRbu ){
3972 int (*xShmLock)(sqlite3_file*,int,int,int) = p->pReal->pMethods->xShmLock;
3973 int i;
3974 for(i=0; i<SQLITE_SHM_NLOCK;i++){
3975 if( (1<<i) & p->pRbu->mLock ){
3976 xShmLock(p->pReal, i, 1, SQLITE_SHM_UNLOCK|SQLITE_SHM_EXCLUSIVE);
3977 }
3978 }
3979 p->pRbu->mLock = 0;
3980 }
3981 }
3982
3983 /*
3984 ** Close an rbu file.
3985 */
rbuVfsClose(sqlite3_file * pFile)3986 static int rbuVfsClose(sqlite3_file *pFile){
3987 rbu_file *p = (rbu_file*)pFile;
3988 int rc;
3989 int i;
3990
3991 /* Free the contents of the apShm[] array. And the array itself. */
3992 for(i=0; i<p->nShm; i++){
3993 sqlite3_free(p->apShm[i]);
3994 }
3995 sqlite3_free(p->apShm);
3996 p->apShm = 0;
3997 sqlite3_free(p->zDel);
3998
3999 if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
4000 rbu_file **pp;
4001 sqlite3_mutex_enter(p->pRbuVfs->mutex);
4002 for(pp=&p->pRbuVfs->pMain; *pp!=p; pp=&((*pp)->pMainNext));
4003 *pp = p->pMainNext;
4004 sqlite3_mutex_leave(p->pRbuVfs->mutex);
4005 rbuUnlockShm(p);
4006 p->pReal->pMethods->xShmUnmap(p->pReal, 0);
4007 }
4008
4009 /* Close the underlying file handle */
4010 rc = p->pReal->pMethods->xClose(p->pReal);
4011 return rc;
4012 }
4013
4014
4015 /*
4016 ** Read and return an unsigned 32-bit big-endian integer from the buffer
4017 ** passed as the only argument.
4018 */
rbuGetU32(u8 * aBuf)4019 static u32 rbuGetU32(u8 *aBuf){
4020 return ((u32)aBuf[0] << 24)
4021 + ((u32)aBuf[1] << 16)
4022 + ((u32)aBuf[2] << 8)
4023 + ((u32)aBuf[3]);
4024 }
4025
4026 /*
4027 ** Write an unsigned 32-bit value in big-endian format to the supplied
4028 ** buffer.
4029 */
rbuPutU32(u8 * aBuf,u32 iVal)4030 static void rbuPutU32(u8 *aBuf, u32 iVal){
4031 aBuf[0] = (iVal >> 24) & 0xFF;
4032 aBuf[1] = (iVal >> 16) & 0xFF;
4033 aBuf[2] = (iVal >> 8) & 0xFF;
4034 aBuf[3] = (iVal >> 0) & 0xFF;
4035 }
4036
rbuPutU16(u8 * aBuf,u16 iVal)4037 static void rbuPutU16(u8 *aBuf, u16 iVal){
4038 aBuf[0] = (iVal >> 8) & 0xFF;
4039 aBuf[1] = (iVal >> 0) & 0xFF;
4040 }
4041
4042 /*
4043 ** Read data from an rbuVfs-file.
4044 */
rbuVfsRead(sqlite3_file * pFile,void * zBuf,int iAmt,sqlite_int64 iOfst)4045 static int rbuVfsRead(
4046 sqlite3_file *pFile,
4047 void *zBuf,
4048 int iAmt,
4049 sqlite_int64 iOfst
4050 ){
4051 rbu_file *p = (rbu_file*)pFile;
4052 sqlite3rbu *pRbu = p->pRbu;
4053 int rc;
4054
4055 if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){
4056 assert( p->openFlags & SQLITE_OPEN_WAL );
4057 rc = rbuCaptureWalRead(p->pRbu, iOfst, iAmt);
4058 }else{
4059 if( pRbu && pRbu->eStage==RBU_STAGE_OAL
4060 && (p->openFlags & SQLITE_OPEN_WAL)
4061 && iOfst>=pRbu->iOalSz
4062 ){
4063 rc = SQLITE_OK;
4064 memset(zBuf, 0, iAmt);
4065 }else{
4066 rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst);
4067 #if 1
4068 /* If this is being called to read the first page of the target
4069 ** database as part of an rbu vacuum operation, synthesize the
4070 ** contents of the first page if it does not yet exist. Otherwise,
4071 ** SQLite will not check for a *-wal file. */
4072 if( pRbu && rbuIsVacuum(pRbu)
4073 && rc==SQLITE_IOERR_SHORT_READ && iOfst==0
4074 && (p->openFlags & SQLITE_OPEN_MAIN_DB)
4075 && pRbu->rc==SQLITE_OK
4076 ){
4077 sqlite3_file *pFd = (sqlite3_file*)pRbu->pRbuFd;
4078 rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst);
4079 if( rc==SQLITE_OK ){
4080 u8 *aBuf = (u8*)zBuf;
4081 u32 iRoot = rbuGetU32(&aBuf[52]) ? 1 : 0;
4082 rbuPutU32(&aBuf[52], iRoot); /* largest root page number */
4083 rbuPutU32(&aBuf[36], 0); /* number of free pages */
4084 rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */
4085 rbuPutU32(&aBuf[28], 1); /* size of db file in pages */
4086 rbuPutU32(&aBuf[24], pRbu->pRbuFd->iCookie+1); /* Change counter */
4087
4088 if( iAmt>100 ){
4089 memset(&aBuf[100], 0, iAmt-100);
4090 rbuPutU16(&aBuf[105], iAmt & 0xFFFF);
4091 aBuf[100] = 0x0D;
4092 }
4093 }
4094 }
4095 #endif
4096 }
4097 if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
4098 /* These look like magic numbers. But they are stable, as they are part
4099 ** of the definition of the SQLite file format, which may not change. */
4100 u8 *pBuf = (u8*)zBuf;
4101 p->iCookie = rbuGetU32(&pBuf[24]);
4102 p->iWriteVer = pBuf[19];
4103 }
4104 }
4105 return rc;
4106 }
4107
4108 /*
4109 ** Write data to an rbuVfs-file.
4110 */
rbuVfsWrite(sqlite3_file * pFile,const void * zBuf,int iAmt,sqlite_int64 iOfst)4111 static int rbuVfsWrite(
4112 sqlite3_file *pFile,
4113 const void *zBuf,
4114 int iAmt,
4115 sqlite_int64 iOfst
4116 ){
4117 rbu_file *p = (rbu_file*)pFile;
4118 sqlite3rbu *pRbu = p->pRbu;
4119 int rc;
4120
4121 if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){
4122 assert( p->openFlags & SQLITE_OPEN_MAIN_DB );
4123 rc = rbuCaptureDbWrite(p->pRbu, iOfst);
4124 }else{
4125 if( pRbu && pRbu->eStage==RBU_STAGE_OAL
4126 && (p->openFlags & SQLITE_OPEN_WAL)
4127 && iOfst>=pRbu->iOalSz
4128 ){
4129 pRbu->iOalSz = iAmt + iOfst;
4130 }
4131 rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst);
4132 if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
4133 /* These look like magic numbers. But they are stable, as they are part
4134 ** of the definition of the SQLite file format, which may not change. */
4135 u8 *pBuf = (u8*)zBuf;
4136 p->iCookie = rbuGetU32(&pBuf[24]);
4137 p->iWriteVer = pBuf[19];
4138 }
4139 }
4140 return rc;
4141 }
4142
4143 /*
4144 ** Truncate an rbuVfs-file.
4145 */
rbuVfsTruncate(sqlite3_file * pFile,sqlite_int64 size)4146 static int rbuVfsTruncate(sqlite3_file *pFile, sqlite_int64 size){
4147 rbu_file *p = (rbu_file*)pFile;
4148 return p->pReal->pMethods->xTruncate(p->pReal, size);
4149 }
4150
4151 /*
4152 ** Sync an rbuVfs-file.
4153 */
rbuVfsSync(sqlite3_file * pFile,int flags)4154 static int rbuVfsSync(sqlite3_file *pFile, int flags){
4155 rbu_file *p = (rbu_file *)pFile;
4156 if( p->pRbu && p->pRbu->eStage==RBU_STAGE_CAPTURE ){
4157 if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
4158 return SQLITE_INTERNAL;
4159 }
4160 return SQLITE_OK;
4161 }
4162 return p->pReal->pMethods->xSync(p->pReal, flags);
4163 }
4164
4165 /*
4166 ** Return the current file-size of an rbuVfs-file.
4167 */
rbuVfsFileSize(sqlite3_file * pFile,sqlite_int64 * pSize)4168 static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
4169 rbu_file *p = (rbu_file *)pFile;
4170 int rc;
4171 rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
4172
4173 /* If this is an RBU vacuum operation and this is the target database,
4174 ** pretend that it has at least one page. Otherwise, SQLite will not
4175 ** check for the existance of a *-wal file. rbuVfsRead() contains
4176 ** similar logic. */
4177 if( rc==SQLITE_OK && *pSize==0
4178 && p->pRbu && rbuIsVacuum(p->pRbu)
4179 && (p->openFlags & SQLITE_OPEN_MAIN_DB)
4180 ){
4181 *pSize = 1024;
4182 }
4183 return rc;
4184 }
4185
4186 /*
4187 ** Lock an rbuVfs-file.
4188 */
rbuVfsLock(sqlite3_file * pFile,int eLock)4189 static int rbuVfsLock(sqlite3_file *pFile, int eLock){
4190 rbu_file *p = (rbu_file*)pFile;
4191 sqlite3rbu *pRbu = p->pRbu;
4192 int rc = SQLITE_OK;
4193
4194 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
4195 if( eLock==SQLITE_LOCK_EXCLUSIVE
4196 && (p->bNolock || (pRbu && pRbu->eStage!=RBU_STAGE_DONE))
4197 ){
4198 /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this
4199 ** prevents it from checkpointing the database from sqlite3_close(). */
4200 rc = SQLITE_BUSY;
4201 }else{
4202 rc = p->pReal->pMethods->xLock(p->pReal, eLock);
4203 }
4204
4205 return rc;
4206 }
4207
4208 /*
4209 ** Unlock an rbuVfs-file.
4210 */
rbuVfsUnlock(sqlite3_file * pFile,int eLock)4211 static int rbuVfsUnlock(sqlite3_file *pFile, int eLock){
4212 rbu_file *p = (rbu_file *)pFile;
4213 return p->pReal->pMethods->xUnlock(p->pReal, eLock);
4214 }
4215
4216 /*
4217 ** Check if another file-handle holds a RESERVED lock on an rbuVfs-file.
4218 */
rbuVfsCheckReservedLock(sqlite3_file * pFile,int * pResOut)4219 static int rbuVfsCheckReservedLock(sqlite3_file *pFile, int *pResOut){
4220 rbu_file *p = (rbu_file *)pFile;
4221 return p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut);
4222 }
4223
4224 /*
4225 ** File control method. For custom operations on an rbuVfs-file.
4226 */
rbuVfsFileControl(sqlite3_file * pFile,int op,void * pArg)4227 static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){
4228 rbu_file *p = (rbu_file *)pFile;
4229 int (*xControl)(sqlite3_file*,int,void*) = p->pReal->pMethods->xFileControl;
4230 int rc;
4231
4232 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB)
4233 || p->openFlags & (SQLITE_OPEN_TRANSIENT_DB|SQLITE_OPEN_TEMP_JOURNAL)
4234 );
4235 if( op==SQLITE_FCNTL_RBU ){
4236 sqlite3rbu *pRbu = (sqlite3rbu*)pArg;
4237
4238 /* First try to find another RBU vfs lower down in the vfs stack. If
4239 ** one is found, this vfs will operate in pass-through mode. The lower
4240 ** level vfs will do the special RBU handling. */
4241 rc = xControl(p->pReal, op, pArg);
4242
4243 if( rc==SQLITE_NOTFOUND ){
4244 /* Now search for a zipvfs instance lower down in the VFS stack. If
4245 ** one is found, this is an error. */
4246 void *dummy = 0;
4247 rc = xControl(p->pReal, SQLITE_FCNTL_ZIPVFS, &dummy);
4248 if( rc==SQLITE_OK ){
4249 rc = SQLITE_ERROR;
4250 pRbu->zErrmsg = sqlite3_mprintf("rbu/zipvfs setup error");
4251 }else if( rc==SQLITE_NOTFOUND ){
4252 pRbu->pTargetFd = p;
4253 p->pRbu = pRbu;
4254 if( p->pWalFd ) p->pWalFd->pRbu = pRbu;
4255 rc = SQLITE_OK;
4256 }
4257 }
4258 return rc;
4259 }
4260 else if( op==SQLITE_FCNTL_RBUCNT ){
4261 sqlite3rbu *pRbu = (sqlite3rbu*)pArg;
4262 pRbu->nRbu++;
4263 pRbu->pRbuFd = p;
4264 p->bNolock = 1;
4265 }
4266
4267 rc = xControl(p->pReal, op, pArg);
4268 if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
4269 rbu_vfs *pRbuVfs = p->pRbuVfs;
4270 char *zIn = *(char**)pArg;
4271 char *zOut = sqlite3_mprintf("rbu(%s)/%z", pRbuVfs->base.zName, zIn);
4272 *(char**)pArg = zOut;
4273 if( zOut==0 ) rc = SQLITE_NOMEM;
4274 }
4275
4276 return rc;
4277 }
4278
4279 /*
4280 ** Return the sector-size in bytes for an rbuVfs-file.
4281 */
rbuVfsSectorSize(sqlite3_file * pFile)4282 static int rbuVfsSectorSize(sqlite3_file *pFile){
4283 rbu_file *p = (rbu_file *)pFile;
4284 return p->pReal->pMethods->xSectorSize(p->pReal);
4285 }
4286
4287 /*
4288 ** Return the device characteristic flags supported by an rbuVfs-file.
4289 */
rbuVfsDeviceCharacteristics(sqlite3_file * pFile)4290 static int rbuVfsDeviceCharacteristics(sqlite3_file *pFile){
4291 rbu_file *p = (rbu_file *)pFile;
4292 return p->pReal->pMethods->xDeviceCharacteristics(p->pReal);
4293 }
4294
4295 /*
4296 ** Take or release a shared-memory lock.
4297 */
rbuVfsShmLock(sqlite3_file * pFile,int ofst,int n,int flags)4298 static int rbuVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){
4299 rbu_file *p = (rbu_file*)pFile;
4300 sqlite3rbu *pRbu = p->pRbu;
4301 int rc = SQLITE_OK;
4302
4303 #ifdef SQLITE_AMALGAMATION
4304 assert( WAL_CKPT_LOCK==1 );
4305 #endif
4306
4307 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
4308 if( pRbu && (pRbu->eStage==RBU_STAGE_OAL || pRbu->eStage==RBU_STAGE_MOVE) ){
4309 /* Magic number 1 is the WAL_CKPT_LOCK lock. Preventing SQLite from
4310 ** taking this lock also prevents any checkpoints from occurring.
4311 ** todo: really, it's not clear why this might occur, as
4312 ** wal_autocheckpoint ought to be turned off. */
4313 if( ofst==WAL_LOCK_CKPT && n==1 ) rc = SQLITE_BUSY;
4314 }else{
4315 int bCapture = 0;
4316 if( n==1 && (flags & SQLITE_SHM_EXCLUSIVE)
4317 && pRbu && pRbu->eStage==RBU_STAGE_CAPTURE
4318 && (ofst==WAL_LOCK_WRITE || ofst==WAL_LOCK_CKPT || ofst==WAL_LOCK_READ0)
4319 ){
4320 bCapture = 1;
4321 }
4322
4323 if( bCapture==0 || 0==(flags & SQLITE_SHM_UNLOCK) ){
4324 rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags);
4325 if( bCapture && rc==SQLITE_OK ){
4326 pRbu->mLock |= (1 << ofst);
4327 }
4328 }
4329 }
4330
4331 return rc;
4332 }
4333
4334 /*
4335 ** Obtain a pointer to a mapping of a single 32KiB page of the *-shm file.
4336 */
rbuVfsShmMap(sqlite3_file * pFile,int iRegion,int szRegion,int isWrite,void volatile ** pp)4337 static int rbuVfsShmMap(
4338 sqlite3_file *pFile,
4339 int iRegion,
4340 int szRegion,
4341 int isWrite,
4342 void volatile **pp
4343 ){
4344 rbu_file *p = (rbu_file*)pFile;
4345 int rc = SQLITE_OK;
4346 int eStage = (p->pRbu ? p->pRbu->eStage : 0);
4347
4348 /* If not in RBU_STAGE_OAL, allow this call to pass through. Or, if this
4349 ** rbu is in the RBU_STAGE_OAL state, use heap memory for *-shm space
4350 ** instead of a file on disk. */
4351 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
4352 if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){
4353 if( iRegion<=p->nShm ){
4354 int nByte = (iRegion+1) * sizeof(char*);
4355 char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte);
4356 if( apNew==0 ){
4357 rc = SQLITE_NOMEM;
4358 }else{
4359 memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm));
4360 p->apShm = apNew;
4361 p->nShm = iRegion+1;
4362 }
4363 }
4364
4365 if( rc==SQLITE_OK && p->apShm[iRegion]==0 ){
4366 char *pNew = (char*)sqlite3_malloc64(szRegion);
4367 if( pNew==0 ){
4368 rc = SQLITE_NOMEM;
4369 }else{
4370 memset(pNew, 0, szRegion);
4371 p->apShm[iRegion] = pNew;
4372 }
4373 }
4374
4375 if( rc==SQLITE_OK ){
4376 *pp = p->apShm[iRegion];
4377 }else{
4378 *pp = 0;
4379 }
4380 }else{
4381 assert( p->apShm==0 );
4382 rc = p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp);
4383 }
4384
4385 return rc;
4386 }
4387
4388 /*
4389 ** Memory barrier.
4390 */
rbuVfsShmBarrier(sqlite3_file * pFile)4391 static void rbuVfsShmBarrier(sqlite3_file *pFile){
4392 rbu_file *p = (rbu_file *)pFile;
4393 p->pReal->pMethods->xShmBarrier(p->pReal);
4394 }
4395
4396 /*
4397 ** The xShmUnmap method.
4398 */
rbuVfsShmUnmap(sqlite3_file * pFile,int delFlag)4399 static int rbuVfsShmUnmap(sqlite3_file *pFile, int delFlag){
4400 rbu_file *p = (rbu_file*)pFile;
4401 int rc = SQLITE_OK;
4402 int eStage = (p->pRbu ? p->pRbu->eStage : 0);
4403
4404 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
4405 if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){
4406 /* no-op */
4407 }else{
4408 /* Release the checkpointer and writer locks */
4409 rbuUnlockShm(p);
4410 rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag);
4411 }
4412 return rc;
4413 }
4414
4415 /*
4416 ** Given that zWal points to a buffer containing a wal file name passed to
4417 ** either the xOpen() or xAccess() VFS method, return a pointer to the
4418 ** file-handle opened by the same database connection on the corresponding
4419 ** database file.
4420 */
rbuFindMaindb(rbu_vfs * pRbuVfs,const char * zWal)4421 static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal){
4422 rbu_file *pDb;
4423 sqlite3_mutex_enter(pRbuVfs->mutex);
4424 for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){}
4425 sqlite3_mutex_leave(pRbuVfs->mutex);
4426 return pDb;
4427 }
4428
4429 /*
4430 ** A main database named zName has just been opened. The following
4431 ** function returns a pointer to a buffer owned by SQLite that contains
4432 ** the name of the *-wal file this db connection will use. SQLite
4433 ** happens to pass a pointer to this buffer when using xAccess()
4434 ** or xOpen() to operate on the *-wal file.
4435 */
rbuMainToWal(const char * zName,int flags)4436 static const char *rbuMainToWal(const char *zName, int flags){
4437 int n = (int)strlen(zName);
4438 const char *z = &zName[n];
4439 if( flags & SQLITE_OPEN_URI ){
4440 int odd = 0;
4441 while( 1 ){
4442 if( z[0]==0 ){
4443 odd = 1 - odd;
4444 if( odd && z[1]==0 ) break;
4445 }
4446 z++;
4447 }
4448 z += 2;
4449 }else{
4450 while( *z==0 ) z++;
4451 }
4452 z += (n + 8 + 1);
4453 return z;
4454 }
4455
4456 /*
4457 ** Open an rbu file handle.
4458 */
rbuVfsOpen(sqlite3_vfs * pVfs,const char * zName,sqlite3_file * pFile,int flags,int * pOutFlags)4459 static int rbuVfsOpen(
4460 sqlite3_vfs *pVfs,
4461 const char *zName,
4462 sqlite3_file *pFile,
4463 int flags,
4464 int *pOutFlags
4465 ){
4466 static sqlite3_io_methods rbuvfs_io_methods = {
4467 2, /* iVersion */
4468 rbuVfsClose, /* xClose */
4469 rbuVfsRead, /* xRead */
4470 rbuVfsWrite, /* xWrite */
4471 rbuVfsTruncate, /* xTruncate */
4472 rbuVfsSync, /* xSync */
4473 rbuVfsFileSize, /* xFileSize */
4474 rbuVfsLock, /* xLock */
4475 rbuVfsUnlock, /* xUnlock */
4476 rbuVfsCheckReservedLock, /* xCheckReservedLock */
4477 rbuVfsFileControl, /* xFileControl */
4478 rbuVfsSectorSize, /* xSectorSize */
4479 rbuVfsDeviceCharacteristics, /* xDeviceCharacteristics */
4480 rbuVfsShmMap, /* xShmMap */
4481 rbuVfsShmLock, /* xShmLock */
4482 rbuVfsShmBarrier, /* xShmBarrier */
4483 rbuVfsShmUnmap, /* xShmUnmap */
4484 0, 0 /* xFetch, xUnfetch */
4485 };
4486 rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs;
4487 sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs;
4488 rbu_file *pFd = (rbu_file *)pFile;
4489 int rc = SQLITE_OK;
4490 const char *zOpen = zName;
4491 int oflags = flags;
4492
4493 memset(pFd, 0, sizeof(rbu_file));
4494 pFd->pReal = (sqlite3_file*)&pFd[1];
4495 pFd->pRbuVfs = pRbuVfs;
4496 pFd->openFlags = flags;
4497 if( zName ){
4498 if( flags & SQLITE_OPEN_MAIN_DB ){
4499 /* A main database has just been opened. The following block sets
4500 ** (pFd->zWal) to point to a buffer owned by SQLite that contains
4501 ** the name of the *-wal file this db connection will use. SQLite
4502 ** happens to pass a pointer to this buffer when using xAccess()
4503 ** or xOpen() to operate on the *-wal file. */
4504 pFd->zWal = rbuMainToWal(zName, flags);
4505 }
4506 else if( flags & SQLITE_OPEN_WAL ){
4507 rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName);
4508 if( pDb ){
4509 if( pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
4510 /* This call is to open a *-wal file. Intead, open the *-oal. This
4511 ** code ensures that the string passed to xOpen() is terminated by a
4512 ** pair of '\0' bytes in case the VFS attempts to extract a URI
4513 ** parameter from it. */
4514 const char *zBase = zName;
4515 size_t nCopy;
4516 char *zCopy;
4517 if( rbuIsVacuum(pDb->pRbu) ){
4518 zBase = sqlite3_db_filename(pDb->pRbu->dbRbu, "main");
4519 zBase = rbuMainToWal(zBase, SQLITE_OPEN_URI);
4520 }
4521 nCopy = strlen(zBase);
4522 zCopy = sqlite3_malloc64(nCopy+2);
4523 if( zCopy ){
4524 memcpy(zCopy, zBase, nCopy);
4525 zCopy[nCopy-3] = 'o';
4526 zCopy[nCopy] = '\0';
4527 zCopy[nCopy+1] = '\0';
4528 zOpen = (const char*)(pFd->zDel = zCopy);
4529 }else{
4530 rc = SQLITE_NOMEM;
4531 }
4532 pFd->pRbu = pDb->pRbu;
4533 }
4534 pDb->pWalFd = pFd;
4535 }
4536 }
4537 }
4538
4539 if( oflags & SQLITE_OPEN_MAIN_DB
4540 && sqlite3_uri_boolean(zName, "rbu_memory", 0)
4541 ){
4542 assert( oflags & SQLITE_OPEN_MAIN_DB );
4543 oflags = SQLITE_OPEN_TEMP_DB | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
4544 SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE;
4545 zOpen = 0;
4546 }
4547
4548 if( rc==SQLITE_OK ){
4549 rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, oflags, pOutFlags);
4550 }
4551 if( pFd->pReal->pMethods ){
4552 /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods
4553 ** pointer and, if the file is a main database file, link it into the
4554 ** mutex protected linked list of all such files. */
4555 pFile->pMethods = &rbuvfs_io_methods;
4556 if( flags & SQLITE_OPEN_MAIN_DB ){
4557 sqlite3_mutex_enter(pRbuVfs->mutex);
4558 pFd->pMainNext = pRbuVfs->pMain;
4559 pRbuVfs->pMain = pFd;
4560 sqlite3_mutex_leave(pRbuVfs->mutex);
4561 }
4562 }else{
4563 sqlite3_free(pFd->zDel);
4564 }
4565
4566 return rc;
4567 }
4568
4569 /*
4570 ** Delete the file located at zPath.
4571 */
rbuVfsDelete(sqlite3_vfs * pVfs,const char * zPath,int dirSync)4572 static int rbuVfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
4573 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
4574 return pRealVfs->xDelete(pRealVfs, zPath, dirSync);
4575 }
4576
4577 /*
4578 ** Test for access permissions. Return true if the requested permission
4579 ** is available, or false otherwise.
4580 */
rbuVfsAccess(sqlite3_vfs * pVfs,const char * zPath,int flags,int * pResOut)4581 static int rbuVfsAccess(
4582 sqlite3_vfs *pVfs,
4583 const char *zPath,
4584 int flags,
4585 int *pResOut
4586 ){
4587 rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs;
4588 sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs;
4589 int rc;
4590
4591 rc = pRealVfs->xAccess(pRealVfs, zPath, flags, pResOut);
4592
4593 /* If this call is to check if a *-wal file associated with an RBU target
4594 ** database connection exists, and the RBU update is in RBU_STAGE_OAL,
4595 ** the following special handling is activated:
4596 **
4597 ** a) if the *-wal file does exist, return SQLITE_CANTOPEN. This
4598 ** ensures that the RBU extension never tries to update a database
4599 ** in wal mode, even if the first page of the database file has
4600 ** been damaged.
4601 **
4602 ** b) if the *-wal file does not exist, claim that it does anyway,
4603 ** causing SQLite to call xOpen() to open it. This call will also
4604 ** be intercepted (see the rbuVfsOpen() function) and the *-oal
4605 ** file opened instead.
4606 */
4607 if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS ){
4608 rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath);
4609 if( pDb && pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
4610 if( *pResOut ){
4611 rc = SQLITE_CANTOPEN;
4612 }else{
4613 *pResOut = 1;
4614 }
4615 }
4616 }
4617
4618 return rc;
4619 }
4620
4621 /*
4622 ** Populate buffer zOut with the full canonical pathname corresponding
4623 ** to the pathname in zPath. zOut is guaranteed to point to a buffer
4624 ** of at least (DEVSYM_MAX_PATHNAME+1) bytes.
4625 */
rbuVfsFullPathname(sqlite3_vfs * pVfs,const char * zPath,int nOut,char * zOut)4626 static int rbuVfsFullPathname(
4627 sqlite3_vfs *pVfs,
4628 const char *zPath,
4629 int nOut,
4630 char *zOut
4631 ){
4632 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
4633 return pRealVfs->xFullPathname(pRealVfs, zPath, nOut, zOut);
4634 }
4635
4636 #ifndef SQLITE_OMIT_LOAD_EXTENSION
4637 /*
4638 ** Open the dynamic library located at zPath and return a handle.
4639 */
rbuVfsDlOpen(sqlite3_vfs * pVfs,const char * zPath)4640 static void *rbuVfsDlOpen(sqlite3_vfs *pVfs, const char *zPath){
4641 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
4642 return pRealVfs->xDlOpen(pRealVfs, zPath);
4643 }
4644
4645 /*
4646 ** Populate the buffer zErrMsg (size nByte bytes) with a human readable
4647 ** utf-8 string describing the most recent error encountered associated
4648 ** with dynamic libraries.
4649 */
rbuVfsDlError(sqlite3_vfs * pVfs,int nByte,char * zErrMsg)4650 static void rbuVfsDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
4651 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
4652 pRealVfs->xDlError(pRealVfs, nByte, zErrMsg);
4653 }
4654
4655 /*
4656 ** Return a pointer to the symbol zSymbol in the dynamic library pHandle.
4657 */
rbuVfsDlSym(sqlite3_vfs * pVfs,void * pArg,const char * zSym)4658 static void (*rbuVfsDlSym(
4659 sqlite3_vfs *pVfs,
4660 void *pArg,
4661 const char *zSym
4662 ))(void){
4663 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
4664 return pRealVfs->xDlSym(pRealVfs, pArg, zSym);
4665 }
4666
4667 /*
4668 ** Close the dynamic library handle pHandle.
4669 */
rbuVfsDlClose(sqlite3_vfs * pVfs,void * pHandle)4670 static void rbuVfsDlClose(sqlite3_vfs *pVfs, void *pHandle){
4671 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
4672 pRealVfs->xDlClose(pRealVfs, pHandle);
4673 }
4674 #endif /* SQLITE_OMIT_LOAD_EXTENSION */
4675
4676 /*
4677 ** Populate the buffer pointed to by zBufOut with nByte bytes of
4678 ** random data.
4679 */
rbuVfsRandomness(sqlite3_vfs * pVfs,int nByte,char * zBufOut)4680 static int rbuVfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
4681 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
4682 return pRealVfs->xRandomness(pRealVfs, nByte, zBufOut);
4683 }
4684
4685 /*
4686 ** Sleep for nMicro microseconds. Return the number of microseconds
4687 ** actually slept.
4688 */
rbuVfsSleep(sqlite3_vfs * pVfs,int nMicro)4689 static int rbuVfsSleep(sqlite3_vfs *pVfs, int nMicro){
4690 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
4691 return pRealVfs->xSleep(pRealVfs, nMicro);
4692 }
4693
4694 /*
4695 ** Return the current time as a Julian Day number in *pTimeOut.
4696 */
rbuVfsCurrentTime(sqlite3_vfs * pVfs,double * pTimeOut)4697 static int rbuVfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
4698 sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs;
4699 return pRealVfs->xCurrentTime(pRealVfs, pTimeOut);
4700 }
4701
4702 /*
4703 ** No-op.
4704 */
rbuVfsGetLastError(sqlite3_vfs * pVfs,int a,char * b)4705 static int rbuVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){
4706 return 0;
4707 }
4708
4709 /*
4710 ** Deregister and destroy an RBU vfs created by an earlier call to
4711 ** sqlite3rbu_create_vfs().
4712 */
sqlite3rbu_destroy_vfs(const char * zName)4713 void sqlite3rbu_destroy_vfs(const char *zName){
4714 sqlite3_vfs *pVfs = sqlite3_vfs_find(zName);
4715 if( pVfs && pVfs->xOpen==rbuVfsOpen ){
4716 sqlite3_mutex_free(((rbu_vfs*)pVfs)->mutex);
4717 sqlite3_vfs_unregister(pVfs);
4718 sqlite3_free(pVfs);
4719 }
4720 }
4721
4722 /*
4723 ** Create an RBU VFS named zName that accesses the underlying file-system
4724 ** via existing VFS zParent. The new object is registered as a non-default
4725 ** VFS with SQLite before returning.
4726 */
sqlite3rbu_create_vfs(const char * zName,const char * zParent)4727 int sqlite3rbu_create_vfs(const char *zName, const char *zParent){
4728
4729 /* Template for VFS */
4730 static sqlite3_vfs vfs_template = {
4731 1, /* iVersion */
4732 0, /* szOsFile */
4733 0, /* mxPathname */
4734 0, /* pNext */
4735 0, /* zName */
4736 0, /* pAppData */
4737 rbuVfsOpen, /* xOpen */
4738 rbuVfsDelete, /* xDelete */
4739 rbuVfsAccess, /* xAccess */
4740 rbuVfsFullPathname, /* xFullPathname */
4741
4742 #ifndef SQLITE_OMIT_LOAD_EXTENSION
4743 rbuVfsDlOpen, /* xDlOpen */
4744 rbuVfsDlError, /* xDlError */
4745 rbuVfsDlSym, /* xDlSym */
4746 rbuVfsDlClose, /* xDlClose */
4747 #else
4748 0, 0, 0, 0,
4749 #endif
4750
4751 rbuVfsRandomness, /* xRandomness */
4752 rbuVfsSleep, /* xSleep */
4753 rbuVfsCurrentTime, /* xCurrentTime */
4754 rbuVfsGetLastError, /* xGetLastError */
4755 0, /* xCurrentTimeInt64 (version 2) */
4756 0, 0, 0 /* Unimplemented version 3 methods */
4757 };
4758
4759 rbu_vfs *pNew = 0; /* Newly allocated VFS */
4760 int rc = SQLITE_OK;
4761 size_t nName;
4762 size_t nByte;
4763
4764 nName = strlen(zName);
4765 nByte = sizeof(rbu_vfs) + nName + 1;
4766 pNew = (rbu_vfs*)sqlite3_malloc64(nByte);
4767 if( pNew==0 ){
4768 rc = SQLITE_NOMEM;
4769 }else{
4770 sqlite3_vfs *pParent; /* Parent VFS */
4771 memset(pNew, 0, nByte);
4772 pParent = sqlite3_vfs_find(zParent);
4773 if( pParent==0 ){
4774 rc = SQLITE_NOTFOUND;
4775 }else{
4776 char *zSpace;
4777 memcpy(&pNew->base, &vfs_template, sizeof(sqlite3_vfs));
4778 pNew->base.mxPathname = pParent->mxPathname;
4779 pNew->base.szOsFile = sizeof(rbu_file) + pParent->szOsFile;
4780 pNew->pRealVfs = pParent;
4781 pNew->base.zName = (const char*)(zSpace = (char*)&pNew[1]);
4782 memcpy(zSpace, zName, nName);
4783
4784 /* Allocate the mutex and register the new VFS (not as the default) */
4785 pNew->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE);
4786 if( pNew->mutex==0 ){
4787 rc = SQLITE_NOMEM;
4788 }else{
4789 rc = sqlite3_vfs_register(&pNew->base, 0);
4790 }
4791 }
4792
4793 if( rc!=SQLITE_OK ){
4794 sqlite3_mutex_free(pNew->mutex);
4795 sqlite3_free(pNew);
4796 }
4797 }
4798
4799 return rc;
4800 }
4801
4802
4803 /**************************************************************************/
4804
4805 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) */
4806