1 /*
2 ** 2015-04-06
3 **
4 ** The author disclaims copyright to this source code.  In place of
5 ** a legal notice, here is a blessing:
6 **
7 **    May you do good and not evil.
8 **    May you find forgiveness for yourself and forgive others.
9 **    May you share freely, never taking more than you give.
10 **
11 *************************************************************************
12 **
13 ** This is a utility program that computes the differences in content
14 ** between two SQLite databases.
15 **
16 ** To compile, simply link against SQLite.
17 **
18 ** See the showHelp() routine below for a brief description of how to
19 ** run the utility.
20 */
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <stdarg.h>
24 #include <ctype.h>
25 #include <string.h>
26 #include <assert.h>
27 #include "sqlite3.h"
28 
29 /*
30 ** All global variables are gathered into the "g" singleton.
31 */
32 struct GlobalVars {
33   const char *zArgv0;       /* Name of program */
34   int bSchemaOnly;          /* Only show schema differences */
35   int bSchemaPK;            /* Use the schema-defined PK, not the true PK */
36   int bHandleVtab;          /* Handle fts3, fts4, fts5 and rtree vtabs */
37   unsigned fDebug;          /* Debug flags */
38   sqlite3 *db;              /* The database connection */
39 } g;
40 
41 /*
42 ** Allowed values for g.fDebug
43 */
44 #define DEBUG_COLUMN_NAMES  0x000001
45 #define DEBUG_DIFF_SQL      0x000002
46 
47 /*
48 ** Dynamic string object
49 */
50 typedef struct Str Str;
51 struct Str {
52   char *z;        /* Text of the string */
53   int nAlloc;     /* Bytes allocated in z[] */
54   int nUsed;      /* Bytes actually used in z[] */
55 };
56 
57 /*
58 ** Initialize a Str object
59 */
strInit(Str * p)60 static void strInit(Str *p){
61   p->z = 0;
62   p->nAlloc = 0;
63   p->nUsed = 0;
64 }
65 
66 /*
67 ** Print an error resulting from faulting command-line arguments and
68 ** abort the program.
69 */
cmdlineError(const char * zFormat,...)70 static void cmdlineError(const char *zFormat, ...){
71   va_list ap;
72   fprintf(stderr, "%s: ", g.zArgv0);
73   va_start(ap, zFormat);
74   vfprintf(stderr, zFormat, ap);
75   va_end(ap);
76   fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0);
77   exit(1);
78 }
79 
80 /*
81 ** Print an error message for an error that occurs at runtime, then
82 ** abort the program.
83 */
runtimeError(const char * zFormat,...)84 static void runtimeError(const char *zFormat, ...){
85   va_list ap;
86   fprintf(stderr, "%s: ", g.zArgv0);
87   va_start(ap, zFormat);
88   vfprintf(stderr, zFormat, ap);
89   va_end(ap);
90   fprintf(stderr, "\n");
91   exit(1);
92 }
93 
94 /*
95 ** Free all memory held by a Str object
96 */
strFree(Str * p)97 static void strFree(Str *p){
98   sqlite3_free(p->z);
99   strInit(p);
100 }
101 
102 /*
103 ** Add formatted text to the end of a Str object
104 */
strPrintf(Str * p,const char * zFormat,...)105 static void strPrintf(Str *p, const char *zFormat, ...){
106   int nNew;
107   for(;;){
108     if( p->z ){
109       va_list ap;
110       va_start(ap, zFormat);
111       sqlite3_vsnprintf(p->nAlloc-p->nUsed, p->z+p->nUsed, zFormat, ap);
112       va_end(ap);
113       nNew = (int)strlen(p->z + p->nUsed);
114     }else{
115       nNew = p->nAlloc;
116     }
117     if( p->nUsed+nNew < p->nAlloc-1 ){
118       p->nUsed += nNew;
119       break;
120     }
121     p->nAlloc = p->nAlloc*2 + 1000;
122     p->z = sqlite3_realloc(p->z, p->nAlloc);
123     if( p->z==0 ) runtimeError("out of memory");
124   }
125 }
126 
127 
128 
129 /* Safely quote an SQL identifier.  Use the minimum amount of transformation
130 ** necessary to allow the string to be used with %s.
131 **
132 ** Space to hold the returned string is obtained from sqlite3_malloc().  The
133 ** caller is responsible for ensuring this space is freed when no longer
134 ** needed.
135 */
safeId(const char * zId)136 static char *safeId(const char *zId){
137   int i, x;
138   char c;
139   if( zId[0]==0 ) return sqlite3_mprintf("\"\"");
140   for(i=x=0; (c = zId[i])!=0; i++){
141     if( !isalpha(c) && c!='_' ){
142       if( i>0 && isdigit(c) ){
143         x++;
144       }else{
145         return sqlite3_mprintf("\"%w\"", zId);
146       }
147     }
148   }
149   if( x || !sqlite3_keyword_check(zId,i) ){
150     return sqlite3_mprintf("%s", zId);
151   }
152   return sqlite3_mprintf("\"%w\"", zId);
153 }
154 
155 /*
156 ** Prepare a new SQL statement.  Print an error and abort if anything
157 ** goes wrong.
158 */
db_vprepare(const char * zFormat,va_list ap)159 static sqlite3_stmt *db_vprepare(const char *zFormat, va_list ap){
160   char *zSql;
161   int rc;
162   sqlite3_stmt *pStmt;
163 
164   zSql = sqlite3_vmprintf(zFormat, ap);
165   if( zSql==0 ) runtimeError("out of memory");
166   rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0);
167   if( rc ){
168     runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g.db),
169                  zSql);
170   }
171   sqlite3_free(zSql);
172   return pStmt;
173 }
db_prepare(const char * zFormat,...)174 static sqlite3_stmt *db_prepare(const char *zFormat, ...){
175   va_list ap;
176   sqlite3_stmt *pStmt;
177   va_start(ap, zFormat);
178   pStmt = db_vprepare(zFormat, ap);
179   va_end(ap);
180   return pStmt;
181 }
182 
183 /*
184 ** Free a list of strings
185 */
namelistFree(char ** az)186 static void namelistFree(char **az){
187   if( az ){
188     int i;
189     for(i=0; az[i]; i++) sqlite3_free(az[i]);
190     sqlite3_free(az);
191   }
192 }
193 
194 /*
195 ** Return a list of column names for the table zDb.zTab.  Space to
196 ** hold the list is obtained from sqlite3_malloc() and should released
197 ** using namelistFree() when no longer needed.
198 **
199 ** Primary key columns are listed first, followed by data columns.
200 ** The number of columns in the primary key is returned in *pnPkey.
201 **
202 ** Normally, the "primary key" in the previous sentence is the true
203 ** primary key - the rowid or INTEGER PRIMARY KEY for ordinary tables
204 ** or the declared PRIMARY KEY for WITHOUT ROWID tables.  However, if
205 ** the g.bSchemaPK flag is set, then the schema-defined PRIMARY KEY is
206 ** used in all cases.  In that case, entries that have NULL values in
207 ** any of their primary key fields will be excluded from the analysis.
208 **
209 ** If the primary key for a table is the rowid but rowid is inaccessible,
210 ** then this routine returns a NULL pointer.
211 **
212 ** Examples:
213 **    CREATE TABLE t1(a INT UNIQUE, b INTEGER, c TEXT, PRIMARY KEY(c));
214 **    *pnPKey = 1;
215 **    az = { "rowid", "a", "b", "c", 0 }  // Normal case
216 **    az = { "c", "a", "b", 0 }           // g.bSchemaPK==1
217 **
218 **    CREATE TABLE t2(a INT UNIQUE, b INTEGER, c TEXT, PRIMARY KEY(b));
219 **    *pnPKey = 1;
220 **    az = { "b", "a", "c", 0 }
221 **
222 **    CREATE TABLE t3(x,y,z,PRIMARY KEY(y,z));
223 **    *pnPKey = 1                         // Normal case
224 **    az = { "rowid", "x", "y", "z", 0 }  // Normal case
225 **    *pnPKey = 2                         // g.bSchemaPK==1
226 **    az = { "y", "x", "z", 0 }           // g.bSchemaPK==1
227 **
228 **    CREATE TABLE t4(x,y,z,PRIMARY KEY(y,z)) WITHOUT ROWID;
229 **    *pnPKey = 2
230 **    az = { "y", "z", "x", 0 }
231 **
232 **    CREATE TABLE t5(rowid,_rowid_,oid);
233 **    az = 0     // The rowid is not accessible
234 */
columnNames(const char * zDb,const char * zTab,int * pnPKey,int * pbRowid)235 static char **columnNames(
236   const char *zDb,                /* Database ("main" or "aux") to query */
237   const char *zTab,               /* Name of table to return details of */
238   int *pnPKey,                    /* OUT: Number of PK columns */
239   int *pbRowid                    /* OUT: True if PK is an implicit rowid */
240 ){
241   char **az = 0;           /* List of column names to be returned */
242   int naz = 0;             /* Number of entries in az[] */
243   sqlite3_stmt *pStmt;     /* SQL statement being run */
244   char *zPkIdxName = 0;    /* Name of the PRIMARY KEY index */
245   int truePk = 0;          /* PRAGMA table_info indentifies the PK to use */
246   int nPK = 0;             /* Number of PRIMARY KEY columns */
247   int i, j;                /* Loop counters */
248 
249   if( g.bSchemaPK==0 ){
250     /* Normal case:  Figure out what the true primary key is for the table.
251     **   *  For WITHOUT ROWID tables, the true primary key is the same as
252     **      the schema PRIMARY KEY, which is guaranteed to be present.
253     **   *  For rowid tables with an INTEGER PRIMARY KEY, the true primary
254     **      key is the INTEGER PRIMARY KEY.
255     **   *  For all other rowid tables, the rowid is the true primary key.
256     */
257     pStmt = db_prepare("PRAGMA %s.index_list=%Q", zDb, zTab);
258     while( SQLITE_ROW==sqlite3_step(pStmt) ){
259       if( sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,3),"pk")==0 ){
260         zPkIdxName = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
261         break;
262       }
263     }
264     sqlite3_finalize(pStmt);
265     if( zPkIdxName ){
266       int nKey = 0;
267       int nCol = 0;
268       truePk = 0;
269       pStmt = db_prepare("PRAGMA %s.index_xinfo=%Q", zDb, zPkIdxName);
270       while( SQLITE_ROW==sqlite3_step(pStmt) ){
271         nCol++;
272         if( sqlite3_column_int(pStmt,5) ){ nKey++; continue; }
273         if( sqlite3_column_int(pStmt,1)>=0 ) truePk = 1;
274       }
275       if( nCol==nKey ) truePk = 1;
276       if( truePk ){
277         nPK = nKey;
278       }else{
279         nPK = 1;
280       }
281       sqlite3_finalize(pStmt);
282       sqlite3_free(zPkIdxName);
283     }else{
284       truePk = 1;
285       nPK = 1;
286     }
287     pStmt = db_prepare("PRAGMA %s.table_info=%Q", zDb, zTab);
288   }else{
289     /* The g.bSchemaPK==1 case:  Use whatever primary key is declared
290     ** in the schema.  The "rowid" will still be used as the primary key
291     ** if the table definition does not contain a PRIMARY KEY.
292     */
293     nPK = 0;
294     pStmt = db_prepare("PRAGMA %s.table_info=%Q", zDb, zTab);
295     while( SQLITE_ROW==sqlite3_step(pStmt) ){
296       if( sqlite3_column_int(pStmt,5)>0 ) nPK++;
297     }
298     sqlite3_reset(pStmt);
299     if( nPK==0 ) nPK = 1;
300     truePk = 1;
301   }
302   *pnPKey = nPK;
303   naz = nPK;
304   az = sqlite3_malloc( sizeof(char*)*(nPK+1) );
305   if( az==0 ) runtimeError("out of memory");
306   memset(az, 0, sizeof(char*)*(nPK+1));
307   while( SQLITE_ROW==sqlite3_step(pStmt) ){
308     int iPKey;
309     if( truePk && (iPKey = sqlite3_column_int(pStmt,5))>0 ){
310       az[iPKey-1] = safeId((char*)sqlite3_column_text(pStmt,1));
311     }else{
312       az = sqlite3_realloc(az, sizeof(char*)*(naz+2) );
313       if( az==0 ) runtimeError("out of memory");
314       az[naz++] = safeId((char*)sqlite3_column_text(pStmt,1));
315     }
316   }
317   sqlite3_finalize(pStmt);
318   if( az ) az[naz] = 0;
319 
320   /* If it is non-NULL, set *pbRowid to indicate whether or not the PK of
321   ** this table is an implicit rowid (*pbRowid==1) or not (*pbRowid==0).  */
322   if( pbRowid ) *pbRowid = (az[0]==0);
323 
324   /* If this table has an implicit rowid for a PK, figure out how to refer
325   ** to it. There are three options - "rowid", "_rowid_" and "oid". Any
326   ** of these will work, unless the table has an explicit column of the
327   ** same name.  */
328   if( az[0]==0 ){
329     const char *azRowid[] = { "rowid", "_rowid_", "oid" };
330     for(i=0; i<sizeof(azRowid)/sizeof(azRowid[0]); i++){
331       for(j=1; j<naz; j++){
332         if( sqlite3_stricmp(az[j], azRowid[i])==0 ) break;
333       }
334       if( j>=naz ){
335         az[0] = sqlite3_mprintf("%s", azRowid[i]);
336         break;
337       }
338     }
339     if( az[0]==0 ){
340       for(i=1; i<naz; i++) sqlite3_free(az[i]);
341       sqlite3_free(az);
342       az = 0;
343     }
344   }
345   return az;
346 }
347 
348 /*
349 ** Print the sqlite3_value X as an SQL literal.
350 */
printQuoted(FILE * out,sqlite3_value * X)351 static void printQuoted(FILE *out, sqlite3_value *X){
352   switch( sqlite3_value_type(X) ){
353     case SQLITE_FLOAT: {
354       double r1;
355       char zBuf[50];
356       r1 = sqlite3_value_double(X);
357       sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1);
358       fprintf(out, "%s", zBuf);
359       break;
360     }
361     case SQLITE_INTEGER: {
362       fprintf(out, "%lld", sqlite3_value_int64(X));
363       break;
364     }
365     case SQLITE_BLOB: {
366       const unsigned char *zBlob = sqlite3_value_blob(X);
367       int nBlob = sqlite3_value_bytes(X);
368       if( zBlob ){
369         int i;
370         fprintf(out, "x'");
371         for(i=0; i<nBlob; i++){
372           fprintf(out, "%02x", zBlob[i]);
373         }
374         fprintf(out, "'");
375       }else{
376         /* Could be an OOM, could be a zero-byte blob */
377         fprintf(out, "X''");
378       }
379       break;
380     }
381     case SQLITE_TEXT: {
382       const unsigned char *zArg = sqlite3_value_text(X);
383       int i, j;
384 
385       if( zArg==0 ){
386         fprintf(out, "NULL");
387       }else{
388         fprintf(out, "'");
389         for(i=j=0; zArg[i]; i++){
390           if( zArg[i]=='\'' ){
391             fprintf(out, "%.*s'", i-j+1, &zArg[j]);
392             j = i+1;
393           }
394         }
395         fprintf(out, "%s'", &zArg[j]);
396       }
397       break;
398     }
399     case SQLITE_NULL: {
400       fprintf(out, "NULL");
401       break;
402     }
403   }
404 }
405 
406 /*
407 ** Output SQL that will recreate the aux.zTab table.
408 */
dump_table(const char * zTab,FILE * out)409 static void dump_table(const char *zTab, FILE *out){
410   char *zId = safeId(zTab); /* Name of the table */
411   char **az = 0;            /* List of columns */
412   int nPk;                  /* Number of true primary key columns */
413   int nCol;                 /* Number of data columns */
414   int i;                    /* Loop counter */
415   sqlite3_stmt *pStmt;      /* SQL statement */
416   const char *zSep;         /* Separator string */
417   Str ins;                  /* Beginning of the INSERT statement */
418 
419   pStmt = db_prepare("SELECT sql FROM aux.sqlite_master WHERE name=%Q", zTab);
420   if( SQLITE_ROW==sqlite3_step(pStmt) ){
421     fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
422   }
423   sqlite3_finalize(pStmt);
424   if( !g.bSchemaOnly ){
425     az = columnNames("aux", zTab, &nPk, 0);
426     strInit(&ins);
427     if( az==0 ){
428       pStmt = db_prepare("SELECT * FROM aux.%s", zId);
429       strPrintf(&ins,"INSERT INTO %s VALUES", zId);
430     }else{
431       Str sql;
432       strInit(&sql);
433       zSep =  "SELECT";
434       for(i=0; az[i]; i++){
435         strPrintf(&sql, "%s %s", zSep, az[i]);
436         zSep = ",";
437       }
438       strPrintf(&sql," FROM aux.%s", zId);
439       zSep = " ORDER BY";
440       for(i=1; i<=nPk; i++){
441         strPrintf(&sql, "%s %d", zSep, i);
442         zSep = ",";
443       }
444       pStmt = db_prepare("%s", sql.z);
445       strFree(&sql);
446       strPrintf(&ins, "INSERT INTO %s", zId);
447       zSep = "(";
448       for(i=0; az[i]; i++){
449         strPrintf(&ins, "%s%s", zSep, az[i]);
450         zSep = ",";
451       }
452       strPrintf(&ins,") VALUES");
453       namelistFree(az);
454     }
455     nCol = sqlite3_column_count(pStmt);
456     while( SQLITE_ROW==sqlite3_step(pStmt) ){
457       fprintf(out, "%s",ins.z);
458       zSep = "(";
459       for(i=0; i<nCol; i++){
460         fprintf(out, "%s",zSep);
461         printQuoted(out, sqlite3_column_value(pStmt,i));
462         zSep = ",";
463       }
464       fprintf(out, ");\n");
465     }
466     sqlite3_finalize(pStmt);
467     strFree(&ins);
468   } /* endif !g.bSchemaOnly */
469   pStmt = db_prepare("SELECT sql FROM aux.sqlite_master"
470                      " WHERE type='index' AND tbl_name=%Q AND sql IS NOT NULL",
471                      zTab);
472   while( SQLITE_ROW==sqlite3_step(pStmt) ){
473     fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
474   }
475   sqlite3_finalize(pStmt);
476 }
477 
478 
479 /*
480 ** Compute all differences for a single table.
481 */
diff_one_table(const char * zTab,FILE * out)482 static void diff_one_table(const char *zTab, FILE *out){
483   char *zId = safeId(zTab); /* Name of table (translated for us in SQL) */
484   char **az = 0;            /* Columns in main */
485   char **az2 = 0;           /* Columns in aux */
486   int nPk;                  /* Primary key columns in main */
487   int nPk2;                 /* Primary key columns in aux */
488   int n = 0;                /* Number of columns in main */
489   int n2;                   /* Number of columns in aux */
490   int nQ;                   /* Number of output columns in the diff query */
491   int i;                    /* Loop counter */
492   const char *zSep;         /* Separator string */
493   Str sql;                  /* Comparison query */
494   sqlite3_stmt *pStmt;      /* Query statement to do the diff */
495 
496   strInit(&sql);
497   if( g.fDebug==DEBUG_COLUMN_NAMES ){
498     /* Simply run columnNames() on all tables of the origin
499     ** database and show the results.  This is used for testing
500     ** and debugging of the columnNames() function.
501     */
502     az = columnNames("aux",zTab, &nPk, 0);
503     if( az==0 ){
504       printf("Rowid not accessible for %s\n", zId);
505     }else{
506       printf("%s:", zId);
507       for(i=0; az[i]; i++){
508         printf(" %s", az[i]);
509         if( i+1==nPk ) printf(" *");
510       }
511       printf("\n");
512     }
513     goto end_diff_one_table;
514   }
515 
516 
517   if( sqlite3_table_column_metadata(g.db,"aux",zTab,0,0,0,0,0,0) ){
518     if( !sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
519       /* Table missing from second database. */
520       fprintf(out, "DROP TABLE %s;\n", zId);
521     }
522     goto end_diff_one_table;
523   }
524 
525   if( sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
526     /* Table missing from source */
527     dump_table(zTab, out);
528     goto end_diff_one_table;
529   }
530 
531   az = columnNames("main", zTab, &nPk, 0);
532   az2 = columnNames("aux", zTab, &nPk2, 0);
533   if( az && az2 ){
534     for(n=0; az[n] && az2[n]; n++){
535       if( sqlite3_stricmp(az[n],az2[n])!=0 ) break;
536     }
537   }
538   if( az==0
539    || az2==0
540    || nPk!=nPk2
541    || az[n]
542   ){
543     /* Schema mismatch */
544     fprintf(out, "DROP TABLE %s; -- due to schema mismatch\n", zId);
545     dump_table(zTab, out);
546     goto end_diff_one_table;
547   }
548 
549   /* Build the comparison query */
550   for(n2=n; az2[n2]; n2++){
551     fprintf(out, "ALTER TABLE %s ADD COLUMN %s;\n", zId, safeId(az2[n2]));
552   }
553   nQ = nPk2+1+2*(n2-nPk2);
554   if( n2>nPk2 ){
555     zSep = "SELECT ";
556     for(i=0; i<nPk; i++){
557       strPrintf(&sql, "%sB.%s", zSep, az[i]);
558       zSep = ", ";
559     }
560     strPrintf(&sql, ", 1%s -- changed row\n", nPk==n ? "" : ",");
561     while( az[i] ){
562       strPrintf(&sql, "       A.%s IS NOT B.%s, B.%s%s\n",
563                 az[i], az2[i], az2[i], az2[i+1]==0 ? "" : ",");
564       i++;
565     }
566     while( az2[i] ){
567       strPrintf(&sql, "       B.%s IS NOT NULL, B.%s%s\n",
568                 az2[i], az2[i], az2[i+1]==0 ? "" : ",");
569       i++;
570     }
571     strPrintf(&sql, "  FROM main.%s A, aux.%s B\n", zId, zId);
572     zSep = " WHERE";
573     for(i=0; i<nPk; i++){
574       strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
575       zSep = " AND";
576     }
577     zSep = "\n   AND (";
578     while( az[i] ){
579       strPrintf(&sql, "%sA.%s IS NOT B.%s%s\n",
580                 zSep, az[i], az2[i], az2[i+1]==0 ? ")" : "");
581       zSep = "        OR ";
582       i++;
583     }
584     while( az2[i] ){
585       strPrintf(&sql, "%sB.%s IS NOT NULL%s\n",
586                 zSep, az2[i], az2[i+1]==0 ? ")" : "");
587       zSep = "        OR ";
588       i++;
589     }
590     strPrintf(&sql, " UNION ALL\n");
591   }
592   zSep = "SELECT ";
593   for(i=0; i<nPk; i++){
594     strPrintf(&sql, "%sA.%s", zSep, az[i]);
595     zSep = ", ";
596   }
597   strPrintf(&sql, ", 2%s -- deleted row\n", nPk==n ? "" : ",");
598   while( az2[i] ){
599     strPrintf(&sql, "       NULL, NULL%s\n", i==n2-1 ? "" : ",");
600     i++;
601   }
602   strPrintf(&sql, "  FROM main.%s A\n", zId);
603   strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId);
604   zSep =          "                   WHERE";
605   for(i=0; i<nPk; i++){
606     strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
607     zSep = " AND";
608   }
609   strPrintf(&sql, ")\n");
610   zSep = " UNION ALL\nSELECT ";
611   for(i=0; i<nPk; i++){
612     strPrintf(&sql, "%sB.%s", zSep, az[i]);
613     zSep = ", ";
614   }
615   strPrintf(&sql, ", 3%s -- inserted row\n", nPk==n ? "" : ",");
616   while( az2[i] ){
617     strPrintf(&sql, "       1, B.%s%s\n", az2[i], az2[i+1]==0 ? "" : ",");
618     i++;
619   }
620   strPrintf(&sql, "  FROM aux.%s B\n", zId);
621   strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId);
622   zSep =          "                   WHERE";
623   for(i=0; i<nPk; i++){
624     strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
625     zSep = " AND";
626   }
627   strPrintf(&sql, ")\n ORDER BY");
628   zSep = " ";
629   for(i=1; i<=nPk; i++){
630     strPrintf(&sql, "%s%d", zSep, i);
631     zSep = ", ";
632   }
633   strPrintf(&sql, ";\n");
634 
635   if( g.fDebug & DEBUG_DIFF_SQL ){
636     printf("SQL for %s:\n%s\n", zId, sql.z);
637     goto end_diff_one_table;
638   }
639 
640   /* Drop indexes that are missing in the destination */
641   pStmt = db_prepare(
642     "SELECT name FROM main.sqlite_master"
643     " WHERE type='index' AND tbl_name=%Q"
644     "   AND sql IS NOT NULL"
645     "   AND sql NOT IN (SELECT sql FROM aux.sqlite_master"
646     "                    WHERE type='index' AND tbl_name=%Q"
647     "                      AND sql IS NOT NULL)",
648     zTab, zTab);
649   while( SQLITE_ROW==sqlite3_step(pStmt) ){
650     char *z = safeId((const char*)sqlite3_column_text(pStmt,0));
651     fprintf(out, "DROP INDEX %s;\n", z);
652     sqlite3_free(z);
653   }
654   sqlite3_finalize(pStmt);
655 
656   /* Run the query and output differences */
657   if( !g.bSchemaOnly ){
658     pStmt = db_prepare("%s", sql.z);
659     while( SQLITE_ROW==sqlite3_step(pStmt) ){
660       int iType = sqlite3_column_int(pStmt, nPk);
661       if( iType==1 || iType==2 ){
662         if( iType==1 ){       /* Change the content of a row */
663           fprintf(out, "UPDATE %s", zId);
664           zSep = " SET";
665           for(i=nPk+1; i<nQ; i+=2){
666             if( sqlite3_column_int(pStmt,i)==0 ) continue;
667             fprintf(out, "%s %s=", zSep, az2[(i+nPk-1)/2]);
668             zSep = ",";
669             printQuoted(out, sqlite3_column_value(pStmt,i+1));
670           }
671         }else{                /* Delete a row */
672           fprintf(out, "DELETE FROM %s", zId);
673         }
674         zSep = " WHERE";
675         for(i=0; i<nPk; i++){
676           fprintf(out, "%s %s=", zSep, az2[i]);
677           printQuoted(out, sqlite3_column_value(pStmt,i));
678           zSep = " AND";
679         }
680         fprintf(out, ";\n");
681       }else{                  /* Insert a row */
682         fprintf(out, "INSERT INTO %s(%s", zId, az2[0]);
683         for(i=1; az2[i]; i++) fprintf(out, ",%s", az2[i]);
684         fprintf(out, ") VALUES");
685         zSep = "(";
686         for(i=0; i<nPk2; i++){
687           fprintf(out, "%s", zSep);
688           zSep = ",";
689           printQuoted(out, sqlite3_column_value(pStmt,i));
690         }
691         for(i=nPk2+2; i<nQ; i+=2){
692           fprintf(out, ",");
693           printQuoted(out, sqlite3_column_value(pStmt,i));
694         }
695         fprintf(out, ");\n");
696       }
697     }
698     sqlite3_finalize(pStmt);
699   } /* endif !g.bSchemaOnly */
700 
701   /* Create indexes that are missing in the source */
702   pStmt = db_prepare(
703     "SELECT sql FROM aux.sqlite_master"
704     " WHERE type='index' AND tbl_name=%Q"
705     "   AND sql IS NOT NULL"
706     "   AND sql NOT IN (SELECT sql FROM main.sqlite_master"
707     "                    WHERE type='index' AND tbl_name=%Q"
708     "                      AND sql IS NOT NULL)",
709     zTab, zTab);
710   while( SQLITE_ROW==sqlite3_step(pStmt) ){
711     fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
712   }
713   sqlite3_finalize(pStmt);
714 
715 end_diff_one_table:
716   strFree(&sql);
717   sqlite3_free(zId);
718   namelistFree(az);
719   namelistFree(az2);
720   return;
721 }
722 
723 /*
724 ** Check that table zTab exists and has the same schema in both the "main"
725 ** and "aux" databases currently opened by the global db handle. If they
726 ** do not, output an error message on stderr and exit(1). Otherwise, if
727 ** the schemas do match, return control to the caller.
728 */
checkSchemasMatch(const char * zTab)729 static void checkSchemasMatch(const char *zTab){
730   sqlite3_stmt *pStmt = db_prepare(
731       "SELECT A.sql=B.sql FROM main.sqlite_master A, aux.sqlite_master B"
732       " WHERE A.name=%Q AND B.name=%Q", zTab, zTab
733   );
734   if( SQLITE_ROW==sqlite3_step(pStmt) ){
735     if( sqlite3_column_int(pStmt,0)==0 ){
736       runtimeError("schema changes for table %s", safeId(zTab));
737     }
738   }else{
739     runtimeError("table %s missing from one or both databases", safeId(zTab));
740   }
741   sqlite3_finalize(pStmt);
742 }
743 
744 /**************************************************************************
745 ** The following code is copied from fossil. It is used to generate the
746 ** fossil delta blobs sometimes used in RBU update records.
747 */
748 
749 typedef unsigned short u16;
750 typedef unsigned int u32;
751 typedef unsigned char u8;
752 
753 /*
754 ** The width of a hash window in bytes.  The algorithm only works if this
755 ** is a power of 2.
756 */
757 #define NHASH 16
758 
759 /*
760 ** The current state of the rolling hash.
761 **
762 ** z[] holds the values that have been hashed.  z[] is a circular buffer.
763 ** z[i] is the first entry and z[(i+NHASH-1)%NHASH] is the last entry of
764 ** the window.
765 **
766 ** Hash.a is the sum of all elements of hash.z[].  Hash.b is a weighted
767 ** sum.  Hash.b is z[i]*NHASH + z[i+1]*(NHASH-1) + ... + z[i+NHASH-1]*1.
768 ** (Each index for z[] should be module NHASH, of course.  The %NHASH operator
769 ** is omitted in the prior expression for brevity.)
770 */
771 typedef struct hash hash;
772 struct hash {
773   u16 a, b;         /* Hash values */
774   u16 i;            /* Start of the hash window */
775   char z[NHASH];    /* The values that have been hashed */
776 };
777 
778 /*
779 ** Initialize the rolling hash using the first NHASH characters of z[]
780 */
hash_init(hash * pHash,const char * z)781 static void hash_init(hash *pHash, const char *z){
782   u16 a, b, i;
783   a = b = 0;
784   for(i=0; i<NHASH; i++){
785     a += z[i];
786     b += (NHASH-i)*z[i];
787     pHash->z[i] = z[i];
788   }
789   pHash->a = a & 0xffff;
790   pHash->b = b & 0xffff;
791   pHash->i = 0;
792 }
793 
794 /*
795 ** Advance the rolling hash by a single character "c"
796 */
hash_next(hash * pHash,int c)797 static void hash_next(hash *pHash, int c){
798   u16 old = pHash->z[pHash->i];
799   pHash->z[pHash->i] = (char)c;
800   pHash->i = (pHash->i+1)&(NHASH-1);
801   pHash->a = pHash->a - old + (char)c;
802   pHash->b = pHash->b - NHASH*old + pHash->a;
803 }
804 
805 /*
806 ** Return a 32-bit hash value
807 */
hash_32bit(hash * pHash)808 static u32 hash_32bit(hash *pHash){
809   return (pHash->a & 0xffff) | (((u32)(pHash->b & 0xffff))<<16);
810 }
811 
812 /*
813 ** Write an base-64 integer into the given buffer.
814 */
putInt(unsigned int v,char ** pz)815 static void putInt(unsigned int v, char **pz){
816   static const char zDigits[] =
817     "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~";
818   /*  123456789 123456789 123456789 123456789 123456789 123456789 123 */
819   int i, j;
820   char zBuf[20];
821   if( v==0 ){
822     *(*pz)++ = '0';
823     return;
824   }
825   for(i=0; v>0; i++, v>>=6){
826     zBuf[i] = zDigits[v&0x3f];
827   }
828   for(j=i-1; j>=0; j--){
829     *(*pz)++ = zBuf[j];
830   }
831 }
832 
833 /*
834 ** Return the number digits in the base-64 representation of a positive integer
835 */
digit_count(int v)836 static int digit_count(int v){
837   unsigned int i, x;
838   for(i=1, x=64; (unsigned int)v>=x; i++, x <<= 6){}
839   return i;
840 }
841 
842 /*
843 ** Compute a 32-bit checksum on the N-byte buffer.  Return the result.
844 */
checksum(const char * zIn,size_t N)845 static unsigned int checksum(const char *zIn, size_t N){
846   const unsigned char *z = (const unsigned char *)zIn;
847   unsigned sum0 = 0;
848   unsigned sum1 = 0;
849   unsigned sum2 = 0;
850   unsigned sum3 = 0;
851   while(N >= 16){
852     sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]);
853     sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]);
854     sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]);
855     sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]);
856     z += 16;
857     N -= 16;
858   }
859   while(N >= 4){
860     sum0 += z[0];
861     sum1 += z[1];
862     sum2 += z[2];
863     sum3 += z[3];
864     z += 4;
865     N -= 4;
866   }
867   sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24);
868   switch(N){
869     case 3:   sum3 += (z[2] << 8);
870     case 2:   sum3 += (z[1] << 16);
871     case 1:   sum3 += (z[0] << 24);
872     default:  ;
873   }
874   return sum3;
875 }
876 
877 /*
878 ** Create a new delta.
879 **
880 ** The delta is written into a preallocated buffer, zDelta, which
881 ** should be at least 60 bytes longer than the target file, zOut.
882 ** The delta string will be NUL-terminated, but it might also contain
883 ** embedded NUL characters if either the zSrc or zOut files are
884 ** binary.  This function returns the length of the delta string
885 ** in bytes, excluding the final NUL terminator character.
886 **
887 ** Output Format:
888 **
889 ** The delta begins with a base64 number followed by a newline.  This
890 ** number is the number of bytes in the TARGET file.  Thus, given a
891 ** delta file z, a program can compute the size of the output file
892 ** simply by reading the first line and decoding the base-64 number
893 ** found there.  The delta_output_size() routine does exactly this.
894 **
895 ** After the initial size number, the delta consists of a series of
896 ** literal text segments and commands to copy from the SOURCE file.
897 ** A copy command looks like this:
898 **
899 **     NNN@MMM,
900 **
901 ** where NNN is the number of bytes to be copied and MMM is the offset
902 ** into the source file of the first byte (both base-64).   If NNN is 0
903 ** it means copy the rest of the input file.  Literal text is like this:
904 **
905 **     NNN:TTTTT
906 **
907 ** where NNN is the number of bytes of text (base-64) and TTTTT is the text.
908 **
909 ** The last term is of the form
910 **
911 **     NNN;
912 **
913 ** In this case, NNN is a 32-bit bigendian checksum of the output file
914 ** that can be used to verify that the delta applied correctly.  All
915 ** numbers are in base-64.
916 **
917 ** Pure text files generate a pure text delta.  Binary files generate a
918 ** delta that may contain some binary data.
919 **
920 ** Algorithm:
921 **
922 ** The encoder first builds a hash table to help it find matching
923 ** patterns in the source file.  16-byte chunks of the source file
924 ** sampled at evenly spaced intervals are used to populate the hash
925 ** table.
926 **
927 ** Next we begin scanning the target file using a sliding 16-byte
928 ** window.  The hash of the 16-byte window in the target is used to
929 ** search for a matching section in the source file.  When a match
930 ** is found, a copy command is added to the delta.  An effort is
931 ** made to extend the matching section to regions that come before
932 ** and after the 16-byte hash window.  A copy command is only issued
933 ** if the result would use less space that just quoting the text
934 ** literally. Literal text is added to the delta for sections that
935 ** do not match or which can not be encoded efficiently using copy
936 ** commands.
937 */
rbuDeltaCreate(const char * zSrc,unsigned int lenSrc,const char * zOut,unsigned int lenOut,char * zDelta)938 static int rbuDeltaCreate(
939   const char *zSrc,      /* The source or pattern file */
940   unsigned int lenSrc,   /* Length of the source file */
941   const char *zOut,      /* The target file */
942   unsigned int lenOut,   /* Length of the target file */
943   char *zDelta           /* Write the delta into this buffer */
944 ){
945   unsigned int i, base;
946   char *zOrigDelta = zDelta;
947   hash h;
948   int nHash;                 /* Number of hash table entries */
949   int *landmark;             /* Primary hash table */
950   int *collide;              /* Collision chain */
951   int lastRead = -1;         /* Last byte of zSrc read by a COPY command */
952 
953   /* Add the target file size to the beginning of the delta
954   */
955   putInt(lenOut, &zDelta);
956   *(zDelta++) = '\n';
957 
958   /* If the source file is very small, it means that we have no
959   ** chance of ever doing a copy command.  Just output a single
960   ** literal segment for the entire target and exit.
961   */
962   if( lenSrc<=NHASH ){
963     putInt(lenOut, &zDelta);
964     *(zDelta++) = ':';
965     memcpy(zDelta, zOut, lenOut);
966     zDelta += lenOut;
967     putInt(checksum(zOut, lenOut), &zDelta);
968     *(zDelta++) = ';';
969     return (int)(zDelta - zOrigDelta);
970   }
971 
972   /* Compute the hash table used to locate matching sections in the
973   ** source file.
974   */
975   nHash = lenSrc/NHASH;
976   collide = sqlite3_malloc( nHash*2*sizeof(int) );
977   landmark = &collide[nHash];
978   memset(landmark, -1, nHash*sizeof(int));
979   memset(collide, -1, nHash*sizeof(int));
980   for(i=0; i<lenSrc-NHASH; i+=NHASH){
981     int hv;
982     hash_init(&h, &zSrc[i]);
983     hv = hash_32bit(&h) % nHash;
984     collide[i/NHASH] = landmark[hv];
985     landmark[hv] = i/NHASH;
986   }
987 
988   /* Begin scanning the target file and generating copy commands and
989   ** literal sections of the delta.
990   */
991   base = 0;    /* We have already generated everything before zOut[base] */
992   while( base+NHASH<lenOut ){
993     int iSrc, iBlock;
994     int bestCnt, bestOfst=0, bestLitsz=0;
995     hash_init(&h, &zOut[base]);
996     i = 0;     /* Trying to match a landmark against zOut[base+i] */
997     bestCnt = 0;
998     while( 1 ){
999       int hv;
1000       int limit = 250;
1001 
1002       hv = hash_32bit(&h) % nHash;
1003       iBlock = landmark[hv];
1004       while( iBlock>=0 && (limit--)>0 ){
1005         /*
1006         ** The hash window has identified a potential match against
1007         ** landmark block iBlock.  But we need to investigate further.
1008         **
1009         ** Look for a region in zOut that matches zSrc. Anchor the search
1010         ** at zSrc[iSrc] and zOut[base+i].  Do not include anything prior to
1011         ** zOut[base] or after zOut[outLen] nor anything after zSrc[srcLen].
1012         **
1013         ** Set cnt equal to the length of the match and set ofst so that
1014         ** zSrc[ofst] is the first element of the match.  litsz is the number
1015         ** of characters between zOut[base] and the beginning of the match.
1016         ** sz will be the overhead (in bytes) needed to encode the copy
1017         ** command.  Only generate copy command if the overhead of the
1018         ** copy command is less than the amount of literal text to be copied.
1019         */
1020         int cnt, ofst, litsz;
1021         int j, k, x, y;
1022         int sz;
1023 
1024         /* Beginning at iSrc, match forwards as far as we can.  j counts
1025         ** the number of characters that match */
1026         iSrc = iBlock*NHASH;
1027         for(
1028           j=0, x=iSrc, y=base+i;
1029           (unsigned int)x<lenSrc && (unsigned int)y<lenOut;
1030           j++, x++, y++
1031         ){
1032           if( zSrc[x]!=zOut[y] ) break;
1033         }
1034         j--;
1035 
1036         /* Beginning at iSrc-1, match backwards as far as we can.  k counts
1037         ** the number of characters that match */
1038         for(k=1; k<iSrc && (unsigned int)k<=i; k++){
1039           if( zSrc[iSrc-k]!=zOut[base+i-k] ) break;
1040         }
1041         k--;
1042 
1043         /* Compute the offset and size of the matching region */
1044         ofst = iSrc-k;
1045         cnt = j+k+1;
1046         litsz = i-k;  /* Number of bytes of literal text before the copy */
1047         /* sz will hold the number of bytes needed to encode the "insert"
1048         ** command and the copy command, not counting the "insert" text */
1049         sz = digit_count(i-k)+digit_count(cnt)+digit_count(ofst)+3;
1050         if( cnt>=sz && cnt>bestCnt ){
1051           /* Remember this match only if it is the best so far and it
1052           ** does not increase the file size */
1053           bestCnt = cnt;
1054           bestOfst = iSrc-k;
1055           bestLitsz = litsz;
1056         }
1057 
1058         /* Check the next matching block */
1059         iBlock = collide[iBlock];
1060       }
1061 
1062       /* We have a copy command that does not cause the delta to be larger
1063       ** than a literal insert.  So add the copy command to the delta.
1064       */
1065       if( bestCnt>0 ){
1066         if( bestLitsz>0 ){
1067           /* Add an insert command before the copy */
1068           putInt(bestLitsz,&zDelta);
1069           *(zDelta++) = ':';
1070           memcpy(zDelta, &zOut[base], bestLitsz);
1071           zDelta += bestLitsz;
1072           base += bestLitsz;
1073         }
1074         base += bestCnt;
1075         putInt(bestCnt, &zDelta);
1076         *(zDelta++) = '@';
1077         putInt(bestOfst, &zDelta);
1078         *(zDelta++) = ',';
1079         if( bestOfst + bestCnt -1 > lastRead ){
1080           lastRead = bestOfst + bestCnt - 1;
1081         }
1082         bestCnt = 0;
1083         break;
1084       }
1085 
1086       /* If we reach this point, it means no match is found so far */
1087       if( base+i+NHASH>=lenOut ){
1088         /* We have reached the end of the file and have not found any
1089         ** matches.  Do an "insert" for everything that does not match */
1090         putInt(lenOut-base, &zDelta);
1091         *(zDelta++) = ':';
1092         memcpy(zDelta, &zOut[base], lenOut-base);
1093         zDelta += lenOut-base;
1094         base = lenOut;
1095         break;
1096       }
1097 
1098       /* Advance the hash by one character.  Keep looking for a match */
1099       hash_next(&h, zOut[base+i+NHASH]);
1100       i++;
1101     }
1102   }
1103   /* Output a final "insert" record to get all the text at the end of
1104   ** the file that does not match anything in the source file.
1105   */
1106   if( base<lenOut ){
1107     putInt(lenOut-base, &zDelta);
1108     *(zDelta++) = ':';
1109     memcpy(zDelta, &zOut[base], lenOut-base);
1110     zDelta += lenOut-base;
1111   }
1112   /* Output the final checksum record. */
1113   putInt(checksum(zOut, lenOut), &zDelta);
1114   *(zDelta++) = ';';
1115   sqlite3_free(collide);
1116   return (int)(zDelta - zOrigDelta);
1117 }
1118 
1119 /*
1120 ** End of code copied from fossil.
1121 **************************************************************************/
1122 
strPrintfArray(Str * pStr,const char * zSep,const char * zFmt,char ** az,int n)1123 static void strPrintfArray(
1124   Str *pStr,                      /* String object to append to */
1125   const char *zSep,               /* Separator string */
1126   const char *zFmt,               /* Format for each entry */
1127   char **az, int n                /* Array of strings & its size (or -1) */
1128 ){
1129   int i;
1130   for(i=0; az[i] && (i<n || n<0); i++){
1131     if( i!=0 ) strPrintf(pStr, "%s", zSep);
1132     strPrintf(pStr, zFmt, az[i], az[i], az[i]);
1133   }
1134 }
1135 
getRbudiffQuery(const char * zTab,char ** azCol,int nPK,int bOtaRowid,Str * pSql)1136 static void getRbudiffQuery(
1137   const char *zTab,
1138   char **azCol,
1139   int nPK,
1140   int bOtaRowid,
1141   Str *pSql
1142 ){
1143   int i;
1144 
1145   /* First the newly inserted rows: **/
1146   strPrintf(pSql, "SELECT ");
1147   strPrintfArray(pSql, ", ", "%s", azCol, -1);
1148   strPrintf(pSql, ", 0, ");       /* Set ota_control to 0 for an insert */
1149   strPrintfArray(pSql, ", ", "NULL", azCol, -1);
1150   strPrintf(pSql, " FROM aux.%Q AS n WHERE NOT EXISTS (\n", zTab);
1151   strPrintf(pSql, "    SELECT 1 FROM ", zTab);
1152   strPrintf(pSql, " main.%Q AS o WHERE ", zTab);
1153   strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
1154   strPrintf(pSql, "\n) AND ");
1155   strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK);
1156 
1157   /* Deleted rows: */
1158   strPrintf(pSql, "\nUNION ALL\nSELECT ");
1159   strPrintfArray(pSql, ", ", "%s", azCol, nPK);
1160   if( azCol[nPK] ){
1161     strPrintf(pSql, ", ");
1162     strPrintfArray(pSql, ", ", "NULL", &azCol[nPK], -1);
1163   }
1164   strPrintf(pSql, ", 1, ");       /* Set ota_control to 1 for a delete */
1165   strPrintfArray(pSql, ", ", "NULL", azCol, -1);
1166   strPrintf(pSql, " FROM main.%Q AS n WHERE NOT EXISTS (\n", zTab);
1167   strPrintf(pSql, "    SELECT 1 FROM ", zTab);
1168   strPrintf(pSql, " aux.%Q AS o WHERE ", zTab);
1169   strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
1170   strPrintf(pSql, "\n) AND ");
1171   strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK);
1172 
1173   /* Updated rows. If all table columns are part of the primary key, there
1174   ** can be no updates. In this case this part of the compound SELECT can
1175   ** be omitted altogether. */
1176   if( azCol[nPK] ){
1177     strPrintf(pSql, "\nUNION ALL\nSELECT ");
1178     strPrintfArray(pSql, ", ", "n.%s", azCol, nPK);
1179     strPrintf(pSql, ",\n");
1180     strPrintfArray(pSql, " ,\n",
1181         "    CASE WHEN n.%s IS o.%s THEN NULL ELSE n.%s END", &azCol[nPK], -1
1182     );
1183 
1184     if( bOtaRowid==0 ){
1185       strPrintf(pSql, ", '");
1186       strPrintfArray(pSql, "", ".", azCol, nPK);
1187       strPrintf(pSql, "' ||\n");
1188     }else{
1189       strPrintf(pSql, ",\n");
1190     }
1191     strPrintfArray(pSql, " ||\n",
1192         "    CASE WHEN n.%s IS o.%s THEN '.' ELSE 'x' END", &azCol[nPK], -1
1193     );
1194     strPrintf(pSql, "\nAS ota_control, ");
1195     strPrintfArray(pSql, ", ", "NULL", azCol, nPK);
1196     strPrintf(pSql, ",\n");
1197     strPrintfArray(pSql, " ,\n",
1198         "    CASE WHEN n.%s IS o.%s THEN NULL ELSE o.%s END", &azCol[nPK], -1
1199     );
1200 
1201     strPrintf(pSql, "\nFROM main.%Q AS o, aux.%Q AS n\nWHERE ", zTab, zTab);
1202     strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
1203     strPrintf(pSql, " AND ota_control LIKE '%%x%%'");
1204   }
1205 
1206   /* Now add an ORDER BY clause to sort everything by PK. */
1207   strPrintf(pSql, "\nORDER BY ");
1208   for(i=1; i<=nPK; i++) strPrintf(pSql, "%s%d", ((i>1)?", ":""), i);
1209 }
1210 
rbudiff_one_table(const char * zTab,FILE * out)1211 static void rbudiff_one_table(const char *zTab, FILE *out){
1212   int bOtaRowid;                  /* True to use an ota_rowid column */
1213   int nPK;                        /* Number of primary key columns in table */
1214   char **azCol;                   /* NULL terminated array of col names */
1215   int i;
1216   int nCol;
1217   Str ct = {0, 0, 0};             /* The "CREATE TABLE data_xxx" statement */
1218   Str sql = {0, 0, 0};            /* Query to find differences */
1219   Str insert = {0, 0, 0};         /* First part of output INSERT statement */
1220   sqlite3_stmt *pStmt = 0;
1221   int nRow = 0;                   /* Total rows in data_xxx table */
1222 
1223   /* --rbu mode must use real primary keys. */
1224   g.bSchemaPK = 1;
1225 
1226   /* Check that the schemas of the two tables match. Exit early otherwise. */
1227   checkSchemasMatch(zTab);
1228 
1229   /* Grab the column names and PK details for the table(s). If no usable PK
1230   ** columns are found, bail out early.  */
1231   azCol = columnNames("main", zTab, &nPK, &bOtaRowid);
1232   if( azCol==0 ){
1233     runtimeError("table %s has no usable PK columns", zTab);
1234   }
1235   for(nCol=0; azCol[nCol]; nCol++);
1236 
1237   /* Build and output the CREATE TABLE statement for the data_xxx table */
1238   strPrintf(&ct, "CREATE TABLE IF NOT EXISTS 'data_%q'(", zTab);
1239   if( bOtaRowid ) strPrintf(&ct, "rbu_rowid, ");
1240   strPrintfArray(&ct, ", ", "%s", &azCol[bOtaRowid], -1);
1241   strPrintf(&ct, ", rbu_control);");
1242 
1243   /* Get the SQL for the query to retrieve data from the two databases */
1244   getRbudiffQuery(zTab, azCol, nPK, bOtaRowid, &sql);
1245 
1246   /* Build the first part of the INSERT statement output for each row
1247   ** in the data_xxx table. */
1248   strPrintf(&insert, "INSERT INTO 'data_%q' (", zTab);
1249   if( bOtaRowid ) strPrintf(&insert, "rbu_rowid, ");
1250   strPrintfArray(&insert, ", ", "%s", &azCol[bOtaRowid], -1);
1251   strPrintf(&insert, ", rbu_control) VALUES(");
1252 
1253   pStmt = db_prepare("%s", sql.z);
1254 
1255   while( sqlite3_step(pStmt)==SQLITE_ROW ){
1256 
1257     /* If this is the first row output, print out the CREATE TABLE
1258     ** statement first. And then set ct.z to NULL so that it is not
1259     ** printed again.  */
1260     if( ct.z ){
1261       fprintf(out, "%s\n", ct.z);
1262       strFree(&ct);
1263     }
1264 
1265     /* Output the first part of the INSERT statement */
1266     fprintf(out, "%s", insert.z);
1267     nRow++;
1268 
1269     if( sqlite3_column_type(pStmt, nCol)==SQLITE_INTEGER ){
1270       for(i=0; i<=nCol; i++){
1271         if( i>0 ) fprintf(out, ", ");
1272         printQuoted(out, sqlite3_column_value(pStmt, i));
1273       }
1274     }else{
1275       char *zOtaControl;
1276       int nOtaControl = sqlite3_column_bytes(pStmt, nCol);
1277 
1278       zOtaControl = (char*)sqlite3_malloc(nOtaControl+1);
1279       memcpy(zOtaControl, sqlite3_column_text(pStmt, nCol), nOtaControl+1);
1280 
1281       for(i=0; i<nCol; i++){
1282         int bDone = 0;
1283         if( i>=nPK
1284             && sqlite3_column_type(pStmt, i)==SQLITE_BLOB
1285             && sqlite3_column_type(pStmt, nCol+1+i)==SQLITE_BLOB
1286         ){
1287           const char *aSrc = sqlite3_column_blob(pStmt, nCol+1+i);
1288           int nSrc = sqlite3_column_bytes(pStmt, nCol+1+i);
1289           const char *aFinal = sqlite3_column_blob(pStmt, i);
1290           int nFinal = sqlite3_column_bytes(pStmt, i);
1291           char *aDelta;
1292           int nDelta;
1293 
1294           aDelta = sqlite3_malloc(nFinal + 60);
1295           nDelta = rbuDeltaCreate(aSrc, nSrc, aFinal, nFinal, aDelta);
1296           if( nDelta<nFinal ){
1297             int j;
1298             fprintf(out, "x'");
1299             for(j=0; j<nDelta; j++) fprintf(out, "%02x", (u8)aDelta[j]);
1300             fprintf(out, "'");
1301             zOtaControl[i-bOtaRowid] = 'f';
1302             bDone = 1;
1303           }
1304           sqlite3_free(aDelta);
1305         }
1306 
1307         if( bDone==0 ){
1308           printQuoted(out, sqlite3_column_value(pStmt, i));
1309         }
1310         fprintf(out, ", ");
1311       }
1312       fprintf(out, "'%s'", zOtaControl);
1313       sqlite3_free(zOtaControl);
1314     }
1315 
1316     /* And the closing bracket of the insert statement */
1317     fprintf(out, ");\n");
1318   }
1319 
1320   sqlite3_finalize(pStmt);
1321   if( nRow>0 ){
1322     Str cnt = {0, 0, 0};
1323     strPrintf(&cnt, "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab, nRow);
1324     fprintf(out, "%s\n", cnt.z);
1325     strFree(&cnt);
1326   }
1327 
1328   strFree(&ct);
1329   strFree(&sql);
1330   strFree(&insert);
1331 }
1332 
1333 /*
1334 ** Display a summary of differences between two versions of the same
1335 ** table table.
1336 **
1337 **   *  Number of rows changed
1338 **   *  Number of rows added
1339 **   *  Number of rows deleted
1340 **   *  Number of identical rows
1341 */
summarize_one_table(const char * zTab,FILE * out)1342 static void summarize_one_table(const char *zTab, FILE *out){
1343   char *zId = safeId(zTab); /* Name of table (translated for us in SQL) */
1344   char **az = 0;            /* Columns in main */
1345   char **az2 = 0;           /* Columns in aux */
1346   int nPk;                  /* Primary key columns in main */
1347   int nPk2;                 /* Primary key columns in aux */
1348   int n = 0;                /* Number of columns in main */
1349   int n2;                   /* Number of columns in aux */
1350   int i;                    /* Loop counter */
1351   const char *zSep;         /* Separator string */
1352   Str sql;                  /* Comparison query */
1353   sqlite3_stmt *pStmt;      /* Query statement to do the diff */
1354   sqlite3_int64 nUpdate;    /* Number of updated rows */
1355   sqlite3_int64 nUnchanged; /* Number of unmodified rows */
1356   sqlite3_int64 nDelete;    /* Number of deleted rows */
1357   sqlite3_int64 nInsert;    /* Number of inserted rows */
1358 
1359   strInit(&sql);
1360   if( sqlite3_table_column_metadata(g.db,"aux",zTab,0,0,0,0,0,0) ){
1361     if( !sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
1362       /* Table missing from second database. */
1363       fprintf(out, "%s: missing from second database\n", zTab);
1364     }
1365     goto end_summarize_one_table;
1366   }
1367 
1368   if( sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
1369     /* Table missing from source */
1370     fprintf(out, "%s: missing from first database\n", zTab);
1371     goto end_summarize_one_table;
1372   }
1373 
1374   az = columnNames("main", zTab, &nPk, 0);
1375   az2 = columnNames("aux", zTab, &nPk2, 0);
1376   if( az && az2 ){
1377     for(n=0; az[n]; n++){
1378       if( sqlite3_stricmp(az[n],az2[n])!=0 ) break;
1379     }
1380   }
1381   if( az==0
1382    || az2==0
1383    || nPk!=nPk2
1384    || az[n]
1385   ){
1386     /* Schema mismatch */
1387     fprintf(out, "%s: incompatible schema\n", zTab);
1388     goto end_summarize_one_table;
1389   }
1390 
1391   /* Build the comparison query */
1392   for(n2=n; az[n2]; n2++){}
1393   strPrintf(&sql, "SELECT 1, count(*)");
1394   if( n2==nPk2 ){
1395     strPrintf(&sql, ", 0\n");
1396   }else{
1397     zSep = ", sum(";
1398     for(i=nPk; az[i]; i++){
1399       strPrintf(&sql, "%sA.%s IS NOT B.%s", zSep, az[i], az[i]);
1400       zSep = " OR ";
1401     }
1402     strPrintf(&sql, ")\n");
1403   }
1404   strPrintf(&sql, "  FROM main.%s A, aux.%s B\n", zId, zId);
1405   zSep = " WHERE";
1406   for(i=0; i<nPk; i++){
1407     strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
1408     zSep = " AND";
1409   }
1410   strPrintf(&sql, " UNION ALL\n");
1411   strPrintf(&sql, "SELECT 2, count(*), 0\n");
1412   strPrintf(&sql, "  FROM main.%s A\n", zId);
1413   strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B ", zId);
1414   zSep = "WHERE";
1415   for(i=0; i<nPk; i++){
1416     strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
1417     zSep = " AND";
1418   }
1419   strPrintf(&sql, ")\n");
1420   strPrintf(&sql, " UNION ALL\n");
1421   strPrintf(&sql, "SELECT 3, count(*), 0\n");
1422   strPrintf(&sql, "  FROM aux.%s B\n", zId);
1423   strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A ", zId);
1424   zSep = "WHERE";
1425   for(i=0; i<nPk; i++){
1426     strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
1427     zSep = " AND";
1428   }
1429   strPrintf(&sql, ")\n ORDER BY 1;\n");
1430 
1431   if( (g.fDebug & DEBUG_DIFF_SQL)!=0 ){
1432     printf("SQL for %s:\n%s\n", zId, sql.z);
1433     goto end_summarize_one_table;
1434   }
1435 
1436   /* Run the query and output difference summary */
1437   pStmt = db_prepare("%s", sql.z);
1438   nUpdate = 0;
1439   nInsert = 0;
1440   nDelete = 0;
1441   nUnchanged = 0;
1442   while( SQLITE_ROW==sqlite3_step(pStmt) ){
1443     switch( sqlite3_column_int(pStmt,0) ){
1444       case 1:
1445         nUpdate = sqlite3_column_int64(pStmt,2);
1446         nUnchanged = sqlite3_column_int64(pStmt,1) - nUpdate;
1447         break;
1448       case 2:
1449         nDelete = sqlite3_column_int64(pStmt,1);
1450         break;
1451       case 3:
1452         nInsert = sqlite3_column_int64(pStmt,1);
1453         break;
1454     }
1455   }
1456   sqlite3_finalize(pStmt);
1457   fprintf(out, "%s: %lld changes, %lld inserts, %lld deletes, %lld unchanged\n",
1458           zTab, nUpdate, nInsert, nDelete, nUnchanged);
1459 
1460 end_summarize_one_table:
1461   strFree(&sql);
1462   sqlite3_free(zId);
1463   namelistFree(az);
1464   namelistFree(az2);
1465   return;
1466 }
1467 
1468 /*
1469 ** Write a 64-bit signed integer as a varint onto out
1470 */
putsVarint(FILE * out,sqlite3_uint64 v)1471 static void putsVarint(FILE *out, sqlite3_uint64 v){
1472   int i, n;
1473   unsigned char p[12];
1474   if( v & (((sqlite3_uint64)0xff000000)<<32) ){
1475     p[8] = (unsigned char)v;
1476     v >>= 8;
1477     for(i=7; i>=0; i--){
1478       p[i] = (unsigned char)((v & 0x7f) | 0x80);
1479       v >>= 7;
1480     }
1481     fwrite(p, 8, 1, out);
1482   }else{
1483     n = 9;
1484     do{
1485       p[n--] = (unsigned char)((v & 0x7f) | 0x80);
1486       v >>= 7;
1487     }while( v!=0 );
1488     p[9] &= 0x7f;
1489     fwrite(p+n+1, 9-n, 1, out);
1490   }
1491 }
1492 
1493 /*
1494 ** Write an SQLite value onto out.
1495 */
putValue(FILE * out,sqlite3_stmt * pStmt,int k)1496 static void putValue(FILE *out, sqlite3_stmt *pStmt, int k){
1497   int iDType = sqlite3_column_type(pStmt, k);
1498   sqlite3_int64 iX;
1499   double rX;
1500   sqlite3_uint64 uX;
1501   int j;
1502 
1503   putc(iDType, out);
1504   switch( iDType ){
1505     case SQLITE_INTEGER:
1506       iX = sqlite3_column_int64(pStmt, k);
1507       memcpy(&uX, &iX, 8);
1508       for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out);
1509       break;
1510     case SQLITE_FLOAT:
1511       rX = sqlite3_column_double(pStmt, k);
1512       memcpy(&uX, &rX, 8);
1513       for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out);
1514       break;
1515     case SQLITE_TEXT:
1516       iX = sqlite3_column_bytes(pStmt, k);
1517       putsVarint(out, (sqlite3_uint64)iX);
1518       fwrite(sqlite3_column_text(pStmt, k),1,(size_t)iX,out);
1519       break;
1520     case SQLITE_BLOB:
1521       iX = sqlite3_column_bytes(pStmt, k);
1522       putsVarint(out, (sqlite3_uint64)iX);
1523       fwrite(sqlite3_column_blob(pStmt, k),1,(size_t)iX,out);
1524       break;
1525     case SQLITE_NULL:
1526       break;
1527   }
1528 }
1529 
1530 /*
1531 ** Generate a CHANGESET for all differences from main.zTab to aux.zTab.
1532 */
changeset_one_table(const char * zTab,FILE * out)1533 static void changeset_one_table(const char *zTab, FILE *out){
1534   sqlite3_stmt *pStmt;          /* SQL statment */
1535   char *zId = safeId(zTab);     /* Escaped name of the table */
1536   char **azCol = 0;             /* List of escaped column names */
1537   int nCol = 0;                 /* Number of columns */
1538   int *aiFlg = 0;               /* 0 if column is not part of PK */
1539   int *aiPk = 0;                /* Column numbers for each PK column */
1540   int nPk = 0;                  /* Number of PRIMARY KEY columns */
1541   Str sql;                      /* SQL for the diff query */
1542   int i, k;                     /* Loop counters */
1543   const char *zSep;             /* List separator */
1544 
1545   /* Check that the schemas of the two tables match. Exit early otherwise. */
1546   checkSchemasMatch(zTab);
1547 
1548   pStmt = db_prepare("PRAGMA main.table_info=%Q", zTab);
1549   while( SQLITE_ROW==sqlite3_step(pStmt) ){
1550     nCol++;
1551     azCol = sqlite3_realloc(azCol, sizeof(char*)*nCol);
1552     if( azCol==0 ) runtimeError("out of memory");
1553     aiFlg = sqlite3_realloc(aiFlg, sizeof(int)*nCol);
1554     if( aiFlg==0 ) runtimeError("out of memory");
1555     azCol[nCol-1] = safeId((const char*)sqlite3_column_text(pStmt,1));
1556     aiFlg[nCol-1] = i = sqlite3_column_int(pStmt,5);
1557     if( i>0 ){
1558       if( i>nPk ){
1559         nPk = i;
1560         aiPk = sqlite3_realloc(aiPk, sizeof(int)*nPk);
1561         if( aiPk==0 ) runtimeError("out of memory");
1562       }
1563       aiPk[i-1] = nCol-1;
1564     }
1565   }
1566   sqlite3_finalize(pStmt);
1567   if( nPk==0 ) goto end_changeset_one_table;
1568   strInit(&sql);
1569   if( nCol>nPk ){
1570     strPrintf(&sql, "SELECT %d", SQLITE_UPDATE);
1571     for(i=0; i<nCol; i++){
1572       if( aiFlg[i] ){
1573         strPrintf(&sql, ",\n       A.%s", azCol[i]);
1574       }else{
1575         strPrintf(&sql, ",\n       A.%s IS NOT B.%s, A.%s, B.%s",
1576                   azCol[i], azCol[i], azCol[i], azCol[i]);
1577       }
1578     }
1579     strPrintf(&sql,"\n  FROM main.%s A, aux.%s B\n", zId, zId);
1580     zSep = " WHERE";
1581     for(i=0; i<nPk; i++){
1582       strPrintf(&sql, "%s A.%s=B.%s", zSep, azCol[aiPk[i]], azCol[aiPk[i]]);
1583       zSep = " AND";
1584     }
1585     zSep = "\n   AND (";
1586     for(i=0; i<nCol; i++){
1587       if( aiFlg[i] ) continue;
1588       strPrintf(&sql, "%sA.%s IS NOT B.%s", zSep, azCol[i], azCol[i]);
1589       zSep = " OR\n        ";
1590     }
1591     strPrintf(&sql,")\n UNION ALL\n");
1592   }
1593   strPrintf(&sql, "SELECT %d", SQLITE_DELETE);
1594   for(i=0; i<nCol; i++){
1595     if( aiFlg[i] ){
1596       strPrintf(&sql, ",\n       A.%s", azCol[i]);
1597     }else{
1598       strPrintf(&sql, ",\n       1, A.%s, NULL", azCol[i]);
1599     }
1600   }
1601   strPrintf(&sql, "\n  FROM main.%s A\n", zId);
1602   strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId);
1603   zSep =          "                   WHERE";
1604   for(i=0; i<nPk; i++){
1605     strPrintf(&sql, "%s A.%s=B.%s", zSep, azCol[aiPk[i]], azCol[aiPk[i]]);
1606     zSep = " AND";
1607   }
1608   strPrintf(&sql, ")\n UNION ALL\n");
1609   strPrintf(&sql, "SELECT %d", SQLITE_INSERT);
1610   for(i=0; i<nCol; i++){
1611     if( aiFlg[i] ){
1612       strPrintf(&sql, ",\n       B.%s", azCol[i]);
1613     }else{
1614       strPrintf(&sql, ",\n       1, NULL, B.%s", azCol[i]);
1615     }
1616   }
1617   strPrintf(&sql, "\n  FROM aux.%s B\n", zId);
1618   strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId);
1619   zSep =          "                   WHERE";
1620   for(i=0; i<nPk; i++){
1621     strPrintf(&sql, "%s A.%s=B.%s", zSep, azCol[aiPk[i]], azCol[aiPk[i]]);
1622     zSep = " AND";
1623   }
1624   strPrintf(&sql, ")\n");
1625   strPrintf(&sql, " ORDER BY");
1626   zSep = " ";
1627   for(i=0; i<nPk; i++){
1628     strPrintf(&sql, "%s %d", zSep, aiPk[i]+2);
1629     zSep = ",";
1630   }
1631   strPrintf(&sql, ";\n");
1632 
1633   if( g.fDebug & DEBUG_DIFF_SQL ){
1634     printf("SQL for %s:\n%s\n", zId, sql.z);
1635     goto end_changeset_one_table;
1636   }
1637 
1638   putc('T', out);
1639   putsVarint(out, (sqlite3_uint64)nCol);
1640   for(i=0; i<nCol; i++) putc(aiFlg[i], out);
1641   fwrite(zTab, 1, strlen(zTab), out);
1642   putc(0, out);
1643 
1644   pStmt = db_prepare("%s", sql.z);
1645   while( SQLITE_ROW==sqlite3_step(pStmt) ){
1646     int iType = sqlite3_column_int(pStmt,0);
1647     putc(iType, out);
1648     putc(0, out);
1649     switch( sqlite3_column_int(pStmt,0) ){
1650       case SQLITE_UPDATE: {
1651         for(k=1, i=0; i<nCol; i++){
1652           if( aiFlg[i] ){
1653             putValue(out, pStmt, k);
1654             k++;
1655           }else if( sqlite3_column_int(pStmt,k) ){
1656             putValue(out, pStmt, k+1);
1657             k += 3;
1658           }else{
1659             putc(0, out);
1660             k += 3;
1661           }
1662         }
1663         for(k=1, i=0; i<nCol; i++){
1664           if( aiFlg[i] ){
1665             putc(0, out);
1666             k++;
1667           }else if( sqlite3_column_int(pStmt,k) ){
1668             putValue(out, pStmt, k+2);
1669             k += 3;
1670           }else{
1671             putc(0, out);
1672             k += 3;
1673           }
1674         }
1675         break;
1676       }
1677       case SQLITE_INSERT: {
1678         for(k=1, i=0; i<nCol; i++){
1679           if( aiFlg[i] ){
1680             putValue(out, pStmt, k);
1681             k++;
1682           }else{
1683             putValue(out, pStmt, k+2);
1684             k += 3;
1685           }
1686         }
1687         break;
1688       }
1689       case SQLITE_DELETE: {
1690         for(k=1, i=0; i<nCol; i++){
1691           if( aiFlg[i] ){
1692             putValue(out, pStmt, k);
1693             k++;
1694           }else{
1695             putValue(out, pStmt, k+1);
1696             k += 3;
1697           }
1698         }
1699         break;
1700       }
1701     }
1702   }
1703   sqlite3_finalize(pStmt);
1704 
1705 end_changeset_one_table:
1706   while( nCol>0 ) sqlite3_free(azCol[--nCol]);
1707   sqlite3_free(azCol);
1708   sqlite3_free(aiPk);
1709   sqlite3_free(zId);
1710 }
1711 
1712 /*
1713 ** Extract the next SQL keyword or quoted string from buffer zIn and copy it
1714 ** (or a prefix of it if it will not fit) into buffer zBuf, size nBuf bytes.
1715 ** Return a pointer to the character within zIn immediately following
1716 ** the token or quoted string just extracted.
1717 */
gobble_token(const char * zIn,char * zBuf,int nBuf)1718 const char *gobble_token(const char *zIn, char *zBuf, int nBuf){
1719   const char *p = zIn;
1720   char *pOut = zBuf;
1721   char *pEnd = &pOut[nBuf-1];
1722   char q = 0;                     /* quote character, if any */
1723 
1724   if( p==0 ) return 0;
1725   while( *p==' ' ) p++;
1726   switch( *p ){
1727     case '"': q = '"'; break;
1728     case '\'': q = '\''; break;
1729     case '`': q = '`'; break;
1730     case '[': q = ']'; break;
1731   }
1732 
1733   if( q ){
1734     p++;
1735     while( *p && pOut<pEnd ){
1736       if( *p==q ){
1737         p++;
1738         if( *p!=q ) break;
1739       }
1740       if( pOut<pEnd ) *pOut++ = *p;
1741       p++;
1742     }
1743   }else{
1744     while( *p && *p!=' ' && *p!='(' ){
1745       if( pOut<pEnd ) *pOut++ = *p;
1746       p++;
1747     }
1748   }
1749 
1750   *pOut = '\0';
1751   return p;
1752 }
1753 
1754 /*
1755 ** This function is the implementation of SQL scalar function "module_name":
1756 **
1757 **   module_name(SQL)
1758 **
1759 ** The only argument should be an SQL statement of the type that may appear
1760 ** in the sqlite_master table. If the statement is a "CREATE VIRTUAL TABLE"
1761 ** statement, then the value returned is the name of the module that it
1762 ** uses. Otherwise, if the statement is not a CVT, NULL is returned.
1763 */
module_name_func(sqlite3_context * pCtx,int nVal,sqlite3_value ** apVal)1764 static void module_name_func(
1765   sqlite3_context *pCtx,
1766   int nVal, sqlite3_value **apVal
1767 ){
1768   const char *zSql;
1769   char zToken[32];
1770 
1771   assert( nVal==1 );
1772   zSql = (const char*)sqlite3_value_text(apVal[0]);
1773 
1774   zSql = gobble_token(zSql, zToken, sizeof(zToken));
1775   if( zSql==0 || sqlite3_stricmp(zToken, "create") ) return;
1776   zSql = gobble_token(zSql, zToken, sizeof(zToken));
1777   if( zSql==0 || sqlite3_stricmp(zToken, "virtual") ) return;
1778   zSql = gobble_token(zSql, zToken, sizeof(zToken));
1779   if( zSql==0 || sqlite3_stricmp(zToken, "table") ) return;
1780   zSql = gobble_token(zSql, zToken, sizeof(zToken));
1781   if( zSql==0 ) return;
1782   zSql = gobble_token(zSql, zToken, sizeof(zToken));
1783   if( zSql==0 || sqlite3_stricmp(zToken, "using") ) return;
1784   zSql = gobble_token(zSql, zToken, sizeof(zToken));
1785 
1786   sqlite3_result_text(pCtx, zToken, -1, SQLITE_TRANSIENT);
1787 }
1788 
1789 /*
1790 ** Return the text of an SQL statement that itself returns the list of
1791 ** tables to process within the database.
1792 */
all_tables_sql()1793 const char *all_tables_sql(){
1794   if( g.bHandleVtab ){
1795     int rc;
1796 
1797     rc = sqlite3_exec(g.db,
1798         "CREATE TEMP TABLE tblmap(module COLLATE nocase, postfix);"
1799         "INSERT INTO temp.tblmap VALUES"
1800         "('fts3', '_content'), ('fts3', '_segments'), ('fts3', '_segdir'),"
1801 
1802         "('fts4', '_content'), ('fts4', '_segments'), ('fts4', '_segdir'),"
1803         "('fts4', '_docsize'), ('fts4', '_stat'),"
1804 
1805         "('fts5', '_data'), ('fts5', '_idx'), ('fts5', '_content'),"
1806         "('fts5', '_docsize'), ('fts5', '_config'),"
1807 
1808         "('rtree', '_node'), ('rtree', '_rowid'), ('rtree', '_parent');"
1809         , 0, 0, 0
1810     );
1811     assert( rc==SQLITE_OK );
1812 
1813     rc = sqlite3_create_function(
1814         g.db, "module_name", 1, SQLITE_UTF8, 0, module_name_func, 0, 0
1815     );
1816     assert( rc==SQLITE_OK );
1817 
1818     return
1819       "SELECT name FROM main.sqlite_master\n"
1820       " WHERE type='table' AND (\n"
1821       "    module_name(sql) IS NULL OR \n"
1822       "    module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
1823       " ) AND name NOT IN (\n"
1824       "  SELECT a.name || b.postfix \n"
1825         "FROM main.sqlite_master AS a, temp.tblmap AS b \n"
1826         "WHERE module_name(a.sql) = b.module\n"
1827       " )\n"
1828       "UNION \n"
1829       "SELECT name FROM aux.sqlite_master\n"
1830       " WHERE type='table' AND (\n"
1831       "    module_name(sql) IS NULL OR \n"
1832       "    module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
1833       " ) AND name NOT IN (\n"
1834       "  SELECT a.name || b.postfix \n"
1835         "FROM aux.sqlite_master AS a, temp.tblmap AS b \n"
1836         "WHERE module_name(a.sql) = b.module\n"
1837       " )\n"
1838       " ORDER BY name";
1839   }else{
1840     return
1841       "SELECT name FROM main.sqlite_master\n"
1842       " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
1843       " UNION\n"
1844       "SELECT name FROM aux.sqlite_master\n"
1845       " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
1846       " ORDER BY name";
1847   }
1848 }
1849 
1850 /*
1851 ** Print sketchy documentation for this utility program
1852 */
showHelp(void)1853 static void showHelp(void){
1854   printf("Usage: %s [options] DB1 DB2\n", g.zArgv0);
1855   printf(
1856 "Output SQL text that would transform DB1 into DB2.\n"
1857 "Options:\n"
1858 "  --changeset FILE      Write a CHANGESET into FILE\n"
1859 "  -L|--lib LIBRARY      Load an SQLite extension library\n"
1860 "  --primarykey          Use schema-defined PRIMARY KEYs\n"
1861 "  --rbu                 Output SQL to create/populate RBU table(s)\n"
1862 "  --schema              Show only differences in the schema\n"
1863 "  --summary             Show only a summary of the differences\n"
1864 "  --table TAB           Show only differences in table TAB\n"
1865 "  --transaction         Show SQL output inside a transaction\n"
1866 "  --vtab                Handle fts3, fts4, fts5 and rtree tables\n"
1867   );
1868 }
1869 
main(int argc,char ** argv)1870 int main(int argc, char **argv){
1871   const char *zDb1 = 0;
1872   const char *zDb2 = 0;
1873   int i;
1874   int rc;
1875   char *zErrMsg = 0;
1876   char *zSql;
1877   sqlite3_stmt *pStmt;
1878   char *zTab = 0;
1879   FILE *out = stdout;
1880   void (*xDiff)(const char*,FILE*) = diff_one_table;
1881 #ifndef SQLITE_OMIT_LOAD_EXTENSION
1882   int nExt = 0;
1883   char **azExt = 0;
1884 #endif
1885   int useTransaction = 0;
1886   int neverUseTransaction = 0;
1887 
1888   g.zArgv0 = argv[0];
1889   sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
1890   for(i=1; i<argc; i++){
1891     const char *z = argv[i];
1892     if( z[0]=='-' ){
1893       z++;
1894       if( z[0]=='-' ) z++;
1895       if( strcmp(z,"changeset")==0 ){
1896         if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
1897         out = fopen(argv[++i], "wb");
1898         if( out==0 ) cmdlineError("cannot open: %s", argv[i]);
1899         xDiff = changeset_one_table;
1900         neverUseTransaction = 1;
1901       }else
1902       if( strcmp(z,"debug")==0 ){
1903         if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
1904         g.fDebug = strtol(argv[++i], 0, 0);
1905       }else
1906       if( strcmp(z,"help")==0 ){
1907         showHelp();
1908         return 0;
1909       }else
1910 #ifndef SQLITE_OMIT_LOAD_EXTENSION
1911       if( strcmp(z,"lib")==0 || strcmp(z,"L")==0 ){
1912         if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
1913         azExt = realloc(azExt, sizeof(azExt[0])*(nExt+1));
1914         if( azExt==0 ) cmdlineError("out of memory");
1915         azExt[nExt++] = argv[++i];
1916       }else
1917 #endif
1918       if( strcmp(z,"primarykey")==0 ){
1919         g.bSchemaPK = 1;
1920       }else
1921       if( strcmp(z,"rbu")==0 ){
1922         xDiff = rbudiff_one_table;
1923       }else
1924       if( strcmp(z,"schema")==0 ){
1925         g.bSchemaOnly = 1;
1926       }else
1927       if( strcmp(z,"summary")==0 ){
1928         xDiff = summarize_one_table;
1929       }else
1930       if( strcmp(z,"table")==0 ){
1931         if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
1932         zTab = argv[++i];
1933       }else
1934       if( strcmp(z,"transaction")==0 ){
1935         useTransaction = 1;
1936       }else
1937       if( strcmp(z,"vtab")==0 ){
1938         g.bHandleVtab = 1;
1939       }else
1940       {
1941         cmdlineError("unknown option: %s", argv[i]);
1942       }
1943     }else if( zDb1==0 ){
1944       zDb1 = argv[i];
1945     }else if( zDb2==0 ){
1946       zDb2 = argv[i];
1947     }else{
1948       cmdlineError("unknown argument: %s", argv[i]);
1949     }
1950   }
1951   if( zDb2==0 ){
1952     cmdlineError("two database arguments required");
1953   }
1954   rc = sqlite3_open(zDb1, &g.db);
1955   if( rc ){
1956     cmdlineError("cannot open database file \"%s\"", zDb1);
1957   }
1958   rc = sqlite3_exec(g.db, "SELECT * FROM sqlite_master", 0, 0, &zErrMsg);
1959   if( rc || zErrMsg ){
1960     cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb1);
1961   }
1962 #ifndef SQLITE_OMIT_LOAD_EXTENSION
1963   sqlite3_enable_load_extension(g.db, 1);
1964   for(i=0; i<nExt; i++){
1965     rc = sqlite3_load_extension(g.db, azExt[i], 0, &zErrMsg);
1966     if( rc || zErrMsg ){
1967       cmdlineError("error loading %s: %s", azExt[i], zErrMsg);
1968     }
1969   }
1970   free(azExt);
1971 #endif
1972   zSql = sqlite3_mprintf("ATTACH %Q as aux;", zDb2);
1973   rc = sqlite3_exec(g.db, zSql, 0, 0, &zErrMsg);
1974   if( rc || zErrMsg ){
1975     cmdlineError("cannot attach database \"%s\"", zDb2);
1976   }
1977   rc = sqlite3_exec(g.db, "SELECT * FROM aux.sqlite_master", 0, 0, &zErrMsg);
1978   if( rc || zErrMsg ){
1979     cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb2);
1980   }
1981 
1982   if( neverUseTransaction ) useTransaction = 0;
1983   if( useTransaction ) fprintf(out, "BEGIN TRANSACTION;\n");
1984   if( xDiff==rbudiff_one_table ){
1985     fprintf(out, "CREATE TABLE IF NOT EXISTS rbu_count"
1986            "(tbl TEXT PRIMARY KEY COLLATE NOCASE, cnt INTEGER) "
1987            "WITHOUT ROWID;\n"
1988     );
1989   }
1990   if( zTab ){
1991     xDiff(zTab, out);
1992   }else{
1993     /* Handle tables one by one */
1994     pStmt = db_prepare("%s", all_tables_sql() );
1995     while( SQLITE_ROW==sqlite3_step(pStmt) ){
1996       xDiff((const char*)sqlite3_column_text(pStmt,0), out);
1997     }
1998     sqlite3_finalize(pStmt);
1999   }
2000   if( useTransaction ) printf("COMMIT;\n");
2001 
2002   /* TBD: Handle trigger differences */
2003   /* TBD: Handle view differences */
2004   sqlite3_close(g.db);
2005   return 0;
2006 }
2007