1 /*
2 ** Copyright (c) 2007 D. Richard Hipp
3 **
4 ** This program is free software; you can redistribute it and/or
5 ** modify it under the terms of the Simplified BSD License (also
6 ** known as the "2-Clause License" or "FreeBSD License".)
7 **
8 ** This program is distributed in the hope that it will be useful,
9 ** but without any warranty; without even the implied warranty of
10 ** merchantability or fitness for a particular purpose.
11 **
12 ** Author contact information:
13 ** drh@hwaci.com
14 ** http://www.hwaci.com/drh/
15 **
16 *******************************************************************************
17 **
18 ** This file contains code to implement the file transfer protocol.
19 */
20 #include "config.h"
21 #include "xfer.h"
22
23 #include <time.h>
24
25 /*
26 ** Maximum number of HTTP redirects that any http_exchange() call will
27 ** follow before throwing a fatal error. Most browsers use a limit of 20.
28 */
29 #define MAX_REDIRECTS 20
30
31 /*
32 ** This structure holds information about the current state of either
33 ** a client or a server that is participating in xfer.
34 */
35 typedef struct Xfer Xfer;
36 struct Xfer {
37 Blob *pIn; /* Input text from the other side */
38 Blob *pOut; /* Compose our reply here */
39 Blob line; /* The current line of input */
40 Blob aToken[6]; /* Tokenized version of line */
41 Blob err; /* Error message text */
42 int nToken; /* Number of tokens in line */
43 int nIGotSent; /* Number of "igot" cards sent */
44 int nPrivIGot; /* Number of private "igot" cards */
45 int nGimmeSent; /* Number of gimme cards sent */
46 int nFileSent; /* Number of files sent */
47 int nDeltaSent; /* Number of deltas sent */
48 int nFileRcvd; /* Number of files received */
49 int nDeltaRcvd; /* Number of deltas received */
50 int nDanglingFile; /* Number of dangling deltas received */
51 int mxSend; /* Stop sending "file" when pOut reaches this size */
52 int resync; /* Send igot cards for all holdings */
53 u8 syncPrivate; /* True to enable syncing private content */
54 u8 nextIsPrivate; /* If true, next "file" received is a private */
55 u32 remoteVersion; /* Version of fossil running on the other side */
56 u32 remoteDate; /* Date for specific client software edition */
57 u32 remoteTime; /* Time of date correspoding on remoteDate */
58 time_t maxTime; /* Time when this transfer should be finished */
59 };
60
61
62 /*
63 ** The input blob contains an artifact. Convert it into a record ID.
64 ** Create a phantom record if no prior record exists and
65 ** phantomize is true.
66 **
67 ** Compare to uuid_to_rid(). This routine takes a blob argument
68 ** and does less error checking.
69 */
rid_from_uuid(Blob * pUuid,int phantomize,int isPrivate)70 static int rid_from_uuid(Blob *pUuid, int phantomize, int isPrivate){
71 static Stmt q;
72 int rid;
73
74 db_static_prepare(&q, "SELECT rid FROM blob WHERE uuid=:uuid");
75 db_bind_str(&q, ":uuid", pUuid);
76 if( db_step(&q)==SQLITE_ROW ){
77 rid = db_column_int(&q, 0);
78 }else{
79 rid = 0;
80 }
81 db_reset(&q);
82 if( rid==0 && phantomize ){
83 rid = content_new(blob_str(pUuid), isPrivate);
84 }
85 return rid;
86 }
87
88 /*
89 ** Remember that the other side of the connection already has a copy
90 ** of the file rid.
91 */
remote_has(int rid)92 static void remote_has(int rid){
93 if( rid ){
94 static Stmt q;
95 db_static_prepare(&q, "INSERT OR IGNORE INTO onremote VALUES(:r)");
96 db_bind_int(&q, ":r", rid);
97 db_step(&q);
98 db_reset(&q);
99 }
100 }
101
102 /*
103 ** Remember that the other side of the connection lacks a copy of
104 ** the artifact with the given hash.
105 */
remote_unk(Blob * pHash)106 static void remote_unk(Blob *pHash){
107 static Stmt q;
108 db_static_prepare(&q, "INSERT OR IGNORE INTO unk VALUES(:h)");
109 db_bind_text(&q, ":h", blob_str(pHash));
110 db_step(&q);
111 db_reset(&q);
112 }
113
114 /*
115 ** The aToken[0..nToken-1] blob array is a parse of a "file" line
116 ** message. This routine finishes parsing that message and does
117 ** a record insert of the file.
118 **
119 ** The file line is in one of the following two forms:
120 **
121 ** file HASH SIZE \n CONTENT
122 ** file HASH DELTASRC SIZE \n CONTENT
123 **
124 ** The content is SIZE bytes immediately following the newline.
125 ** If DELTASRC exists, then the CONTENT is a delta against the
126 ** content of DELTASRC.
127 **
128 ** If any error occurs, write a message into pErr which has already
129 ** be initialized to an empty string.
130 **
131 ** Any artifact successfully received by this routine is considered to
132 ** be public and is therefore removed from the "private" table.
133 */
xfer_accept_file(Xfer * pXfer,int cloneFlag,char ** pzUuidList,int * pnUuidList)134 static void xfer_accept_file(
135 Xfer *pXfer,
136 int cloneFlag,
137 char **pzUuidList,
138 int *pnUuidList
139 ){
140 int n;
141 int rid;
142 int srcid = 0;
143 Blob content;
144 int isPriv;
145 Blob *pUuid;
146
147 isPriv = pXfer->nextIsPrivate;
148 pXfer->nextIsPrivate = 0;
149 if( pXfer->nToken<3
150 || pXfer->nToken>4
151 || !blob_is_hname(&pXfer->aToken[1])
152 || !blob_is_int(&pXfer->aToken[pXfer->nToken-1], &n)
153 || n<0
154 || (pXfer->nToken==4 && !blob_is_hname(&pXfer->aToken[2]))
155 ){
156 blob_appendf(&pXfer->err, "malformed file line");
157 return;
158 }
159 blob_zero(&content);
160 blob_extract(pXfer->pIn, n, &content);
161 pUuid = &pXfer->aToken[1];
162 if( !cloneFlag && uuid_is_shunned(blob_str(pUuid)) ){
163 /* Ignore files that have been shunned */
164 blob_reset(&content);
165 return;
166 }
167 if( isPriv && !g.perm.Private ){
168 /* Do not accept private files if not authorized */
169 blob_reset(&content);
170 return;
171 }
172 if( cloneFlag ){
173 if( pXfer->nToken==4 ){
174 srcid = rid_from_uuid(&pXfer->aToken[2], 1, isPriv);
175 pXfer->nDeltaRcvd++;
176 }else{
177 srcid = 0;
178 pXfer->nFileRcvd++;
179 }
180 rid = content_put_ex(&content, blob_str(pUuid), srcid,
181 0, isPriv);
182 Th_AppendToList(pzUuidList, pnUuidList, blob_str(pUuid),
183 blob_size(pUuid));
184 remote_has(rid);
185 blob_reset(&content);
186 return;
187 }
188 if( pXfer->nToken==4 ){
189 Blob src, next;
190 srcid = rid_from_uuid(&pXfer->aToken[2], 1, isPriv);
191 if( content_get(srcid, &src)==0 ){
192 rid = content_put_ex(&content, blob_str(pUuid), srcid,
193 0, isPriv);
194 Th_AppendToList(pzUuidList, pnUuidList, blob_str(pUuid),
195 blob_size(pUuid));
196 pXfer->nDanglingFile++;
197 db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
198 if( !isPriv ) content_make_public(rid);
199 blob_reset(&src);
200 blob_reset(&content);
201 return;
202 }
203 pXfer->nDeltaRcvd++;
204 blob_delta_apply(&src, &content, &next);
205 blob_reset(&src);
206 blob_reset(&content);
207 content = next;
208 }else{
209 pXfer->nFileRcvd++;
210 }
211 if( hname_verify_hash(&content, blob_buffer(pUuid), blob_size(pUuid))==0 ){
212 blob_appendf(&pXfer->err, "wrong hash on received artifact: %b", pUuid);
213 }
214 rid = content_put_ex(&content, blob_str(pUuid), 0, 0, isPriv);
215 Th_AppendToList(pzUuidList, pnUuidList, blob_str(pUuid), blob_size(pUuid));
216 if( rid==0 ){
217 blob_appendf(&pXfer->err, "%s", g.zErrMsg);
218 blob_reset(&content);
219 }else{
220 if( !isPriv ) content_make_public(rid);
221 manifest_crosslink(rid, &content, MC_NO_ERRORS);
222 }
223 assert( blob_is_reset(&content) );
224 remote_has(rid);
225 }
226
227 /*
228 ** The aToken[0..nToken-1] blob array is a parse of a "cfile" line
229 ** message. This routine finishes parsing that message and does
230 ** a record insert of the file. The difference between "file" and
231 ** "cfile" is that with "cfile" the content is already compressed.
232 **
233 ** The file line is in one of the following two forms:
234 **
235 ** cfile HASH USIZE CSIZE \n CONTENT
236 ** cfile HASH DELTASRC USIZE CSIZE \n CONTENT
237 **
238 ** The content is CSIZE bytes immediately following the newline.
239 ** If DELTASRC exists, then the CONTENT is a delta against the
240 ** content of DELTASRC.
241 **
242 ** The original size of the HASH artifact is USIZE.
243 **
244 ** If any error occurs, write a message into pErr which has already
245 ** be initialized to an empty string.
246 **
247 ** Any artifact successfully received by this routine is considered to
248 ** be public and is therefore removed from the "private" table.
249 */
xfer_accept_compressed_file(Xfer * pXfer,char ** pzUuidList,int * pnUuidList)250 static void xfer_accept_compressed_file(
251 Xfer *pXfer,
252 char **pzUuidList,
253 int *pnUuidList
254 ){
255 int szC; /* CSIZE */
256 int szU; /* USIZE */
257 int rid;
258 int srcid = 0;
259 Blob content;
260 int isPriv;
261
262 isPriv = pXfer->nextIsPrivate;
263 pXfer->nextIsPrivate = 0;
264 if( pXfer->nToken<4
265 || pXfer->nToken>5
266 || !blob_is_hname(&pXfer->aToken[1])
267 || !blob_is_int(&pXfer->aToken[pXfer->nToken-2], &szU)
268 || !blob_is_int(&pXfer->aToken[pXfer->nToken-1], &szC)
269 || szC<0 || szU<0
270 || (pXfer->nToken==5 && !blob_is_hname(&pXfer->aToken[2]))
271 ){
272 blob_appendf(&pXfer->err, "malformed cfile line");
273 return;
274 }
275 if( isPriv && !g.perm.Private ){
276 /* Do not accept private files if not authorized */
277 return;
278 }
279 blob_zero(&content);
280 blob_extract(pXfer->pIn, szC, &content);
281 if( uuid_is_shunned(blob_str(&pXfer->aToken[1])) ){
282 /* Ignore files that have been shunned */
283 blob_reset(&content);
284 return;
285 }
286 if( pXfer->nToken==5 ){
287 srcid = rid_from_uuid(&pXfer->aToken[2], 1, isPriv);
288 pXfer->nDeltaRcvd++;
289 }else{
290 srcid = 0;
291 pXfer->nFileRcvd++;
292 }
293 rid = content_put_ex(&content, blob_str(&pXfer->aToken[1]), srcid,
294 szC, isPriv);
295 Th_AppendToList(pzUuidList, pnUuidList, blob_str(&pXfer->aToken[1]),
296 blob_size(&pXfer->aToken[1]));
297 remote_has(rid);
298 blob_reset(&content);
299 }
300
301 /*
302 ** The aToken[0..nToken-1] blob array is a parse of a "uvfile" line
303 ** message. This routine finishes parsing that message and adds the
304 ** unversioned file to the "unversioned" table.
305 **
306 ** The file line is in one of the following two forms:
307 **
308 ** uvfile NAME MTIME HASH SIZE FLAGS
309 ** uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
310 **
311 ** If the 0x0001 bit of FLAGS is set, that means the file has been
312 ** deleted, SIZE is zero, the HASH is "-", and the "\n CONTENT" is omitted.
313 **
314 ** SIZE is the number of bytes of CONTENT. The CONTENT is uncompressed.
315 ** HASH is the artifact hash of CONTENT.
316 **
317 ** If the 0x0004 bit of FLAGS is set, that means the CONTENT is omitted.
318 ** The sender might have omitted the content because it is too big to
319 ** transmit, or because it is unchanged and this record exists purely
320 ** to update the MTIME.
321 */
xfer_accept_unversioned_file(Xfer * pXfer,int isWriter)322 static void xfer_accept_unversioned_file(Xfer *pXfer, int isWriter){
323 sqlite3_int64 mtime; /* The MTIME */
324 Blob *pHash; /* The HASH value */
325 int sz; /* The SIZE */
326 int flags; /* The FLAGS */
327 Blob content; /* The CONTENT */
328 Blob x; /* Compressed content */
329 Stmt q; /* SQL statements for comparison and insert */
330 int isDelete; /* HASH is "-" indicating this is a delete */
331 int nullContent; /* True of CONTENT is NULL */
332 int iStatus; /* Result from unversioned_status() */
333
334 pHash = &pXfer->aToken[3];
335 if( pXfer->nToken==5
336 || !blob_is_filename(&pXfer->aToken[1])
337 || !blob_is_int64(&pXfer->aToken[2], &mtime)
338 || (!blob_eq(pHash,"-") && !blob_is_hname(pHash))
339 || !blob_is_int(&pXfer->aToken[4], &sz)
340 || !blob_is_int(&pXfer->aToken[5], &flags)
341 ){
342 blob_appendf(&pXfer->err, "malformed uvfile line");
343 return;
344 }
345 blob_init(&content, 0, 0);
346 blob_init(&x, 0, 0);
347 if( sz>0 && (flags & 0x0005)==0 ){
348 blob_extract(pXfer->pIn, sz, &content);
349 nullContent = 0;
350 if( hname_verify_hash(&content, blob_buffer(pHash), blob_size(pHash))==0 ){
351 blob_appendf(&pXfer->err, "in uvfile line, HASH does not match CONTENT");
352 goto end_accept_unversioned_file;
353 }
354 }else{
355 nullContent = 1;
356 }
357
358 /* The isWriter flag must be true in order to land the new file */
359 if( !isWriter ) goto end_accept_unversioned_file;
360
361 /* Make sure we have a valid g.rcvid marker */
362 content_rcvid_init(0);
363
364 /* Check to see if current content really should be overwritten. Ideally,
365 ** a uvfile card should never have been sent unless the overwrite should
366 ** occur. But do not trust the sender. Double-check.
367 */
368 iStatus = unversioned_status(blob_str(&pXfer->aToken[1]), mtime,
369 blob_str(pHash));
370 if( iStatus>=3 ) goto end_accept_unversioned_file;
371
372 /* Store the content */
373 isDelete = blob_eq(pHash, "-");
374 if( isDelete ){
375 db_prepare(&q,
376 "UPDATE unversioned"
377 " SET rcvid=:rcvid, mtime=:mtime, hash=NULL,"
378 " sz=0, encoding=0, content=NULL"
379 " WHERE name=:name"
380 );
381 db_bind_int(&q, ":rcvid", g.rcvid);
382 }else if( iStatus==2 ){
383 db_prepare(&q, "UPDATE unversioned SET mtime=:mtime WHERE name=:name");
384 }else{
385 db_prepare(&q,
386 "REPLACE INTO unversioned(name,rcvid,mtime,hash,sz,encoding,content)"
387 " VALUES(:name,:rcvid,:mtime,:hash,:sz,:encoding,:content)"
388 );
389 db_bind_int(&q, ":rcvid", g.rcvid);
390 db_bind_text(&q, ":hash", blob_str(pHash));
391 db_bind_int(&q, ":sz", blob_size(&content));
392 if( !nullContent ){
393 blob_compress(&content, &x);
394 if( blob_size(&x) < 0.8*blob_size(&content) ){
395 db_bind_blob(&q, ":content", &x);
396 db_bind_int(&q, ":encoding", 1);
397 }else{
398 db_bind_blob(&q, ":content", &content);
399 db_bind_int(&q, ":encoding", 0);
400 }
401 }else{
402 db_bind_int(&q, ":encoding", 0);
403 }
404 }
405 db_bind_text(&q, ":name", blob_str(&pXfer->aToken[1]));
406 db_bind_int64(&q, ":mtime", mtime);
407 db_step(&q);
408 db_finalize(&q);
409 db_unset("uv-hash", 0);
410
411 end_accept_unversioned_file:
412 blob_reset(&x);
413 blob_reset(&content);
414 }
415
416 /*
417 ** Try to send a file as a delta against its parent.
418 ** If successful, return the number of bytes in the delta.
419 ** If we cannot generate an appropriate delta, then send
420 ** nothing and return zero.
421 **
422 ** Never send a delta against a private artifact.
423 */
send_delta_parent(Xfer * pXfer,int rid,int isPrivate,Blob * pContent,Blob * pUuid)424 static int send_delta_parent(
425 Xfer *pXfer, /* The transfer context */
426 int rid, /* record id of the file to send */
427 int isPrivate, /* True if rid is a private artifact */
428 Blob *pContent, /* The content of the file to send */
429 Blob *pUuid /* The HASH of the file to send */
430 ){
431 static const char *const azQuery[] = {
432 "SELECT pid FROM plink x"
433 " WHERE cid=%d"
434 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)",
435
436 "SELECT pid, min(mtime) FROM mlink, event ON mlink.mid=event.objid"
437 " WHERE fid=%d"
438 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
439 };
440 int i;
441 Blob src, delta;
442 int size = 0;
443 int srcId = 0;
444
445 for(i=0; srcId==0 && i<count(azQuery); i++){
446 srcId = db_int(0, azQuery[i] /*works-like:"%d"*/, rid);
447 }
448 if( srcId>0
449 && (pXfer->syncPrivate || !content_is_private(srcId))
450 && content_get(srcId, &src)
451 ){
452 char *zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", srcId);
453 blob_delta_create(&src, pContent, &delta);
454 size = blob_size(&delta);
455 if( size>=blob_size(pContent)-50 ){
456 size = 0;
457 }else if( uuid_is_shunned(zUuid) ){
458 size = 0;
459 }else{
460 if( isPrivate ) blob_append(pXfer->pOut, "private\n", -1);
461 blob_appendf(pXfer->pOut, "file %b %s %d\n", pUuid, zUuid, size);
462 blob_append(pXfer->pOut, blob_buffer(&delta), size);
463 }
464 blob_reset(&delta);
465 free(zUuid);
466 blob_reset(&src);
467 }
468 return size;
469 }
470
471 /*
472 ** Try to send a file as a native delta.
473 ** If successful, return the number of bytes in the delta.
474 ** If we cannot generate an appropriate delta, then send
475 ** nothing and return zero.
476 **
477 ** Never send a delta against a private artifact.
478 */
send_delta_native(Xfer * pXfer,int rid,int isPrivate,Blob * pUuid)479 static int send_delta_native(
480 Xfer *pXfer, /* The transfer context */
481 int rid, /* record id of the file to send */
482 int isPrivate, /* True if rid is a private artifact */
483 Blob *pUuid /* The HASH of the file to send */
484 ){
485 Blob src, delta;
486 int size = 0;
487 int srcId;
488
489 srcId = db_int(0, "SELECT srcid FROM delta WHERE rid=%d", rid);
490 if( srcId>0
491 && (pXfer->syncPrivate || !content_is_private(srcId))
492 ){
493 blob_zero(&src);
494 db_blob(&src, "SELECT uuid FROM blob WHERE rid=%d", srcId);
495 if( uuid_is_shunned(blob_str(&src)) ){
496 blob_reset(&src);
497 return 0;
498 }
499 blob_zero(&delta);
500 db_blob(&delta, "SELECT content FROM blob WHERE rid=%d", rid);
501 blob_uncompress(&delta, &delta);
502 if( isPrivate ) blob_append(pXfer->pOut, "private\n", -1);
503 blob_appendf(pXfer->pOut, "file %b %b %d\n",
504 pUuid, &src, blob_size(&delta));
505 blob_append(pXfer->pOut, blob_buffer(&delta), blob_size(&delta));
506 size = blob_size(&delta);
507 blob_reset(&delta);
508 blob_reset(&src);
509 }else{
510 size = 0;
511 }
512 return size;
513 }
514
515 /*
516 ** Push an error message to alert the older client that the repository
517 ** has SHA3 content and cannot be synced or cloned.
518 */
xfer_cannot_send_sha3_error(Xfer * pXfer)519 static void xfer_cannot_send_sha3_error(Xfer *pXfer){
520 blob_appendf(pXfer->pOut,
521 "error Fossil\\sversion\\s2.0\\sor\\slater\\srequired.\n"
522 );
523 }
524
525
526 /*
527 ** Send the file identified by rid.
528 **
529 ** The pUuid can be NULL in which case the correct hash is computed
530 ** from the rid.
531 **
532 ** Try to send the file as a native delta if nativeDelta is true, or
533 ** as a parent delta if nativeDelta is false.
534 **
535 ** It should never be the case that rid is a private artifact. But
536 ** as a precaution, this routine does check on rid and if it is private
537 ** this routine becomes a no-op.
538 */
send_file(Xfer * pXfer,int rid,Blob * pUuid,int nativeDelta)539 static void send_file(Xfer *pXfer, int rid, Blob *pUuid, int nativeDelta){
540 Blob content, uuid;
541 int size = 0;
542 int isPriv = content_is_private(rid);
543
544 if( isPriv && pXfer->syncPrivate==0 ){
545 if( pXfer->remoteDate>=20200413 && pUuid && blob_size(pUuid)>0 ){
546 /* If the artifact is private and we are not doing a private sync,
547 ** at least tell the other side that the artifact exists and is
548 ** known to be private. But only do this for newer clients since
549 ** older ones will throw an error if they get a private igot card
550 ** and private syncing is disallowed */
551 blob_appendf(pXfer->pOut, "igot %b 1\n", pUuid);
552 pXfer->nIGotSent++;
553 pXfer->nPrivIGot++;
554 }
555 return;
556 }
557 if( db_exists("SELECT 1 FROM onremote WHERE rid=%d", rid) ){
558 return;
559 }
560 blob_zero(&uuid);
561 db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d AND size>=0", rid);
562 if( blob_size(&uuid)==0 ){
563 return;
564 }
565 if( blob_size(&uuid)>HNAME_LEN_SHA1 && pXfer->remoteVersion<20000 ){
566 xfer_cannot_send_sha3_error(pXfer);
567 return;
568 }
569 if( pUuid ){
570 if( blob_compare(pUuid, &uuid)!=0 ){
571 blob_reset(&uuid);
572 return;
573 }
574 }else{
575 pUuid = &uuid;
576 }
577 if( uuid_is_shunned(blob_str(pUuid)) ){
578 blob_reset(&uuid);
579 return;
580 }
581 if( (pXfer->maxTime != -1 && time(NULL) >= pXfer->maxTime) ||
582 pXfer->mxSend<=blob_size(pXfer->pOut) ){
583 const char *zFormat = isPriv ? "igot %b 1\n" : "igot %b\n";
584 blob_appendf(pXfer->pOut, zFormat /*works-like:"%b"*/, pUuid);
585 pXfer->nIGotSent++;
586 blob_reset(&uuid);
587 return;
588 }
589 if( nativeDelta ){
590 size = send_delta_native(pXfer, rid, isPriv, pUuid);
591 if( size ){
592 pXfer->nDeltaSent++;
593 }
594 }
595 if( size==0 ){
596 content_get(rid, &content);
597
598 if( !nativeDelta && blob_size(&content)>100 ){
599 size = send_delta_parent(pXfer, rid, isPriv, &content, pUuid);
600 }
601 if( size==0 ){
602 int size = blob_size(&content);
603 if( isPriv ) blob_append(pXfer->pOut, "private\n", -1);
604 blob_appendf(pXfer->pOut, "file %b %d\n", pUuid, size);
605 blob_append(pXfer->pOut, blob_buffer(&content), size);
606 pXfer->nFileSent++;
607 }else{
608 pXfer->nDeltaSent++;
609 }
610 blob_reset(&content);
611 }
612 remote_has(rid);
613 blob_reset(&uuid);
614 #if 0
615 if( blob_buffer(pXfer->pOut)[blob_size(pXfer->pOut)-1]!='\n' ){
616 blob_append(pXfer->pOut, "\n", 1);
617 }
618 #endif
619 }
620
621 /*
622 ** Send the file identified by rid as a compressed artifact. Basically,
623 ** send the content exactly as it appears in the BLOB table using
624 ** a "cfile" card.
625 */
send_compressed_file(Xfer * pXfer,int rid)626 static void send_compressed_file(Xfer *pXfer, int rid){
627 const char *zContent;
628 const char *zUuid;
629 const char *zDelta;
630 int szU;
631 int szC;
632 int rc;
633 int isPrivate;
634 int srcIsPrivate;
635 static Stmt q1;
636 Blob fullContent;
637
638 isPrivate = content_is_private(rid);
639 if( isPrivate && pXfer->syncPrivate==0 ) return;
640 db_static_prepare(&q1,
641 "SELECT uuid, size, content, delta.srcid IN private,"
642 " (SELECT uuid FROM blob WHERE rid=delta.srcid)"
643 " FROM blob LEFT JOIN delta ON (blob.rid=delta.rid)"
644 " WHERE blob.rid=:rid"
645 " AND blob.size>=0"
646 " AND NOT EXISTS(SELECT 1 FROM shun WHERE shun.uuid=blob.uuid)"
647 );
648 db_bind_int(&q1, ":rid", rid);
649 rc = db_step(&q1);
650 if( rc==SQLITE_ROW ){
651 zUuid = db_column_text(&q1, 0);
652 szU = db_column_int(&q1, 1);
653 szC = db_column_bytes(&q1, 2);
654 zContent = db_column_raw(&q1, 2);
655 srcIsPrivate = db_column_int(&q1, 3);
656 zDelta = db_column_text(&q1, 4);
657 if( isPrivate ) blob_append(pXfer->pOut, "private\n", -1);
658 if( pXfer->remoteVersion<20000 && db_column_bytes(&q1,0)!=HNAME_LEN_SHA1 ){
659 xfer_cannot_send_sha3_error(pXfer);
660 db_reset(&q1);
661 return;
662 }
663 blob_appendf(pXfer->pOut, "cfile %s ", zUuid);
664 if( !isPrivate && srcIsPrivate ){
665 content_get(rid, &fullContent);
666 szU = blob_size(&fullContent);
667 blob_compress(&fullContent, &fullContent);
668 szC = blob_size(&fullContent);
669 zContent = blob_buffer(&fullContent);
670 zDelta = 0;
671 }
672 if( zDelta ){
673 blob_appendf(pXfer->pOut, "%s ", zDelta);
674 pXfer->nDeltaSent++;
675 }else{
676 pXfer->nFileSent++;
677 }
678 blob_appendf(pXfer->pOut, "%d %d\n", szU, szC);
679 blob_append(pXfer->pOut, zContent, szC);
680 if( blob_buffer(pXfer->pOut)[blob_size(pXfer->pOut)-1]!='\n' ){
681 blob_append(pXfer->pOut, "\n", 1);
682 }
683 if( !isPrivate && srcIsPrivate ){
684 blob_reset(&fullContent);
685 }
686 }
687 db_reset(&q1);
688 }
689
690 /*
691 ** Send the unversioned file identified by zName by generating the
692 ** appropriate "uvfile" card.
693 **
694 ** uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
695 **
696 ** If the noContent flag is set, omit the CONTENT and set the 0x0004
697 ** flag in FLAGS.
698 */
send_unversioned_file(Xfer * pXfer,const char * zName,int noContent)699 static void send_unversioned_file(
700 Xfer *pXfer, /* Transfer context */
701 const char *zName, /* Name of unversioned file to be sent */
702 int noContent /* True to omit the content */
703 ){
704 Stmt q1;
705
706 if( blob_size(pXfer->pOut)>=pXfer->mxSend ) noContent = 1;
707 if( noContent ){
708 db_prepare(&q1,
709 "SELECT mtime, hash, encoding, sz FROM unversioned WHERE name=%Q",
710 zName
711 );
712 }else{
713 db_prepare(&q1,
714 "SELECT mtime, hash, encoding, sz, content FROM unversioned"
715 " WHERE name=%Q",
716 zName
717 );
718 }
719 if( db_step(&q1)==SQLITE_ROW ){
720 sqlite3_int64 mtime = db_column_int64(&q1, 0);
721 const char *zHash = db_column_text(&q1, 1);
722 if( pXfer->remoteVersion<20000 && db_column_bytes(&q1,1)>HNAME_LEN_SHA1 ){
723 xfer_cannot_send_sha3_error(pXfer);
724 db_reset(&q1);
725 return;
726 }
727 if( blob_size(pXfer->pOut)>=pXfer->mxSend ){
728 /* If we have already reached the send size limit, send a (short)
729 ** uvigot card rather than a uvfile card. This only happens on the
730 ** server side. The uvigot card will provoke the client to resend
731 ** another uvgimme on the next cycle. */
732 blob_appendf(pXfer->pOut, "uvigot %s %lld %s %d\n",
733 zName, mtime, zHash, db_column_int(&q1,3));
734 }else{
735 blob_appendf(pXfer->pOut, "uvfile %s %lld", zName, mtime);
736 if( zHash==0 ){
737 blob_append(pXfer->pOut, " - 0 1\n", -1);
738 }else if( noContent ){
739 blob_appendf(pXfer->pOut, " %s %d 4\n", zHash, db_column_int(&q1,3));
740 }else{
741 Blob content;
742 blob_init(&content, 0, 0);
743 db_column_blob(&q1, 4, &content);
744 if( db_column_int(&q1, 2) ){
745 blob_uncompress(&content, &content);
746 }
747 blob_appendf(pXfer->pOut, " %s %d 0\n", zHash, blob_size(&content));
748 blob_append(pXfer->pOut, blob_buffer(&content), blob_size(&content));
749 blob_reset(&content);
750 }
751 }
752 }
753 db_finalize(&q1);
754 }
755
756 /*
757 ** Send a gimme message for every phantom.
758 **
759 ** Except: do not request shunned artifacts. And do not request
760 ** private artifacts if we are not doing a private transfer.
761 */
request_phantoms(Xfer * pXfer,int maxReq)762 static void request_phantoms(Xfer *pXfer, int maxReq){
763 Stmt q;
764 db_prepare(&q,
765 "SELECT uuid FROM phantom CROSS JOIN blob USING(rid) /*scan*/"
766 " WHERE NOT EXISTS(SELECT 1 FROM unk WHERE unk.uuid=blob.uuid)"
767 " AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid) %s",
768 (pXfer->syncPrivate ? "" :
769 " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)")
770 );
771 while( db_step(&q)==SQLITE_ROW && maxReq-- > 0 ){
772 const char *zUuid = db_column_text(&q, 0);
773 blob_appendf(pXfer->pOut, "gimme %s\n", zUuid);
774 pXfer->nGimmeSent++;
775 }
776 db_finalize(&q);
777 }
778
779 /*
780 ** Compute an hash on the tail of pMsg. Verify that it matches the
781 ** the hash given in pHash. Return non-zero for an error and 0 on success.
782 **
783 ** The type of hash computed (SHA1, SHA3-256) is determined by
784 ** the length of the input hash in pHash.
785 */
check_tail_hash(Blob * pHash,Blob * pMsg)786 static int check_tail_hash(Blob *pHash, Blob *pMsg){
787 Blob tail;
788 int rc;
789 blob_tail(pMsg, &tail);
790 rc = hname_verify_hash(&tail, blob_buffer(pHash), blob_size(pHash));
791 blob_reset(&tail);
792 return rc==HNAME_ERROR;
793 }
794
795 /*
796 ** Check the signature on an application/x-fossil payload received by
797 ** the HTTP server. The signature is a line of the following form:
798 **
799 ** login LOGIN NONCE SIGNATURE
800 **
801 ** The NONCE is the SHA1 hash of the remainder of the input.
802 ** SIGNATURE is the SHA1 checksum of the NONCE concatenated
803 ** with the users password.
804 **
805 ** The parameters to this routine are ephemeral blobs holding the
806 ** LOGIN, NONCE and SIGNATURE.
807 **
808 ** This routine attempts to locate the user and verify the signature.
809 ** If everything checks out, the USER.CAP column for the USER table
810 ** is consulted to set privileges in the global g variable.
811 **
812 ** If anything fails to check out, no changes are made to privileges.
813 **
814 ** Signature generation on the client side is handled by the
815 ** http_exchange() routine.
816 **
817 ** Return non-zero for a login failure and zero for success.
818 */
check_login(Blob * pLogin,Blob * pNonce,Blob * pSig)819 int check_login(Blob *pLogin, Blob *pNonce, Blob *pSig){
820 Stmt q;
821 int rc = -1;
822 char *zLogin = blob_terminate(pLogin);
823 defossilize(zLogin);
824
825 if( fossil_strcmp(zLogin, "nobody")==0
826 || fossil_strcmp(zLogin,"anonymous")==0
827 ){
828 return 0; /* Anybody is allowed to sync as "nobody" or "anonymous" */
829 }
830 if( fossil_strcmp(P("REMOTE_USER"), zLogin)==0
831 && db_get_boolean("remote_user_ok",0) ){
832 return 0; /* Accept Basic Authorization */
833 }
834 db_prepare(&q,
835 "SELECT pw, cap, uid FROM user"
836 " WHERE login=%Q"
837 " AND login NOT IN ('anonymous','nobody','developer','reader')"
838 " AND length(pw)>0",
839 zLogin
840 );
841 if( db_step(&q)==SQLITE_ROW ){
842 int szPw;
843 Blob pw, combined, hash;
844 blob_zero(&pw);
845 db_ephemeral_blob(&q, 0, &pw);
846 szPw = blob_size(&pw);
847 blob_zero(&combined);
848 blob_copy(&combined, pNonce);
849 blob_append(&combined, blob_buffer(&pw), szPw);
850 sha1sum_blob(&combined, &hash);
851 assert( blob_size(&hash)==40 );
852 rc = blob_constant_time_cmp(&hash, pSig);
853 blob_reset(&hash);
854 blob_reset(&combined);
855 if( rc!=0 && szPw!=40 ){
856 /* If this server stores cleartext passwords and the password did not
857 ** match, then perhaps the client is sending SHA1 passwords. Try
858 ** again with the SHA1 password.
859 */
860 const char *zPw = db_column_text(&q, 0);
861 char *zSecret = sha1_shared_secret(zPw, blob_str(pLogin), 0);
862 blob_zero(&combined);
863 blob_copy(&combined, pNonce);
864 blob_append(&combined, zSecret, -1);
865 free(zSecret);
866 sha1sum_blob(&combined, &hash);
867 rc = blob_constant_time_cmp(&hash, pSig);
868 blob_reset(&hash);
869 blob_reset(&combined);
870 }
871 if( rc==0 ){
872 const char *zCap;
873 zCap = db_column_text(&q, 1);
874 login_set_capabilities(zCap, 0);
875 g.userUid = db_column_int(&q, 2);
876 g.zLogin = mprintf("%b", pLogin);
877 g.zNonce = mprintf("%b", pNonce);
878 }
879 }
880 db_finalize(&q);
881 return rc;
882 }
883
884 /*
885 ** Send the content of all files in the unsent table.
886 **
887 ** This is really just an optimization. If you clear the
888 ** unsent table, all the right files will still get transferred.
889 ** It just might require an extra round trip or two.
890 */
send_unsent(Xfer * pXfer)891 static void send_unsent(Xfer *pXfer){
892 Stmt q;
893 db_prepare(&q, "SELECT rid FROM unsent EXCEPT SELECT rid FROM private");
894 while( db_step(&q)==SQLITE_ROW ){
895 int rid = db_column_int(&q, 0);
896 send_file(pXfer, rid, 0, 0);
897 }
898 db_finalize(&q);
899 db_multi_exec("DELETE FROM unsent");
900 }
901
902 /*
903 ** Check to see if the number of unclustered entries is greater than
904 ** 100 and if it is, form a new cluster. Unclustered phantoms do not
905 ** count toward the 100 total. And phantoms are never added to a new
906 ** cluster.
907 */
create_cluster(void)908 void create_cluster(void){
909 Blob cluster, cksum;
910 Blob deleteWhere;
911 Stmt q;
912 int nUncl;
913 int nRow = 0;
914 int rid;
915
916 #if 0
917 /* We should not ever get any private artifacts in the unclustered table.
918 ** But if we do (because of a bug) now is a good time to delete them. */
919 db_multi_exec(
920 "DELETE FROM unclustered WHERE rid IN (SELECT rid FROM private)"
921 );
922 #endif
923
924 nUncl = db_int(0, "SELECT count(*) FROM unclustered /*scan*/"
925 " WHERE NOT EXISTS(SELECT 1 FROM phantom"
926 " WHERE rid=unclustered.rid)");
927 if( nUncl>=100 ){
928 blob_zero(&cluster);
929 blob_zero(&deleteWhere);
930 db_prepare(&q, "SELECT uuid FROM unclustered, blob"
931 " WHERE NOT EXISTS(SELECT 1 FROM phantom"
932 " WHERE rid=unclustered.rid)"
933 " AND unclustered.rid=blob.rid"
934 " AND NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
935 " ORDER BY 1");
936 while( db_step(&q)==SQLITE_ROW ){
937 blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
938 nRow++;
939 if( nRow>=800 && nUncl>nRow+100 ){
940 md5sum_blob(&cluster, &cksum);
941 blob_appendf(&cluster, "Z %b\n", &cksum);
942 blob_reset(&cksum);
943 rid = content_put(&cluster);
944 manifest_crosslink(rid, &cluster, MC_NONE);
945 blob_reset(&cluster);
946 nUncl -= nRow;
947 nRow = 0;
948 blob_append_sql(&deleteWhere, ",%d", rid);
949 }
950 }
951 db_finalize(&q);
952 db_multi_exec(
953 "DELETE FROM unclustered WHERE rid NOT IN (0 %s)"
954 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=unclustered.rid)",
955 blob_sql_text(&deleteWhere)
956 );
957 blob_reset(&deleteWhere);
958 if( nRow>0 ){
959 md5sum_blob(&cluster, &cksum);
960 blob_appendf(&cluster, "Z %b\n", &cksum);
961 blob_reset(&cksum);
962 rid = content_put(&cluster);
963 manifest_crosslink(rid, &cluster, MC_NONE);
964 blob_reset(&cluster);
965 }
966 }
967 }
968
969 /*
970 ** Send igot messages for every private artifact
971 */
send_private(Xfer * pXfer)972 static int send_private(Xfer *pXfer){
973 int cnt = 0;
974 Stmt q;
975 if( pXfer->syncPrivate ){
976 db_prepare(&q, "SELECT uuid FROM private JOIN blob USING(rid)");
977 while( db_step(&q)==SQLITE_ROW ){
978 blob_appendf(pXfer->pOut, "igot %s 1\n", db_column_text(&q,0));
979 cnt++;
980 }
981 db_finalize(&q);
982 }
983 return cnt;
984 }
985
986 /*
987 ** Send an igot message for every entry in unclustered table.
988 ** Return the number of cards sent.
989 **
990 ** Except:
991 ** * Do not send igot cards for shunned artifacts
992 ** * Do not send igot cards for phantoms
993 ** * Do not send igot cards for private artifacts
994 ** * Do not send igot cards for any artifact that is in the
995 ** ONREMOTE table, if that table exists.
996 **
997 ** If the pXfer->resync flag is set, that means we are doing a "--verily"
998 ** sync and all artifacts that don't meet the restrictions above should
999 ** be sent.
1000 */
send_unclustered(Xfer * pXfer)1001 static int send_unclustered(Xfer *pXfer){
1002 Stmt q;
1003 int cnt = 0;
1004 const char *zExtra;
1005 if( db_table_exists("temp","onremote") ){
1006 zExtra = " AND NOT EXISTS(SELECT 1 FROM onremote WHERE rid=blob.rid)";
1007 }else{
1008 zExtra = "";
1009 }
1010 if( pXfer->resync ){
1011 db_prepare(&q,
1012 "SELECT uuid, rid FROM blob"
1013 " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
1014 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
1015 " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)%s"
1016 " AND blob.rid<=%d"
1017 " ORDER BY blob.rid DESC",
1018 zExtra /*safe-for-%s*/, pXfer->resync
1019 );
1020 }else{
1021 db_prepare(&q,
1022 "SELECT uuid FROM unclustered JOIN blob USING(rid) /*scan*/"
1023 " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
1024 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
1025 " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)%s",
1026 zExtra /*safe-for-%s*/
1027 );
1028 }
1029 while( db_step(&q)==SQLITE_ROW ){
1030 blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
1031 cnt++;
1032 if( pXfer->resync && pXfer->mxSend<blob_size(pXfer->pOut) ){
1033 pXfer->resync = db_column_int(&q, 1)-1;
1034 }
1035 }
1036 db_finalize(&q);
1037 if( cnt==0 ) pXfer->resync = 0;
1038 return cnt;
1039 }
1040
1041 /*
1042 ** Send an igot message for every artifact.
1043 */
send_all(Xfer * pXfer)1044 static void send_all(Xfer *pXfer){
1045 Stmt q;
1046 db_prepare(&q,
1047 "SELECT uuid FROM blob "
1048 " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
1049 " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)"
1050 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
1051 );
1052 while( db_step(&q)==SQLITE_ROW ){
1053 blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
1054 }
1055 db_finalize(&q);
1056 }
1057
1058 /*
1059 ** pXfer is a "pragma uv-hash HASH" card.
1060 **
1061 ** If HASH is different from the unversioned content hash on this server,
1062 ** then send a bunch of uvigot cards, one for each entry unversioned file
1063 ** on this server.
1064 */
send_unversioned_catalog(Xfer * pXfer)1065 static void send_unversioned_catalog(Xfer *pXfer){
1066 int nUvIgot = 0;
1067 Stmt uvq;
1068 unversioned_schema();
1069 db_prepare(&uvq,
1070 "SELECT name, mtime, hash, sz FROM unversioned"
1071 );
1072 while( db_step(&uvq)==SQLITE_ROW ){
1073 const char *zName = db_column_text(&uvq,0);
1074 sqlite3_int64 mtime = db_column_int64(&uvq,1);
1075 const char *zHash = db_column_text(&uvq,2);
1076 int sz = db_column_int(&uvq,3);
1077 nUvIgot++;
1078 if( zHash==0 ){ sz = 0; zHash = "-"; }
1079 blob_appendf(pXfer->pOut, "uvigot %s %lld %s %d\n",
1080 zName, mtime, zHash, sz);
1081 }
1082 db_finalize(&uvq);
1083 }
1084
1085 /*
1086 ** Called when there is an attempt to transfer private content to and
1087 ** from a server without authorization.
1088 */
server_private_xfer_not_authorized(void)1089 static void server_private_xfer_not_authorized(void){
1090 @ error not\sauthorized\sto\ssync\sprivate\scontent
1091 }
1092
1093 /*
1094 ** Return the common TH1 code to evaluate prior to evaluating any other
1095 ** TH1 transfer notification scripts.
1096 */
xfer_common_code(void)1097 const char *xfer_common_code(void){
1098 return db_get("xfer-common-script", 0);
1099 }
1100
1101 /*
1102 ** Return the TH1 code to evaluate when a push is processed.
1103 */
xfer_push_code(void)1104 const char *xfer_push_code(void){
1105 return db_get("xfer-push-script", 0);
1106 }
1107
1108 /*
1109 ** Return the TH1 code to evaluate when a commit is processed.
1110 */
xfer_commit_code(void)1111 const char *xfer_commit_code(void){
1112 return db_get("xfer-commit-script", 0);
1113 }
1114
1115 /*
1116 ** Return the TH1 code to evaluate when a ticket change is processed.
1117 */
xfer_ticket_code(void)1118 const char *xfer_ticket_code(void){
1119 return db_get("xfer-ticket-script", 0);
1120 }
1121
1122 /*
1123 ** Run the specified TH1 script, if any, and returns 1 on error.
1124 */
xfer_run_script(const char * zScript,const char * zUuidOrList,int bIsList)1125 int xfer_run_script(
1126 const char *zScript,
1127 const char *zUuidOrList,
1128 int bIsList
1129 ){
1130 int rc = TH_OK;
1131 if( !zScript ) return rc;
1132 Th_FossilInit(TH_INIT_DEFAULT);
1133 Th_Store(bIsList ? "uuids" : "uuid", zUuidOrList ? zUuidOrList : "");
1134 rc = Th_Eval(g.interp, 0, zScript, -1);
1135 if( rc!=TH_OK ){
1136 fossil_error(1, "%s", Th_GetResult(g.interp, 0));
1137 }
1138 return rc;
1139 }
1140
1141 /*
1142 ** Runs the pre-transfer TH1 script, if any, and returns its return code.
1143 ** This script may be run multiple times. If the script performs actions
1144 ** that cannot be redone, it should use an internal [if] guard similar to
1145 ** the following:
1146 **
1147 ** if {![info exists common_done]} {
1148 ** # ... code here
1149 ** set common_done 1
1150 ** }
1151 */
xfer_run_common_script(void)1152 int xfer_run_common_script(void){
1153 return xfer_run_script(xfer_common_code(), 0, 0);
1154 }
1155
1156 /*
1157 ** If this variable is set, disable login checks. Used for debugging
1158 ** only.
1159 */
1160 static int disableLogin = 0;
1161
1162 /*
1163 ** The CGI/HTTP preprocessor always redirects requests with a content-type
1164 ** of application/x-fossil or application/x-fossil-debug to this page,
1165 ** regardless of what path was specified in the HTTP header. This allows
1166 ** clone clients to specify a URL that omits default pathnames, such
1167 ** as "http://fossil-scm.org/" instead of "http://fossil-scm.org/index.cgi".
1168 **
1169 ** WEBPAGE: xfer raw-content
1170 **
1171 ** This is the transfer handler on the server side. The transfer
1172 ** message has been uncompressed and placed in the g.cgiIn blob.
1173 ** Process this message and form an appropriate reply.
1174 */
page_xfer(void)1175 void page_xfer(void){
1176 int isPull = 0;
1177 int isPush = 0;
1178 int nErr = 0;
1179 Xfer xfer;
1180 int deltaFlag = 0;
1181 int isClone = 0;
1182 int nGimme = 0;
1183 int size;
1184 char *zNow;
1185 int rc;
1186 const char *zScript = 0;
1187 char *zUuidList = 0;
1188 int nUuidList = 0;
1189 char **pzUuidList = 0;
1190 int *pnUuidList = 0;
1191 int uvCatalogSent = 0;
1192
1193 if( fossil_strcmp(PD("REQUEST_METHOD","POST"),"POST") ){
1194 fossil_redirect_home();
1195 }
1196 g.zLogin = "anonymous";
1197 login_set_anon_nobody_capabilities();
1198 login_check_credentials();
1199 memset(&xfer, 0, sizeof(xfer));
1200 blobarray_zero(xfer.aToken, count(xfer.aToken));
1201 cgi_set_content_type(g.zContentType);
1202 cgi_reset_content();
1203 if( db_schema_is_outofdate() ){
1204 @ error database\sschema\sis\sout-of-date\son\sthe\sserver.
1205 return;
1206 }
1207 blob_zero(&xfer.err);
1208 xfer.pIn = &g.cgiIn;
1209 xfer.pOut = cgi_output_blob();
1210 xfer.mxSend = db_get_int("max-download", 5000000);
1211 xfer.maxTime = db_get_int("max-download-time", 30);
1212 if( xfer.maxTime<1 ) xfer.maxTime = 1;
1213 xfer.maxTime += time(NULL);
1214 g.xferPanic = 1;
1215
1216 db_begin_write();
1217 db_multi_exec(
1218 "CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
1219 "CREATE TEMP TABLE unk(uuid TEXT PRIMARY KEY) WITHOUT ROWID;"
1220 );
1221 manifest_crosslink_begin();
1222 rc = xfer_run_common_script();
1223 if( rc==TH_ERROR ){
1224 cgi_reset_content();
1225 @ error common\sscript\sfailed:\s%F(g.zErrMsg)
1226 nErr++;
1227 }
1228 zScript = xfer_push_code();
1229 if( zScript ){ /* NOTE: Are TH1 transfer hooks enabled? */
1230 pzUuidList = &zUuidList;
1231 pnUuidList = &nUuidList;
1232 }
1233 while( blob_line(xfer.pIn, &xfer.line) ){
1234 if( blob_buffer(&xfer.line)[0]=='#' ) continue;
1235 if( blob_size(&xfer.line)==0 ) continue;
1236 xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
1237
1238 /* file HASH SIZE \n CONTENT
1239 ** file HASH DELTASRC SIZE \n CONTENT
1240 **
1241 ** Server accepts a file from the client.
1242 */
1243 if( blob_eq(&xfer.aToken[0], "file") ){
1244 if( !isPush ){
1245 cgi_reset_content();
1246 @ error not\sauthorized\sto\swrite
1247 nErr++;
1248 break;
1249 }
1250 xfer_accept_file(&xfer, 0, pzUuidList, pnUuidList);
1251 if( blob_size(&xfer.err) ){
1252 cgi_reset_content();
1253 @ error %T(blob_str(&xfer.err))
1254 nErr++;
1255 break;
1256 }
1257 }else
1258
1259 /* cfile HASH USIZE CSIZE \n CONTENT
1260 ** cfile HASH DELTASRC USIZE CSIZE \n CONTENT
1261 **
1262 ** Server accepts a compressed file from the client.
1263 */
1264 if( blob_eq(&xfer.aToken[0], "cfile") ){
1265 if( !isPush ){
1266 cgi_reset_content();
1267 @ error not\sauthorized\sto\swrite
1268 nErr++;
1269 break;
1270 }
1271 xfer_accept_compressed_file(&xfer, pzUuidList, pnUuidList);
1272 if( blob_size(&xfer.err) ){
1273 cgi_reset_content();
1274 @ error %T(blob_str(&xfer.err))
1275 nErr++;
1276 break;
1277 }
1278 }else
1279
1280 /* uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
1281 **
1282 ** Server accepts an unversioned file from the client.
1283 */
1284 if( blob_eq(&xfer.aToken[0], "uvfile") ){
1285 xfer_accept_unversioned_file(&xfer, g.perm.WrUnver);
1286 if( blob_size(&xfer.err) ){
1287 cgi_reset_content();
1288 @ error %T(blob_str(&xfer.err))
1289 nErr++;
1290 break;
1291 }
1292 }else
1293
1294 /* gimme HASH
1295 **
1296 ** Client is requesting a file from the server. Send it.
1297 */
1298 if( blob_eq(&xfer.aToken[0], "gimme")
1299 && xfer.nToken==2
1300 && blob_is_hname(&xfer.aToken[1])
1301 ){
1302 nGimme++;
1303 remote_unk(&xfer.aToken[1]);
1304 if( isPull ){
1305 int rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
1306 if( rid ){
1307 send_file(&xfer, rid, &xfer.aToken[1], deltaFlag);
1308 }
1309 }
1310 }else
1311
1312 /* uvgimme NAME
1313 **
1314 ** Client is requesting an unversioned file from the server. Send it.
1315 */
1316 if( blob_eq(&xfer.aToken[0], "uvgimme")
1317 && xfer.nToken==2
1318 && blob_is_filename(&xfer.aToken[1])
1319 ){
1320 send_unversioned_file(&xfer, blob_str(&xfer.aToken[1]), 0);
1321 }else
1322
1323 /* igot HASH ?ISPRIVATE?
1324 **
1325 ** Client announces that it has a particular file. If the ISPRIVATE
1326 ** argument exists and is "1", then the file is a private file.
1327 */
1328 if( xfer.nToken>=2
1329 && blob_eq(&xfer.aToken[0], "igot")
1330 && blob_is_hname(&xfer.aToken[1])
1331 ){
1332 if( isPush ){
1333 int rid = 0;
1334 int isPriv = 0;
1335 if( xfer.nToken==2 || blob_eq(&xfer.aToken[2],"1")==0 ){
1336 /* Client says the artifact is public */
1337 rid = rid_from_uuid(&xfer.aToken[1], 1, 0);
1338 }else if( g.perm.Private ){
1339 /* Client says the artifact is private and the client has
1340 ** permission to push private content. Create a new phantom
1341 ** artifact that is marked private. */
1342 rid = rid_from_uuid(&xfer.aToken[1], 1, 1);
1343 isPriv = 1;
1344 }else{
1345 /* Client says the artifact is private and the client is unable
1346 ** or unwilling to send us the artifact. If we already hold the
1347 ** artifact here on the server as a phantom, make sure that
1348 ** phantom is marked as private so that we don't keep asking about
1349 ** it in subsequent sync requests. */
1350 rid = rid_from_uuid(&xfer.aToken[1], 0, 1);
1351 isPriv = 1;
1352 }
1353 if( rid ){
1354 remote_has(rid);
1355 if( isPriv ){
1356 content_make_private(rid);
1357 }else{
1358 content_make_public(rid);
1359 }
1360 }
1361 }
1362 }else
1363
1364
1365 /* pull SERVERCODE PROJECTCODE
1366 ** push SERVERCODE PROJECTCODE
1367 **
1368 ** The client wants either send or receive. The server should
1369 ** verify that the project code matches. The server code is ignored.
1370 */
1371 if( xfer.nToken==3
1372 && (blob_eq(&xfer.aToken[0], "pull") || blob_eq(&xfer.aToken[0], "push"))
1373 && blob_is_hname(&xfer.aToken[2])
1374 ){
1375 const char *zPCode;
1376 zPCode = db_get("project-code", 0);
1377 if( zPCode==0 ){
1378 fossil_fatal("missing project code");
1379 }
1380 if( !blob_eq_str(&xfer.aToken[2], zPCode, -1) ){
1381 cgi_reset_content();
1382 @ error wrong\sproject
1383 nErr++;
1384 break;
1385 }
1386 login_check_credentials();
1387 if( blob_eq(&xfer.aToken[0], "pull") ){
1388 if( !g.perm.Read ){
1389 cgi_reset_content();
1390 @ error not\sauthorized\sto\sread
1391 nErr++;
1392 break;
1393 }
1394 isPull = 1;
1395 }else{
1396 if( !g.perm.Write ){
1397 if( !isPull ){
1398 cgi_reset_content();
1399 @ error not\sauthorized\sto\swrite
1400 nErr++;
1401 }else{
1402 @ message pull\sonly\s-\snot\sauthorized\sto\spush
1403 }
1404 }else{
1405 isPush = 1;
1406 }
1407 }
1408 }else
1409
1410 /* clone ?PROTOCOL-VERSION? ?SEQUENCE-NUMBER?
1411 **
1412 ** The client knows nothing. Tell all.
1413 */
1414 if( blob_eq(&xfer.aToken[0], "clone") ){
1415 int iVers;
1416 login_check_credentials();
1417 if( !g.perm.Clone ){
1418 cgi_reset_content();
1419 @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
1420 @ error not\sauthorized\sto\sclone
1421 nErr++;
1422 break;
1423 }
1424 if( db_get_boolean("uv-sync",0) && !uvCatalogSent ){
1425 @ pragma uv-pull-only
1426 send_unversioned_catalog(&xfer);
1427 uvCatalogSent = 1;
1428 }
1429 if( xfer.nToken==3
1430 && blob_is_int(&xfer.aToken[1], &iVers)
1431 && iVers>=2
1432 ){
1433 int seqno, max;
1434 if( iVers>=3 ){
1435 cgi_set_content_type("application/x-fossil-uncompressed");
1436 }
1437 blob_is_int(&xfer.aToken[2], &seqno);
1438 max = db_int(0, "SELECT max(rid) FROM blob");
1439 while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max){
1440 if( time(NULL) >= xfer.maxTime ) break;
1441 if( iVers>=3 ){
1442 send_compressed_file(&xfer, seqno);
1443 }else{
1444 send_file(&xfer, seqno, 0, 1);
1445 }
1446 seqno++;
1447 }
1448 if( seqno>max ) seqno = 0;
1449 @ clone_seqno %d(seqno)
1450 }else{
1451 isClone = 1;
1452 isPull = 1;
1453 deltaFlag = 1;
1454 }
1455 @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
1456 }else
1457
1458 /* login USER NONCE SIGNATURE
1459 **
1460 ** The client has sent login credentials to the server.
1461 ** Validate the login. This has to happen before anything else.
1462 ** The client can send multiple logins. Permissions are cumulative.
1463 */
1464 if( blob_eq(&xfer.aToken[0], "login")
1465 && xfer.nToken==4
1466 ){
1467 if( disableLogin ){
1468 g.perm.Read = g.perm.Write = g.perm.Private = g.perm.Admin = 1;
1469 }else{
1470 if( check_tail_hash(&xfer.aToken[2], xfer.pIn)
1471 || check_login(&xfer.aToken[1], &xfer.aToken[2], &xfer.aToken[3])
1472 ){
1473 cgi_reset_content();
1474 @ error login\sfailed
1475 nErr++;
1476 break;
1477 }
1478 }
1479 }else
1480
1481 /* reqconfig NAME
1482 **
1483 ** Client is requesting a configuration value from the server
1484 */
1485 if( blob_eq(&xfer.aToken[0], "reqconfig")
1486 && xfer.nToken==2
1487 ){
1488 if( g.perm.Read ){
1489 char *zName = blob_str(&xfer.aToken[1]);
1490 if( zName[0]=='/' ){
1491 /* New style configuration transfer */
1492 int groupMask = configure_name_to_mask(&zName[1], 0);
1493 if( !g.perm.Admin ) groupMask &= ~(CONFIGSET_USER|CONFIGSET_SCRIBER);
1494 if( !g.perm.RdAddr ) groupMask &= ~CONFIGSET_ADDR;
1495 configure_send_group(xfer.pOut, groupMask, 0);
1496 }
1497 }
1498 }else
1499
1500 /* config NAME SIZE \n CONTENT
1501 **
1502 ** Client has sent a configuration value to the server.
1503 ** This is only permitted for high-privilege users.
1504 */
1505 if( blob_eq(&xfer.aToken[0],"config") && xfer.nToken==3
1506 && blob_is_int(&xfer.aToken[2], &size) ){
1507 const char *zName = blob_str(&xfer.aToken[1]);
1508 Blob content;
1509 blob_zero(&content);
1510 blob_extract(xfer.pIn, size, &content);
1511 if( !g.perm.Admin ){
1512 cgi_reset_content();
1513 @ error not\sauthorized\sto\spush\sconfiguration
1514 nErr++;
1515 break;
1516 }
1517 configure_receive(zName, &content, CONFIGSET_ALL);
1518 blob_reset(&content);
1519 blob_seek(xfer.pIn, 1, BLOB_SEEK_CUR);
1520 }else
1521
1522
1523 /* cookie TEXT
1524 **
1525 ** A cookie contains a arbitrary-length argument that is server-defined.
1526 ** The argument must be encoded so as not to contain any whitespace.
1527 ** The server can optionally send a cookie to the client. The client
1528 ** might then return the same cookie back to the server on its next
1529 ** communication. The cookie might record information that helps
1530 ** the server optimize a push or pull.
1531 **
1532 ** The client is not required to return a cookie. So the server
1533 ** must not depend on the cookie. The cookie should be an optimization
1534 ** only. The client might also send a cookie that came from a different
1535 ** server. So the server must be prepared to distinguish its own cookie
1536 ** from cookies originating from other servers. The client might send
1537 ** back several different cookies to the server. The server should be
1538 ** prepared to sift through the cookies and pick the one that it wants.
1539 */
1540 if( blob_eq(&xfer.aToken[0], "cookie") && xfer.nToken==2 ){
1541 /* Process the cookie */
1542 }else
1543
1544
1545 /* private
1546 **
1547 ** The client card indicates that the next "file" or "cfile" will contain
1548 ** private content.
1549 */
1550 if( blob_eq(&xfer.aToken[0], "private") ){
1551 if( !g.perm.Private ){
1552 server_private_xfer_not_authorized();
1553 }else{
1554 xfer.nextIsPrivate = 1;
1555 }
1556 }else
1557
1558
1559 /* pragma NAME VALUE...
1560 **
1561 ** The client issue pragmas to try to influence the behavior of the
1562 ** server. These are requests only. Unknown pragmas are silently
1563 ** ignored.
1564 */
1565 if( blob_eq(&xfer.aToken[0], "pragma") && xfer.nToken>=2 ){
1566
1567 /* pragma send-private
1568 **
1569 ** The client is requesting private artifacts.
1570 **
1571 ** If the user has the "x" privilege (which must be set explicitly -
1572 ** it is not automatic with "a" or "s") then this pragma causes
1573 ** private information to be pulled in addition to public records.
1574 */
1575 if( blob_eq(&xfer.aToken[1], "send-private") ){
1576 login_check_credentials();
1577 if( !g.perm.Private ){
1578 server_private_xfer_not_authorized();
1579 }else{
1580 xfer.syncPrivate = 1;
1581 }
1582 }
1583
1584 /* pragma send-catalog
1585 **
1586 ** The client wants to see igot cards for all known artifacts.
1587 ** This is used as part of "sync --verily" to help ensure that
1588 ** no artifacts have been missed on prior syncs.
1589 */
1590 if( blob_eq(&xfer.aToken[1], "send-catalog") ){
1591 xfer.resync = 0x7fffffff;
1592 }
1593
1594 /* pragma client-version VERSION ?DATE? ?TIME?
1595 **
1596 ** The client announces to the server what version of Fossil it
1597 ** is running. The DATE and TIME are a pure numeric ISO8601 time
1598 ** for the specific check-in of the client.
1599 */
1600 if( xfer.nToken>=3 && blob_eq(&xfer.aToken[1], "client-version") ){
1601 xfer.remoteVersion = atoi(blob_str(&xfer.aToken[2]));
1602 if( xfer.nToken>=5 ){
1603 xfer.remoteDate = atoi(blob_str(&xfer.aToken[3]));
1604 xfer.remoteTime = atoi(blob_str(&xfer.aToken[4]));
1605 @ pragma server-version %d(RELEASE_VERSION_NUMBER) \
1606 @ %d(MANIFEST_NUMERIC_DATE) %d(MANIFEST_NUMERIC_TIME)
1607 }
1608 }
1609
1610 /* pragma uv-hash HASH
1611 **
1612 ** The client wants to make sure that unversioned files are all synced.
1613 ** If the HASH does not match, send a complete catalog of
1614 ** "uvigot" cards.
1615 */
1616 if( blob_eq(&xfer.aToken[1], "uv-hash")
1617 && blob_is_hname(&xfer.aToken[2])
1618 ){
1619 if( !uvCatalogSent
1620 && g.perm.Read
1621 && !blob_eq_str(&xfer.aToken[2], unversioned_content_hash(0),-1)
1622 ){
1623 if( g.perm.WrUnver ){
1624 @ pragma uv-push-ok
1625 }else if( g.perm.Read ){
1626 @ pragma uv-pull-only
1627 }
1628 send_unversioned_catalog(&xfer);
1629 }
1630 uvCatalogSent = 1;
1631 }
1632
1633 /* pragma ci-lock CHECKIN-HASH CLIENT-ID
1634 **
1635 ** The client wants to make non-branch commit against the check-in
1636 ** identified by CHECKIN-HASH. The server will remember this and
1637 ** subsequent ci-lock requests from different clients will generate
1638 ** a ci-lock-fail pragma in the reply.
1639 */
1640 if( blob_eq(&xfer.aToken[1], "ci-lock")
1641 && xfer.nToken==4
1642 && blob_is_hname(&xfer.aToken[2])
1643 ){
1644 Stmt q;
1645 sqlite3_int64 iNow = time(0);
1646 sqlite3_int64 maxAge = db_get_int("lock-timeout",60);
1647 int seenFault = 0;
1648 db_prepare(&q,
1649 "SELECT json_extract(value,'$.login'),"
1650 " mtime,"
1651 " json_extract(value,'$.clientid'),"
1652 " (SELECT rid FROM blob WHERE uuid=substr(name,9)),"
1653 " name"
1654 " FROM config WHERE name GLOB 'ci-lock-*'"
1655 );
1656 while( db_step(&q)==SQLITE_ROW ){
1657 int x = db_column_int(&q,3);
1658 const char *zName = db_column_text(&q,4);
1659 if( db_column_int64(&q,1)<=iNow-maxAge || !is_a_leaf(x) ){
1660 /* check-in locks expire after maxAge seconds, or when the
1661 ** check-in is no longer a leaf */
1662 db_unprotect(PROTECT_CONFIG);
1663 db_multi_exec("DELETE FROM config WHERE name=%Q", zName);
1664 db_protect_pop();
1665 continue;
1666 }
1667 if( fossil_strcmp(zName+8, blob_str(&xfer.aToken[2]))==0 ){
1668 const char *zClientId = db_column_text(&q, 2);
1669 const char *zLogin = db_column_text(&q,0);
1670 sqlite3_int64 mtime = db_column_int64(&q, 1);
1671 if( fossil_strcmp(zClientId, blob_str(&xfer.aToken[3]))!=0 ){
1672 @ pragma ci-lock-fail %F(zLogin) %lld(mtime)
1673 }
1674 seenFault = 1;
1675 }
1676 }
1677 db_finalize(&q);
1678 if( !seenFault ){
1679 db_unprotect(PROTECT_CONFIG);
1680 db_multi_exec(
1681 "REPLACE INTO config(name,value,mtime)"
1682 "VALUES('ci-lock-%q',json_object('login',%Q,'clientid',%Q),now())",
1683 blob_str(&xfer.aToken[2]), g.zLogin,
1684 blob_str(&xfer.aToken[3])
1685 );
1686 db_protect_pop();
1687 }
1688 if( db_get_boolean("forbid-delta-manifests",0) ){
1689 @ pragma avoid-delta-manifests
1690 }
1691 }
1692
1693 /* pragma ci-unlock CLIENT-ID
1694 **
1695 ** Remove any locks previously held by CLIENT-ID. Clients send this
1696 ** pragma with their own ID whenever they know that they no longer
1697 ** have any commits pending.
1698 */
1699 if( blob_eq(&xfer.aToken[1], "ci-unlock")
1700 && xfer.nToken==3
1701 && blob_is_hname(&xfer.aToken[2])
1702 ){
1703 db_unprotect(PROTECT_CONFIG);
1704 db_multi_exec(
1705 "DELETE FROM config"
1706 " WHERE name GLOB 'ci-lock-*'"
1707 " AND json_extract(value,'$.clientid')=%Q",
1708 blob_str(&xfer.aToken[2])
1709 );
1710 db_protect_pop();
1711 }
1712
1713 }else
1714
1715 /* Unknown message
1716 */
1717 {
1718 cgi_reset_content();
1719 @ error bad\scommand:\s%F(blob_str(&xfer.line))
1720 }
1721 blobarray_reset(xfer.aToken, xfer.nToken);
1722 blob_reset(&xfer.line);
1723 }
1724 if( isPush ){
1725 if( rc==TH_OK ){
1726 rc = xfer_run_script(zScript, zUuidList, 1);
1727 if( rc==TH_ERROR ){
1728 cgi_reset_content();
1729 @ error push\sscript\sfailed:\s%F(g.zErrMsg)
1730 nErr++;
1731 }
1732 }
1733 request_phantoms(&xfer, 500);
1734 }
1735 if( zUuidList ){
1736 Th_Free(g.interp, zUuidList);
1737 }
1738 if( isClone && nGimme==0 ){
1739 /* The initial "clone" message from client to server contains no
1740 ** "gimme" cards. On that initial message, send the client an "igot"
1741 ** card for every artifact currently in the repository. This will
1742 ** cause the client to create phantoms for all artifacts, which will
1743 ** in turn make sure that the entire repository is sent efficiently
1744 ** and expeditiously.
1745 */
1746 send_all(&xfer);
1747 if( xfer.syncPrivate ) send_private(&xfer);
1748 }else if( isPull ){
1749 create_cluster();
1750 send_unclustered(&xfer);
1751 if( xfer.syncPrivate ) send_private(&xfer);
1752 }
1753 hook_expecting_more_artifacts(xfer.nGimmeSent?60:0);
1754 db_multi_exec("DROP TABLE onremote; DROP TABLE unk;");
1755 manifest_crosslink_end(MC_PERMIT_HOOKS);
1756
1757 /* Send the server timestamp last, in case prior processing happened
1758 ** to use up a significant fraction of our time window.
1759 */
1760 zNow = db_text(0, "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S', 'now')");
1761 @ # timestamp %s(zNow)
1762 free(zNow);
1763
1764 db_commit_transaction();
1765 configure_rebuild();
1766 }
1767
1768 /*
1769 ** COMMAND: test-xfer
1770 **
1771 ** This command is used for debugging the server. There is a single
1772 ** argument which is the uncompressed content of an "xfer" message
1773 ** from client to server. This command interprets that message as
1774 ** if had been received by the server.
1775 **
1776 ** On the client side, run:
1777 **
1778 ** fossil push http://bogus/ --httptrace
1779 **
1780 ** Or a similar command to provide the output. The content of the
1781 ** message will appear on standard output. Capture this message
1782 ** into a file named (for example) out.txt. Then run the
1783 ** server in gdb:
1784 **
1785 ** gdb fossil
1786 ** r test-xfer out.txt
1787 */
cmd_test_xfer(void)1788 void cmd_test_xfer(void){
1789 db_find_and_open_repository(0,0);
1790 if( g.argc!=2 && g.argc!=3 ){
1791 usage("?MESSAGEFILE?");
1792 }
1793 blob_zero(&g.cgiIn);
1794 blob_read_from_file(&g.cgiIn, g.argc==2 ? "-" : g.argv[2], ExtFILE);
1795 disableLogin = 1;
1796 page_xfer();
1797 fossil_print("%s\n", cgi_extract_content());
1798 }
1799
1800 /*
1801 ** Format strings for progress reporting.
1802 */
1803 static const char zLabelFormat[] = "%-10s %10s %10s %10s %10s\n";
1804 static const char zValueFormat[] = "\r%-10s %10d %10d %10d %10d\n";
1805 static const char zBriefFormat[] =
1806 "Round-trips: %d Artifacts sent: %d received: %d\r";
1807
1808 #if INTERFACE
1809 /*
1810 ** Flag options for controlling client_sync()
1811 */
1812 #define SYNC_PUSH 0x0001 /* push content client to server */
1813 #define SYNC_PULL 0x0002 /* pull content server to client */
1814 #define SYNC_CLONE 0x0004 /* clone the repository */
1815 #define SYNC_PRIVATE 0x0008 /* Also transfer private content */
1816 #define SYNC_VERBOSE 0x0010 /* Extra diagnostics */
1817 #define SYNC_RESYNC 0x0020 /* --verily */
1818 #define SYNC_FROMPARENT 0x0040 /* Pull from the parent project */
1819 #define SYNC_UNVERSIONED 0x0100 /* Sync unversioned content */
1820 #define SYNC_UV_REVERT 0x0200 /* Copy server unversioned to client */
1821 #define SYNC_UV_TRACE 0x0400 /* Describe UV activities */
1822 #define SYNC_UV_DRYRUN 0x0800 /* Do not actually exchange files */
1823 #define SYNC_IFABLE 0x1000 /* Inability to sync is not fatal */
1824 #define SYNC_CKIN_LOCK 0x2000 /* Lock the current check-in */
1825 #define SYNC_NOHTTPCOMPRESS 0x4000 /* Do not compression HTTP messages */
1826 #endif
1827
1828 /*
1829 ** Floating-point absolute value
1830 */
fossil_fabs(double x)1831 static double fossil_fabs(double x){
1832 return x>0.0 ? x : -x;
1833 }
1834
1835 /*
1836 ** Sync to the host identified in g.url.name and g.url.path. This
1837 ** routine is called by the client.
1838 **
1839 ** Records are pushed to the server if pushFlag is true. Records
1840 ** are pulled if pullFlag is true. A full sync occurs if both are
1841 ** true.
1842 */
client_sync(unsigned syncFlags,unsigned configRcvMask,unsigned configSendMask,const char * zAltPCode)1843 int client_sync(
1844 unsigned syncFlags, /* Mask of SYNC_* flags */
1845 unsigned configRcvMask, /* Receive these configuration items */
1846 unsigned configSendMask, /* Send these configuration items */
1847 const char *zAltPCode /* Alternative project code (usually NULL) */
1848 ){
1849 int go = 1; /* Loop until zero */
1850 int nCardSent = 0; /* Number of cards sent */
1851 int nCardRcvd = 0; /* Number of cards received */
1852 int nCycle = 0; /* Number of round trips to the server */
1853 int size; /* Size of a config value or uvfile */
1854 int origConfigRcvMask; /* Original value of configRcvMask */
1855 int nFileRecv; /* Number of files received */
1856 int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
1857 const char *zCookie; /* Server cookie */
1858 i64 nUncSent, nUncRcvd; /* Bytes sent and received (before compression) */
1859 i64 nSent, nRcvd; /* Bytes sent and received (after compression) */
1860 int cloneSeqno = 1; /* Sequence number for clones */
1861 Blob send; /* Text we are sending to the server */
1862 Blob recv; /* Reply we got back from the server */
1863 Xfer xfer; /* Transfer data */
1864 int pctDone; /* Percentage done with a message */
1865 int lastPctDone = -1; /* Last displayed pctDone */
1866 double rArrivalTime; /* Time at which a message arrived */
1867 const char *zSCode = db_get("server-code", "x");
1868 const char *zPCode = db_get("project-code", 0);
1869 int nErr = 0; /* Number of errors */
1870 int nRoundtrip= 0; /* Number of HTTP requests */
1871 int nArtifactSent = 0; /* Total artifacts sent */
1872 int nArtifactRcvd = 0; /* Total artifacts received */
1873 int nPriorArtifact = 0; /* Artifacts received on prior round-trips */
1874 const char *zOpType = 0;/* Push, Pull, Sync, Clone */
1875 double rSkew = 0.0; /* Maximum time skew */
1876 int uvHashSent = 0; /* The "pragma uv-hash" message has been sent */
1877 int uvDoPush = 0; /* Generate uvfile messages to send to server */
1878 int uvPullOnly = 0; /* 1: pull-only. 2: pull-only warning issued */
1879 int nUvGimmeSent = 0; /* Number of uvgimme cards sent on this cycle */
1880 int nUvFileRcvd = 0; /* Number of uvfile cards received on this cycle */
1881 sqlite3_int64 mtime; /* Modification time on a UV file */
1882 int autopushFailed = 0; /* Autopush following commit failed if true */
1883 const char *zCkinLock; /* Name of check-in to lock. NULL for none */
1884 const char *zClientId; /* A unique identifier for this check-out */
1885 unsigned int mHttpFlags;/* Flags for the http_exchange() subsystem */
1886
1887 if( db_get_boolean("dont-push", 0) ) syncFlags &= ~SYNC_PUSH;
1888 if( (syncFlags & (SYNC_PUSH|SYNC_PULL|SYNC_CLONE|SYNC_UNVERSIONED))==0
1889 && configRcvMask==0 && configSendMask==0 ) return 0;
1890 if( syncFlags & SYNC_FROMPARENT ){
1891 configRcvMask = 0;
1892 configSendMask = 0;
1893 syncFlags &= ~(SYNC_PUSH);
1894 zPCode = db_get("parent-project-code", 0);
1895 if( zPCode==0 || db_get("parent-project-name",0)==0 ){
1896 fossil_fatal("there is no parent project: set the 'parent-project-code'"
1897 " and 'parent-project-name' config parameters in order"
1898 " to pull from a parent project");
1899 }
1900 }
1901
1902 transport_stats(0, 0, 1);
1903 socket_global_init();
1904 memset(&xfer, 0, sizeof(xfer));
1905 xfer.pIn = &recv;
1906 xfer.pOut = &send;
1907 xfer.mxSend = db_get_int("max-upload", 250000);
1908 xfer.maxTime = -1;
1909 xfer.remoteVersion = RELEASE_VERSION_NUMBER;
1910 if( syncFlags & SYNC_PRIVATE ){
1911 g.perm.Private = 1;
1912 xfer.syncPrivate = 1;
1913 }
1914
1915 blobarray_zero(xfer.aToken, count(xfer.aToken));
1916 blob_zero(&send);
1917 blob_zero(&recv);
1918 blob_zero(&xfer.err);
1919 blob_zero(&xfer.line);
1920 origConfigRcvMask = 0;
1921 nUncSent = nUncRcvd = 0;
1922
1923 /* Send the send-private pragma if we are trying to sync private data */
1924 if( syncFlags & SYNC_PRIVATE ){
1925 blob_append(&send, "pragma send-private\n", -1);
1926 }
1927
1928 /* Figure out which check-in to lock */
1929 if( syncFlags & SYNC_CKIN_LOCK ){
1930 int vid = db_lget_int("checkout",0);
1931 zCkinLock = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", vid);
1932 }else{
1933 zCkinLock = 0;
1934 }
1935 zClientId = g.localOpen ? db_lget("client-id", 0) : 0;
1936
1937 /* When syncing unversioned files, create a TEMP table in which to store
1938 ** the names of files that need to be sent from client to server.
1939 **
1940 ** The initial assumption is that all unversioned files need to be sent
1941 ** to the other side. But "uvigot" cards received back from the remote
1942 ** side will normally cause many of these entries to be removed since they
1943 ** do not really need to be sent.
1944 */
1945 if( (syncFlags & (SYNC_UNVERSIONED|SYNC_CLONE))!=0 ){
1946 unversioned_schema();
1947 db_multi_exec(
1948 "CREATE TEMP TABLE uv_tosend("
1949 " name TEXT PRIMARY KEY," /* Name of file to send client->server */
1950 " mtimeOnly BOOLEAN" /* True to only send mtime, not content */
1951 ") WITHOUT ROWID;"
1952 "INSERT INTO uv_toSend(name,mtimeOnly)"
1953 " SELECT name, 0 FROM unversioned WHERE hash IS NOT NULL;"
1954 );
1955 }
1956
1957 /*
1958 ** The request from the client always begin with a clone, pull,
1959 ** or push message.
1960 */
1961 blob_appendf(&send, "pragma client-version %d %d %d\n",
1962 RELEASE_VERSION_NUMBER, MANIFEST_NUMERIC_DATE,
1963 MANIFEST_NUMERIC_TIME);
1964 if( syncFlags & SYNC_CLONE ){
1965 blob_appendf(&send, "clone 3 %d\n", cloneSeqno);
1966 syncFlags &= ~(SYNC_PUSH|SYNC_PULL);
1967 nCardSent++;
1968 /* TBD: Request all transferable configuration values */
1969 content_enable_dephantomize(0);
1970 zOpType = "Clone";
1971 }else if( syncFlags & SYNC_PULL ){
1972 blob_appendf(&send, "pull %s %s\n", zSCode,
1973 zAltPCode ? zAltPCode : zPCode);
1974 nCardSent++;
1975 zOpType = (syncFlags & SYNC_PUSH)?"Sync":"Pull";
1976 if( (syncFlags & SYNC_RESYNC)!=0 && nCycle<2 ){
1977 blob_appendf(&send, "pragma send-catalog\n");
1978 nCardSent++;
1979 }
1980 }
1981 if( syncFlags & SYNC_PUSH ){
1982 blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
1983 nCardSent++;
1984 if( (syncFlags & SYNC_PULL)==0 ) zOpType = "Push";
1985 if( (syncFlags & SYNC_RESYNC)!=0 ) xfer.resync = 0x7fffffff;
1986 }
1987 if( syncFlags & SYNC_VERBOSE ){
1988 fossil_print(zLabelFormat /*works-like:"%s%s%s%s%d"*/,
1989 "", "Bytes", "Cards", "Artifacts", "Deltas");
1990 }
1991
1992 while( go ){
1993 int newPhantom = 0;
1994 char *zRandomness;
1995 db_begin_transaction();
1996 db_record_repository_filename(0);
1997 db_multi_exec(
1998 "CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
1999 "CREATE TEMP TABLE unk(uuid TEXT PRIMARY KEY) WITHOUT ROWID;"
2000 );
2001 manifest_crosslink_begin();
2002
2003
2004 /* Client sends the most recently received cookie back to the server.
2005 ** Let the server figure out if this is a cookie that it cares about.
2006 */
2007 zCookie = db_get("cookie", 0);
2008 if( zCookie ){
2009 blob_appendf(&send, "cookie %s\n", zCookie);
2010 }
2011
2012 /* Client sends gimme cards for phantoms
2013 */
2014 if( (syncFlags & SYNC_PULL)!=0
2015 || ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
2016 ){
2017 request_phantoms(&xfer, mxPhantomReq);
2018 }
2019 if( syncFlags & SYNC_PUSH ){
2020 send_unsent(&xfer);
2021 nCardSent += send_unclustered(&xfer);
2022 if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);
2023 }
2024
2025 /* Client sends configuration parameter requests. On a clone, delay sending
2026 ** this until the second cycle since the login card might fail on
2027 ** the first cycle.
2028 */
2029 if( configRcvMask && ((syncFlags & SYNC_CLONE)==0 || nCycle>0) ){
2030 const char *zName;
2031 if( zOpType==0 ) zOpType = "Pull";
2032 zName = configure_first_name(configRcvMask);
2033 while( zName ){
2034 blob_appendf(&send, "reqconfig %s\n", zName);
2035 zName = configure_next_name(configRcvMask);
2036 nCardSent++;
2037 }
2038 origConfigRcvMask = configRcvMask;
2039 configRcvMask = 0;
2040 }
2041
2042 /* Client sends a request to sync unversioned files.
2043 ** On a clone, delay sending this until the second cycle since
2044 ** the login card might fail on the first cycle.
2045 */
2046 if( (syncFlags & SYNC_UNVERSIONED)!=0
2047 && ((syncFlags & SYNC_CLONE)==0 || nCycle>0)
2048 && !uvHashSent
2049 ){
2050 blob_appendf(&send, "pragma uv-hash %s\n", unversioned_content_hash(0));
2051 nCardSent++;
2052 uvHashSent = 1;
2053 }
2054
2055 /* On a "fossil config push", the client send configuration parameters
2056 ** being pushed up to the server */
2057 if( configSendMask ){
2058 if( zOpType==0 ) zOpType = "Push";
2059 nCardSent += configure_send_group(xfer.pOut, configSendMask, 0);
2060 configSendMask = 0;
2061 }
2062
2063 /* Send unversioned files present here on the client but missing or
2064 ** obsolete on the server.
2065 **
2066 ** Or, if the SYNC_UV_REVERT flag is set, delete the local unversioned
2067 ** files that do not exist on the server.
2068 **
2069 ** This happens on the second exchange, since we do not know what files
2070 ** need to be sent until after the uvigot cards from the first exchange
2071 ** have been processed.
2072 */
2073 if( uvDoPush ){
2074 assert( (syncFlags & SYNC_UNVERSIONED)!=0 );
2075 if( syncFlags & SYNC_UV_DRYRUN ){
2076 uvDoPush = 0;
2077 }else if( syncFlags & SYNC_UV_REVERT ){
2078 db_multi_exec(
2079 "DELETE FROM unversioned"
2080 " WHERE name IN (SELECT name FROM uv_tosend);"
2081 "DELETE FROM uv_tosend;"
2082 );
2083 uvDoPush = 0;
2084 }else{
2085 Stmt uvq;
2086 int rc = SQLITE_OK;
2087 db_prepare(&uvq, "SELECT name, mtimeOnly FROM uv_tosend");
2088 while( (rc = db_step(&uvq))==SQLITE_ROW ){
2089 const char *zName = db_column_text(&uvq, 0);
2090 send_unversioned_file(&xfer, zName, db_column_int(&uvq,1));
2091 nCardSent++;
2092 nArtifactSent++;
2093 db_multi_exec("DELETE FROM uv_tosend WHERE name=%Q", zName);
2094 if( syncFlags & SYNC_VERBOSE ){
2095 fossil_print("\rUnversioned-file sent: %s\n", zName);
2096 }
2097 if( blob_size(xfer.pOut)>xfer.mxSend ) break;
2098 }
2099 db_finalize(&uvq);
2100 if( rc==SQLITE_DONE ) uvDoPush = 0;
2101 }
2102 }
2103
2104 /* Lock the current check-out */
2105 if( zCkinLock ){
2106 if( zClientId==0 ){
2107 zClientId = db_text(0, "SELECT lower(hex(randomblob(20)))");
2108 db_lset("client-id", zClientId);
2109 }
2110 blob_appendf(&send, "pragma ci-lock %s %s\n", zCkinLock, zClientId);
2111 zCkinLock = 0;
2112 }else if( zClientId ){
2113 blob_appendf(&send, "pragma ci-unlock %s\n", zClientId);
2114 }
2115
2116 /* Append randomness to the end of the uplink message. This makes all
2117 ** messages unique so that that the login-card nonce will always
2118 ** be unique.
2119 */
2120 zRandomness = db_text(0, "SELECT hex(randomblob(20))");
2121 blob_appendf(&send, "# %s\n", zRandomness);
2122 free(zRandomness);
2123
2124 if( syncFlags & SYNC_VERBOSE ){
2125 fossil_print("waiting for server...");
2126 }
2127 fflush(stdout);
2128 /* Exchange messages with the server */
2129 if( (syncFlags & SYNC_CLONE)!=0 && nCycle==0 ){
2130 /* Do not send a login card on the first round-trip of a clone */
2131 mHttpFlags = 0;
2132 }else{
2133 mHttpFlags = HTTP_USE_LOGIN;
2134 }
2135 if( syncFlags & SYNC_NOHTTPCOMPRESS ){
2136 mHttpFlags |= HTTP_NOCOMPRESS;
2137 }
2138
2139 /* Do the round-trip to the server */
2140 if( http_exchange(&send, &recv, mHttpFlags, MAX_REDIRECTS, 0) ){
2141 nErr++;
2142 go = 2;
2143 break;
2144 }
2145
2146 /* Remember the URL of the sync target in the config file on the
2147 ** first successful round-trip */
2148 if( nCycle==0 && db_is_writeable("repository") ){
2149 db_unprotect(PROTECT_CONFIG);
2150 db_multi_exec("REPLACE INTO config(name,value,mtime)"
2151 "VALUES('syncwith:%q',1,now())", g.url.canonical);
2152 db_protect_pop();
2153 }
2154
2155 /* Output current stats */
2156 if( syncFlags & SYNC_VERBOSE ){
2157 fossil_print(zValueFormat /*works-like:"%s%d%d%d%d"*/, "Sent:",
2158 blob_size(&send), nCardSent+xfer.nGimmeSent+xfer.nIGotSent,
2159 xfer.nFileSent, xfer.nDeltaSent);
2160 }else{
2161 nRoundtrip++;
2162 nArtifactSent += xfer.nFileSent + xfer.nDeltaSent;
2163 fossil_print(zBriefFormat /*works-like:"%d%d%d"*/,
2164 nRoundtrip, nArtifactSent, nArtifactRcvd);
2165 }
2166 nCardSent = 0;
2167 nCardRcvd = 0;
2168 xfer.nFileSent = 0;
2169 xfer.nDeltaSent = 0;
2170 xfer.nGimmeSent = 0;
2171 xfer.nIGotSent = 0;
2172 xfer.nPrivIGot = 0;
2173
2174 lastPctDone = -1;
2175 nUncSent += blob_size(&send);
2176 blob_reset(&send);
2177 blob_appendf(&send, "pragma client-version %d %d %d\n",
2178 RELEASE_VERSION_NUMBER, MANIFEST_NUMERIC_DATE,
2179 MANIFEST_NUMERIC_TIME);
2180 rArrivalTime = db_double(0.0, "SELECT julianday('now')");
2181
2182 /* Send the send-private pragma if we are trying to sync private data */
2183 if( syncFlags & SYNC_PRIVATE ){
2184 blob_append(&send, "pragma send-private\n", -1);
2185 }
2186
2187 /* Begin constructing the next message (which might never be
2188 ** sent) by beginning with the pull or push cards
2189 */
2190 if( syncFlags & SYNC_PULL ){
2191 blob_appendf(&send, "pull %s %s\n", zSCode,
2192 zAltPCode ? zAltPCode : zPCode);
2193 nCardSent++;
2194 }
2195 if( syncFlags & SYNC_PUSH ){
2196 blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
2197 nCardSent++;
2198 }
2199 go = 0;
2200 nUvGimmeSent = 0;
2201 nUvFileRcvd = 0;
2202 nPriorArtifact = nArtifactRcvd;
2203
2204 /* Process the reply that came back from the server */
2205 while( blob_line(&recv, &xfer.line) ){
2206 if( blob_buffer(&xfer.line)[0]=='#' ){
2207 const char *zLine = blob_buffer(&xfer.line);
2208 if( memcmp(zLine, "# timestamp ", 12)==0 ){
2209 char zTime[20];
2210 double rDiff;
2211 sqlite3_snprintf(sizeof(zTime), zTime, "%.19s", &zLine[12]);
2212 rDiff = db_double(9e99, "SELECT julianday('%q') - %.17g",
2213 zTime, rArrivalTime);
2214 if( rDiff>9e98 || rDiff<-9e98 ) rDiff = 0.0;
2215 if( rDiff*24.0*3600.0 >= -(blob_size(&recv)/5000.0 + 20) ){
2216 rDiff = 0.0;
2217 }
2218 if( fossil_fabs(rDiff)>fossil_fabs(rSkew) ) rSkew = rDiff;
2219 }
2220 nCardRcvd++;
2221 continue;
2222 }
2223 xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
2224 nCardRcvd++;
2225 if( (syncFlags & SYNC_VERBOSE)!=0 && recv.nUsed>0 ){
2226 pctDone = (recv.iCursor*100)/recv.nUsed;
2227 if( pctDone!=lastPctDone ){
2228 fossil_print("\rprocessed: %d%% ", pctDone);
2229 lastPctDone = pctDone;
2230 fflush(stdout);
2231 }
2232 }
2233
2234 /* file HASH SIZE \n CONTENT
2235 ** file HASH DELTASRC SIZE \n CONTENT
2236 **
2237 ** Client receives a file transmitted from the server.
2238 */
2239 if( blob_eq(&xfer.aToken[0],"file") ){
2240 xfer_accept_file(&xfer, (syncFlags & SYNC_CLONE)!=0, 0, 0);
2241 nArtifactRcvd++;
2242 }else
2243
2244 /* cfile HASH USIZE CSIZE \n CONTENT
2245 ** cfile HASH DELTASRC USIZE CSIZE \n CONTENT
2246 **
2247 ** Client receives a compressed file transmitted from the server.
2248 */
2249 if( blob_eq(&xfer.aToken[0],"cfile") ){
2250 xfer_accept_compressed_file(&xfer, 0, 0);
2251 nArtifactRcvd++;
2252 }else
2253
2254 /* uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT
2255 **
2256 ** Client accepts an unversioned file from the server.
2257 */
2258 if( blob_eq(&xfer.aToken[0], "uvfile") ){
2259 xfer_accept_unversioned_file(&xfer, 1);
2260 nArtifactRcvd++;
2261 nUvFileRcvd++;
2262 if( syncFlags & SYNC_VERBOSE ){
2263 fossil_print("\rUnversioned-file received: %s\n",
2264 blob_str(&xfer.aToken[1]));
2265 }
2266 }else
2267
2268 /* gimme HASH
2269 **
2270 ** Client receives an artifact request from the server.
2271 ** If the file is a manifest, assume that the server will also want
2272 ** to know all of the content artifacts associated with the manifest
2273 ** and send those too.
2274 */
2275 if( blob_eq(&xfer.aToken[0], "gimme")
2276 && xfer.nToken==2
2277 && blob_is_hname(&xfer.aToken[1])
2278 ){
2279 remote_unk(&xfer.aToken[1]);
2280 if( syncFlags & SYNC_PUSH ){
2281 int rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
2282 if( rid ) send_file(&xfer, rid, &xfer.aToken[1], 0);
2283 }
2284 }else
2285
2286 /* igot HASH ?PRIVATEFLAG?
2287 **
2288 ** Server announces that it has a particular file. If this is
2289 ** not a file that we have and we are pulling, then create a
2290 ** phantom to cause this file to be requested on the next cycle.
2291 ** Always remember that the server has this file so that we do
2292 ** not transmit it by accident.
2293 **
2294 ** If the PRIVATE argument exists and is 1, then the file is
2295 ** private. Pretend it does not exists if we are not pulling
2296 ** private files.
2297 */
2298 if( xfer.nToken>=2
2299 && blob_eq(&xfer.aToken[0], "igot")
2300 && blob_is_hname(&xfer.aToken[1])
2301 ){
2302 int rid;
2303 int isPriv = xfer.nToken>=3 && blob_eq(&xfer.aToken[2],"1");
2304 rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
2305 if( rid>0 ){
2306 if( isPriv ){
2307 content_make_private(rid);
2308 }else{
2309 content_make_public(rid);
2310 }
2311 }else if( isPriv && !g.perm.Private ){
2312 /* ignore private files */
2313 }else if( (syncFlags & (SYNC_PULL|SYNC_CLONE))!=0 ){
2314 rid = content_new(blob_str(&xfer.aToken[1]), isPriv);
2315 if( rid ) newPhantom = 1;
2316 }
2317 remote_has(rid);
2318 }else
2319
2320 /* uvigot NAME MTIME HASH SIZE
2321 **
2322 ** Server announces that it has a particular unversioned file. The
2323 ** server will only send this card if the client had previously sent
2324 ** a "pragma uv-hash" card with a hash that does not match.
2325 **
2326 ** If the identified file needs to be transferred, then setup for the
2327 ** transfer. Generate a "uvgimme" card in the reply if the server
2328 ** version is newer than the client. Generate a "uvfile" card if
2329 ** the client version is newer than the server. If HASH is "-"
2330 ** (indicating that the file has been deleted) and MTIME is newer,
2331 ** then do the deletion.
2332 */
2333 if( xfer.nToken==5
2334 && blob_eq(&xfer.aToken[0], "uvigot")
2335 && blob_is_filename(&xfer.aToken[1])
2336 && blob_is_int64(&xfer.aToken[2], &mtime)
2337 && blob_is_int(&xfer.aToken[4], &size)
2338 && (blob_eq(&xfer.aToken[3],"-") || blob_is_hname(&xfer.aToken[3]))
2339 ){
2340 const char *zName = blob_str(&xfer.aToken[1]);
2341 const char *zHash = blob_str(&xfer.aToken[3]);
2342 int iStatus;
2343 iStatus = unversioned_status(zName, mtime, zHash);
2344 if( (syncFlags & SYNC_UV_REVERT)!=0 ){
2345 if( iStatus==4 ) iStatus = 2;
2346 if( iStatus==5 ) iStatus = 1;
2347 }
2348 if( syncFlags & (SYNC_UV_TRACE|SYNC_UV_DRYRUN) ){
2349 const char *zMsg = 0;
2350 switch( iStatus ){
2351 case 0:
2352 case 1: zMsg = "UV-PULL"; break;
2353 case 2: zMsg = "UV-PULL-MTIME-ONLY"; break;
2354 case 4: zMsg = "UV-PUSH-MTIME-ONLY"; break;
2355 case 5: zMsg = "UV-PUSH"; break;
2356 }
2357 if( zMsg ) fossil_print("\r%s: %s\n", zMsg, zName);
2358 if( syncFlags & SYNC_UV_DRYRUN ){
2359 iStatus = 99; /* Prevent any changes or reply messages */
2360 }
2361 }
2362 if( iStatus<=1 ){
2363 if( zHash[0]!='-' ){
2364 blob_appendf(xfer.pOut, "uvgimme %s\n", zName);
2365 nCardSent++;
2366 nUvGimmeSent++;
2367 db_multi_exec("DELETE FROM unversioned WHERE name=%Q", zName);
2368 }else if( iStatus==1 ){
2369 db_multi_exec(
2370 "UPDATE unversioned"
2371 " SET mtime=%lld, hash=NULL, sz=0, encoding=0, content=NULL"
2372 " WHERE name=%Q", mtime, zName
2373 );
2374 db_unset("uv-hash", 0);
2375 }
2376 }else if( iStatus==2 ){
2377 db_multi_exec(
2378 "UPDATE unversioned SET mtime=%lld WHERE name=%Q", mtime, zName
2379 );
2380 db_unset("uv-hash", 0);
2381 }
2382 if( iStatus>=4 && uvPullOnly==1 ){
2383 fossil_warning(
2384 "Warning: uv-pull-only \n"
2385 " Unable to push unversioned content because you lack\n"
2386 " sufficient permission on the server\n"
2387 );
2388 uvPullOnly = 2;
2389 }
2390 if( iStatus<=3 || uvPullOnly ){
2391 db_multi_exec("DELETE FROM uv_tosend WHERE name=%Q", zName);
2392 }else if( iStatus==4 ){
2393 db_multi_exec("UPDATE uv_tosend SET mtimeOnly=1 WHERE name=%Q",zName);
2394 }else if( iStatus==5 ){
2395 db_multi_exec("REPLACE INTO uv_tosend(name,mtimeOnly) VALUES(%Q,0)",
2396 zName);
2397 }
2398 }else
2399
2400 /* push SERVERCODE PRODUCTCODE
2401 **
2402 ** Should only happen in response to a clone. This message tells
2403 ** the client what product code to use for the new database.
2404 */
2405 if( blob_eq(&xfer.aToken[0],"push")
2406 && xfer.nToken==3
2407 && (syncFlags & SYNC_CLONE)!=0
2408 && blob_is_hname(&xfer.aToken[2])
2409 ){
2410 if( zPCode==0 ){
2411 zPCode = mprintf("%b", &xfer.aToken[2]);
2412 db_set("project-code", zPCode, 0);
2413 }
2414 if( cloneSeqno>0 ) blob_appendf(&send, "clone 3 %d\n", cloneSeqno);
2415 nCardSent++;
2416 }else
2417
2418 /* config NAME SIZE \n CONTENT
2419 **
2420 ** Client receive a configuration value from the server.
2421 **
2422 ** The received configuration setting is silently ignored if it was
2423 ** not requested by a prior "reqconfig" sent from client to server.
2424 */
2425 if( blob_eq(&xfer.aToken[0],"config") && xfer.nToken==3
2426 && blob_is_int(&xfer.aToken[2], &size) ){
2427 const char *zName = blob_str(&xfer.aToken[1]);
2428 Blob content;
2429 blob_zero(&content);
2430 blob_extract(xfer.pIn, size, &content);
2431 g.perm.Admin = g.perm.RdAddr = 1;
2432 configure_receive(zName, &content, origConfigRcvMask);
2433 nCardRcvd++;
2434 nArtifactRcvd++;
2435 blob_reset(&content);
2436 blob_seek(xfer.pIn, 1, BLOB_SEEK_CUR);
2437 }else
2438
2439
2440 /* cookie TEXT
2441 **
2442 ** The client reserves a cookie from the server. The client
2443 ** should remember this cookie and send it back to the server
2444 ** in its next query.
2445 **
2446 ** Each cookie received overwrites the prior cookie from the
2447 ** same server.
2448 */
2449 if( blob_eq(&xfer.aToken[0], "cookie") && xfer.nToken==2 ){
2450 db_set("cookie", blob_str(&xfer.aToken[1]), 0);
2451 }else
2452
2453
2454 /* private
2455 **
2456 ** The server tells the client that the next "file" or "cfile" will
2457 ** contain private content.
2458 */
2459 if( blob_eq(&xfer.aToken[0], "private") ){
2460 xfer.nextIsPrivate = 1;
2461 }else
2462
2463
2464 /* clone_seqno N
2465 **
2466 ** When doing a clone, the server tries to send all of its artifacts
2467 ** in sequence. This card indicates the sequence number of the next
2468 ** blob that needs to be sent. If N<=0 that indicates that all blobs
2469 ** have been sent.
2470 */
2471 if( blob_eq(&xfer.aToken[0], "clone_seqno") && xfer.nToken==2 ){
2472 blob_is_int(&xfer.aToken[1], &cloneSeqno);
2473 }else
2474
2475 /* message MESSAGE
2476 **
2477 ** A message is received from the server. Print it.
2478 ** Similar to "error" but does not stop processing.
2479 **
2480 ** If the "login failed" message is seen, clear the sync password prior
2481 ** to the next cycle.
2482 */
2483 if( blob_eq(&xfer.aToken[0],"message") && xfer.nToken==2 ){
2484 char *zMsg = blob_terminate(&xfer.aToken[1]);
2485 defossilize(zMsg);
2486 if( (syncFlags & SYNC_PUSH) && zMsg
2487 && sqlite3_strglob("pull only *", zMsg)==0 ){
2488 syncFlags &= ~SYNC_PUSH;
2489 zMsg = 0;
2490 }
2491 if( zMsg && zMsg[0] ){
2492 fossil_force_newline();
2493 fossil_print("Server says: %s\n", zMsg);
2494 }
2495 }else
2496
2497 /* pragma NAME VALUE...
2498 **
2499 ** The server can send pragmas to try to convey meta-information to
2500 ** the client. These are informational only. Unknown pragmas are
2501 ** silently ignored.
2502 */
2503 if( blob_eq(&xfer.aToken[0], "pragma") && xfer.nToken>=2 ){
2504 /* pragma server-version VERSION ?DATE? ?TIME?
2505 **
2506 ** The servger announces to the server what version of Fossil it
2507 ** is running. The DATE and TIME are a pure numeric ISO8601 time
2508 ** for the specific check-in of the client.
2509 */
2510 if( xfer.nToken>=3 && blob_eq(&xfer.aToken[1], "server-version") ){
2511 xfer.remoteVersion = atoi(blob_str(&xfer.aToken[2]));
2512 if( xfer.nToken>=5 ){
2513 xfer.remoteDate = atoi(blob_str(&xfer.aToken[3]));
2514 xfer.remoteTime = atoi(blob_str(&xfer.aToken[4]));
2515 }
2516 }
2517
2518 /* pragma uv-pull-only
2519 ** pragma uv-push-ok
2520 **
2521 ** If the server is unwill to accept new unversioned content (because
2522 ** this client lacks the necessary permissions) then it sends a
2523 ** "uv-pull-only" pragma so that the client will know not to waste
2524 ** bandwidth trying to upload unversioned content. If the server
2525 ** does accept new unversioned content, it sends "uv-push-ok".
2526 */
2527 if( syncFlags & SYNC_UNVERSIONED ){
2528 if( blob_eq(&xfer.aToken[1], "uv-pull-only") ){
2529 uvPullOnly = 1;
2530 if( syncFlags & SYNC_UV_REVERT ) uvDoPush = 1;
2531 }else if( blob_eq(&xfer.aToken[1], "uv-push-ok") ){
2532 uvDoPush = 1;
2533 }
2534 }
2535
2536 /* pragma ci-lock-fail USER-HOLDING-LOCK LOCK-TIME
2537 **
2538 ** The server generates this message when a "pragma ci-lock"
2539 ** is attempted on a check-in for which there is an existing
2540 ** lock. USER-HOLDING-LOCK is the name of the user who originated
2541 ** the lock, and LOCK-TIME is the timestamp (seconds since 1970)
2542 ** when the lock was taken.
2543 */
2544 else if( blob_eq(&xfer.aToken[1], "ci-lock-fail") && xfer.nToken==4 ){
2545 char *zUser = blob_terminate(&xfer.aToken[2]);
2546 sqlite3_int64 mtime, iNow;
2547 defossilize(zUser);
2548 iNow = time(NULL);
2549 if( blob_is_int64(&xfer.aToken[3], &mtime) && iNow>mtime ){
2550 iNow = time(NULL);
2551 fossil_print("\nParent check-in locked by %s %s ago\n",
2552 zUser, human_readable_age((iNow+1-mtime)/86400.0));
2553 }else{
2554 fossil_print("\nParent check-in locked by %s\n", zUser);
2555 }
2556 g.ckinLockFail = fossil_strdup(zUser);
2557 }
2558
2559 /* pragma avoid-delta-manifests
2560 **
2561 ** Discourage the use of delta manifests. The remote side sends
2562 ** this pragma when its forbid-delta-manifests setting is true.
2563 */
2564 else if( blob_eq(&xfer.aToken[1], "avoid-delta-manifests") ){
2565 g.bAvoidDeltaManifests = 1;
2566 }
2567 }else
2568
2569 /* error MESSAGE
2570 **
2571 ** The server is reporting an error. The client will abandon
2572 ** the sync session.
2573 **
2574 ** Except, when cloning we will sometimes get an error on the
2575 ** first message exchange because the project-code is unknown
2576 ** and so the login card on the request was invalid. The project-code
2577 ** is returned in the reply before the error card, so second and
2578 ** subsequent messages should be OK. Nevertheless, we need to ignore
2579 ** the error card on the first message of a clone.
2580 **
2581 ** Also ignore "not authorized to write" errors if this is an
2582 ** autopush following a commit.
2583 */
2584 if( blob_eq(&xfer.aToken[0],"error") && xfer.nToken==2 ){
2585 char *zMsg = blob_terminate(&xfer.aToken[1]);
2586 defossilize(zMsg);
2587 if( (syncFlags & SYNC_IFABLE)!=0
2588 && sqlite3_strlike("%not authorized to write%",zMsg,0)==0 ){
2589 autopushFailed = 1;
2590 nErr++;
2591 }else if( (syncFlags & SYNC_CLONE)==0 || nCycle>0 ){
2592 fossil_force_newline();
2593 fossil_print("Error: %s\n", zMsg);
2594 blob_appendf(&xfer.err, "server says: %s\n", zMsg);
2595 nErr++;
2596 break;
2597 }
2598 }else
2599
2600 /* Unknown message */
2601 if( xfer.nToken>0 ){
2602 if( blob_str(&xfer.aToken[0])[0]=='<' ){
2603 fossil_warning(
2604 "server replies with HTML instead of fossil sync protocol:\n%b",
2605 &recv
2606 );
2607 nErr++;
2608 break;
2609 }
2610 blob_appendf(&xfer.err, "unknown command: [%b]\n", &xfer.aToken[0]);
2611 }
2612
2613 if( blob_size(&xfer.err) ){
2614 fossil_force_newline();
2615 fossil_warning("%b", &xfer.err);
2616 nErr++;
2617 break;
2618 }
2619 blobarray_reset(xfer.aToken, xfer.nToken);
2620 blob_reset(&xfer.line);
2621 }
2622 origConfigRcvMask = 0;
2623 if( nCardRcvd>0 && (syncFlags & SYNC_VERBOSE) ){
2624 fossil_print(zValueFormat /*works-like:"%s%d%d%d%d"*/, "Received:",
2625 blob_size(&recv), nCardRcvd,
2626 xfer.nFileRcvd, xfer.nDeltaRcvd + xfer.nDanglingFile);
2627 }else{
2628 fossil_print(zBriefFormat /*works-like:"%d%d%d"*/,
2629 nRoundtrip, nArtifactSent, nArtifactRcvd);
2630 }
2631 nUncRcvd += blob_size(&recv);
2632 blob_reset(&recv);
2633 nCycle++;
2634
2635 /* Set go to 1 if we need to continue the sync/push/pull/clone for
2636 ** another round. Set go to 0 if it is time to quit. */
2637 nFileRecv = xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile;
2638 if( (nFileRecv>0 || newPhantom) && db_exists("SELECT 1 FROM phantom") ){
2639 go = 1;
2640 mxPhantomReq = nFileRecv*2;
2641 if( mxPhantomReq<200 ) mxPhantomReq = 200;
2642 }else if( (syncFlags & SYNC_CLONE)!=0 && nFileRecv>0 ){
2643 go = 1;
2644 }else if( xfer.nFileSent+xfer.nDeltaSent>0 || uvDoPush ){
2645 /* Go another round if files are queued to send */
2646 go = 1;
2647 }else if( xfer.nPrivIGot>0 && nCycle==1 ){
2648 go = 1;
2649 }else if( (syncFlags & SYNC_CLONE)!=0 ){
2650 if( nCycle==1 ){
2651 go = 1; /* go at least two rounds on a clone */
2652 }else if( cloneSeqno>0 && nArtifactRcvd>nPriorArtifact ){
2653 /* Continue the clone until we see the clone_seqno 0" card or
2654 ** until we stop receiving artifacts */
2655 go = 1;
2656 }
2657 }else if( nUvGimmeSent>0 && (nUvFileRcvd>0 || nCycle<3) ){
2658 /* Continue looping as long as new uvfile cards are being received
2659 ** and uvgimme cards are being sent. */
2660 go = 1;
2661 }
2662
2663 nCardRcvd = 0;
2664 xfer.nFileRcvd = 0;
2665 xfer.nDeltaRcvd = 0;
2666 xfer.nDanglingFile = 0;
2667 db_multi_exec("DROP TABLE onremote; DROP TABLE unk;");
2668 if( go ){
2669 manifest_crosslink_end(MC_PERMIT_HOOKS);
2670 }else{
2671 manifest_crosslink_end(MC_PERMIT_HOOKS);
2672 content_enable_dephantomize(1);
2673 }
2674 db_end_transaction(0);
2675 };
2676 transport_stats(&nSent, &nRcvd, 1);
2677 if( (rSkew*24.0*3600.0) > 10.0 ){
2678 fossil_warning("*** time skew *** server is fast by %s",
2679 db_timespan_name(rSkew));
2680 g.clockSkewSeen = 1;
2681 }else if( rSkew*24.0*3600.0 < -10.0 ){
2682 fossil_warning("*** time skew *** server is slow by %s",
2683 db_timespan_name(-rSkew));
2684 g.clockSkewSeen = 1;
2685 }
2686
2687 fossil_force_newline();
2688 fossil_print(
2689 "%s done, wire bytes sent: %lld received: %lld ip: %s\n",
2690 zOpType, nSent, nRcvd, g.zIpAddr);
2691 if( syncFlags & SYNC_VERBOSE ){
2692 fossil_print(
2693 "Uncompressed payload sent: %lld received: %lld\n", nUncSent, nUncRcvd);
2694 }
2695 transport_close(&g.url);
2696 transport_global_shutdown(&g.url);
2697 if( nErr && go==2 ){
2698 db_multi_exec("DROP TABLE onremote; DROP TABLE unk;");
2699 manifest_crosslink_end(MC_PERMIT_HOOKS);
2700 content_enable_dephantomize(1);
2701 db_end_transaction(0);
2702 }
2703 if( nErr && autopushFailed ){
2704 fossil_warning(
2705 "Warning: The check-in was successful and is saved locally but you\n"
2706 " are not authorized to push the changes back to the server\n"
2707 " at %s",
2708 g.url.canonical
2709 );
2710 nErr--;
2711 }
2712 if( (syncFlags & SYNC_CLONE)==0 && g.rcvid && fossil_any_has_fork(g.rcvid) ){
2713 fossil_warning("***** WARNING: a fork has occurred *****\n"
2714 "use \"fossil leaves -multiple\" for more details.");
2715 }
2716 return nErr;
2717 }
2718