1 /*
2  * Routines for read/writing/managing file attributes
3  *
4  * Copyright (C) 2013 Craig Barratt.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 3 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License along
17  * with this program; if not, visit the http://fsf.org website.
18  */
19 
20 #include "backuppc.h"
21 
22 /*
23  * Type of attribute file.  This is saved as a magic number at the
24  * start of the file.  This type is for V3 and earlier.
25  */
26 #define BPC_ATTRIB_TYPE_UNIX    (0x17555555)
27 
28 /*
29  * super set of UNIX, including extended attribs and digest, for 4.x+
30  */
31 #define BPC_ATTRIB_TYPE_XATTR   (0x17565353)
32 
33 /*
34  * starting in 4.x, attrib files in the pc backup tree are
35  * just digests that give the location of the real attrib file
36  * in the pool.  attrib files written in the pc backup tree
37  * start with this magic number, followed by the digest.
38  */
39 #define BPC_ATTRIB_TYPE_DIGEST  (0x17585451)
40 
41 static char *FileType2Text[] = {
42     "file",
43     "hardlink",
44     "symlink",
45     "chardev",
46     "blockdev",
47     "dir",
48     "fifo",
49     "?",
50     "socket",
51     "?",
52     "deleted",
53 };
54 
55 /*
56  * Ensure by default we use old-style "attrib" files for <= 4.0.0alpha3 compatibility.
57  * Newer versions of BackupPC turn this off to use new attribute file naming.
58  */
59 static int WriteOldStyleAttribFile = 1;
60 static int KeepOldAttribFiles      = 1;
61 
bpc_attrib_backwardCompat(int writeOldStyleAttribFile,int keepOldAttribFiles)62 void bpc_attrib_backwardCompat(int writeOldStyleAttribFile, int keepOldAttribFiles)
63 {
64     if ( writeOldStyleAttribFile >= 0 ) WriteOldStyleAttribFile = writeOldStyleAttribFile;
65     if ( keepOldAttribFiles >= 0 )      KeepOldAttribFiles = keepOldAttribFiles;
66     if ( BPC_LogLevel >= 5 ) {
67         bpc_logMsgf("bpc_attrib_backwardCompat: WriteOldStyleAttribFile = %d, KeepOldAttribFiles = %d\n",
68                      WriteOldStyleAttribFile, KeepOldAttribFiles);
69     }
70 }
71 
72 
73 #define CONV_BUF_TO_UINT32(buf)    ((buf)[0] << 24 | (buf)[1] << 16 | (buf)[2] << 8 | (buf)[3])
74 
75 #define CONV_UINT32_TO_BUF(buf, val)   { *(buf)++ = ((val) >> 24) & 0xff;               \
76                                          *(buf)++ = ((val) >> 16) & 0xff;               \
77                                          *(buf)++ = ((val) >> 8)  & 0xff;               \
78                                          *(buf)++ = ((val) >> 0)  & 0xff; }
79 
80 /*
81  * Note on xattr keys: they are treated as opaque strings of bytes, and the convention
82  * is to include the '\0' byte termination in the keyLen (ie: it is strlen(key) + 1).
83  */
bpc_attrib_xattrGet(bpc_attrib_file * file,void * key,int keyLen,int allocate_if_missing)84 bpc_attrib_xattr *bpc_attrib_xattrGet(bpc_attrib_file *file, void *key, int keyLen, int allocate_if_missing)
85 {
86     return (bpc_attrib_xattr*)bpc_hashtable_find(&file->xattrHT, key, keyLen, allocate_if_missing);
87 }
88 
bpc_attrib_xattrDestroy(bpc_attrib_xattr * xattr)89 void bpc_attrib_xattrDestroy(bpc_attrib_xattr *xattr)
90 {
91     if ( xattr->key.key ) free(xattr->key.key);
92     if ( xattr->value )   free(xattr->value);
93 }
94 
bpc_attrib_xattrDelete(bpc_attrib_file * file,void * key,int keyLen)95 int bpc_attrib_xattrDelete(bpc_attrib_file *file, void *key, int keyLen)
96 {
97     bpc_attrib_xattr *xattr = bpc_hashtable_find(&file->xattrHT, key, keyLen, 0);
98 
99     if ( !xattr ) return -1;
100     bpc_attrib_xattrDestroy(xattr);
101     bpc_hashtable_nodeDelete(&file->xattrHT, xattr);
102     return 0;
103 }
104 
bpc_attrib_xattrDeleteNode(bpc_attrib_xattr * xattr,bpc_attrib_file * file)105 static void bpc_attrib_xattrDeleteNode(bpc_attrib_xattr *xattr, bpc_attrib_file *file)
106 {
107     bpc_attrib_xattrDestroy(xattr);
108     bpc_hashtable_nodeDelete(&file->xattrHT, xattr);
109 }
110 
bpc_attrib_xattrDeleteAll(bpc_attrib_file * file)111 int bpc_attrib_xattrDeleteAll(bpc_attrib_file *file)
112 {
113     bpc_hashtable_iterate(&file->xattrHT, (void*)bpc_attrib_xattrDeleteNode, file);
114     return 0;
115 }
116 
117 /*
118  * returns >0 if the new value is the same as the current value.
119  * returns 0  if the new value was set correctly
120  * returns <0 on error.
121  */
bpc_attrib_xattrSetValue(bpc_attrib_file * file,void * key,int keyLen,void * value,uint32 valueLen)122 int bpc_attrib_xattrSetValue(bpc_attrib_file *file, void *key, int keyLen, void *value, uint32 valueLen)
123 {
124     bpc_attrib_xattr *xattr;
125     char keyCopy[BPC_MAXPATHLEN];
126 
127     if ( ((char*)key)[keyLen - 1] != 0x0 ) {
128         if ( keyLen >= BPC_MAXPATHLEN - 8 ) {
129             bpc_logMsgf("bpc_attrib_xattrSetValue: BOTCH: key not 0x0 terminated; too long to repair (keyLen %u)\n", keyLen);
130             return -1;
131         } else {
132             memcpy(keyCopy, key, keyLen);
133             keyCopy[keyLen++] = 0x0;
134             key = keyCopy;
135             if ( BPC_LogLevel >= 6 ) {
136                 bpc_logMsgf("bpc_attrib_xattrSetValue: fixup: appended 0x0 to xattr name '%s' (keyLen now %u)\n", (char*)key, keyLen);
137             }
138         }
139     }
140     xattr = bpc_attrib_xattrGet(file, key, keyLen, 1);
141 
142     if ( !xattr->value ) {
143         /*
144          * new entry
145          */
146         if ( !(xattr->key.key = malloc(keyLen)) ) {
147             bpc_logErrf("bpc_attrib_xattrSetValue: can't allocate %d bytes for key\n", keyLen);
148             return -1;
149         }
150         memcpy(xattr->key.key, key, keyLen);
151         xattr->key.keyLen = keyLen;
152     } else {
153         /*
154          * existing entry - no need to recopy key.  If value array isn't big enough then create another.
155          */
156         if ( valueLen > xattr->valueLen ) {
157             free(xattr->value);
158             xattr->value = NULL;
159         } else if ( valueLen == xattr->valueLen && !memcmp(xattr->value, value, valueLen) ) {
160             /*
161              * same value - no change
162              */
163             return 1;
164         }
165     }
166     if ( !xattr->value && !(xattr->value = malloc(valueLen)) ) {
167         bpc_logErrf("bpc_attrib_xattrSetValue: can't allocate %d bytes for value\n", valueLen);
168         return -1;
169     }
170     memcpy(xattr->value, value, valueLen);
171     xattr->valueLen = valueLen;
172     return 0;
173 }
174 
bpc_attrib_xattrCopy(bpc_attrib_xattr * xattrSrc,bpc_attrib_file * fileDest)175 void bpc_attrib_xattrCopy(bpc_attrib_xattr *xattrSrc, bpc_attrib_file *fileDest)
176 {
177     bpc_attrib_xattr *xattr;
178     uchar *key   = (uchar*)malloc(xattrSrc->key.keyLen > 0 ? xattrSrc->key.keyLen : 1);
179     uchar *value = (uchar*)malloc(xattrSrc->valueLen > 0 ? xattrSrc->valueLen : 1);
180 
181     if ( !key || !value ) {
182         bpc_logErrf("bpc_attrib_xattrCopy: can't allocate %d,%d bytes\n", xattrSrc->key.keyLen + 1, xattrSrc->valueLen + 1);
183         return;
184     }
185 
186     memcpy(key, xattrSrc->key.key, xattrSrc->key.keyLen);
187     memcpy(value, xattrSrc->value, xattrSrc->valueLen);
188 
189     xattr = bpc_attrib_xattrGet(fileDest, key, xattrSrc->key.keyLen, 1);
190 
191     if ( xattr->value ) {
192         /*
193          * Shouldn't be present, but if so clear it out and write the new key
194          */
195         bpc_attrib_xattrDestroy(xattr);
196         xattr->key.key    = key;
197         xattr->key.keyLen = xattrSrc->key.keyLen;
198     }
199     xattr->value    = value;
200     xattr->valueLen = xattrSrc->valueLen;
201 }
202 
bpc_attrib_xattrCount(bpc_attrib_file * file)203 int bpc_attrib_xattrCount(bpc_attrib_file *file)
204 {
205     return bpc_hashtable_entryCount(&file->xattrHT);
206 }
207 
208 typedef struct {
209     char *list;
210     ssize_t idx;
211     ssize_t listLen;
212     int ignoreRsyncACLs;
213 } xattrList_info;
214 
bpc_attrib_xattrListKey(bpc_attrib_xattr * xattr,xattrList_info * info)215 static void bpc_attrib_xattrListKey(bpc_attrib_xattr *xattr, xattrList_info *info)
216 {
217     if ( info->idx < 0 ) return;
218 
219     if ( info->ignoreRsyncACLs ) {
220         static struct {
221             char *str;
222             unsigned int len;
223         } ignoreKeys[] = {
224             { "user.rsync.%aacl", sizeof("user.rsync.%aacl"), },    /* note: sizeof() includes the \0 terminator */
225             { "user.rsync.%dacl", sizeof("user.rsync.%dacl"), },
226         };
227         uint i;
228 
229         for ( i = 0 ; i < sizeof(ignoreKeys) / sizeof(ignoreKeys[0]) ; i++ ) {
230             if ( xattr->key.keyLen == ignoreKeys[i].len
231                     && !memcmp(xattr->key.key, ignoreKeys[i].str, xattr->key.keyLen) ) {
232                 return;
233             }
234         }
235     }
236     if ( info->list ) {
237         if ( info->idx + (signed)xattr->key.keyLen > info->listLen ) {
238             info->idx = -1;
239             return;
240         }
241         /*
242          * confirm that keyLen includes the \0 terminating byte
243          */
244         memcpy(info->list + info->idx, xattr->key.key, xattr->key.keyLen);
245         if ( xattr->key.keyLen >= 1 && info->list[info->idx + xattr->key.keyLen - 1] != 0x0 ) {
246             info->list[info->idx + xattr->key.keyLen - 1] = 0x0;
247             bpc_logMsgf("bpc_attrib_xattrListKey: BOTCH: truncated xattr name '%s' to match keyLen %u\n", info->list + info->idx, xattr->key.keyLen);
248         }
249         if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_attrib_xattrListKey: adding %s\n", info->list + info->idx);
250         info->idx += xattr->key.keyLen;
251     } else {
252         info->idx += xattr->key.keyLen;
253     }
254 }
255 
256 /*
257  * Concatenate all the xattr keys, (which the caller has ensured are \0 terminated),
258  * into a single string.  Return the number of bytes in the output string.
259  * Returns -1 if listLen is too short to fit all the keys.
260  * If list is NULL, instead returns the number of bytes required to store all the keys.
261  */
bpc_attrib_xattrList(bpc_attrib_file * file,char * list,size_t listLen,int ignoreRsyncACLs)262 size_t bpc_attrib_xattrList(bpc_attrib_file *file, char *list, size_t listLen, int ignoreRsyncACLs)
263 {
264     xattrList_info info;
265 
266     info.list            = list;
267     info.idx             = 0;
268     info.listLen         = listLen;
269     info.ignoreRsyncACLs = ignoreRsyncACLs;
270 
271     bpc_hashtable_iterate(&file->xattrHT, (void*)bpc_attrib_xattrListKey, &info);
272     return info.idx;
273 }
274 
bpc_attrib_fileDestroy(bpc_attrib_file * file)275 void bpc_attrib_fileDestroy(bpc_attrib_file *file)
276 {
277     if ( file->name) free(file->name);
278     bpc_hashtable_iterate(&file->xattrHT, (void*)bpc_attrib_xattrDestroy, NULL);
279     bpc_hashtable_destroy(&file->xattrHT);
280 }
281 
282 /*
283  * Return the attributes for the given file.
284  * If allocate_if_missing == 0 and not present, then NULL is returned.
285  * If allocate_if_missing != 0 and not present, then an empty struct is returned with the key filled in,
286  * and file->name is NULL.
287  */
bpc_attrib_fileGet(bpc_attrib_dir * dir,char * fileName,int allocate_if_missing)288 bpc_attrib_file *bpc_attrib_fileGet(bpc_attrib_dir *dir, char *fileName, int allocate_if_missing)
289 {
290     return bpc_hashtable_find(&dir->filesHT, (uchar*)fileName, strlen(fileName), allocate_if_missing);
291 }
292 
293 /*
294  * Initialize an empty file structure (ie: one returned by bpc_attrib_fileGet() that is empty)
295  */
bpc_attrib_fileInit(bpc_attrib_file * file,char * fileName,int xattrNumEntries)296 void bpc_attrib_fileInit(bpc_attrib_file *file, char *fileName, int xattrNumEntries)
297 {
298     int fileNameLen = strlen(fileName);
299 
300     if ( file->name ) bpc_attrib_fileDestroy(file);
301     file->name = (char*)malloc(fileNameLen + 1);
302     if ( !file->name ) {
303         bpc_logErrf("bpc_attrib_fileInit: can't allocate %d bytes for file name\n", fileNameLen + 1);
304         return;
305     }
306     memcpy(file->name, fileName, fileNameLen + 1);
307     file->isTemp  = 0;
308     file->key.key = file->name;
309     bpc_hashtable_create(&file->xattrHT, 16 + xattrNumEntries, sizeof(bpc_attrib_xattr));
310 }
311 
312 /*
313  * Copy all the attributes from fileSrc to fileDest.  fileDest should already have a
314  * valid allocated fileName and allocated xattr hash.  The fileDest xattr hash is
315  * emptied before the copy, meaning it is over written.
316  *
317  * If overwriteEmptyDigest == 0, an empty digest in fileSrc will not overwrite fileDest.
318  */
bpc_attrib_fileCopyOpt(bpc_attrib_file * fileDest,bpc_attrib_file * fileSrc,int overwriteEmptyDigest)319 void bpc_attrib_fileCopyOpt(bpc_attrib_file *fileDest, bpc_attrib_file *fileSrc, int overwriteEmptyDigest)
320 {
321     if ( fileDest == fileSrc ) return;
322 
323     fileDest->type      = fileSrc->type;
324     fileDest->compress  = fileSrc->compress;
325     fileDest->mode      = fileSrc->mode;
326     fileDest->isTemp    = fileSrc->isTemp;
327     fileDest->uid       = fileSrc->uid;
328     fileDest->gid       = fileSrc->gid;
329     fileDest->nlinks    = fileSrc->nlinks;
330     fileDest->mtime     = fileSrc->mtime;
331     fileDest->size      = fileSrc->size;
332     fileDest->inode     = fileSrc->inode;
333     fileDest->backupNum = fileSrc->backupNum;
334     if ( fileSrc->digest.len > 0 || overwriteEmptyDigest ) {
335         fileDest->digest = fileSrc->digest;
336     }
337     bpc_hashtable_iterate(&fileDest->xattrHT, (void*)bpc_attrib_xattrDestroy, NULL);
338     bpc_hashtable_erase(&fileDest->xattrHT);
339     bpc_hashtable_iterate(&fileSrc->xattrHT, (void*)bpc_attrib_xattrCopy, fileDest);
340 }
341 
342 /*
343  * Copy all the attributes from fileSrc to fileDest.  fileDest should already have a
344  * valid allocated fileName and allocated xattr hash.  The fileDest xattr hash is
345  * emptied before the copy, meaning it is over written.
346  */
bpc_attrib_fileCopy(bpc_attrib_file * fileDest,bpc_attrib_file * fileSrc)347 void bpc_attrib_fileCopy(bpc_attrib_file *fileDest, bpc_attrib_file *fileSrc)
348 {
349     if ( fileDest == fileSrc ) return;
350 
351     bpc_attrib_fileCopyOpt(fileDest, fileSrc, 1);
352 }
353 
354 /*
355  * Check if two file attribute structures are the same.  Returns 0 if they are the same.
356  */
bpc_attrib_fileCompare(bpc_attrib_file * file0,bpc_attrib_file * file1)357 int bpc_attrib_fileCompare(bpc_attrib_file *file0, bpc_attrib_file *file1)
358 {
359     uint idx = 0;
360 
361     if ( file0->type != file1->type
362             || file0->compress   != file1->compress
363             || file0->mode       != file1->mode
364             || file0->uid        != file1->uid
365             || file0->gid        != file1->gid
366             || file0->nlinks     != file1->nlinks
367             || file0->mtime      != file1->mtime
368             || file0->size       != file1->size
369             || file0->inode      != file1->inode
370             || file0->digest.len != file1->digest.len
371             || memcmp(file0->digest.digest, file1->digest.digest, file0->digest.len)
372             || bpc_attrib_xattrCount(file0) != bpc_attrib_xattrCount(file1) ) {
373         if ( BPC_LogLevel >= 9 ) bpc_logMsgf("bpc_attrib_fileCompare: %s %s differ\n", file0->name, file1->name);
374         return 1;
375     }
376     while ( 1 ) {
377         bpc_attrib_xattr *xattr0 = bpc_hashtable_nextEntry(&file0->xattrHT, &idx), *xattr1;
378         if ( !xattr0 ) return 0;
379         if ( !(xattr1 = bpc_attrib_xattrGet(file1, xattr0->key.key, xattr0->key.keyLen, 0)) ) return 1;
380         if ( xattr0->valueLen != xattr1->valueLen || memcmp(xattr0->value, xattr1->value, xattr0->valueLen) ) return 1;
381     }
382 }
383 
bpc_attrib_fileDeleteName(bpc_attrib_dir * dir,char * fileName)384 void bpc_attrib_fileDeleteName(bpc_attrib_dir *dir, char *fileName)
385 {
386     bpc_attrib_file *file = bpc_hashtable_find(&dir->filesHT, (uchar*)fileName, strlen(fileName), 0);
387 
388     if ( !file ) return;
389     bpc_attrib_fileDestroy(file);
390     bpc_hashtable_nodeDelete(&dir->filesHT, file);
391 }
392 
bpc_attrib_fileIterate(bpc_attrib_dir * dir,bpc_attrib_file ** file,uint * idx)393 int bpc_attrib_fileIterate(bpc_attrib_dir *dir, bpc_attrib_file **file, uint *idx)
394 {
395     *file = bpc_hashtable_nextEntry(&dir->filesHT, idx);
396     if ( !*file ) return -1;
397     return 0;
398 }
399 
bpc_attrib_fileCount(bpc_attrib_dir * dir)400 int bpc_attrib_fileCount(bpc_attrib_dir *dir)
401 {
402     return bpc_hashtable_entryCount(&dir->filesHT);
403 }
404 
bpc_attrib_fileType2Text(int type)405 char *bpc_attrib_fileType2Text(int type)
406 {
407     if ( type < 0 || type >= (int)(sizeof(FileType2Text) / sizeof(FileType2Text[0])) ) return "?";
408     return FileType2Text[type];
409 }
410 
bpc_attrib_dirInit(bpc_attrib_dir * dir,int compressLevel)411 void bpc_attrib_dirInit(bpc_attrib_dir *dir, int compressLevel)
412 {
413     dir->digest.len = 0;
414     dir->compress = compressLevel;
415     dir->needRewrite = 0;
416     bpc_hashtable_create(&dir->filesHT, 512, sizeof(bpc_attrib_file));
417 }
418 
bpc_attrib_dirDestroy(bpc_attrib_dir * dir)419 void bpc_attrib_dirDestroy(bpc_attrib_dir *dir)
420 {
421     bpc_hashtable_iterate(&dir->filesHT, (void*)bpc_attrib_fileDestroy, NULL);
422     bpc_hashtable_destroy(&dir->filesHT);
423 }
424 
bpc_attrib_dirNeedRewrite(bpc_attrib_dir * dir)425 int bpc_attrib_dirNeedRewrite(bpc_attrib_dir *dir)
426 {
427     return dir->needRewrite;
428 }
429 
430 typedef struct {
431     bpc_deltaCount_info *deltaInfo;
432     int incr;
433     unsigned int *inodeMax;
434 } fileRefCnt_info;
435 
bpc_attrib_fileRefCount(bpc_attrib_file * file,fileRefCnt_info * info)436 static void bpc_attrib_fileRefCount(bpc_attrib_file *file, fileRefCnt_info *info)
437 {
438     if ( file->digest.len > 0 ) {
439         char hexStr[BPC_DIGEST_LEN_MAX * 2 + 1];
440         bpc_digest_digest2str(&file->digest, hexStr);
441         if ( BPC_LogLevel >= 7 ) bpc_logMsgf("bpc_attrib_fileRefCount: file %s digest %s delta %d\n", file->name, hexStr, info->incr);
442         bpc_poolRefDeltaUpdate(info->deltaInfo, file->compress, &file->digest, info->incr);
443     }
444     if ( info->inodeMax && file->inode > *info->inodeMax ) {
445         *info->inodeMax = file->inode;
446     }
447 }
448 
449 /*
450  * call refDeltaUpdate with incr (typically +/-1) for every entry in the directory,
451  * as well as the dir itself.
452  */
bpc_attrib_dirRefCountInodeMax(bpc_deltaCount_info * deltaInfo,bpc_attrib_dir * dir,int incr,unsigned int * inodeMax)453 void bpc_attrib_dirRefCountInodeMax(bpc_deltaCount_info *deltaInfo, bpc_attrib_dir *dir, int incr, unsigned int *inodeMax)
454 {
455     fileRefCnt_info info;
456 
457     info.deltaInfo = deltaInfo;
458     info.incr      = incr;
459     info.inodeMax  = inodeMax;
460     bpc_hashtable_iterate(&dir->filesHT, (void*)bpc_attrib_fileRefCount, &info);
461     if ( dir->digest.len > 0 ) {
462         char hexStr[BPC_DIGEST_LEN_MAX * 2 + 1];
463         bpc_digest_digest2str(&dir->digest, hexStr);
464         if ( BPC_LogLevel >= 7 ) bpc_logMsgf("bpc_attrib_dirRefCount: attrib digest %s delta = %d\n", hexStr, incr);
465         bpc_poolRefDeltaUpdate(deltaInfo, dir->compress, &dir->digest, incr);
466     } else {
467         if ( BPC_LogLevel >= 7 ) bpc_logMsgf("bpc_attrib_dirRefCount: no attrib digest -> no delta\n");
468     }
469 }
470 
471 /*
472  * call refDeltaUpdate with incr (typically +/-1) for every entry in the directory,
473  * as well as the dir itself.
474  */
bpc_attrib_dirRefCount(bpc_deltaCount_info * deltaInfo,bpc_attrib_dir * dir,int incr)475 void bpc_attrib_dirRefCount(bpc_deltaCount_info *deltaInfo, bpc_attrib_dir *dir, int incr)
476 {
477     bpc_attrib_dirRefCountInodeMax(deltaInfo, dir, incr, NULL);
478 }
479 
480 typedef struct {
481     char *entries;
482     ssize_t entryIdx;
483     ssize_t entrySize;
484 } dirEntry_info;
485 
bpc_attrib_getDirEntry(bpc_attrib_file * file,dirEntry_info * info)486 static void bpc_attrib_getDirEntry(bpc_attrib_file *file, dirEntry_info *info)
487 {
488     ssize_t len = strlen(file->name) + 1;
489 
490     if ( info->entryIdx < 0 ) return;
491     if ( info->entries ) {
492         if ( info->entryIdx + len > info->entrySize ) {
493             info->entryIdx = -1;
494             return;
495         }
496         memcpy(info->entries + info->entryIdx, file->name, len);
497     }
498     info->entryIdx += len;
499 }
500 
bpc_attrib_getEntries(bpc_attrib_dir * dir,char * entries,ssize_t entrySize)501 ssize_t bpc_attrib_getEntries(bpc_attrib_dir *dir, char *entries, ssize_t entrySize)
502 {
503     dirEntry_info info;
504 
505     info.entries   = entries;
506     info.entryIdx  = 0;
507     info.entrySize = entrySize;
508 
509     bpc_hashtable_iterate(&dir->filesHT, (void*)bpc_attrib_getDirEntry, &info);
510     return info.entryIdx;
511 }
512 
bpc_attrib_attribFilePath(char * path,char * dir,char * attribFileName)513 void bpc_attrib_attribFilePath(char *path, char *dir, char *attribFileName)
514 {
515     if ( !dir ) {
516         snprintf(path, BPC_MAXPATHLEN, "%s", attribFileName);
517     } else {
518         snprintf(path, BPC_MAXPATHLEN, "%s/%s", dir, attribFileName ? attribFileName : "attrib");
519     }
520 }
521 
bpc_attrib_dirDigestGet(bpc_attrib_dir * dir)522 bpc_digest *bpc_attrib_dirDigestGet(bpc_attrib_dir *dir)
523 {
524     return &dir->digest;
525 }
526 
read_more_data(bpc_fileZIO_fd * fd,uchar * buf,size_t bufSize,size_t * nRead,uchar ** bufPP,char * attribPath)527 static int read_more_data(bpc_fileZIO_fd *fd, uchar *buf, size_t bufSize, size_t *nRead, uchar **bufPP, char *attribPath)
528 {
529     int thisRead;
530     /*
531      * move the remaining part of the buffer down, and read more data
532      */
533     *nRead = (buf + *nRead) - *bufPP;
534     memmove(buf, *bufPP, *nRead);
535     *bufPP = buf;
536     thisRead = bpc_fileZIO_read(fd, buf + *nRead, bufSize - *nRead);
537     if ( thisRead < 0 ) {
538         bpc_logErrf("bpc_attrib_dirRead: can't read more bytes from %s\n", attribPath);
539         return -1;
540     }
541     *nRead += thisRead;
542     return 0;
543 }
544 
545 /*
546  * Read variable-length unsigned integer in 7 bit chunks, LSB first.
547  */
getVarInt(uchar ** bufPP,uchar * bufEnd)548 static int64 getVarInt(uchar **bufPP, uchar *bufEnd)
549 {
550     int64 result = 0;
551     uchar *bufP = *bufPP;
552     int i = 0;
553 
554     while ( bufP < bufEnd ) {
555         uchar c = *bufP++;
556         result |= ((int64)(c & 0x7f)) << i;
557         if ( !(c & 0x80) ) {
558             *bufPP = bufP;
559             return result;
560         }
561         i += 7;
562     }
563     /*
564      * we ran out of data... make sure bufP is greater than bufEnd, since
565      * returning it to be equal (ie: bufP) will be incorrectly interpreted as
566      * meaning the integer correctly ended right at the end of the buffer.
567      */
568     *bufPP = bufEnd + 1;
569     return result;
570 }
571 
572 /*
573  * V3 variable length integer read, MSB first, which is compatible with perl pack("w")
574  */
getVarInt_v3(uchar ** bufPP,uchar * bufEnd)575 static int64 getVarInt_v3(uchar **bufPP, uchar *bufEnd)
576 {
577     int64 result = 0;
578     uchar *bufP = *bufPP;
579 
580     while ( bufP < bufEnd ) {
581         uchar c = *bufP++;
582         result = (result << 7) | (c & 0x7f);
583         if ( !(c & 0x80) ) {
584             *bufPP = bufP;
585             return result;
586         }
587     }
588     /*
589      * we ran out of data... make sure bufP is greater than bufEnd, since
590      * returning it to be equal (ie: bufP) will be incorrectly interpreted as
591      * meaning the integer correctly ended right at the end of the buffer.
592      */
593     *bufPP = bufEnd + 1;
594     return result;
595 }
596 
597 /*
598  * Write variable-length unsigned integer in 7 bit chunks, LSB first
599  */
setVarInt(uchar ** bufPP,uchar * bufEnd,int64 value)600 static void setVarInt(uchar **bufPP, uchar *bufEnd, int64 value)
601 {
602     uchar *bufP = *bufPP;
603     int maxBytes = (sizeof(value) * 8 + 6) / 7;
604 
605     do {
606         uchar c = value & 0x7f;
607         value >>= 7;
608         maxBytes--;
609         if ( value && maxBytes > 0 ) c |= 0x80;
610         if ( bufP < bufEnd ) {
611             *bufP++ = c;
612         } else {
613             bufP++;
614         }
615     } while ( value && maxBytes > 0 );
616     *bufPP = bufP;
617 }
618 
619 /*
620  * Unpack the data in buf[] into the file structure, after the file name and xattr entry
621  * count have been extracted.  Returns next unused buffer location.
622  *
623  * If there isn't enough data to extract a complete file structure, the return value
624  * will be greater than bufEnd.  You should gather more data and re-call the function.
625  */
bpc_attrib_buf2file(bpc_attrib_file * file,uchar * buf,uchar * bufEnd,int xattrNumEntries,int * xattrFixup)626 uchar *bpc_attrib_buf2file(bpc_attrib_file *file, uchar *buf, uchar *bufEnd, int xattrNumEntries, int *xattrFixup)
627 {
628     uchar *bufP   = buf;
629     int i;
630 
631     file->type       = getVarInt(&bufP, bufEnd);
632     file->mtime      = getVarInt(&bufP, bufEnd);
633     file->mode       = getVarInt(&bufP, bufEnd);
634     file->uid        = getVarInt(&bufP, bufEnd);
635     file->gid        = getVarInt(&bufP, bufEnd);
636     file->size       = getVarInt(&bufP, bufEnd);
637     file->inode      = getVarInt(&bufP, bufEnd);
638     file->compress   = getVarInt(&bufP, bufEnd);
639     file->nlinks     = getVarInt(&bufP, bufEnd);
640     file->digest.len = getVarInt(&bufP, bufEnd);
641     file->isTemp     = 0;
642 
643     if ( file->digest.len > 0 && bufP + file->digest.len <= bufEnd ) {
644         memcpy(file->digest.digest, bufP, file->digest.len);
645     }
646     bufP += file->digest.len;
647 
648     for ( i = 0 ; i < xattrNumEntries ; i++ ) {
649         uint keyLen   = getVarInt(&bufP, bufEnd);
650         uint valueLen = getVarInt(&bufP, bufEnd);
651 
652         if ( bufP + keyLen + valueLen <= bufEnd ) {
653             if ( xattrFixup && bufP[keyLen - 1] != 0x0 ) {
654                 *xattrFixup = 1;
655             }
656             bpc_attrib_xattrSetValue(file, bufP, keyLen, bufP + keyLen, valueLen);
657         }
658         bufP += keyLen + valueLen;
659     }
660     return bufP;
661 }
662 
663 /*
664  * Extract an entire packed file structure, starting with the fileName length varint.
665  * Returns next unused buffer location.  It is assumed the file structure is already
666  * initialized and has a valid fileName allocated, so we don't allocate it here.
667  *
668  * If there isn't enough data to extract a complete file structure, the return value
669  * will be greater than bufEnd.  You should gather more data and re-call the function.
670  * On certain errors, returns NULL;
671  */
bpc_attrib_buf2fileFull(bpc_attrib_file * file,uchar * bufP,uchar * bufEnd)672 uchar *bpc_attrib_buf2fileFull(bpc_attrib_file *file, uchar *bufP, uchar *bufEnd)
673 {
674     uint fileNameLen, xattrNumEntries;
675 
676     fileNameLen = getVarInt(&bufP, bufEnd);
677     if ( fileNameLen > BPC_MAXPATHLEN - 1 ) {
678         bpc_logErrf("bpc_attrib_buf2fileFull: got unreasonable file name length %d\n", fileNameLen);
679         return NULL;
680     }
681     bufP += fileNameLen;
682     bpc_attrib_xattrDeleteAll(file);
683     xattrNumEntries = getVarInt(&bufP, bufEnd);
684     if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_attrib_buf2fileFull: xattrNumEntries = %d\n", xattrNumEntries);
685     bufP = bpc_attrib_buf2file(file, bufP, bufEnd, xattrNumEntries, NULL);
686     return bufP;
687 }
688 
689 /*
690  * Read the attribute file at dirPath/attribFilePath and populate dir
691  */
bpc_attrib_dirRead(bpc_attrib_dir * dir,char * dirPath,char * attribFilePath,int backupNum)692 int bpc_attrib_dirRead(bpc_attrib_dir *dir, char *dirPath, char *attribFilePath, int backupNum)
693 {
694     char attribPath[BPC_MAXPATHLEN];
695     bpc_fileZIO_fd fd;
696     size_t nRead;
697     uint32 magic = 0;
698     uchar buf[8 * 65536], *bufP;
699     STRUCT_STAT st;
700     char *p, *attribFileName;
701 
702     bpc_attrib_attribFilePath(attribPath, dirPath, attribFilePath);
703     dir->digest.len = 0;
704 
705     /*
706      * attribFileName points to the last portion of attribFilePath, or the whole
707      * string if it doesn't contain '/'
708      */
709     if ( (attribFileName = strrchr(attribFilePath, '/')) ) {
710         attribFileName++;
711     } else {
712         attribFileName = attribFilePath;
713     }
714 
715     if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_attrib_dirRead(%s); dirPath = %s, attribFilePath = %s, attribFileName = %s\n",
716                                          attribPath, dirPath, attribFilePath, attribFileName);
717 
718     if ( (p = strchr(attribFileName, '_')) && !stat(attribPath, &st) && S_ISREG(st.st_mode) ) {
719         /*
720          * Explicit path name to new-style attrib file, and it exists; extract digest
721          */
722         if ( !strcmp(p + 1, "0") ) return 0;
723         bpc_digest_str2digest(&dir->digest, p + 1);
724         if ( BPC_LogLevel >= 6 ) {
725             char str[256];
726             bpc_digest_digest2str(&dir->digest, str);
727             bpc_logMsgf("bpc_attrib_dirRead: called with attrib file %s: digest = %s, len = %d\n",
728                                               attribPath, str, dir->digest.len);
729         }
730         /*
731          * Write new type attrib files (since we found a new-style one)
732          */
733 	WriteOldStyleAttribFile = 0;
734         magic = BPC_ATTRIB_TYPE_XATTR;
735     } else if ( stat(attribPath, &st) || !S_ISREG(st.st_mode) || strchr(attribFileName, '_') ) {
736         DIR *dirOs;
737         struct dirent *dp;
738         int attribFileNameLen = strlen(attribFileName);
739         char attribDirPath[BPC_MAXPATHLEN];
740         /*
741          * Starting in 0.50, the attrib files are zero length with the digest encoded in
742          * the file name, so there is no file just called "attrib".  Look in the directory
743          * to find it.
744          */
745         strcpy(attribDirPath, attribPath);
746         if ( (p = strrchr(attribDirPath, '/')) ) {
747             *p = '\0';
748         } else {
749             strcpy(attribDirPath, ".");
750         }
751 	if ( !(dirOs = opendir(attribDirPath)) ) {
752             /*
753              * This is a benign error - just return as though there is an empty attrib file
754              */
755             if ( BPC_LogLevel >= 8 ) bpc_logMsgf("bpc_attrib_dirRead: can't opendir %s (note: this is ok)\n", attribDirPath);
756 	    return 0;
757 	}
758 	while ( (dp = readdir(dirOs)) ) {
759 	    if ( strncmp(dp->d_name, attribFileName, attribFileNameLen) ) continue;
760             p = dp->d_name + attribFileNameLen;
761             if ( p[0] != '_' ) continue;
762             p++;
763 	    if ( !strcmp(p, "0") ) {
764                 /*
765                  * An empty attrib file is legit; just return with no entries
766                  */
767                 if ( BPC_LogLevel >= 6 ) {
768                     bpc_logMsgf("bpc_attrib_dirRead: Got empty attrib file %s\n", dp->d_name);
769                 }
770 		closedir(dirOs);
771 		return 0;
772 	    }
773             bpc_digest_str2digest(&dir->digest, p);
774             if ( BPC_LogLevel >= 6 ) {
775                 char str[256];
776                 bpc_digest_digest2str(&dir->digest, str);
777                 bpc_logMsgf("bpc_attrib_dirRead: Got attrib file %s: digest = %s, len = %d\n",
778                                                   dp->d_name, str, dir->digest.len);
779             }
780             /*
781              * Write new type attrib files (since we found a new-style one)
782              */
783             WriteOldStyleAttribFile = 0;
784             break;
785 	}
786 	closedir(dirOs);
787         if ( dir->digest.len == 0 ) return 0;
788         magic = BPC_ATTRIB_TYPE_XATTR;
789     } else {
790         if ( bpc_fileZIO_open(&fd, attribPath, 0, dir->compress) ) {
791             bpc_logErrf("bpc_attrib_dirRead: can't open %s\n", attribPath);
792             return -1;
793         }
794         nRead = bpc_fileZIO_read(&fd, buf, sizeof(buf));
795         if ( nRead == 0 ) {
796             /*
797              * an empty file is legit - this means an empty directory (ie: zero attrib entries).
798              * indicate this with an empty digest and empty hash of entries.
799              */
800             bpc_fileZIO_close(&fd);
801             if ( !strcmp(attribFileName, "attrib") ) {
802                 char attribPathTemp[BPC_MAXPATHLEN + 16];
803 
804                 strcpy(attribPathTemp, attribPath);
805                 strcat(attribPathTemp, "_0");
806                 if ( rename(attribPath, attribPathTemp) ) {
807                     bpc_logErrf("bpc_attrib_dirRead: rename of empty attrib file from %s to %s failed\n", attribPath, attribPathTemp);
808                     return -1;
809                 } else if ( BPC_LogLevel >= 6 ) {
810                     bpc_logMsgf("bpc_attrib_dirRead: renamed empty attrib file %s -> %s\n", attribPath, attribPathTemp);
811                 }
812             }
813             return 0;
814         }
815         if ( nRead < 4 ) {
816             bpc_logErrf("bpc_attrib_dirRead: can't read at least 4 bytes from %s\n", attribPath);
817             bpc_fileZIO_close(&fd);
818             return -1;
819         }
820         magic = CONV_BUF_TO_UINT32(buf);
821     }
822 
823     if ( magic == BPC_ATTRIB_TYPE_DIGEST ) {
824         char attribPathNew[BPC_MAXPATHLEN];
825         int fdNum;
826         size_t digestLen = nRead - 4, attribPathLen = strlen(attribPath);
827 
828         if ( nRead < 20 ) {
829             bpc_logErrf("bpc_attrib_dirRead: can't read at least 20 bytes from %s\n", attribPath);
830             return -1;
831         }
832         bpc_fileZIO_close(&fd);
833         if ( digestLen > sizeof(dir->digest.digest) ) digestLen = sizeof(dir->digest.digest);
834         memcpy(dir->digest.digest, buf + 4, digestLen);
835         dir->digest.len = digestLen;
836 
837         if ( !KeepOldAttribFiles ) {
838             /*
839              * replace the attrib file with a new-style attrib file where the digest is encoded
840              * in a zero length file name
841              */
842             if ( attribPathLen + dir->digest.len * 2 + 2 >= sizeof(attribPathNew) ) {
843                 bpc_logErrf("bpc_attrib_dirRead: new digest path too long (%d, %d, %d)\n",
844                                         attribPathLen, dir->digest.len, sizeof(attribPathNew));
845                 return -1;
846             }
847             strcpy(attribPathNew, attribPath);
848             attribPathNew[attribPathLen++] = '_';
849             bpc_digest_digest2str(&dir->digest, attribPathNew + attribPathLen);
850             if ( (fdNum = open(attribPathNew, O_WRONLY | O_CREAT | O_TRUNC, 0660)) < 0 ) {
851                 bpc_logErrf("bpc_attrib_dirRead: can't open/create empty attrib file %s\n", attribPathNew);
852                 return -1;
853             }
854             close(fdNum);
855             unlink(attribPath);
856             if ( BPC_LogLevel >= 4 ) bpc_logMsgf("bpc_attrib_dirRead: replaced %s with %s\n",
857                                                     attribPath, attribPathNew);
858         }
859     }
860 
861     if ( dir->digest.len > 0 ) {
862         /*
863          * Handle V4+ case - open the pool file directly
864          * For V3, digest.len == 0 since we opened the attrib file above (it is stored hardlinked in the backup
865          * directory; there is no digest)
866          */
867         bpc_digest_md52path(attribPath, dir->compress, &dir->digest);
868         if ( bpc_fileZIO_open(&fd, attribPath, 0, dir->compress) ) {
869             bpc_logErrf("bpc_attrib_dirRead: can't open %s\n", attribPath);
870             return -1;
871         }
872         nRead = bpc_fileZIO_read(&fd, buf, sizeof(buf));
873         if ( nRead < 4 ) {
874             bpc_logErrf("bpc_attrib_dirRead: can't read at least 4 bytes from %s\n", attribPath);
875             bpc_fileZIO_close(&fd);
876             return -1;
877         }
878         magic = CONV_BUF_TO_UINT32(buf);
879     }
880     bufP = buf + 4;
881 
882     if ( magic == BPC_ATTRIB_TYPE_XATTR ) {
883         int retry = 0;
884         while ( bufP < buf + nRead ) {
885             uint fileNameLen, xattrNumEntries;
886             char *fileName;
887             bpc_attrib_file *file;
888             uchar *bufPsave = bufP;
889             int xattrFixup = 0;
890 
891             if ( nRead == sizeof(buf) && bufP > buf + nRead - 2 * BPC_MAXPATHLEN
892                     && read_more_data(&fd, buf, sizeof(buf), &nRead, &bufP, attribPath) ) {
893                 bpc_fileZIO_close(&fd);
894                 return -1;
895             }
896 
897             fileNameLen = getVarInt(&bufP, buf + nRead);
898             if ( fileNameLen > BPC_MAXPATHLEN - 1 ) {
899                 bpc_logErrf("bpc_attrib_dirRead: got unreasonable file name length %d\n", fileNameLen);
900                 bpc_fileZIO_close(&fd);
901                 return -1;
902             }
903 
904             /*
905              * Save the fileName, but it's not NULL terminated yet.
906              * After we consume the next varint, we can safely NULL-terminate
907              * the fileName, which allows us to look up or create the file entry.
908              */
909             fileName = (char*)bufP;
910             bufP    += fileNameLen;
911             xattrNumEntries = getVarInt(&bufP, buf + nRead);
912             fileName[fileNameLen] = '\0';
913 
914             file = bpc_attrib_fileGet(dir, fileName, 1);
915             bpc_attrib_fileInit(file, fileName, xattrNumEntries);
916             file->backupNum = backupNum;
917 
918             bufP = bpc_attrib_buf2file(file, bufP, buf + nRead, xattrNumEntries, &xattrFixup);
919             dir->needRewrite |= xattrFixup;
920             if ( bufP > buf + nRead ) {
921                 /*
922                  * Need to get more data and try again.  We have allocated file->name,
923                  * and perhaps partially filled the xattr structure, which will be ok
924                  * on the retry since the same structure will be used.
925                  */
926                 if ( retry ) {
927                     bpc_logErrf("bpc_attrib_dirRead: BOTCH: couldn't complete file conversion on retry (%ld,%ld,%ld)\n",
928                                         bufP - buf, bufPsave - buf, nRead);
929                     bpc_fileZIO_close(&fd);
930                     return -1;
931                 }
932                 if ( BPC_LogLevel >= 7 ) bpc_logMsgf("bpc_attrib_dirRead: retrying file conversion\n");
933                 bufP = bufPsave;
934                 if ( read_more_data(&fd, buf, sizeof(buf), &nRead, &bufP, attribPath) ) {
935                     bpc_fileZIO_close(&fd);
936                     return -1;
937                 }
938                 retry = 1;
939             } else {
940                 retry = 0;
941             }
942             if ( !retry && BPC_LogLevel >= 8 ) bpc_logMsgf("bpc_attrib_dirRead(%s): Got file %s: type = %d, mode = 0%o, uid/gid = %d/%d, size = %d\n",
943                                                   attribPath, file->name, file->type, file->mode, file->uid, file->gid, file->size);
944         }
945     } else if ( magic == BPC_ATTRIB_TYPE_UNIX ) {
946         while ( bufP < buf + nRead ) {
947             uint fileNameLen;
948             char *fileName;
949             bpc_attrib_file *file;
950             int64 sizeDiv4GB;
951             uint type;
952 
953             if ( nRead == sizeof(buf) && bufP > buf + nRead - 2 * BPC_MAXPATHLEN
954                     && read_more_data(&fd, buf, sizeof(buf), &nRead, &bufP, attribPath) ) {
955                 bpc_fileZIO_close(&fd);
956                 return -1;
957             }
958 
959             fileNameLen = getVarInt_v3(&bufP, buf + nRead);
960             if ( fileNameLen > 2 * BPC_MAXPATHLEN - 16 ) {
961                 bpc_logErrf("bpc_attrib_dirRead: got unreasonable file name length %d\n", fileNameLen);
962                 bpc_fileZIO_close(&fd);
963                 return -1;
964             }
965 
966             /*
967              * Save the fileName, but it's not NULL terminated yet.
968              * After we get the next data, we can safely NULL-terminate the fileName.
969              */
970             fileName = (char*)bufP;
971             bufP    += fileNameLen;
972             type     = getVarInt_v3(&bufP, buf + nRead);
973             fileName[fileNameLen] = '\0';
974 
975             file = bpc_attrib_fileGet(dir, fileName, 1);
976             bpc_attrib_fileInit(file, fileName, 0);
977 
978             file->type      = type;
979             file->mode      = getVarInt_v3(&bufP, buf + nRead);
980             file->uid       = getVarInt_v3(&bufP, buf + nRead);
981             file->gid       = getVarInt_v3(&bufP, buf + nRead);
982             sizeDiv4GB      = getVarInt_v3(&bufP, buf + nRead);
983             file->size      = (sizeDiv4GB << 32) + getVarInt_v3(&bufP, buf + nRead);
984             file->mtime     = CONV_BUF_TO_UINT32(bufP); bufP += 4;
985             file->compress  = dir->compress;
986             file->backupNum = backupNum;
987 
988             if ( BPC_LogLevel >= 8 ) bpc_logMsgf("bpc_attrib_dirRead(%s): Got v3 file %s: type = %d, mode = 0%o, uid/gid = %d/%d, size = %d\n",
989                                                   attribPath, file->name, file->type, file->mode, file->uid, file->gid, file->size);
990         }
991     } else {
992         bpc_logErrf("Unexpected magic number 0x%x read from %s\n", magic, attribPath);
993         return -1;
994     }
995     /* TODO: make sure we are at EOF? */
996     bpc_fileZIO_close(&fd);
997     return 0;
998 }
999 
1000 typedef struct {
1001     uchar *bufP;
1002     uchar *bufEnd;
1003     uint numEntries;
1004 } buf_info;
1005 
1006 typedef struct {
1007     bpc_poolWrite_info fd;
1008     uchar buf[4 * 65536];
1009     uchar *bufP;
1010 } write_info;
1011 
write_file_flush(write_info * info)1012 static void write_file_flush(write_info *info)
1013 {
1014     if ( info->bufP > info->buf ) {
1015         if ( BPC_LogLevel >= 7 ) bpc_logMsgf("write_file_flush: writing %lu bytes to pool\n", (unsigned long)(info->bufP - info->buf));
1016         bpc_poolWrite_write(&info->fd, info->buf, info->bufP - info->buf);
1017     }
1018     info->bufP = info->buf;
1019 }
1020 
bpc_attrib_xattrWrite(bpc_attrib_xattr * xattr,buf_info * info)1021 static void bpc_attrib_xattrWrite(bpc_attrib_xattr *xattr, buf_info *info)
1022 {
1023     setVarInt(&info->bufP, info->bufEnd, xattr->key.keyLen);
1024     setVarInt(&info->bufP, info->bufEnd, xattr->valueLen);
1025 
1026     if ( xattr->key.keyLen >= 1 && info->bufP + xattr->key.keyLen <= info->bufEnd ) {
1027         memcpy(info->bufP, xattr->key.key, xattr->key.keyLen);
1028         if ( info->bufP[xattr->key.keyLen - 1] != 0x0 ) {
1029             info->bufP[xattr->key.keyLen - 1] = 0x0;
1030             bpc_logMsgf("bpc_attrib_xattrWrite: BOTCH: truncated xattr name '%s' to match keyLen %u\n", info->bufP, xattr->key.keyLen);
1031         }
1032     }
1033     info->bufP += xattr->key.keyLen;
1034 
1035     if ( info->bufP + xattr->valueLen <= info->bufEnd ) {
1036         memcpy(info->bufP, xattr->value, xattr->valueLen);
1037     }
1038     info->bufP += xattr->valueLen;
1039     info->numEntries++;
1040 }
1041 
1042 /*
1043  * Write a file structure to the memory buffer.  Returns the next unused buffer
1044  * pointer.  If the buffer is exhausted, no data is written past the buffer end,
1045  * Therefore, if the return value is greater than bufEnd, then the conversion
1046  * failed to fit.  The routine can be called again with at least (bufP - buf)
1047  * bytes allocated.
1048  */
bpc_attrib_file2buf(bpc_attrib_file * file,uchar * buf,uchar * bufEnd)1049 uchar *bpc_attrib_file2buf(bpc_attrib_file *file, uchar *buf, uchar *bufEnd)
1050 {
1051     uchar *bufP = buf;
1052     size_t fileNameLen = strlen(file->name);
1053     uint xattrEntryCnt = bpc_hashtable_entryCount(&file->xattrHT);
1054     buf_info info;
1055 
1056     setVarInt(&bufP, bufEnd, fileNameLen);
1057     if ( bufP + fileNameLen < bufEnd ) {
1058         memcpy(bufP, file->name, fileNameLen);
1059     }
1060     bufP += fileNameLen;
1061 
1062     setVarInt(&bufP, bufEnd, xattrEntryCnt);
1063     setVarInt(&bufP, bufEnd, file->type);
1064     setVarInt(&bufP, bufEnd, file->mtime);
1065     setVarInt(&bufP, bufEnd, file->mode);
1066     setVarInt(&bufP, bufEnd, file->uid);
1067     setVarInt(&bufP, bufEnd, file->gid);
1068     setVarInt(&bufP, bufEnd, file->size);
1069     setVarInt(&bufP, bufEnd, file->inode);
1070     setVarInt(&bufP, bufEnd, file->compress);
1071     setVarInt(&bufP, bufEnd, file->nlinks);
1072     setVarInt(&bufP, bufEnd, file->digest.len);
1073 
1074     if ( bufP + file->digest.len <= bufEnd ) {
1075         memcpy(bufP, file->digest.digest, file->digest.len);
1076     }
1077     bufP += file->digest.len;
1078 
1079     info.bufEnd     = bufEnd;
1080     info.bufP       = bufP;
1081     info.numEntries = 0;
1082     bpc_hashtable_iterate(&file->xattrHT, (void*)bpc_attrib_xattrWrite, &info);
1083     if ( info.numEntries != xattrEntryCnt ) {
1084         bpc_logErrf("bpc_attrib_file2buf: BOTCH: wrote %u xattr entries vs %u; attrib file corrupted\n", info.numEntries, xattrEntryCnt);
1085     }
1086     return info.bufP;
1087 }
1088 
bpc_attrib_fileWrite(bpc_attrib_file * file,write_info * info)1089 static void bpc_attrib_fileWrite(bpc_attrib_file *file, write_info *info)
1090 {
1091     uchar *bufP;
1092 
1093     if ( file->isTemp ) {
1094         if ( BPC_LogLevel >= 6 ) bpc_logMsgf("Skipping temp file %s: type = %d, mode = 0%o, uid/gid = %lu/%lu, size = %lu, inode = %lu, nlinks = %d, digest = 0x%02x%02x%02x..., bufUsed = %lu\n",
1095                 file->name, file->type, file->mode,
1096                 (unsigned long)file->uid, (unsigned long)file->gid,
1097                 (unsigned long)file->size, (unsigned long)file->inode, file->nlinks,
1098                 file->digest.digest[0], file->digest.digest[1], file->digest.digest[2],
1099                 (unsigned long)(info->bufP - info->buf));
1100         return;
1101     }
1102     bufP = bpc_attrib_file2buf(file, info->bufP, info->buf + sizeof(info->buf));
1103     if ( BPC_LogLevel >= 6 ) bpc_logMsgf("Wrote file %s: type = %d, mode = 0%o, uid/gid = %lu/%lu, size = %lu, inode = %lu, nlinks = %d, digest = 0x%02x%02x%02x..., bufUsed = %lu\n",
1104                 file->name, file->type, file->mode,
1105                 (unsigned long)file->uid, (unsigned long)file->gid,
1106                 (unsigned long)file->size, (unsigned long)file->inode, file->nlinks,
1107                 file->digest.digest[0], file->digest.digest[1], file->digest.digest[2],
1108                 (unsigned long)(info->bufP - info->buf));
1109 
1110     if ( bufP <= info->buf + sizeof(info->buf) ) {
1111         /*
1112          * it fit into the buffer
1113          */
1114         info->bufP = bufP;
1115         return;
1116     }
1117     /*
1118      * we overflowed the buffer - flush and try again
1119      */
1120     write_file_flush(info);
1121     bufP = bpc_attrib_file2buf(file, info->bufP, info->buf + sizeof(info->buf));
1122     if ( bufP <= info->buf + sizeof(info->buf) ) {
1123         info->bufP = bufP;
1124         return;
1125     }
1126     bpc_logErrf("bpc_attrib_fileWrite: BOTCH: can't fit file into buffer (%ld, %ld)\n", bufP - info->buf, sizeof(info->buf));
1127 }
1128 
1129 /*
1130  * Pre 0.50 attribute writing.  Writes a small file that contains the file hash of the attrib file
1131  */
bpc_attrib_dirWriteOld(bpc_deltaCount_info * deltaInfo,bpc_attrib_dir * dir,char * dirPath,char * attribFileName,bpc_digest * oldDigest)1132 static int bpc_attrib_dirWriteOld(bpc_deltaCount_info *deltaInfo, bpc_attrib_dir *dir,
1133                                   char *dirPath, char *attribFileName, bpc_digest *oldDigest)
1134 {
1135     char attribPath[BPC_MAXPATHLEN], attribPathTemp[BPC_MAXPATHLEN];
1136     bpc_fileZIO_fd fd;
1137     bpc_digest digest;
1138     int status;
1139     OFF_T poolFileSize;
1140     int errorCnt;
1141     static write_info info;
1142     char *p;
1143 
1144     bpc_attrib_attribFilePath(attribPath, dirPath, attribFileName);
1145     if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_attrib_dirWriteOld(%s)\n", attribPath);
1146     snprintf(attribPathTemp, BPC_MAXPATHLEN, "%s.%d", attribPath, getpid());
1147     if ( (p = strrchr(attribPathTemp, '/')) ) {
1148         *p = '\0';
1149         if ( bpc_path_create(attribPathTemp) ) return -1;
1150         *p = '/';
1151     }
1152 
1153     if ( bpc_hashtable_entryCount(&dir->filesHT) == 0 ) {
1154         int fdNum;
1155         /*
1156          * Empty directory - we just generate an empty attrib file, which we don't pool
1157          */
1158         if ( (fdNum = open(attribPathTemp, O_WRONLY | O_CREAT | O_TRUNC, 0660)) < 0 ) {
1159             bpc_logErrf("bpc_attrib_dirWrite: can't open/create raw %s for writing\n", attribPathTemp);
1160             return -1;
1161         }
1162         close(fdNum);
1163         if ( rename(attribPathTemp, attribPath) ) {
1164             bpc_logErrf("bpc_attrib_dirWrite: rename from %s to %s failed\n", attribPathTemp, attribPath);
1165             return -1;
1166         }
1167         if ( oldDigest ) bpc_poolRefDeltaUpdate(deltaInfo, dir->compress, oldDigest, -1);
1168 	dir->digest.len = 0;
1169         return 0;
1170     }
1171 
1172     info.bufP = info.buf;
1173     CONV_UINT32_TO_BUF(info.bufP, BPC_ATTRIB_TYPE_XATTR);
1174 
1175     bpc_poolWrite_open(&info.fd, dir->compress, NULL);
1176     bpc_hashtable_iterate(&dir->filesHT, (void*)bpc_attrib_fileWrite, &info);
1177     write_file_flush(&info);
1178     bpc_poolWrite_close(&info.fd, &status, &digest, &poolFileSize, &errorCnt);
1179 
1180     if ( errorCnt ) return -1;
1181 
1182     /*
1183      * Now write the small atttib file, which just contains a magic number and the digest
1184      */
1185     if ( bpc_fileZIO_open(&fd, attribPathTemp, 1, dir->compress) ) {
1186         bpc_logErrf("bpc_attrib_dirWrite: can't open/create %s for writing\n", attribPathTemp);
1187         return -1;
1188     }
1189     info.bufP = info.buf;
1190     CONV_UINT32_TO_BUF(info.bufP, BPC_ATTRIB_TYPE_DIGEST);
1191     if ( digest.len > 0 ) {
1192         memcpy(info.bufP, digest.digest, digest.len);
1193         info.bufP += digest.len;
1194     }
1195     if ( bpc_fileZIO_write(&fd, info.buf, info.bufP - info.buf) < 0 ) {
1196         bpc_logErrf("bpc_attrib_dirWrite: can't write to %s\n", attribPathTemp);
1197         bpc_fileZIO_close(&fd);
1198         return -1;
1199     }
1200     bpc_fileZIO_close(&fd);
1201     if ( rename(attribPathTemp, attribPath) ) {
1202         bpc_logErrf("bpc_attrib_dirWrite: rename from %s to %s failed\n", attribPathTemp, attribPath);
1203         return -1;
1204     }
1205     if ( BPC_LogLevel >= 8 ) bpc_logMsgf("bpc_attrib_dirWrite: new attrib digest = 0x%02x%02x%02x..., oldDigest = 0x%02x%02x...\n",
1206                 digest.digest[0], digest.digest[1], digest.digest[2],
1207                 oldDigest ? oldDigest->digest[0] : 0x0, oldDigest ? oldDigest->digest[1] : 0x0);
1208 
1209     if ( oldDigest ) bpc_poolRefDeltaUpdate(deltaInfo, dir->compress, oldDigest, -1);
1210     bpc_poolRefDeltaUpdate(deltaInfo, dir->compress, &digest, 1);
1211 
1212     /*
1213      * update with the new digest
1214      */
1215     memcpy(&dir->digest, &digest, sizeof(digest));
1216 
1217     return 0;
1218 }
1219 
bpc_attrib_dirWrite(bpc_deltaCount_info * deltaInfo,bpc_attrib_dir * dir,char * dirPath,char * attribFileName,bpc_digest * oldDigest)1220 int bpc_attrib_dirWrite(bpc_deltaCount_info *deltaInfo, bpc_attrib_dir *dir, char *dirPath, char *attribFileName, bpc_digest *oldDigest)
1221 {
1222     char attribPath[BPC_MAXPATHLEN], attribPathTemp[BPC_MAXPATHLEN], *baseAttribFileName;
1223     bpc_digest digest;
1224     int status;
1225     OFF_T poolFileSize;
1226     int errorCnt;
1227     static write_info info;
1228     char *p;
1229     int fdNum;
1230     size_t attribPathLen, baseAttribFileNameLen;
1231     int digestChanged;
1232 
1233     if ( WriteOldStyleAttribFile ) return bpc_attrib_dirWriteOld(deltaInfo, dir, dirPath, attribFileName, oldDigest);
1234 
1235     /*
1236      * baseAttribFileName points to the last portion of attribFileName, or the whole
1237      * string if it doesn't contain '/'
1238      */
1239     if ( (baseAttribFileName = strrchr(attribFileName, '/')) ) {
1240         baseAttribFileName++;
1241     } else {
1242         baseAttribFileName = attribFileName;
1243     }
1244     baseAttribFileNameLen = strlen(baseAttribFileName);
1245 
1246     bpc_attrib_attribFilePath(attribPath, dirPath, attribFileName);
1247     if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_attrib_dirWrite(%s): dirPath = %s, attribFileName = %s, baseAttribFileName = %s\n",
1248                                           attribPath, dirPath, attribFileName, baseAttribFileName);
1249     snprintf(attribPathTemp, BPC_MAXPATHLEN, "%s.%d", attribPath, getpid());
1250     if ( (p = strrchr(attribPathTemp, '/')) ) {
1251         *p = '\0';
1252         if ( bpc_path_create(attribPathTemp) ) return -1;
1253         *p = '/';
1254     }
1255     attribPathLen = strlen(attribPath);
1256 
1257     if ( bpc_hashtable_entryCount(&dir->filesHT) > 0 ) {
1258         /*
1259          * Write the attribute file to the pool
1260          */
1261         info.bufP = info.buf;
1262         CONV_UINT32_TO_BUF(info.bufP, BPC_ATTRIB_TYPE_XATTR);
1263 
1264         bpc_poolWrite_open(&info.fd, dir->compress, NULL);
1265         bpc_hashtable_iterate(&dir->filesHT, (void*)bpc_attrib_fileWrite, &info);
1266         write_file_flush(&info);
1267         bpc_poolWrite_close(&info.fd, &status, &digest, &poolFileSize, &errorCnt);
1268 
1269         if ( errorCnt ) return -1;
1270 
1271         /*
1272          * Starting in 0.50, the attrib file is always empty (so it takes no extra blocks)
1273          * and we simply include the digest in the file name by appending the hex digits.
1274          *
1275          * An empty attrib file is simply "attrib_0", and a legacy attrib file (pre <0.50)
1276          * is "attrib".
1277          */
1278         if ( attribPathLen + digest.len * 2 + 2 >= sizeof(attribPath) ) {
1279             bpc_logErrf("bpc_attrib_dirWrite: path too long (%d, %d, %d)\n", strlen(attribPath), digest.len, sizeof(attribPath));
1280             return -1;
1281         }
1282         attribPath[attribPathLen++] = '_';
1283         bpc_digest_digest2str(&digest, attribPath + attribPathLen);
1284         /*
1285          * Now create an empty attrib file with the file name digest
1286          */
1287         if ( (fdNum = open(attribPathTemp, O_WRONLY | O_CREAT | O_TRUNC, 0660)) < 0 ) {
1288             bpc_logErrf("bpc_attrib_dirWrite: can't open/create raw %s for writing\n", attribPathTemp);
1289             return -1;
1290         }
1291         close(fdNum);
1292         if ( rename(attribPathTemp, attribPath) ) {
1293             bpc_logErrf("bpc_attrib_dirWrite: rename from %s to %s failed\n", attribPathTemp, attribPath);
1294             return -1;
1295         }
1296         if ( BPC_LogLevel >= 5 ) bpc_logMsgf("bpc_attrib_dirWrite: created new attrib file %s\n", attribPath);
1297     } else {
1298         memset(&digest, 0, sizeof(digest));
1299         attribPath[attribPathLen++] = '_';
1300         strcpy(attribPath + attribPathLen, "0");
1301         if ( BPC_LogLevel >= 5 ) bpc_logMsgf("bpc_attrib_dirWrite: skipping creating new empty attrib file %s\n", attribPath);
1302         unlink(attribPath);
1303     }
1304 
1305     if ( BPC_LogLevel >= 8 ) bpc_logMsgf("bpc_attrib_dirWrite: new attrib digest = 0x%02x%02x%02x..., oldDigest = 0x%02x%02x...\n",
1306                 digest.digest[0], digest.digest[1], digest.digest[2],
1307                 oldDigest ? oldDigest->digest[0] : 0x0, oldDigest ? oldDigest->digest[1] : 0x0);
1308 
1309     digestChanged = !oldDigest || bpc_digest_compare(&digest, oldDigest);
1310     if ( digestChanged && digest.len > 0 ) {
1311         bpc_poolRefDeltaUpdate(deltaInfo, dir->compress, &digest, 1);
1312     }
1313 
1314     if ( oldDigest ) {
1315         if ( !digestChanged ) {
1316             if ( BPC_LogLevel >= 4 ) bpc_logMsgf("bpc_attrib_dirWrite: old attrib has same digest; no changes to ref counts\n");
1317             return 0;
1318         }
1319         if ( attribPathLen + oldDigest->len * 2 + 2 >= sizeof(attribPath) ) {
1320             bpc_logErrf("bpc_attrib_dirWrite: oldDigest path too long (%d, %d, %d)\n", strlen(attribPath), oldDigest->len, sizeof(attribPath));
1321             return -1;
1322         }
1323         strcpy(attribPathTemp, attribPath);
1324         if ( oldDigest->len > 0 ) {
1325             bpc_poolRefDeltaUpdate(deltaInfo, dir->compress, oldDigest, -1);
1326             bpc_digest_digest2str(oldDigest, attribPathTemp + attribPathLen);
1327         } else {
1328             strcpy(attribPathTemp + attribPathLen, "0");
1329         }
1330         if ( !unlink(attribPathTemp) ) {
1331             if ( BPC_LogLevel >= 5 ) bpc_logMsgf("bpc_attrib_dirWrite: removed old attrib file %s\n", attribPathTemp);
1332         } else {
1333             DIR *dirOs;
1334             struct dirent *dp;
1335             char deletePath[BPC_MAXPATHLEN];
1336 
1337             /*
1338              * Scan the directory and remove any other attribute files that have the
1339              * same root.
1340              */
1341             if ( !(p = strrchr(attribPath, '/')) ) {
1342                 bpc_logErrf("bpc_attrib_dirWrite: can't find a '/' in %s\n", attribPath);
1343                 return -1;
1344             }
1345             *p++ = '\0';
1346             if ( !(dirOs = opendir(attribPath)) ) {
1347                 bpc_logErrf("bpc_attrib_dirWrite: can't opendir %s\n", attribPath);
1348                 return -1;
1349             }
1350             while ( (dp = readdir(dirOs)) ) {
1351                 if ( strncmp(dp->d_name, baseAttribFileName, baseAttribFileNameLen) || !strcmp(dp->d_name, p) ) continue;
1352                 snprintf(deletePath, sizeof(deletePath), "%s/%s", attribPath, dp->d_name);
1353                 unlink(deletePath);
1354                 if ( BPC_LogLevel >= 5 ) bpc_logMsgf("bpc_attrib_dirWrite: removed other old attrib file %s\n", deletePath);
1355             }
1356             closedir(dirOs);
1357         }
1358     }
1359 
1360     /*
1361      * update with the new digest
1362      */
1363     memcpy(&dir->digest, &digest, sizeof(digest));
1364 
1365     return 0;
1366 }
1367