1 /*
2 * Routines for read/writing/managing file attributes
3 *
4 * Copyright (C) 2013 Craig Barratt.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 3 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along
17 * with this program; if not, visit the http://fsf.org website.
18 */
19
20 #include "backuppc.h"
21
22 /*
23 * Type of attribute file. This is saved as a magic number at the
24 * start of the file. This type is for V3 and earlier.
25 */
26 #define BPC_ATTRIB_TYPE_UNIX (0x17555555)
27
28 /*
29 * super set of UNIX, including extended attribs and digest, for 4.x+
30 */
31 #define BPC_ATTRIB_TYPE_XATTR (0x17565353)
32
33 /*
34 * starting in 4.x, attrib files in the pc backup tree are
35 * just digests that give the location of the real attrib file
36 * in the pool. attrib files written in the pc backup tree
37 * start with this magic number, followed by the digest.
38 */
39 #define BPC_ATTRIB_TYPE_DIGEST (0x17585451)
40
41 static char *FileType2Text[] = {
42 "file",
43 "hardlink",
44 "symlink",
45 "chardev",
46 "blockdev",
47 "dir",
48 "fifo",
49 "?",
50 "socket",
51 "?",
52 "deleted",
53 };
54
55 /*
56 * Ensure by default we use old-style "attrib" files for <= 4.0.0alpha3 compatibility.
57 * Newer versions of BackupPC turn this off to use new attribute file naming.
58 */
59 static int WriteOldStyleAttribFile = 1;
60 static int KeepOldAttribFiles = 1;
61
bpc_attrib_backwardCompat(int writeOldStyleAttribFile,int keepOldAttribFiles)62 void bpc_attrib_backwardCompat(int writeOldStyleAttribFile, int keepOldAttribFiles)
63 {
64 if ( writeOldStyleAttribFile >= 0 ) WriteOldStyleAttribFile = writeOldStyleAttribFile;
65 if ( keepOldAttribFiles >= 0 ) KeepOldAttribFiles = keepOldAttribFiles;
66 if ( BPC_LogLevel >= 5 ) {
67 bpc_logMsgf("bpc_attrib_backwardCompat: WriteOldStyleAttribFile = %d, KeepOldAttribFiles = %d\n",
68 WriteOldStyleAttribFile, KeepOldAttribFiles);
69 }
70 }
71
72
73 #define CONV_BUF_TO_UINT32(buf) ((buf)[0] << 24 | (buf)[1] << 16 | (buf)[2] << 8 | (buf)[3])
74
75 #define CONV_UINT32_TO_BUF(buf, val) { *(buf)++ = ((val) >> 24) & 0xff; \
76 *(buf)++ = ((val) >> 16) & 0xff; \
77 *(buf)++ = ((val) >> 8) & 0xff; \
78 *(buf)++ = ((val) >> 0) & 0xff; }
79
80 /*
81 * Note on xattr keys: they are treated as opaque strings of bytes, and the convention
82 * is to include the '\0' byte termination in the keyLen (ie: it is strlen(key) + 1).
83 */
bpc_attrib_xattrGet(bpc_attrib_file * file,void * key,int keyLen,int allocate_if_missing)84 bpc_attrib_xattr *bpc_attrib_xattrGet(bpc_attrib_file *file, void *key, int keyLen, int allocate_if_missing)
85 {
86 return (bpc_attrib_xattr*)bpc_hashtable_find(&file->xattrHT, key, keyLen, allocate_if_missing);
87 }
88
bpc_attrib_xattrDestroy(bpc_attrib_xattr * xattr)89 void bpc_attrib_xattrDestroy(bpc_attrib_xattr *xattr)
90 {
91 if ( xattr->key.key ) free(xattr->key.key);
92 if ( xattr->value ) free(xattr->value);
93 }
94
bpc_attrib_xattrDelete(bpc_attrib_file * file,void * key,int keyLen)95 int bpc_attrib_xattrDelete(bpc_attrib_file *file, void *key, int keyLen)
96 {
97 bpc_attrib_xattr *xattr = bpc_hashtable_find(&file->xattrHT, key, keyLen, 0);
98
99 if ( !xattr ) return -1;
100 bpc_attrib_xattrDestroy(xattr);
101 bpc_hashtable_nodeDelete(&file->xattrHT, xattr);
102 return 0;
103 }
104
bpc_attrib_xattrDeleteNode(bpc_attrib_xattr * xattr,bpc_attrib_file * file)105 static void bpc_attrib_xattrDeleteNode(bpc_attrib_xattr *xattr, bpc_attrib_file *file)
106 {
107 bpc_attrib_xattrDestroy(xattr);
108 bpc_hashtable_nodeDelete(&file->xattrHT, xattr);
109 }
110
bpc_attrib_xattrDeleteAll(bpc_attrib_file * file)111 int bpc_attrib_xattrDeleteAll(bpc_attrib_file *file)
112 {
113 bpc_hashtable_iterate(&file->xattrHT, (void*)bpc_attrib_xattrDeleteNode, file);
114 return 0;
115 }
116
117 /*
118 * returns >0 if the new value is the same as the current value.
119 * returns 0 if the new value was set correctly
120 * returns <0 on error.
121 */
bpc_attrib_xattrSetValue(bpc_attrib_file * file,void * key,int keyLen,void * value,uint32 valueLen)122 int bpc_attrib_xattrSetValue(bpc_attrib_file *file, void *key, int keyLen, void *value, uint32 valueLen)
123 {
124 bpc_attrib_xattr *xattr;
125 char keyCopy[BPC_MAXPATHLEN];
126
127 if ( ((char*)key)[keyLen - 1] != 0x0 ) {
128 if ( keyLen >= BPC_MAXPATHLEN - 8 ) {
129 bpc_logMsgf("bpc_attrib_xattrSetValue: BOTCH: key not 0x0 terminated; too long to repair (keyLen %u)\n", keyLen);
130 return -1;
131 } else {
132 memcpy(keyCopy, key, keyLen);
133 keyCopy[keyLen++] = 0x0;
134 key = keyCopy;
135 if ( BPC_LogLevel >= 6 ) {
136 bpc_logMsgf("bpc_attrib_xattrSetValue: fixup: appended 0x0 to xattr name '%s' (keyLen now %u)\n", (char*)key, keyLen);
137 }
138 }
139 }
140 xattr = bpc_attrib_xattrGet(file, key, keyLen, 1);
141
142 if ( !xattr->value ) {
143 /*
144 * new entry
145 */
146 if ( !(xattr->key.key = malloc(keyLen)) ) {
147 bpc_logErrf("bpc_attrib_xattrSetValue: can't allocate %d bytes for key\n", keyLen);
148 return -1;
149 }
150 memcpy(xattr->key.key, key, keyLen);
151 xattr->key.keyLen = keyLen;
152 } else {
153 /*
154 * existing entry - no need to recopy key. If value array isn't big enough then create another.
155 */
156 if ( valueLen > xattr->valueLen ) {
157 free(xattr->value);
158 xattr->value = NULL;
159 } else if ( valueLen == xattr->valueLen && !memcmp(xattr->value, value, valueLen) ) {
160 /*
161 * same value - no change
162 */
163 return 1;
164 }
165 }
166 if ( !xattr->value && !(xattr->value = malloc(valueLen)) ) {
167 bpc_logErrf("bpc_attrib_xattrSetValue: can't allocate %d bytes for value\n", valueLen);
168 return -1;
169 }
170 memcpy(xattr->value, value, valueLen);
171 xattr->valueLen = valueLen;
172 return 0;
173 }
174
bpc_attrib_xattrCopy(bpc_attrib_xattr * xattrSrc,bpc_attrib_file * fileDest)175 void bpc_attrib_xattrCopy(bpc_attrib_xattr *xattrSrc, bpc_attrib_file *fileDest)
176 {
177 bpc_attrib_xattr *xattr;
178 uchar *key = (uchar*)malloc(xattrSrc->key.keyLen > 0 ? xattrSrc->key.keyLen : 1);
179 uchar *value = (uchar*)malloc(xattrSrc->valueLen > 0 ? xattrSrc->valueLen : 1);
180
181 if ( !key || !value ) {
182 bpc_logErrf("bpc_attrib_xattrCopy: can't allocate %d,%d bytes\n", xattrSrc->key.keyLen + 1, xattrSrc->valueLen + 1);
183 return;
184 }
185
186 memcpy(key, xattrSrc->key.key, xattrSrc->key.keyLen);
187 memcpy(value, xattrSrc->value, xattrSrc->valueLen);
188
189 xattr = bpc_attrib_xattrGet(fileDest, key, xattrSrc->key.keyLen, 1);
190
191 if ( xattr->value ) {
192 /*
193 * Shouldn't be present, but if so clear it out and write the new key
194 */
195 bpc_attrib_xattrDestroy(xattr);
196 xattr->key.key = key;
197 xattr->key.keyLen = xattrSrc->key.keyLen;
198 }
199 xattr->value = value;
200 xattr->valueLen = xattrSrc->valueLen;
201 }
202
bpc_attrib_xattrCount(bpc_attrib_file * file)203 int bpc_attrib_xattrCount(bpc_attrib_file *file)
204 {
205 return bpc_hashtable_entryCount(&file->xattrHT);
206 }
207
208 typedef struct {
209 char *list;
210 ssize_t idx;
211 ssize_t listLen;
212 int ignoreRsyncACLs;
213 } xattrList_info;
214
bpc_attrib_xattrListKey(bpc_attrib_xattr * xattr,xattrList_info * info)215 static void bpc_attrib_xattrListKey(bpc_attrib_xattr *xattr, xattrList_info *info)
216 {
217 if ( info->idx < 0 ) return;
218
219 if ( info->ignoreRsyncACLs ) {
220 static struct {
221 char *str;
222 unsigned int len;
223 } ignoreKeys[] = {
224 { "user.rsync.%aacl", sizeof("user.rsync.%aacl"), }, /* note: sizeof() includes the \0 terminator */
225 { "user.rsync.%dacl", sizeof("user.rsync.%dacl"), },
226 };
227 uint i;
228
229 for ( i = 0 ; i < sizeof(ignoreKeys) / sizeof(ignoreKeys[0]) ; i++ ) {
230 if ( xattr->key.keyLen == ignoreKeys[i].len
231 && !memcmp(xattr->key.key, ignoreKeys[i].str, xattr->key.keyLen) ) {
232 return;
233 }
234 }
235 }
236 if ( info->list ) {
237 if ( info->idx + (signed)xattr->key.keyLen > info->listLen ) {
238 info->idx = -1;
239 return;
240 }
241 /*
242 * confirm that keyLen includes the \0 terminating byte
243 */
244 memcpy(info->list + info->idx, xattr->key.key, xattr->key.keyLen);
245 if ( xattr->key.keyLen >= 1 && info->list[info->idx + xattr->key.keyLen - 1] != 0x0 ) {
246 info->list[info->idx + xattr->key.keyLen - 1] = 0x0;
247 bpc_logMsgf("bpc_attrib_xattrListKey: BOTCH: truncated xattr name '%s' to match keyLen %u\n", info->list + info->idx, xattr->key.keyLen);
248 }
249 if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_attrib_xattrListKey: adding %s\n", info->list + info->idx);
250 info->idx += xattr->key.keyLen;
251 } else {
252 info->idx += xattr->key.keyLen;
253 }
254 }
255
256 /*
257 * Concatenate all the xattr keys, (which the caller has ensured are \0 terminated),
258 * into a single string. Return the number of bytes in the output string.
259 * Returns -1 if listLen is too short to fit all the keys.
260 * If list is NULL, instead returns the number of bytes required to store all the keys.
261 */
bpc_attrib_xattrList(bpc_attrib_file * file,char * list,size_t listLen,int ignoreRsyncACLs)262 size_t bpc_attrib_xattrList(bpc_attrib_file *file, char *list, size_t listLen, int ignoreRsyncACLs)
263 {
264 xattrList_info info;
265
266 info.list = list;
267 info.idx = 0;
268 info.listLen = listLen;
269 info.ignoreRsyncACLs = ignoreRsyncACLs;
270
271 bpc_hashtable_iterate(&file->xattrHT, (void*)bpc_attrib_xattrListKey, &info);
272 return info.idx;
273 }
274
bpc_attrib_fileDestroy(bpc_attrib_file * file)275 void bpc_attrib_fileDestroy(bpc_attrib_file *file)
276 {
277 if ( file->name) free(file->name);
278 bpc_hashtable_iterate(&file->xattrHT, (void*)bpc_attrib_xattrDestroy, NULL);
279 bpc_hashtable_destroy(&file->xattrHT);
280 }
281
282 /*
283 * Return the attributes for the given file.
284 * If allocate_if_missing == 0 and not present, then NULL is returned.
285 * If allocate_if_missing != 0 and not present, then an empty struct is returned with the key filled in,
286 * and file->name is NULL.
287 */
bpc_attrib_fileGet(bpc_attrib_dir * dir,char * fileName,int allocate_if_missing)288 bpc_attrib_file *bpc_attrib_fileGet(bpc_attrib_dir *dir, char *fileName, int allocate_if_missing)
289 {
290 return bpc_hashtable_find(&dir->filesHT, (uchar*)fileName, strlen(fileName), allocate_if_missing);
291 }
292
293 /*
294 * Initialize an empty file structure (ie: one returned by bpc_attrib_fileGet() that is empty)
295 */
bpc_attrib_fileInit(bpc_attrib_file * file,char * fileName,int xattrNumEntries)296 void bpc_attrib_fileInit(bpc_attrib_file *file, char *fileName, int xattrNumEntries)
297 {
298 int fileNameLen = strlen(fileName);
299
300 if ( file->name ) bpc_attrib_fileDestroy(file);
301 file->name = (char*)malloc(fileNameLen + 1);
302 if ( !file->name ) {
303 bpc_logErrf("bpc_attrib_fileInit: can't allocate %d bytes for file name\n", fileNameLen + 1);
304 return;
305 }
306 memcpy(file->name, fileName, fileNameLen + 1);
307 file->isTemp = 0;
308 file->key.key = file->name;
309 bpc_hashtable_create(&file->xattrHT, 16 + xattrNumEntries, sizeof(bpc_attrib_xattr));
310 }
311
312 /*
313 * Copy all the attributes from fileSrc to fileDest. fileDest should already have a
314 * valid allocated fileName and allocated xattr hash. The fileDest xattr hash is
315 * emptied before the copy, meaning it is over written.
316 *
317 * If overwriteEmptyDigest == 0, an empty digest in fileSrc will not overwrite fileDest.
318 */
bpc_attrib_fileCopyOpt(bpc_attrib_file * fileDest,bpc_attrib_file * fileSrc,int overwriteEmptyDigest)319 void bpc_attrib_fileCopyOpt(bpc_attrib_file *fileDest, bpc_attrib_file *fileSrc, int overwriteEmptyDigest)
320 {
321 if ( fileDest == fileSrc ) return;
322
323 fileDest->type = fileSrc->type;
324 fileDest->compress = fileSrc->compress;
325 fileDest->mode = fileSrc->mode;
326 fileDest->isTemp = fileSrc->isTemp;
327 fileDest->uid = fileSrc->uid;
328 fileDest->gid = fileSrc->gid;
329 fileDest->nlinks = fileSrc->nlinks;
330 fileDest->mtime = fileSrc->mtime;
331 fileDest->size = fileSrc->size;
332 fileDest->inode = fileSrc->inode;
333 fileDest->backupNum = fileSrc->backupNum;
334 if ( fileSrc->digest.len > 0 || overwriteEmptyDigest ) {
335 fileDest->digest = fileSrc->digest;
336 }
337 bpc_hashtable_iterate(&fileDest->xattrHT, (void*)bpc_attrib_xattrDestroy, NULL);
338 bpc_hashtable_erase(&fileDest->xattrHT);
339 bpc_hashtable_iterate(&fileSrc->xattrHT, (void*)bpc_attrib_xattrCopy, fileDest);
340 }
341
342 /*
343 * Copy all the attributes from fileSrc to fileDest. fileDest should already have a
344 * valid allocated fileName and allocated xattr hash. The fileDest xattr hash is
345 * emptied before the copy, meaning it is over written.
346 */
bpc_attrib_fileCopy(bpc_attrib_file * fileDest,bpc_attrib_file * fileSrc)347 void bpc_attrib_fileCopy(bpc_attrib_file *fileDest, bpc_attrib_file *fileSrc)
348 {
349 if ( fileDest == fileSrc ) return;
350
351 bpc_attrib_fileCopyOpt(fileDest, fileSrc, 1);
352 }
353
354 /*
355 * Check if two file attribute structures are the same. Returns 0 if they are the same.
356 */
bpc_attrib_fileCompare(bpc_attrib_file * file0,bpc_attrib_file * file1)357 int bpc_attrib_fileCompare(bpc_attrib_file *file0, bpc_attrib_file *file1)
358 {
359 uint idx = 0;
360
361 if ( file0->type != file1->type
362 || file0->compress != file1->compress
363 || file0->mode != file1->mode
364 || file0->uid != file1->uid
365 || file0->gid != file1->gid
366 || file0->nlinks != file1->nlinks
367 || file0->mtime != file1->mtime
368 || file0->size != file1->size
369 || file0->inode != file1->inode
370 || file0->digest.len != file1->digest.len
371 || memcmp(file0->digest.digest, file1->digest.digest, file0->digest.len)
372 || bpc_attrib_xattrCount(file0) != bpc_attrib_xattrCount(file1) ) {
373 if ( BPC_LogLevel >= 9 ) bpc_logMsgf("bpc_attrib_fileCompare: %s %s differ\n", file0->name, file1->name);
374 return 1;
375 }
376 while ( 1 ) {
377 bpc_attrib_xattr *xattr0 = bpc_hashtable_nextEntry(&file0->xattrHT, &idx), *xattr1;
378 if ( !xattr0 ) return 0;
379 if ( !(xattr1 = bpc_attrib_xattrGet(file1, xattr0->key.key, xattr0->key.keyLen, 0)) ) return 1;
380 if ( xattr0->valueLen != xattr1->valueLen || memcmp(xattr0->value, xattr1->value, xattr0->valueLen) ) return 1;
381 }
382 }
383
bpc_attrib_fileDeleteName(bpc_attrib_dir * dir,char * fileName)384 void bpc_attrib_fileDeleteName(bpc_attrib_dir *dir, char *fileName)
385 {
386 bpc_attrib_file *file = bpc_hashtable_find(&dir->filesHT, (uchar*)fileName, strlen(fileName), 0);
387
388 if ( !file ) return;
389 bpc_attrib_fileDestroy(file);
390 bpc_hashtable_nodeDelete(&dir->filesHT, file);
391 }
392
bpc_attrib_fileIterate(bpc_attrib_dir * dir,bpc_attrib_file ** file,uint * idx)393 int bpc_attrib_fileIterate(bpc_attrib_dir *dir, bpc_attrib_file **file, uint *idx)
394 {
395 *file = bpc_hashtable_nextEntry(&dir->filesHT, idx);
396 if ( !*file ) return -1;
397 return 0;
398 }
399
bpc_attrib_fileCount(bpc_attrib_dir * dir)400 int bpc_attrib_fileCount(bpc_attrib_dir *dir)
401 {
402 return bpc_hashtable_entryCount(&dir->filesHT);
403 }
404
bpc_attrib_fileType2Text(int type)405 char *bpc_attrib_fileType2Text(int type)
406 {
407 if ( type < 0 || type >= (int)(sizeof(FileType2Text) / sizeof(FileType2Text[0])) ) return "?";
408 return FileType2Text[type];
409 }
410
bpc_attrib_dirInit(bpc_attrib_dir * dir,int compressLevel)411 void bpc_attrib_dirInit(bpc_attrib_dir *dir, int compressLevel)
412 {
413 dir->digest.len = 0;
414 dir->compress = compressLevel;
415 dir->needRewrite = 0;
416 bpc_hashtable_create(&dir->filesHT, 512, sizeof(bpc_attrib_file));
417 }
418
bpc_attrib_dirDestroy(bpc_attrib_dir * dir)419 void bpc_attrib_dirDestroy(bpc_attrib_dir *dir)
420 {
421 bpc_hashtable_iterate(&dir->filesHT, (void*)bpc_attrib_fileDestroy, NULL);
422 bpc_hashtable_destroy(&dir->filesHT);
423 }
424
bpc_attrib_dirNeedRewrite(bpc_attrib_dir * dir)425 int bpc_attrib_dirNeedRewrite(bpc_attrib_dir *dir)
426 {
427 return dir->needRewrite;
428 }
429
430 typedef struct {
431 bpc_deltaCount_info *deltaInfo;
432 int incr;
433 unsigned int *inodeMax;
434 } fileRefCnt_info;
435
bpc_attrib_fileRefCount(bpc_attrib_file * file,fileRefCnt_info * info)436 static void bpc_attrib_fileRefCount(bpc_attrib_file *file, fileRefCnt_info *info)
437 {
438 if ( file->digest.len > 0 ) {
439 char hexStr[BPC_DIGEST_LEN_MAX * 2 + 1];
440 bpc_digest_digest2str(&file->digest, hexStr);
441 if ( BPC_LogLevel >= 7 ) bpc_logMsgf("bpc_attrib_fileRefCount: file %s digest %s delta %d\n", file->name, hexStr, info->incr);
442 bpc_poolRefDeltaUpdate(info->deltaInfo, file->compress, &file->digest, info->incr);
443 }
444 if ( info->inodeMax && file->inode > *info->inodeMax ) {
445 *info->inodeMax = file->inode;
446 }
447 }
448
449 /*
450 * call refDeltaUpdate with incr (typically +/-1) for every entry in the directory,
451 * as well as the dir itself.
452 */
bpc_attrib_dirRefCountInodeMax(bpc_deltaCount_info * deltaInfo,bpc_attrib_dir * dir,int incr,unsigned int * inodeMax)453 void bpc_attrib_dirRefCountInodeMax(bpc_deltaCount_info *deltaInfo, bpc_attrib_dir *dir, int incr, unsigned int *inodeMax)
454 {
455 fileRefCnt_info info;
456
457 info.deltaInfo = deltaInfo;
458 info.incr = incr;
459 info.inodeMax = inodeMax;
460 bpc_hashtable_iterate(&dir->filesHT, (void*)bpc_attrib_fileRefCount, &info);
461 if ( dir->digest.len > 0 ) {
462 char hexStr[BPC_DIGEST_LEN_MAX * 2 + 1];
463 bpc_digest_digest2str(&dir->digest, hexStr);
464 if ( BPC_LogLevel >= 7 ) bpc_logMsgf("bpc_attrib_dirRefCount: attrib digest %s delta = %d\n", hexStr, incr);
465 bpc_poolRefDeltaUpdate(deltaInfo, dir->compress, &dir->digest, incr);
466 } else {
467 if ( BPC_LogLevel >= 7 ) bpc_logMsgf("bpc_attrib_dirRefCount: no attrib digest -> no delta\n");
468 }
469 }
470
471 /*
472 * call refDeltaUpdate with incr (typically +/-1) for every entry in the directory,
473 * as well as the dir itself.
474 */
bpc_attrib_dirRefCount(bpc_deltaCount_info * deltaInfo,bpc_attrib_dir * dir,int incr)475 void bpc_attrib_dirRefCount(bpc_deltaCount_info *deltaInfo, bpc_attrib_dir *dir, int incr)
476 {
477 bpc_attrib_dirRefCountInodeMax(deltaInfo, dir, incr, NULL);
478 }
479
480 typedef struct {
481 char *entries;
482 ssize_t entryIdx;
483 ssize_t entrySize;
484 } dirEntry_info;
485
bpc_attrib_getDirEntry(bpc_attrib_file * file,dirEntry_info * info)486 static void bpc_attrib_getDirEntry(bpc_attrib_file *file, dirEntry_info *info)
487 {
488 ssize_t len = strlen(file->name) + 1;
489
490 if ( info->entryIdx < 0 ) return;
491 if ( info->entries ) {
492 if ( info->entryIdx + len > info->entrySize ) {
493 info->entryIdx = -1;
494 return;
495 }
496 memcpy(info->entries + info->entryIdx, file->name, len);
497 }
498 info->entryIdx += len;
499 }
500
bpc_attrib_getEntries(bpc_attrib_dir * dir,char * entries,ssize_t entrySize)501 ssize_t bpc_attrib_getEntries(bpc_attrib_dir *dir, char *entries, ssize_t entrySize)
502 {
503 dirEntry_info info;
504
505 info.entries = entries;
506 info.entryIdx = 0;
507 info.entrySize = entrySize;
508
509 bpc_hashtable_iterate(&dir->filesHT, (void*)bpc_attrib_getDirEntry, &info);
510 return info.entryIdx;
511 }
512
bpc_attrib_attribFilePath(char * path,char * dir,char * attribFileName)513 void bpc_attrib_attribFilePath(char *path, char *dir, char *attribFileName)
514 {
515 if ( !dir ) {
516 snprintf(path, BPC_MAXPATHLEN, "%s", attribFileName);
517 } else {
518 snprintf(path, BPC_MAXPATHLEN, "%s/%s", dir, attribFileName ? attribFileName : "attrib");
519 }
520 }
521
bpc_attrib_dirDigestGet(bpc_attrib_dir * dir)522 bpc_digest *bpc_attrib_dirDigestGet(bpc_attrib_dir *dir)
523 {
524 return &dir->digest;
525 }
526
read_more_data(bpc_fileZIO_fd * fd,uchar * buf,size_t bufSize,size_t * nRead,uchar ** bufPP,char * attribPath)527 static int read_more_data(bpc_fileZIO_fd *fd, uchar *buf, size_t bufSize, size_t *nRead, uchar **bufPP, char *attribPath)
528 {
529 int thisRead;
530 /*
531 * move the remaining part of the buffer down, and read more data
532 */
533 *nRead = (buf + *nRead) - *bufPP;
534 memmove(buf, *bufPP, *nRead);
535 *bufPP = buf;
536 thisRead = bpc_fileZIO_read(fd, buf + *nRead, bufSize - *nRead);
537 if ( thisRead < 0 ) {
538 bpc_logErrf("bpc_attrib_dirRead: can't read more bytes from %s\n", attribPath);
539 return -1;
540 }
541 *nRead += thisRead;
542 return 0;
543 }
544
545 /*
546 * Read variable-length unsigned integer in 7 bit chunks, LSB first.
547 */
getVarInt(uchar ** bufPP,uchar * bufEnd)548 static int64 getVarInt(uchar **bufPP, uchar *bufEnd)
549 {
550 int64 result = 0;
551 uchar *bufP = *bufPP;
552 int i = 0;
553
554 while ( bufP < bufEnd ) {
555 uchar c = *bufP++;
556 result |= ((int64)(c & 0x7f)) << i;
557 if ( !(c & 0x80) ) {
558 *bufPP = bufP;
559 return result;
560 }
561 i += 7;
562 }
563 /*
564 * we ran out of data... make sure bufP is greater than bufEnd, since
565 * returning it to be equal (ie: bufP) will be incorrectly interpreted as
566 * meaning the integer correctly ended right at the end of the buffer.
567 */
568 *bufPP = bufEnd + 1;
569 return result;
570 }
571
572 /*
573 * V3 variable length integer read, MSB first, which is compatible with perl pack("w")
574 */
getVarInt_v3(uchar ** bufPP,uchar * bufEnd)575 static int64 getVarInt_v3(uchar **bufPP, uchar *bufEnd)
576 {
577 int64 result = 0;
578 uchar *bufP = *bufPP;
579
580 while ( bufP < bufEnd ) {
581 uchar c = *bufP++;
582 result = (result << 7) | (c & 0x7f);
583 if ( !(c & 0x80) ) {
584 *bufPP = bufP;
585 return result;
586 }
587 }
588 /*
589 * we ran out of data... make sure bufP is greater than bufEnd, since
590 * returning it to be equal (ie: bufP) will be incorrectly interpreted as
591 * meaning the integer correctly ended right at the end of the buffer.
592 */
593 *bufPP = bufEnd + 1;
594 return result;
595 }
596
597 /*
598 * Write variable-length unsigned integer in 7 bit chunks, LSB first
599 */
setVarInt(uchar ** bufPP,uchar * bufEnd,int64 value)600 static void setVarInt(uchar **bufPP, uchar *bufEnd, int64 value)
601 {
602 uchar *bufP = *bufPP;
603 int maxBytes = (sizeof(value) * 8 + 6) / 7;
604
605 do {
606 uchar c = value & 0x7f;
607 value >>= 7;
608 maxBytes--;
609 if ( value && maxBytes > 0 ) c |= 0x80;
610 if ( bufP < bufEnd ) {
611 *bufP++ = c;
612 } else {
613 bufP++;
614 }
615 } while ( value && maxBytes > 0 );
616 *bufPP = bufP;
617 }
618
619 /*
620 * Unpack the data in buf[] into the file structure, after the file name and xattr entry
621 * count have been extracted. Returns next unused buffer location.
622 *
623 * If there isn't enough data to extract a complete file structure, the return value
624 * will be greater than bufEnd. You should gather more data and re-call the function.
625 */
bpc_attrib_buf2file(bpc_attrib_file * file,uchar * buf,uchar * bufEnd,int xattrNumEntries,int * xattrFixup)626 uchar *bpc_attrib_buf2file(bpc_attrib_file *file, uchar *buf, uchar *bufEnd, int xattrNumEntries, int *xattrFixup)
627 {
628 uchar *bufP = buf;
629 int i;
630
631 file->type = getVarInt(&bufP, bufEnd);
632 file->mtime = getVarInt(&bufP, bufEnd);
633 file->mode = getVarInt(&bufP, bufEnd);
634 file->uid = getVarInt(&bufP, bufEnd);
635 file->gid = getVarInt(&bufP, bufEnd);
636 file->size = getVarInt(&bufP, bufEnd);
637 file->inode = getVarInt(&bufP, bufEnd);
638 file->compress = getVarInt(&bufP, bufEnd);
639 file->nlinks = getVarInt(&bufP, bufEnd);
640 file->digest.len = getVarInt(&bufP, bufEnd);
641 file->isTemp = 0;
642
643 if ( file->digest.len > 0 && bufP + file->digest.len <= bufEnd ) {
644 memcpy(file->digest.digest, bufP, file->digest.len);
645 }
646 bufP += file->digest.len;
647
648 for ( i = 0 ; i < xattrNumEntries ; i++ ) {
649 uint keyLen = getVarInt(&bufP, bufEnd);
650 uint valueLen = getVarInt(&bufP, bufEnd);
651
652 if ( bufP + keyLen + valueLen <= bufEnd ) {
653 if ( xattrFixup && bufP[keyLen - 1] != 0x0 ) {
654 *xattrFixup = 1;
655 }
656 bpc_attrib_xattrSetValue(file, bufP, keyLen, bufP + keyLen, valueLen);
657 }
658 bufP += keyLen + valueLen;
659 }
660 return bufP;
661 }
662
663 /*
664 * Extract an entire packed file structure, starting with the fileName length varint.
665 * Returns next unused buffer location. It is assumed the file structure is already
666 * initialized and has a valid fileName allocated, so we don't allocate it here.
667 *
668 * If there isn't enough data to extract a complete file structure, the return value
669 * will be greater than bufEnd. You should gather more data and re-call the function.
670 * On certain errors, returns NULL;
671 */
bpc_attrib_buf2fileFull(bpc_attrib_file * file,uchar * bufP,uchar * bufEnd)672 uchar *bpc_attrib_buf2fileFull(bpc_attrib_file *file, uchar *bufP, uchar *bufEnd)
673 {
674 uint fileNameLen, xattrNumEntries;
675
676 fileNameLen = getVarInt(&bufP, bufEnd);
677 if ( fileNameLen > BPC_MAXPATHLEN - 1 ) {
678 bpc_logErrf("bpc_attrib_buf2fileFull: got unreasonable file name length %d\n", fileNameLen);
679 return NULL;
680 }
681 bufP += fileNameLen;
682 bpc_attrib_xattrDeleteAll(file);
683 xattrNumEntries = getVarInt(&bufP, bufEnd);
684 if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_attrib_buf2fileFull: xattrNumEntries = %d\n", xattrNumEntries);
685 bufP = bpc_attrib_buf2file(file, bufP, bufEnd, xattrNumEntries, NULL);
686 return bufP;
687 }
688
689 /*
690 * Read the attribute file at dirPath/attribFilePath and populate dir
691 */
bpc_attrib_dirRead(bpc_attrib_dir * dir,char * dirPath,char * attribFilePath,int backupNum)692 int bpc_attrib_dirRead(bpc_attrib_dir *dir, char *dirPath, char *attribFilePath, int backupNum)
693 {
694 char attribPath[BPC_MAXPATHLEN];
695 bpc_fileZIO_fd fd;
696 size_t nRead;
697 uint32 magic = 0;
698 uchar buf[8 * 65536], *bufP;
699 STRUCT_STAT st;
700 char *p, *attribFileName;
701
702 bpc_attrib_attribFilePath(attribPath, dirPath, attribFilePath);
703 dir->digest.len = 0;
704
705 /*
706 * attribFileName points to the last portion of attribFilePath, or the whole
707 * string if it doesn't contain '/'
708 */
709 if ( (attribFileName = strrchr(attribFilePath, '/')) ) {
710 attribFileName++;
711 } else {
712 attribFileName = attribFilePath;
713 }
714
715 if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_attrib_dirRead(%s); dirPath = %s, attribFilePath = %s, attribFileName = %s\n",
716 attribPath, dirPath, attribFilePath, attribFileName);
717
718 if ( (p = strchr(attribFileName, '_')) && !stat(attribPath, &st) && S_ISREG(st.st_mode) ) {
719 /*
720 * Explicit path name to new-style attrib file, and it exists; extract digest
721 */
722 if ( !strcmp(p + 1, "0") ) return 0;
723 bpc_digest_str2digest(&dir->digest, p + 1);
724 if ( BPC_LogLevel >= 6 ) {
725 char str[256];
726 bpc_digest_digest2str(&dir->digest, str);
727 bpc_logMsgf("bpc_attrib_dirRead: called with attrib file %s: digest = %s, len = %d\n",
728 attribPath, str, dir->digest.len);
729 }
730 /*
731 * Write new type attrib files (since we found a new-style one)
732 */
733 WriteOldStyleAttribFile = 0;
734 magic = BPC_ATTRIB_TYPE_XATTR;
735 } else if ( stat(attribPath, &st) || !S_ISREG(st.st_mode) || strchr(attribFileName, '_') ) {
736 DIR *dirOs;
737 struct dirent *dp;
738 int attribFileNameLen = strlen(attribFileName);
739 char attribDirPath[BPC_MAXPATHLEN];
740 /*
741 * Starting in 0.50, the attrib files are zero length with the digest encoded in
742 * the file name, so there is no file just called "attrib". Look in the directory
743 * to find it.
744 */
745 strcpy(attribDirPath, attribPath);
746 if ( (p = strrchr(attribDirPath, '/')) ) {
747 *p = '\0';
748 } else {
749 strcpy(attribDirPath, ".");
750 }
751 if ( !(dirOs = opendir(attribDirPath)) ) {
752 /*
753 * This is a benign error - just return as though there is an empty attrib file
754 */
755 if ( BPC_LogLevel >= 8 ) bpc_logMsgf("bpc_attrib_dirRead: can't opendir %s (note: this is ok)\n", attribDirPath);
756 return 0;
757 }
758 while ( (dp = readdir(dirOs)) ) {
759 if ( strncmp(dp->d_name, attribFileName, attribFileNameLen) ) continue;
760 p = dp->d_name + attribFileNameLen;
761 if ( p[0] != '_' ) continue;
762 p++;
763 if ( !strcmp(p, "0") ) {
764 /*
765 * An empty attrib file is legit; just return with no entries
766 */
767 if ( BPC_LogLevel >= 6 ) {
768 bpc_logMsgf("bpc_attrib_dirRead: Got empty attrib file %s\n", dp->d_name);
769 }
770 closedir(dirOs);
771 return 0;
772 }
773 bpc_digest_str2digest(&dir->digest, p);
774 if ( BPC_LogLevel >= 6 ) {
775 char str[256];
776 bpc_digest_digest2str(&dir->digest, str);
777 bpc_logMsgf("bpc_attrib_dirRead: Got attrib file %s: digest = %s, len = %d\n",
778 dp->d_name, str, dir->digest.len);
779 }
780 /*
781 * Write new type attrib files (since we found a new-style one)
782 */
783 WriteOldStyleAttribFile = 0;
784 break;
785 }
786 closedir(dirOs);
787 if ( dir->digest.len == 0 ) return 0;
788 magic = BPC_ATTRIB_TYPE_XATTR;
789 } else {
790 if ( bpc_fileZIO_open(&fd, attribPath, 0, dir->compress) ) {
791 bpc_logErrf("bpc_attrib_dirRead: can't open %s\n", attribPath);
792 return -1;
793 }
794 nRead = bpc_fileZIO_read(&fd, buf, sizeof(buf));
795 if ( nRead == 0 ) {
796 /*
797 * an empty file is legit - this means an empty directory (ie: zero attrib entries).
798 * indicate this with an empty digest and empty hash of entries.
799 */
800 bpc_fileZIO_close(&fd);
801 if ( !strcmp(attribFileName, "attrib") ) {
802 char attribPathTemp[BPC_MAXPATHLEN + 16];
803
804 strcpy(attribPathTemp, attribPath);
805 strcat(attribPathTemp, "_0");
806 if ( rename(attribPath, attribPathTemp) ) {
807 bpc_logErrf("bpc_attrib_dirRead: rename of empty attrib file from %s to %s failed\n", attribPath, attribPathTemp);
808 return -1;
809 } else if ( BPC_LogLevel >= 6 ) {
810 bpc_logMsgf("bpc_attrib_dirRead: renamed empty attrib file %s -> %s\n", attribPath, attribPathTemp);
811 }
812 }
813 return 0;
814 }
815 if ( nRead < 4 ) {
816 bpc_logErrf("bpc_attrib_dirRead: can't read at least 4 bytes from %s\n", attribPath);
817 bpc_fileZIO_close(&fd);
818 return -1;
819 }
820 magic = CONV_BUF_TO_UINT32(buf);
821 }
822
823 if ( magic == BPC_ATTRIB_TYPE_DIGEST ) {
824 char attribPathNew[BPC_MAXPATHLEN];
825 int fdNum;
826 size_t digestLen = nRead - 4, attribPathLen = strlen(attribPath);
827
828 if ( nRead < 20 ) {
829 bpc_logErrf("bpc_attrib_dirRead: can't read at least 20 bytes from %s\n", attribPath);
830 return -1;
831 }
832 bpc_fileZIO_close(&fd);
833 if ( digestLen > sizeof(dir->digest.digest) ) digestLen = sizeof(dir->digest.digest);
834 memcpy(dir->digest.digest, buf + 4, digestLen);
835 dir->digest.len = digestLen;
836
837 if ( !KeepOldAttribFiles ) {
838 /*
839 * replace the attrib file with a new-style attrib file where the digest is encoded
840 * in a zero length file name
841 */
842 if ( attribPathLen + dir->digest.len * 2 + 2 >= sizeof(attribPathNew) ) {
843 bpc_logErrf("bpc_attrib_dirRead: new digest path too long (%d, %d, %d)\n",
844 attribPathLen, dir->digest.len, sizeof(attribPathNew));
845 return -1;
846 }
847 strcpy(attribPathNew, attribPath);
848 attribPathNew[attribPathLen++] = '_';
849 bpc_digest_digest2str(&dir->digest, attribPathNew + attribPathLen);
850 if ( (fdNum = open(attribPathNew, O_WRONLY | O_CREAT | O_TRUNC, 0660)) < 0 ) {
851 bpc_logErrf("bpc_attrib_dirRead: can't open/create empty attrib file %s\n", attribPathNew);
852 return -1;
853 }
854 close(fdNum);
855 unlink(attribPath);
856 if ( BPC_LogLevel >= 4 ) bpc_logMsgf("bpc_attrib_dirRead: replaced %s with %s\n",
857 attribPath, attribPathNew);
858 }
859 }
860
861 if ( dir->digest.len > 0 ) {
862 /*
863 * Handle V4+ case - open the pool file directly
864 * For V3, digest.len == 0 since we opened the attrib file above (it is stored hardlinked in the backup
865 * directory; there is no digest)
866 */
867 bpc_digest_md52path(attribPath, dir->compress, &dir->digest);
868 if ( bpc_fileZIO_open(&fd, attribPath, 0, dir->compress) ) {
869 bpc_logErrf("bpc_attrib_dirRead: can't open %s\n", attribPath);
870 return -1;
871 }
872 nRead = bpc_fileZIO_read(&fd, buf, sizeof(buf));
873 if ( nRead < 4 ) {
874 bpc_logErrf("bpc_attrib_dirRead: can't read at least 4 bytes from %s\n", attribPath);
875 bpc_fileZIO_close(&fd);
876 return -1;
877 }
878 magic = CONV_BUF_TO_UINT32(buf);
879 }
880 bufP = buf + 4;
881
882 if ( magic == BPC_ATTRIB_TYPE_XATTR ) {
883 int retry = 0;
884 while ( bufP < buf + nRead ) {
885 uint fileNameLen, xattrNumEntries;
886 char *fileName;
887 bpc_attrib_file *file;
888 uchar *bufPsave = bufP;
889 int xattrFixup = 0;
890
891 if ( nRead == sizeof(buf) && bufP > buf + nRead - 2 * BPC_MAXPATHLEN
892 && read_more_data(&fd, buf, sizeof(buf), &nRead, &bufP, attribPath) ) {
893 bpc_fileZIO_close(&fd);
894 return -1;
895 }
896
897 fileNameLen = getVarInt(&bufP, buf + nRead);
898 if ( fileNameLen > BPC_MAXPATHLEN - 1 ) {
899 bpc_logErrf("bpc_attrib_dirRead: got unreasonable file name length %d\n", fileNameLen);
900 bpc_fileZIO_close(&fd);
901 return -1;
902 }
903
904 /*
905 * Save the fileName, but it's not NULL terminated yet.
906 * After we consume the next varint, we can safely NULL-terminate
907 * the fileName, which allows us to look up or create the file entry.
908 */
909 fileName = (char*)bufP;
910 bufP += fileNameLen;
911 xattrNumEntries = getVarInt(&bufP, buf + nRead);
912 fileName[fileNameLen] = '\0';
913
914 file = bpc_attrib_fileGet(dir, fileName, 1);
915 bpc_attrib_fileInit(file, fileName, xattrNumEntries);
916 file->backupNum = backupNum;
917
918 bufP = bpc_attrib_buf2file(file, bufP, buf + nRead, xattrNumEntries, &xattrFixup);
919 dir->needRewrite |= xattrFixup;
920 if ( bufP > buf + nRead ) {
921 /*
922 * Need to get more data and try again. We have allocated file->name,
923 * and perhaps partially filled the xattr structure, which will be ok
924 * on the retry since the same structure will be used.
925 */
926 if ( retry ) {
927 bpc_logErrf("bpc_attrib_dirRead: BOTCH: couldn't complete file conversion on retry (%ld,%ld,%ld)\n",
928 bufP - buf, bufPsave - buf, nRead);
929 bpc_fileZIO_close(&fd);
930 return -1;
931 }
932 if ( BPC_LogLevel >= 7 ) bpc_logMsgf("bpc_attrib_dirRead: retrying file conversion\n");
933 bufP = bufPsave;
934 if ( read_more_data(&fd, buf, sizeof(buf), &nRead, &bufP, attribPath) ) {
935 bpc_fileZIO_close(&fd);
936 return -1;
937 }
938 retry = 1;
939 } else {
940 retry = 0;
941 }
942 if ( !retry && BPC_LogLevel >= 8 ) bpc_logMsgf("bpc_attrib_dirRead(%s): Got file %s: type = %d, mode = 0%o, uid/gid = %d/%d, size = %d\n",
943 attribPath, file->name, file->type, file->mode, file->uid, file->gid, file->size);
944 }
945 } else if ( magic == BPC_ATTRIB_TYPE_UNIX ) {
946 while ( bufP < buf + nRead ) {
947 uint fileNameLen;
948 char *fileName;
949 bpc_attrib_file *file;
950 int64 sizeDiv4GB;
951 uint type;
952
953 if ( nRead == sizeof(buf) && bufP > buf + nRead - 2 * BPC_MAXPATHLEN
954 && read_more_data(&fd, buf, sizeof(buf), &nRead, &bufP, attribPath) ) {
955 bpc_fileZIO_close(&fd);
956 return -1;
957 }
958
959 fileNameLen = getVarInt_v3(&bufP, buf + nRead);
960 if ( fileNameLen > 2 * BPC_MAXPATHLEN - 16 ) {
961 bpc_logErrf("bpc_attrib_dirRead: got unreasonable file name length %d\n", fileNameLen);
962 bpc_fileZIO_close(&fd);
963 return -1;
964 }
965
966 /*
967 * Save the fileName, but it's not NULL terminated yet.
968 * After we get the next data, we can safely NULL-terminate the fileName.
969 */
970 fileName = (char*)bufP;
971 bufP += fileNameLen;
972 type = getVarInt_v3(&bufP, buf + nRead);
973 fileName[fileNameLen] = '\0';
974
975 file = bpc_attrib_fileGet(dir, fileName, 1);
976 bpc_attrib_fileInit(file, fileName, 0);
977
978 file->type = type;
979 file->mode = getVarInt_v3(&bufP, buf + nRead);
980 file->uid = getVarInt_v3(&bufP, buf + nRead);
981 file->gid = getVarInt_v3(&bufP, buf + nRead);
982 sizeDiv4GB = getVarInt_v3(&bufP, buf + nRead);
983 file->size = (sizeDiv4GB << 32) + getVarInt_v3(&bufP, buf + nRead);
984 file->mtime = CONV_BUF_TO_UINT32(bufP); bufP += 4;
985 file->compress = dir->compress;
986 file->backupNum = backupNum;
987
988 if ( BPC_LogLevel >= 8 ) bpc_logMsgf("bpc_attrib_dirRead(%s): Got v3 file %s: type = %d, mode = 0%o, uid/gid = %d/%d, size = %d\n",
989 attribPath, file->name, file->type, file->mode, file->uid, file->gid, file->size);
990 }
991 } else {
992 bpc_logErrf("Unexpected magic number 0x%x read from %s\n", magic, attribPath);
993 return -1;
994 }
995 /* TODO: make sure we are at EOF? */
996 bpc_fileZIO_close(&fd);
997 return 0;
998 }
999
1000 typedef struct {
1001 uchar *bufP;
1002 uchar *bufEnd;
1003 uint numEntries;
1004 } buf_info;
1005
1006 typedef struct {
1007 bpc_poolWrite_info fd;
1008 uchar buf[4 * 65536];
1009 uchar *bufP;
1010 } write_info;
1011
write_file_flush(write_info * info)1012 static void write_file_flush(write_info *info)
1013 {
1014 if ( info->bufP > info->buf ) {
1015 if ( BPC_LogLevel >= 7 ) bpc_logMsgf("write_file_flush: writing %lu bytes to pool\n", (unsigned long)(info->bufP - info->buf));
1016 bpc_poolWrite_write(&info->fd, info->buf, info->bufP - info->buf);
1017 }
1018 info->bufP = info->buf;
1019 }
1020
bpc_attrib_xattrWrite(bpc_attrib_xattr * xattr,buf_info * info)1021 static void bpc_attrib_xattrWrite(bpc_attrib_xattr *xattr, buf_info *info)
1022 {
1023 setVarInt(&info->bufP, info->bufEnd, xattr->key.keyLen);
1024 setVarInt(&info->bufP, info->bufEnd, xattr->valueLen);
1025
1026 if ( xattr->key.keyLen >= 1 && info->bufP + xattr->key.keyLen <= info->bufEnd ) {
1027 memcpy(info->bufP, xattr->key.key, xattr->key.keyLen);
1028 if ( info->bufP[xattr->key.keyLen - 1] != 0x0 ) {
1029 info->bufP[xattr->key.keyLen - 1] = 0x0;
1030 bpc_logMsgf("bpc_attrib_xattrWrite: BOTCH: truncated xattr name '%s' to match keyLen %u\n", info->bufP, xattr->key.keyLen);
1031 }
1032 }
1033 info->bufP += xattr->key.keyLen;
1034
1035 if ( info->bufP + xattr->valueLen <= info->bufEnd ) {
1036 memcpy(info->bufP, xattr->value, xattr->valueLen);
1037 }
1038 info->bufP += xattr->valueLen;
1039 info->numEntries++;
1040 }
1041
1042 /*
1043 * Write a file structure to the memory buffer. Returns the next unused buffer
1044 * pointer. If the buffer is exhausted, no data is written past the buffer end,
1045 * Therefore, if the return value is greater than bufEnd, then the conversion
1046 * failed to fit. The routine can be called again with at least (bufP - buf)
1047 * bytes allocated.
1048 */
bpc_attrib_file2buf(bpc_attrib_file * file,uchar * buf,uchar * bufEnd)1049 uchar *bpc_attrib_file2buf(bpc_attrib_file *file, uchar *buf, uchar *bufEnd)
1050 {
1051 uchar *bufP = buf;
1052 size_t fileNameLen = strlen(file->name);
1053 uint xattrEntryCnt = bpc_hashtable_entryCount(&file->xattrHT);
1054 buf_info info;
1055
1056 setVarInt(&bufP, bufEnd, fileNameLen);
1057 if ( bufP + fileNameLen < bufEnd ) {
1058 memcpy(bufP, file->name, fileNameLen);
1059 }
1060 bufP += fileNameLen;
1061
1062 setVarInt(&bufP, bufEnd, xattrEntryCnt);
1063 setVarInt(&bufP, bufEnd, file->type);
1064 setVarInt(&bufP, bufEnd, file->mtime);
1065 setVarInt(&bufP, bufEnd, file->mode);
1066 setVarInt(&bufP, bufEnd, file->uid);
1067 setVarInt(&bufP, bufEnd, file->gid);
1068 setVarInt(&bufP, bufEnd, file->size);
1069 setVarInt(&bufP, bufEnd, file->inode);
1070 setVarInt(&bufP, bufEnd, file->compress);
1071 setVarInt(&bufP, bufEnd, file->nlinks);
1072 setVarInt(&bufP, bufEnd, file->digest.len);
1073
1074 if ( bufP + file->digest.len <= bufEnd ) {
1075 memcpy(bufP, file->digest.digest, file->digest.len);
1076 }
1077 bufP += file->digest.len;
1078
1079 info.bufEnd = bufEnd;
1080 info.bufP = bufP;
1081 info.numEntries = 0;
1082 bpc_hashtable_iterate(&file->xattrHT, (void*)bpc_attrib_xattrWrite, &info);
1083 if ( info.numEntries != xattrEntryCnt ) {
1084 bpc_logErrf("bpc_attrib_file2buf: BOTCH: wrote %u xattr entries vs %u; attrib file corrupted\n", info.numEntries, xattrEntryCnt);
1085 }
1086 return info.bufP;
1087 }
1088
bpc_attrib_fileWrite(bpc_attrib_file * file,write_info * info)1089 static void bpc_attrib_fileWrite(bpc_attrib_file *file, write_info *info)
1090 {
1091 uchar *bufP;
1092
1093 if ( file->isTemp ) {
1094 if ( BPC_LogLevel >= 6 ) bpc_logMsgf("Skipping temp file %s: type = %d, mode = 0%o, uid/gid = %lu/%lu, size = %lu, inode = %lu, nlinks = %d, digest = 0x%02x%02x%02x..., bufUsed = %lu\n",
1095 file->name, file->type, file->mode,
1096 (unsigned long)file->uid, (unsigned long)file->gid,
1097 (unsigned long)file->size, (unsigned long)file->inode, file->nlinks,
1098 file->digest.digest[0], file->digest.digest[1], file->digest.digest[2],
1099 (unsigned long)(info->bufP - info->buf));
1100 return;
1101 }
1102 bufP = bpc_attrib_file2buf(file, info->bufP, info->buf + sizeof(info->buf));
1103 if ( BPC_LogLevel >= 6 ) bpc_logMsgf("Wrote file %s: type = %d, mode = 0%o, uid/gid = %lu/%lu, size = %lu, inode = %lu, nlinks = %d, digest = 0x%02x%02x%02x..., bufUsed = %lu\n",
1104 file->name, file->type, file->mode,
1105 (unsigned long)file->uid, (unsigned long)file->gid,
1106 (unsigned long)file->size, (unsigned long)file->inode, file->nlinks,
1107 file->digest.digest[0], file->digest.digest[1], file->digest.digest[2],
1108 (unsigned long)(info->bufP - info->buf));
1109
1110 if ( bufP <= info->buf + sizeof(info->buf) ) {
1111 /*
1112 * it fit into the buffer
1113 */
1114 info->bufP = bufP;
1115 return;
1116 }
1117 /*
1118 * we overflowed the buffer - flush and try again
1119 */
1120 write_file_flush(info);
1121 bufP = bpc_attrib_file2buf(file, info->bufP, info->buf + sizeof(info->buf));
1122 if ( bufP <= info->buf + sizeof(info->buf) ) {
1123 info->bufP = bufP;
1124 return;
1125 }
1126 bpc_logErrf("bpc_attrib_fileWrite: BOTCH: can't fit file into buffer (%ld, %ld)\n", bufP - info->buf, sizeof(info->buf));
1127 }
1128
1129 /*
1130 * Pre 0.50 attribute writing. Writes a small file that contains the file hash of the attrib file
1131 */
bpc_attrib_dirWriteOld(bpc_deltaCount_info * deltaInfo,bpc_attrib_dir * dir,char * dirPath,char * attribFileName,bpc_digest * oldDigest)1132 static int bpc_attrib_dirWriteOld(bpc_deltaCount_info *deltaInfo, bpc_attrib_dir *dir,
1133 char *dirPath, char *attribFileName, bpc_digest *oldDigest)
1134 {
1135 char attribPath[BPC_MAXPATHLEN], attribPathTemp[BPC_MAXPATHLEN];
1136 bpc_fileZIO_fd fd;
1137 bpc_digest digest;
1138 int status;
1139 OFF_T poolFileSize;
1140 int errorCnt;
1141 static write_info info;
1142 char *p;
1143
1144 bpc_attrib_attribFilePath(attribPath, dirPath, attribFileName);
1145 if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_attrib_dirWriteOld(%s)\n", attribPath);
1146 snprintf(attribPathTemp, BPC_MAXPATHLEN, "%s.%d", attribPath, getpid());
1147 if ( (p = strrchr(attribPathTemp, '/')) ) {
1148 *p = '\0';
1149 if ( bpc_path_create(attribPathTemp) ) return -1;
1150 *p = '/';
1151 }
1152
1153 if ( bpc_hashtable_entryCount(&dir->filesHT) == 0 ) {
1154 int fdNum;
1155 /*
1156 * Empty directory - we just generate an empty attrib file, which we don't pool
1157 */
1158 if ( (fdNum = open(attribPathTemp, O_WRONLY | O_CREAT | O_TRUNC, 0660)) < 0 ) {
1159 bpc_logErrf("bpc_attrib_dirWrite: can't open/create raw %s for writing\n", attribPathTemp);
1160 return -1;
1161 }
1162 close(fdNum);
1163 if ( rename(attribPathTemp, attribPath) ) {
1164 bpc_logErrf("bpc_attrib_dirWrite: rename from %s to %s failed\n", attribPathTemp, attribPath);
1165 return -1;
1166 }
1167 if ( oldDigest ) bpc_poolRefDeltaUpdate(deltaInfo, dir->compress, oldDigest, -1);
1168 dir->digest.len = 0;
1169 return 0;
1170 }
1171
1172 info.bufP = info.buf;
1173 CONV_UINT32_TO_BUF(info.bufP, BPC_ATTRIB_TYPE_XATTR);
1174
1175 bpc_poolWrite_open(&info.fd, dir->compress, NULL);
1176 bpc_hashtable_iterate(&dir->filesHT, (void*)bpc_attrib_fileWrite, &info);
1177 write_file_flush(&info);
1178 bpc_poolWrite_close(&info.fd, &status, &digest, &poolFileSize, &errorCnt);
1179
1180 if ( errorCnt ) return -1;
1181
1182 /*
1183 * Now write the small atttib file, which just contains a magic number and the digest
1184 */
1185 if ( bpc_fileZIO_open(&fd, attribPathTemp, 1, dir->compress) ) {
1186 bpc_logErrf("bpc_attrib_dirWrite: can't open/create %s for writing\n", attribPathTemp);
1187 return -1;
1188 }
1189 info.bufP = info.buf;
1190 CONV_UINT32_TO_BUF(info.bufP, BPC_ATTRIB_TYPE_DIGEST);
1191 if ( digest.len > 0 ) {
1192 memcpy(info.bufP, digest.digest, digest.len);
1193 info.bufP += digest.len;
1194 }
1195 if ( bpc_fileZIO_write(&fd, info.buf, info.bufP - info.buf) < 0 ) {
1196 bpc_logErrf("bpc_attrib_dirWrite: can't write to %s\n", attribPathTemp);
1197 bpc_fileZIO_close(&fd);
1198 return -1;
1199 }
1200 bpc_fileZIO_close(&fd);
1201 if ( rename(attribPathTemp, attribPath) ) {
1202 bpc_logErrf("bpc_attrib_dirWrite: rename from %s to %s failed\n", attribPathTemp, attribPath);
1203 return -1;
1204 }
1205 if ( BPC_LogLevel >= 8 ) bpc_logMsgf("bpc_attrib_dirWrite: new attrib digest = 0x%02x%02x%02x..., oldDigest = 0x%02x%02x...\n",
1206 digest.digest[0], digest.digest[1], digest.digest[2],
1207 oldDigest ? oldDigest->digest[0] : 0x0, oldDigest ? oldDigest->digest[1] : 0x0);
1208
1209 if ( oldDigest ) bpc_poolRefDeltaUpdate(deltaInfo, dir->compress, oldDigest, -1);
1210 bpc_poolRefDeltaUpdate(deltaInfo, dir->compress, &digest, 1);
1211
1212 /*
1213 * update with the new digest
1214 */
1215 memcpy(&dir->digest, &digest, sizeof(digest));
1216
1217 return 0;
1218 }
1219
bpc_attrib_dirWrite(bpc_deltaCount_info * deltaInfo,bpc_attrib_dir * dir,char * dirPath,char * attribFileName,bpc_digest * oldDigest)1220 int bpc_attrib_dirWrite(bpc_deltaCount_info *deltaInfo, bpc_attrib_dir *dir, char *dirPath, char *attribFileName, bpc_digest *oldDigest)
1221 {
1222 char attribPath[BPC_MAXPATHLEN], attribPathTemp[BPC_MAXPATHLEN], *baseAttribFileName;
1223 bpc_digest digest;
1224 int status;
1225 OFF_T poolFileSize;
1226 int errorCnt;
1227 static write_info info;
1228 char *p;
1229 int fdNum;
1230 size_t attribPathLen, baseAttribFileNameLen;
1231 int digestChanged;
1232
1233 if ( WriteOldStyleAttribFile ) return bpc_attrib_dirWriteOld(deltaInfo, dir, dirPath, attribFileName, oldDigest);
1234
1235 /*
1236 * baseAttribFileName points to the last portion of attribFileName, or the whole
1237 * string if it doesn't contain '/'
1238 */
1239 if ( (baseAttribFileName = strrchr(attribFileName, '/')) ) {
1240 baseAttribFileName++;
1241 } else {
1242 baseAttribFileName = attribFileName;
1243 }
1244 baseAttribFileNameLen = strlen(baseAttribFileName);
1245
1246 bpc_attrib_attribFilePath(attribPath, dirPath, attribFileName);
1247 if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_attrib_dirWrite(%s): dirPath = %s, attribFileName = %s, baseAttribFileName = %s\n",
1248 attribPath, dirPath, attribFileName, baseAttribFileName);
1249 snprintf(attribPathTemp, BPC_MAXPATHLEN, "%s.%d", attribPath, getpid());
1250 if ( (p = strrchr(attribPathTemp, '/')) ) {
1251 *p = '\0';
1252 if ( bpc_path_create(attribPathTemp) ) return -1;
1253 *p = '/';
1254 }
1255 attribPathLen = strlen(attribPath);
1256
1257 if ( bpc_hashtable_entryCount(&dir->filesHT) > 0 ) {
1258 /*
1259 * Write the attribute file to the pool
1260 */
1261 info.bufP = info.buf;
1262 CONV_UINT32_TO_BUF(info.bufP, BPC_ATTRIB_TYPE_XATTR);
1263
1264 bpc_poolWrite_open(&info.fd, dir->compress, NULL);
1265 bpc_hashtable_iterate(&dir->filesHT, (void*)bpc_attrib_fileWrite, &info);
1266 write_file_flush(&info);
1267 bpc_poolWrite_close(&info.fd, &status, &digest, &poolFileSize, &errorCnt);
1268
1269 if ( errorCnt ) return -1;
1270
1271 /*
1272 * Starting in 0.50, the attrib file is always empty (so it takes no extra blocks)
1273 * and we simply include the digest in the file name by appending the hex digits.
1274 *
1275 * An empty attrib file is simply "attrib_0", and a legacy attrib file (pre <0.50)
1276 * is "attrib".
1277 */
1278 if ( attribPathLen + digest.len * 2 + 2 >= sizeof(attribPath) ) {
1279 bpc_logErrf("bpc_attrib_dirWrite: path too long (%d, %d, %d)\n", strlen(attribPath), digest.len, sizeof(attribPath));
1280 return -1;
1281 }
1282 attribPath[attribPathLen++] = '_';
1283 bpc_digest_digest2str(&digest, attribPath + attribPathLen);
1284 /*
1285 * Now create an empty attrib file with the file name digest
1286 */
1287 if ( (fdNum = open(attribPathTemp, O_WRONLY | O_CREAT | O_TRUNC, 0660)) < 0 ) {
1288 bpc_logErrf("bpc_attrib_dirWrite: can't open/create raw %s for writing\n", attribPathTemp);
1289 return -1;
1290 }
1291 close(fdNum);
1292 if ( rename(attribPathTemp, attribPath) ) {
1293 bpc_logErrf("bpc_attrib_dirWrite: rename from %s to %s failed\n", attribPathTemp, attribPath);
1294 return -1;
1295 }
1296 if ( BPC_LogLevel >= 5 ) bpc_logMsgf("bpc_attrib_dirWrite: created new attrib file %s\n", attribPath);
1297 } else {
1298 memset(&digest, 0, sizeof(digest));
1299 attribPath[attribPathLen++] = '_';
1300 strcpy(attribPath + attribPathLen, "0");
1301 if ( BPC_LogLevel >= 5 ) bpc_logMsgf("bpc_attrib_dirWrite: skipping creating new empty attrib file %s\n", attribPath);
1302 unlink(attribPath);
1303 }
1304
1305 if ( BPC_LogLevel >= 8 ) bpc_logMsgf("bpc_attrib_dirWrite: new attrib digest = 0x%02x%02x%02x..., oldDigest = 0x%02x%02x...\n",
1306 digest.digest[0], digest.digest[1], digest.digest[2],
1307 oldDigest ? oldDigest->digest[0] : 0x0, oldDigest ? oldDigest->digest[1] : 0x0);
1308
1309 digestChanged = !oldDigest || bpc_digest_compare(&digest, oldDigest);
1310 if ( digestChanged && digest.len > 0 ) {
1311 bpc_poolRefDeltaUpdate(deltaInfo, dir->compress, &digest, 1);
1312 }
1313
1314 if ( oldDigest ) {
1315 if ( !digestChanged ) {
1316 if ( BPC_LogLevel >= 4 ) bpc_logMsgf("bpc_attrib_dirWrite: old attrib has same digest; no changes to ref counts\n");
1317 return 0;
1318 }
1319 if ( attribPathLen + oldDigest->len * 2 + 2 >= sizeof(attribPath) ) {
1320 bpc_logErrf("bpc_attrib_dirWrite: oldDigest path too long (%d, %d, %d)\n", strlen(attribPath), oldDigest->len, sizeof(attribPath));
1321 return -1;
1322 }
1323 strcpy(attribPathTemp, attribPath);
1324 if ( oldDigest->len > 0 ) {
1325 bpc_poolRefDeltaUpdate(deltaInfo, dir->compress, oldDigest, -1);
1326 bpc_digest_digest2str(oldDigest, attribPathTemp + attribPathLen);
1327 } else {
1328 strcpy(attribPathTemp + attribPathLen, "0");
1329 }
1330 if ( !unlink(attribPathTemp) ) {
1331 if ( BPC_LogLevel >= 5 ) bpc_logMsgf("bpc_attrib_dirWrite: removed old attrib file %s\n", attribPathTemp);
1332 } else {
1333 DIR *dirOs;
1334 struct dirent *dp;
1335 char deletePath[BPC_MAXPATHLEN];
1336
1337 /*
1338 * Scan the directory and remove any other attribute files that have the
1339 * same root.
1340 */
1341 if ( !(p = strrchr(attribPath, '/')) ) {
1342 bpc_logErrf("bpc_attrib_dirWrite: can't find a '/' in %s\n", attribPath);
1343 return -1;
1344 }
1345 *p++ = '\0';
1346 if ( !(dirOs = opendir(attribPath)) ) {
1347 bpc_logErrf("bpc_attrib_dirWrite: can't opendir %s\n", attribPath);
1348 return -1;
1349 }
1350 while ( (dp = readdir(dirOs)) ) {
1351 if ( strncmp(dp->d_name, baseAttribFileName, baseAttribFileNameLen) || !strcmp(dp->d_name, p) ) continue;
1352 snprintf(deletePath, sizeof(deletePath), "%s/%s", attribPath, dp->d_name);
1353 unlink(deletePath);
1354 if ( BPC_LogLevel >= 5 ) bpc_logMsgf("bpc_attrib_dirWrite: removed other old attrib file %s\n", deletePath);
1355 }
1356 closedir(dirOs);
1357 }
1358 }
1359
1360 /*
1361 * update with the new digest
1362 */
1363 memcpy(&dir->digest, &digest, sizeof(digest));
1364
1365 return 0;
1366 }
1367