1 /* 2 * Definitions for BackupPC libraries. 3 * 4 * Copyright (C) 2013 Craig Barratt. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; either version 3 of the License, or 9 * (at your option) any later version. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License along 17 * with this program; if not, visit the http://fsf.org website. 18 */ 19 20 #ifndef _BACKUPPC_H_ 21 22 #define _BACKUPPC_H_ 23 24 #include <rsync.h> 25 #include "zlib/zlib.h" 26 27 #define BPC_MAXPATHLEN (2 * MAXPATHLEN) 28 29 extern char BPC_PoolDir[]; 30 extern char BPC_CPoolDir[]; 31 extern char BPC_PoolDir3[]; 32 extern char BPC_CPoolDir3[]; 33 extern char BPC_TopDir[]; 34 extern int BPC_HardLinkMax; 35 extern int BPC_PoolV3Enabled; 36 extern int BPC_TmpFileUnique; 37 extern int BPC_LogLevel; 38 39 /* 40 * Maximum length of a digest - 16 bytes for MD5, but 4 bytes of collision counting 41 */ 42 #define BPC_DIGEST_LEN_MAX 20 43 44 #define uint64 unsigned int64 45 46 typedef struct { 47 uchar digest[BPC_DIGEST_LEN_MAX]; 48 int len; 49 } bpc_digest; 50 51 /* 52 * Simple hash table functions. 53 * 54 * Any structure stored in a hash table should start with a bpc_hashtable_key entry for the key. 55 */ 56 57 typedef struct { 58 void *key; /* a NULL key means this node is empty or deleted; also used for free list */ 59 uint32 keyLen; /* with a NULL key, a zero value means empty; non-zero means deleted */ 60 uint32 keyHash; 61 } bpc_hashtable_key; 62 63 typedef struct { 64 bpc_hashtable_key **nodes; 65 uint32 nodeSize; 66 uint32 size; 67 uint32 entries; /* total number of user entries */ 68 uint32 entriesDel; /* number of entries flagged as deleted */ 69 } bpc_hashtable; 70 71 void bpc_hashtable_create(bpc_hashtable *tbl, uint32 size, uint32 nodeSize); 72 void bpc_hashtable_destroy(bpc_hashtable *tbl); 73 void bpc_hashtable_erase(bpc_hashtable *tbl); 74 uint32 bpc_hashtable_hash(uchar *key, uint32 keyLen); 75 void *bpc_hashtable_find(bpc_hashtable *tbl, unsigned char *key, unsigned int keyLen, int allocate_if_missing); 76 void bpc_hashtable_growSize(bpc_hashtable *tbl, uint32 newSize); 77 void bpc_hashtable_nodeDelete(bpc_hashtable *tbl, void *node); 78 void bpc_hashtable_iterate(bpc_hashtable *tbl, void (*callback)(void*, void*), void *arg1); 79 void *bpc_hashtable_nextEntry(bpc_hashtable *tbl, uint *idx); 80 int bpc_hashtable_entryCount(bpc_hashtable *tbl); 81 82 /* 83 * Reference counting 84 */ 85 typedef struct { 86 bpc_hashtable ht; 87 int initDone; 88 } bpc_refCount_info; 89 90 void bpc_poolRefInit(bpc_refCount_info *info, int entryCnt); 91 void bpc_poolRefDestroy(bpc_refCount_info *info); 92 void bpc_poolRefSet(bpc_refCount_info *info, bpc_digest *digest, int32 count); 93 int bpc_poolRefDelete(bpc_refCount_info *info, bpc_digest *digest); 94 int bpc_poolRefGet(bpc_refCount_info *info, bpc_digest *digest, int32 *count); 95 int bpc_poolRefIncr(bpc_refCount_info *info, bpc_digest *digest, int32 delta); 96 int bpc_poolRefIterate(bpc_refCount_info *info, bpc_digest *digest, int32 *count, uint *idx); 97 void bpc_poolRefCountPrint(bpc_refCount_info *info); 98 int bpc_poolRefFileWrite(bpc_refCount_info *info, char *fileName); 99 int bpc_poolRefFileRead(bpc_refCount_info *info, char *fileName); 100 void bpc_poolRefRequestFsck(char *targetDir, int ext); 101 102 /* 103 * Delta counting 104 */ 105 typedef struct { 106 bpc_refCount_info refCnt[2]; 107 char targetDir[BPC_MAXPATHLEN]; 108 } bpc_deltaCount_info; 109 110 void bpc_poolRefDeltaFileInit(bpc_deltaCount_info *info, char *hostDir); 111 void bpc_poolRefDeltaFileDestroy(bpc_deltaCount_info *info); 112 uint32 bpc_poolRefDeltaFileFlush(bpc_deltaCount_info *info); 113 void bpc_poolRefDeltaUpdate(bpc_deltaCount_info *info, int compress, bpc_digest *digest, int32 count); 114 void bpc_poolRefDeltaPrint(bpc_deltaCount_info *info); 115 116 void bpc_poolRefDeltaFileInitOld(char *hostDir); 117 uint32 bpc_poolRefDeltaFileFlushOld(void); 118 void bpc_poolRefDeltaUpdateOld(int compress, bpc_digest *digest, int32 count); 119 void bpc_poolRefDeltaPrintOld(void); 120 121 /* 122 * Compressed file IO. A compressed file descriptor contains a buffer for compressed data. 123 */ 124 typedef struct { 125 z_stream strm; 126 char *buf; 127 size_t bufSize; 128 int fd; 129 int first; 130 int write; 131 int eof; 132 int error; 133 int compressLevel; 134 int writeTeeStderr; 135 /* 136 * readLine buffer 137 */ 138 char *lineBuf; 139 size_t lineBufSize; 140 size_t lineBufLen; 141 size_t lineBufIdx; 142 int lineBufEof; 143 } bpc_fileZIO_fd; 144 145 int bpc_fileZIO_open(bpc_fileZIO_fd *fd, char *fileName, int writeFile, int compressLevel); 146 int bpc_fileZIO_fdopen(bpc_fileZIO_fd *fd, FILE *stream, int writeFile, int compressLevel); 147 void bpc_fileZIO_writeTeeStderr(bpc_fileZIO_fd *fd, int tee); 148 ssize_t bpc_fileZIO_read(bpc_fileZIO_fd *fd, uchar *buf, size_t nRead); 149 ssize_t bpc_fileZIO_write(bpc_fileZIO_fd *fd, uchar *buf, size_t nWrite); 150 int bpc_fileZIO_readLine(bpc_fileZIO_fd *fd, char **str, size_t *strLen); 151 int bpc_fileZIO_close(bpc_fileZIO_fd *fd); 152 int bpc_fileZIO_rewind(bpc_fileZIO_fd *fd); 153 154 #define BPC_POOL_WRITE_BUF_SZ (8 * 1048576) /* 8 MB - must be at least 1MB so the V3 digest calculation can occur */ 155 #define BPC_POOL_WRITE_CONCURRENT_MATCH (16) /* number of pool files we concurrently match */ 156 157 typedef struct _bpc_candidate_file { 158 bpc_digest digest; 159 OFF_T fileSize; 160 int v3File; 161 char fileName[BPC_MAXPATHLEN]; 162 struct _bpc_candidate_file *next; 163 } bpc_candidate_file; 164 165 typedef struct { 166 bpc_fileZIO_fd fd; 167 int used; 168 int v3File; 169 OFF_T fileSize; 170 bpc_digest digest; 171 char fileName[BPC_MAXPATHLEN]; 172 } bpc_candidate_match; 173 174 typedef struct { 175 int compress; 176 int state; 177 int eof; 178 int retValue; 179 int retryCnt; 180 OFF_T fileSize; 181 OFF_T poolFileSize; 182 bpc_digest digest; 183 bpc_digest digest_v3; 184 md_context md5; 185 /* 186 * Set of active potential file matches. All files match up to matchPosn. 187 */ 188 OFF_T matchPosn; 189 bpc_candidate_match match[BPC_POOL_WRITE_CONCURRENT_MATCH]; 190 bpc_candidate_file *candidateList; 191 /* 192 * When we first build the candidate match list, we remember where the first 193 * zero-length file is (if any), and the next open slot. If these change 194 * before we insert a new file, we know to try again (since someone probably 195 * won a race to get there first). 196 */ 197 int digestExtZeroLen, digestExtOpen; 198 /* 199 * Temporary output file if the in-memory buffer is too small 200 */ 201 int fdOpen; 202 bpc_fileZIO_fd fd; 203 char tmpFileName[BPC_MAXPATHLEN]; 204 /* 205 * Error count 206 */ 207 int errorCnt; 208 /* 209 * Initial file buffer - used if the entire file fits, or otherwise keeps the first 1MB 210 * of the file so we can compute the V3 digest. If we have the entire file in memory 211 * then fileWritten == 0. 212 * 213 * This buffer is allocated to be size BPC_POOL_WRITE_BUF_SZ on open() and freed on close(). 214 */ 215 uint32 bufferIdx; 216 uchar *buffer; 217 } bpc_poolWrite_info; 218 219 int bpc_poolWrite_open(bpc_poolWrite_info *info, int compress, bpc_digest *digest); 220 int bpc_poolWrite_write(bpc_poolWrite_info *info, uchar *data, size_t dataLen); 221 int bpc_poolWrite_createPoolDir(bpc_poolWrite_info *info, bpc_digest *digest); 222 void bpc_poolWrite_close(bpc_poolWrite_info *info, int *match, bpc_digest *digest, OFF_T *poolFileSize, int *errorCnt); 223 void bpc_poolWrite_cleanup(bpc_poolWrite_info *info); 224 void bpc_poolWrite_repeatPoolWrite(bpc_poolWrite_info *info, char *fileName); 225 int bpc_poolWrite_copyToPool(bpc_poolWrite_info *info, char *poolPath, char *fileName); 226 void bpc_poolWrite_addToPool(bpc_poolWrite_info *info, char *fileName, int v3PoolFile); 227 int bpc_poolWrite_unmarkPendingDelete(char *poolPath); 228 229 /* 230 * General library routines 231 */ 232 void bpc_lib_conf_init(char *topDir, int hardLinkMax, int poolV3Enabled, int logLevel); 233 void bpc_lib_setTmpFileUnique(int val); 234 int bpc_lib_setLogLevel(int logLevel); 235 void bpc_byte2hex(char *outStr, int byte); 236 uchar bpc_hexStr2byte(char c1, char c2); 237 void bpc_digest_buffer2MD5(bpc_digest *digest, uchar *buffer, size_t bufferLen); 238 void bpc_digest_append_ext(bpc_digest *digest, uint32 ext); 239 void bpc_digest_digest2str(bpc_digest *digest, char *hexStr); 240 void bpc_digest_str2digest(bpc_digest *digest, char *hexStr); 241 int bpc_digest_compare(bpc_digest *digest1, bpc_digest *digest2); 242 void bpc_digest_md52path(char *path, int compress, bpc_digest *digest); 243 void bpc_digest_md52path_v3(char *path, int compress, bpc_digest *digest); 244 void bpc_digest_buffer2MD5_v3(bpc_digest *digest, uchar *buffer, size_t bufferLen); 245 void bpc_fileNameEltMangle(char *path, int pathSize, char *pathUM); 246 void bpc_fileNameMangle(char *path, int pathSize, char *pathUM); 247 void bpc_logMsgf(char *fmt, ...); 248 void bpc_logErrf(char *fmt, ...); 249 void bpc_logMsgGet(char **mesg, size_t *mesgLen); 250 void bpc_logMsgErrorCntGet(unsigned long *errorCnt); 251 void bpc_logMsgCBSet(void (*cb)(int errFlag, char *mesg, size_t mesgLen)); 252 253 /* 254 * Compute file digest 255 */ 256 int bpc_fileDigest(char *fileName, int compress, bpc_digest *digest); 257 258 /* 259 * Directory operations 260 */ 261 int bpc_path_create(char *path); 262 int bpc_path_remove(bpc_deltaCount_info *deltaInfo, char *path, int compress); 263 int bpc_path_refCountAll(bpc_deltaCount_info *deltaInfo, char *path, int compress, int incr); 264 int bpc_path_refCountAllInodeMax(bpc_deltaCount_info *deltaInfo, char *path, int compress, int incr, unsigned int *inodeMax); 265 int bpc_lockRangeFd(int fd, OFF_T offset, OFF_T len, int block); 266 int bpc_unlockRangeFd(int fd, OFF_T offset, OFF_T len); 267 int bpc_lockRangeFile(char *lockFile, OFF_T offset, OFF_T len, int block); 268 void bpc_unlockRangeFile(int lockFd); 269 270 /* 271 * File attribs 272 */ 273 typedef struct { 274 bpc_hashtable_key key; 275 void *value; 276 uint32 valueLen; 277 } bpc_attrib_xattr; 278 279 typedef struct { 280 bpc_hashtable_key key; 281 char *name; 282 ushort type; 283 ushort compress; 284 /* 285 * isTemp is set if this is a temporary attribute entry (eg: mkstemp), that 286 * doesn't have referencing counting for the digest. Therefore, when a 287 * temporary file is created or deleted, there is no change to the 288 * reference counts. 289 */ 290 ushort isTemp; 291 uint32 mode; 292 uid_t uid; 293 gid_t gid; 294 uint32 nlinks; 295 time_t mtime; 296 OFF_T size; 297 ino_t inode; 298 int32 backupNum; 299 bpc_digest digest; 300 /* 301 * hash table of bpc_attrib_xattr entries, indexed by xattr key 302 */ 303 bpc_hashtable xattrHT; 304 } bpc_attrib_file; 305 306 /* 307 * A directory is a hash table of file attributes, indexed by file name 308 */ 309 typedef struct { 310 bpc_digest digest; 311 ushort compress; 312 ushort needRewrite; 313 /* 314 * hash table of bpc_attrib_file entries, indexed by file name 315 */ 316 bpc_hashtable filesHT; 317 } bpc_attrib_dir; 318 319 bpc_attrib_xattr *bpc_attrib_xattrGet(bpc_attrib_file *file, void *key, int keyLen, int allocate_if_missing); 320 void bpc_attrib_xattrDestroy(bpc_attrib_xattr *xattr); 321 int bpc_attrib_xattrDelete(bpc_attrib_file *file, void *key, int keyLen); 322 int bpc_attrib_xattrDeleteAll(bpc_attrib_file *file); 323 int bpc_attrib_xattrSetValue(bpc_attrib_file *file, void *key, int keyLen, void *value, uint32 valueLen); 324 int bpc_attrib_xattrCount(bpc_attrib_file *file); 325 size_t bpc_attrib_xattrList(bpc_attrib_file *file, char *list, size_t listLen, int ignoreRsyncACLs); 326 void bpc_attrib_fileInit(bpc_attrib_file *file, char *fileName, int xattrNumEntries); 327 void bpc_attrib_fileDestroy(bpc_attrib_file *file); 328 bpc_attrib_file *bpc_attrib_fileGet(bpc_attrib_dir *dir, char *fileName, int allocate_if_missing); 329 int bpc_attrib_fileIterate(bpc_attrib_dir *dir, bpc_attrib_file **file, uint *idx); 330 void bpc_attrib_fileCopyOpt(bpc_attrib_file *fileDest, bpc_attrib_file *fileSrc, int overwriteEmptyDigest); 331 void bpc_attrib_fileCopy(bpc_attrib_file *fileDest, bpc_attrib_file *fileSrc); 332 int bpc_attrib_fileCompare(bpc_attrib_file *file0, bpc_attrib_file *file1); 333 void bpc_attrib_fileDeleteName(bpc_attrib_dir *dir, char *fileName); 334 int bpc_attrib_fileCount(bpc_attrib_dir *dir); 335 char *bpc_attrib_fileType2Text(int type); 336 void bpc_attrib_dirInit(bpc_attrib_dir *dir, int compressLevel); 337 void bpc_attrib_dirDestroy(bpc_attrib_dir *dir); 338 int bpc_attrib_dirNeedRewrite(bpc_attrib_dir *dir); 339 ssize_t bpc_attrib_getEntries(bpc_attrib_dir *dir, char *entries, ssize_t entrySize); 340 void bpc_attrib_dirRefCount(bpc_deltaCount_info *deltaInfo, bpc_attrib_dir *dir, int incr); 341 void bpc_attrib_dirRefCountInodeMax(bpc_deltaCount_info *deltaInfo, bpc_attrib_dir *dir, int incr, unsigned int *inodeMax); 342 void bpc_attrib_attribFilePath(char *path, char *dir, char *attribFileName); 343 bpc_digest *bpc_attrib_dirDigestGet(bpc_attrib_dir *dir); 344 uchar *bpc_attrib_buf2file(bpc_attrib_file *file, uchar *buf, uchar *bufEnd, int xattrNumEntries, int *xattrFixup); 345 uchar *bpc_attrib_buf2fileFull(bpc_attrib_file *file, uchar *buf, uchar *bufEnd); 346 uchar *bpc_attrib_file2buf(bpc_attrib_file *file, uchar *buf, uchar *bufEnd); 347 int bpc_attrib_digestRead(bpc_attrib_dir *dir, bpc_digest *digest, char *attribPath); 348 int bpc_attrib_dirRead(bpc_attrib_dir *dir, char *dirPath, char *attribFileName, int backupNum); 349 int bpc_attrib_dirWrite(bpc_deltaCount_info *deltaInfo, bpc_attrib_dir *dir, char *dirPath, char *attribFileName, bpc_digest *oldDigest); 350 void bpc_attrib_backwardCompat(int writeOldStyleAttribFile, int keepOldAttribFiles); 351 352 /* 353 * Attrib caching 354 */ 355 356 #define BPC_FTYPE_FILE (0) 357 #define BPC_FTYPE_HARDLINK (1) 358 #define BPC_FTYPE_SYMLINK (2) 359 #define BPC_FTYPE_CHARDEV (3) 360 #define BPC_FTYPE_BLOCKDEV (4) 361 #define BPC_FTYPE_DIR (5) 362 #define BPC_FTYPE_FIFO (6) 363 #define BPC_FTYPE_SOCKET (8) 364 #define BPC_FTYPE_UNKNOWN (9) 365 #define BPC_FTYPE_DELETED (10) 366 #define BPC_FTYPE_INVALID (11) 367 368 typedef struct { 369 int num; 370 int compress; 371 int version; 372 } bpc_backup_info; 373 374 typedef struct { 375 int backupNum; 376 int compress; 377 int readOnly; 378 uint cacheLruCnt; 379 380 /* 381 * optional merging of backups to create view for restore 382 */ 383 bpc_backup_info *bkupMergeList; 384 int bkupMergeCnt; 385 386 /* 387 * Hash table of cached file attributes. 388 * Key is the mangled attrib path (excluding backupTopDir[], and including attrib file name). 389 * Value is a bpc_attrib_dir structure. 390 * - Keys of the bpc_attrib_dir hash table are the file names in that directory. 391 */ 392 bpc_hashtable attrHT; 393 394 /* 395 * Hash table of cached inode attributes. 396 * Key is the inode attribute path (excluding backupTopDir[]). 397 * Value is a bpc_attrib_dir structure. 398 * - Keys of the bpc_attrib_dir hash table are the inode numbers converted to ascii hex, lsb first. 399 */ 400 bpc_hashtable inodeHT; 401 402 /* 403 * Delta reference count for any changes as we write/change files or attributes 404 */ 405 bpc_deltaCount_info *deltaInfo; 406 407 char shareName[BPC_MAXPATHLEN]; 408 int shareNameLen; 409 char shareNameUM[BPC_MAXPATHLEN]; 410 char hostName[BPC_MAXPATHLEN]; 411 char hostDir[BPC_MAXPATHLEN]; 412 char backupTopDir[BPC_MAXPATHLEN]; 413 char currentDir[BPC_MAXPATHLEN]; 414 } bpc_attribCache_info; 415 416 typedef struct { 417 bpc_hashtable_key key; 418 int dirty; 419 /* 420 * We flag directories whose parents either don't exist or aren't directories. 421 * We ignore attributes on bad directories. 422 * Initially this flag is zero, meaning we don't know if this directory is ok. 423 * After we check, > 0 means parent does exist and is a directory ; < 0 means dir is bad 424 */ 425 int dirOk; 426 uint lruCnt; 427 bpc_attrib_dir dir; 428 } bpc_attribCache_dir; 429 430 void bpc_attribCache_init(bpc_attribCache_info *ac, char *host, int backupNum, char *shareNameUM, int compress); 431 void bpc_attribCache_setDeltaInfo(bpc_attribCache_info *ac, bpc_deltaCount_info *deltaInfo); 432 void bpc_attribCache_setMergeList(bpc_attribCache_info *ac, bpc_backup_info *bkupList, int bkupCnt); 433 void bpc_attribCache_destroy(bpc_attribCache_info *ac); 434 int bpc_attribCache_readOnly(bpc_attribCache_info *ac, int readOnly); 435 void bpc_attribCache_setCurrentDirectory(bpc_attribCache_info *ac, char *dir); 436 bpc_attrib_file *bpc_attribCache_getFile(bpc_attribCache_info *ac, char *path, int allocate_if_missing, int dontReadInode); 437 int bpc_attribCache_setFile(bpc_attribCache_info *ac, char *path, bpc_attrib_file *file, int dontOverwriteInode); 438 int bpc_attribCache_deleteFile(bpc_attribCache_info *ac, char *path); 439 bpc_attrib_file *bpc_attribCache_getInode(bpc_attribCache_info *ac, ino_t inode, int allocate_if_missing); 440 int bpc_attribCache_setInode(bpc_attribCache_info *ac, ino_t inode, bpc_attrib_file *inodeSrc); 441 int bpc_attribCache_deleteInode(bpc_attribCache_info *ac, ino_t inode); 442 int bpc_attribCache_getDirEntryCnt(bpc_attribCache_info *ac, char *path); 443 ssize_t bpc_attribCache_getDirEntries(bpc_attribCache_info *ac, char *path, char *entries, ssize_t entrySize); 444 void bpc_attribCache_flush(bpc_attribCache_info *ac, int all, char *path); 445 void bpc_attribCache_getFullMangledPath(bpc_attribCache_info *ac, char *path, char *dirName, int backupNum); 446 447 #endif 448