1 /* xdelta3 - delta compression tools and library 2 Copyright 2016 Joshua MacDonald 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 #ifndef XDELTA3_INTERNAL_H__ 17 #define XDELTA3_INTERNAL_H__ 18 19 #include "xdelta3.h" 20 21 typedef struct _main_file main_file; 22 typedef struct _main_extcomp main_extcomp; 23 24 void main_buffree (void *ptr); 25 void* main_bufalloc (size_t size); 26 void main_file_init (main_file *xfile); 27 int main_file_close (main_file *xfile); 28 void main_file_cleanup (main_file *xfile); 29 int main_file_isopen (main_file *xfile); 30 int main_file_open (main_file *xfile, const char* name, int mode); 31 int main_file_exists (main_file *xfile); 32 int main_file_stat (main_file *xfile, xoff_t *size); 33 int xd3_whole_append_window (xd3_stream *stream); 34 int xd3_main_cmdline (int argc, char **argv); 35 int main_file_read (main_file *ifile, 36 uint8_t *buf, 37 size_t size, 38 size_t *nread, 39 const char *msg); 40 int main_file_write (main_file *ofile, uint8_t *buf, 41 usize_t size, const char *msg); 42 void* main_malloc (size_t size); 43 void main_free (void *ptr); 44 45 int test_compare_files (const char* f0, const char* f1); 46 usize_t xd3_bytes_on_srcblk (xd3_source *src, xoff_t blkno); 47 xoff_t xd3_source_eof(const xd3_source *src); 48 49 uint32_t xd3_large_cksum_update (uint32_t cksum, 50 const uint8_t *base, 51 usize_t look); 52 int xd3_emit_byte (xd3_stream *stream, 53 xd3_output **outputp, 54 uint8_t code); 55 56 int xd3_emit_bytes (xd3_stream *stream, 57 xd3_output **outputp, 58 const uint8_t *base, 59 usize_t size); 60 xd3_output* xd3_alloc_output (xd3_stream *stream, 61 xd3_output *old_output); 62 63 int xd3_encode_init_full (xd3_stream *stream); 64 usize_t xd3_pow2_roundup (usize_t x); 65 long get_millisecs_now (void); 66 int xd3_process_stream (int is_encode, 67 xd3_stream *stream, 68 int (*func) (xd3_stream *), 69 int close_stream, 70 const uint8_t *input, 71 usize_t input_size, 72 uint8_t *output, 73 usize_t *output_size, 74 usize_t output_size_max); 75 76 #if PYTHON_MODULE || SWIG_MODULE || NOT_MAIN 77 int xd3_main_cmdline (int argc, char **argv); 78 #endif 79 80 #if REGRESSION_TEST 81 int xd3_selftest (void); 82 #endif 83 84 /* main_file->mode values */ 85 typedef enum 86 { 87 XO_READ = 0, 88 XO_WRITE = 1 89 } main_file_modes; 90 91 #ifndef XD3_POSIX 92 #define XD3_POSIX 0 93 #endif 94 #ifndef XD3_STDIO 95 #define XD3_STDIO 0 96 #endif 97 #ifndef XD3_WIN32 98 #define XD3_WIN32 0 99 #endif 100 #ifndef NOT_MAIN 101 #define NOT_MAIN 0 102 #endif 103 104 /* If none are set, default to posix. */ 105 #if (XD3_POSIX + XD3_STDIO + XD3_WIN32) == 0 106 #undef XD3_POSIX 107 #define XD3_POSIX 1 108 #endif 109 110 struct _main_file 111 { 112 #if XD3_WIN32 113 HANDLE file; 114 #elif XD3_STDIO 115 FILE *file; 116 #elif XD3_POSIX 117 int file; 118 #endif 119 120 int mode; /* XO_READ and XO_WRITE */ 121 const char *filename; /* File name or /dev/stdin, 122 * /dev/stdout, /dev/stderr. */ 123 char *filename_copy; /* File name or /dev/stdin, 124 * /dev/stdout, /dev/stderr. */ 125 const char *realname; /* File name or /dev/stdin, 126 * /dev/stdout, /dev/stderr. */ 127 const main_extcomp *compressor; /* External compression struct. */ 128 int flags; /* RD_FIRST, RD_NONEXTERNAL, ... */ 129 xoff_t nread; /* for input position */ 130 xoff_t nwrite; /* for output position */ 131 uint8_t *snprintf_buf; /* internal snprintf() use */ 132 int size_known; /* Set by main_set_souze */ 133 xoff_t source_position; /* for avoiding seek in getblk_func */ 134 int seek_failed; /* after seek fails once, try FIFO */ 135 }; 136 137 #ifndef UINT32_MAX 138 #define UINT32_MAX 4294967295U 139 #endif 140 141 #ifndef UINT64_MAX 142 #define UINT64_MAX 18446744073709551615ULL 143 #endif 144 145 #define UINT32_OFLOW_MASK 0xfe000000U 146 #define UINT64_OFLOW_MASK 0xfe00000000000000ULL 147 148 /********************************************************************* 149 Integer encoder/decoder functions 150 **********************************************************************/ 151 152 /* Consume N bytes of input, only used by the decoder. */ 153 #define DECODE_INPUT(n) \ 154 do { \ 155 stream->total_in += (xoff_t) (n); \ 156 stream->avail_in -= (n); \ 157 stream->next_in += (n); \ 158 } while (0) 159 160 #define DECODE_INTEGER_TYPE(PART,OFLOW) \ 161 while (stream->avail_in != 0) \ 162 { \ 163 usize_t next = stream->next_in[0]; \ 164 \ 165 DECODE_INPUT(1); \ 166 \ 167 if (PART & OFLOW) \ 168 { \ 169 stream->msg = "overflow in decode_integer"; \ 170 return XD3_INVALID_INPUT; \ 171 } \ 172 \ 173 PART = (PART << 7) | (next & 127); \ 174 \ 175 if ((next & 128) == 0) \ 176 { \ 177 (*val) = PART; \ 178 PART = 0; \ 179 return 0; \ 180 } \ 181 } \ 182 \ 183 stream->msg = "further input required"; \ 184 return XD3_INPUT 185 186 #define READ_INTEGER_TYPE(TYPE, OFLOW) \ 187 TYPE val = 0; \ 188 const uint8_t *inp = (*inpp); \ 189 usize_t next; \ 190 \ 191 do \ 192 { \ 193 if (inp == maxp) \ 194 { \ 195 stream->msg = "end-of-input in read_integer"; \ 196 return XD3_INVALID_INPUT; \ 197 } \ 198 \ 199 if (val & OFLOW) \ 200 { \ 201 stream->msg = "overflow in read_intger"; \ 202 return XD3_INVALID_INPUT; \ 203 } \ 204 \ 205 next = (*inp++); \ 206 val = (val << 7) | (next & 127); \ 207 } \ 208 while (next & 128); \ 209 \ 210 (*valp) = val; \ 211 (*inpp) = inp; \ 212 \ 213 return 0 214 215 #define EMIT_INTEGER_TYPE() \ 216 /* max 64-bit value in base-7 encoding is 9.1 bytes */ \ 217 uint8_t buf[10]; \ 218 usize_t bufi = 10; \ 219 \ 220 /* This loop performs division and turns on all MSBs. */ \ 221 do \ 222 { \ 223 buf[--bufi] = (num & 127) | 128; \ 224 num >>= 7U; \ 225 } \ 226 while (num != 0); \ 227 \ 228 /* Turn off MSB of the last byte. */ \ 229 buf[9] &= 127; \ 230 \ 231 return xd3_emit_bytes (stream, output, buf + bufi, 10 - bufi) 232 233 #define IF_SIZEOF32(x) if (num < (1U << (7 * (x)))) return (x); 234 #define IF_SIZEOF64(x) if (num < (1ULL << (7 * (x)))) return (x); 235 236 #if USE_UINT32 237 static inline uint32_t 238 xd3_sizeof_uint32_t (uint32_t num) 239 { 240 IF_SIZEOF32(1); 241 IF_SIZEOF32(2); 242 IF_SIZEOF32(3); 243 IF_SIZEOF32(4); 244 return 5; 245 } 246 247 static inline int 248 xd3_decode_uint32_t (xd3_stream *stream, uint32_t *val) 249 { DECODE_INTEGER_TYPE (stream->dec_32part, UINT32_OFLOW_MASK); } 250 251 static inline int 252 xd3_read_uint32_t (xd3_stream *stream, const uint8_t **inpp, 253 const uint8_t *maxp, uint32_t *valp) 254 { READ_INTEGER_TYPE (uint32_t, UINT32_OFLOW_MASK); } 255 256 #if XD3_ENCODER 257 static inline int 258 xd3_emit_uint32_t (xd3_stream *stream, xd3_output **output, uint32_t num) 259 { EMIT_INTEGER_TYPE (); } 260 #endif /* XD3_ENCODER */ 261 #endif /* USE_UINT32 */ 262 263 #if USE_UINT64 264 static inline uint32_t 265 xd3_sizeof_uint64_t (uint64_t num) 266 { 267 IF_SIZEOF64(1); 268 IF_SIZEOF64(2); 269 IF_SIZEOF64(3); 270 IF_SIZEOF64(4); 271 IF_SIZEOF64(5); 272 IF_SIZEOF64(6); 273 IF_SIZEOF64(7); 274 IF_SIZEOF64(8); 275 IF_SIZEOF64(9); 276 277 return 10; 278 } 279 280 static inline int 281 xd3_decode_uint64_t (xd3_stream *stream, uint64_t *val) 282 { DECODE_INTEGER_TYPE (stream->dec_64part, UINT64_OFLOW_MASK); } 283 284 static inline int 285 xd3_read_uint64_t (xd3_stream *stream, const uint8_t **inpp, 286 const uint8_t *maxp, uint64_t *valp) 287 { READ_INTEGER_TYPE (uint64_t, UINT64_OFLOW_MASK); } 288 289 #if XD3_ENCODER 290 static inline int 291 xd3_emit_uint64_t (xd3_stream *stream, xd3_output **output, uint64_t num) 292 { EMIT_INTEGER_TYPE (); } 293 #endif /* XD3_ENCODER */ 294 #endif /* USE_UINT64 */ 295 296 #if SIZEOF_USIZE_T == 4 297 #define USIZE_T_MAX UINT32_MAX 298 #define USIZE_T_MAXBLKSZ 0x80000000U 299 #define XD3_MAXSRCWINSZ (1ULL << 31) 300 #define xd3_large_cksum xd3_large32_cksum 301 #define xd3_large_cksum_update xd3_large32_cksum_update 302 #define xd3_hash_multiplier xd3_hash_multiplier32 303 304 static inline uint32_t xd3_sizeof_size (usize_t num) 305 { return xd3_sizeof_uint32_t (num); } 306 static inline int xd3_decode_size (xd3_stream *stream, usize_t *valp) 307 { return xd3_decode_uint32_t (stream, (uint32_t*) valp); } 308 static inline int xd3_read_size (xd3_stream *stream, const uint8_t **inpp, 309 const uint8_t *maxp, usize_t *valp) 310 { return xd3_read_uint32_t (stream, inpp, maxp, (uint32_t*) valp); } 311 #if XD3_ENCODER 312 static inline int xd3_emit_size (xd3_stream *stream, xd3_output **output, usize_t num) 313 { return xd3_emit_uint32_t (stream, output, num); } 314 #endif 315 316 #elif SIZEOF_USIZE_T == 8 317 #define USIZE_T_MAX UINT64_MAX 318 #define USIZE_T_MAXBLKSZ 0x8000000000000000ULL 319 #define XD3_MAXSRCWINSZ (1ULL << 61) 320 #define xd3_large_cksum xd3_large64_cksum 321 #define xd3_large_cksum_update xd3_large64_cksum_update 322 #define xd3_hash_multiplier xd3_hash_multiplier64 323 324 static inline uint32_t xd3_sizeof_size (usize_t num) 325 { return xd3_sizeof_uint64_t (num); } 326 static inline int xd3_decode_size (xd3_stream *stream, usize_t *valp) 327 { return xd3_decode_uint64_t (stream, (uint64_t*) valp); } 328 static inline int xd3_read_size (xd3_stream *stream, const uint8_t **inpp, 329 const uint8_t *maxp, usize_t *valp) 330 { return xd3_read_uint64_t (stream, inpp, maxp, (uint64_t*) valp); } 331 #if XD3_ENCODER 332 static inline int xd3_emit_size (xd3_stream *stream, xd3_output **output, usize_t num) 333 { return xd3_emit_uint64_t (stream, output, num); } 334 #endif 335 336 #endif /* SIZEOF_USIZE_T */ 337 338 #if SIZEOF_XOFF_T == 4 339 #define XOFF_T_MAX UINT32_MAX 340 341 static inline int xd3_decode_offset (xd3_stream *stream, xoff_t *valp) 342 { return xd3_decode_uint32_t (stream, (uint32_t*) valp); } 343 #if XD3_ENCODER 344 static inline int xd3_emit_offset (xd3_stream *stream, xd3_output **output, xoff_t num) 345 { return xd3_emit_uint32_t (stream, output, num); } 346 #endif 347 348 #elif SIZEOF_XOFF_T == 8 349 #define XOFF_T_MAX UINT64_MAX 350 351 static inline int xd3_decode_offset (xd3_stream *stream, xoff_t *valp) 352 { return xd3_decode_uint64_t (stream, (uint64_t*) valp); } 353 #if XD3_ENCODER 354 static inline int xd3_emit_offset (xd3_stream *stream, xd3_output **output, xoff_t num) 355 { return xd3_emit_uint64_t (stream, output, num); } 356 #endif 357 358 #endif 359 360 #define USIZE_T_OVERFLOW(a,b) ((USIZE_T_MAX - (usize_t) (a)) < (usize_t) (b)) 361 #define XOFF_T_OVERFLOW(a,b) ((XOFF_T_MAX - (xoff_t) (a)) < (xoff_t) (b)) 362 363 int xd3_size_hashtable (xd3_stream *stream, 364 usize_t slots, 365 usize_t look, 366 xd3_hash_cfg *cfg); 367 368 usize_t xd3_checksum_hash (const xd3_hash_cfg *cfg, const usize_t cksum); 369 370 #if USE_UINT32 371 uint32_t xd3_large32_cksum (xd3_hash_cfg *cfg, const uint8_t *base, const usize_t look); 372 uint32_t xd3_large32_cksum_update (xd3_hash_cfg *cfg, const uint32_t cksum, 373 const uint8_t *base, const usize_t look); 374 #endif /* USE_UINT32 */ 375 376 #if USE_UINT64 377 uint64_t xd3_large64_cksum (xd3_hash_cfg *cfg, const uint8_t *base, const usize_t look); 378 uint64_t xd3_large64_cksum_update (xd3_hash_cfg *cfg, const uint64_t cksum, 379 const uint8_t *base, const usize_t look); 380 #endif /* USE_UINT64 */ 381 382 #define MAX_LRU_SIZE 32U 383 #define XD3_MINSRCWINSZ (XD3_ALLOCSIZE * MAX_LRU_SIZE) 384 385 #endif // XDELTA3_INTERNAL_H__ 386