1 /**********************************************************************
2 Copyright(c) 2011-2016 Intel Corporation All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions
6 are met:
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in
11 the documentation and/or other materials provided with the
12 distribution.
13 * Neither the name of Intel Corporation nor the names of its
14 contributors may be used to endorse or promote products derived
15 from this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 **********************************************************************/
29
30 #define ASM
31
32 #include <assert.h>
33 #include <string.h>
34 #include <wchar.h>
35 #ifdef _WIN32
36 # include <intrin.h>
37 #endif
38
39 #define MAX_WRITE_BITS_SIZE 8
40 #define FORCE_FLUSH 64
41 #define MIN_OBUF_SIZE 224
42 #define NON_EMPTY_BLOCK_SIZE 6
43 #define MAX_SYNC_FLUSH_SIZE NON_EMPTY_BLOCK_SIZE + MAX_WRITE_BITS_SIZE
44
45 #include "huffman.h"
46 #include "bitbuf2.h"
47 #include "igzip_lib.h"
48 #include "crc.h"
49 #include "repeated_char_result.h"
50 #include "huff_codes.h"
51 #include "encode_df.h"
52 #include "igzip_level_buf_structs.h"
53 #include "igzip_checksums.h"
54 #include "igzip_wrapper.h"
55 #include "unaligned.h"
56
57 #if defined(__FreeBSD__) || defined(__DragonFly__)
58 #include <sys/types.h>
59 #include <sys/endian.h>
60 # define to_be32(x) bswap32(x)
61 #elif defined (__APPLE__)
62 #include <libkern/OSByteOrder.h>
63 # define to_be32(x) OSSwapInt32(x)
64 #elif defined (__GNUC__) && !defined (__MINGW32__)
65 # include <byteswap.h>
66 # define to_be32(x) bswap_32(x)
67 #elif defined _WIN64
68 # define to_be32(x) _byteswap_ulong(x)
69 #endif
70
71 extern void isal_deflate_hash_lvl0(uint16_t *, uint32_t, uint32_t, uint8_t *, uint32_t);
72 extern void isal_deflate_hash_lvl1(uint16_t *, uint32_t, uint32_t, uint8_t *, uint32_t);
73 extern void isal_deflate_hash_lvl2(uint16_t *, uint32_t, uint32_t, uint8_t *, uint32_t);
74 extern void isal_deflate_hash_lvl3(uint16_t *, uint32_t, uint32_t, uint8_t *, uint32_t);
75 extern const uint8_t gzip_hdr[];
76 extern const uint32_t gzip_hdr_bytes;
77 extern const uint32_t gzip_trl_bytes;
78 extern const uint8_t zlib_hdr[];
79 extern const uint32_t zlib_hdr_bytes;
80 extern const uint32_t zlib_trl_bytes;
81 extern const struct isal_hufftables hufftables_default;
82 extern const struct isal_hufftables hufftables_static;
83
84 static uint32_t write_stored_block(struct isal_zstream *stream);
85
86 static int write_stream_header_stateless(struct isal_zstream *stream);
87 static void write_stream_header(struct isal_zstream *stream);
88 static int write_deflate_header_stateless(struct isal_zstream *stream);
89 static int write_deflate_header_unaligned_stateless(struct isal_zstream *stream);
90
91 #define TYPE0_HDR_LEN 4
92 #define TYPE0_BLK_HDR_LEN 5
93 #define TYPE0_MAX_BLK_LEN 65535
94
95 void isal_deflate_body(struct isal_zstream *stream);
96 void isal_deflate_finish(struct isal_zstream *stream);
97
98 void isal_deflate_icf_body(struct isal_zstream *stream);
99 void isal_deflate_icf_finish_lvl1(struct isal_zstream *stream);
100 void isal_deflate_icf_finish_lvl2(struct isal_zstream *stream);
101 void isal_deflate_icf_finish_lvl3(struct isal_zstream *stream);
102 /*****************************************************************/
103
104 /* Forward declarations */
105 static inline void reset_match_history(struct isal_zstream *stream);
106 static void write_header(struct isal_zstream *stream, uint8_t * deflate_hdr,
107 uint32_t deflate_hdr_count, uint32_t extra_bits_count,
108 uint32_t next_state, uint32_t toggle_end_of_stream);
109 static void write_trailer(struct isal_zstream *stream);
110
111 struct slver {
112 uint16_t snum;
113 uint8_t ver;
114 uint8_t core;
115 };
116
117 /* Version info */
118 struct slver isal_deflate_init_slver_01030081;
119 struct slver isal_deflate_init_slver = { 0x0081, 0x03, 0x01 };
120
121 struct slver isal_deflate_reset_slver_0001008e;
122 struct slver isal_deflate_reset_slver = { 0x008e, 0x01, 0x00 };
123
124 struct slver isal_deflate_stateless_init_slver_00010084;
125 struct slver isal_deflate_stateless_init_slver = { 0x0084, 0x01, 0x00 };
126
127 struct slver isal_deflate_slver_01030082;
128 struct slver isal_deflate_slver = { 0x0082, 0x03, 0x01 };
129
130 struct slver isal_deflate_stateless_slver_01010083;
131 struct slver isal_deflate_stateless_slver = { 0x0083, 0x01, 0x01 };
132
133 struct slver isal_deflate_set_hufftables_slver_0001008b;
134 struct slver isal_deflate_set_hufftables_slver = { 0x008b, 0x01, 0x00 };
135
136 struct slver isal_deflate_set_dict_slver_0001008c;
137 struct slver isal_deflate_set_dict_slver = { 0x008c, 0x01, 0x00 };
138
139 /*****************************************************************/
140
141 // isal_adler32_bam1 - adler with (B | A minus 1) storage
142
isal_adler32_bam1(uint32_t adler32,const unsigned char * start,uint64_t length)143 uint32_t isal_adler32_bam1(uint32_t adler32, const unsigned char *start, uint64_t length)
144 {
145 uint64_t a;
146
147 /* Internally the checksum is being stored as B | (A-1) so crc and
148 * addler have same init value */
149 a = adler32 & 0xffff;
150 a = (a == ADLER_MOD - 1) ? 0 : a + 1;
151 adler32 = isal_adler32((adler32 & 0xffff0000) | a, start, length);
152 a = (adler32 & 0xffff);
153 a = (a == 0) ? ADLER_MOD - 1 : a - 1;
154
155 return (adler32 & 0xffff0000) | a;
156 }
157
update_checksum(struct isal_zstream * stream,uint8_t * start_in,uint64_t length)158 static void update_checksum(struct isal_zstream *stream, uint8_t * start_in, uint64_t length)
159 {
160 struct isal_zstate *state = &stream->internal_state;
161 switch (stream->gzip_flag) {
162 case IGZIP_GZIP:
163 case IGZIP_GZIP_NO_HDR:
164 state->crc = crc32_gzip_refl(state->crc, start_in, length);
165 break;
166 case IGZIP_ZLIB:
167 case IGZIP_ZLIB_NO_HDR:
168 state->crc = isal_adler32_bam1(state->crc, start_in, length);
169 break;
170 }
171 }
172
173 static
sync_flush(struct isal_zstream * stream)174 void sync_flush(struct isal_zstream *stream)
175 {
176 struct isal_zstate *state = &stream->internal_state;
177 uint64_t bits_to_write = 0xFFFF0000, bits_len;
178 uint64_t bytes;
179 int flush_size;
180
181 if (stream->avail_out >= 8) {
182 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
183
184 flush_size = (-(state->bitbuf.m_bit_count + 3)) % 8;
185
186 bits_to_write <<= flush_size + 3;
187 bits_len = 32 + flush_size + 3;
188
189 state->state = ZSTATE_NEW_HDR;
190 state->has_eob = 0;
191
192 write_bits(&state->bitbuf, bits_to_write, bits_len);
193
194 bytes = buffer_used(&state->bitbuf);
195 stream->next_out = buffer_ptr(&state->bitbuf);
196 stream->avail_out -= bytes;
197 stream->total_out += bytes;
198
199 if (stream->flush == FULL_FLUSH) {
200 /* Clear match history so there are no cross
201 * block length distance pairs */
202 state->has_hist = IGZIP_NO_HIST;
203 }
204 }
205 }
206
flush_write_buffer(struct isal_zstream * stream)207 static void flush_write_buffer(struct isal_zstream *stream)
208 {
209 struct isal_zstate *state = &stream->internal_state;
210 int bytes = 0;
211 if (stream->avail_out >= 8) {
212 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
213 flush(&state->bitbuf);
214 stream->next_out = buffer_ptr(&state->bitbuf);
215 bytes = buffer_used(&state->bitbuf);
216 stream->avail_out -= bytes;
217 stream->total_out += bytes;
218 state->state = ZSTATE_NEW_HDR;
219 }
220 }
221
flush_icf_block(struct isal_zstream * stream)222 static void flush_icf_block(struct isal_zstream *stream)
223 {
224 struct isal_zstate *state = &stream->internal_state;
225 struct level_buf *level_buf = (struct level_buf *)stream->level_buf;
226 struct BitBuf2 *write_buf = &state->bitbuf;
227 struct deflate_icf *icf_buf_encoded_next;
228
229 set_buf(write_buf, stream->next_out, stream->avail_out);
230
231 icf_buf_encoded_next = encode_deflate_icf(level_buf->icf_buf_start + state->count,
232 level_buf->icf_buf_next, write_buf,
233 &level_buf->encode_tables);
234
235 state->count = icf_buf_encoded_next - level_buf->icf_buf_start;
236 stream->next_out = buffer_ptr(write_buf);
237 stream->total_out += buffer_used(write_buf);
238 stream->avail_out -= buffer_used(write_buf);
239
240 if (level_buf->icf_buf_next <= icf_buf_encoded_next) {
241 state->count = 0;
242 if (stream->avail_in == 0 && stream->end_of_stream)
243 state->state = ZSTATE_TRL;
244 else if (stream->avail_in == 0 && stream->flush != NO_FLUSH)
245 state->state = ZSTATE_SYNC_FLUSH;
246 else
247 state->state = ZSTATE_NEW_HDR;
248 }
249 }
250
check_level_req(struct isal_zstream * stream)251 static int check_level_req(struct isal_zstream *stream)
252 {
253 if (stream->level == 0)
254 return 0;
255
256 if (stream->level_buf == NULL)
257 return ISAL_INVALID_LEVEL_BUF;
258
259 switch (stream->level) {
260 case 3:
261 if (stream->level_buf_size < ISAL_DEF_LVL3_MIN)
262 return ISAL_INVALID_LEVEL;
263 break;
264
265 case 2:
266 if (stream->level_buf_size < ISAL_DEF_LVL2_MIN)
267 return ISAL_INVALID_LEVEL;
268 break;
269 case 1:
270 if (stream->level_buf_size < ISAL_DEF_LVL1_MIN)
271 return ISAL_INVALID_LEVEL;
272 break;
273 default:
274 return ISAL_INVALID_LEVEL;
275 }
276
277 return 0;
278 }
279
init_hash8k_buf(struct isal_zstream * stream)280 static int init_hash8k_buf(struct isal_zstream *stream)
281 {
282 struct isal_zstate *state = &stream->internal_state;
283 struct level_buf *level_buf = (struct level_buf *)stream->level_buf;
284 state->has_level_buf_init = 1;
285 return sizeof(struct level_buf) - MAX_LVL_BUF_SIZE + sizeof(level_buf->hash8k);
286 }
287
init_hash_hist_buf(struct isal_zstream * stream)288 static int init_hash_hist_buf(struct isal_zstream *stream)
289 {
290 struct isal_zstate *state = &stream->internal_state;
291 struct level_buf *level_buf = (struct level_buf *)stream->level_buf;
292 state->has_level_buf_init = 1;
293 return sizeof(struct level_buf) - MAX_LVL_BUF_SIZE + sizeof(level_buf->hash_hist);
294 }
295
init_hash_map_buf(struct isal_zstream * stream)296 static int init_hash_map_buf(struct isal_zstream *stream)
297 {
298 struct isal_zstate *state = &stream->internal_state;
299 struct level_buf *level_buf = (struct level_buf *)stream->level_buf;
300 if (!state->has_level_buf_init) {
301 level_buf->hash_map.matches_next = level_buf->hash_map.matches;
302 level_buf->hash_map.matches_end = level_buf->hash_map.matches;
303 }
304 state->has_level_buf_init = 1;
305
306 return sizeof(struct level_buf) - MAX_LVL_BUF_SIZE + sizeof(level_buf->hash_map);
307
308 }
309
310 /* returns the size of the level specific buffer */
init_lvlX_buf(struct isal_zstream * stream)311 static int init_lvlX_buf(struct isal_zstream *stream)
312 {
313 switch (stream->level) {
314 case 3:
315 return init_hash_map_buf(stream);
316 case 2:
317 return init_hash_hist_buf(stream);
318 default:
319 return init_hash8k_buf(stream);
320 }
321
322 }
323
init_new_icf_block(struct isal_zstream * stream)324 static void init_new_icf_block(struct isal_zstream *stream)
325 {
326 struct isal_zstate *state = &stream->internal_state;
327 struct level_buf *level_buf = (struct level_buf *)stream->level_buf;
328 int level_struct_size;
329
330 level_struct_size = init_lvlX_buf(stream);
331
332 state->block_next = state->block_end;
333 level_buf->icf_buf_start =
334 (struct deflate_icf *)(stream->level_buf + level_struct_size);
335
336 level_buf->icf_buf_next = level_buf->icf_buf_start;
337 level_buf->icf_buf_avail_out =
338 stream->level_buf_size - level_struct_size - sizeof(struct deflate_icf);
339
340 memset(&level_buf->hist, 0, sizeof(struct isal_mod_hist));
341 state->state = ZSTATE_BODY;
342 }
343
are_buffers_empty_hashX(struct isal_zstream * stream)344 static int are_buffers_empty_hashX(struct isal_zstream *stream)
345 {
346 return !stream->avail_in;
347 }
348
are_buffers_empty_hash_map(struct isal_zstream * stream)349 static int are_buffers_empty_hash_map(struct isal_zstream *stream)
350 {
351 struct level_buf *level_buf = (struct level_buf *)stream->level_buf;
352
353 return (!stream->avail_in
354 && level_buf->hash_map.matches_next >= level_buf->hash_map.matches_end);
355 }
356
are_buffers_empty(struct isal_zstream * stream)357 static int are_buffers_empty(struct isal_zstream *stream)
358 {
359
360 switch (stream->level) {
361 case 3:
362 return are_buffers_empty_hash_map(stream);
363 case 2:
364 return are_buffers_empty_hashX(stream);
365 default:
366 return are_buffers_empty_hashX(stream);
367 }
368 }
369
create_icf_block_hdr(struct isal_zstream * stream,uint8_t * start_in)370 static void create_icf_block_hdr(struct isal_zstream *stream, uint8_t * start_in)
371 {
372 struct isal_zstate *state = &stream->internal_state;
373 struct level_buf *level_buf = (struct level_buf *)stream->level_buf;
374 struct BitBuf2 *write_buf = &state->bitbuf;
375 struct BitBuf2 write_buf_tmp;
376 uint32_t out_size = stream->avail_out;
377 uint32_t avail_output, block_start_offset;
378 uint8_t *end_out = stream->next_out + out_size;
379 uint64_t cur_in_processed;
380 uint64_t bit_count;
381 uint64_t block_in_size = state->block_end - state->block_next;
382 uint64_t block_size;
383 int buffer_header = 0;
384
385 memcpy(&write_buf_tmp, write_buf, sizeof(struct BitBuf2));
386
387 /* Calculate the bytes required to store a type 0 block. Need to account
388 * for bits stored in the bitbuf. Since 3 bits correspond to the deflate
389 * type 0 header, we need to add one byte more when the number of bits
390 * is at least 6 mod 8. */
391 block_size = (TYPE0_BLK_HDR_LEN) * ((block_in_size + TYPE0_MAX_BLK_LEN - 1) /
392 TYPE0_MAX_BLK_LEN) + block_in_size;
393 block_size = block_size ? block_size : TYPE0_BLK_HDR_LEN;
394 block_size += (write_buf->m_bit_count + 2) / 8;
395
396 /* Write EOB in icf_buf */
397 level_buf->hist.ll_hist[256] = 1;
398 level_buf->icf_buf_next->lit_len = 0x100;
399 level_buf->icf_buf_next->lit_dist = NULL_DIST_SYM;
400 level_buf->icf_buf_next->dist_extra = 0;
401 level_buf->icf_buf_next++;
402
403 state->has_eob_hdr = (stream->end_of_stream && are_buffers_empty(stream)) ? 1 : 0;
404
405 if (end_out - stream->next_out >= ISAL_DEF_MAX_HDR_SIZE) {
406 /* Assumes ISAL_DEF_MAX_HDR_SIZE is large enough to contain a
407 * max length header and a gzip header */
408 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB)
409 write_stream_header_stateless(stream);
410 set_buf(write_buf, stream->next_out, stream->avail_out);
411 buffer_header = 0;
412
413 } else {
414 /* Start writing into temporary buffer */
415 set_buf(write_buf, level_buf->deflate_hdr, ISAL_DEF_MAX_HDR_SIZE);
416 buffer_header = 1;
417 }
418
419 bit_count = create_hufftables_icf(write_buf, &level_buf->encode_tables,
420 &level_buf->hist, state->has_eob_hdr);
421
422 /* Assumes that type 0 block has size less than 4G */
423 block_start_offset = (stream->total_in - state->block_next);
424 cur_in_processed = stream->next_in - start_in;
425 avail_output = stream->avail_out + sizeof(state->buffer) -
426 (stream->total_in - state->block_end);
427
428 if (bit_count / 8 >= block_size && cur_in_processed >= block_start_offset
429 && block_size <= avail_output) {
430 /* Reset stream for writing out a type0 block */
431 state->has_eob_hdr = 0;
432 memcpy(write_buf, &write_buf_tmp, sizeof(struct BitBuf2));
433 state->state = ZSTATE_TYPE0_HDR;
434
435 } else if (buffer_header) {
436 /* Setup stream to write out a buffered header */
437 level_buf->deflate_hdr_count = buffer_used(write_buf);
438 level_buf->deflate_hdr_extra_bits = write_buf->m_bit_count;
439 flush(write_buf);
440 memcpy(write_buf, &write_buf_tmp, sizeof(struct BitBuf2));
441 write_buf->m_bits = 0;
442 write_buf->m_bit_count = 0;
443 state->state = ZSTATE_HDR;
444
445 } else {
446 stream->next_out = buffer_ptr(write_buf);
447 stream->total_out += buffer_used(write_buf);
448 stream->avail_out -= buffer_used(write_buf);
449 state->state = ZSTATE_FLUSH_ICF_BUFFER;
450 }
451 }
452
isal_deflate_pass(struct isal_zstream * stream)453 static void isal_deflate_pass(struct isal_zstream *stream)
454 {
455 struct isal_zstate *state = &stream->internal_state;
456 struct isal_hufftables *hufftables = stream->hufftables;
457 uint8_t *start_in = stream->next_in;
458
459 if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) {
460 if (state->count == 0)
461 /* Assume the final header is being written since the header
462 * stored in hufftables is the final header. */
463 state->has_eob_hdr = 1;
464 write_header(stream, hufftables->deflate_hdr, hufftables->deflate_hdr_count,
465 hufftables->deflate_hdr_extra_bits, ZSTATE_BODY,
466 !stream->end_of_stream);
467 }
468
469 if (state->state == ZSTATE_BODY)
470 isal_deflate_body(stream);
471
472 if (state->state == ZSTATE_FLUSH_READ_BUFFER)
473 isal_deflate_finish(stream);
474 if (state->state == ZSTATE_SYNC_FLUSH)
475 sync_flush(stream);
476
477 if (state->state == ZSTATE_FLUSH_WRITE_BUFFER)
478 flush_write_buffer(stream);
479
480 if (stream->gzip_flag)
481 update_checksum(stream, start_in, stream->next_in - start_in);
482
483 if (state->state == ZSTATE_TRL)
484 write_trailer(stream);
485 }
486
isal_deflate_icf_finish(struct isal_zstream * stream)487 static void isal_deflate_icf_finish(struct isal_zstream *stream)
488 {
489 switch (stream->level) {
490 case 3:
491 isal_deflate_icf_finish_lvl3(stream);
492 break;
493 case 2:
494 isal_deflate_icf_finish_lvl2(stream);
495 break;
496 default:
497 isal_deflate_icf_finish_lvl1(stream);
498 }
499 }
500
isal_deflate_icf_pass(struct isal_zstream * stream,uint8_t * inbuf_start)501 static void isal_deflate_icf_pass(struct isal_zstream *stream, uint8_t * inbuf_start)
502 {
503 uint8_t *start_in = stream->next_in;
504 struct isal_zstate *state = &stream->internal_state;
505 struct level_buf *level_buf = (struct level_buf *)stream->level_buf;
506
507 do {
508 if (state->state == ZSTATE_NEW_HDR)
509 init_new_icf_block(stream);
510
511 if (state->state == ZSTATE_BODY)
512 isal_deflate_icf_body(stream);
513
514 if (state->state == ZSTATE_FLUSH_READ_BUFFER)
515 isal_deflate_icf_finish(stream);
516
517 if (state->state == ZSTATE_CREATE_HDR)
518 create_icf_block_hdr(stream, inbuf_start);
519
520 if (state->state == ZSTATE_HDR)
521 /* Note that the header may be prepended by the
522 * remaining bits in the previous block, as such the
523 * toggle header flag cannot be used */
524 write_header(stream, level_buf->deflate_hdr,
525 level_buf->deflate_hdr_count,
526 level_buf->deflate_hdr_extra_bits,
527 ZSTATE_FLUSH_ICF_BUFFER, 0);
528
529 if (state->state == ZSTATE_FLUSH_ICF_BUFFER)
530 flush_icf_block(stream);
531
532 if (state->state == ZSTATE_TYPE0_HDR || state->state == ZSTATE_TYPE0_BODY) {
533 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB)
534 write_stream_header(stream);
535 write_stored_block(stream);
536 }
537
538 }
539 while (state->state == ZSTATE_NEW_HDR);
540
541 if (state->state == ZSTATE_SYNC_FLUSH)
542 sync_flush(stream);
543
544 if (state->state == ZSTATE_FLUSH_WRITE_BUFFER)
545 flush_write_buffer(stream);
546
547 if (stream->gzip_flag)
548 update_checksum(stream, start_in, stream->next_in - start_in);
549
550 if (state->state == ZSTATE_TRL)
551 write_trailer(stream);
552 }
553
isal_deflate_int(struct isal_zstream * stream,uint8_t * start_in)554 static void isal_deflate_int(struct isal_zstream *stream, uint8_t * start_in)
555 {
556 struct isal_zstate *state = &stream->internal_state;
557 uint32_t size;
558
559 /* Move data from temporary output buffer to output buffer */
560 if (state->state >= ZSTATE_TMP_OFFSET) {
561 size = state->tmp_out_end - state->tmp_out_start;
562 if (size > stream->avail_out)
563 size = stream->avail_out;
564 memcpy(stream->next_out, state->tmp_out_buff + state->tmp_out_start, size);
565 stream->next_out += size;
566 stream->avail_out -= size;
567 stream->total_out += size;
568 state->tmp_out_start += size;
569
570 if (state->tmp_out_start == state->tmp_out_end)
571 state->state -= ZSTATE_TMP_OFFSET;
572
573 if (stream->avail_out == 0 || state->state == ZSTATE_END
574 // or do not write out empty blocks since the outbuffer was processed
575 || (state->state == ZSTATE_NEW_HDR && stream->avail_out == 0))
576 return;
577 }
578 assert(state->tmp_out_start == state->tmp_out_end);
579
580 if (stream->level == 0)
581 isal_deflate_pass(stream);
582 else
583 isal_deflate_icf_pass(stream, start_in);
584
585 /* Fill temporary output buffer then complete filling output buffer */
586 if (stream->avail_out > 0 && stream->avail_out < 8 && state->state != ZSTATE_NEW_HDR) {
587 uint8_t *next_out;
588 uint32_t avail_out;
589 uint32_t total_out;
590
591 next_out = stream->next_out;
592 avail_out = stream->avail_out;
593 total_out = stream->total_out;
594
595 stream->next_out = state->tmp_out_buff;
596 stream->avail_out = sizeof(state->tmp_out_buff);
597 stream->total_out = 0;
598
599 if (stream->level == 0)
600 isal_deflate_pass(stream);
601 else
602 isal_deflate_icf_pass(stream, start_in);
603
604 state->tmp_out_start = 0;
605 state->tmp_out_end = stream->total_out;
606
607 stream->next_out = next_out;
608 stream->avail_out = avail_out;
609 stream->total_out = total_out;
610 if (state->tmp_out_end) {
611 size = state->tmp_out_end;
612 if (size > stream->avail_out)
613 size = stream->avail_out;
614 memcpy(stream->next_out, state->tmp_out_buff, size);
615 stream->next_out += size;
616 stream->avail_out -= size;
617 stream->total_out += size;
618 state->tmp_out_start += size;
619 if (state->tmp_out_start != state->tmp_out_end)
620 state->state += ZSTATE_TMP_OFFSET;
621
622 }
623 }
624
625 }
626
write_constant_compressed_stateless(struct isal_zstream * stream,uint32_t repeated_length)627 static void write_constant_compressed_stateless(struct isal_zstream *stream,
628 uint32_t repeated_length)
629 {
630 /* Assumes repeated_length is at least 1.
631 * Assumes the input end_of_stream is either 0 or 1. */
632 struct isal_zstate *state = &stream->internal_state;
633 uint32_t rep_bits = ((repeated_length - 1) / 258) * 2;
634 uint32_t rep_bytes = rep_bits / 8;
635 uint32_t rep_extra = (repeated_length - 1) % 258;
636 uint32_t bytes;
637 uint32_t repeated_char = *stream->next_in;
638 uint8_t *start_in = stream->next_in;
639
640 /* Guarantee there is enough space for the header even in the worst case */
641 if (stream->avail_out < HEADER_LENGTH + MAX_FIXUP_CODE_LENGTH + rep_bytes + 8)
642 return;
643
644 /* Assumes the repeated char is either 0 or 0xFF. */
645 memcpy(stream->next_out, repeated_char_header[repeated_char & 1], HEADER_LENGTH);
646
647 if (stream->avail_in == repeated_length && stream->end_of_stream > 0) {
648 stream->next_out[0] |= 1;
649 state->has_eob_hdr = 1;
650 state->has_eob = 1;
651 state->state = ZSTATE_TRL;
652 } else {
653 state->state = ZSTATE_NEW_HDR;
654 }
655
656 memset(stream->next_out + HEADER_LENGTH, 0, rep_bytes);
657 stream->avail_out -= HEADER_LENGTH + rep_bytes;
658 stream->next_out += HEADER_LENGTH + rep_bytes;
659 stream->total_out += HEADER_LENGTH + rep_bytes;
660
661 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
662
663 /* These two lines are basically a modified version of init. */
664 state->bitbuf.m_bits = 0;
665 state->bitbuf.m_bit_count = rep_bits % 8;
666
667 /* Add smaller repeat codes as necessary. Code280 can describe repeat
668 * lengths of 115-130 bits. Code10 can describe repeat lengths of 10
669 * bits. If more than 230 bits, fill code with two code280s. Else if
670 * more than 115 repeates, fill with code10s until one code280 can
671 * finish the rest of the repeats. Else, fill with code10s and
672 * literals */
673 if (rep_extra > 115) {
674 while (rep_extra > 130 && rep_extra < 230) {
675 write_bits(&state->bitbuf, CODE_10, CODE_10_LENGTH);
676 rep_extra -= 10;
677 }
678
679 if (rep_extra >= 230) {
680 write_bits(&state->bitbuf,
681 CODE_280 | ((rep_extra / 2 - 115) <<
682 CODE_280_LENGTH), CODE_280_TOTAL_LENGTH);
683 rep_extra -= rep_extra / 2;
684 }
685
686 write_bits(&state->bitbuf,
687 CODE_280 | ((rep_extra - 115) << CODE_280_LENGTH),
688 CODE_280_TOTAL_LENGTH);
689
690 } else {
691 while (rep_extra >= 10) {
692
693 write_bits(&state->bitbuf, CODE_10, CODE_10_LENGTH);
694 rep_extra -= 10;
695 }
696
697 for (; rep_extra > 0; rep_extra--)
698 write_bits(&state->bitbuf, CODE_LIT, CODE_LIT_LENGTH);
699 }
700
701 write_bits(&state->bitbuf, END_OF_BLOCK, END_OF_BLOCK_LEN);
702
703 stream->next_in += repeated_length;
704 stream->avail_in -= repeated_length;
705 stream->total_in += repeated_length;
706 state->block_end += repeated_length;
707
708 bytes = buffer_used(&state->bitbuf);
709 stream->next_out = buffer_ptr(&state->bitbuf);
710 stream->avail_out -= bytes;
711 stream->total_out += bytes;
712
713 if (stream->gzip_flag)
714 update_checksum(stream, start_in, stream->next_in - start_in);
715
716 return;
717 }
718
detect_repeated_char_length(uint8_t * in,uint32_t length)719 static int detect_repeated_char_length(uint8_t * in, uint32_t length)
720 {
721 /* This currently assumes the first 8 bytes are the same character.
722 * This won't work effectively if the input stream isn't aligned well. */
723 uint8_t *p_8, *end = in + length;
724 uint64_t *p_64 = (uint64_t *) in;
725 uint64_t w = *p_64;
726 uint8_t c = (uint8_t) w;
727
728 for (; (p_64 <= (uint64_t *) (end - 8)) && (w == *p_64); p_64++) ;
729
730 p_8 = (uint8_t *) p_64;
731
732 for (; (p_8 < end) && (c == *p_8); p_8++) ;
733
734 return p_8 - in;
735 }
736
isal_deflate_int_stateless(struct isal_zstream * stream)737 static int isal_deflate_int_stateless(struct isal_zstream *stream)
738 {
739 uint32_t repeat_length;
740 struct isal_zstate *state = &stream->internal_state;
741
742 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB)
743 if (write_stream_header_stateless(stream))
744 return STATELESS_OVERFLOW;
745
746 if (stream->avail_in >= 8
747 && (load_u64(stream->next_in) == 0
748 || load_u64(stream->next_in) == ~(uint64_t) 0)) {
749 repeat_length = detect_repeated_char_length(stream->next_in, stream->avail_in);
750
751 if (stream->avail_in == repeat_length || repeat_length >= MIN_REPEAT_LEN)
752 write_constant_compressed_stateless(stream, repeat_length);
753 }
754
755 if (stream->level == 0) {
756 if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) {
757 write_deflate_header_unaligned_stateless(stream);
758 if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR)
759 return STATELESS_OVERFLOW;
760
761 reset_match_history(stream);
762 }
763
764 isal_deflate_pass(stream);
765
766 } else if (stream->level <= ISAL_DEF_MAX_LEVEL) {
767 if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR)
768 reset_match_history(stream);
769
770 state->count = 0;
771 isal_deflate_icf_pass(stream, stream->next_in);
772
773 }
774
775 if (state->state == ZSTATE_END
776 || (state->state == ZSTATE_NEW_HDR && stream->flush == FULL_FLUSH))
777 return COMP_OK;
778 else
779 return STATELESS_OVERFLOW;
780 }
781
write_type0_header(struct isal_zstream * stream)782 static void write_type0_header(struct isal_zstream *stream)
783 {
784 struct isal_zstate *state = &stream->internal_state;
785 uint64_t stored_blk_hdr;
786 uint32_t copy_size;
787 uint32_t memcpy_len, avail_in;
788 uint32_t block_in_size = state->block_end - state->block_next;
789 uint32_t block_next_offset;
790 struct BitBuf2 *bitbuf = &stream->internal_state.bitbuf;
791
792 if (block_in_size > TYPE0_MAX_BLK_LEN) {
793 stored_blk_hdr = 0xFFFF;
794 copy_size = TYPE0_MAX_BLK_LEN;
795 } else {
796 stored_blk_hdr = ~block_in_size;
797 stored_blk_hdr <<= 16;
798 stored_blk_hdr |= (block_in_size & 0xFFFF);
799 copy_size = block_in_size;
800
801 /* Handle BFINAL bit */
802 block_next_offset = stream->total_in - state->block_next;
803 avail_in = stream->avail_in + block_next_offset;
804 if (stream->end_of_stream && avail_in == block_in_size)
805 stream->internal_state.has_eob_hdr = 1;
806 }
807
808 if (bitbuf->m_bit_count == 0 && stream->avail_out >= TYPE0_HDR_LEN + 1) {
809 stored_blk_hdr = stored_blk_hdr << 8;
810 stored_blk_hdr |= stream->internal_state.has_eob_hdr;
811 memcpy_len = TYPE0_HDR_LEN + 1;
812 memcpy(stream->next_out, &stored_blk_hdr, memcpy_len);
813 } else if (stream->avail_out >= 8) {
814 set_buf(bitbuf, stream->next_out, stream->avail_out);
815 write_bits_flush(bitbuf, stream->internal_state.has_eob_hdr, 3);
816 stream->next_out = buffer_ptr(bitbuf);
817 stream->total_out += buffer_used(bitbuf);
818 stream->avail_out -= buffer_used(bitbuf);
819 memcpy_len = TYPE0_HDR_LEN;
820 memcpy(stream->next_out, &stored_blk_hdr, memcpy_len);
821 } else {
822 stream->internal_state.has_eob_hdr = 0;
823 return;
824 }
825
826 stream->next_out += memcpy_len;
827 stream->avail_out -= memcpy_len;
828 stream->total_out += memcpy_len;
829 stream->internal_state.state = ZSTATE_TYPE0_BODY;
830
831 stream->internal_state.count = copy_size;
832 }
833
write_stored_block(struct isal_zstream * stream)834 static uint32_t write_stored_block(struct isal_zstream *stream)
835 {
836 uint32_t copy_size, avail_in, block_next_offset;
837 uint8_t *next_in;
838 struct isal_zstate *state = &stream->internal_state;
839
840 do {
841 if (state->state == ZSTATE_TYPE0_HDR) {
842 write_type0_header(stream);
843 if (state->state == ZSTATE_TYPE0_HDR)
844 break;
845 }
846
847 assert(state->count <= state->block_end - state->block_next);
848 copy_size = state->count;
849
850 block_next_offset = stream->total_in - state->block_next;
851 next_in = stream->next_in - block_next_offset;
852 avail_in = stream->avail_in + block_next_offset;
853
854 if (copy_size > stream->avail_out || copy_size > avail_in) {
855 state->count = copy_size;
856 copy_size = (stream->avail_out <= avail_in) ?
857 stream->avail_out : avail_in;
858
859 memcpy(stream->next_out, next_in, copy_size);
860 state->count -= copy_size;
861 } else {
862 memcpy(stream->next_out, next_in, copy_size);
863
864 state->count = 0;
865 state->state = ZSTATE_TYPE0_HDR;
866 }
867
868 state->block_next += copy_size;
869 stream->next_out += copy_size;
870 stream->avail_out -= copy_size;
871 stream->total_out += copy_size;
872
873 if (state->block_next == state->block_end) {
874 state->state = state->has_eob_hdr ? ZSTATE_TRL : ZSTATE_NEW_HDR;
875 if (stream->flush == FULL_FLUSH && state->state == ZSTATE_NEW_HDR
876 && are_buffers_empty(stream)) {
877 /* Clear match history so there are no cross
878 * block length distance pairs */
879 reset_match_history(stream);
880 }
881 }
882 } while (state->state == ZSTATE_TYPE0_HDR);
883
884 return state->block_end - state->block_next;
885 }
886
reset_match_history(struct isal_zstream * stream)887 static inline void reset_match_history(struct isal_zstream *stream)
888 {
889 struct isal_zstate *state = &stream->internal_state;
890 struct level_buf *level_buf = (struct level_buf *)stream->level_buf;
891 uint16_t *hash_table;
892 uint32_t hash_table_size;
893
894 hash_table_size = 2 * (state->hash_mask + 1);
895
896 switch (stream->level) {
897 case 3:
898 hash_table = level_buf->lvl3.hash_table;
899 break;
900 case 2:
901 hash_table = level_buf->lvl2.hash_table;
902 break;
903 case 1:
904 hash_table = level_buf->lvl1.hash_table;
905 break;
906 default:
907 hash_table = state->head;
908 }
909
910 state->has_hist = IGZIP_NO_HIST;
911
912 /* There is definitely more than 16 bytes in the hash table. Set this
913 * minimum to avoid a wmemset of size 0 */
914 if (hash_table_size <= sizeof(wchar_t))
915 hash_table_size = sizeof(wchar_t);
916
917 if (sizeof(wchar_t) == 2) {
918 uint16_t hash_init_val;
919
920 hash_init_val = stream->total_in & 0xffff;
921 wmemset((wchar_t *)hash_table, hash_init_val,
922 hash_table_size / sizeof(wchar_t));
923
924 } else if (sizeof(wchar_t) == 4) {
925 uint32_t hash_init_val;
926 int rep_bits;
927
928 hash_init_val = stream->total_in & 0xffff;
929 for (rep_bits = sizeof(uint16_t) * 8; rep_bits < sizeof(wchar_t) * 8;
930 rep_bits *= 2)
931 hash_init_val |= hash_init_val << rep_bits;
932
933 wmemset((wchar_t *)hash_table, hash_init_val,
934 hash_table_size / sizeof(wchar_t));
935 } else {
936 if ((stream->total_in & 0xFFFF) == 0)
937 memset(hash_table, 0, hash_table_size);
938 else {
939 int i;
940 for (i = 0; i < hash_table_size / 2; i++) {
941 hash_table[i] = (uint16_t) (stream->total_in);
942 }
943 }
944 }
945
946 }
947
set_dist_mask(struct isal_zstream * stream)948 static void inline set_dist_mask(struct isal_zstream *stream)
949 {
950 struct isal_zstate *state = &stream->internal_state;
951 uint32_t hist_size;
952
953 if (stream->hist_bits > ISAL_DEF_MAX_HIST_BITS || stream->hist_bits == 0)
954 stream->hist_bits = ISAL_DEF_MAX_HIST_BITS;
955
956 hist_size = (1 << (stream->hist_bits));
957 state->dist_mask = hist_size - 1;
958
959 if (IGZIP_HIST_SIZE < ISAL_DEF_HIST_SIZE && state->dist_mask > IGZIP_HIST_SIZE - 1)
960 state->dist_mask = IGZIP_HIST_SIZE - 1;
961 }
962
set_hash_mask(struct isal_zstream * stream)963 static void inline set_hash_mask(struct isal_zstream *stream)
964 {
965 struct isal_zstate *state = &stream->internal_state;
966
967 switch (stream->level) {
968 case 3:
969 state->hash_mask = LVL3_HASH_MASK;
970 break;
971 case 2:
972 state->hash_mask = LVL2_HASH_MASK;
973 break;
974 case 1:
975 state->hash_mask = LVL1_HASH_MASK;
976 break;
977 case 0:
978 state->hash_mask = LVL0_HASH_MASK;
979 }
980 }
981
isal_deflate_init(struct isal_zstream * stream)982 void isal_deflate_init(struct isal_zstream *stream)
983 {
984 struct isal_zstate *state = &stream->internal_state;
985
986 stream->total_in = 0;
987 stream->total_out = 0;
988 stream->hufftables = (struct isal_hufftables *)&hufftables_default;
989 stream->level = 0;
990 stream->level_buf = NULL;
991 stream->level_buf_size = 0;
992 stream->end_of_stream = 0;
993 stream->flush = NO_FLUSH;
994 stream->gzip_flag = 0;
995 stream->hist_bits = 0;
996
997 state->block_next = 0;
998 state->block_end = 0;
999 state->b_bytes_valid = 0;
1000 state->b_bytes_processed = 0;
1001 state->total_in_start = 0;
1002 state->has_wrap_hdr = 0;
1003 state->has_eob = 0;
1004 state->has_eob_hdr = 0;
1005 state->has_hist = IGZIP_NO_HIST;
1006 state->has_level_buf_init = 0;
1007 state->state = ZSTATE_NEW_HDR;
1008 state->count = 0;
1009
1010 state->tmp_out_start = 0;
1011 state->tmp_out_end = 0;
1012
1013 init(&state->bitbuf);
1014
1015 state->crc = 0;
1016
1017 return;
1018 }
1019
isal_deflate_reset(struct isal_zstream * stream)1020 void isal_deflate_reset(struct isal_zstream *stream)
1021 {
1022 struct isal_zstate *state = &stream->internal_state;
1023
1024 stream->total_in = 0;
1025 stream->total_out = 0;
1026
1027 state->block_next = 0;
1028 state->block_end = 0;
1029 state->b_bytes_valid = 0;
1030 state->b_bytes_processed = 0;
1031 state->total_in_start = 0;
1032 state->has_wrap_hdr = 0;
1033 state->has_eob = 0;
1034 state->has_level_buf_init = 0;
1035 state->has_eob_hdr = 0;
1036 state->has_hist = IGZIP_NO_HIST;
1037 state->state = ZSTATE_NEW_HDR;
1038 state->count = 0;
1039
1040 state->tmp_out_start = 0;
1041 state->tmp_out_end = 0;
1042
1043 init(&state->bitbuf);
1044
1045 state->crc = 0;
1046
1047 }
1048
isal_gzip_header_init(struct isal_gzip_header * gz_hdr)1049 void isal_gzip_header_init(struct isal_gzip_header *gz_hdr)
1050 {
1051 gz_hdr->text = 0;
1052 gz_hdr->time = 0;
1053 gz_hdr->xflags = 0;
1054 gz_hdr->os = 0xff;
1055 gz_hdr->extra = NULL;
1056 gz_hdr->extra_buf_len = 0;
1057 gz_hdr->extra_len = 0;
1058 gz_hdr->name = NULL;
1059 gz_hdr->name_buf_len = 0;
1060 gz_hdr->comment = NULL;
1061 gz_hdr->comment_buf_len = 0;
1062 gz_hdr->hcrc = 0;
1063 }
1064
isal_write_gzip_header(struct isal_zstream * stream,struct isal_gzip_header * gz_hdr)1065 uint32_t isal_write_gzip_header(struct isal_zstream *stream, struct isal_gzip_header *gz_hdr)
1066 {
1067 uint32_t flags = 0, hcrc, hdr_size = GZIP_HDR_BASE;
1068 uint8_t *out_buf = stream->next_out, *out_buf_start = stream->next_out;
1069 uint32_t name_len = 0, comment_len = 0;
1070
1071 if (gz_hdr->text)
1072 flags |= TEXT_FLAG;
1073 if (gz_hdr->extra) {
1074 flags |= EXTRA_FLAG;
1075 hdr_size += GZIP_EXTRA_LEN + gz_hdr->extra_len;
1076 }
1077 if (gz_hdr->name) {
1078 flags |= NAME_FLAG;
1079 name_len = strnlen(gz_hdr->name, gz_hdr->name_buf_len);
1080 if (name_len < gz_hdr->name_buf_len)
1081 name_len++;
1082 hdr_size += name_len;
1083 }
1084 if (gz_hdr->comment) {
1085 flags |= COMMENT_FLAG;
1086 comment_len = strnlen(gz_hdr->comment, gz_hdr->comment_buf_len);
1087 if (comment_len < gz_hdr->comment_buf_len)
1088 comment_len++;
1089 hdr_size += comment_len;
1090 }
1091 if (gz_hdr->hcrc) {
1092 flags |= HCRC_FLAG;
1093 hdr_size += GZIP_HCRC_LEN;
1094 }
1095
1096 if (stream->avail_out < hdr_size)
1097 return hdr_size;
1098
1099 out_buf[0] = 0x1f;
1100 out_buf[1] = 0x8b;
1101 out_buf[2] = DEFLATE_METHOD;
1102 out_buf[3] = flags;
1103 store_u32(out_buf + 4, gz_hdr->time);
1104 out_buf[8] = gz_hdr->xflags;
1105 out_buf[9] = gz_hdr->os;
1106
1107 out_buf += GZIP_HDR_BASE;
1108 if (flags & EXTRA_FLAG) {
1109 store_u16(out_buf, gz_hdr->extra_len);
1110 out_buf += GZIP_EXTRA_LEN;
1111
1112 memcpy(out_buf, gz_hdr->extra, gz_hdr->extra_len);
1113 out_buf += gz_hdr->extra_len;
1114 }
1115
1116 if (flags & NAME_FLAG) {
1117 memcpy(out_buf, gz_hdr->name, name_len);
1118 out_buf += name_len;
1119 }
1120
1121 if (flags & COMMENT_FLAG) {
1122 memcpy(out_buf, gz_hdr->comment, comment_len);
1123 out_buf += comment_len;
1124 }
1125
1126 if (flags & HCRC_FLAG) {
1127 hcrc = crc32_gzip_refl(0, out_buf_start, out_buf - out_buf_start);
1128 store_u16(out_buf, hcrc);
1129 out_buf += GZIP_HCRC_LEN;
1130 }
1131
1132 stream->next_out += hdr_size;
1133 stream->total_out += hdr_size;
1134 stream->avail_out -= hdr_size;
1135
1136 return ISAL_DECOMP_OK;
1137 }
1138
isal_write_zlib_header(struct isal_zstream * stream,struct isal_zlib_header * z_hdr)1139 uint32_t isal_write_zlib_header(struct isal_zstream *stream, struct isal_zlib_header *z_hdr)
1140 {
1141 uint32_t cmf, flg, dict_flag = 0, hdr_size = ZLIB_HDR_BASE;
1142 uint8_t *out_buf = stream->next_out;
1143
1144 if (z_hdr->dict_flag) {
1145 dict_flag = ZLIB_DICT_FLAG;
1146 hdr_size = ZLIB_HDR_BASE + ZLIB_DICT_LEN;
1147 }
1148
1149 if (stream->avail_out < hdr_size)
1150 return hdr_size;
1151
1152 cmf = DEFLATE_METHOD | (z_hdr->info << 4);
1153 flg = (z_hdr->level << 6) | dict_flag;
1154
1155 flg += 31 - ((256 * cmf + flg) % 31);
1156
1157 out_buf[0] = cmf;
1158 out_buf[1] = flg;
1159
1160 if (dict_flag)
1161 store_u32(out_buf + 2, z_hdr->dict_id);
1162
1163 stream->next_out += hdr_size;
1164 stream->total_out += hdr_size;
1165 stream->avail_out -= hdr_size;
1166
1167 return ISAL_DECOMP_OK;
1168 }
1169
isal_deflate_set_hufftables(struct isal_zstream * stream,struct isal_hufftables * hufftables,int type)1170 int isal_deflate_set_hufftables(struct isal_zstream *stream,
1171 struct isal_hufftables *hufftables, int type)
1172 {
1173 if (stream->internal_state.state != ZSTATE_NEW_HDR)
1174 return ISAL_INVALID_OPERATION;
1175
1176 switch (type) {
1177 case IGZIP_HUFFTABLE_DEFAULT:
1178 stream->hufftables = (struct isal_hufftables *)&hufftables_default;
1179 break;
1180 case IGZIP_HUFFTABLE_STATIC:
1181 stream->hufftables = (struct isal_hufftables *)&hufftables_static;
1182 break;
1183 case IGZIP_HUFFTABLE_CUSTOM:
1184 if (hufftables != NULL) {
1185 stream->hufftables = hufftables;
1186 break;
1187 }
1188 default:
1189 return ISAL_INVALID_OPERATION;
1190 }
1191
1192 return COMP_OK;
1193 }
1194
isal_deflate_stateless_init(struct isal_zstream * stream)1195 void isal_deflate_stateless_init(struct isal_zstream *stream)
1196 {
1197 stream->total_in = 0;
1198 stream->total_out = 0;
1199 stream->hufftables = (struct isal_hufftables *)&hufftables_default;
1200 stream->level = 0;
1201 stream->level_buf = NULL;
1202 stream->level_buf_size = 0;
1203 stream->end_of_stream = 0;
1204 stream->flush = NO_FLUSH;
1205 stream->gzip_flag = 0;
1206 stream->internal_state.has_wrap_hdr = 0;
1207 stream->internal_state.state = ZSTATE_NEW_HDR;
1208 return;
1209 }
1210
isal_deflate_hash(struct isal_zstream * stream,uint8_t * dict,uint32_t dict_len)1211 void isal_deflate_hash(struct isal_zstream *stream, uint8_t * dict, uint32_t dict_len)
1212 {
1213 /* Reset history to prevent out of bounds matches this works because
1214 * dictionary must set at least 1 element in the history */
1215 struct level_buf *level_buf = (struct level_buf *)stream->level_buf;
1216 uint32_t hash_mask = stream->internal_state.hash_mask;
1217
1218 switch (stream->level) {
1219 case 3:
1220 memset(level_buf->lvl3.hash_table, -1, sizeof(level_buf->lvl3.hash_table));
1221 isal_deflate_hash_lvl3(level_buf->lvl3.hash_table, hash_mask,
1222 stream->total_in, dict, dict_len);
1223 break;
1224
1225 case 2:
1226 memset(level_buf->lvl2.hash_table, -1, sizeof(level_buf->lvl2.hash_table));
1227 isal_deflate_hash_lvl2(level_buf->lvl2.hash_table, hash_mask,
1228 stream->total_in, dict, dict_len);
1229 break;
1230 case 1:
1231 memset(level_buf->lvl1.hash_table, -1, sizeof(level_buf->lvl1.hash_table));
1232 isal_deflate_hash_lvl1(level_buf->lvl1.hash_table, hash_mask,
1233 stream->total_in, dict, dict_len);
1234 break;
1235 default:
1236 memset(stream->internal_state.head, -1, sizeof(stream->internal_state.head));
1237 isal_deflate_hash_lvl0(stream->internal_state.head, hash_mask,
1238 stream->total_in, dict, dict_len);
1239 }
1240
1241 stream->internal_state.has_hist = IGZIP_HIST;
1242 }
1243
isal_deflate_process_dict(struct isal_zstream * stream,struct isal_dict * dict,uint8_t * dict_data,uint32_t dict_len)1244 int isal_deflate_process_dict(struct isal_zstream *stream, struct isal_dict *dict,
1245 uint8_t * dict_data, uint32_t dict_len)
1246 {
1247 if ((dict == NULL)
1248 || (dict_len == 0)
1249 || (dict->level > ISAL_DEF_MAX_LEVEL))
1250 return ISAL_INVALID_STATE;
1251
1252 if (dict_len > IGZIP_HIST_SIZE) {
1253 dict_data = dict_data + dict_len - IGZIP_HIST_SIZE;
1254 dict_len = IGZIP_HIST_SIZE;
1255 }
1256
1257 dict->level = stream->level;
1258 dict->hist_size = dict_len;
1259 memcpy(dict->history, dict_data, dict_len);
1260 memset(dict->hashtable, -1, sizeof(dict->hashtable));
1261
1262 switch (stream->level) {
1263 case 3:
1264 dict->hash_size = IGZIP_LVL3_HASH_SIZE;
1265 isal_deflate_hash_lvl3(dict->hashtable, LVL3_HASH_MASK,
1266 0, dict_data, dict_len);
1267 break;
1268
1269 case 2:
1270 dict->hash_size = IGZIP_LVL2_HASH_SIZE;
1271 isal_deflate_hash_lvl2(dict->hashtable, LVL2_HASH_MASK,
1272 0, dict_data, dict_len);
1273 break;
1274 case 1:
1275 dict->hash_size = IGZIP_LVL1_HASH_SIZE;
1276 isal_deflate_hash_lvl1(dict->hashtable, LVL1_HASH_MASK,
1277 0, dict_data, dict_len);
1278 break;
1279 default:
1280 dict->hash_size = IGZIP_LVL0_HASH_SIZE;
1281 isal_deflate_hash_lvl0(dict->hashtable, LVL0_HASH_MASK,
1282 0, dict_data, dict_len);
1283 }
1284 return COMP_OK;
1285 }
1286
isal_deflate_reset_dict(struct isal_zstream * stream,struct isal_dict * dict)1287 int isal_deflate_reset_dict(struct isal_zstream *stream, struct isal_dict *dict)
1288 {
1289 struct isal_zstate *state = &stream->internal_state;
1290 struct level_buf *level_buf = (struct level_buf *)stream->level_buf;
1291 int ret;
1292
1293 if ((state->state != ZSTATE_NEW_HDR)
1294 || (state->b_bytes_processed != state->b_bytes_valid)
1295 || (dict->level != stream->level)
1296 || (dict->hist_size == 0)
1297 || (dict->hist_size > IGZIP_HIST_SIZE)
1298 || (dict->hash_size > IGZIP_LVL3_HASH_SIZE))
1299 return ISAL_INVALID_STATE;
1300
1301 ret = check_level_req(stream);
1302 if (ret)
1303 return ret;
1304
1305 memcpy(state->buffer, dict->history, dict->hist_size);
1306 state->b_bytes_processed = dict->hist_size;
1307 state->b_bytes_valid = dict->hist_size;
1308 state->has_hist = IGZIP_DICT_HASH_SET;
1309
1310 switch (stream->level) {
1311 case 3:
1312 memcpy(level_buf->lvl3.hash_table, dict->hashtable,
1313 sizeof(level_buf->lvl3.hash_table));
1314 break;
1315
1316 case 2:
1317 memcpy(level_buf->lvl2.hash_table, dict->hashtable,
1318 sizeof(level_buf->lvl2.hash_table));
1319 break;
1320 case 1:
1321 memcpy(level_buf->lvl1.hash_table, dict->hashtable,
1322 sizeof(level_buf->lvl1.hash_table));
1323 break;
1324 default:
1325 memcpy(stream->internal_state.head, dict->hashtable,
1326 sizeof(stream->internal_state.head));
1327 }
1328
1329 return COMP_OK;
1330 }
1331
isal_deflate_set_dict(struct isal_zstream * stream,uint8_t * dict,uint32_t dict_len)1332 int isal_deflate_set_dict(struct isal_zstream *stream, uint8_t * dict, uint32_t dict_len)
1333 {
1334 struct isal_zstate *state = &stream->internal_state;
1335
1336 if (state->state != ZSTATE_NEW_HDR || state->b_bytes_processed != state->b_bytes_valid)
1337 return ISAL_INVALID_STATE;
1338
1339 if (dict_len <= 0)
1340 return COMP_OK;
1341
1342 if (dict_len > IGZIP_HIST_SIZE) {
1343 dict = dict + dict_len - IGZIP_HIST_SIZE;
1344 dict_len = IGZIP_HIST_SIZE;
1345 }
1346
1347 memcpy(state->buffer, dict, dict_len);
1348 state->b_bytes_processed = dict_len;
1349 state->b_bytes_valid = dict_len;
1350
1351 state->has_hist = IGZIP_DICT_HIST;
1352
1353 return COMP_OK;
1354 }
1355
isal_deflate_stateless(struct isal_zstream * stream)1356 int isal_deflate_stateless(struct isal_zstream *stream)
1357 {
1358 struct isal_zstate *state = &stream->internal_state;
1359 uint8_t *next_in = stream->next_in;
1360 const uint32_t avail_in = stream->avail_in;
1361 const uint32_t total_in = stream->total_in;
1362
1363 uint8_t *next_out = stream->next_out;
1364 const uint32_t avail_out = stream->avail_out;
1365 const uint32_t total_out = stream->total_out;
1366 const uint32_t gzip_flag = stream->gzip_flag;
1367 const uint32_t has_wrap_hdr = state->has_wrap_hdr;
1368
1369 int level_check;
1370 uint64_t stored_len;
1371
1372 /* Final block has already been written */
1373 state->block_next = stream->total_in;
1374 state->block_end = stream->total_in;
1375 state->has_eob_hdr = 0;
1376 init(&state->bitbuf);
1377 state->state = ZSTATE_NEW_HDR;
1378 state->crc = 0;
1379 state->has_level_buf_init = 0;
1380 set_dist_mask(stream);
1381
1382 if (stream->flush == NO_FLUSH)
1383 stream->end_of_stream = 1;
1384
1385 if (stream->flush != NO_FLUSH && stream->flush != FULL_FLUSH)
1386 return INVALID_FLUSH;
1387
1388 level_check = check_level_req(stream);
1389 if (level_check) {
1390 if (stream->level == 1 && stream->level_buf == NULL) {
1391 /* Default to internal buffer if invalid size is supplied */
1392 stream->level_buf = state->buffer;
1393 stream->level_buf_size = sizeof(state->buffer) + sizeof(state->head);
1394 } else
1395 return level_check;
1396 }
1397
1398 set_hash_mask(stream);
1399
1400 if (state->hash_mask > 2 * avail_in)
1401 state->hash_mask = (1 << bsr(avail_in)) - 1;
1402
1403 if (avail_in == 0)
1404 stored_len = TYPE0_BLK_HDR_LEN;
1405 else {
1406 stored_len = TYPE0_BLK_HDR_LEN * ((avail_in + TYPE0_MAX_BLK_LEN - 1) /
1407 TYPE0_MAX_BLK_LEN);
1408 stored_len += avail_in;
1409 }
1410
1411 /*
1412 at least 1 byte compressed data in the case of empty dynamic block which only
1413 contains the EOB
1414 */
1415 if (stream->gzip_flag == IGZIP_GZIP)
1416 stored_len += gzip_hdr_bytes + gzip_trl_bytes;
1417 else if (stream->gzip_flag == IGZIP_GZIP_NO_HDR)
1418 stored_len += gzip_trl_bytes;
1419
1420 else if (stream->gzip_flag == IGZIP_ZLIB)
1421 stored_len += zlib_hdr_bytes + zlib_trl_bytes;
1422
1423 else if (stream->gzip_flag == IGZIP_ZLIB_NO_HDR)
1424 stored_len += zlib_trl_bytes;
1425
1426 if (avail_out >= stored_len)
1427 stream->avail_out = stored_len;
1428
1429 if (isal_deflate_int_stateless(stream) == COMP_OK) {
1430 if (avail_out >= stored_len)
1431 stream->avail_out += avail_out - stored_len;
1432 return COMP_OK;
1433 } else {
1434 if (avail_out >= stored_len)
1435 stream->avail_out += avail_out - stored_len;
1436 if (stream->flush == FULL_FLUSH) {
1437 reset_match_history(stream);
1438 }
1439 stream->internal_state.has_eob_hdr = 0;
1440 }
1441
1442 if (avail_out < stored_len)
1443 return STATELESS_OVERFLOW;
1444
1445 stream->next_in = next_in + avail_in;
1446 stream->avail_in = 0;
1447 stream->total_in = avail_in;
1448
1449 state->block_next = stream->total_in - avail_in;
1450 state->block_end = stream->total_in;
1451
1452 stream->next_out = next_out;
1453 stream->avail_out = avail_out;
1454 stream->total_out = total_out;
1455
1456 stream->gzip_flag = gzip_flag;
1457 state->has_wrap_hdr = has_wrap_hdr;
1458 init(&stream->internal_state.bitbuf);
1459 stream->internal_state.count = 0;
1460
1461 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB)
1462 write_stream_header_stateless(stream);
1463
1464 stream->internal_state.state = ZSTATE_TYPE0_HDR;
1465
1466 write_stored_block(stream);
1467
1468 stream->total_in = total_in + avail_in;
1469
1470 if (stream->gzip_flag) {
1471 stream->internal_state.crc = 0;
1472 update_checksum(stream, next_in, avail_in);
1473 }
1474
1475 if (stream->end_of_stream)
1476 write_trailer(stream);
1477
1478 return COMP_OK;
1479
1480 }
1481
get_hist_size(struct isal_zstream * stream,uint8_t * start_in,int32_t buf_hist_start)1482 static inline uint32_t get_hist_size(struct isal_zstream *stream, uint8_t * start_in,
1483 int32_t buf_hist_start)
1484 {
1485 struct isal_zstate *state = &stream->internal_state;
1486 uint32_t history_size;
1487 uint32_t buffered_history;
1488 uint32_t buffered_size = state->b_bytes_valid - state->b_bytes_processed;
1489 uint32_t input_history;
1490
1491 buffered_history = (state->has_hist) ? state->b_bytes_processed - buf_hist_start : 0;
1492 input_history = stream->next_in - start_in;
1493
1494 /* Calculate history required for deflate window */
1495 history_size = (buffered_history >= input_history) ? buffered_history : input_history;
1496 if (history_size > IGZIP_HIST_SIZE)
1497 history_size = IGZIP_HIST_SIZE;
1498
1499 /* Calculate history required based on internal state */
1500 if (state->state == ZSTATE_TYPE0_HDR
1501 || state->state == ZSTATE_TYPE0_BODY
1502 || state->state == ZSTATE_TMP_TYPE0_HDR || state->state == ZSTATE_TMP_TYPE0_BODY) {
1503 if (stream->total_in - state->block_next > history_size) {
1504 history_size = (stream->total_in - state->block_next);
1505 }
1506 } else if (stream->avail_in + buffered_size == 0
1507 && (stream->end_of_stream || stream->flush == FULL_FLUSH)) {
1508 history_size = 0;
1509 }
1510 return history_size;
1511 }
1512
isal_deflate(struct isal_zstream * stream)1513 int isal_deflate(struct isal_zstream *stream)
1514 {
1515 struct isal_zstate *state = &stream->internal_state;
1516 int ret = COMP_OK;
1517 uint8_t *next_in, *start_in, *buf_start_in, *next_in_pre;
1518 uint32_t avail_in, total_start, hist_size, future_size;
1519 uint32_t in_size, in_size_initial, out_size, out_size_initial;
1520 uint32_t processed, buffered_size = state->b_bytes_valid - state->b_bytes_processed;
1521 uint32_t flush_type = stream->flush;
1522 uint32_t end_of_stream = stream->end_of_stream;
1523 uint32_t size = 0;
1524 int32_t buf_hist_start = 0;
1525 uint8_t *copy_down_src = NULL;
1526 uint64_t copy_down_size = 0, copy_start_offset;
1527 int internal;
1528
1529 if (stream->flush >= 3)
1530 return INVALID_FLUSH;
1531
1532 ret = check_level_req(stream);
1533 if (ret)
1534 return ret;
1535
1536 start_in = stream->next_in;
1537 total_start = stream->total_in;
1538
1539 hist_size = get_hist_size(stream, start_in, buf_hist_start);
1540
1541 if (state->has_hist == IGZIP_NO_HIST) {
1542 set_dist_mask(stream);
1543 set_hash_mask(stream);
1544 if (state->hash_mask > 2 * stream->avail_in
1545 && (stream->flush == FULL_FLUSH || stream->end_of_stream))
1546 state->hash_mask = (1 << bsr(2 * stream->avail_in)) - 1;
1547 stream->total_in -= buffered_size;
1548 reset_match_history(stream);
1549 stream->total_in += buffered_size;
1550 buf_hist_start = state->b_bytes_processed;
1551
1552 } else if (state->has_hist == IGZIP_DICT_HIST) {
1553 set_dist_mask(stream);
1554 set_hash_mask(stream);
1555 isal_deflate_hash(stream, state->buffer, state->b_bytes_processed);
1556 } else if (state->has_hist == IGZIP_DICT_HASH_SET) {
1557 set_dist_mask(stream);
1558 set_hash_mask(stream);
1559 }
1560
1561 in_size = stream->avail_in + buffered_size;
1562 out_size = stream->total_out;
1563 do {
1564 in_size_initial = in_size;
1565 out_size_initial = out_size;
1566 buf_start_in = start_in;
1567 internal = 0;
1568
1569 /* Setup to compress from internal buffer if insufficient history */
1570 if (stream->total_in - total_start < hist_size + buffered_size) {
1571 /* On entry there should always be sufficient history bufferd */
1572 /* assert(state->b_bytes_processed >= hist_size); */
1573
1574 internal = 1;
1575 /* Shift down internal buffer if it contains more data
1576 * than required */
1577 if (state->b_bytes_processed > hist_size) {
1578 copy_start_offset = state->b_bytes_processed - hist_size;
1579
1580 copy_down_src = &state->buffer[copy_start_offset];
1581 copy_down_size = state->b_bytes_valid - copy_start_offset;
1582 memmove(state->buffer, copy_down_src, copy_down_size);
1583
1584 state->b_bytes_valid -= copy_down_src - state->buffer;
1585 state->b_bytes_processed -= copy_down_src - state->buffer;
1586 buf_hist_start -= copy_down_src - state->buffer;
1587 if (buf_hist_start < 0)
1588 buf_hist_start = 0;
1589 }
1590
1591 size = stream->avail_in;
1592 if (size > sizeof(state->buffer) - state->b_bytes_valid)
1593 size = sizeof(state->buffer) - state->b_bytes_valid;
1594
1595 memcpy(&state->buffer[state->b_bytes_valid], stream->next_in, size);
1596
1597 stream->next_in += size;
1598 stream->avail_in -= size;
1599 stream->total_in += size;
1600 state->b_bytes_valid += size;
1601 buffered_size += size;
1602
1603 /* Save off next_in and avail_in if compression is
1604 * performed in internal buffer, total_in can be
1605 * recovered from knowledge of the size of the buffered
1606 * input */
1607 next_in = stream->next_in;
1608 avail_in = stream->avail_in;
1609
1610 /* If not much data is buffered and there is no need to
1611 * flush the buffer, just continue rather than attempt
1612 * to compress */
1613 if (avail_in == 0 && buffered_size <= IGZIP_HIST_SIZE
1614 && stream->total_in - buffered_size - state->block_next <=
1615 IGZIP_HIST_SIZE && !stream->end_of_stream
1616 && stream->flush == NO_FLUSH)
1617 continue;
1618
1619 if (avail_in) {
1620 stream->flush = NO_FLUSH;
1621 stream->end_of_stream = 0;
1622 }
1623
1624 stream->next_in = &state->buffer[state->b_bytes_processed];
1625 stream->avail_in = buffered_size;
1626 stream->total_in -= buffered_size;
1627
1628 buf_start_in = state->buffer;
1629
1630 } else if (buffered_size) {
1631 /* The user provided buffer has sufficient data, reset
1632 * the user supplied buffer to included any data already
1633 * buffered */
1634 stream->next_in -= buffered_size;
1635 stream->avail_in += buffered_size;
1636 stream->total_in -= buffered_size;
1637 state->b_bytes_valid = 0;
1638 state->b_bytes_processed = 0;
1639 buffered_size = 0;
1640 }
1641
1642 next_in_pre = stream->next_in;
1643 isal_deflate_int(stream, buf_start_in);
1644 processed = stream->next_in - next_in_pre;
1645 hist_size = get_hist_size(stream, buf_start_in, buf_hist_start);
1646
1647 /* Restore compression to unbuffered input when compressing to internal buffer */
1648 if (internal) {
1649 state->b_bytes_processed += processed;
1650 buffered_size -= processed;
1651
1652 stream->flush = flush_type;
1653 stream->end_of_stream = end_of_stream;
1654 stream->total_in += buffered_size;
1655
1656 stream->next_in = next_in;
1657 stream->avail_in = avail_in;
1658 }
1659
1660 in_size = stream->avail_in + buffered_size;
1661 out_size = stream->total_out;
1662
1663 } while (internal && stream->avail_in > 0 && stream->avail_out > 0
1664 && (in_size_initial != in_size || out_size_initial != out_size));
1665
1666 /* Buffer history if data was pulled from the external buffer and future
1667 * calls to deflate will be required */
1668 if (!internal && (state->state != ZSTATE_END && state->state != ZSTATE_TRL)) {
1669 /* If the external buffer was used, sufficient history must
1670 * exist in the user input buffer */
1671 /* assert(stream->total_in - total_start >= */
1672 /* hist_size + buffered_size); */
1673
1674 stream->next_in -= buffered_size;
1675 stream->avail_in += buffered_size;
1676 stream->total_in -= buffered_size;
1677
1678 memmove(state->buffer, stream->next_in - hist_size, hist_size);
1679 state->b_bytes_processed = hist_size;
1680 state->b_bytes_valid = hist_size;
1681 buffered_size = 0;
1682 }
1683
1684 /* Buffer input data if it is necessary for continued execution */
1685 if (stream->avail_in > 0 && (stream->avail_out > 0 || stream->level == 3)) {
1686 /* Determine how much data to buffer */
1687 future_size = sizeof(state->buffer) - state->b_bytes_valid;
1688 if (stream->avail_in < future_size)
1689 /* Buffer all data if it fits as it will need to be buffered
1690 * on the next call anyways*/
1691 future_size = stream->avail_in;
1692 else if (ISAL_LOOK_AHEAD < future_size)
1693 /* Buffer a minimum look ahead required for level 3 */
1694 future_size = ISAL_LOOK_AHEAD;
1695
1696 memcpy(&state->buffer[state->b_bytes_valid], stream->next_in, future_size);
1697
1698 state->b_bytes_valid += future_size;
1699 buffered_size += future_size;
1700 stream->next_in += future_size;
1701 stream->total_in += future_size;
1702 stream->avail_in -= future_size;
1703
1704 }
1705
1706 return ret;
1707 }
1708
write_stream_header_stateless(struct isal_zstream * stream)1709 static int write_stream_header_stateless(struct isal_zstream *stream)
1710 {
1711 uint32_t hdr_bytes;
1712 const uint8_t *hdr;
1713 uint32_t next_flag;
1714
1715 if (stream->internal_state.has_wrap_hdr)
1716 return COMP_OK;
1717
1718 if (stream->gzip_flag == IGZIP_ZLIB) {
1719 hdr_bytes = zlib_hdr_bytes;
1720 hdr = zlib_hdr;
1721 next_flag = IGZIP_ZLIB_NO_HDR;
1722
1723 } else {
1724 hdr_bytes = gzip_hdr_bytes;
1725 hdr = gzip_hdr;
1726 next_flag = IGZIP_GZIP_NO_HDR;
1727 }
1728
1729 if (hdr_bytes >= stream->avail_out)
1730 return STATELESS_OVERFLOW;
1731
1732 stream->avail_out -= hdr_bytes;
1733 stream->total_out += hdr_bytes;
1734
1735 memcpy(stream->next_out, hdr, hdr_bytes);
1736
1737 stream->next_out += hdr_bytes;
1738 stream->internal_state.has_wrap_hdr = 1;
1739 stream->gzip_flag = next_flag;
1740
1741 return COMP_OK;
1742 }
1743
write_stream_header(struct isal_zstream * stream)1744 static void write_stream_header(struct isal_zstream *stream)
1745 {
1746 struct isal_zstate *state = &stream->internal_state;
1747 int bytes_to_write;
1748 uint32_t hdr_bytes;
1749 const uint8_t *hdr;
1750
1751 if (stream->internal_state.has_wrap_hdr)
1752 return;
1753
1754 if (stream->gzip_flag == IGZIP_ZLIB) {
1755 hdr_bytes = zlib_hdr_bytes;
1756 hdr = zlib_hdr;
1757 } else {
1758 hdr_bytes = gzip_hdr_bytes;
1759 hdr = gzip_hdr;
1760 }
1761
1762 bytes_to_write = hdr_bytes;
1763 bytes_to_write -= state->count;
1764
1765 if (bytes_to_write > stream->avail_out)
1766 bytes_to_write = stream->avail_out;
1767
1768 memcpy(stream->next_out, hdr + state->count, bytes_to_write);
1769 state->count += bytes_to_write;
1770
1771 if (state->count == hdr_bytes) {
1772 state->count = 0;
1773 state->has_wrap_hdr = 1;
1774 }
1775
1776 stream->avail_out -= bytes_to_write;
1777 stream->total_out += bytes_to_write;
1778 stream->next_out += bytes_to_write;
1779
1780 }
1781
write_deflate_header_stateless(struct isal_zstream * stream)1782 static int write_deflate_header_stateless(struct isal_zstream *stream)
1783 {
1784 struct isal_zstate *state = &stream->internal_state;
1785 struct isal_hufftables *hufftables = stream->hufftables;
1786 uint64_t hdr_extra_bits = hufftables->deflate_hdr[hufftables->deflate_hdr_count];
1787 uint32_t count;
1788
1789 if (hufftables->deflate_hdr_count + 8 >= stream->avail_out)
1790 return STATELESS_OVERFLOW;
1791
1792 memcpy(stream->next_out, hufftables->deflate_hdr, hufftables->deflate_hdr_count);
1793
1794 if (stream->end_of_stream == 0) {
1795 if (hufftables->deflate_hdr_count > 0)
1796 *stream->next_out -= 1;
1797 else
1798 hdr_extra_bits -= 1;
1799 } else
1800 state->has_eob_hdr = 1;
1801
1802 stream->avail_out -= hufftables->deflate_hdr_count;
1803 stream->total_out += hufftables->deflate_hdr_count;
1804 stream->next_out += hufftables->deflate_hdr_count;
1805
1806 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
1807
1808 write_bits(&state->bitbuf, hdr_extra_bits, hufftables->deflate_hdr_extra_bits);
1809
1810 count = buffer_used(&state->bitbuf);
1811 stream->next_out = buffer_ptr(&state->bitbuf);
1812 stream->avail_out -= count;
1813 stream->total_out += count;
1814
1815 state->state = ZSTATE_BODY;
1816
1817 return COMP_OK;
1818 }
1819
write_deflate_header_unaligned_stateless(struct isal_zstream * stream)1820 static int write_deflate_header_unaligned_stateless(struct isal_zstream *stream)
1821 {
1822 struct isal_zstate *state = &stream->internal_state;
1823 struct isal_hufftables *hufftables = stream->hufftables;
1824 unsigned int count;
1825 uint64_t bit_count;
1826 uint8_t *header_next;
1827 uint8_t *header_end;
1828 uint64_t header_bits;
1829
1830 if (state->bitbuf.m_bit_count == 0)
1831 return write_deflate_header_stateless(stream);
1832
1833 if (hufftables->deflate_hdr_count + 16 >= stream->avail_out)
1834 return STATELESS_OVERFLOW;
1835
1836 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
1837
1838 header_next = hufftables->deflate_hdr;
1839 header_end = header_next +
1840 (hufftables->deflate_hdr_count / sizeof(header_bits)) * sizeof(header_bits);
1841
1842 header_bits = load_u64(header_next);
1843
1844 if (stream->end_of_stream == 0)
1845 header_bits--;
1846 else
1847 state->has_eob_hdr = 1;
1848
1849 header_next += sizeof(header_bits);
1850
1851 /* Write out Complete Header bits */
1852 for (; header_next <= header_end; header_next += sizeof(header_bits)) {
1853 write_bits(&state->bitbuf, header_bits, 32);
1854 header_bits >>= 32;
1855 write_bits(&state->bitbuf, header_bits, 32);
1856 header_bits = load_u64(header_next);
1857 }
1858 bit_count =
1859 (hufftables->deflate_hdr_count & 0x7) * 8 + hufftables->deflate_hdr_extra_bits;
1860
1861 if (bit_count > MAX_BITBUF_BIT_WRITE) {
1862 write_bits(&state->bitbuf, header_bits, MAX_BITBUF_BIT_WRITE);
1863 header_bits >>= MAX_BITBUF_BIT_WRITE;
1864 bit_count -= MAX_BITBUF_BIT_WRITE;
1865
1866 }
1867
1868 write_bits(&state->bitbuf, header_bits, bit_count);
1869
1870 /* check_space flushes extra bytes in bitbuf. Required because
1871 * write_bits_always fails when the next commit makes the buffer
1872 * length exceed 64 bits */
1873 check_space(&state->bitbuf, FORCE_FLUSH);
1874
1875 count = buffer_used(&state->bitbuf);
1876 stream->next_out = buffer_ptr(&state->bitbuf);
1877 stream->avail_out -= count;
1878 stream->total_out += count;
1879
1880 state->state = ZSTATE_BODY;
1881
1882 return COMP_OK;
1883 }
1884
1885 /* Toggle end of stream only works when deflate header is aligned */
write_header(struct isal_zstream * stream,uint8_t * deflate_hdr,uint32_t deflate_hdr_count,uint32_t extra_bits_count,uint32_t next_state,uint32_t toggle_end_of_stream)1886 static void write_header(struct isal_zstream *stream, uint8_t * deflate_hdr,
1887 uint32_t deflate_hdr_count, uint32_t extra_bits_count,
1888 uint32_t next_state, uint32_t toggle_end_of_stream)
1889 {
1890 struct isal_zstate *state = &stream->internal_state;
1891 uint32_t hdr_extra_bits = deflate_hdr[deflate_hdr_count];
1892 uint32_t count;
1893 state->state = ZSTATE_HDR;
1894
1895 if (state->bitbuf.m_bit_count != 0) {
1896 if (stream->avail_out < 8)
1897 return;
1898 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
1899 flush(&state->bitbuf);
1900 count = buffer_used(&state->bitbuf);
1901 stream->next_out = buffer_ptr(&state->bitbuf);
1902 stream->avail_out -= count;
1903 stream->total_out += count;
1904 }
1905
1906 if (stream->gzip_flag == IGZIP_GZIP || stream->gzip_flag == IGZIP_ZLIB)
1907 write_stream_header(stream);
1908
1909 count = deflate_hdr_count - state->count;
1910
1911 if (count != 0) {
1912 if (count > stream->avail_out)
1913 count = stream->avail_out;
1914
1915 memcpy(stream->next_out, deflate_hdr + state->count, count);
1916
1917 if (toggle_end_of_stream && state->count == 0 && count > 0) {
1918 /* Assumes the final block bit is the first bit */
1919 *stream->next_out ^= 1;
1920 state->has_eob_hdr = !state->has_eob_hdr;
1921 }
1922
1923 stream->next_out += count;
1924 stream->avail_out -= count;
1925 stream->total_out += count;
1926 state->count += count;
1927
1928 count = deflate_hdr_count - state->count;
1929 } else if (toggle_end_of_stream && deflate_hdr_count == 0) {
1930 /* Assumes the final block bit is the first bit */
1931 hdr_extra_bits ^= 1;
1932 state->has_eob_hdr = !state->has_eob_hdr;
1933 }
1934
1935 if ((count == 0) && (stream->avail_out >= 8)) {
1936
1937 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
1938
1939 write_bits(&state->bitbuf, hdr_extra_bits, extra_bits_count);
1940
1941 state->state = next_state;
1942 state->count = 0;
1943
1944 count = buffer_used(&state->bitbuf);
1945 stream->next_out = buffer_ptr(&state->bitbuf);
1946 stream->avail_out -= count;
1947 stream->total_out += count;
1948 }
1949
1950 }
1951
write_trailer(struct isal_zstream * stream)1952 static void write_trailer(struct isal_zstream *stream)
1953 {
1954 struct isal_zstate *state = &stream->internal_state;
1955 unsigned int bytes = 0;
1956 uint32_t crc = state->crc;
1957
1958 set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
1959
1960 if (!state->has_eob_hdr) {
1961 /* If the final header has not been written, write a
1962 * final block. This block is a static huffman block
1963 * which only contains the end of block symbol. The code
1964 * that happens to do this is the fist 10 bits of
1965 * 0x003 */
1966 if (stream->avail_out < 8)
1967 return;
1968
1969 state->has_eob_hdr = 1;
1970 write_bits(&state->bitbuf, 0x003, 10);
1971 if (is_full(&state->bitbuf)) {
1972 stream->next_out = buffer_ptr(&state->bitbuf);
1973 bytes = buffer_used(&state->bitbuf);
1974 stream->avail_out -= bytes;
1975 stream->total_out += bytes;
1976 return;
1977 }
1978 }
1979
1980 if (state->bitbuf.m_bit_count) {
1981 /* the flush() will pad to the next byte and write up to 8 bytes
1982 * to the output stream/buffer.
1983 */
1984 if (stream->avail_out < 8)
1985 return;
1986
1987 flush(&state->bitbuf);
1988 }
1989
1990 stream->next_out = buffer_ptr(&state->bitbuf);
1991 bytes = buffer_used(&state->bitbuf);
1992
1993 switch (stream->gzip_flag) {
1994 case IGZIP_GZIP:
1995 case IGZIP_GZIP_NO_HDR:
1996 if (stream->avail_out - bytes >= gzip_trl_bytes) {
1997 store_u64(stream->next_out, ((uint64_t) stream->total_in << 32) | crc);
1998 stream->next_out += gzip_trl_bytes;
1999 bytes += gzip_trl_bytes;
2000 state->state = ZSTATE_END;
2001 }
2002 break;
2003
2004 case IGZIP_ZLIB:
2005 case IGZIP_ZLIB_NO_HDR:
2006 if (stream->avail_out - bytes >= zlib_trl_bytes) {
2007 store_u32(stream->next_out,
2008 to_be32((crc & 0xFFFF0000) | ((crc & 0xFFFF) + 1) %
2009 ADLER_MOD));
2010 stream->next_out += zlib_trl_bytes;
2011 bytes += zlib_trl_bytes;
2012 state->state = ZSTATE_END;
2013 }
2014 break;
2015
2016 default:
2017 state->state = ZSTATE_END;
2018 }
2019
2020 stream->avail_out -= bytes;
2021 stream->total_out += bytes;
2022 }
2023