1 // SPDX-License-Identifier: 0BSD
2 
3 ///////////////////////////////////////////////////////////////////////////////
4 //
5 /// \file       lzma_encoder.c
6 /// \brief      LZMA encoder
7 ///
8 //  Authors:    Igor Pavlov
9 //              Lasse Collin
10 //
11 ///////////////////////////////////////////////////////////////////////////////
12 
13 #include "lzma2_encoder.h"
14 #include "lzma_encoder_private.h"
15 #include "fastpos.h"
16 
17 
18 /////////////
19 // Literal //
20 /////////////
21 
22 static inline void
literal_matched(lzma_range_encoder * rc,probability * subcoder,uint32_t match_byte,uint32_t symbol)23 literal_matched(lzma_range_encoder *rc, probability *subcoder,
24 		uint32_t match_byte, uint32_t symbol)
25 {
26 	uint32_t offset = 0x100;
27 	symbol += UINT32_C(1) << 8;
28 
29 	do {
30 		match_byte <<= 1;
31 		const uint32_t match_bit = match_byte & offset;
32 		const uint32_t subcoder_index
33 				= offset + match_bit + (symbol >> 8);
34 		const uint32_t bit = (symbol >> 7) & 1;
35 		rc_bit(rc, &subcoder[subcoder_index], bit);
36 
37 		symbol <<= 1;
38 		offset &= ~(match_byte ^ symbol);
39 
40 	} while (symbol < (UINT32_C(1) << 16));
41 }
42 
43 
44 static inline void
literal(lzma_lzma1_encoder * coder,lzma_mf * mf,uint32_t position)45 literal(lzma_lzma1_encoder *coder, lzma_mf *mf, uint32_t position)
46 {
47 	// Locate the literal byte to be encoded and the subcoder.
48 	const uint8_t cur_byte = mf->buffer[
49 			mf->read_pos - mf->read_ahead];
50 	probability *subcoder = literal_subcoder(coder->literal,
51 			coder->literal_context_bits, coder->literal_mask,
52 			position, mf->buffer[mf->read_pos - mf->read_ahead - 1]);
53 
54 	if (is_literal_state(coder->state)) {
55 		// Previous LZMA-symbol was a literal. Encode a normal
56 		// literal without a match byte.
57 		update_literal_normal(coder->state);
58 		rc_bittree(&coder->rc, subcoder, 8, cur_byte);
59 	} else {
60 		// Previous LZMA-symbol was a match. Use the last byte of
61 		// the match as a "match byte". That is, compare the bits
62 		// of the current literal and the match byte.
63 		update_literal_matched(coder->state);
64 		const uint8_t match_byte = mf->buffer[
65 				mf->read_pos - coder->reps[0] - 1
66 				- mf->read_ahead];
67 		literal_matched(&coder->rc, subcoder, match_byte, cur_byte);
68 	}
69 }
70 
71 
72 //////////////////
73 // Match length //
74 //////////////////
75 
76 static void
length_update_prices(lzma_length_encoder * lc,const uint32_t pos_state)77 length_update_prices(lzma_length_encoder *lc, const uint32_t pos_state)
78 {
79 	const uint32_t table_size = lc->table_size;
80 	lc->counters[pos_state] = table_size;
81 
82 	const uint32_t a0 = rc_bit_0_price(lc->choice);
83 	const uint32_t a1 = rc_bit_1_price(lc->choice);
84 	const uint32_t b0 = a1 + rc_bit_0_price(lc->choice2);
85 	const uint32_t b1 = a1 + rc_bit_1_price(lc->choice2);
86 	uint32_t *const prices = lc->prices[pos_state];
87 
88 	uint32_t i;
89 	for (i = 0; i < table_size && i < LEN_LOW_SYMBOLS; ++i)
90 		prices[i] = a0 + rc_bittree_price(lc->low[pos_state],
91 				LEN_LOW_BITS, i);
92 
93 	for (; i < table_size && i < LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; ++i)
94 		prices[i] = b0 + rc_bittree_price(lc->mid[pos_state],
95 				LEN_MID_BITS, i - LEN_LOW_SYMBOLS);
96 
97 	for (; i < table_size; ++i)
98 		prices[i] = b1 + rc_bittree_price(lc->high, LEN_HIGH_BITS,
99 				i - LEN_LOW_SYMBOLS - LEN_MID_SYMBOLS);
100 
101 	return;
102 }
103 
104 
105 static inline void
length(lzma_range_encoder * rc,lzma_length_encoder * lc,const uint32_t pos_state,uint32_t len,const bool fast_mode)106 length(lzma_range_encoder *rc, lzma_length_encoder *lc,
107 		const uint32_t pos_state, uint32_t len, const bool fast_mode)
108 {
109 	assert(len <= MATCH_LEN_MAX);
110 	len -= MATCH_LEN_MIN;
111 
112 	if (len < LEN_LOW_SYMBOLS) {
113 		rc_bit(rc, &lc->choice, 0);
114 		rc_bittree(rc, lc->low[pos_state], LEN_LOW_BITS, len);
115 	} else {
116 		rc_bit(rc, &lc->choice, 1);
117 		len -= LEN_LOW_SYMBOLS;
118 
119 		if (len < LEN_MID_SYMBOLS) {
120 			rc_bit(rc, &lc->choice2, 0);
121 			rc_bittree(rc, lc->mid[pos_state], LEN_MID_BITS, len);
122 		} else {
123 			rc_bit(rc, &lc->choice2, 1);
124 			len -= LEN_MID_SYMBOLS;
125 			rc_bittree(rc, lc->high, LEN_HIGH_BITS, len);
126 		}
127 	}
128 
129 	// Only getoptimum uses the prices so don't update the table when
130 	// in fast mode.
131 	if (!fast_mode)
132 		if (--lc->counters[pos_state] == 0)
133 			length_update_prices(lc, pos_state);
134 }
135 
136 
137 ///////////
138 // Match //
139 ///////////
140 
141 static inline void
match(lzma_lzma1_encoder * coder,const uint32_t pos_state,const uint32_t distance,const uint32_t len)142 match(lzma_lzma1_encoder *coder, const uint32_t pos_state,
143 		const uint32_t distance, const uint32_t len)
144 {
145 	update_match(coder->state);
146 
147 	length(&coder->rc, &coder->match_len_encoder, pos_state, len,
148 			coder->fast_mode);
149 
150 	const uint32_t dist_slot = get_dist_slot(distance);
151 	const uint32_t dist_state = get_dist_state(len);
152 	rc_bittree(&coder->rc, coder->dist_slot[dist_state],
153 			DIST_SLOT_BITS, dist_slot);
154 
155 	if (dist_slot >= DIST_MODEL_START) {
156 		const uint32_t footer_bits = (dist_slot >> 1) - 1;
157 		const uint32_t base = (2 | (dist_slot & 1)) << footer_bits;
158 		const uint32_t dist_reduced = distance - base;
159 
160 		if (dist_slot < DIST_MODEL_END) {
161 			// Careful here: base - dist_slot - 1 can be -1, but
162 			// rc_bittree_reverse starts at probs[1], not probs[0].
163 			rc_bittree_reverse(&coder->rc,
164 				coder->dist_special + base - dist_slot - 1,
165 				footer_bits, dist_reduced);
166 		} else {
167 			rc_direct(&coder->rc, dist_reduced >> ALIGN_BITS,
168 					footer_bits - ALIGN_BITS);
169 			rc_bittree_reverse(
170 					&coder->rc, coder->dist_align,
171 					ALIGN_BITS, dist_reduced & ALIGN_MASK);
172 			++coder->align_price_count;
173 		}
174 	}
175 
176 	coder->reps[3] = coder->reps[2];
177 	coder->reps[2] = coder->reps[1];
178 	coder->reps[1] = coder->reps[0];
179 	coder->reps[0] = distance;
180 	++coder->match_price_count;
181 }
182 
183 
184 ////////////////////
185 // Repeated match //
186 ////////////////////
187 
188 static inline void
rep_match(lzma_lzma1_encoder * coder,const uint32_t pos_state,const uint32_t rep,const uint32_t len)189 rep_match(lzma_lzma1_encoder *coder, const uint32_t pos_state,
190 		const uint32_t rep, const uint32_t len)
191 {
192 	if (rep == 0) {
193 		rc_bit(&coder->rc, &coder->is_rep0[coder->state], 0);
194 		rc_bit(&coder->rc,
195 				&coder->is_rep0_long[coder->state][pos_state],
196 				len != 1);
197 	} else {
198 		const uint32_t distance = coder->reps[rep];
199 		rc_bit(&coder->rc, &coder->is_rep0[coder->state], 1);
200 
201 		if (rep == 1) {
202 			rc_bit(&coder->rc, &coder->is_rep1[coder->state], 0);
203 		} else {
204 			rc_bit(&coder->rc, &coder->is_rep1[coder->state], 1);
205 			rc_bit(&coder->rc, &coder->is_rep2[coder->state],
206 					rep - 2);
207 
208 			if (rep == 3)
209 				coder->reps[3] = coder->reps[2];
210 
211 			coder->reps[2] = coder->reps[1];
212 		}
213 
214 		coder->reps[1] = coder->reps[0];
215 		coder->reps[0] = distance;
216 	}
217 
218 	if (len == 1) {
219 		update_short_rep(coder->state);
220 	} else {
221 		length(&coder->rc, &coder->rep_len_encoder, pos_state, len,
222 				coder->fast_mode);
223 		update_long_rep(coder->state);
224 	}
225 }
226 
227 
228 //////////
229 // Main //
230 //////////
231 
232 static void
encode_symbol(lzma_lzma1_encoder * coder,lzma_mf * mf,uint32_t back,uint32_t len,uint32_t position)233 encode_symbol(lzma_lzma1_encoder *coder, lzma_mf *mf,
234 		uint32_t back, uint32_t len, uint32_t position)
235 {
236 	const uint32_t pos_state = position & coder->pos_mask;
237 
238 	if (back == UINT32_MAX) {
239 		// Literal i.e. eight-bit byte
240 		assert(len == 1);
241 		rc_bit(&coder->rc,
242 				&coder->is_match[coder->state][pos_state], 0);
243 		literal(coder, mf, position);
244 	} else {
245 		// Some type of match
246 		rc_bit(&coder->rc,
247 			&coder->is_match[coder->state][pos_state], 1);
248 
249 		if (back < REPS) {
250 			// It's a repeated match i.e. the same distance
251 			// has been used earlier.
252 			rc_bit(&coder->rc, &coder->is_rep[coder->state], 1);
253 			rep_match(coder, pos_state, back, len);
254 		} else {
255 			// Normal match
256 			rc_bit(&coder->rc, &coder->is_rep[coder->state], 0);
257 			match(coder, pos_state, back - REPS, len);
258 		}
259 	}
260 
261 	assert(mf->read_ahead >= len);
262 	mf->read_ahead -= len;
263 }
264 
265 
266 static bool
encode_init(lzma_lzma1_encoder * coder,lzma_mf * mf)267 encode_init(lzma_lzma1_encoder *coder, lzma_mf *mf)
268 {
269 	assert(mf_position(mf) == 0);
270 	assert(coder->uncomp_size == 0);
271 
272 	if (mf->read_pos == mf->read_limit) {
273 		if (mf->action == LZMA_RUN)
274 			return false; // We cannot do anything.
275 
276 		// We are finishing (we cannot get here when flushing).
277 		assert(mf->write_pos == mf->read_pos);
278 		assert(mf->action == LZMA_FINISH);
279 	} else {
280 		// Do the actual initialization. The first LZMA symbol must
281 		// always be a literal.
282 		mf_skip(mf, 1);
283 		mf->read_ahead = 0;
284 		rc_bit(&coder->rc, &coder->is_match[0][0], 0);
285 		rc_bittree(&coder->rc, coder->literal + 0, 8, mf->buffer[0]);
286 		++coder->uncomp_size;
287 	}
288 
289 	// Initialization is done (except if empty file).
290 	coder->is_initialized = true;
291 
292 	return true;
293 }
294 
295 
296 static void
encode_eopm(lzma_lzma1_encoder * coder,uint32_t position)297 encode_eopm(lzma_lzma1_encoder *coder, uint32_t position)
298 {
299 	const uint32_t pos_state = position & coder->pos_mask;
300 	rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1);
301 	rc_bit(&coder->rc, &coder->is_rep[coder->state], 0);
302 	match(coder, pos_state, UINT32_MAX, MATCH_LEN_MIN);
303 }
304 
305 
306 /// Number of bytes that a single encoding loop in lzma_lzma_encode() can
307 /// consume from the dictionary. This limit comes from lzma_lzma_optimum()
308 /// and may need to be updated if that function is significantly modified.
309 #define LOOP_INPUT_MAX (OPTS + 1)
310 
311 
312 extern lzma_ret
lzma_lzma_encode(lzma_lzma1_encoder * restrict coder,lzma_mf * restrict mf,uint8_t * restrict out,size_t * restrict out_pos,size_t out_size,uint32_t limit)313 lzma_lzma_encode(lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf,
314 		uint8_t *restrict out, size_t *restrict out_pos,
315 		size_t out_size, uint32_t limit)
316 {
317 	// Initialize the stream if no data has been encoded yet.
318 	if (!coder->is_initialized && !encode_init(coder, mf))
319 		return LZMA_OK;
320 
321 	// Encode pending output bytes from the range encoder.
322 	// At the start of the stream, encode_init() encodes one literal.
323 	// Later there can be pending output only with LZMA1 because LZMA2
324 	// ensures that there is always enough output space. Thus when using
325 	// LZMA2, rc_encode() calls in this function will always return false.
326 	if (rc_encode(&coder->rc, out, out_pos, out_size)) {
327 		// We don't get here with LZMA2.
328 		assert(limit == UINT32_MAX);
329 		return LZMA_OK;
330 	}
331 
332 	// If the range encoder was flushed in an earlier call to this
333 	// function but there wasn't enough output buffer space, those
334 	// bytes would have now been encoded by the above rc_encode() call
335 	// and the stream has now been finished. This can only happen with
336 	// LZMA1 as LZMA2 always provides enough output buffer space.
337 	if (coder->is_flushed) {
338 		assert(limit == UINT32_MAX);
339 		return LZMA_STREAM_END;
340 	}
341 
342 	while (true) {
343 		// With LZMA2 we need to take care that compressed size of
344 		// a chunk doesn't get too big.
345 		// FIXME? Check if this could be improved.
346 		if (limit != UINT32_MAX
347 				&& (mf->read_pos - mf->read_ahead >= limit
348 					|| *out_pos + rc_pending(&coder->rc)
349 						>= LZMA2_CHUNK_MAX
350 							- LOOP_INPUT_MAX))
351 			break;
352 
353 		// Check that there is some input to process.
354 		if (mf->read_pos >= mf->read_limit) {
355 			if (mf->action == LZMA_RUN)
356 				return LZMA_OK;
357 
358 			if (mf->read_ahead == 0)
359 				break;
360 		}
361 
362 		// Get optimal match (repeat position and length).
363 		// Value ranges for pos:
364 		//   - [0, REPS): repeated match
365 		//   - [REPS, UINT32_MAX):
366 		//     match at (pos - REPS)
367 		//   - UINT32_MAX: not a match but a literal
368 		// Value ranges for len:
369 		//   - [MATCH_LEN_MIN, MATCH_LEN_MAX]
370 		uint32_t len;
371 		uint32_t back;
372 
373 		if (coder->fast_mode)
374 			lzma_lzma_optimum_fast(coder, mf, &back, &len);
375 		else
376 			lzma_lzma_optimum_normal(coder, mf, &back, &len,
377 					(uint32_t)(coder->uncomp_size));
378 
379 		encode_symbol(coder, mf, back, len,
380 				(uint32_t)(coder->uncomp_size));
381 
382 		// If output size limiting is active (out_limit != 0), check
383 		// if encoding this LZMA symbol would make the output size
384 		// exceed the specified limit.
385 		if (coder->out_limit != 0 && rc_encode_dummy(
386 				&coder->rc, coder->out_limit)) {
387 			// The most recent LZMA symbol would make the output
388 			// too big. Throw it away.
389 			rc_forget(&coder->rc);
390 
391 			// FIXME: Tell the LZ layer to not read more input as
392 			// it would be waste of time. This doesn't matter if
393 			// output-size-limited encoding is done with a single
394 			// call though.
395 
396 			break;
397 		}
398 
399 		// This symbol will be encoded so update the uncompressed size.
400 		coder->uncomp_size += len;
401 
402 		// Encode the LZMA symbol.
403 		if (rc_encode(&coder->rc, out, out_pos, out_size)) {
404 			// Once again, this can only happen with LZMA1.
405 			assert(limit == UINT32_MAX);
406 			return LZMA_OK;
407 		}
408 	}
409 
410 	// Make the uncompressed size available to the application.
411 	if (coder->uncomp_size_ptr != NULL)
412 		*coder->uncomp_size_ptr = coder->uncomp_size;
413 
414 	// LZMA2 doesn't use EOPM at LZMA level.
415 	//
416 	// Plain LZMA streams without EOPM aren't supported except when
417 	// output size limiting is enabled.
418 	if (coder->use_eopm)
419 		encode_eopm(coder, (uint32_t)(coder->uncomp_size));
420 
421 	// Flush the remaining bytes from the range encoder.
422 	rc_flush(&coder->rc);
423 
424 	// Copy the remaining bytes to the output buffer. If there
425 	// isn't enough output space, we will copy out the remaining
426 	// bytes on the next call to this function.
427 	if (rc_encode(&coder->rc, out, out_pos, out_size)) {
428 		// This cannot happen with LZMA2.
429 		assert(limit == UINT32_MAX);
430 
431 		coder->is_flushed = true;
432 		return LZMA_OK;
433 	}
434 
435 	return LZMA_STREAM_END;
436 }
437 
438 
439 static lzma_ret
lzma_encode(void * coder,lzma_mf * restrict mf,uint8_t * restrict out,size_t * restrict out_pos,size_t out_size)440 lzma_encode(void *coder, lzma_mf *restrict mf,
441 		uint8_t *restrict out, size_t *restrict out_pos,
442 		size_t out_size)
443 {
444 	// Plain LZMA has no support for sync-flushing.
445 	if (unlikely(mf->action == LZMA_SYNC_FLUSH))
446 		return LZMA_OPTIONS_ERROR;
447 
448 	return lzma_lzma_encode(coder, mf, out, out_pos, out_size, UINT32_MAX);
449 }
450 
451 
452 static lzma_ret
lzma_lzma_set_out_limit(void * coder_ptr,uint64_t * uncomp_size,uint64_t out_limit)453 lzma_lzma_set_out_limit(
454 		void *coder_ptr, uint64_t *uncomp_size, uint64_t out_limit)
455 {
456 	// Minimum output size is 5 bytes but that cannot hold any output
457 	// so we use 6 bytes.
458 	if (out_limit < 6)
459 		return LZMA_BUF_ERROR;
460 
461 	lzma_lzma1_encoder *coder = coder_ptr;
462 	coder->out_limit = out_limit;
463 	coder->uncomp_size_ptr = uncomp_size;
464 	coder->use_eopm = false;
465 	return LZMA_OK;
466 }
467 
468 
469 ////////////////////
470 // Initialization //
471 ////////////////////
472 
473 static bool
is_options_valid(const lzma_options_lzma * options)474 is_options_valid(const lzma_options_lzma *options)
475 {
476 	// Validate some of the options. LZ encoder validates nice_len too
477 	// but we need a valid value here earlier.
478 	return is_lclppb_valid(options)
479 			&& options->nice_len >= MATCH_LEN_MIN
480 			&& options->nice_len <= MATCH_LEN_MAX
481 			&& (options->mode == LZMA_MODE_FAST
482 				|| options->mode == LZMA_MODE_NORMAL);
483 }
484 
485 
486 static void
set_lz_options(lzma_lz_options * lz_options,const lzma_options_lzma * options)487 set_lz_options(lzma_lz_options *lz_options, const lzma_options_lzma *options)
488 {
489 	// LZ encoder initialization does the validation for these so we
490 	// don't need to validate here.
491 	lz_options->before_size = OPTS;
492 	lz_options->dict_size = options->dict_size;
493 	lz_options->after_size = LOOP_INPUT_MAX;
494 	lz_options->match_len_max = MATCH_LEN_MAX;
495 	lz_options->nice_len = my_max(mf_get_hash_bytes(options->mf),
496 				options->nice_len);
497 	lz_options->match_finder = options->mf;
498 	lz_options->depth = options->depth;
499 	lz_options->preset_dict = options->preset_dict;
500 	lz_options->preset_dict_size = options->preset_dict_size;
501 	return;
502 }
503 
504 
505 static void
length_encoder_reset(lzma_length_encoder * lencoder,const uint32_t num_pos_states,const bool fast_mode)506 length_encoder_reset(lzma_length_encoder *lencoder,
507 		const uint32_t num_pos_states, const bool fast_mode)
508 {
509 	bit_reset(lencoder->choice);
510 	bit_reset(lencoder->choice2);
511 
512 	for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) {
513 		bittree_reset(lencoder->low[pos_state], LEN_LOW_BITS);
514 		bittree_reset(lencoder->mid[pos_state], LEN_MID_BITS);
515 	}
516 
517 	bittree_reset(lencoder->high, LEN_HIGH_BITS);
518 
519 	if (!fast_mode)
520 		for (uint32_t pos_state = 0; pos_state < num_pos_states;
521 				++pos_state)
522 			length_update_prices(lencoder, pos_state);
523 
524 	return;
525 }
526 
527 
528 extern lzma_ret
lzma_lzma_encoder_reset(lzma_lzma1_encoder * coder,const lzma_options_lzma * options)529 lzma_lzma_encoder_reset(lzma_lzma1_encoder *coder,
530 		const lzma_options_lzma *options)
531 {
532 	if (!is_options_valid(options))
533 		return LZMA_OPTIONS_ERROR;
534 
535 	coder->pos_mask = (1U << options->pb) - 1;
536 	coder->literal_context_bits = options->lc;
537 	coder->literal_mask = literal_mask_calc(options->lc, options->lp);
538 
539 	// Range coder
540 	rc_reset(&coder->rc);
541 
542 	// State
543 	coder->state = STATE_LIT_LIT;
544 	for (size_t i = 0; i < REPS; ++i)
545 		coder->reps[i] = 0;
546 
547 	literal_init(coder->literal, options->lc, options->lp);
548 
549 	// Bit encoders
550 	for (size_t i = 0; i < STATES; ++i) {
551 		for (size_t j = 0; j <= coder->pos_mask; ++j) {
552 			bit_reset(coder->is_match[i][j]);
553 			bit_reset(coder->is_rep0_long[i][j]);
554 		}
555 
556 		bit_reset(coder->is_rep[i]);
557 		bit_reset(coder->is_rep0[i]);
558 		bit_reset(coder->is_rep1[i]);
559 		bit_reset(coder->is_rep2[i]);
560 	}
561 
562 	for (size_t i = 0; i < FULL_DISTANCES - DIST_MODEL_END; ++i)
563 		bit_reset(coder->dist_special[i]);
564 
565 	// Bit tree encoders
566 	for (size_t i = 0; i < DIST_STATES; ++i)
567 		bittree_reset(coder->dist_slot[i], DIST_SLOT_BITS);
568 
569 	bittree_reset(coder->dist_align, ALIGN_BITS);
570 
571 	// Length encoders
572 	length_encoder_reset(&coder->match_len_encoder,
573 			1U << options->pb, coder->fast_mode);
574 
575 	length_encoder_reset(&coder->rep_len_encoder,
576 			1U << options->pb, coder->fast_mode);
577 
578 	// Price counts are incremented every time appropriate probabilities
579 	// are changed. price counts are set to zero when the price tables
580 	// are updated, which is done when the appropriate price counts have
581 	// big enough value, and lzma_mf.read_ahead == 0 which happens at
582 	// least every OPTS (a few thousand) possible price count increments.
583 	//
584 	// By resetting price counts to UINT32_MAX / 2, we make sure that the
585 	// price tables will be initialized before they will be used (since
586 	// the value is definitely big enough), and that it is OK to increment
587 	// price counts without risk of integer overflow (since UINT32_MAX / 2
588 	// is small enough). The current code doesn't increment price counts
589 	// before initializing price tables, but it maybe done in future if
590 	// we add support for saving the state between LZMA2 chunks.
591 	coder->match_price_count = UINT32_MAX / 2;
592 	coder->align_price_count = UINT32_MAX / 2;
593 
594 	coder->opts_end_index = 0;
595 	coder->opts_current_index = 0;
596 
597 	return LZMA_OK;
598 }
599 
600 
601 extern lzma_ret
lzma_lzma_encoder_create(void ** coder_ptr,const lzma_allocator * allocator,lzma_vli id,const lzma_options_lzma * options,lzma_lz_options * lz_options)602 lzma_lzma_encoder_create(void **coder_ptr, const lzma_allocator *allocator,
603 		lzma_vli id, const lzma_options_lzma *options,
604 		lzma_lz_options *lz_options)
605 {
606 	assert(id == LZMA_FILTER_LZMA1 || id == LZMA_FILTER_LZMA1EXT
607 			|| id == LZMA_FILTER_LZMA2);
608 
609 	// Allocate lzma_lzma1_encoder if it wasn't already allocated.
610 	if (*coder_ptr == NULL) {
611 		*coder_ptr = lzma_alloc(sizeof(lzma_lzma1_encoder), allocator);
612 		if (*coder_ptr == NULL)
613 			return LZMA_MEM_ERROR;
614 	}
615 
616 	lzma_lzma1_encoder *coder = *coder_ptr;
617 
618 	// Set compression mode. Note that we haven't validated the options
619 	// yet. Invalid options will get rejected by lzma_lzma_encoder_reset()
620 	// call at the end of this function.
621 	switch (options->mode) {
622 		case LZMA_MODE_FAST:
623 			coder->fast_mode = true;
624 			break;
625 
626 		case LZMA_MODE_NORMAL: {
627 			coder->fast_mode = false;
628 
629 			// Set dist_table_size.
630 			// Round the dictionary size up to next 2^n.
631 			//
632 			// Currently the maximum encoder dictionary size
633 			// is 1.5 GiB due to lz_encoder.c and here we need
634 			// to be below 2 GiB to make the rounded up value
635 			// fit in an uint32_t and avoid an infinite while-loop
636 			// (and undefined behavior due to a too large shift).
637 			// So do the same check as in LZ encoder,
638 			// limiting to 1.5 GiB.
639 			if (options->dict_size > (UINT32_C(1) << 30)
640 					+ (UINT32_C(1) << 29))
641 				return LZMA_OPTIONS_ERROR;
642 
643 			uint32_t log_size = 0;
644 			while ((UINT32_C(1) << log_size) < options->dict_size)
645 				++log_size;
646 
647 			coder->dist_table_size = log_size * 2;
648 
649 			// Length encoders' price table size
650 			const uint32_t nice_len = my_max(
651 					mf_get_hash_bytes(options->mf),
652 					options->nice_len);
653 
654 			coder->match_len_encoder.table_size
655 					= nice_len + 1 - MATCH_LEN_MIN;
656 			coder->rep_len_encoder.table_size
657 					= nice_len + 1 - MATCH_LEN_MIN;
658 			break;
659 		}
660 
661 		default:
662 			return LZMA_OPTIONS_ERROR;
663 	}
664 
665 	// We don't need to write the first byte as literal if there is
666 	// a non-empty preset dictionary. encode_init() wouldn't even work
667 	// if there is a non-empty preset dictionary, because encode_init()
668 	// assumes that position is zero and previous byte is also zero.
669 	coder->is_initialized = options->preset_dict != NULL
670 			&& options->preset_dict_size > 0;
671 	coder->is_flushed = false;
672 	coder->uncomp_size = 0;
673 	coder->uncomp_size_ptr = NULL;
674 
675 	// Output size limiting is disabled by default.
676 	coder->out_limit = 0;
677 
678 	// Determine if end marker is wanted:
679 	//   - It is never used with LZMA2.
680 	//   - It is always used with LZMA_FILTER_LZMA1 (unless
681 	//     lzma_lzma_set_out_limit() is called later).
682 	//   - LZMA_FILTER_LZMA1EXT has a flag for it in the options.
683 	coder->use_eopm = (id == LZMA_FILTER_LZMA1);
684 	if (id == LZMA_FILTER_LZMA1EXT) {
685 		// Check if unsupported flags are present.
686 		if (options->ext_flags & ~LZMA_LZMA1EXT_ALLOW_EOPM)
687 			return LZMA_OPTIONS_ERROR;
688 
689 		coder->use_eopm = (options->ext_flags
690 				& LZMA_LZMA1EXT_ALLOW_EOPM) != 0;
691 
692 		// TODO? As long as there are no filters that change the size
693 		// of the data, it is enough to look at lzma_stream.total_in
694 		// after encoding has been finished to know the uncompressed
695 		// size of the LZMA1 stream. But in the future there could be
696 		// filters that change the size of the data and then total_in
697 		// doesn't work as the LZMA1 stream size might be different
698 		// due to another filter in the chain. The problem is simple
699 		// to solve: Add another flag to ext_flags and then set
700 		// coder->uncomp_size_ptr to the address stored in
701 		// lzma_options_lzma.reserved_ptr2 (or _ptr1).
702 	}
703 
704 	set_lz_options(lz_options, options);
705 
706 	return lzma_lzma_encoder_reset(coder, options);
707 }
708 
709 
710 static lzma_ret
lzma_encoder_init(lzma_lz_encoder * lz,const lzma_allocator * allocator,lzma_vli id,const void * options,lzma_lz_options * lz_options)711 lzma_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator,
712 		lzma_vli id, const void *options, lzma_lz_options *lz_options)
713 {
714         if (options == NULL)
715                 return LZMA_PROG_ERROR;
716 
717 	lz->code = &lzma_encode;
718 	lz->set_out_limit = &lzma_lzma_set_out_limit;
719 	return lzma_lzma_encoder_create(
720 			&lz->coder, allocator, id, options, lz_options);
721 }
722 
723 
724 extern lzma_ret
lzma_lzma_encoder_init(lzma_next_coder * next,const lzma_allocator * allocator,const lzma_filter_info * filters)725 lzma_lzma_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
726 		const lzma_filter_info *filters)
727 {
728 	return lzma_lz_encoder_init(
729 			next, allocator, filters, &lzma_encoder_init);
730 }
731 
732 
733 extern uint64_t
lzma_lzma_encoder_memusage(const void * options)734 lzma_lzma_encoder_memusage(const void *options)
735 {
736 	if (!is_options_valid(options))
737 		return UINT64_MAX;
738 
739 	lzma_lz_options lz_options;
740 	set_lz_options(&lz_options, options);
741 
742 	const uint64_t lz_memusage = lzma_lz_encoder_memusage(&lz_options);
743 	if (lz_memusage == UINT64_MAX)
744 		return UINT64_MAX;
745 
746 	return (uint64_t)(sizeof(lzma_lzma1_encoder)) + lz_memusage;
747 }
748 
749 
750 extern bool
lzma_lzma_lclppb_encode(const lzma_options_lzma * options,uint8_t * byte)751 lzma_lzma_lclppb_encode(const lzma_options_lzma *options, uint8_t *byte)
752 {
753 	if (!is_lclppb_valid(options))
754 		return true;
755 
756 	*byte = (options->pb * 5 + options->lp) * 9 + options->lc;
757 	assert(*byte <= (4 * 5 + 4) * 9 + 8);
758 
759 	return false;
760 }
761 
762 
763 #ifdef HAVE_ENCODER_LZMA1
764 extern lzma_ret
lzma_lzma_props_encode(const void * options,uint8_t * out)765 lzma_lzma_props_encode(const void *options, uint8_t *out)
766 {
767 	if (options == NULL)
768 		return LZMA_PROG_ERROR;
769 
770 	const lzma_options_lzma *const opt = options;
771 
772 	if (lzma_lzma_lclppb_encode(opt, out))
773 		return LZMA_PROG_ERROR;
774 
775 	write32le(out + 1, opt->dict_size);
776 
777 	return LZMA_OK;
778 }
779 #endif
780 
781 
782 extern LZMA_API(lzma_bool)
lzma_mode_is_supported(lzma_mode mode)783 lzma_mode_is_supported(lzma_mode mode)
784 {
785 	return mode == LZMA_MODE_FAST || mode == LZMA_MODE_NORMAL;
786 }
787