1 /* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
2
3 #include "lib.h"
4
5 #ifdef HAVE_LZMA
6
7 #include "istream-private.h"
8 #include "istream-zlib.h"
9 #include <lzma.h>
10
11 #define CHUNK_SIZE (1024*64)
12
13 #define LZMA_MEMORY_LIMIT (1024*1024*80)
14
15 struct lzma_istream {
16 struct istream_private istream;
17
18 lzma_stream strm;
19 uoff_t eof_offset;
20 struct stat last_parent_statbuf;
21
22 bool hdr_read:1;
23 bool marked:1;
24 bool strm_closed:1;
25 };
26
i_stream_lzma_close(struct iostream_private * stream,bool close_parent)27 static void i_stream_lzma_close(struct iostream_private *stream,
28 bool close_parent)
29 {
30 struct lzma_istream *zstream = (struct lzma_istream *)stream;
31
32 if (!zstream->strm_closed) {
33 lzma_end(&zstream->strm);
34 zstream->strm_closed = TRUE;
35 }
36 if (close_parent)
37 i_stream_close(zstream->istream.parent);
38 }
39
lzma_read_error(struct lzma_istream * zstream,const char * error)40 static void lzma_read_error(struct lzma_istream *zstream, const char *error)
41 {
42 io_stream_set_error(&zstream->istream.iostream,
43 "lzma.read(%s): %s at %"PRIuUOFF_T,
44 i_stream_get_name(&zstream->istream.istream), error,
45 i_stream_get_absolute_offset(&zstream->istream.istream));
46 }
47
lzma_handle_error(struct lzma_istream * zstream,lzma_ret lzma_err)48 static int lzma_handle_error(struct lzma_istream *zstream, lzma_ret lzma_err)
49 {
50 struct istream_private *stream = &zstream->istream;
51 switch (lzma_err) {
52 case LZMA_OK:
53 break;
54 case LZMA_DATA_ERROR:
55 case LZMA_BUF_ERROR:
56 lzma_read_error(zstream, "corrupted data");
57 stream->istream.stream_errno = EINVAL;
58 return -1;
59 case LZMA_FORMAT_ERROR:
60 lzma_read_error(zstream, "wrong magic in header (not xz file?)");
61 stream->istream.stream_errno = EINVAL;
62 return -1;
63 case LZMA_OPTIONS_ERROR:
64 lzma_read_error(zstream, "Unsupported xz options");
65 stream->istream.stream_errno = EIO;
66 return -1;
67 case LZMA_MEM_ERROR:
68 i_fatal_status(FATAL_OUTOFMEM, "lzma.read(%s): Out of memory",
69 i_stream_get_name(&stream->istream));
70 case LZMA_STREAM_END:
71 break;
72 default:
73 lzma_read_error(zstream, t_strdup_printf(
74 "lzma_code() failed with %d", lzma_err));
75 stream->istream.stream_errno = EIO;
76 return -1;
77 }
78 return 0;
79 }
80
lzma_stream_end(struct lzma_istream * zstream)81 static void lzma_stream_end(struct lzma_istream *zstream)
82 {
83 zstream->eof_offset = zstream->istream.istream.v_offset +
84 (zstream->istream.pos - zstream->istream.skip);
85 zstream->istream.cached_stream_size = zstream->eof_offset;
86 }
87
i_stream_lzma_read(struct istream_private * stream)88 static ssize_t i_stream_lzma_read(struct istream_private *stream)
89 {
90 struct lzma_istream *zstream = (struct lzma_istream *)stream;
91 const unsigned char *data;
92 uoff_t high_offset;
93 size_t size, out_size;
94 lzma_ret ret;
95
96 high_offset = stream->istream.v_offset + (stream->pos - stream->skip);
97 if (zstream->eof_offset == high_offset) {
98 stream->istream.eof = TRUE;
99 return -1;
100 }
101
102 if (!zstream->marked) {
103 if (!i_stream_try_alloc(stream, CHUNK_SIZE, &out_size))
104 return -2; /* buffer full */
105 } else {
106 /* try to avoid compressing, so we can quickly seek backwards */
107 if (!i_stream_try_alloc_avoid_compress(stream, CHUNK_SIZE, &out_size))
108 return -2; /* buffer full */
109 }
110
111 if (i_stream_read_more(stream->parent, &data, &size) < 0) {
112 if (stream->parent->stream_errno != 0) {
113 stream->istream.stream_errno =
114 stream->parent->stream_errno;
115 } else {
116 i_assert(stream->parent->eof);
117 lzma_stream_end(zstream);
118 ret = lzma_code(&zstream->strm, LZMA_FINISH);
119 if (lzma_handle_error(zstream, ret) < 0)
120 ;
121 else if (!zstream->hdr_read) {
122 lzma_read_error(zstream, "file too small (not xz file?)");
123 stream->istream.stream_errno = EINVAL;
124 } else if (ret != LZMA_STREAM_END) {
125 lzma_read_error(zstream, "unexpected EOF");
126 stream->istream.stream_errno = EPIPE;
127 }
128 stream->istream.eof = TRUE;
129 }
130 return -1;
131 }
132 if (size == 0) {
133 /* no more input */
134 i_assert(!stream->istream.blocking);
135 return 0;
136 }
137
138 zstream->strm.next_in = data;
139 zstream->strm.avail_in = size;
140
141 zstream->strm.next_out = stream->w_buffer + stream->pos;
142 zstream->strm.avail_out = out_size;
143 if (!zstream->hdr_read && size > LZMA_STREAM_HEADER_SIZE)
144 zstream->hdr_read = TRUE;
145 ret = lzma_code(&zstream->strm, LZMA_RUN);
146
147 out_size -= zstream->strm.avail_out;
148 stream->pos += out_size;
149
150 size_t bytes_consumed = size - zstream->strm.avail_in;
151 i_stream_skip(stream->parent, bytes_consumed);
152 if (i_stream_get_data_size(stream->parent) > 0 &&
153 (bytes_consumed > 0 || out_size > 0)) {
154 /* Parent stream was only partially consumed. Set the stream's
155 IO as pending to avoid hangs. */
156 i_stream_set_input_pending(&stream->istream, TRUE);
157 }
158
159 if (lzma_handle_error(zstream, ret) < 0) {
160 return -1;
161 } else if (ret == LZMA_STREAM_END) {
162 lzma_stream_end(zstream);
163 if (out_size == 0) {
164 stream->istream.eof = TRUE;
165 return -1;
166 }
167 }
168 if (out_size == 0) {
169 /* read more input */
170 return i_stream_lzma_read(stream);
171 }
172 return out_size;
173 }
174
i_stream_lzma_init(struct lzma_istream * zstream)175 static void i_stream_lzma_init(struct lzma_istream *zstream)
176 {
177 lzma_ret ret;
178
179 ret = lzma_stream_decoder(&zstream->strm, LZMA_MEMORY_LIMIT,
180 LZMA_CONCATENATED);
181 switch (ret) {
182 case LZMA_OK:
183 break;
184 case LZMA_MEM_ERROR:
185 i_fatal_status(FATAL_OUTOFMEM, "lzma: Out of memory");
186 default:
187 i_fatal("lzma_stream_decoder() failed with ret=%d", ret);
188 }
189 }
190
i_stream_lzma_reset(struct lzma_istream * zstream)191 static void i_stream_lzma_reset(struct lzma_istream *zstream)
192 {
193 struct istream_private *stream = &zstream->istream;
194
195 i_stream_seek(stream->parent, stream->parent_start_offset);
196 zstream->eof_offset = UOFF_T_MAX;
197 zstream->strm.next_in = NULL;
198 zstream->strm.avail_in = 0;
199
200 stream->parent_expected_offset = stream->parent_start_offset;
201 stream->skip = stream->pos = 0;
202 stream->istream.v_offset = 0;
203
204 lzma_end(&zstream->strm);
205 i_stream_lzma_init(zstream);
206 }
207
208 static void
i_stream_lzma_seek(struct istream_private * stream,uoff_t v_offset,bool mark)209 i_stream_lzma_seek(struct istream_private *stream, uoff_t v_offset, bool mark)
210 {
211 struct lzma_istream *zstream = (struct lzma_istream *) stream;
212
213 if (i_stream_nonseekable_try_seek(stream, v_offset))
214 return;
215
216 /* have to seek backwards - reset state and retry */
217 i_stream_lzma_reset(zstream);
218 if (!i_stream_nonseekable_try_seek(stream, v_offset))
219 i_unreached();
220
221 if (mark)
222 zstream->marked = TRUE;
223 }
224
i_stream_lzma_sync(struct istream_private * stream)225 static void i_stream_lzma_sync(struct istream_private *stream)
226 {
227 struct lzma_istream *zstream = (struct lzma_istream *) stream;
228 const struct stat *st;
229
230 if (i_stream_stat(stream->parent, FALSE, &st) == 0) {
231 if (memcmp(&zstream->last_parent_statbuf,
232 st, sizeof(*st)) == 0) {
233 /* a compressed file doesn't change unexpectedly,
234 don't clear our caches unnecessarily */
235 return;
236 }
237 zstream->last_parent_statbuf = *st;
238 }
239 i_stream_lzma_reset(zstream);
240 }
241
i_stream_create_lzma(struct istream * input)242 struct istream *i_stream_create_lzma(struct istream *input)
243 {
244 struct lzma_istream *zstream;
245
246 zstream = i_new(struct lzma_istream, 1);
247 zstream->eof_offset = UOFF_T_MAX;
248
249 i_stream_lzma_init(zstream);
250
251 zstream->istream.iostream.close = i_stream_lzma_close;
252 zstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
253 zstream->istream.read = i_stream_lzma_read;
254 zstream->istream.seek = i_stream_lzma_seek;
255 zstream->istream.sync = i_stream_lzma_sync;
256
257 zstream->istream.istream.readable_fd = FALSE;
258 zstream->istream.istream.blocking = input->blocking;
259 zstream->istream.istream.seekable = input->seekable;
260
261 return i_stream_create(&zstream->istream, input,
262 i_stream_get_fd(input), 0);
263 }
264 #endif
265