1 ///////////////////////////////////////////////////////////////////////////////
2 //
3 /// \file 04_compress_easy_mt.c
4 /// \brief Compress in multi-call mode using LZMA2 in multi-threaded mode
5 ///
6 /// Usage: ./04_compress_easy_mt < INFILE > OUTFILE
7 ///
8 /// Example: ./04_compress_easy_mt < foo > foo.xz
9 //
10 // Author: Lasse Collin
11 //
12 // This file has been put into the public domain.
13 // You can do whatever you want with this file.
14 //
15 ///////////////////////////////////////////////////////////////////////////////
16
17 #include <stdbool.h>
18 #include <stdlib.h>
19 #include <stdio.h>
20 #include <string.h>
21 #include <errno.h>
22 #include <lzma.h>
23
24
25 static bool
init_encoder(lzma_stream * strm)26 init_encoder(lzma_stream *strm)
27 {
28 // The threaded encoder takes the options as pointer to
29 // a lzma_mt structure.
30 lzma_mt mt = {
31 // No flags are needed.
32 .flags = 0,
33
34 // Let liblzma determine a sane block size.
35 .block_size = 0,
36
37 // Use no timeout for lzma_code() calls by setting timeout
38 // to zero. That is, sometimes lzma_code() might block for
39 // a long time (from several seconds to even minutes).
40 // If this is not OK, for example due to progress indicator
41 // needing updates, specify a timeout in milliseconds here.
42 // See the documentation of lzma_mt in lzma/container.h for
43 // information how to choose a reasonable timeout.
44 .timeout = 0,
45
46 // Use the default preset (6) for LZMA2.
47 // To use a preset, filters must be set to NULL.
48 .preset = LZMA_PRESET_DEFAULT,
49 .filters = NULL,
50
51 // Use CRC64 for integrity checking. See also
52 // 01_compress_easy.c about choosing the integrity check.
53 .check = LZMA_CHECK_CRC64,
54 };
55
56 // Detect how many threads the CPU supports.
57 mt.threads = lzma_cputhreads();
58
59 // If the number of CPU cores/threads cannot be detected,
60 // use one thread. Note that this isn't the same as the normal
61 // single-threaded mode as this will still split the data into
62 // blocks and use more RAM than the normal single-threaded mode.
63 // You may want to consider using lzma_easy_encoder() or
64 // lzma_stream_encoder() instead of lzma_stream_encoder_mt() if
65 // lzma_cputhreads() returns 0 or 1.
66 if (mt.threads == 0)
67 mt.threads = 1;
68
69 // If the number of CPU cores/threads exceeds threads_max,
70 // limit the number of threads to keep memory usage lower.
71 // The number 8 is arbitrarily chosen and may be too low or
72 // high depending on the compression preset and the computer
73 // being used.
74 //
75 // FIXME: A better way could be to check the amount of RAM
76 // (or available RAM) and use lzma_stream_encoder_mt_memusage()
77 // to determine if the number of threads should be reduced.
78 const uint32_t threads_max = 8;
79 if (mt.threads > threads_max)
80 mt.threads = threads_max;
81
82 // Initialize the threaded encoder.
83 lzma_ret ret = lzma_stream_encoder_mt(strm, &mt);
84
85 if (ret == LZMA_OK)
86 return true;
87
88 const char *msg;
89 switch (ret) {
90 case LZMA_MEM_ERROR:
91 msg = "Memory allocation failed";
92 break;
93
94 case LZMA_OPTIONS_ERROR:
95 // We are no longer using a plain preset so this error
96 // message has been edited accordingly compared to
97 // 01_compress_easy.c.
98 msg = "Specified filter chain is not supported";
99 break;
100
101 case LZMA_UNSUPPORTED_CHECK:
102 msg = "Specified integrity check is not supported";
103 break;
104
105 default:
106 msg = "Unknown error, possibly a bug";
107 break;
108 }
109
110 fprintf(stderr, "Error initializing the encoder: %s (error code %u)\n",
111 msg, ret);
112 return false;
113 }
114
115
116 // This function is identical to the one in 01_compress_easy.c.
117 static bool
compress(lzma_stream * strm,FILE * infile,FILE * outfile)118 compress(lzma_stream *strm, FILE *infile, FILE *outfile)
119 {
120 lzma_action action = LZMA_RUN;
121
122 uint8_t inbuf[BUFSIZ];
123 uint8_t outbuf[BUFSIZ];
124
125 strm->next_in = NULL;
126 strm->avail_in = 0;
127 strm->next_out = outbuf;
128 strm->avail_out = sizeof(outbuf);
129
130 while (true) {
131 if (strm->avail_in == 0 && !feof(infile)) {
132 strm->next_in = inbuf;
133 strm->avail_in = fread(inbuf, 1, sizeof(inbuf),
134 infile);
135
136 if (ferror(infile)) {
137 fprintf(stderr, "Read error: %s\n",
138 strerror(errno));
139 return false;
140 }
141
142 if (feof(infile))
143 action = LZMA_FINISH;
144 }
145
146 lzma_ret ret = lzma_code(strm, action);
147
148 if (strm->avail_out == 0 || ret == LZMA_STREAM_END) {
149 size_t write_size = sizeof(outbuf) - strm->avail_out;
150
151 if (fwrite(outbuf, 1, write_size, outfile)
152 != write_size) {
153 fprintf(stderr, "Write error: %s\n",
154 strerror(errno));
155 return false;
156 }
157
158 strm->next_out = outbuf;
159 strm->avail_out = sizeof(outbuf);
160 }
161
162 if (ret != LZMA_OK) {
163 if (ret == LZMA_STREAM_END)
164 return true;
165
166 const char *msg;
167 switch (ret) {
168 case LZMA_MEM_ERROR:
169 msg = "Memory allocation failed";
170 break;
171
172 case LZMA_DATA_ERROR:
173 msg = "File size limits exceeded";
174 break;
175
176 default:
177 msg = "Unknown error, possibly a bug";
178 break;
179 }
180
181 fprintf(stderr, "Encoder error: %s (error code %u)\n",
182 msg, ret);
183 return false;
184 }
185 }
186 }
187
188
189 extern int
main(void)190 main(void)
191 {
192 lzma_stream strm = LZMA_STREAM_INIT;
193
194 bool success = init_encoder(&strm);
195 if (success)
196 success = compress(&strm, stdin, stdout);
197
198 lzma_end(&strm);
199
200 if (fclose(stdout)) {
201 fprintf(stderr, "Write error: %s\n", strerror(errno));
202 success = false;
203 }
204
205 return success ? EXIT_SUCCESS : EXIT_FAILURE;
206 }
207