1 // SPDX-License-Identifier: 0BSD
2
3 ///////////////////////////////////////////////////////////////////////////////
4 //
5 /// \file hardware.c
6 /// \brief Detection of available hardware resources
7 //
8 // Author: Lasse Collin
9 //
10 ///////////////////////////////////////////////////////////////////////////////
11
12 #include "private.h"
13
14
15 /// Maximum number of worker threads. This can be set with
16 /// the --threads=NUM command line option.
17 static uint32_t threads_max;
18
19 /// True when the number of threads is automatically determined based
20 /// on the available hardware threads.
21 static bool threads_are_automatic = false;
22
23 /// If true, then try to use multi-threaded mode (if memlimit allows)
24 /// even if only one thread was requested explicitly (-T+1).
25 static bool use_mt_mode_with_one_thread = false;
26
27 /// Memory usage limit for compression
28 static uint64_t memlimit_compress = 0;
29
30 /// Memory usage limit for decompression
31 static uint64_t memlimit_decompress = 0;
32
33 /// Default memory usage for multithreaded modes:
34 ///
35 /// - Default value for --memlimit-compress when automatic number of threads
36 /// is used. However, if the limit wouldn't allow even one thread then
37 /// the limit is ignored in coder.c and one thread will be used anyway.
38 /// This mess is a compromise: we wish to prevent -T0 from using too
39 /// many threads but we also don't want xz to give an error due to
40 /// a memlimit that the user didn't explicitly set.
41 ///
42 /// - Default value for --memlimit-mt-decompress
43 ///
44 /// This value is calculated in hardware_init() and cannot be changed later.
45 static uint64_t memlimit_mt_default;
46
47 /// Memory usage limit for multithreaded decompression. This is a soft limit:
48 /// if reducing the number of threads to one isn't enough to keep memory
49 /// usage below this limit, then one thread is used and this limit is ignored.
50 /// memlimit_decompress is still obeyed.
51 ///
52 /// This can be set with --memlimit-mt-decompress. The default value for
53 /// this is memlimit_mt_default.
54 static uint64_t memlimit_mtdec;
55
56 /// Total amount of physical RAM
57 static uint64_t total_ram;
58
59
60 extern void
hardware_threads_set(uint32_t n)61 hardware_threads_set(uint32_t n)
62 {
63 // Reset these to false first and set them to true when appropriate.
64 threads_are_automatic = false;
65 use_mt_mode_with_one_thread = false;
66
67 if (n == 0) {
68 // Automatic number of threads was requested.
69 // If there is only one hardware thread, multi-threaded
70 // mode will still be used if memory limit allows.
71 threads_are_automatic = true;
72 use_mt_mode_with_one_thread = true;
73
74 // If threading support was enabled at build time,
75 // use the number of available CPU cores. Otherwise
76 // use one thread since disabling threading support
77 // omits lzma_cputhreads() from liblzma.
78 #ifdef MYTHREAD_ENABLED
79 threads_max = lzma_cputhreads();
80 if (threads_max == 0)
81 threads_max = 1;
82 #else
83 threads_max = 1;
84 #endif
85 } else if (n == UINT32_MAX) {
86 use_mt_mode_with_one_thread = true;
87 threads_max = 1;
88 } else {
89 threads_max = n;
90 }
91
92 return;
93 }
94
95
96 extern uint32_t
hardware_threads_get(void)97 hardware_threads_get(void)
98 {
99 return threads_max;
100 }
101
102
103 extern bool
hardware_threads_is_mt(void)104 hardware_threads_is_mt(void)
105 {
106 #ifdef MYTHREAD_ENABLED
107 return threads_max > 1 || use_mt_mode_with_one_thread;
108 #else
109 return false;
110 #endif
111 }
112
113
114 extern void
hardware_memlimit_set(uint64_t new_memlimit,bool set_compress,bool set_decompress,bool set_mtdec,bool is_percentage)115 hardware_memlimit_set(uint64_t new_memlimit,
116 bool set_compress, bool set_decompress, bool set_mtdec,
117 bool is_percentage)
118 {
119 if (is_percentage) {
120 assert(new_memlimit > 0);
121 assert(new_memlimit <= 100);
122 new_memlimit = (uint32_t)new_memlimit * total_ram / 100;
123 }
124
125 if (set_compress) {
126 memlimit_compress = new_memlimit;
127
128 #if SIZE_MAX == UINT32_MAX
129 // FIXME?
130 //
131 // When running a 32-bit xz on a system with a lot of RAM and
132 // using a percentage-based memory limit, the result can be
133 // bigger than the 32-bit address space. Limiting the limit
134 // below SIZE_MAX for compression (not decompression) makes
135 // xz lower the compression settings (or number of threads)
136 // to a level that *might* work. In practice it has worked
137 // when using a 64-bit kernel that gives full 4 GiB address
138 // space to 32-bit programs. In other situations this might
139 // still be too high, like 32-bit kernels that may give much
140 // less than 4 GiB to a single application.
141 //
142 // So this is an ugly hack but I will keep it here while
143 // it does more good than bad.
144 //
145 // Use a value less than SIZE_MAX so that there's some room
146 // for the xz program and so on. Don't use 4000 MiB because
147 // it could look like someone mixed up base-2 and base-10.
148 #ifdef __mips__
149 // For MIPS32, due to architectural peculiarities,
150 // the limit is even lower.
151 const uint64_t limit_max = UINT64_C(2000) << 20;
152 #else
153 const uint64_t limit_max = UINT64_C(4020) << 20;
154 #endif
155
156 // UINT64_MAX is a special case for the string "max" so
157 // that has to be handled specially.
158 if (memlimit_compress != UINT64_MAX
159 && memlimit_compress > limit_max)
160 memlimit_compress = limit_max;
161 #endif
162 }
163
164 if (set_decompress)
165 memlimit_decompress = new_memlimit;
166
167 if (set_mtdec)
168 memlimit_mtdec = new_memlimit;
169
170 return;
171 }
172
173
174 extern uint64_t
hardware_memlimit_get(enum operation_mode mode)175 hardware_memlimit_get(enum operation_mode mode)
176 {
177 // 0 is a special value that indicates the default.
178 // It disables the limit in single-threaded mode.
179 //
180 // NOTE: For multithreaded decompression, this is the hard limit
181 // (memlimit_stop). hardware_memlimit_mtdec_get() gives the
182 // soft limit (memlimit_threaded).
183 const uint64_t memlimit = mode == MODE_COMPRESS
184 ? memlimit_compress : memlimit_decompress;
185 return memlimit != 0 ? memlimit : UINT64_MAX;
186 }
187
188
189 extern uint64_t
hardware_memlimit_mtenc_get(void)190 hardware_memlimit_mtenc_get(void)
191 {
192 return hardware_memlimit_mtenc_is_default()
193 ? memlimit_mt_default
194 : hardware_memlimit_get(MODE_COMPRESS);
195 }
196
197
198 extern bool
hardware_memlimit_mtenc_is_default(void)199 hardware_memlimit_mtenc_is_default(void)
200 {
201 return memlimit_compress == 0 && threads_are_automatic;
202 }
203
204
205 extern uint64_t
hardware_memlimit_mtdec_get(void)206 hardware_memlimit_mtdec_get(void)
207 {
208 uint64_t m = memlimit_mtdec != 0
209 ? memlimit_mtdec
210 : memlimit_mt_default;
211
212 // Cap the value to memlimit_decompress if it has been specified.
213 // This is nice for --info-memory. It wouldn't be needed for liblzma
214 // since it does this anyway.
215 if (memlimit_decompress != 0 && m > memlimit_decompress)
216 m = memlimit_decompress;
217
218 return m;
219 }
220
221
222 /// Helper for hardware_memlimit_show() to print one human-readable info line.
223 static void
memlimit_show(const char * str,size_t str_columns,uint64_t value)224 memlimit_show(const char *str, size_t str_columns, uint64_t value)
225 {
226 // Calculate the field width so that str will be padded to take
227 // str_columns on the terminal.
228 //
229 // NOTE: If the string is invalid, this will be -1. Using -1 as
230 // the field width is fine here so it's not handled specially.
231 const int fw = tuklib_mbstr_fw(str, (int)(str_columns));
232
233 // The memory usage limit is considered to be disabled if value
234 // is 0 or UINT64_MAX. This might get a bit more complex once there
235 // is threading support. See the comment in hardware_memlimit_get().
236 if (value == 0 || value == UINT64_MAX)
237 printf(" %-*s %s\n", fw, str, _("Disabled"));
238 else
239 printf(" %-*s %s MiB (%s B)\n", fw, str,
240 uint64_to_str(round_up_to_mib(value), 0),
241 uint64_to_str(value, 1));
242
243 return;
244 }
245
246
247 extern void
hardware_memlimit_show(void)248 hardware_memlimit_show(void)
249 {
250 uint32_t cputhreads = 1;
251 #ifdef MYTHREAD_ENABLED
252 cputhreads = lzma_cputhreads();
253 if (cputhreads == 0)
254 cputhreads = 1;
255 #endif
256
257 if (opt_robot) {
258 printf("%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64
259 "\t%" PRIu64 "\t%" PRIu32 "\n",
260 total_ram,
261 memlimit_compress,
262 memlimit_decompress,
263 hardware_memlimit_mtdec_get(),
264 memlimit_mt_default,
265 cputhreads);
266 } else {
267 const char *msgs[] = {
268 _("Amount of physical memory (RAM):"),
269 _("Number of processor threads:"),
270 _("Compression:"),
271 _("Decompression:"),
272 _("Multi-threaded decompression:"),
273 _("Default for -T0:"),
274 };
275
276 size_t width_max = 1;
277 for (unsigned i = 0; i < ARRAY_SIZE(msgs); ++i) {
278 size_t w = tuklib_mbstr_width(msgs[i], NULL);
279
280 // When debugging, catch invalid strings with
281 // an assertion. Otherwise fallback to 1 so
282 // that the columns just won't be aligned.
283 assert(w != (size_t)-1);
284 if (w == (size_t)-1)
285 w = 1;
286
287 if (width_max < w)
288 width_max = w;
289 }
290
291 puts(_("Hardware information:"));
292 memlimit_show(msgs[0], width_max, total_ram);
293 printf(" %-*s %" PRIu32 "\n",
294 tuklib_mbstr_fw(msgs[1], (int)(width_max)),
295 msgs[1], cputhreads);
296
297 putchar('\n');
298 puts(_("Memory usage limits:"));
299 memlimit_show(msgs[2], width_max, memlimit_compress);
300 memlimit_show(msgs[3], width_max, memlimit_decompress);
301 memlimit_show(msgs[4], width_max,
302 hardware_memlimit_mtdec_get());
303 memlimit_show(msgs[5], width_max, memlimit_mt_default);
304 }
305
306 tuklib_exit(E_SUCCESS, E_ERROR, message_verbosity_get() != V_SILENT);
307 }
308
309
310 extern void
hardware_init(void)311 hardware_init(void)
312 {
313 // Get the amount of RAM. If we cannot determine it,
314 // use the assumption defined by the configure script.
315 total_ram = lzma_physmem();
316 if (total_ram == 0)
317 total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024;
318
319 // FIXME? There may be better methods to determine the default value.
320 // One Linux-specific suggestion is to use MemAvailable from
321 // /proc/meminfo as the starting point.
322 memlimit_mt_default = total_ram / 4;
323
324 #if SIZE_MAX == UINT32_MAX
325 // A too high value may cause 32-bit xz to run out of address space.
326 // Use a conservative maximum value here. A few typical address space
327 // sizes with Linux:
328 // - x86-64 with 32-bit xz: 4 GiB
329 // - x86: 3 GiB
330 // - MIPS32: 2 GiB
331 const size_t mem_ceiling = 1400U << 20;
332 if (memlimit_mt_default > mem_ceiling)
333 memlimit_mt_default = mem_ceiling;
334 #endif
335
336 // Enable threaded mode by default. xz 5.4.x and older
337 // used single-threaded mode by default.
338 hardware_threads_set(0);
339
340 return;
341 }
342