1 // SPDX-License-Identifier: 0BSD 2 3 /////////////////////////////////////////////////////////////////////////////// 4 // 5 /// \file hardware.c 6 /// \brief Detection of available hardware resources 7 // 8 // Author: Lasse Collin 9 // 10 /////////////////////////////////////////////////////////////////////////////// 11 12 #include "private.h" 13 14 15 /// Maximum number of worker threads. This can be set with 16 /// the --threads=NUM command line option. 17 static uint32_t threads_max; 18 19 /// True when the number of threads is automatically determined based 20 /// on the available hardware threads. 21 static bool threads_are_automatic = false; 22 23 /// If true, then try to use multi-threaded mode (if memlimit allows) 24 /// even if only one thread was requested explicitly (-T+1). 25 static bool use_mt_mode_with_one_thread = false; 26 27 /// Memory usage limit for compression 28 static uint64_t memlimit_compress = 0; 29 30 /// Memory usage limit for decompression 31 static uint64_t memlimit_decompress = 0; 32 33 /// Default memory usage for multithreaded modes: 34 /// 35 /// - Default value for --memlimit-compress when automatic number of threads 36 /// is used. However, if the limit wouldn't allow even one thread then 37 /// the limit is ignored in coder.c and one thread will be used anyway. 38 /// This mess is a compromise: we wish to prevent -T0 from using too 39 /// many threads but we also don't want xz to give an error due to 40 /// a memlimit that the user didn't explicitly set. 41 /// 42 /// - Default value for --memlimit-mt-decompress 43 /// 44 /// This value is calculated in hardware_init() and cannot be changed later. 45 static uint64_t memlimit_mt_default; 46 47 /// Memory usage limit for multithreaded decompression. This is a soft limit: 48 /// if reducing the number of threads to one isn't enough to keep memory 49 /// usage below this limit, then one thread is used and this limit is ignored. 50 /// memlimit_decompress is still obeyed. 51 /// 52 /// This can be set with --memlimit-mt-decompress. The default value for 53 /// this is memlimit_mt_default. 54 static uint64_t memlimit_mtdec; 55 56 /// Total amount of physical RAM 57 static uint64_t total_ram; 58 59 60 extern void 61 hardware_threads_set(uint32_t n) 62 { 63 // Reset these to false first and set them to true when appropriate. 64 threads_are_automatic = false; 65 use_mt_mode_with_one_thread = false; 66 67 if (n == 0) { 68 // Automatic number of threads was requested. 69 // If there is only one hardware thread, multi-threaded 70 // mode will still be used if memory limit allows. 71 threads_are_automatic = true; 72 use_mt_mode_with_one_thread = true; 73 74 // If threading support was enabled at build time, 75 // use the number of available CPU cores. Otherwise 76 // use one thread since disabling threading support 77 // omits lzma_cputhreads() from liblzma. 78 #ifdef MYTHREAD_ENABLED 79 threads_max = lzma_cputhreads(); 80 if (threads_max == 0) 81 threads_max = 1; 82 #else 83 threads_max = 1; 84 #endif 85 } else if (n == UINT32_MAX) { 86 use_mt_mode_with_one_thread = true; 87 threads_max = 1; 88 } else { 89 threads_max = n; 90 } 91 92 return; 93 } 94 95 96 extern uint32_t 97 hardware_threads_get(void) 98 { 99 return threads_max; 100 } 101 102 103 extern bool 104 hardware_threads_is_mt(void) 105 { 106 #ifdef MYTHREAD_ENABLED 107 return threads_max > 1 || use_mt_mode_with_one_thread; 108 #else 109 return false; 110 #endif 111 } 112 113 114 extern void 115 hardware_memlimit_set(uint64_t new_memlimit, 116 bool set_compress, bool set_decompress, bool set_mtdec, 117 bool is_percentage) 118 { 119 if (is_percentage) { 120 assert(new_memlimit > 0); 121 assert(new_memlimit <= 100); 122 new_memlimit = (uint32_t)new_memlimit * total_ram / 100; 123 } 124 125 if (set_compress) { 126 memlimit_compress = new_memlimit; 127 128 #if SIZE_MAX == UINT32_MAX 129 // FIXME? 130 // 131 // When running a 32-bit xz on a system with a lot of RAM and 132 // using a percentage-based memory limit, the result can be 133 // bigger than the 32-bit address space. Limiting the limit 134 // below SIZE_MAX for compression (not decompression) makes 135 // xz lower the compression settings (or number of threads) 136 // to a level that *might* work. In practice it has worked 137 // when using a 64-bit kernel that gives full 4 GiB address 138 // space to 32-bit programs. In other situations this might 139 // still be too high, like 32-bit kernels that may give much 140 // less than 4 GiB to a single application. 141 // 142 // So this is an ugly hack but I will keep it here while 143 // it does more good than bad. 144 // 145 // Use a value less than SIZE_MAX so that there's some room 146 // for the xz program and so on. Don't use 4000 MiB because 147 // it could look like someone mixed up base-2 and base-10. 148 #ifdef __mips__ 149 // For MIPS32, due to architectural peculiarities, 150 // the limit is even lower. 151 const uint64_t limit_max = UINT64_C(2000) << 20; 152 #else 153 const uint64_t limit_max = UINT64_C(4020) << 20; 154 #endif 155 156 // UINT64_MAX is a special case for the string "max" so 157 // that has to be handled specially. 158 if (memlimit_compress != UINT64_MAX 159 && memlimit_compress > limit_max) 160 memlimit_compress = limit_max; 161 #endif 162 } 163 164 if (set_decompress) 165 memlimit_decompress = new_memlimit; 166 167 if (set_mtdec) 168 memlimit_mtdec = new_memlimit; 169 170 return; 171 } 172 173 174 extern uint64_t 175 hardware_memlimit_get(enum operation_mode mode) 176 { 177 // 0 is a special value that indicates the default. 178 // It disables the limit in single-threaded mode. 179 // 180 // NOTE: For multithreaded decompression, this is the hard limit 181 // (memlimit_stop). hardware_memlimit_mtdec_get() gives the 182 // soft limit (memlimit_threaded). 183 const uint64_t memlimit = mode == MODE_COMPRESS 184 ? memlimit_compress : memlimit_decompress; 185 return memlimit != 0 ? memlimit : UINT64_MAX; 186 } 187 188 189 extern uint64_t 190 hardware_memlimit_mtenc_get(void) 191 { 192 return hardware_memlimit_mtenc_is_default() 193 ? memlimit_mt_default 194 : hardware_memlimit_get(MODE_COMPRESS); 195 } 196 197 198 extern bool 199 hardware_memlimit_mtenc_is_default(void) 200 { 201 return memlimit_compress == 0 && threads_are_automatic; 202 } 203 204 205 extern uint64_t 206 hardware_memlimit_mtdec_get(void) 207 { 208 uint64_t m = memlimit_mtdec != 0 209 ? memlimit_mtdec 210 : memlimit_mt_default; 211 212 // Cap the value to memlimit_decompress if it has been specified. 213 // This is nice for --info-memory. It wouldn't be needed for liblzma 214 // since it does this anyway. 215 if (memlimit_decompress != 0 && m > memlimit_decompress) 216 m = memlimit_decompress; 217 218 return m; 219 } 220 221 222 /// Helper for hardware_memlimit_show() to print one human-readable info line. 223 static void 224 memlimit_show(const char *str, size_t str_columns, uint64_t value) 225 { 226 // Calculate the field width so that str will be padded to take 227 // str_columns on the terminal. 228 // 229 // NOTE: If the string is invalid, this will be -1. Using -1 as 230 // the field width is fine here so it's not handled specially. 231 const int fw = tuklib_mbstr_fw(str, (int)(str_columns)); 232 233 // The memory usage limit is considered to be disabled if value 234 // is 0 or UINT64_MAX. This might get a bit more complex once there 235 // is threading support. See the comment in hardware_memlimit_get(). 236 if (value == 0 || value == UINT64_MAX) 237 printf(" %-*s %s\n", fw, str, _("Disabled")); 238 else 239 printf(" %-*s %s MiB (%s B)\n", fw, str, 240 uint64_to_str(round_up_to_mib(value), 0), 241 uint64_to_str(value, 1)); 242 243 return; 244 } 245 246 247 extern void 248 hardware_memlimit_show(void) 249 { 250 uint32_t cputhreads = 1; 251 #ifdef MYTHREAD_ENABLED 252 cputhreads = lzma_cputhreads(); 253 if (cputhreads == 0) 254 cputhreads = 1; 255 #endif 256 257 if (opt_robot) { 258 printf("%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 259 "\t%" PRIu64 "\t%" PRIu32 "\n", 260 total_ram, 261 memlimit_compress, 262 memlimit_decompress, 263 hardware_memlimit_mtdec_get(), 264 memlimit_mt_default, 265 cputhreads); 266 } else { 267 const char *msgs[] = { 268 _("Amount of physical memory (RAM):"), 269 _("Number of processor threads:"), 270 _("Compression:"), 271 _("Decompression:"), 272 _("Multi-threaded decompression:"), 273 _("Default for -T0:"), 274 }; 275 276 size_t width_max = 1; 277 for (unsigned i = 0; i < ARRAY_SIZE(msgs); ++i) { 278 size_t w = tuklib_mbstr_width(msgs[i], NULL); 279 280 // When debugging, catch invalid strings with 281 // an assertion. Otherwise fallback to 1 so 282 // that the columns just won't be aligned. 283 assert(w != (size_t)-1); 284 if (w == (size_t)-1) 285 w = 1; 286 287 if (width_max < w) 288 width_max = w; 289 } 290 291 puts(_("Hardware information:")); 292 memlimit_show(msgs[0], width_max, total_ram); 293 printf(" %-*s %" PRIu32 "\n", 294 tuklib_mbstr_fw(msgs[1], (int)(width_max)), 295 msgs[1], cputhreads); 296 297 putchar('\n'); 298 puts(_("Memory usage limits:")); 299 memlimit_show(msgs[2], width_max, memlimit_compress); 300 memlimit_show(msgs[3], width_max, memlimit_decompress); 301 memlimit_show(msgs[4], width_max, 302 hardware_memlimit_mtdec_get()); 303 memlimit_show(msgs[5], width_max, memlimit_mt_default); 304 } 305 306 tuklib_exit(E_SUCCESS, E_ERROR, message_verbosity_get() != V_SILENT); 307 } 308 309 310 extern void 311 hardware_init(void) 312 { 313 // Get the amount of RAM. If we cannot determine it, 314 // use the assumption defined by the configure script. 315 total_ram = lzma_physmem(); 316 if (total_ram == 0) 317 total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024; 318 319 // FIXME? There may be better methods to determine the default value. 320 // One Linux-specific suggestion is to use MemAvailable from 321 // /proc/meminfo as the starting point. 322 memlimit_mt_default = total_ram / 4; 323 324 #if SIZE_MAX == UINT32_MAX 325 // A too high value may cause 32-bit xz to run out of address space. 326 // Use a conservative maximum value here. A few typical address space 327 // sizes with Linux: 328 // - x86-64 with 32-bit xz: 4 GiB 329 // - x86: 3 GiB 330 // - MIPS32: 2 GiB 331 const size_t mem_ceiling = 1400U << 20; 332 if (memlimit_mt_default > mem_ceiling) 333 memlimit_mt_default = mem_ceiling; 334 #endif 335 336 // Enable threaded mode by default. xz 5.4.x and older 337 // used single-threaded mode by default. 338 hardware_threads_set(0); 339 340 return; 341 } 342