1 /////////////////////////////////////////////////////////////////////////////// 2 // 3 /// \file hardware.c 4 /// \brief Detection of available hardware resources 5 // 6 // Author: Lasse Collin 7 // 8 // This file has been put into the public domain. 9 // You can do whatever you want with this file. 10 // 11 /////////////////////////////////////////////////////////////////////////////// 12 13 #include "private.h" 14 15 16 /// Maximum number of worker threads. This can be set with 17 /// the --threads=NUM command line option. 18 static uint32_t threads_max = 1; 19 20 /// True when the number of threads is automatically determined based 21 /// on the available hardware threads. 22 static bool threads_are_automatic = false; 23 24 /// If true, then try to use multi-threaded mode (if memlimit allows) 25 /// even if only one thread was requested explicitly (-T+1). 26 static bool use_mt_mode_with_one_thread = false; 27 28 /// Memory usage limit for compression 29 static uint64_t memlimit_compress = 0; 30 31 /// Memory usage limit for decompression 32 static uint64_t memlimit_decompress = 0; 33 34 /// Default memory usage for multithreaded modes: 35 /// 36 /// - Default value for --memlimit-compress when automatic number of threads 37 /// is used. However, if the limit wouldn't allow even one thread then 38 /// the limit is ignored in coder.c and one thread will be used anyway. 39 /// This mess is a compromise: we wish to prevent -T0 from using too 40 /// many threads but we also don't want xz to give an error due to 41 /// a memlimit that the user didn't explicitly set. 42 /// 43 /// - Default value for --memlimit-mt-decompress 44 /// 45 /// This value is calculated in hardware_init() and cannot be changed later. 46 static uint64_t memlimit_mt_default; 47 48 /// Memory usage limit for multithreaded decompression. This is a soft limit: 49 /// if reducing the number of threads to one isn't enough to keep memory 50 /// usage below this limit, then one thread is used and this limit is ignored. 51 /// memlimit_decompress is still obeyed. 52 /// 53 /// This can be set with --memlimit-mt-decompress. The default value for 54 /// this is memlimit_mt_default. 55 static uint64_t memlimit_mtdec; 56 57 /// Total amount of physical RAM 58 static uint64_t total_ram; 59 60 61 extern void 62 hardware_threads_set(uint32_t n) 63 { 64 // Reset these to false first and set them to true when appropriate. 65 threads_are_automatic = false; 66 use_mt_mode_with_one_thread = false; 67 68 if (n == 0) { 69 // Automatic number of threads was requested. 70 // If there is only one hardware thread, multi-threaded 71 // mode will still be used if memory limit allows. 72 threads_are_automatic = true; 73 use_mt_mode_with_one_thread = true; 74 75 // If threading support was enabled at build time, 76 // use the number of available CPU cores. Otherwise 77 // use one thread since disabling threading support 78 // omits lzma_cputhreads() from liblzma. 79 #ifdef MYTHREAD_ENABLED 80 threads_max = lzma_cputhreads(); 81 if (threads_max == 0) 82 threads_max = 1; 83 #else 84 threads_max = 1; 85 #endif 86 } else if (n == UINT32_MAX) { 87 use_mt_mode_with_one_thread = true; 88 threads_max = 1; 89 } else { 90 threads_max = n; 91 } 92 93 return; 94 } 95 96 97 extern uint32_t 98 hardware_threads_get(void) 99 { 100 return threads_max; 101 } 102 103 104 extern bool 105 hardware_threads_is_mt(void) 106 { 107 #ifdef MYTHREAD_ENABLED 108 return threads_max > 1 || use_mt_mode_with_one_thread; 109 #else 110 return false; 111 #endif 112 } 113 114 115 extern void 116 hardware_memlimit_set(uint64_t new_memlimit, 117 bool set_compress, bool set_decompress, bool set_mtdec, 118 bool is_percentage) 119 { 120 if (is_percentage) { 121 assert(new_memlimit > 0); 122 assert(new_memlimit <= 100); 123 new_memlimit = (uint32_t)new_memlimit * total_ram / 100; 124 } 125 126 if (set_compress) { 127 memlimit_compress = new_memlimit; 128 129 #if SIZE_MAX == UINT32_MAX 130 // FIXME? 131 // 132 // When running a 32-bit xz on a system with a lot of RAM and 133 // using a percentage-based memory limit, the result can be 134 // bigger than the 32-bit address space. Limiting the limit 135 // below SIZE_MAX for compression (not decompression) makes 136 // xz lower the compression settings (or number of threads) 137 // to a level that *might* work. In practice it has worked 138 // when using a 64-bit kernel that gives full 4 GiB address 139 // space to 32-bit programs. In other situations this might 140 // still be too high, like 32-bit kernels that may give much 141 // less than 4 GiB to a single application. 142 // 143 // So this is an ugly hack but I will keep it here while 144 // it does more good than bad. 145 // 146 // Use a value less than SIZE_MAX so that there's some room 147 // for the xz program and so on. Don't use 4000 MiB because 148 // it could look like someone mixed up base-2 and base-10. 149 #ifdef __mips__ 150 // For MIPS32, due to architectural peculiarities, 151 // the limit is even lower. 152 const uint64_t limit_max = UINT64_C(2000) << 20; 153 #else 154 const uint64_t limit_max = UINT64_C(4020) << 20; 155 #endif 156 157 // UINT64_MAX is a special case for the string "max" so 158 // that has to be handled specially. 159 if (memlimit_compress != UINT64_MAX 160 && memlimit_compress > limit_max) 161 memlimit_compress = limit_max; 162 #endif 163 } 164 165 if (set_decompress) 166 memlimit_decompress = new_memlimit; 167 168 if (set_mtdec) 169 memlimit_mtdec = new_memlimit; 170 171 return; 172 } 173 174 175 extern uint64_t 176 hardware_memlimit_get(enum operation_mode mode) 177 { 178 // 0 is a special value that indicates the default. 179 // It disables the limit in single-threaded mode. 180 // 181 // NOTE: For multithreaded decompression, this is the hard limit 182 // (memlimit_stop). hardware_memlimit_mtdec_get() gives the 183 // soft limit (memlimit_threaded). 184 const uint64_t memlimit = mode == MODE_COMPRESS 185 ? memlimit_compress : memlimit_decompress; 186 return memlimit != 0 ? memlimit : UINT64_MAX; 187 } 188 189 190 extern uint64_t 191 hardware_memlimit_mtenc_get(void) 192 { 193 return hardware_memlimit_mtenc_is_default() 194 ? memlimit_mt_default 195 : hardware_memlimit_get(MODE_COMPRESS); 196 } 197 198 199 extern bool 200 hardware_memlimit_mtenc_is_default(void) 201 { 202 return memlimit_compress == 0 && threads_are_automatic; 203 } 204 205 206 extern uint64_t 207 hardware_memlimit_mtdec_get(void) 208 { 209 uint64_t m = memlimit_mtdec != 0 210 ? memlimit_mtdec 211 : memlimit_mt_default; 212 213 // Cap the value to memlimit_decompress if it has been specified. 214 // This is nice for --info-memory. It wouldn't be needed for liblzma 215 // since it does this anyway. 216 if (memlimit_decompress != 0 && m > memlimit_decompress) 217 m = memlimit_decompress; 218 219 return m; 220 } 221 222 223 /// Helper for hardware_memlimit_show() to print one human-readable info line. 224 static void 225 memlimit_show(const char *str, size_t str_columns, uint64_t value) 226 { 227 // Calculate the field width so that str will be padded to take 228 // str_columns on the terminal. 229 // 230 // NOTE: If the string is invalid, this will be -1. Using -1 as 231 // the field width is fine here so it's not handled specially. 232 const int fw = tuklib_mbstr_fw(str, (int)(str_columns)); 233 234 // The memory usage limit is considered to be disabled if value 235 // is 0 or UINT64_MAX. This might get a bit more complex once there 236 // is threading support. See the comment in hardware_memlimit_get(). 237 if (value == 0 || value == UINT64_MAX) 238 printf(" %-*s %s\n", fw, str, _("Disabled")); 239 else 240 printf(" %-*s %s MiB (%s B)\n", fw, str, 241 uint64_to_str(round_up_to_mib(value), 0), 242 uint64_to_str(value, 1)); 243 244 return; 245 } 246 247 248 extern void 249 hardware_memlimit_show(void) 250 { 251 uint32_t cputhreads = 1; 252 #ifdef MYTHREAD_ENABLED 253 cputhreads = lzma_cputhreads(); 254 if (cputhreads == 0) 255 cputhreads = 1; 256 #endif 257 258 if (opt_robot) { 259 printf("%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 260 "\t%" PRIu64 "\t%" PRIu32 "\n", 261 total_ram, 262 memlimit_compress, 263 memlimit_decompress, 264 hardware_memlimit_mtdec_get(), 265 memlimit_mt_default, 266 cputhreads); 267 } else { 268 const char *msgs[] = { 269 _("Amount of physical memory (RAM):"), 270 _("Number of processor threads:"), 271 _("Compression:"), 272 _("Decompression:"), 273 _("Multi-threaded decompression:"), 274 _("Default for -T0:"), 275 }; 276 277 size_t width_max = 1; 278 for (unsigned i = 0; i < ARRAY_SIZE(msgs); ++i) { 279 size_t w = tuklib_mbstr_width(msgs[i], NULL); 280 281 // When debugging, catch invalid strings with 282 // an assertion. Otherwise fallback to 1 so 283 // that the columns just won't be aligned. 284 assert(w != (size_t)-1); 285 if (w == (size_t)-1) 286 w = 1; 287 288 if (width_max < w) 289 width_max = w; 290 } 291 292 puts(_("Hardware information:")); 293 memlimit_show(msgs[0], width_max, total_ram); 294 printf(" %-*s %" PRIu32 "\n", 295 tuklib_mbstr_fw(msgs[1], (int)(width_max)), 296 msgs[1], cputhreads); 297 298 putchar('\n'); 299 puts(_("Memory usage limits:")); 300 memlimit_show(msgs[2], width_max, memlimit_compress); 301 memlimit_show(msgs[3], width_max, memlimit_decompress); 302 memlimit_show(msgs[4], width_max, 303 hardware_memlimit_mtdec_get()); 304 memlimit_show(msgs[5], width_max, memlimit_mt_default); 305 } 306 307 tuklib_exit(E_SUCCESS, E_ERROR, message_verbosity_get() != V_SILENT); 308 } 309 310 311 extern void 312 hardware_init(void) 313 { 314 // Get the amount of RAM. If we cannot determine it, 315 // use the assumption defined by the configure script. 316 total_ram = lzma_physmem(); 317 if (total_ram == 0) 318 total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024; 319 320 // FIXME? There may be better methods to determine the default value. 321 // One Linux-specific suggestion is to use MemAvailable from 322 // /proc/meminfo as the starting point. 323 memlimit_mt_default = total_ram / 4; 324 325 #if SIZE_MAX == UINT32_MAX 326 // A too high value may cause 32-bit xz to run out of address space. 327 // Use a conservative maximum value here. A few typical address space 328 // sizes with Linux: 329 // - x86-64 with 32-bit xz: 4 GiB 330 // - x86: 3 GiB 331 // - MIPS32: 2 GiB 332 const size_t mem_ceiling = 1400U << 20; 333 if (memlimit_mt_default > mem_ceiling) 334 memlimit_mt_default = mem_ceiling; 335 #endif 336 337 return; 338 } 339