1import datetime 2import os 3import platform 4import re 5import socket 6import sys 7import time 8try: 9 import resource 10except ImportError: 11 resource = None 12 13try: 14 # Optional dependency 15 import psutil 16except ImportError: 17 psutil = None 18 19import pyperf 20from pyperf._cli import format_metadata 21from pyperf._cpu_utils import (format_cpu_list, 22 parse_cpu_list, get_isolated_cpus, 23 get_logical_cpu_count, format_cpu_infos, 24 set_cpu_affinity) 25from pyperf._formatter import format_timedelta, format_datetime 26from pyperf._utils import (MS_WINDOWS, 27 open_text, read_first_line, sysfs_path, proc_path) 28if MS_WINDOWS: 29 from pyperf._win_memory import check_tracking_memory, get_peak_pagefile_usage 30 31 32def normalize_text(text): 33 text = str(text) 34 text = re.sub(r'\s+', ' ', text) 35 return text.strip() 36 37 38def collect_python_metadata(metadata): 39 # Implementation 40 impl = pyperf.python_implementation() 41 metadata['python_implementation'] = impl 42 43 # Version 44 version = platform.python_version() 45 46 match = re.search(r'\[(PyPy [^ ]+)', sys.version) 47 if match: 48 version = '%s (Python %s)' % (match.group(1), version) 49 50 bits = platform.architecture()[0] 51 if bits: 52 if bits == '64bit': 53 bits = '64-bit' 54 elif bits == '32bit': 55 bits = '32-bit' 56 version = '%s (%s)' % (version, bits) 57 58 # '74667320778e' in 'Python 2.7.12+ (2.7:74667320778e,' 59 match = re.search(r'^[^(]+\([^:]+:([a-f0-9]{6,}\+?),', sys.version) 60 if match: 61 revision = match.group(1) 62 else: 63 # 'bbd45126bc691f669c4ebdfbd74456cd274c6b92' 64 # in 'Python 2.7.10 (bbd45126bc691f669c4ebdfbd74456cd274c6b92,' 65 match = re.search(r'^[^(]+\(([a-f0-9]{6,}\+?),', sys.version) 66 if match: 67 revision = match.group(1) 68 else: 69 revision = None 70 if revision: 71 version = '%s revision %s' % (version, revision) 72 metadata['python_version'] = version 73 74 if sys.executable: 75 metadata['python_executable'] = sys.executable 76 77 # timer 78 info = time.get_clock_info('perf_counter') 79 metadata['timer'] = ('%s, resolution: %s' 80 % (info.implementation, 81 format_timedelta(info.resolution))) 82 83 # PYTHONHASHSEED 84 if os.environ.get('PYTHONHASHSEED'): 85 hash_seed = os.environ['PYTHONHASHSEED'] 86 try: 87 if hash_seed != "random": 88 hash_seed = int(hash_seed) 89 except ValueError: 90 pass 91 else: 92 metadata['python_hash_seed'] = hash_seed 93 94 # compiler 95 python_compiler = normalize_text(platform.python_compiler()) 96 if python_compiler: 97 metadata['python_compiler'] = python_compiler 98 99 # CFLAGS 100 try: 101 import sysconfig 102 except ImportError: 103 pass 104 else: 105 cflags = sysconfig.get_config_var('CFLAGS') 106 if cflags: 107 cflags = normalize_text(cflags) 108 metadata['python_cflags'] = cflags 109 110 # GC disabled? 111 try: 112 import gc 113 except ImportError: 114 pass 115 else: 116 if not gc.isenabled(): 117 metadata['python_gc'] = 'disabled' 118 119 120def read_proc(path): 121 path = proc_path(path) 122 try: 123 with open_text(path) as fp: 124 for line in fp: 125 yield line.rstrip() 126 except (OSError, IOError): 127 return 128 129 130def collect_linux_metadata(metadata): 131 # ASLR 132 for line in read_proc('sys/kernel/randomize_va_space'): 133 if line == '0': 134 metadata['aslr'] = 'No randomization' 135 elif line == '1': 136 metadata['aslr'] = 'Conservative randomization' 137 elif line == '2': 138 metadata['aslr'] = 'Full randomization' 139 break 140 141 142def get_cpu_affinity(): 143 if hasattr(os, 'sched_getaffinity'): 144 return os.sched_getaffinity(0) 145 146 if psutil is not None: 147 proc = psutil.Process() 148 # cpu_affinity() is only available on Linux, Windows and FreeBSD 149 if hasattr(proc, 'cpu_affinity'): 150 return proc.cpu_affinity() 151 152 return None 153 154 155def collect_system_metadata(metadata): 156 metadata['platform'] = platform.platform(True, False) 157 if sys.platform.startswith('linux'): 158 collect_linux_metadata(metadata) 159 160 # on linux, load average over 1 minute 161 for line in read_proc("loadavg"): 162 fields = line.split() 163 loadavg = fields[0] 164 metadata['load_avg_1min'] = float(loadavg) 165 166 if len(fields) >= 4 and '/' in fields[3]: 167 runnable_threads = fields[3].split('/', 1)[0] 168 runnable_threads = int(runnable_threads) 169 metadata['runnable_threads'] = runnable_threads 170 171 if 'load_avg_1min' not in metadata and hasattr(os, 'getloadavg'): 172 metadata['load_avg_1min'] = os.getloadavg()[0] 173 174 # Hostname 175 hostname = socket.gethostname() 176 if hostname: 177 metadata['hostname'] = hostname 178 179 # Boot time 180 boot_time = None 181 for line in read_proc("stat"): 182 if not line.startswith("btime "): 183 continue 184 boot_time = int(line[6:]) 185 break 186 187 if boot_time is None and psutil: 188 boot_time = psutil.boot_time() 189 190 if boot_time is not None: 191 btime = datetime.datetime.fromtimestamp(boot_time) 192 metadata['boot_time'] = format_datetime(btime) 193 metadata['uptime'] = time.time() - boot_time 194 195 196def collect_memory_metadata(metadata): 197 if resource is not None: 198 usage = resource.getrusage(resource.RUSAGE_SELF) 199 max_rss = usage.ru_maxrss 200 if max_rss: 201 metadata['mem_max_rss'] = max_rss * 1024 202 203 # Note: Don't collect VmPeak of /proc/self/status on Linux because it is 204 # not accurate. See pyperf._linux_memory for more accurate memory metrics. 205 206 # On Windows, use GetProcessMemoryInfo() if available 207 if MS_WINDOWS and not check_tracking_memory(): 208 usage = get_peak_pagefile_usage() 209 if usage: 210 metadata['mem_peak_pagefile_usage'] = usage 211 212 213def collect_cpu_freq(metadata, cpus): 214 # Parse /proc/cpuinfo: search for 'cpu MHz' (Intel) or 'clock' (Power8) 215 cpu_set = set(cpus) 216 cpu_freq = {} 217 cpu = None 218 for line in read_proc('cpuinfo'): 219 line = line.rstrip() 220 221 if line.startswith('processor'): 222 # Intel format, example where \t is a tab (U+0009 character): 223 # processor\t: 7 224 # model name\t: Intel(R) Core(TM) i7-6820HQ CPU @ 2.70GHz 225 # cpu MHz\t\t: 800.009 226 match = re.match(r'^processor\s*: ([0-9]+)', line) 227 if match is None: 228 # IBM Z 229 # Example: "processor 0: version = 00, identification = [...]" 230 match = re.match(r'^processor ([0-9]+): ', line) 231 if match is None: 232 raise Exception 233 # unknown /proc/cpuinfo format: silently ignore and exit 234 return 235 236 cpu = int(match.group(1)) 237 if cpu not in cpu_set: 238 # skip this CPU 239 cpu = None 240 241 elif line.startswith('cpu MHz') and cpu is not None: 242 # Intel: 'cpu MHz : 1261.613' 243 mhz = line.split(':', 1)[-1].strip() 244 mhz = float(mhz) 245 mhz = int(round(mhz)) 246 cpu_freq[cpu] = '%s MHz' % mhz 247 248 elif line.startswith('clock') and line.endswith('MHz') and cpu is not None: 249 # Power8: 'clock : 3425.000000MHz' 250 mhz = line[:-3].split(':', 1)[-1].strip() 251 mhz = float(mhz) 252 mhz = int(round(mhz)) 253 cpu_freq[cpu] = '%s MHz' % mhz 254 255 if not cpu_freq: 256 return 257 258 metadata['cpu_freq'] = '; '.join(format_cpu_infos(cpu_freq)) 259 260 261def get_cpu_config(cpu): 262 sys_cpu_path = sysfs_path("devices/system/cpu") 263 info = [] 264 265 path = os.path.join(sys_cpu_path, "cpu%s/cpufreq/scaling_driver" % cpu) 266 scaling_driver = read_first_line(path) 267 if scaling_driver: 268 info.append('driver:%s' % scaling_driver) 269 270 if scaling_driver == 'intel_pstate': 271 path = os.path.join(sys_cpu_path, "intel_pstate/no_turbo") 272 no_turbo = read_first_line(path) 273 if no_turbo == '1': 274 info.append('intel_pstate:no turbo') 275 elif no_turbo == '0': 276 info.append('intel_pstate:turbo') 277 278 path = os.path.join(sys_cpu_path, "cpu%s/cpufreq/scaling_governor" % cpu) 279 scaling_governor = read_first_line(path) 280 if scaling_governor: 281 info.append('governor:%s' % scaling_governor) 282 283 return info 284 285 286def collect_cpu_config(metadata, cpus): 287 nohz_full = read_first_line(sysfs_path('devices/system/cpu/nohz_full')) 288 if nohz_full: 289 nohz_full = parse_cpu_list(nohz_full) 290 291 isolated = get_isolated_cpus() 292 if isolated: 293 isolated = set(isolated) 294 295 configs = {} 296 for cpu in cpus: 297 config = get_cpu_config(cpu) 298 if nohz_full and cpu in nohz_full: 299 config.append('nohz_full') 300 if isolated and cpu in isolated: 301 config.append('isolated') 302 if config: 303 configs[cpu] = ', '.join(config) 304 config = format_cpu_infos(configs) 305 306 cpuidle = read_first_line('/sys/devices/system/cpu/cpuidle/current_driver') 307 if cpuidle: 308 config.append('idle:%s' % cpuidle) 309 310 if not config: 311 return 312 metadata['cpu_config'] = '; '.join(config) 313 314 315def get_cpu_temperature(path, cpu_temp): 316 hwmon_name = read_first_line(os.path.join(path, 'name')) 317 if not hwmon_name.startswith('coretemp'): 318 return 319 320 index = 1 321 while True: 322 template = os.path.join(path, "temp%s_%%s" % index) 323 324 try: 325 temp_label = read_first_line(template % 'label', error=True) 326 except IOError: 327 break 328 329 temp_input = read_first_line(template % 'input', error=True) 330 temp_input = float(temp_input) / 1000 331 # On Python 2, u"%.0f\xb0C" introduces unicode errors if the 332 # locale encoding is ASCII, so use a space. 333 temp_input = "%.0f C" % temp_input 334 335 item = '%s:%s=%s' % (hwmon_name, temp_label, temp_input) 336 cpu_temp.append(item) 337 338 index += 1 339 340 341def collect_cpu_temperatures(metadata): 342 path = sysfs_path("class/hwmon") 343 try: 344 names = os.listdir(path) 345 except OSError: 346 return None 347 348 cpu_temp = [] 349 for name in names: 350 hwmon = os.path.join(path, name) 351 get_cpu_temperature(hwmon, cpu_temp) 352 if not cpu_temp: 353 return None 354 355 metadata['cpu_temp'] = ', '.join(cpu_temp) 356 357 358def collect_cpu_affinity(metadata, cpu_affinity, cpu_count): 359 if not cpu_affinity: 360 return 361 if not cpu_count: 362 return 363 364 # CPU affinity 365 if set(cpu_affinity) == set(range(cpu_count)): 366 return 367 368 metadata['cpu_affinity'] = format_cpu_list(cpu_affinity) 369 370 371def collect_cpu_model(metadata): 372 for line in read_proc("cpuinfo"): 373 if line.startswith('model name'): 374 model_name = line.split(':', 1)[1].strip() 375 if model_name: 376 metadata['cpu_model_name'] = model_name 377 break 378 379 if line.startswith('machine'): 380 machine = line.split(':', 1)[1].strip() 381 if machine: 382 metadata['cpu_machine'] = machine 383 break 384 385 386def collect_cpu_metadata(metadata): 387 collect_cpu_model(metadata) 388 389 # CPU count 390 cpu_count = get_logical_cpu_count() 391 if cpu_count: 392 metadata['cpu_count'] = cpu_count 393 394 cpu_affinity = get_cpu_affinity() 395 collect_cpu_affinity(metadata, cpu_affinity, cpu_count) 396 397 all_cpus = cpu_affinity 398 if not all_cpus and cpu_count: 399 all_cpus = tuple(range(cpu_count)) 400 401 if all_cpus: 402 collect_cpu_freq(metadata, all_cpus) 403 collect_cpu_config(metadata, all_cpus) 404 405 collect_cpu_temperatures(metadata) 406 407 408def collect_metadata(process=True): 409 metadata = {} 410 metadata['perf_version'] = pyperf.__version__ 411 metadata['date'] = format_datetime(datetime.datetime.now()) 412 413 collect_system_metadata(metadata) 414 collect_cpu_metadata(metadata) 415 if process: 416 collect_python_metadata(metadata) 417 collect_memory_metadata(metadata) 418 419 return metadata 420 421 422def cmd_collect_metadata(args): 423 filename = args.output 424 if filename and os.path.exists(filename): 425 print("ERROR: The JSON file %r already exists" % filename) 426 sys.exit(1) 427 428 cpus = args.affinity 429 if cpus: 430 if not set_cpu_affinity(cpus): 431 print("ERROR: failed to set the CPU affinity") 432 sys.exit(1) 433 else: 434 cpus = get_isolated_cpus() 435 if cpus: 436 set_cpu_affinity(cpus) 437 # ignore if set_cpu_affinity() failed 438 439 run = pyperf.Run([1.0]) 440 metadata = run.get_metadata() 441 if metadata: 442 print("Metadata:") 443 for line in format_metadata(metadata): 444 print(line) 445 446 if filename: 447 run = run._update_metadata({'name': 'metadata'}) 448 bench = pyperf.Benchmark([run]) 449 bench.dump(filename) 450