1import datetime
2import os
3import platform
4import re
5import socket
6import sys
7import time
8try:
9    import resource
10except ImportError:
11    resource = None
12
13try:
14    # Optional dependency
15    import psutil
16except ImportError:
17    psutil = None
18
19import pyperf
20from pyperf._cli import format_metadata
21from pyperf._cpu_utils import (format_cpu_list,
22                               parse_cpu_list, get_isolated_cpus,
23                               get_logical_cpu_count, format_cpu_infos,
24                               set_cpu_affinity)
25from pyperf._formatter import format_timedelta, format_datetime
26from pyperf._utils import (MS_WINDOWS,
27                           open_text, read_first_line, sysfs_path, proc_path)
28if MS_WINDOWS:
29    from pyperf._win_memory import check_tracking_memory, get_peak_pagefile_usage
30
31
32def normalize_text(text):
33    text = str(text)
34    text = re.sub(r'\s+', ' ', text)
35    return text.strip()
36
37
38def collect_python_metadata(metadata):
39    # Implementation
40    impl = pyperf.python_implementation()
41    metadata['python_implementation'] = impl
42
43    # Version
44    version = platform.python_version()
45
46    match = re.search(r'\[(PyPy [^ ]+)', sys.version)
47    if match:
48        version = '%s (Python %s)' % (match.group(1), version)
49
50    bits = platform.architecture()[0]
51    if bits:
52        if bits == '64bit':
53            bits = '64-bit'
54        elif bits == '32bit':
55            bits = '32-bit'
56        version = '%s (%s)' % (version, bits)
57
58    # '74667320778e' in 'Python 2.7.12+ (2.7:74667320778e,'
59    match = re.search(r'^[^(]+\([^:]+:([a-f0-9]{6,}\+?),', sys.version)
60    if match:
61        revision = match.group(1)
62    else:
63        # 'bbd45126bc691f669c4ebdfbd74456cd274c6b92'
64        # in 'Python 2.7.10 (bbd45126bc691f669c4ebdfbd74456cd274c6b92,'
65        match = re.search(r'^[^(]+\(([a-f0-9]{6,}\+?),', sys.version)
66        if match:
67            revision = match.group(1)
68        else:
69            revision = None
70    if revision:
71        version = '%s revision %s' % (version, revision)
72    metadata['python_version'] = version
73
74    if sys.executable:
75        metadata['python_executable'] = sys.executable
76
77    # timer
78    info = time.get_clock_info('perf_counter')
79    metadata['timer'] = ('%s, resolution: %s'
80                         % (info.implementation,
81                            format_timedelta(info.resolution)))
82
83    # PYTHONHASHSEED
84    if os.environ.get('PYTHONHASHSEED'):
85        hash_seed = os.environ['PYTHONHASHSEED']
86        try:
87            if hash_seed != "random":
88                hash_seed = int(hash_seed)
89        except ValueError:
90            pass
91        else:
92            metadata['python_hash_seed'] = hash_seed
93
94    # compiler
95    python_compiler = normalize_text(platform.python_compiler())
96    if python_compiler:
97        metadata['python_compiler'] = python_compiler
98
99    # CFLAGS
100    try:
101        import sysconfig
102    except ImportError:
103        pass
104    else:
105        cflags = sysconfig.get_config_var('CFLAGS')
106        if cflags:
107            cflags = normalize_text(cflags)
108            metadata['python_cflags'] = cflags
109
110    # GC disabled?
111    try:
112        import gc
113    except ImportError:
114        pass
115    else:
116        if not gc.isenabled():
117            metadata['python_gc'] = 'disabled'
118
119
120def read_proc(path):
121    path = proc_path(path)
122    try:
123        with open_text(path) as fp:
124            for line in fp:
125                yield line.rstrip()
126    except (OSError, IOError):
127        return
128
129
130def collect_linux_metadata(metadata):
131    # ASLR
132    for line in read_proc('sys/kernel/randomize_va_space'):
133        if line == '0':
134            metadata['aslr'] = 'No randomization'
135        elif line == '1':
136            metadata['aslr'] = 'Conservative randomization'
137        elif line == '2':
138            metadata['aslr'] = 'Full randomization'
139        break
140
141
142def get_cpu_affinity():
143    if hasattr(os, 'sched_getaffinity'):
144        return os.sched_getaffinity(0)
145
146    if psutil is not None:
147        proc = psutil.Process()
148        # cpu_affinity() is only available on Linux, Windows and FreeBSD
149        if hasattr(proc, 'cpu_affinity'):
150            return proc.cpu_affinity()
151
152    return None
153
154
155def collect_system_metadata(metadata):
156    metadata['platform'] = platform.platform(True, False)
157    if sys.platform.startswith('linux'):
158        collect_linux_metadata(metadata)
159
160    # on linux, load average over 1 minute
161    for line in read_proc("loadavg"):
162        fields = line.split()
163        loadavg = fields[0]
164        metadata['load_avg_1min'] = float(loadavg)
165
166        if len(fields) >= 4 and '/' in fields[3]:
167            runnable_threads = fields[3].split('/', 1)[0]
168            runnable_threads = int(runnable_threads)
169            metadata['runnable_threads'] = runnable_threads
170
171    if 'load_avg_1min' not in metadata and hasattr(os, 'getloadavg'):
172        metadata['load_avg_1min'] = os.getloadavg()[0]
173
174    # Hostname
175    hostname = socket.gethostname()
176    if hostname:
177        metadata['hostname'] = hostname
178
179    # Boot time
180    boot_time = None
181    for line in read_proc("stat"):
182        if not line.startswith("btime "):
183            continue
184        boot_time = int(line[6:])
185        break
186
187    if boot_time is None and psutil:
188        boot_time = psutil.boot_time()
189
190    if boot_time is not None:
191        btime = datetime.datetime.fromtimestamp(boot_time)
192        metadata['boot_time'] = format_datetime(btime)
193        metadata['uptime'] = time.time() - boot_time
194
195
196def collect_memory_metadata(metadata):
197    if resource is not None:
198        usage = resource.getrusage(resource.RUSAGE_SELF)
199        max_rss = usage.ru_maxrss
200        if max_rss:
201            metadata['mem_max_rss'] = max_rss * 1024
202
203    # Note: Don't collect VmPeak of /proc/self/status on Linux because it is
204    # not accurate. See pyperf._linux_memory for more accurate memory metrics.
205
206    # On Windows, use GetProcessMemoryInfo() if available
207    if MS_WINDOWS and not check_tracking_memory():
208        usage = get_peak_pagefile_usage()
209        if usage:
210            metadata['mem_peak_pagefile_usage'] = usage
211
212
213def collect_cpu_freq(metadata, cpus):
214    # Parse /proc/cpuinfo: search for 'cpu MHz' (Intel) or 'clock' (Power8)
215    cpu_set = set(cpus)
216    cpu_freq = {}
217    cpu = None
218    for line in read_proc('cpuinfo'):
219        line = line.rstrip()
220
221        if line.startswith('processor'):
222            # Intel format, example where \t is a tab (U+0009 character):
223            # processor\t: 7
224            # model name\t: Intel(R) Core(TM) i7-6820HQ CPU @ 2.70GHz
225            # cpu MHz\t\t: 800.009
226            match = re.match(r'^processor\s*: ([0-9]+)', line)
227            if match is None:
228                # IBM Z
229                # Example: "processor 0: version = 00,  identification = [...]"
230                match = re.match(r'^processor ([0-9]+): ', line)
231                if match is None:
232                    raise Exception
233                    # unknown /proc/cpuinfo format: silently ignore and exit
234                    return
235
236            cpu = int(match.group(1))
237            if cpu not in cpu_set:
238                # skip this CPU
239                cpu = None
240
241        elif line.startswith('cpu MHz') and cpu is not None:
242            # Intel: 'cpu MHz : 1261.613'
243            mhz = line.split(':', 1)[-1].strip()
244            mhz = float(mhz)
245            mhz = int(round(mhz))
246            cpu_freq[cpu] = '%s MHz' % mhz
247
248        elif line.startswith('clock') and line.endswith('MHz') and cpu is not None:
249            # Power8: 'clock : 3425.000000MHz'
250            mhz = line[:-3].split(':', 1)[-1].strip()
251            mhz = float(mhz)
252            mhz = int(round(mhz))
253            cpu_freq[cpu] = '%s MHz' % mhz
254
255    if not cpu_freq:
256        return
257
258    metadata['cpu_freq'] = '; '.join(format_cpu_infos(cpu_freq))
259
260
261def get_cpu_config(cpu):
262    sys_cpu_path = sysfs_path("devices/system/cpu")
263    info = []
264
265    path = os.path.join(sys_cpu_path, "cpu%s/cpufreq/scaling_driver" % cpu)
266    scaling_driver = read_first_line(path)
267    if scaling_driver:
268        info.append('driver:%s' % scaling_driver)
269
270    if scaling_driver == 'intel_pstate':
271        path = os.path.join(sys_cpu_path, "intel_pstate/no_turbo")
272        no_turbo = read_first_line(path)
273        if no_turbo == '1':
274            info.append('intel_pstate:no turbo')
275        elif no_turbo == '0':
276            info.append('intel_pstate:turbo')
277
278    path = os.path.join(sys_cpu_path, "cpu%s/cpufreq/scaling_governor" % cpu)
279    scaling_governor = read_first_line(path)
280    if scaling_governor:
281        info.append('governor:%s' % scaling_governor)
282
283    return info
284
285
286def collect_cpu_config(metadata, cpus):
287    nohz_full = read_first_line(sysfs_path('devices/system/cpu/nohz_full'))
288    if nohz_full:
289        nohz_full = parse_cpu_list(nohz_full)
290
291    isolated = get_isolated_cpus()
292    if isolated:
293        isolated = set(isolated)
294
295    configs = {}
296    for cpu in cpus:
297        config = get_cpu_config(cpu)
298        if nohz_full and cpu in nohz_full:
299            config.append('nohz_full')
300        if isolated and cpu in isolated:
301            config.append('isolated')
302        if config:
303            configs[cpu] = ', '.join(config)
304    config = format_cpu_infos(configs)
305
306    cpuidle = read_first_line('/sys/devices/system/cpu/cpuidle/current_driver')
307    if cpuidle:
308        config.append('idle:%s' % cpuidle)
309
310    if not config:
311        return
312    metadata['cpu_config'] = '; '.join(config)
313
314
315def get_cpu_temperature(path, cpu_temp):
316    hwmon_name = read_first_line(os.path.join(path, 'name'))
317    if not hwmon_name.startswith('coretemp'):
318        return
319
320    index = 1
321    while True:
322        template = os.path.join(path, "temp%s_%%s" % index)
323
324        try:
325            temp_label = read_first_line(template % 'label', error=True)
326        except IOError:
327            break
328
329        temp_input = read_first_line(template % 'input', error=True)
330        temp_input = float(temp_input) / 1000
331        # On Python 2, u"%.0f\xb0C" introduces unicode errors if the
332        # locale encoding is ASCII, so use a space.
333        temp_input = "%.0f C" % temp_input
334
335        item = '%s:%s=%s' % (hwmon_name, temp_label, temp_input)
336        cpu_temp.append(item)
337
338        index += 1
339
340
341def collect_cpu_temperatures(metadata):
342    path = sysfs_path("class/hwmon")
343    try:
344        names = os.listdir(path)
345    except OSError:
346        return None
347
348    cpu_temp = []
349    for name in names:
350        hwmon = os.path.join(path, name)
351        get_cpu_temperature(hwmon, cpu_temp)
352    if not cpu_temp:
353        return None
354
355    metadata['cpu_temp'] = ', '.join(cpu_temp)
356
357
358def collect_cpu_affinity(metadata, cpu_affinity, cpu_count):
359    if not cpu_affinity:
360        return
361    if not cpu_count:
362        return
363
364    # CPU affinity
365    if set(cpu_affinity) == set(range(cpu_count)):
366        return
367
368    metadata['cpu_affinity'] = format_cpu_list(cpu_affinity)
369
370
371def collect_cpu_model(metadata):
372    for line in read_proc("cpuinfo"):
373        if line.startswith('model name'):
374            model_name = line.split(':', 1)[1].strip()
375            if model_name:
376                metadata['cpu_model_name'] = model_name
377            break
378
379        if line.startswith('machine'):
380            machine = line.split(':', 1)[1].strip()
381            if machine:
382                metadata['cpu_machine'] = machine
383            break
384
385
386def collect_cpu_metadata(metadata):
387    collect_cpu_model(metadata)
388
389    # CPU count
390    cpu_count = get_logical_cpu_count()
391    if cpu_count:
392        metadata['cpu_count'] = cpu_count
393
394    cpu_affinity = get_cpu_affinity()
395    collect_cpu_affinity(metadata, cpu_affinity, cpu_count)
396
397    all_cpus = cpu_affinity
398    if not all_cpus and cpu_count:
399        all_cpus = tuple(range(cpu_count))
400
401    if all_cpus:
402        collect_cpu_freq(metadata, all_cpus)
403        collect_cpu_config(metadata, all_cpus)
404
405    collect_cpu_temperatures(metadata)
406
407
408def collect_metadata(process=True):
409    metadata = {}
410    metadata['perf_version'] = pyperf.__version__
411    metadata['date'] = format_datetime(datetime.datetime.now())
412
413    collect_system_metadata(metadata)
414    collect_cpu_metadata(metadata)
415    if process:
416        collect_python_metadata(metadata)
417        collect_memory_metadata(metadata)
418
419    return metadata
420
421
422def cmd_collect_metadata(args):
423    filename = args.output
424    if filename and os.path.exists(filename):
425        print("ERROR: The JSON file %r already exists" % filename)
426        sys.exit(1)
427
428    cpus = args.affinity
429    if cpus:
430        if not set_cpu_affinity(cpus):
431            print("ERROR: failed to set the CPU affinity")
432            sys.exit(1)
433    else:
434        cpus = get_isolated_cpus()
435        if cpus:
436            set_cpu_affinity(cpus)
437            # ignore if set_cpu_affinity() failed
438
439    run = pyperf.Run([1.0])
440    metadata = run.get_metadata()
441    if metadata:
442        print("Metadata:")
443        for line in format_metadata(metadata):
444            print(line)
445
446    if filename:
447        run = run._update_metadata({'name': 'metadata'})
448        bench = pyperf.Benchmark([run])
449        bench.dump(filename)
450