1#!/usr/bin/env python3 2# This file is part of Xpra. 3# Copyright (C) 2012, 2013 Antoine Martin <antoine@xpra.org> 4# Xpra is released under the terms of the GNU GPL v2, or, at your option, any 5# later version. See the file COPYING for details. 6# 7# To create multiple output files which can be used to generate charts (using test_measure_perf_charts.py) 8# build a config class (copy from perf_config_default.py -- make changes as necessary). 9# 10# Then determine the values of the following variables: 11# prefix: a string to identify the data set 12# id: a string to identify the variable that the data set is testing 13# (for example '14' because we're testing xpra v14 in this data set) 14# repetitions: decide how many times you want to run the tests 15# 16# The data file names you will produce will then be in the format: 17# prefix_id_rep#.csv 18# 19# With this information in hand you can now create a script that will run the tests, containing commands like: 20# ./test_measure_perf.py as an example: 21 22# For example: 23# 24# ./test_measure_perf.py all_tests_40 ./data/all_tests_40_14_1.csv 1 14 > ./data//all_tests_40_14_1.log 25# ./test_measure_perf.py all_tests_40 ./data/all_tests_40_14_2.csv 2 14 > ./data//all_tests_40_14_2.log 26# 27# In this example script, I'm running test_measure_perf 2 times, using a config class named "all_tests_40.py", 28# and outputting the results to data files using the prefix "all_tests_40", for version 14. 29# 30# The additional arguments "1 14", "2 14" are custom paramaters which will be written to the "Custom Params" column 31# in the corresponding data files. 32# 33# Where you see "1", "2" in the file names or params, that's referring to the corresponding repetition of the tests. 34# 35# Once this script has run, you can open up test_measure_perf_charts.py and take a look at the 36# instructions there for generating the charts. 37# 38 39import re 40import sys 41import time 42import os.path 43from subprocess import Popen, PIPE, STDOUT 44 45from xpra.exit_codes import EXIT_STR 46from xpra.gtk_common.gtk_util import get_root_size 47from xpra.log import Logger 48 49log = Logger("util") 50 51 52def getoutput(cmd, env=None): 53 try: 54 process = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT, env=env) 55 except Exception as e: 56 print("error running %s: %s" % (cmd, e)) 57 raise e 58 out, err = process.communicate() 59 code = process.poll() 60 if code!=0: 61 raise Exception("command '%s' returned error code %i: %s, out=%s, err=%s" % 62 (cmd, code, EXIT_STR.get(code), out, err)) 63 return out 64 65def get_config(config_name): 66 try: 67 mod = __import__(config_name) 68 except (ImportError, SyntaxError) as e: 69 sys.stderr.write("Error loading module %s (%s)\n" % (config_name, e)) 70 return None 71 return mod.Config() 72 73if len(sys.argv) > 1: 74 config_name = sys.argv[1] 75else: 76 config_name = 'perf_config_default' 77config = get_config(config_name) 78if not config: 79 raise Exception("Could not load config file") 80 81XPRA_BIN = "/usr/bin/xpra" 82XPRA_VERSION_OUTPUT = getoutput([XPRA_BIN, "--version"]) 83XPRA_VERSION = "" 84for x in XPRA_VERSION_OUTPUT.splitlines(): 85 if x.startswith("xpra v"): 86 XPRA_VERSION = x[len("xpra v"):].replace("\n", "").replace("\r", "") 87XPRA_VERSION = XPRA_VERSION.split("-")[0] 88XPRA_VERSION_NO = [int(x) for x in XPRA_VERSION.split(".")] 89XPRA_SERVER_STOP_COMMANDS = [ 90 "%s stop :%s" % (XPRA_BIN, config.DISPLAY_NO), 91 "ps -ef | grep -i [X]org-for-Xpra-:%s | awk '{print $2}' | xargs kill" % config.DISPLAY_NO 92 ] 93XPRA_INFO_COMMAND = [XPRA_BIN, "info", "tcp:%s:%s" % (config.IP, config.PORT)] 94print ("XPRA_VERSION_NO=%s" % XPRA_VERSION_NO) 95 96STRICT_ENCODINGS = False 97if STRICT_ENCODINGS: 98 #beware: only enable this flag if the version being tested 99 # also supports the same environment overrides, 100 # or the comparison will not be fair. 101 os.environ["XPRA_ENCODING_STRICT_MODE"] = "1" 102 os.environ["XPRA_MAX_PIXELS_PREFER_RGB"] = "0" 103 os.environ["XPRA_MAX_NONVIDEO_PIXELS"] = "0" 104 105XPRA_SPEAKER_OPTIONS = [None] 106XPRA_MICROPHONE_OPTIONS = [None] 107if config.TEST_SOUND: 108 from xpra.sound.gstreamer_util import CODEC_ORDER, has_codec 109 XPRA_SPEAKER_OPTIONS = [x for x in CODEC_ORDER if has_codec(x)] 110 111if config.XPRA_USE_PASSWORD: 112 password_filename = "./test-password.txt" 113 import uuid 114 with open(password_filename, 'wb') as f: 115 f.write(uuid.uuid4().hex) 116 117check = [config.TRICKLE_BIN] 118if config.TEST_XPRA: 119 check.append(XPRA_BIN) 120if config.TEST_VNC: 121 check.append(config.XVNC_BIN) 122 check.append(config.VNCVIEWER_BIN) 123for x in check: 124 if not os.path.exists(x): 125 raise Exception("cannot run tests: %s is missing!" % x) 126 127HEADERS = ["Test Name", "Remoting Tech", "Server Version", "Client Version", "Custom Params", "SVN Version", 128 "Encoding", "Quality", "Speed","OpenGL", "Test Command", "Sample Duration (s)", "Sample Time (epoch)", 129 "CPU info", "Platform", "Kernel Version", "Xorg version", "OpenGL", "Client Window Manager", "Screen Size", 130 "Compression", "Encryption", "Connect via", "download limit (KB)", "upload limit (KB)", "latency (ms)", 131 "packets in/s", "packets in: bytes/s", "packets out/s", "packets out: bytes/s", 132 "Regions/s", "Pixels/s Sent", "Encoding Pixels/s", "Decoding Pixels/s", 133 "Application packets in/s", "Application bytes in/s", 134 "Application packets out/s", "Application bytes out/s", "mmap bytes/s", 135 "Frame Total Latency", "Client Frame Latency", 136 "Video Encoder", "CSC", "CSC Mode", "Scaling", 137 ] 138for x in ("client", "server"): 139 HEADERS += [x+" user cpu_pct", x+" system cpu pct", x+" number of threads", x+" vsize (MB)", x+" rss (MB)"] 140#all these headers have min/max/avg: 141for h in ("Batch Delay (ms)", "Actual Batch Delay (ms)", 142 "Client Latency (ms)", "Client Ping Latency (ms)", "Server Ping Latency (ms)", 143 "Damage Latency (ms)", 144 "Quality", "Speed"): 145 for x in ("Min", "Avg", "Max"): 146 HEADERS.append(x+" "+h) 147 148def is_process_alive(process, grace=0): 149 i = 0 150 while i<grace: 151 if not process or process.poll() is not None: 152 return False 153 time.sleep(1) 154 i += 1 155 return process and process.poll() is None 156 157def try_to_stop(process, grace=0): 158 if is_process_alive(process, grace): 159 try: 160 process.terminate() 161 except Exception as e: 162 print("could not stop process %s: %s" % (process, e)) 163def try_to_kill(process, grace=0): 164 if is_process_alive(process, grace): 165 try: 166 process.kill() 167 except Exception as e: 168 print("could not stop process %s: %s" % (process, e)) 169 170def find_matching_lines(out, pattern): 171 lines = [] 172 for line in out.splitlines(): 173 if line.find(pattern)>=0: 174 lines.append(line) 175 return lines 176 177def getoutput_lines(cmd, pattern): 178 out = getoutput(cmd) 179 return find_matching_lines(out, pattern) 180 181def getoutput_line(cmd, pattern): 182 lines = getoutput_lines(cmd, pattern) 183 if len(lines)!=1: 184 print("WARNING: expected 1 line matching '%s' from %s but found %s" % (pattern, cmd, len(lines))) 185 return "not found" 186 return lines[0] 187 188def get_cpu_info(): 189 lines = getoutput_lines(["cat", "/proc/cpuinfo"], "model name") 190 assert lines, "coult not find 'model name' in '/proc/cpuinfo'" 191 cpu0 = lines[0] 192 n = len(lines) 193 for cpu in lines[1:]: 194 if cpu!=cpu0: 195 return " - ".join(lines), n 196 cpu_name = cpu0.split(":")[1] 197 for o,r in [("Processor", ""), ("(R)", ""), ("(TM)", ""), ("(tm)", ""), (" ", " ")]: 198 while cpu_name.find(o)>=0: 199 cpu_name = cpu_name.replace(o, r) 200 cpu_info = "%sx %s" % (len(lines), cpu_name.strip()) 201 print("CPU_INFO=%s" % cpu_info) 202 return cpu_info, n 203 204XORG_VERSION = getoutput_line([config.XORG_BIN, "-version"], "X.Org X Server") 205print("XORG_VERSION=%s" % XORG_VERSION) 206CPU_INFO, N_CPUS = get_cpu_info() 207KERNEL_VERSION = getoutput(["uname", "-r"]).replace("\n", "").replace("\r", "") 208PAGE_SIZE = int(getoutput(["getconf", "PAGESIZE"]).replace("\n", "").replace("\r", "")) 209PLATFORM = getoutput(["uname", "-p"]).replace("\n", "").replace("\r", "") 210OPENGL_INFO = getoutput_line(["glxinfo"], 211 "OpenGL renderer string").split("OpenGL renderer string:")[1].strip() 212 213SCREEN_SIZE = get_root_size() 214print("screen size=%s" % str(SCREEN_SIZE)) 215 216#detect Xvnc version: 217XVNC_VERSION = "" 218VNCVIEWER_VERSION = "" 219DETECT_XVNC_VERSION_CMD = [config.XVNC_BIN, "--help"] 220DETECT_VNCVIEWER_VERSION_CMD = [config.VNCVIEWER_BIN, "--help"] 221def get_stderr(command): 222 try: 223 process = Popen(command, stdin=PIPE, stdout=PIPE, stderr=PIPE) 224 return process.communicate()[1] 225 except Exception as e: 226 print("error running %s: %s" % (DETECT_XVNC_VERSION_CMD, e)) 227 228err = get_stderr(DETECT_XVNC_VERSION_CMD) 229if err: 230 v_lines = find_matching_lines(err, "Xvnc TigerVNC") 231 if len(v_lines)==1: 232 XVNC_VERSION = " ".join(v_lines[0].split()[:3]) 233print ("XVNC_VERSION=%s" % XVNC_VERSION) 234err = get_stderr(DETECT_VNCVIEWER_VERSION_CMD) 235if err: 236 v_lines = find_matching_lines(err, "TigerVNC Viewer for X version") 237 if len(v_lines)==1: 238 VNCVIEWER_VERSION = "TigerVNC Viewer %s" % (v_lines[0].split()[5]) 239print ("VNCVIEWER_VERSION=%s" % VNCVIEWER_VERSION) 240 241#get svnversion, prefer directly from svn: 242try: 243 SVN_VERSION = getoutput(["svnversion", "-n"]).split(":")[-1].strip() 244except: 245 SVN_VERSION = "" 246if not SVN_VERSION: 247 #fallback to getting it from xpra's src_info: 248 try: 249 from xpra.src_info import REVISION, LOCAL_MODIFICATIONS 250 SVN_VERSION = 'r%s' % REVISION 251 if LOCAL_MODIFICATIONS: 252 SVN_VERSION += "M" 253 except: 254 pass 255if not SVN_VERSION: 256 #fallback to running python: 257 SVN_VERSION = getoutput(["python", "-c", 258 "from xpra.src_info import REVISION,LOCAL_MODIFICATIONS;print(('r%s%s' % (REVISION, ' M'[int(bool(LOCAL_MODIFICATIONS))])).strip())"]) 259print("Found xpra revision: '%s'" % str(SVN_VERSION)) 260 261WINDOW_MANAGER = os.environ.get("DESKTOP_SESSION", "unknown") 262 263def clean_sys_state(): 264 #clear the caches 265 cmd = ["echo", "3", ">", "/proc/sys/vm/drop_caches"] 266 process = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) 267 assert process.wait()==0, "failed to run %s" % str(cmd) 268 269def zero_iptables(): 270 if not config.USE_IPTABLES: 271 return 272 cmds = [config.IPTABLES_CMD+['-Z', 'INPUT'], config.IPTABLES_CMD+['-Z', 'OUTPUT']] 273 for cmd in cmds: 274 getoutput(cmd) 275 #out = getoutput(cmd) 276 #print("output(%s)=%s" % (cmd, out)) 277 278def update_proc_stat(): 279 with open("/proc/stat", "rU") as proc_stat: 280 time_total = 0 281 for line in proc_stat: 282 values = line.split() 283 if values[0]=="cpu": 284 time_total = sum([int(x) for x in values[1:]]) 285 #print("time_total=%s" % time_total) 286 break 287 return time_total 288 289def update_pidstat(pid): 290 with open("/proc/%s/stat" % pid, "rU") as stat_file: 291 data = stat_file.read() 292 pid_stat = data.split() 293 #print("update_pidstat(%s): %s" % (pid, pid_stat)) 294 return pid_stat 295 296def compute_stat(prefix, time_total_diff, old_pid_stat, new_pid_stat): 297 #found help here: 298 #http://stackoverflow.com/questions/1420426/calculating-cpu-usage-of-a-process-in-linux 299 old_utime = int(old_pid_stat[13]) 300 old_stime = int(old_pid_stat[14]) 301 new_utime = int(new_pid_stat[13]) 302 new_stime = int(new_pid_stat[14]) 303 #normalize to 100% (single process) by multiplying by number of CPUs: 304 user_pct = int(N_CPUS * 1000 * (new_utime - old_utime) / time_total_diff)/10.0 305 sys_pct = int(N_CPUS * 1000 * (new_stime - old_stime) / time_total_diff)/10.0 306 nthreads = int((int(old_pid_stat[19])+int(new_pid_stat[19]))/2) 307 vsize = int(max(int(old_pid_stat[22]), int(new_pid_stat[22]))/1024/1024) 308 rss = int(max(int(old_pid_stat[23]), int(new_pid_stat[23]))*PAGE_SIZE/1024/1024) 309 return {prefix+" user cpu_pct" : user_pct, 310 prefix+" system cpu pct" : sys_pct, 311 prefix+" number of threads" : nthreads, 312 prefix+" vsize (MB)" : vsize, 313 prefix+" rss (MB)" : rss, 314 } 315 316def getiptables_line(chain, pattern, setup_info): 317 cmd = config.IPTABLES_CMD + ["-vnL", chain] 318 line = getoutput_line(cmd, pattern) 319 if not line: 320 raise Exception("no line found matching %s, make sure you have a rule like: %s" % (pattern, setup_info)) 321 return line 322 323def parse_ipt(chain, pattern, setup_info): 324 if not config.USE_IPTABLES: 325 return 0, 0 326 line = getiptables_line(chain, pattern, setup_info) 327 parts = line.split() 328 assert len(parts)>2 329 def parse_num(part): 330 U = 1024 331 m = {"K":U, "M":U**2, "G":U**3}.get(part[-1], 1) 332 num = "".join([x for x in part if x in "0123456789"]) 333 return int(num)*m/config.MEASURE_TIME 334 return parse_num(parts[0]), parse_num(parts[1]) 335 336def get_iptables_INPUT_count(): 337 setup = "iptables -I INPUT -p tcp --dport %s -j ACCEPT" % config.PORT 338 return parse_ipt("INPUT", "tcp dpt:%s" % config.PORT, setup) 339 340def get_iptables_OUTPUT_count(): 341 setup = "iptables -I OUTPUT -p tcp --sport %s -j ACCEPT" % config.PORT 342 return parse_ipt("OUTPUT", "tcp spt:%s" % config.PORT, setup) 343 344def measure_client(server_pid, name, cmd, get_stats_cb): 345 print("starting client: %s" % cmd) 346 client_process = None 347 try: 348 client_process = Popen(cmd) 349 #give it time to settle down: 350 time.sleep(config.SETTLE_TIME) 351 if cmd[0].startswith("python"): 352 code = client_process.poll() 353 assert code is None, "client failed to start, return code is %s" % code 354 #clear counters 355 initial_stats = get_stats_cb() 356 zero_iptables() 357 old_time_total = update_proc_stat() 358 old_pid_stat = update_pidstat(client_process.pid) 359 if server_pid>0: 360 old_server_pid_stat = update_pidstat(server_pid) 361 #we start measuring 362 t = 0 363 all_stats = [initial_stats] 364 while t<config.MEASURE_TIME: 365 time.sleep(config.COLLECT_STATS_TIME) 366 t += config.COLLECT_STATS_TIME 367 368 if cmd[0].startswith("python"): 369 code = client_process.poll() 370 assert code is None, "client crashed, return code is %s" % code 371 372 stats = get_stats_cb(initial_stats, all_stats) 373 374 #stop the counters 375 new_time_total = update_proc_stat() 376 new_pid_stat = update_pidstat(client_process.pid) 377 if server_pid>0: 378 new_server_pid_stat = update_pidstat(server_pid) 379 ni,isize = get_iptables_INPUT_count() 380 no,osize = get_iptables_OUTPUT_count() 381 #[ni, isize, no, osize] 382 iptables_stat = {"packets in/s" : ni, 383 "packets in: bytes/s" : isize, 384 "packets out/s" : no, 385 "packets out: bytes/s" : osize} 386 #now collect the data 387 client_process_data = compute_stat("client", new_time_total-old_time_total, old_pid_stat, new_pid_stat) 388 if server_pid>0: 389 server_process_data = compute_stat("server", new_time_total-old_time_total, old_server_pid_stat, new_server_pid_stat) 390 else: 391 server_process_data = [] 392 print("process_data (client/server): %s / %s" % (client_process_data, server_process_data)) 393 print("input/output on tcp PORT %s: %s / %s packets, %s / %s KBytes" % (config.PORT, ni, no, isize, osize)) 394 data = {} 395 data.update(iptables_stat) 396 data.update(stats) 397 data.update(client_process_data) 398 data.update(server_process_data) 399 return data 400 finally: 401 #stop the process 402 if client_process and client_process.poll() is None: 403 try_to_stop(client_process) 404 try_to_kill(client_process, 5) 405 code = client_process.poll() 406 assert code is not None, "failed to stop client!" 407 408def with_server(start_server_command, stop_server_commands, in_tests, get_stats_cb): 409 tests = in_tests[config.STARTING_TEST:config.LIMIT_TESTS] 410 print("going to run %s tests:" % len(tests)) 411 for test in tests: 412 print(" * %s" % test[0]) 413 print("*******************************************") 414 print("ETA: %s minutes" % int((config.SERVER_SETTLE_TIME+config.DEFAULT_TEST_COMMAND_SETTLE_TIME+config.SETTLE_TIME+config.MEASURE_TIME+1)*len(tests)/60)) 415 print("*******************************************") 416 417 server_process = None 418 test_command_process = None 419 env = {} 420 for k,v in os.environ.items(): 421 #whitelist what we want to keep: 422 if k.startswith("XPRA") or k in ( 423 "LOGNAME", "XDG_RUNTIME_DIR", "USER", "HOME", "PATH", 424 "LD_LIBRARY_PATH", "XAUTHORITY", "SHELL", "TERM", 425 "USERNAME", "HOSTNAME", "PWD", 426 ): 427 env[k] = v 428 env["DISPLAY"] = ":%s" % config.DISPLAY_NO 429 errors = 0 430 results = [] 431 count = 0 432 for name, tech_name, server_version, client_version, encoding, quality, speed, \ 433 opengl, compression, encryption, ssh, (down,up,latency), test_command, client_cmd in tests: 434 try: 435 print("**************************************************************") 436 count += 1 437 test_command_settle_time = config.TEST_COMMAND_SETTLE_TIME.get(test_command[0], config.DEFAULT_TEST_COMMAND_SETTLE_TIME) 438 eta = int((config.SERVER_SETTLE_TIME+test_command_settle_time+config.SETTLE_TIME+config.MEASURE_TIME+1)*(len(tests)-count)/60) 439 print("%s/%s: %s ETA=%s minutes" % (count, len(tests), name, eta)) 440 test_command_process = None 441 try: 442 clean_sys_state() 443 #start the server: 444 if config.START_SERVER: 445 print("starting server: %s" % str(start_server_command)) 446 server_process = Popen(start_server_command) 447 #give it time to settle down: 448 t = config.SERVER_SETTLE_TIME 449 if count==1: 450 #first run, give it enough time to cleanup the socket 451 t += 5 452 time.sleep(t) 453 server_pid = server_process.pid 454 code = server_process.poll() 455 assert code is None, "server failed to start, return code is %s, please ensure that you can run the server command line above and that a server does not already exist on that port or DISPLAY" % code 456 else: 457 server_pid = 0 458 459 try: 460 #start the test command: 461 if config.USE_VIRTUALGL: 462 if isinstance(test_command, str): 463 cmd = config.VGLRUN_BIN + " -d "+os.environ.get("DISPLAY")+" -- "+ test_command 464 elif isinstance(test_command, (list, tuple)): 465 cmd = [config.VGLRUN_BIN, "-d", os.environ.get("DISPLAY"), "--"] + list(test_command) 466 else: 467 raise Exception("invalid test command type: %s for %s" % (type(test_command), test_command)) 468 else: 469 cmd = test_command 470 471 print("starting test command: %s with env=%s, settle time=%s" % (cmd, env, test_command_settle_time)) 472 shell = isinstance(cmd, str) 473 test_command_process = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, shell=shell) 474 475 if config.PREVENT_SLEEP: 476 Popen(config.PREVENT_SLEEP_COMMAND) 477 478 time.sleep(test_command_settle_time) 479 code = test_command_process.poll() 480 assert code is None, "test command %s failed to start: exit code is %s" % (cmd, code) 481 print("test command %s is running with pid=%s" % (cmd, test_command_process.pid)) 482 483 #run the client test 484 data = {"Test Name" : name, 485 "Remoting Tech" : tech_name, 486 "Server Version" : server_version, 487 "Client Version" : client_version, 488 "Custom Params" : config.CUSTOM_PARAMS, 489 "SVN Version" : SVN_VERSION, 490 "Encoding" : encoding, 491 "Quality" : quality, 492 "Speed" : speed, 493 "OpenGL" : opengl, 494 "Test Command" : get_command_name(test_command), 495 "Sample Duration (s)" : config.MEASURE_TIME, 496 "Sample Time (epoch)" : time.time(), 497 "CPU info" : CPU_INFO, 498 "Platform" : PLATFORM, 499 "Kernel Version" : KERNEL_VERSION, 500 "Xorg version" : XORG_VERSION, 501 "OpenGL" : OPENGL_INFO, 502 "Client Window Manager" : WINDOW_MANAGER, 503 "Screen Size" : "%sx%s" % get_root_size(), 504 "Compression" : compression, 505 "Encryption" : encryption, 506 "Connect via" : ssh, 507 "download limit (KB)" : down, 508 "upload limit (KB)" : up, 509 "latency (ms)" : latency, 510 } 511 data.update(measure_client(server_pid, name, client_cmd, get_stats_cb)) 512 results.append([data.get(x, "") for x in HEADERS]) 513 except Exception as e: 514 import traceback 515 traceback.print_exc() 516 errors += 1 517 print("error during client command run for %s: %s" % (name, e)) 518 if errors>config.MAX_ERRORS: 519 print("too many errors, aborting tests") 520 break 521 finally: 522 if test_command_process: 523 print("stopping '%s' with pid=%s" % (test_command, test_command_process.pid)) 524 try_to_stop(test_command_process) 525 try_to_kill(test_command_process, 2) 526 if config.START_SERVER: 527 try_to_stop(server_process) 528 time.sleep(2) 529 for s in stop_server_commands: 530 print("stopping server with: %s" % (s)) 531 try: 532 stop_process = Popen(s, stdout=PIPE, stderr=PIPE, shell=True) 533 stop_process.wait() 534 except Exception as e: 535 print("error: %s" % e) 536 try_to_kill(server_process, 5) 537 time.sleep(1) 538 except KeyboardInterrupt as e: 539 print("caught %s: stopping this series of tests" % e) 540 break 541 return results 542 543def trickle_command(down, up, latency): 544 if down<=0 and up<=0 and latency<=0: 545 return [] 546 cmd = [config.TRICKLE_BIN, "-s"] 547 if down>0: 548 cmd += ["-d", str(down)] 549 if up>0: 550 cmd += ["-u", str(up)] 551 if latency>0: 552 cmd += ["-L", str(latency)] 553 return cmd 554 555def trickle_str(down, up, latency): 556 if down<=0 and up<=0 and latency<=0: 557 return "unthrottled" 558 s = "/".join(str(x) for x in [down,up,latency]) 559 return "throttled:%s" % s 560 561def get_command_name(command_arg): 562 try: 563 name = config.TEST_NAMES.get(command_arg) 564 if name: 565 return name 566 except: 567 pass 568 if isinstance(command_arg, list): 569 c = command_arg[0] #["/usr/bin/xterm", "blah"] -> "/usr/bin/xterm" 570 else: 571 c = command_arg.split(" ")[0] #"/usr/bin/xterm -e blah" -> "/usr/bin/xterm" 572 assert isinstance(c, str) 573 return c.split("/")[-1] #/usr/bin/xterm -> xterm 574 575def get_auth_args(server=True): 576 cmd = [] 577 if config.XPRA_USE_PASSWORD: 578 if server: 579 cmd.append("--auth=none") 580 cmd.append("--tcp-auth=file,filename=%s" % password_filename) 581 else: 582 cmd.append("--password-file=%s" % password_filename) 583 return cmd 584 585def xpra_get_stats(initial_stats=None, all_stats=[]): 586 info_cmd = XPRA_INFO_COMMAND[:] + get_auth_args(False) 587 out = getoutput(info_cmd) 588 if not out: 589 return {} 590 #parse output: 591 d = {} 592 for line in out.splitlines(): 593 parts = line.split("=") 594 if len(parts)==2: 595 d[parts[0]] = parts[1] 596 #functions for accessing the data: 597 def iget(names, default_value=""): 598 """ some of the fields got renamed, try both old and new names """ 599 for n in names: 600 v = d.get(n) 601 if v is not None: 602 return int(v) 603 return default_value 604 #values always based on initial data only: 605 #(difference from initial value) 606 lookup = initial_stats or {} 607 initial_input_packetcount = lookup.get("Application packets in/s", 0) 608 initial_input_bytecount = lookup.get("Application bytes in/s", 0) 609 initial_output_packetcount = lookup.get("Application packets out/s", 0) 610 initial_output_bytecount = lookup.get("Application bytes out/s", 0) 611 initial_mmap_bytes = lookup.get("mmap bytes/s", 0) 612 data = { 613 "Application packets in/s" : (iget(["client.connection.input.packetcount", "input_packetcount"], 0)-initial_input_packetcount)/config.MEASURE_TIME, 614 "Application bytes in/s" : (iget(["client.connection.input.bytecount", "input_bytecount"], 0)-initial_input_bytecount)/config.MEASURE_TIME, 615 "Application packets out/s" : (iget(["client.connection.output.packetcount", "output_packetcount"], 0)-initial_output_packetcount)/config.MEASURE_TIME, 616 "Application bytes out/s" : (iget(["client.connection.output.bytecount", "output_bytecount"], 0)-initial_output_bytecount)/config.MEASURE_TIME, 617 "mmap bytes/s" : (iget(["client.connection.output.mmap_bytecount", "output_mmap_bytecount"], 0)-initial_mmap_bytes)/config.MEASURE_TIME, 618 } 619 620 #values that are averages or min/max: 621 def add(prefix, op, name, prop_names): 622 values = [] 623 #cook the property names using the lowercase prefix if needed 624 #(all xpra info properties are lowercase): 625 actual_prop_names = [] 626 full_search = [] 627 for prop_name in prop_names: 628 if prop_name.find("%s")>=0: 629 prop_name = prop_name % prefix.lower() 630 actual_prop_names.append(prop_name) 631 if prop_name.find("*")>=0 or prop_name.find("+")>=0: #ie: "window\[\d+\].encoding.quality.avg" 632 #make it a proper python regex: 633 full_search.append(prop_name) 634 if full_search: 635 for s in full_search: 636 regex = re.compile(s) 637 matches = [d.get(x) for x in d if regex.match(x)] 638 for v in matches: 639 values.append(int(v)) 640 #print("add(%s, %s, %s, %s) values from full_search=%s: %s" % 641 # (prefix, op, name, prop_names, full_search, values)) 642 else: 643 #match just one record: 644 values.append(iget(actual_prop_names)) 645 #print("add(%s, %s, %s, %s) values from iget: %s" % (prefix, op, name, prop_names, values)) 646 #this is the stat property name: 647 full_name = name #ie: "Application packets in/s" 648 if prefix: 649 full_name = prefix+" "+name #ie: "Min" + " " + "Batch Delay" 650 for s in all_stats: #add all previously found values to list 651 values.append(s.get(full_name)) 652 #strip missing values: 653 values = [x for x in values if x is not None and x!=""] 654 if values: 655 v = op(values) #ie: avg([4,5,4]) or max([4,5,4]) 656 #print("%s: %s(%s)=%s" % (full_name, op, values, v)) 657 data[full_name] = v 658 659 def avg(l): 660 return sum(l)/len(l) 661 662 add("", avg, "Regions/s", ["client.encoding.regions_per_second", "encoding.regions_per_second", "regions_per_second"]) 663 add("", avg, "Pixels/s Sent", ["client.encoding.pixels_per_second", "encoding.pixels_per_second", "pixels_per_second"]) 664 add("", avg, "Encoding Pixels/s", ["client.encoding.pixels_encoded_per_second", "encoding.pixels_encoded_per_second", "pixels_encoded_per_second"]) 665 add("", avg, "Decoding Pixels/s", ["client.encoding.pixels_decoded_per_second", "encoding.pixels_decoded_per_second", "pixels_decoded_per_second"]) 666 667 add("", avg, "Frame Total Latency", ["client.damage.frame-total-latency"]) 668 add("", avg, "Client Frame Latency", ["client.damage.client-latency"]) 669 670 for prefix, op in (("Min", min), ("Max", max), ("Avg", avg)): 671 add(prefix, op, "Batch Delay (ms)", ["client.batch.delay.%s", "batch.delay.%s", "batch_delay.%s", "%s_batch_delay"]) 672 add(prefix, op, "Actual Batch Delay (ms)", ["client.window.?[\\d+].batch.actual_delays.%s", "client.batch.actual_delay.%s", "batch.actual_delay.%s"]) 673 add(prefix, op, "Client Latency (ms)", ["client.connection.client.latency.%s", "client.latency.%s", "client_latency.%s", "%s_client_latency"]) 674 add(prefix, op, "Client Ping Latency (ms)", ["client.connection.client.ping_latency.%s", "client.ping_latency.%s", "client_ping_latency.%s"]) 675 add(prefix, op, "Server Ping Latency (ms)", ["client.connection.server.ping_latency.%s", "server.ping_latency.%s", "server_ping_latency.%s", "server_latency.%s", "%s_server_latency"]) 676 add(prefix, op, "Damage Latency (ms)", ["client.damage.in_latency.%s", "damage.in_latency.%s", "damage_in_latency.%s"]) 677 678 add(prefix, op, "Quality", [r"^client.window.?[\d+].encoding.quality.%s$", r"^window[\d+].encoding.quality.%s$"]) 679 add(prefix, op, "Speed", [r"^client.window.?[\d+].encoding.speed.%s$", r"^window[\d+].encoding.speed.%s$"]) 680 681 def addset(name, prop_name): 682 regex = re.compile(prop_name) 683 def getdictvalues(from_dict): 684 return [from_dict.get(x) for x in from_dict.keys() if regex.match(x)] 685 values = getdictvalues(d) 686 for s in all_stats: #add all previously found values to list 687 values += getdictvalues(s) 688 data[name] = list(set(values)) 689 690 #video encoder 691 addset("Video Encoder", r"^window\[\d+\].encoder$") 692 #record CSC: 693 addset("CSC", r"^window\[\d+\].csc$") 694 addset("CSC Mode", r"^window\[\d+\].csc.dst_format$") 695 addset("Scaling", r"^window\[\d+\].scaling$") 696 #packet layer: 697 addset("Compressors", "connection.compression$") 698 addset("Packet Encoders", "connection.encoder$") 699 #add this record to the list: 700 all_stats.append(data) 701 return data 702 703def get_xpra_start_server_command(): 704 cmd = [XPRA_BIN, "--no-daemon", "--bind-tcp=0.0.0.0:%s" % config.PORT] 705 if config.XPRA_FORCE_XDUMMY: 706 cmd.append("--xvfb=%s -nolisten tcp" % config.XORG_BIN 707 +" +extension GLX" 708 +" +extension RANDR" 709 +" +extension RENDER" 710 +" -logfile %s" % config.XORG_LOG 711 +" -config %s" % config.XORG_CONFIG) 712 cmd.append("--no-notifications") 713 cmd += get_auth_args(True) 714 cmd.append("--no-pulseaudio") 715 cmd += ["start", ":%s" % config.DISPLAY_NO] 716 return cmd 717 718 719def test_xpra(): 720 print("") 721 print("*********************************************************") 722 print(" Xpra tests") 723 print("") 724 tests = [] 725 for connect_option, encryption in config.XPRA_CONNECT_OPTIONS: 726 shaping_options = config.TRICKLE_SHAPING_OPTIONS 727 if connect_option=="unix-domain": 728 shaping_options = [config.NO_SHAPING] 729 for down,up,latency in shaping_options: 730 for x11_test_command in config.X11_TEST_COMMANDS: 731 command_name = get_command_name(x11_test_command) 732 for client_type in config.XPRA_CLIENT_TYPES: 733 if client_type=="html5": 734 assert not config.XPRA_USE_PASSWORD 735 if connect_option=="unix-domain": 736 continue 737 for browser_cmd in config.XPRA_HTML5_BROWSERS: 738 cmd = browser_cmd + ["http://localhost:%i/" % (config.PORT)] 739 test_name = "%s : %s" % (browser_cmd[0], command_name) 740 tests.append( 741 ( 742 test_name, "xpra", XPRA_VERSION, XPRA_VERSION, 743 "all", 0, 0, 744 "n/a", "n/a", encryption, connect_option, 745 (down,up,latency), x11_test_command, cmd, 746 ) 747 ) 748 continue 749 750 #python client has more options: 751 encodings = config.XPRA_TEST_ENCODINGS 752 for encoding in encodings: 753 for opengl in config.XPRA_OPENGL_OPTIONS.get(encoding, [True]): 754 quality_options = config.XPRA_ENCODING_QUALITY_OPTIONS.get(encoding, [-1]) 755 for quality in quality_options: 756 speed_options = config.XPRA_ENCODING_SPEED_OPTIONS.get(encoding, [-1]) 757 for speed in speed_options: 758 for speaker in XPRA_SPEAKER_OPTIONS: 759 for mic in XPRA_MICROPHONE_OPTIONS: 760 for comp in config.XPRA_COMPRESSORS_OPTIONS: 761 for compression in config.XPRA_COMPRESSION_LEVEL_OPTIONS: 762 for packet_encoders in config.XPRA_PACKET_ENCODERS_OPTIONS: 763 test_name = "(%s - %s - %s - %s - via %s)" % \ 764 (command_name, compression, encryption, 765 trickle_str(down, up, latency), connect_option) 766 cmd = trickle_command(down, up, latency) 767 cmd += [client_type, XPRA_BIN, "attach"] 768 if connect_option=="ssh": 769 cmd.append("ssh:%s:%s" % (config.IP, config.DISPLAY_NO)) 770 elif connect_option=="tcp": 771 cmd.append("tcp:%s:%s" % (config.IP, config.PORT)) 772 else: 773 cmd.append(":%s" % (config.DISPLAY_NO)) 774 cmd.append("--readonly=yes") 775 cmd += get_auth_args(False) 776 if packet_encoders: 777 cmd += ["--packet-encoders=%s" % packet_encoders] 778 if comp: 779 cmd += ["--compressors=%s" % comp] 780 if compression is not None: 781 cmd += ["-z", str(compression)] 782 cmd.append("--enable-pings") 783 cmd.append("--no-clipboard") 784 cmd.append("--no-bell") 785 cmd.append("--no-cursors") 786 cmd.append("--no-notifications") 787 if config.XPRA_MDNS: 788 cmd.append("--mdns") 789 else: 790 cmd.append("--no-mdns") 791 if encryption: 792 cmd.append("--encryption=%s" % encryption) 793 if speed>=0: 794 cmd.append("--speed=%s" % speed) 795 if quality>=0: 796 cmd.append("--quality=%s" % quality) 797 name = "%s-%s" % (encoding, quality) 798 else: 799 name = encoding 800 if speaker is None: 801 cmd.append("--no-speaker") 802 else: 803 cmd.append("--speaker-codec=%s" % speaker) 804 if mic is None: 805 cmd.append("--no-microphone") 806 else: 807 cmd.append("--microphone-codec=%s" % mic) 808 if encoding!="mmap": 809 cmd.append("--no-mmap") 810 cmd.append("--encoding=%s" % encoding) 811 cmd.append("--opengl=%s" % opengl) 812 813 full_test_name = "%6s %s" % (name, test_name) 814 tests.append( 815 ( 816 full_test_name, "xpra", XPRA_VERSION, XPRA_VERSION, 817 encoding, quality, speed, 818 opengl, compression, encryption, connect_option, 819 (down,up,latency), x11_test_command, cmd, 820 ) 821 ) 822 return with_server(get_xpra_start_server_command(), XPRA_SERVER_STOP_COMMANDS, tests, xpra_get_stats) 823 824def get_x11_client_window_info(display, *app_name_strings): 825 env = os.environ.copy() 826 if display: 827 env["DISPLAY"] = display 828 wininfo = getoutput(["xwininfo", "-root", "-tree"], env) 829 for line in wininfo.splitlines(): 830 if not line: 831 continue 832 found = True 833 for x in app_name_strings: 834 if not line.find(x)>=0: 835 found = False 836 break 837 if not found: 838 continue 839 parts = line.split() 840 if not parts[0].startswith("0x"): 841 continue 842 #found a window which matches the name we are looking for! 843 wid = parts[0] 844 x, y, w, h = 0, 0, 0, 0 845 dims = parts[-2] #ie: 400x300+20+10 846 dp = dims.split("+") #["400x300", "20", "10"] 847 if len(dp)==3: 848 d = dp[0] #"400x300" 849 x = int(dp[1]) #20 850 y = int(dp[2]) #10 851 wh = d.split("x") #["400", "300"] 852 if len(wh)==2: 853 w = int(wh[0]) #400 854 h = int(wh[1]) #300 855 print("Found window for '%s': %s - %sx%s" % (app_name_strings, wid, w, h)) 856 return wid, x, y, w, h 857 return None 858 859def get_vnc_stats(initial_stats=None, all_stats=[]): 860 #print("get_vnc_stats(%s)" % last_record) 861 if initial_stats is None: 862 #this is the initial call, 863 #start the thread to watch the output of tcbench 864 #we first need to figure out the dimensions of the client window 865 #within the Xvnc server, the use those dimensions to tell tcbench 866 #where to look in the vncviewer client window 867 test_window_info = get_x11_client_window_info(":%s" % config.DISPLAY_NO) 868 print("info for client test window: %s" % str(test_window_info)) 869 info = get_x11_client_window_info(None, "TigerVNC: x11", "Vncviewer") 870 if not info: 871 return {} 872 print("info for TigerVNC: %s" % str(info)) 873 wid, _, _, w, h = info 874 if not wid: 875 return {} 876 if test_window_info: 877 _, _, _, w, h = test_window_info 878 command = [config.TCBENCH, "-wh%s" % wid, "-t%s" % (config.MEASURE_TIME-5)] 879 if w>0 and h>0: 880 command.append("-x%s" % int(w/2)) 881 command.append("-y%s" % int(h/2)) 882 if os.path.exists(config.TCBENCH_LOG): 883 os.unlink(config.TCBENCH_LOG) 884 tcbench_log = open(config.TCBENCH_LOG, 'w') 885 try: 886 print("tcbench starting: %s, logging to %s" % (command, config.TCBENCH_LOG)) 887 proc = Popen(command, stdout=tcbench_log, stderr=tcbench_log) 888 return {"tcbench" : proc} 889 except Exception as e: 890 import traceback 891 traceback.print_exc() 892 print("error running %s: %s" % (command, e)) 893 return {} #we failed... 894 regions_s = "" 895 if "tcbench" in initial_stats: 896 #found the process watcher, 897 #parse the tcbench output and look for frames/sec: 898 process = initial_stats.get("tcbench") 899 assert isinstance(process, Popen) 900 #print("get_vnc_stats(%s) process.poll()=%s" % (last_record, process.poll())) 901 if process.poll() is None: 902 try_to_stop(process) 903 try_to_kill(process, 2) 904 else: 905 with open(config.TCBENCH_LOG, mode='rb') as f: 906 out = f.read() 907 #print("get_vnc_stats(%s) tcbench output=%s" % (last_record, out)) 908 for line in out.splitlines(): 909 if not line.find("Frames/sec:")>=0: 910 continue 911 parts = line.split() 912 regions_s = parts[-1] 913 print("Frames/sec=%s" % regions_s) 914 return { 915 "Regions/s" : regions_s, 916 } 917 918def test_vnc(): 919 print("") 920 print("*********************************************************") 921 print(" VNC tests") 922 print("") 923 tests = [] 924 for down,up,latency in config.TRICKLE_SHAPING_OPTIONS: 925 for x11_test_command in config.X11_TEST_COMMANDS: 926 for encoding in config.VNC_ENCODINGS: 927 for zlib in config.VNC_ZLIB_OPTIONS: 928 for compression in config.VNC_COMPRESSION_OPTIONS: 929 jpeg_quality = [8] 930 if encoding=="Tight": 931 jpeg_quality = config.VNC_JPEG_OPTIONS 932 for jpegq in jpeg_quality: 933 cmd = trickle_command(down, up, latency) 934 cmd += [config.VNCVIEWER_BIN, "%s::%s" % (config.IP, config.PORT), 935 "--ViewOnly", 936 "--ZlibLevel=%s" % str(zlib), 937 "--CompressLevel=%s" % str(compression), 938 ] 939 if encoding=="auto": 940 cmd.append("--AutoSelect=1") 941 else: 942 cmd.append("--AutoSelect=0") 943 cmd.append("--PreferredEncoding=%s" % encoding) 944 if jpegq<0: 945 cmd.append("--NoJPEG=1") 946 jpegtxt = "nojpeg" 947 else: 948 cmd.append("--NoJPEG=0") 949 cmd.append("--QualityLevel=%s" % jpegq) 950 jpegtxt = "jpeg=%s" % jpegq 951 #make a descriptive title: 952 if zlib==-1: 953 zlibtxt = "nozlib" 954 else: 955 zlibtxt = "zlib=%s" % zlib 956 command_name = get_command_name(x11_test_command) 957 test_name = "vnc (%s - %s - %s - compression=%s - %s - %s)" % \ 958 (command_name, encoding, zlibtxt, 959 compression, jpegtxt, trickle_str(down, up, latency)) 960 tests.append((test_name, "vnc", XVNC_VERSION, VNCVIEWER_VERSION, \ 961 encoding, False, compression, None, False, \ 962 (down,up,latency), x11_test_command, cmd)) 963 return with_server(config.XVNC_SERVER_START_COMMAND, config.XVNC_SERVER_STOP_COMMANDS, tests, get_vnc_stats) 964 965def main(): 966 #before doing anything, check that the firewall is setup correctly: 967 get_iptables_INPUT_count() 968 get_iptables_OUTPUT_count() 969 970 #If CUSTOM_PARAMS are supplied on the command line, they override what's in config 971 if len(sys.argv) > 3: 972 config.CUSTOM_PARAMS = " ".join(sys.argv[3:]) 973 config.print_options() 974 975 xpra_results = [] 976 if config.TEST_XPRA: 977 xpra_results = test_xpra() 978 vnc_results = [] 979 if config.TEST_VNC: 980 vnc_results = test_vnc() 981 982 if len(sys.argv) > 2: 983 csv_name = sys.argv[2] 984 else: 985 csv_name = None 986 987 print("*"*80) 988 print("RESULTS:") 989 print("") 990 991 out_lines = [] 992 out_line = ", ".join(HEADERS) 993 print(out_line) 994 out_lines.append(out_line) 995 996 def s(x): 997 if x is None: 998 return "" 999 if isinstance(x, (list, tuple, set)): 1000 return '"' + (", ".join(list(x))) + '"' 1001 if isinstance(x, str): 1002 if not x: 1003 return "" 1004 return '"%s"' % x 1005 if isinstance(x, (float, int)): 1006 return str(x) 1007 return "unhandled-type: %s" % type(x) 1008 1009 for result in xpra_results+vnc_results: 1010 out_line = ", ".join([s(x) for x in result]) 1011 print(out_line) 1012 out_lines.append(out_line) 1013 1014 if csv_name: 1015 with open(csv_name, "w") as csv: 1016 for line in out_lines: 1017 csv.write(line+"\n") 1018 1019if __name__ == "__main__": 1020 main() 1021