1""" 2@package core.gcmd 3 4@brief wxGUI command interface 5 6Classes: 7 - gcmd::GError 8 - gcmd::GWarning 9 - gcmd::GMessage 10 - gcmd::GException 11 - gcmd::Popen (from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/440554) 12 - gcmd::Command 13 - gcmd::CommandThread 14 15Functions: 16 - RunCommand 17 - GetDefaultEncoding 18 19(C) 2007-2008, 2010-2011 by the GRASS Development Team 20 21This program is free software under the GNU General Public License 22(>=v2). Read the file COPYING that comes with GRASS for details. 23 24@author Jachym Cepicky 25@author Martin Landa <landa.martin gmail.com> 26""" 27 28from __future__ import print_function 29 30import os 31import sys 32import time 33import errno 34import signal 35import traceback 36import locale 37import subprocess 38from threading import Thread 39import wx 40 41is_mswindows = sys.platform == 'win32' 42if is_mswindows: 43 from win32file import ReadFile, WriteFile 44 from win32pipe import PeekNamedPipe 45 import msvcrt 46else: 47 import select 48 import fcntl 49 50from core.debug import Debug 51from core.globalvar import SCT_EXT 52 53from grass.script import core as grass 54from grass.script.utils import decode, encode 55 56if sys.version_info.major == 2: 57 bytes = str 58 59 60def DecodeString(string): 61 """Decode string using system encoding 62 63 :param string: string to be decoded 64 65 :return: decoded string 66 """ 67 if not string: 68 return string 69 70 if _enc and isinstance(string, bytes): 71 Debug.msg(5, "DecodeString(): enc=%s" % _enc) 72 return string.decode(_enc) 73 return string 74 75 76def EncodeString(string): 77 """Return encoded string using system locales 78 79 :param string: string to be encoded 80 81 :return: encoded string 82 """ 83 if not string: 84 return string 85 if _enc: 86 Debug.msg(5, "EncodeString(): enc=%s" % _enc) 87 return string.encode(_enc) 88 return string 89 90 91class GError: 92 93 def __init__(self, message, parent=None, caption=None, showTraceback=True): 94 """Show error message window 95 96 :param message: error message 97 :param parent: centre window on parent if given 98 :param caption: window caption (if not given "Error") 99 :param showTraceback: True to show also Python traceback 100 """ 101 if not caption: 102 caption = _('Error') 103 style = wx.OK | wx.ICON_ERROR | wx.CENTRE 104 exc_type, exc_value, exc_traceback = sys.exc_info() 105 if exc_traceback: 106 exception = traceback.format_exc() 107 reason = exception.splitlines()[-1].split(':', 1)[-1].strip() 108 109 if Debug.GetLevel() > 0 and exc_traceback: 110 sys.stderr.write(exception) 111 112 if showTraceback and exc_traceback: 113 wx.MessageBox(parent=parent, 114 message=message + '\n\n%s: %s\n\n%s' % 115 (_('Reason'), 116 reason, exception), 117 caption=caption, 118 style=style) 119 else: 120 wx.MessageBox(parent=parent, 121 message=message, 122 caption=caption, 123 style=style) 124 125 126class GWarning: 127 128 def __init__(self, message, parent=None): 129 caption = _('Warning') 130 style = wx.OK | wx.ICON_WARNING | wx.CENTRE 131 wx.MessageBox(parent=parent, 132 message=message, 133 caption=caption, 134 style=style) 135 136 137class GMessage: 138 139 def __init__(self, message, parent=None): 140 caption = _('Message') 141 style = wx.OK | wx.ICON_INFORMATION | wx.CENTRE 142 wx.MessageBox(parent=parent, 143 message=message, 144 caption=caption, 145 style=style) 146 147 148class GException(Exception): 149 150 def __init__(self, value=''): 151 self.value = value 152 153 def __str__(self): 154 return self.value 155 156 def __unicode__(self): 157 return self.value 158 159 160class Popen(subprocess.Popen): 161 """Subclass subprocess.Popen""" 162 163 def __init__(self, args, **kwargs): 164 if is_mswindows: 165 # The Windows shell (cmd.exe) requires some special characters to 166 # be escaped by preceding them with 3 carets (^^^). cmd.exe /? 167 # mentions <space> and &()[]{}^=;!'+,`~. A quick test revealed that 168 # only ^|&<> need to be escaped. A single quote can be escaped by 169 # enclosing it with double quotes and vice versa. 170 for i in range(2, len(args)): 171 # "^" must be the first character in the list to avoid double 172 # escaping. 173 for c in ("^", "|", "&", "<", ">"): 174 if c in args[i]: 175 if "=" in args[i]: 176 a = args[i].split("=") 177 k = a[0] + "=" 178 v = "=".join(a[1:len(a)]) 179 else: 180 k = "" 181 v = args[i] 182 183 # If there are spaces, the argument was already 184 # esscaped with double quotes, so don't escape it 185 # again. 186 if c in v and not " " in v: 187 # Here, we escape each ^ in ^^^ with ^^ and a 188 # <special character> with ^ + <special character>, 189 # so we need 7 carets. 190 191 v = v.replace(c, "^^^^^^^" + c) 192 args[i] = k + v 193 194 subprocess.Popen.__init__(self, args, **kwargs) 195 196 def recv(self, maxsize=None): 197 return self._recv('stdout', maxsize) 198 199 def recv_err(self, maxsize=None): 200 return self._recv('stderr', maxsize) 201 202 def send_recv(self, input='', maxsize=None): 203 return self.send(input), self.recv(maxsize), self.recv_err(maxsize) 204 205 def get_conn_maxsize(self, which, maxsize): 206 if maxsize is None: 207 maxsize = 1024 208 elif maxsize < 1: 209 maxsize = 1 210 return getattr(self, which), maxsize 211 212 def _close(self, which): 213 getattr(self, which).close() 214 setattr(self, which, None) 215 216 def kill(self): 217 """Try to kill running process""" 218 if is_mswindows: 219 import win32api 220 handle = win32api.OpenProcess(1, 0, self.pid) 221 return (0 != win32api.TerminateProcess(handle, 0)) 222 else: 223 try: 224 os.kill(-self.pid, signal.SIGTERM) # kill whole group 225 except OSError: 226 pass 227 228 if sys.platform == 'win32': 229 def send(self, input): 230 if not self.stdin: 231 return None 232 233 import pywintypes 234 try: 235 x = msvcrt.get_osfhandle(self.stdin.fileno()) 236 (errCode, written) = WriteFile(x, input) 237 except ValueError: 238 return self._close('stdin') 239 except (pywintypes.error, Exception) as why: 240 if why.winerror in (109, errno.ESHUTDOWN): 241 return self._close('stdin') 242 raise 243 244 return written 245 246 def _recv(self, which, maxsize): 247 conn, maxsize = self.get_conn_maxsize(which, maxsize) 248 if conn is None: 249 return None 250 251 import pywintypes 252 try: 253 x = msvcrt.get_osfhandle(conn.fileno()) 254 (read, nAvail, nMessage) = PeekNamedPipe(x, 0) 255 if maxsize < nAvail: 256 nAvail = maxsize 257 if nAvail > 0: 258 (errCode, read) = ReadFile(x, nAvail, None) 259 except ValueError: 260 return self._close(which) 261 except (pywintypes.error, Exception) as why: 262 if why.winerror in (109, errno.ESHUTDOWN): 263 return self._close(which) 264 raise 265 266 if self.universal_newlines: 267 read = self._translate_newlines(read) 268 return read 269 270 else: 271 def send(self, input): 272 if not self.stdin: 273 return None 274 275 if not select.select([], [self.stdin], [], 0)[1]: 276 return 0 277 278 try: 279 written = os.write(self.stdin.fileno(), input) 280 except OSError as why: 281 if why[0] == errno.EPIPE: # broken pipe 282 return self._close('stdin') 283 raise 284 285 return written 286 287 def _recv(self, which, maxsize): 288 conn, maxsize = self.get_conn_maxsize(which, maxsize) 289 if conn is None: 290 return None 291 292 flags = fcntl.fcntl(conn, fcntl.F_GETFL) 293 if not conn.closed: 294 fcntl.fcntl(conn, fcntl.F_SETFL, flags | os.O_NONBLOCK) 295 296 try: 297 if not select.select([conn], [], [], 0)[0]: 298 return '' 299 300 r = conn.read() 301 302 if not r: 303 return self._close(which) 304 305 if self.universal_newlines: 306 r = self._translate_newlines(r) 307 return r 308 finally: 309 if not conn.closed: 310 fcntl.fcntl(conn, fcntl.F_SETFL, flags) 311 312message = "Other end disconnected!" 313 314 315def recv_some(p, t=.1, e=1, tr=5, stderr=0): 316 if tr < 1: 317 tr = 1 318 x = time.time() + t 319 y = [] 320 r = '' 321 pr = p.recv 322 if stderr: 323 pr = p.recv_err 324 while time.time() < x or r: 325 r = pr() 326 if r is None: 327 if e: 328 raise Exception(message) 329 else: 330 break 331 elif r: 332 y.append(decode(r)) 333 else: 334 time.sleep(max((x - time.time()) / tr, 0)) 335 return ''.join(y) 336 337 338def send_all(p, data): 339 while len(data): 340 sent = p.send(data) 341 if sent is None: 342 raise Exception(message) 343 data = buffer(data, sent) 344 345 346class Command: 347 """Run command in separate thread. Used for commands launched 348 on the background. 349 350 If stdout/err is redirected, write() method is required for the 351 given classes. 352 353 cmd = Command(cmd=['d.rast', 'elevation.dem'], verbose=3, wait=True) 354 355 if cmd.returncode == None: 356 print 'RUNNING?' 357 elif cmd.returncode == 0: 358 print 'SUCCESS' 359 else: 360 print 'FAILURE (%d)' % cmd.returncode 361 """ 362 363 def __init__(self, cmd, stdin=None, 364 verbose=None, wait=True, rerr=False, 365 stdout=None, stderr=None): 366 """ 367 :param cmd: command given as list 368 :param stdin: standard input stream 369 :param verbose: verbose level [0, 3] (--q, --v) 370 :param wait: wait for child execution terminated 371 :param rerr: error handling (when GException raised). 372 True for redirection to stderr, False for GUI 373 dialog, None for no operation (quiet mode) 374 :param stdout: redirect standard output or None 375 :param stderr: redirect standard error output or None 376 """ 377 Debug.msg(1, "gcmd.Command(): %s" % ' '.join(cmd)) 378 self.cmd = cmd 379 self.stderr = stderr 380 381 # 382 # set verbosity level 383 # 384 verbose_orig = None 385 if ('--q' not in self.cmd and '--quiet' not in self.cmd) and \ 386 ('--v' not in self.cmd and '--verbose' not in self.cmd): 387 if verbose is not None: 388 if verbose == 0: 389 self.cmd.append('--quiet') 390 elif verbose == 3: 391 self.cmd.append('--verbose') 392 else: 393 verbose_orig = os.getenv("GRASS_VERBOSE") 394 os.environ["GRASS_VERBOSE"] = str(verbose) 395 396 # 397 # create command thread 398 # 399 self.cmdThread = CommandThread(cmd, stdin, 400 stdout, stderr) 401 self.cmdThread.start() 402 403 if wait: 404 self.cmdThread.join() 405 if self.cmdThread.module: 406 self.cmdThread.module.wait() 407 self.returncode = self.cmdThread.module.returncode 408 else: 409 self.returncode = 1 410 else: 411 self.cmdThread.join(0.5) 412 self.returncode = None 413 414 if self.returncode is not None: 415 Debug.msg( 416 3, "Command(): cmd='%s', wait=%s, returncode=%d, alive=%s" % 417 (' '.join(cmd), wait, self.returncode, self.cmdThread.isAlive())) 418 if rerr is not None and self.returncode != 0: 419 if rerr is False: # GUI dialog 420 raise GException("%s '%s'%s%s%s %s%s" % 421 (_("Execution failed:"), 422 ' '.join(self.cmd), 423 os.linesep, os.linesep, 424 _("Details:"), 425 os.linesep, 426 _("Error: ") + self.__GetError())) 427 elif rerr == sys.stderr: # redirect message to sys 428 stderr.write("Execution failed: '%s'" % 429 (' '.join(self.cmd))) 430 stderr.write( 431 "%sDetails:%s%s" % 432 (os.linesep, 433 _("Error: ") + 434 self.__GetError(), 435 os.linesep)) 436 else: 437 pass # nop 438 else: 439 Debug.msg( 440 3, "Command(): cmd='%s', wait=%s, returncode=?, alive=%s" % 441 (' '.join(cmd), wait, self.cmdThread.isAlive())) 442 443 if verbose_orig: 444 os.environ["GRASS_VERBOSE"] = verbose_orig 445 elif "GRASS_VERBOSE" in os.environ: 446 del os.environ["GRASS_VERBOSE"] 447 448 def __ReadOutput(self, stream): 449 """Read stream and return list of lines 450 451 :param stream: stream to be read 452 """ 453 lineList = [] 454 455 if stream is None: 456 return lineList 457 458 while True: 459 line = stream.readline() 460 if not line: 461 break 462 line = line.replace('%s' % os.linesep, '').strip() 463 lineList.append(line) 464 465 return lineList 466 467 def __ReadErrOutput(self): 468 """Read standard error output and return list of lines""" 469 return self.__ReadOutput(self.cmdThread.module.stderr) 470 471 def __ProcessStdErr(self): 472 """ 473 Read messages/warnings/errors from stderr 474 475 :return: list of (type, message) 476 """ 477 if self.stderr is None: 478 lines = self.__ReadErrOutput() 479 else: 480 lines = self.cmdThread.error.strip('%s' % os.linesep). \ 481 split('%s' % os.linesep) 482 483 msg = [] 484 485 type = None 486 content = "" 487 for line in lines: 488 if len(line) == 0: 489 continue 490 if 'GRASS_' in line: # error or warning 491 if 'GRASS_INFO_WARNING' in line: # warning 492 type = "WARNING" 493 elif 'GRASS_INFO_ERROR' in line: # error 494 type = "ERROR" 495 elif 'GRASS_INFO_END': # end of message 496 msg.append((type, content)) 497 type = None 498 content = "" 499 500 if type: 501 content += line.split(':', 1)[1].strip() 502 else: # stderr 503 msg.append((None, line.strip())) 504 505 return msg 506 507 def __GetError(self): 508 """Get error message or ''""" 509 if not self.cmdThread.module: 510 return _("Unable to exectute command: '%s'") % ' '.join(self.cmd) 511 512 for type, msg in self.__ProcessStdErr(): 513 if type == 'ERROR': 514 if _enc: 515 return unicode(msg, _enc) 516 return msg 517 518 return '' 519 520 521class CommandThread(Thread): 522 """Create separate thread for command. Used for commands launched 523 on the background.""" 524 525 def __init__(self, cmd, env=None, stdin=None, 526 stdout=sys.stdout, stderr=sys.stderr): 527 """ 528 :param cmd: command (given as list) 529 :param env: environmental variables 530 :param stdin: standard input stream 531 :param stdout: redirect standard output or None 532 :param stderr: redirect standard error output or None 533 """ 534 Thread.__init__(self) 535 536 self.cmd = cmd 537 self.stdin = stdin 538 self.stdout = stdout 539 self.stderr = stderr 540 self.env = env 541 542 self.module = None 543 self.error = '' 544 545 self._want_abort = False 546 self.aborted = False 547 548 self.setDaemon(True) 549 550 # set message formatting 551 self.message_format = os.getenv("GRASS_MESSAGE_FORMAT") 552 os.environ["GRASS_MESSAGE_FORMAT"] = "gui" 553 554 def __del__(self): 555 if self.message_format: 556 os.environ["GRASS_MESSAGE_FORMAT"] = self.message_format 557 else: 558 del os.environ["GRASS_MESSAGE_FORMAT"] 559 560 def run(self): 561 """Run command""" 562 if len(self.cmd) == 0: 563 return 564 565 Debug.msg(1, "gcmd.CommandThread(): %s" % ' '.join(self.cmd)) 566 567 self.startTime = time.time() 568 569 # TODO: replace ugly hack below 570 # this cannot be replaced it can be only improved 571 # also unifying this with 3 other places in code would be nice 572 # changing from one chdir to get_real_command function 573 args = self.cmd 574 if sys.platform == 'win32': 575 if os.path.splitext(args[0])[1] == SCT_EXT: 576 args[0] = args[0][:-3] 577 # using Python executable to run the module if it is a script 578 # expecting at least module name at first position 579 # cannot use make_command for this now because it is used in GUI 580 # The same code is in grass.script.core already twice. 581 args[0] = grass.get_real_command(args[0]) 582 if args[0].endswith('.py'): 583 args.insert(0, sys.executable) 584 585 try: 586 self.module = Popen(args, 587 stdin=subprocess.PIPE, 588 stdout=subprocess.PIPE, 589 stderr=subprocess.PIPE, 590 shell=sys.platform == "win32", 591 env=self.env) 592 593 except OSError as e: 594 self.error = str(e) 595 print(e, file=sys.stderr) 596 return 1 597 598 if self.stdin: # read stdin if requested ... 599 self.module.stdin.write(self.stdin) 600 self.module.stdin.close() 601 602 # redirect standard outputs... 603 self._redirect_stream() 604 605 def _redirect_stream(self): 606 """Redirect stream""" 607 if self.stdout: 608 # make module stdout/stderr non-blocking 609 out_fileno = self.module.stdout.fileno() 610 if not is_mswindows: 611 flags = fcntl.fcntl(out_fileno, fcntl.F_GETFL) 612 fcntl.fcntl(out_fileno, fcntl.F_SETFL, flags | os.O_NONBLOCK) 613 614 if self.stderr: 615 # make module stdout/stderr non-blocking 616 out_fileno = self.module.stderr.fileno() 617 if not is_mswindows: 618 flags = fcntl.fcntl(out_fileno, fcntl.F_GETFL) 619 fcntl.fcntl(out_fileno, fcntl.F_SETFL, flags | os.O_NONBLOCK) 620 621 # wait for the process to end, sucking in stuff until it does end 622 while self.module.poll() is None: 623 if self._want_abort: # abort running process 624 self.module.terminate() 625 self.aborted = True 626 return 627 if self.stdout: 628 line = recv_some(self.module, e=0, stderr=0) 629 self.stdout.write(line) 630 if self.stderr: 631 line = recv_some(self.module, e=0, stderr=1) 632 self.stderr.write(line) 633 if len(line) > 0: 634 self.error = line 635 636 # get the last output 637 if self.stdout: 638 line = recv_some(self.module, e=0, stderr=0) 639 self.stdout.write(line) 640 if self.stderr: 641 line = recv_some(self.module, e=0, stderr=1) 642 self.stderr.write(line) 643 if len(line) > 0: 644 self.error = line 645 646 def abort(self): 647 """Abort running process, used by main thread to signal an abort""" 648 self._want_abort = True 649 650 651def _formatMsg(text): 652 """Format error messages for dialogs 653 """ 654 message = '' 655 for line in text.splitlines(): 656 if len(line) == 0: 657 continue 658 elif 'GRASS_INFO_MESSAGE' in line: 659 message += line.split(':', 1)[1].strip() + '\n' 660 elif 'GRASS_INFO_WARNING' in line: 661 message += line.split(':', 1)[1].strip() + '\n' 662 elif 'GRASS_INFO_ERROR' in line: 663 message += line.split(':', 1)[1].strip() + '\n' 664 elif 'GRASS_INFO_END' in line: 665 return message 666 else: 667 message += line.strip() + '\n' 668 669 return message 670 671 672def RunCommand(prog, flags="", overwrite=False, quiet=False, 673 verbose=False, parent=None, read=False, 674 parse=None, stdin=None, getErrorMsg=False, env=None, **kwargs): 675 """Run GRASS command 676 677 :param prog: program to run 678 :param flags: flags given as a string 679 :param overwrite, quiet, verbose: flags 680 :param parent: parent window for error messages 681 :param read: fetch stdout 682 :param parse: fn to parse stdout (e.g. grass.parse_key_val) or None 683 :param stdin: stdin or None 684 :param getErrorMsg: get error messages on failure 685 :param env: environment (optional, uses os.environ if not provided) 686 :param kwargs: program parameters 687 688 The environment passed to the function (env or os.environ) is not modified (a copy is used internally). 689 690 :return: returncode (read == False and getErrorMsg == False) 691 :return: returncode, messages (read == False and getErrorMsg == True) 692 :return: stdout (read == True and getErrorMsg == False) 693 :return: returncode, stdout, messages (read == True and getErrorMsg == True) 694 :return: stdout, stderr 695 """ 696 cmdString = ' '.join(grass.make_command(prog, flags, overwrite, 697 quiet, verbose, **kwargs)) 698 699 Debug.msg(1, "gcmd.RunCommand(): %s" % cmdString) 700 701 kwargs['stderr'] = subprocess.PIPE 702 703 if read: 704 kwargs['stdout'] = subprocess.PIPE 705 706 if stdin: 707 kwargs['stdin'] = subprocess.PIPE 708 709 # Do not change the environment, only a local copy. 710 if env: 711 env = env.copy() 712 else: 713 env = os.environ.copy() 714 715 if parent: 716 env['GRASS_MESSAGE_FORMAT'] = 'standard' 717 718 start = time.time() 719 720 ps = grass.start_command(prog, flags, overwrite, quiet, verbose, env=env, **kwargs) 721 722 if stdin: 723 ps.stdin.write(encode(stdin)) 724 ps.stdin.close() 725 ps.stdin = None 726 727 stdout, stderr = ps.communicate() 728 stderr = decode(stderr) 729 stdout = decode(stdout) if read else stdout 730 731 ret = ps.returncode 732 Debug.msg(1, "gcmd.RunCommand(): get return code %d (%.6f sec)" % 733 (ret, (time.time() - start))) 734 735 if ret != 0: 736 if stderr: 737 Debug.msg(2, "gcmd.RunCommand(): error %s" % stderr) 738 else: 739 Debug.msg(2, "gcmd.RunCommand(): nothing to print ???") 740 741 if parent: 742 GError(parent=parent, 743 caption=_("Error in %s") % prog, 744 message=stderr) 745 746 if not read: 747 if not getErrorMsg: 748 return ret 749 else: 750 return ret, _formatMsg(stderr) 751 752 if stdout: 753 Debug.msg(3, "gcmd.RunCommand(): return stdout\n'%s'" % stdout) 754 else: 755 Debug.msg(3, "gcmd.RunCommand(): return stdout = None") 756 757 if parse: 758 stdout = parse(stdout) 759 760 if not getErrorMsg: 761 return stdout 762 763 if read and getErrorMsg: 764 return ret, stdout, _formatMsg(stderr) 765 766 return stdout, _formatMsg(stderr) 767 768 769def GetDefaultEncoding(forceUTF8=False): 770 """Get default system encoding 771 772 :param bool forceUTF8: force 'UTF-8' if encoding is not defined 773 774 :return: system encoding (can be None) 775 """ 776 enc = locale.getdefaultlocale()[1] 777 if forceUTF8 and (enc is None or enc == 'UTF8'): 778 return 'UTF-8' 779 780 if enc is None: 781 enc = locale.getpreferredencoding() 782 783 Debug.msg(1, "GetSystemEncoding(): %s" % enc) 784 return enc 785 786_enc = GetDefaultEncoding() # define as global variable 787