1"""
2@package core.gcmd
3
4@brief wxGUI command interface
5
6Classes:
7 - gcmd::GError
8 - gcmd::GWarning
9 - gcmd::GMessage
10 - gcmd::GException
11 - gcmd::Popen (from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/440554)
12 - gcmd::Command
13 - gcmd::CommandThread
14
15Functions:
16 - RunCommand
17 - GetDefaultEncoding
18
19(C) 2007-2008, 2010-2011 by the GRASS Development Team
20
21This program is free software under the GNU General Public License
22(>=v2). Read the file COPYING that comes with GRASS for details.
23
24@author Jachym Cepicky
25@author Martin Landa <landa.martin gmail.com>
26"""
27
28from __future__ import print_function
29
30import os
31import sys
32import time
33import errno
34import signal
35import traceback
36import locale
37import subprocess
38from threading import Thread
39import wx
40
41is_mswindows = sys.platform == 'win32'
42if is_mswindows:
43    from win32file import ReadFile, WriteFile
44    from win32pipe import PeekNamedPipe
45    import msvcrt
46else:
47    import select
48    import fcntl
49
50from core.debug import Debug
51from core.globalvar import SCT_EXT
52
53from grass.script import core as grass
54from grass.script.utils import decode, encode
55
56if sys.version_info.major == 2:
57    bytes = str
58
59
60def DecodeString(string):
61    """Decode string using system encoding
62
63    :param string: string to be decoded
64
65    :return: decoded string
66    """
67    if not string:
68        return string
69
70    if _enc and isinstance(string, bytes):
71        Debug.msg(5, "DecodeString(): enc=%s" % _enc)
72        return string.decode(_enc)
73    return string
74
75
76def EncodeString(string):
77    """Return encoded string using system locales
78
79    :param string: string to be encoded
80
81    :return: encoded string
82    """
83    if not string:
84        return string
85    if _enc:
86        Debug.msg(5, "EncodeString(): enc=%s" % _enc)
87        return string.encode(_enc)
88    return string
89
90
91class GError:
92
93    def __init__(self, message, parent=None, caption=None, showTraceback=True):
94        """Show error message window
95
96        :param message: error message
97        :param parent: centre window on parent if given
98        :param caption: window caption (if not given "Error")
99        :param showTraceback: True to show also Python traceback
100        """
101        if not caption:
102            caption = _('Error')
103        style = wx.OK | wx.ICON_ERROR | wx.CENTRE
104        exc_type, exc_value, exc_traceback = sys.exc_info()
105        if exc_traceback:
106            exception = traceback.format_exc()
107            reason = exception.splitlines()[-1].split(':', 1)[-1].strip()
108
109        if Debug.GetLevel() > 0 and exc_traceback:
110            sys.stderr.write(exception)
111
112        if showTraceback and exc_traceback:
113            wx.MessageBox(parent=parent,
114                          message=message + '\n\n%s: %s\n\n%s' %
115                          (_('Reason'),
116                           reason, exception),
117                          caption=caption,
118                          style=style)
119        else:
120            wx.MessageBox(parent=parent,
121                          message=message,
122                          caption=caption,
123                          style=style)
124
125
126class GWarning:
127
128    def __init__(self, message, parent=None):
129        caption = _('Warning')
130        style = wx.OK | wx.ICON_WARNING | wx.CENTRE
131        wx.MessageBox(parent=parent,
132                      message=message,
133                      caption=caption,
134                      style=style)
135
136
137class GMessage:
138
139    def __init__(self, message, parent=None):
140        caption = _('Message')
141        style = wx.OK | wx.ICON_INFORMATION | wx.CENTRE
142        wx.MessageBox(parent=parent,
143                      message=message,
144                      caption=caption,
145                      style=style)
146
147
148class GException(Exception):
149
150    def __init__(self, value=''):
151        self.value = value
152
153    def __str__(self):
154        return self.value
155
156    def __unicode__(self):
157        return self.value
158
159
160class Popen(subprocess.Popen):
161    """Subclass subprocess.Popen"""
162
163    def __init__(self, args, **kwargs):
164        if is_mswindows:
165            # The Windows shell (cmd.exe) requires some special characters to
166            # be escaped by preceding them with 3 carets (^^^). cmd.exe /?
167            # mentions <space> and &()[]{}^=;!'+,`~. A quick test revealed that
168            # only ^|&<> need to be escaped. A single quote can be escaped by
169            # enclosing it with double quotes and vice versa.
170            for i in range(2, len(args)):
171                # "^" must be the first character in the list to avoid double
172                # escaping.
173                for c in ("^", "|", "&", "<", ">"):
174                    if c in args[i]:
175                        if "=" in args[i]:
176                            a = args[i].split("=")
177                            k = a[0] + "="
178                            v = "=".join(a[1:len(a)])
179                        else:
180                            k = ""
181                            v = args[i]
182
183                        # If there are spaces, the argument was already
184                        # esscaped with double quotes, so don't escape it
185                        # again.
186                        if c in v and not " " in v:
187                            # Here, we escape each ^ in ^^^ with ^^ and a
188                            # <special character> with ^ + <special character>,
189                            # so we need 7 carets.
190
191                            v = v.replace(c, "^^^^^^^" + c)
192                            args[i] = k + v
193
194        subprocess.Popen.__init__(self, args, **kwargs)
195
196    def recv(self, maxsize=None):
197        return self._recv('stdout', maxsize)
198
199    def recv_err(self, maxsize=None):
200        return self._recv('stderr', maxsize)
201
202    def send_recv(self, input='', maxsize=None):
203        return self.send(input), self.recv(maxsize), self.recv_err(maxsize)
204
205    def get_conn_maxsize(self, which, maxsize):
206        if maxsize is None:
207            maxsize = 1024
208        elif maxsize < 1:
209            maxsize = 1
210        return getattr(self, which), maxsize
211
212    def _close(self, which):
213        getattr(self, which).close()
214        setattr(self, which, None)
215
216    def kill(self):
217        """Try to kill running process"""
218        if is_mswindows:
219            import win32api
220            handle = win32api.OpenProcess(1, 0, self.pid)
221            return (0 != win32api.TerminateProcess(handle, 0))
222        else:
223            try:
224                os.kill(-self.pid, signal.SIGTERM)  # kill whole group
225            except OSError:
226                pass
227
228    if sys.platform == 'win32':
229        def send(self, input):
230            if not self.stdin:
231                return None
232
233            import pywintypes
234            try:
235                x = msvcrt.get_osfhandle(self.stdin.fileno())
236                (errCode, written) = WriteFile(x, input)
237            except ValueError:
238                return self._close('stdin')
239            except (pywintypes.error, Exception) as why:
240                if why.winerror in (109, errno.ESHUTDOWN):
241                    return self._close('stdin')
242                raise
243
244            return written
245
246        def _recv(self, which, maxsize):
247            conn, maxsize = self.get_conn_maxsize(which, maxsize)
248            if conn is None:
249                return None
250
251            import pywintypes
252            try:
253                x = msvcrt.get_osfhandle(conn.fileno())
254                (read, nAvail, nMessage) = PeekNamedPipe(x, 0)
255                if maxsize < nAvail:
256                    nAvail = maxsize
257                if nAvail > 0:
258                    (errCode, read) = ReadFile(x, nAvail, None)
259            except ValueError:
260                return self._close(which)
261            except (pywintypes.error, Exception) as why:
262                if why.winerror in (109, errno.ESHUTDOWN):
263                    return self._close(which)
264                raise
265
266            if self.universal_newlines:
267                read = self._translate_newlines(read)
268            return read
269
270    else:
271        def send(self, input):
272            if not self.stdin:
273                return None
274
275            if not select.select([], [self.stdin], [], 0)[1]:
276                return 0
277
278            try:
279                written = os.write(self.stdin.fileno(), input)
280            except OSError as why:
281                if why[0] == errno.EPIPE:  # broken pipe
282                    return self._close('stdin')
283                raise
284
285            return written
286
287        def _recv(self, which, maxsize):
288            conn, maxsize = self.get_conn_maxsize(which, maxsize)
289            if conn is None:
290                return None
291
292            flags = fcntl.fcntl(conn, fcntl.F_GETFL)
293            if not conn.closed:
294                fcntl.fcntl(conn, fcntl.F_SETFL, flags | os.O_NONBLOCK)
295
296            try:
297                if not select.select([conn], [], [], 0)[0]:
298                    return ''
299
300                r = conn.read()
301
302                if not r:
303                    return self._close(which)
304
305                if self.universal_newlines:
306                    r = self._translate_newlines(r)
307                return r
308            finally:
309                if not conn.closed:
310                    fcntl.fcntl(conn, fcntl.F_SETFL, flags)
311
312message = "Other end disconnected!"
313
314
315def recv_some(p, t=.1, e=1, tr=5, stderr=0):
316    if tr < 1:
317        tr = 1
318    x = time.time() + t
319    y = []
320    r = ''
321    pr = p.recv
322    if stderr:
323        pr = p.recv_err
324    while time.time() < x or r:
325        r = pr()
326        if r is None:
327            if e:
328                raise Exception(message)
329            else:
330                break
331        elif r:
332            y.append(decode(r))
333        else:
334            time.sleep(max((x - time.time()) / tr, 0))
335    return ''.join(y)
336
337
338def send_all(p, data):
339    while len(data):
340        sent = p.send(data)
341        if sent is None:
342            raise Exception(message)
343        data = buffer(data, sent)
344
345
346class Command:
347    """Run command in separate thread. Used for commands launched
348    on the background.
349
350    If stdout/err is redirected, write() method is required for the
351    given classes.
352
353        cmd = Command(cmd=['d.rast', 'elevation.dem'], verbose=3, wait=True)
354
355        if cmd.returncode == None:
356            print 'RUNNING?'
357        elif cmd.returncode == 0:
358            print 'SUCCESS'
359        else:
360            print 'FAILURE (%d)' % cmd.returncode
361    """
362
363    def __init__(self, cmd, stdin=None,
364                 verbose=None, wait=True, rerr=False,
365                 stdout=None, stderr=None):
366        """
367        :param cmd: command given as list
368        :param stdin: standard input stream
369        :param verbose: verbose level [0, 3] (--q, --v)
370        :param wait: wait for child execution terminated
371        :param rerr: error handling (when GException raised).
372                     True for redirection to stderr, False for GUI
373                     dialog, None for no operation (quiet mode)
374        :param stdout:  redirect standard output or None
375        :param stderr:  redirect standard error output or None
376        """
377        Debug.msg(1, "gcmd.Command(): %s" % ' '.join(cmd))
378        self.cmd = cmd
379        self.stderr = stderr
380
381        #
382        # set verbosity level
383        #
384        verbose_orig = None
385        if ('--q' not in self.cmd and '--quiet' not in self.cmd) and \
386                ('--v' not in self.cmd and '--verbose' not in self.cmd):
387            if verbose is not None:
388                if verbose == 0:
389                    self.cmd.append('--quiet')
390                elif verbose == 3:
391                    self.cmd.append('--verbose')
392                else:
393                    verbose_orig = os.getenv("GRASS_VERBOSE")
394                    os.environ["GRASS_VERBOSE"] = str(verbose)
395
396        #
397        # create command thread
398        #
399        self.cmdThread = CommandThread(cmd, stdin,
400                                       stdout, stderr)
401        self.cmdThread.start()
402
403        if wait:
404            self.cmdThread.join()
405            if self.cmdThread.module:
406                self.cmdThread.module.wait()
407                self.returncode = self.cmdThread.module.returncode
408            else:
409                self.returncode = 1
410        else:
411            self.cmdThread.join(0.5)
412            self.returncode = None
413
414        if self.returncode is not None:
415            Debug.msg(
416                3, "Command(): cmd='%s', wait=%s, returncode=%d, alive=%s" %
417                (' '.join(cmd), wait, self.returncode, self.cmdThread.isAlive()))
418            if rerr is not None and self.returncode != 0:
419                if rerr is False:  # GUI dialog
420                    raise GException("%s '%s'%s%s%s %s%s" %
421                                     (_("Execution failed:"),
422                                      ' '.join(self.cmd),
423                                      os.linesep, os.linesep,
424                                      _("Details:"),
425                                      os.linesep,
426                                      _("Error: ") + self.__GetError()))
427                elif rerr == sys.stderr:  # redirect message to sys
428                    stderr.write("Execution failed: '%s'" %
429                                 (' '.join(self.cmd)))
430                    stderr.write(
431                        "%sDetails:%s%s" %
432                        (os.linesep,
433                         _("Error: ") +
434                            self.__GetError(),
435                            os.linesep))
436            else:
437                pass  # nop
438        else:
439            Debug.msg(
440                3, "Command(): cmd='%s', wait=%s, returncode=?, alive=%s" %
441                (' '.join(cmd), wait, self.cmdThread.isAlive()))
442
443        if verbose_orig:
444            os.environ["GRASS_VERBOSE"] = verbose_orig
445        elif "GRASS_VERBOSE" in os.environ:
446            del os.environ["GRASS_VERBOSE"]
447
448    def __ReadOutput(self, stream):
449        """Read stream and return list of lines
450
451        :param stream: stream to be read
452        """
453        lineList = []
454
455        if stream is None:
456            return lineList
457
458        while True:
459            line = stream.readline()
460            if not line:
461                break
462            line = line.replace('%s' % os.linesep, '').strip()
463            lineList.append(line)
464
465        return lineList
466
467    def __ReadErrOutput(self):
468        """Read standard error output and return list of lines"""
469        return self.__ReadOutput(self.cmdThread.module.stderr)
470
471    def __ProcessStdErr(self):
472        """
473        Read messages/warnings/errors from stderr
474
475        :return: list of (type, message)
476        """
477        if self.stderr is None:
478            lines = self.__ReadErrOutput()
479        else:
480            lines = self.cmdThread.error.strip('%s' % os.linesep). \
481                split('%s' % os.linesep)
482
483        msg = []
484
485        type = None
486        content = ""
487        for line in lines:
488            if len(line) == 0:
489                continue
490            if 'GRASS_' in line:  # error or warning
491                if 'GRASS_INFO_WARNING' in line:  # warning
492                    type = "WARNING"
493                elif 'GRASS_INFO_ERROR' in line:  # error
494                    type = "ERROR"
495                elif 'GRASS_INFO_END':  # end of message
496                    msg.append((type, content))
497                    type = None
498                    content = ""
499
500                if type:
501                    content += line.split(':', 1)[1].strip()
502            else:  # stderr
503                msg.append((None, line.strip()))
504
505        return msg
506
507    def __GetError(self):
508        """Get error message or ''"""
509        if not self.cmdThread.module:
510            return _("Unable to exectute command: '%s'") % ' '.join(self.cmd)
511
512        for type, msg in self.__ProcessStdErr():
513            if type == 'ERROR':
514                if _enc:
515                    return unicode(msg, _enc)
516                return msg
517
518        return ''
519
520
521class CommandThread(Thread):
522    """Create separate thread for command. Used for commands launched
523    on the background."""
524
525    def __init__(self, cmd, env=None, stdin=None,
526                 stdout=sys.stdout, stderr=sys.stderr):
527        """
528        :param cmd: command (given as list)
529        :param env: environmental variables
530        :param stdin: standard input stream
531        :param stdout: redirect standard output or None
532        :param stderr: redirect standard error output or None
533        """
534        Thread.__init__(self)
535
536        self.cmd = cmd
537        self.stdin = stdin
538        self.stdout = stdout
539        self.stderr = stderr
540        self.env = env
541
542        self.module = None
543        self.error = ''
544
545        self._want_abort = False
546        self.aborted = False
547
548        self.setDaemon(True)
549
550        # set message formatting
551        self.message_format = os.getenv("GRASS_MESSAGE_FORMAT")
552        os.environ["GRASS_MESSAGE_FORMAT"] = "gui"
553
554    def __del__(self):
555        if self.message_format:
556            os.environ["GRASS_MESSAGE_FORMAT"] = self.message_format
557        else:
558            del os.environ["GRASS_MESSAGE_FORMAT"]
559
560    def run(self):
561        """Run command"""
562        if len(self.cmd) == 0:
563            return
564
565        Debug.msg(1, "gcmd.CommandThread(): %s" % ' '.join(self.cmd))
566
567        self.startTime = time.time()
568
569        # TODO: replace ugly hack below
570        # this cannot be replaced it can be only improved
571        # also unifying this with 3 other places in code would be nice
572        # changing from one chdir to get_real_command function
573        args = self.cmd
574        if sys.platform == 'win32':
575            if os.path.splitext(args[0])[1] == SCT_EXT:
576                args[0] = args[0][:-3]
577            # using Python executable to run the module if it is a script
578            # expecting at least module name at first position
579            # cannot use make_command for this now because it is used in GUI
580            # The same code is in grass.script.core already twice.
581            args[0] = grass.get_real_command(args[0])
582            if args[0].endswith('.py'):
583                args.insert(0, sys.executable)
584
585        try:
586            self.module = Popen(args,
587                                stdin=subprocess.PIPE,
588                                stdout=subprocess.PIPE,
589                                stderr=subprocess.PIPE,
590                                shell=sys.platform == "win32",
591                                env=self.env)
592
593        except OSError as e:
594            self.error = str(e)
595            print(e, file=sys.stderr)
596            return 1
597
598        if self.stdin:  # read stdin if requested ...
599            self.module.stdin.write(self.stdin)
600            self.module.stdin.close()
601
602        # redirect standard outputs...
603        self._redirect_stream()
604
605    def _redirect_stream(self):
606        """Redirect stream"""
607        if self.stdout:
608            # make module stdout/stderr non-blocking
609            out_fileno = self.module.stdout.fileno()
610            if not is_mswindows:
611                flags = fcntl.fcntl(out_fileno, fcntl.F_GETFL)
612                fcntl.fcntl(out_fileno, fcntl.F_SETFL, flags | os.O_NONBLOCK)
613
614        if self.stderr:
615            # make module stdout/stderr non-blocking
616            out_fileno = self.module.stderr.fileno()
617            if not is_mswindows:
618                flags = fcntl.fcntl(out_fileno, fcntl.F_GETFL)
619                fcntl.fcntl(out_fileno, fcntl.F_SETFL, flags | os.O_NONBLOCK)
620
621        # wait for the process to end, sucking in stuff until it does end
622        while self.module.poll() is None:
623            if self._want_abort:  # abort running process
624                self.module.terminate()
625                self.aborted = True
626                return
627            if self.stdout:
628                line = recv_some(self.module, e=0, stderr=0)
629                self.stdout.write(line)
630            if self.stderr:
631                line = recv_some(self.module, e=0, stderr=1)
632                self.stderr.write(line)
633                if len(line) > 0:
634                    self.error = line
635
636        # get the last output
637        if self.stdout:
638            line = recv_some(self.module, e=0, stderr=0)
639            self.stdout.write(line)
640        if self.stderr:
641            line = recv_some(self.module, e=0, stderr=1)
642            self.stderr.write(line)
643            if len(line) > 0:
644                self.error = line
645
646    def abort(self):
647        """Abort running process, used by main thread to signal an abort"""
648        self._want_abort = True
649
650
651def _formatMsg(text):
652    """Format error messages for dialogs
653    """
654    message = ''
655    for line in text.splitlines():
656        if len(line) == 0:
657            continue
658        elif 'GRASS_INFO_MESSAGE' in line:
659            message += line.split(':', 1)[1].strip() + '\n'
660        elif 'GRASS_INFO_WARNING' in line:
661            message += line.split(':', 1)[1].strip() + '\n'
662        elif 'GRASS_INFO_ERROR' in line:
663            message += line.split(':', 1)[1].strip() + '\n'
664        elif 'GRASS_INFO_END' in line:
665            return message
666        else:
667            message += line.strip() + '\n'
668
669    return message
670
671
672def RunCommand(prog, flags="", overwrite=False, quiet=False,
673               verbose=False, parent=None, read=False,
674               parse=None, stdin=None, getErrorMsg=False, env=None, **kwargs):
675    """Run GRASS command
676
677    :param prog: program to run
678    :param flags: flags given as a string
679    :param overwrite, quiet, verbose: flags
680    :param parent: parent window for error messages
681    :param read: fetch stdout
682    :param parse: fn to parse stdout (e.g. grass.parse_key_val) or None
683    :param stdin: stdin or None
684    :param getErrorMsg: get error messages on failure
685    :param env: environment (optional, uses os.environ if not provided)
686    :param kwargs: program parameters
687
688    The environment passed to the function (env or os.environ) is not modified (a copy is used internally).
689
690    :return: returncode (read == False and getErrorMsg == False)
691    :return: returncode, messages (read == False and getErrorMsg == True)
692    :return: stdout (read == True and getErrorMsg == False)
693    :return: returncode, stdout, messages (read == True and getErrorMsg == True)
694    :return: stdout, stderr
695    """
696    cmdString = ' '.join(grass.make_command(prog, flags, overwrite,
697                                            quiet, verbose, **kwargs))
698
699    Debug.msg(1, "gcmd.RunCommand(): %s" % cmdString)
700
701    kwargs['stderr'] = subprocess.PIPE
702
703    if read:
704        kwargs['stdout'] = subprocess.PIPE
705
706    if stdin:
707        kwargs['stdin'] = subprocess.PIPE
708
709    # Do not change the environment, only a local copy.
710    if env:
711        env = env.copy()
712    else:
713        env = os.environ.copy()
714
715    if parent:
716        env['GRASS_MESSAGE_FORMAT'] = 'standard'
717
718    start = time.time()
719
720    ps = grass.start_command(prog, flags, overwrite, quiet, verbose, env=env, **kwargs)
721
722    if stdin:
723        ps.stdin.write(encode(stdin))
724        ps.stdin.close()
725        ps.stdin = None
726
727    stdout, stderr = ps.communicate()
728    stderr = decode(stderr)
729    stdout = decode(stdout) if read else stdout
730
731    ret = ps.returncode
732    Debug.msg(1, "gcmd.RunCommand(): get return code %d (%.6f sec)" %
733              (ret, (time.time() - start)))
734
735    if ret != 0:
736        if stderr:
737            Debug.msg(2, "gcmd.RunCommand(): error %s" % stderr)
738        else:
739            Debug.msg(2, "gcmd.RunCommand(): nothing to print ???")
740
741        if parent:
742            GError(parent=parent,
743                   caption=_("Error in %s") % prog,
744                   message=stderr)
745
746    if not read:
747        if not getErrorMsg:
748            return ret
749        else:
750            return ret, _formatMsg(stderr)
751
752    if stdout:
753        Debug.msg(3, "gcmd.RunCommand(): return stdout\n'%s'" % stdout)
754    else:
755        Debug.msg(3, "gcmd.RunCommand(): return stdout = None")
756
757    if parse:
758        stdout = parse(stdout)
759
760    if not getErrorMsg:
761        return stdout
762
763    if read and getErrorMsg:
764        return ret, stdout, _formatMsg(stderr)
765
766    return stdout, _formatMsg(stderr)
767
768
769def GetDefaultEncoding(forceUTF8=False):
770    """Get default system encoding
771
772    :param bool forceUTF8: force 'UTF-8' if encoding is not defined
773
774    :return: system encoding (can be None)
775    """
776    enc = locale.getdefaultlocale()[1]
777    if forceUTF8 and (enc is None or enc == 'UTF8'):
778        return 'UTF-8'
779
780    if enc is None:
781        enc = locale.getpreferredencoding()
782
783    Debug.msg(1, "GetSystemEncoding(): %s" % enc)
784    return enc
785
786_enc = GetDefaultEncoding()  # define as global variable
787