1#    Copyright (C) 2016 Jeremy S. Sanders
2#    Email: Jeremy Sanders <jeremy@jeremysanders.net>
3#
4#    This program is free software; you can redistribute it and/or modify
5#    it under the terms of the GNU General Public License as published by
6#    the Free Software Foundation; either version 2 of the License, or
7#    (at your option) any later version.
8#
9#    This program is distributed in the hope that it will be useful,
10#    but WITHOUT ANY WARRANTY; without even the implied warranty of
11#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12#    GNU General Public License for more details.
13#
14#    You should have received a copy of the GNU General Public License along
15#    with this program; if not, write to the Free Software Foundation, Inc.,
16#    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17##############################################################################
18
19from __future__ import division
20from collections import defaultdict
21import os.path
22import re
23import datetime
24
25import numpy as N
26
27from . import colors
28
29from ..compat import citems, cstr, cexec
30from .. import setting
31from .. import utils
32from .. import datasets
33from .. import qtall as qt
34
35# python identifier
36identifier_re = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$')
37# for splitting
38identifier_split_re = re.compile(r'[A-Za-z_][A-Za-z0-9_]*')
39
40# python module
41module_re = re.compile(r'^[A-Za-z_\.]+$')
42
43# function(arg1, arg2...) for custom functions
44# not quite correct as doesn't check for commas in correct places
45function_re = re.compile(r'''
46^([A-Za-z_][A-Za-z0-9_]*)[ ]*  # identifier
47\((                            # begin args
48(?: [ ]* ,? [ ]* [A-Za-z_][A-Za-z0-9_]* )*     # named args
49(?: [ ]* ,? [ ]* \*[A-Za-z_][A-Za-z0-9_]* )?   # *args
50(?: [ ]* ,? [ ]* \*\*[A-Za-z_][A-Za-z0-9_]* )? # **kwargs
51)\)$                           # endargs''', re.VERBOSE)
52
53def _(text, disambiguation=None, context="Evaluate"):
54    """Translate text."""
55    return qt.QCoreApplication.translate(context, text, disambiguation)
56
57# Notes on Security
58# -----------------
59
60# Security states:
61#  * Secure
62#    - if new document
63#    - if loaded from secure location
64#    - or allow set in dialog
65#  * Insecure (skip in dialog)
66
67# Security context:
68#  * Executing statements when loading
69#  * Importing functions
70#  * Evaluating expressions (non checking Python)
71
72class Evaluate:
73    """Class to manage evaluation of expressions in a special environment."""
74
75    def __init__(self, doc):
76        self.doc = doc
77
78        # directories to examine when importing
79        self.importpath = []
80
81        self.wipe()
82
83    def wipe(self):
84        """Clear current customs."""
85
86        # store custom functions and constants
87        # consists of tuples of (name, type, value)
88        # type is constant or function
89        # we use this format to preserve evaluation order
90        self.def_imports = []
91        self.def_definitions = []
92        self.def_colors = []
93        self.def_colormaps = []
94
95        #self.customs = []
96
97        # this is the context used to evaluate expressions
98        self.context = {}
99
100        # copy default colormaps
101        self.colormaps = utils.ColorMaps()
102        self.colors = colors.Colors()
103
104        self.update()
105
106        # copies of validated compiled expressions
107        self.compiled = {}
108        self.compfailed = set()
109        self.compfailedchangeset = -1
110
111        # cached expressions which have been already evaluated as datasets
112        self.exprdscache = {}
113        self.exprdscachechangeset = None
114
115        # whether we hit security tests
116        self.setSecurity(False)
117
118    def update(self):
119        """To be called after custom constants or functions are changed.
120        This sets up a safe environment where things can be evaluated
121        """
122
123        c = self.context
124        c.clear()
125
126        # add numpy things
127        # we try to avoid various bits and pieces for safety
128        for name, val in citems(N.__dict__):
129            if ( (callable(val) or type(val)==float) and
130                 name not in __builtins__ and
131                 name[:1] != '_' and name[-1:] != '_' ):
132                c[name] = val
133
134        # safe functions
135        c['os_path_join'] = os.path.join
136        c['os_path_dirname'] = os.path.dirname
137        c['veusz_markercodes'] = tuple(utils.MarkerCodes)
138
139        # helpful functions for expansion
140        c['ENVIRON'] = dict(os.environ)
141        c['DATE'] = self._evalformatdate
142        c['TIME'] = self._evalformattime
143        c['DATA'] = self._evaldata
144        c['FILENAME'] = self._evalfilename
145        c['BASENAME'] = self._evalbasename
146        c['ESCAPE'] = utils.latexEscape
147        c['SETTING'] = self._evalsetting
148        c['LANG'] = self._evallang
149
150        for name, val in self.def_imports:
151            self._updateImport(name, val)
152
153        for name, val in self.def_definitions:
154            self._updateDefinition(name, val)
155
156        self.colors.wipe()
157        for name, val in self.def_colors:
158            self.colors.addColor(name, val)
159        self.colors.updateModel()
160
161        self.colormaps.wipe()
162        for name, val in self.def_colormaps:
163            self._updateColormap(name, val)
164
165    def setSecurity(self, secure):
166        """Updated the security context."""
167        oldsecure = getattr(self, 'secure_document', False)
168
169        self.secure_document = secure
170        self.doc.sigSecuritySet.emit(secure)
171
172        if not oldsecure and secure:
173            # if we're now secure, and were not previously, update
174            # context
175            self.exprdscache = {}
176            self.exprdscachechangeset = None
177            self.update()
178
179    def updateSecurityFromPath(self):
180        """Make document secure if in a secure location."""
181        filename = self.doc.filename
182        absfilename = os.path.abspath(filename)
183        paths = setting.settingdb['secure_dirs'] + [
184            utils.exampleDirectory]
185        for dirname in paths:
186            absdirname = os.path.abspath(dirname)
187            if absfilename.startswith(absdirname + os.sep):
188                self.setSecurity(True)
189
190    def inSecureMode(self):
191        """Is the document in a safe location?"""
192        return (
193            setting.transient_settings['unsafe_mode'] or
194            self.secure_document
195        )
196
197    def _updateImport(self, module, val):
198        """Add an import statement to the eval function context."""
199        if module_re.match(module):
200            # work out what is safe to import
201            symbols = identifier_split_re.findall(val)
202            if self._checkImportsSafe():
203                if symbols:
204                    defn = 'from %s import %s' % (
205                        module, ', '.join(symbols))
206                    try:
207                        cexec(defn, self.context)
208                    except Exception:
209                        self.doc.log(_(
210                            "Failed to import '%s' from module '%s'") % (
211                                ', '.join(toimport), module))
212                        return
213                else:
214                    defn = 'import %s' % module
215                    try:
216                        cexec(defn, self.context)
217                    except Exception:
218                        self.doc.log(_(
219                            "Failed to import module '%s'") % module)
220                        return
221            else:
222                if not symbols:
223                    self.doc.log(_("Did not import module '%s'") % module)
224                else:
225                    self.doc.log(_(
226                        "Did not import '%s' from module '%s'") % (
227                            ', '.join(list(symbols)), module))
228
229        else:
230            self.doc.log( _("Invalid module name '%s'") % module )
231
232    def validateProcessColormap(self, colormap):
233        """Validate and process a colormap value.
234
235        Returns a list of B,G,R,alpha tuples or raises ValueError if a problem."""
236
237        try:
238            if len(colormap) < 2:
239                raise ValueError( _("Need at least two entries in colormap") )
240        except TypeError:
241            raise ValueError( _("Invalid type for colormap") )
242
243        out = []
244        for entry in colormap:
245            if entry == (-1,0,0,0):
246                out.append(entry)
247                continue
248
249            for v in entry:
250                try:
251                    v - 0
252                except TypeError:
253                    raise ValueError(
254                        _("Colormap entries should be numerical") )
255                if v < 0 or v > 255:
256                    raise ValueError(
257                        _("Colormap entries should be between 0 and 255") )
258
259            if len(entry) == 3:
260                out.append( (int(entry[2]), int(entry[1]), int(entry[0]),
261                             255) )
262            elif len(entry) == 4:
263                out.append( (int(entry[2]), int(entry[1]), int(entry[0]),
264                             int(entry[3])) )
265            else:
266                raise ValueError( _("Each colormap entry consists of R,G,B "
267                                    "and optionally alpha values") )
268
269        return tuple(out)
270
271    def _updateColormap(self, name, val):
272        """Add a colormap entry."""
273
274        try:
275            cmap = self.validateProcessColormap(val)
276        except ValueError as e:
277            self.doc.log( cstr(e) )
278        else:
279            self.colormaps[ cstr(name) ] = cmap
280
281    def _updateDefinition(self, name, val):
282        """Update a function or constant in eval function context."""
283
284        if identifier_re.match(name):
285            defn = val
286        else:
287            m = function_re.match(name)
288            if not m:
289                self.doc.log(
290                    _("Invalid function or constant specification '%s'") %
291                    name)
292                return
293            name = m.group(1)
294            args = m.group(2)
295            defn = 'lambda %s: %s' % (args, val)
296
297        # evaluate, but we ignore any unsafe commands or exceptions
298        comp = self.compileCheckedExpression(defn)
299        if comp is None:
300            return
301        try:
302            self.context[name] = eval(comp, self.context)
303        except Exception as e:
304            self.doc.log( _(
305                "Error evaluating '%s': '%s'") % (name, cstr(e)) )
306
307    def compileCheckedExpression(self, expr, origexpr=None, log=True):
308        """Compile expression and check for errors.
309
310        origexpr is an expression to show in error messages. This is
311        used if replacements have been done, etc.
312        """
313
314        try:
315            return self.compiled[expr]
316        except KeyError:
317            pass
318
319        # track failed compilations, so we only print them once
320        if self.compfailedchangeset != self.doc.changeset:
321            self.compfailedchangeset = self.doc.changeset
322            self.compfailed.clear()
323        elif expr in self.compfailed:
324            return None
325
326        if origexpr is None:
327            origexpr = expr
328
329        try:
330            checked = utils.compileChecked(
331                expr,
332                ignoresecurity=self.inSecureMode(),
333            )
334        except utils.SafeEvalException as e:
335            if log:
336                self.doc.log(
337                    _("Unsafe expression '%s': %s") % (origexpr, cstr(e)))
338            self.compfailed.add(expr)
339            return None
340        except Exception as e:
341            if log:
342                self.doc.log(
343                    _("Error in expression '%s': %s") % (origexpr, cstr(e)))
344            return None
345        else:
346            self.compiled[expr] = checked
347            return checked
348
349    @staticmethod
350    def _evalformatdate(fmt=None):
351        """DATE() eval: return date with optional format."""
352        d = datetime.date.today()
353        return d.isoformat() if fmt is None else d.strftime(fmt)
354
355    @staticmethod
356    def _evalformattime(fmt=None):
357        """TIME() eval: return time with optional format."""
358        t = datetime.datetime.now()
359        return t.isoformat() if fmt is None else t.strftime(fmt)
360
361    def _evaldata(self, name, part='data'):
362        """DATA(name, [part]) eval: return dataset as array."""
363        if part not in ('data', 'perr', 'serr', 'nerr'):
364            raise RuntimeError("Invalid dataset part '%s'" % part)
365        if name not in self.doc.data:
366            raise RuntimeError("Dataset '%s' does not exist" % name)
367        data = getattr(self.doc.data[name], part)
368
369        if isinstance(data, N.ndarray):
370            return N.array(data)
371        elif isinstance(data, list):
372            return list(data)
373        return data
374
375    def _evalfilename(self):
376        """FILENAME() eval: returns filename."""
377        return utils.latexEscape(self.doc.filename)
378
379    def _evalbasename(self):
380        """BASENAME() eval: returns base filename."""
381        return utils.latexEscape(os.path.basename(self.doc.filename))
382
383    def _evalsetting(self, path):
384        """SETTING() eval: return setting given full path."""
385        return self.doc.resolveSettingPath(None, path).get()
386
387    @staticmethod
388    def _evallang(opts):
389        lang = qt.QLocale().name()
390        if lang in opts:
391            return opts[lang]
392        majorl = lang.split('_')[0]
393        if majorl in opts:
394            return opts[majorl]
395        if 'default' in opts:
396            return opts['default']
397        return utils.latexEscape('NOLANG:%s' % str(lang))
398
399    def evalDatasetExpression(self, expr, part='data', datatype='numeric',
400                              dimensions=1):
401        """Return dataset after evaluating a dataset expression.
402        part is 'data', 'serr', 'perr' or 'nerr' - these are the
403        dataset parts which are evaluated by the expression
404
405        None is returned on error
406        """
407
408        key = (expr, part, datatype, dimensions)
409        if self.exprdscachechangeset != self.doc.changeset:
410            self.exprdscachechangeset = self.doc.changeset
411            self.exprdscache.clear()
412        elif key in self.exprdscache:
413            return self.exprdscache[key]
414
415        self.exprdscache[key] = ds = datasets.evalDatasetExpression(
416            self.doc, expr, part=part, datatype=datatype, dimensions=dimensions)
417        return ds
418
419    def _checkImportsSafe(self):
420        """Check whether symbols are safe to import."""
421
422        # do import anyway
423        if self.inSecureMode():
424            return True
425
426        self.doc.sigAllowedImports.emit()
427        return self.inSecureMode()
428
429    def getColormap(self, name, invert):
430        """Get colormap with name given (returning grey if does not exist)."""
431        cmap = self.colormaps.get(name, self.colormaps['grey'])
432        if invert:
433            if cmap[0][0] >= 0:
434                return cmap[::-1]
435            else:
436                # ignore marker at beginning for stepped maps
437                return tuple([cmap[0]] + list(cmap[-1:0:-1]))
438        return cmap
439
440    def saveCustomDefinitions(self, fileobj):
441        """Save custom constants and functions."""
442        for ctype, defns in (
443                ('import', self.def_imports),
444                ('definition', self.def_definitions),
445                ('color', self.def_colors),
446                ('colormap', self.def_colormaps)):
447
448            for val in defns:
449                fileobj.write(
450                    'AddCustom(%s, %s, %s)\n' % (
451                        utils.rrepr(ctype),
452                        utils.rrepr(val[0]),
453                        utils.rrepr(val[1])))
454
455    def saveCustomFile(self, fileobj):
456        """Export the custom settings to a file."""
457
458        self.doc._writeFileHeader(fileobj, 'custom definitions')
459        self.saveCustomDefinitions(fileobj)
460