1# Copyright (C) 2016 Jeremy S. Sanders 2# Email: Jeremy Sanders <jeremy@jeremysanders.net> 3# 4# This program is free software; you can redistribute it and/or modify 5# it under the terms of the GNU General Public License as published by 6# the Free Software Foundation; either version 2 of the License, or 7# (at your option) any later version. 8# 9# This program is distributed in the hope that it will be useful, 10# but WITHOUT ANY WARRANTY; without even the implied warranty of 11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12# GNU General Public License for more details. 13# 14# You should have received a copy of the GNU General Public License along 15# with this program; if not, write to the Free Software Foundation, Inc., 16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17############################################################################## 18 19from __future__ import division 20from collections import defaultdict 21import os.path 22import re 23import datetime 24 25import numpy as N 26 27from . import colors 28 29from ..compat import citems, cstr, cexec 30from .. import setting 31from .. import utils 32from .. import datasets 33from .. import qtall as qt 34 35# python identifier 36identifier_re = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$') 37# for splitting 38identifier_split_re = re.compile(r'[A-Za-z_][A-Za-z0-9_]*') 39 40# python module 41module_re = re.compile(r'^[A-Za-z_\.]+$') 42 43# function(arg1, arg2...) for custom functions 44# not quite correct as doesn't check for commas in correct places 45function_re = re.compile(r''' 46^([A-Za-z_][A-Za-z0-9_]*)[ ]* # identifier 47\(( # begin args 48(?: [ ]* ,? [ ]* [A-Za-z_][A-Za-z0-9_]* )* # named args 49(?: [ ]* ,? [ ]* \*[A-Za-z_][A-Za-z0-9_]* )? # *args 50(?: [ ]* ,? [ ]* \*\*[A-Za-z_][A-Za-z0-9_]* )? # **kwargs 51)\)$ # endargs''', re.VERBOSE) 52 53def _(text, disambiguation=None, context="Evaluate"): 54 """Translate text.""" 55 return qt.QCoreApplication.translate(context, text, disambiguation) 56 57# Notes on Security 58# ----------------- 59 60# Security states: 61# * Secure 62# - if new document 63# - if loaded from secure location 64# - or allow set in dialog 65# * Insecure (skip in dialog) 66 67# Security context: 68# * Executing statements when loading 69# * Importing functions 70# * Evaluating expressions (non checking Python) 71 72class Evaluate: 73 """Class to manage evaluation of expressions in a special environment.""" 74 75 def __init__(self, doc): 76 self.doc = doc 77 78 # directories to examine when importing 79 self.importpath = [] 80 81 self.wipe() 82 83 def wipe(self): 84 """Clear current customs.""" 85 86 # store custom functions and constants 87 # consists of tuples of (name, type, value) 88 # type is constant or function 89 # we use this format to preserve evaluation order 90 self.def_imports = [] 91 self.def_definitions = [] 92 self.def_colors = [] 93 self.def_colormaps = [] 94 95 #self.customs = [] 96 97 # this is the context used to evaluate expressions 98 self.context = {} 99 100 # copy default colormaps 101 self.colormaps = utils.ColorMaps() 102 self.colors = colors.Colors() 103 104 self.update() 105 106 # copies of validated compiled expressions 107 self.compiled = {} 108 self.compfailed = set() 109 self.compfailedchangeset = -1 110 111 # cached expressions which have been already evaluated as datasets 112 self.exprdscache = {} 113 self.exprdscachechangeset = None 114 115 # whether we hit security tests 116 self.setSecurity(False) 117 118 def update(self): 119 """To be called after custom constants or functions are changed. 120 This sets up a safe environment where things can be evaluated 121 """ 122 123 c = self.context 124 c.clear() 125 126 # add numpy things 127 # we try to avoid various bits and pieces for safety 128 for name, val in citems(N.__dict__): 129 if ( (callable(val) or type(val)==float) and 130 name not in __builtins__ and 131 name[:1] != '_' and name[-1:] != '_' ): 132 c[name] = val 133 134 # safe functions 135 c['os_path_join'] = os.path.join 136 c['os_path_dirname'] = os.path.dirname 137 c['veusz_markercodes'] = tuple(utils.MarkerCodes) 138 139 # helpful functions for expansion 140 c['ENVIRON'] = dict(os.environ) 141 c['DATE'] = self._evalformatdate 142 c['TIME'] = self._evalformattime 143 c['DATA'] = self._evaldata 144 c['FILENAME'] = self._evalfilename 145 c['BASENAME'] = self._evalbasename 146 c['ESCAPE'] = utils.latexEscape 147 c['SETTING'] = self._evalsetting 148 c['LANG'] = self._evallang 149 150 for name, val in self.def_imports: 151 self._updateImport(name, val) 152 153 for name, val in self.def_definitions: 154 self._updateDefinition(name, val) 155 156 self.colors.wipe() 157 for name, val in self.def_colors: 158 self.colors.addColor(name, val) 159 self.colors.updateModel() 160 161 self.colormaps.wipe() 162 for name, val in self.def_colormaps: 163 self._updateColormap(name, val) 164 165 def setSecurity(self, secure): 166 """Updated the security context.""" 167 oldsecure = getattr(self, 'secure_document', False) 168 169 self.secure_document = secure 170 self.doc.sigSecuritySet.emit(secure) 171 172 if not oldsecure and secure: 173 # if we're now secure, and were not previously, update 174 # context 175 self.exprdscache = {} 176 self.exprdscachechangeset = None 177 self.update() 178 179 def updateSecurityFromPath(self): 180 """Make document secure if in a secure location.""" 181 filename = self.doc.filename 182 absfilename = os.path.abspath(filename) 183 paths = setting.settingdb['secure_dirs'] + [ 184 utils.exampleDirectory] 185 for dirname in paths: 186 absdirname = os.path.abspath(dirname) 187 if absfilename.startswith(absdirname + os.sep): 188 self.setSecurity(True) 189 190 def inSecureMode(self): 191 """Is the document in a safe location?""" 192 return ( 193 setting.transient_settings['unsafe_mode'] or 194 self.secure_document 195 ) 196 197 def _updateImport(self, module, val): 198 """Add an import statement to the eval function context.""" 199 if module_re.match(module): 200 # work out what is safe to import 201 symbols = identifier_split_re.findall(val) 202 if self._checkImportsSafe(): 203 if symbols: 204 defn = 'from %s import %s' % ( 205 module, ', '.join(symbols)) 206 try: 207 cexec(defn, self.context) 208 except Exception: 209 self.doc.log(_( 210 "Failed to import '%s' from module '%s'") % ( 211 ', '.join(toimport), module)) 212 return 213 else: 214 defn = 'import %s' % module 215 try: 216 cexec(defn, self.context) 217 except Exception: 218 self.doc.log(_( 219 "Failed to import module '%s'") % module) 220 return 221 else: 222 if not symbols: 223 self.doc.log(_("Did not import module '%s'") % module) 224 else: 225 self.doc.log(_( 226 "Did not import '%s' from module '%s'") % ( 227 ', '.join(list(symbols)), module)) 228 229 else: 230 self.doc.log( _("Invalid module name '%s'") % module ) 231 232 def validateProcessColormap(self, colormap): 233 """Validate and process a colormap value. 234 235 Returns a list of B,G,R,alpha tuples or raises ValueError if a problem.""" 236 237 try: 238 if len(colormap) < 2: 239 raise ValueError( _("Need at least two entries in colormap") ) 240 except TypeError: 241 raise ValueError( _("Invalid type for colormap") ) 242 243 out = [] 244 for entry in colormap: 245 if entry == (-1,0,0,0): 246 out.append(entry) 247 continue 248 249 for v in entry: 250 try: 251 v - 0 252 except TypeError: 253 raise ValueError( 254 _("Colormap entries should be numerical") ) 255 if v < 0 or v > 255: 256 raise ValueError( 257 _("Colormap entries should be between 0 and 255") ) 258 259 if len(entry) == 3: 260 out.append( (int(entry[2]), int(entry[1]), int(entry[0]), 261 255) ) 262 elif len(entry) == 4: 263 out.append( (int(entry[2]), int(entry[1]), int(entry[0]), 264 int(entry[3])) ) 265 else: 266 raise ValueError( _("Each colormap entry consists of R,G,B " 267 "and optionally alpha values") ) 268 269 return tuple(out) 270 271 def _updateColormap(self, name, val): 272 """Add a colormap entry.""" 273 274 try: 275 cmap = self.validateProcessColormap(val) 276 except ValueError as e: 277 self.doc.log( cstr(e) ) 278 else: 279 self.colormaps[ cstr(name) ] = cmap 280 281 def _updateDefinition(self, name, val): 282 """Update a function or constant in eval function context.""" 283 284 if identifier_re.match(name): 285 defn = val 286 else: 287 m = function_re.match(name) 288 if not m: 289 self.doc.log( 290 _("Invalid function or constant specification '%s'") % 291 name) 292 return 293 name = m.group(1) 294 args = m.group(2) 295 defn = 'lambda %s: %s' % (args, val) 296 297 # evaluate, but we ignore any unsafe commands or exceptions 298 comp = self.compileCheckedExpression(defn) 299 if comp is None: 300 return 301 try: 302 self.context[name] = eval(comp, self.context) 303 except Exception as e: 304 self.doc.log( _( 305 "Error evaluating '%s': '%s'") % (name, cstr(e)) ) 306 307 def compileCheckedExpression(self, expr, origexpr=None, log=True): 308 """Compile expression and check for errors. 309 310 origexpr is an expression to show in error messages. This is 311 used if replacements have been done, etc. 312 """ 313 314 try: 315 return self.compiled[expr] 316 except KeyError: 317 pass 318 319 # track failed compilations, so we only print them once 320 if self.compfailedchangeset != self.doc.changeset: 321 self.compfailedchangeset = self.doc.changeset 322 self.compfailed.clear() 323 elif expr in self.compfailed: 324 return None 325 326 if origexpr is None: 327 origexpr = expr 328 329 try: 330 checked = utils.compileChecked( 331 expr, 332 ignoresecurity=self.inSecureMode(), 333 ) 334 except utils.SafeEvalException as e: 335 if log: 336 self.doc.log( 337 _("Unsafe expression '%s': %s") % (origexpr, cstr(e))) 338 self.compfailed.add(expr) 339 return None 340 except Exception as e: 341 if log: 342 self.doc.log( 343 _("Error in expression '%s': %s") % (origexpr, cstr(e))) 344 return None 345 else: 346 self.compiled[expr] = checked 347 return checked 348 349 @staticmethod 350 def _evalformatdate(fmt=None): 351 """DATE() eval: return date with optional format.""" 352 d = datetime.date.today() 353 return d.isoformat() if fmt is None else d.strftime(fmt) 354 355 @staticmethod 356 def _evalformattime(fmt=None): 357 """TIME() eval: return time with optional format.""" 358 t = datetime.datetime.now() 359 return t.isoformat() if fmt is None else t.strftime(fmt) 360 361 def _evaldata(self, name, part='data'): 362 """DATA(name, [part]) eval: return dataset as array.""" 363 if part not in ('data', 'perr', 'serr', 'nerr'): 364 raise RuntimeError("Invalid dataset part '%s'" % part) 365 if name not in self.doc.data: 366 raise RuntimeError("Dataset '%s' does not exist" % name) 367 data = getattr(self.doc.data[name], part) 368 369 if isinstance(data, N.ndarray): 370 return N.array(data) 371 elif isinstance(data, list): 372 return list(data) 373 return data 374 375 def _evalfilename(self): 376 """FILENAME() eval: returns filename.""" 377 return utils.latexEscape(self.doc.filename) 378 379 def _evalbasename(self): 380 """BASENAME() eval: returns base filename.""" 381 return utils.latexEscape(os.path.basename(self.doc.filename)) 382 383 def _evalsetting(self, path): 384 """SETTING() eval: return setting given full path.""" 385 return self.doc.resolveSettingPath(None, path).get() 386 387 @staticmethod 388 def _evallang(opts): 389 lang = qt.QLocale().name() 390 if lang in opts: 391 return opts[lang] 392 majorl = lang.split('_')[0] 393 if majorl in opts: 394 return opts[majorl] 395 if 'default' in opts: 396 return opts['default'] 397 return utils.latexEscape('NOLANG:%s' % str(lang)) 398 399 def evalDatasetExpression(self, expr, part='data', datatype='numeric', 400 dimensions=1): 401 """Return dataset after evaluating a dataset expression. 402 part is 'data', 'serr', 'perr' or 'nerr' - these are the 403 dataset parts which are evaluated by the expression 404 405 None is returned on error 406 """ 407 408 key = (expr, part, datatype, dimensions) 409 if self.exprdscachechangeset != self.doc.changeset: 410 self.exprdscachechangeset = self.doc.changeset 411 self.exprdscache.clear() 412 elif key in self.exprdscache: 413 return self.exprdscache[key] 414 415 self.exprdscache[key] = ds = datasets.evalDatasetExpression( 416 self.doc, expr, part=part, datatype=datatype, dimensions=dimensions) 417 return ds 418 419 def _checkImportsSafe(self): 420 """Check whether symbols are safe to import.""" 421 422 # do import anyway 423 if self.inSecureMode(): 424 return True 425 426 self.doc.sigAllowedImports.emit() 427 return self.inSecureMode() 428 429 def getColormap(self, name, invert): 430 """Get colormap with name given (returning grey if does not exist).""" 431 cmap = self.colormaps.get(name, self.colormaps['grey']) 432 if invert: 433 if cmap[0][0] >= 0: 434 return cmap[::-1] 435 else: 436 # ignore marker at beginning for stepped maps 437 return tuple([cmap[0]] + list(cmap[-1:0:-1])) 438 return cmap 439 440 def saveCustomDefinitions(self, fileobj): 441 """Save custom constants and functions.""" 442 for ctype, defns in ( 443 ('import', self.def_imports), 444 ('definition', self.def_definitions), 445 ('color', self.def_colors), 446 ('colormap', self.def_colormaps)): 447 448 for val in defns: 449 fileobj.write( 450 'AddCustom(%s, %s, %s)\n' % ( 451 utils.rrepr(ctype), 452 utils.rrepr(val[0]), 453 utils.rrepr(val[1]))) 454 455 def saveCustomFile(self, fileobj): 456 """Export the custom settings to a file.""" 457 458 self.doc._writeFileHeader(fileobj, 'custom definitions') 459 self.saveCustomDefinitions(fileobj) 460