1# -*- coding: utf-8 -*- 2 3""" 4*************************************************************************** 5 SagaAlgorithm.py 6 --------------------- 7 Date : August 2012 8 Copyright : (C) 2012 by Victor Olaya 9 Email : volayaf at gmail dot com 10*************************************************************************** 11* * 12* This program is free software; you can redistribute it and/or modify * 13* it under the terms of the GNU General Public License as published by * 14* the Free Software Foundation; either version 2 of the License, or * 15* (at your option) any later version. * 16* * 17*************************************************************************** 18""" 19 20__author__ = 'Victor Olaya' 21__date__ = 'August 2012' 22__copyright__ = '(C) 2012, Victor Olaya' 23 24import os 25import shutil 26import importlib 27from qgis.core import (Qgis, 28 QgsApplication, 29 QgsProcessingUtils, 30 QgsProcessingException, 31 QgsMessageLog, 32 QgsProcessing, 33 QgsProcessingAlgorithm, 34 QgsProcessingParameterRasterLayer, 35 QgsProcessingParameterFeatureSource, 36 QgsProcessingParameterBoolean, 37 QgsProcessingParameterNumber, 38 QgsProcessingParameterEnum, 39 QgsProcessingParameterMultipleLayers, 40 QgsProcessingParameterMatrix, 41 QgsProcessingParameterString, 42 QgsProcessingParameterField, 43 QgsProcessingParameterFile, 44 QgsProcessingParameterExtent, 45 QgsProcessingParameterRasterDestination, 46 QgsProcessingParameterVectorDestination) 47from processing.core.ProcessingConfig import ProcessingConfig 48from processing.algs.help import shortHelp 49from processing.tools.system import getTempFilename 50from processing.algs.saga.SagaNameDecorator import decoratedAlgorithmName, decoratedGroupName 51from processing.algs.saga.SagaParameters import Parameters 52from . import SagaUtils 53from .SagaAlgorithmBase import SagaAlgorithmBase 54 55pluginPath = os.path.normpath(os.path.join( 56 os.path.split(os.path.dirname(__file__))[0], os.pardir)) 57 58sessionExportedLayers = {} 59 60 61class SagaAlgorithm(SagaAlgorithmBase): 62 OUTPUT_EXTENT = 'OUTPUT_EXTENT' 63 64 def __init__(self, descriptionfile): 65 super().__init__() 66 self.hardcoded_strings = [] 67 self.allow_nonmatching_grid_extents = False 68 self.description_file = descriptionfile 69 self.undecorated_group = None 70 self._name = '' 71 self._display_name = '' 72 self._group = '' 73 self._groupId = '' 74 self.params = [] 75 self.known_issues = False 76 self.defineCharacteristicsFromFile() 77 78 def createInstance(self): 79 return SagaAlgorithm(self.description_file) 80 81 def initAlgorithm(self, config=None): 82 for p in self.params: 83 self.addParameter(p) 84 85 def name(self): 86 return self._name 87 88 def displayName(self): 89 return self._display_name 90 91 def group(self): 92 return self._group 93 94 def groupId(self): 95 return self._groupId 96 97 def shortHelpString(self): 98 return shortHelp.get(self.id(), None) 99 100 def icon(self): 101 return QgsApplication.getThemeIcon("/providerSaga.svg") 102 103 def svgIconPath(self): 104 return QgsApplication.iconPath("providerSaga.svg") 105 106 def flags(self): 107 # TODO - maybe it's safe to background thread this? 108 f = super().flags() | QgsProcessingAlgorithm.FlagNoThreading 109 if self.known_issues: 110 f = f | QgsProcessingAlgorithm.FlagKnownIssues 111 return f 112 113 def defineCharacteristicsFromFile(self): 114 with open(self.description_file, encoding="utf-8") as lines: 115 line = lines.readline().strip('\n').strip() 116 117 self._name = line 118 if '|' in self._name: 119 tokens = self._name.split('|') 120 self._name = tokens[0] 121 # cmdname is the name of the algorithm in SAGA, that is, the name to use to call it in the console 122 self.cmdname = tokens[1] 123 124 else: 125 self.cmdname = self._name 126 self._display_name = self.tr(str(self._name)) 127 self._name = decoratedAlgorithmName(self._name) 128 self._display_name = self.tr(str(self._name)) 129 130 self._name = self._name.lower() 131 validChars = \ 132 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789:' 133 self._name = ''.join(c for c in self._name if c in validChars) 134 135 line = lines.readline().strip('\n').strip() 136 if line == '##known_issues': 137 self.known_issues = True 138 line = lines.readline().strip('\n').strip() 139 140 self.undecorated_group = line 141 self._group = self.tr(decoratedGroupName(self.undecorated_group)) 142 143 validChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789:' 144 grpName = decoratedGroupName(self.undecorated_group).lower() 145 self._groupId = ''.join(c for c in grpName if c in validChars) 146 line = lines.readline().strip('\n').strip() 147 while line != '': 148 if line.startswith('Hardcoded'): 149 self.hardcoded_strings.append(line[len('Hardcoded|'):]) 150 elif Parameters.is_parameter_line(line): 151 self.params.append(Parameters.create_parameter_from_line(line)) 152 elif line.startswith('AllowUnmatching'): 153 self.allow_nonmatching_grid_extents = True 154 else: 155 pass # TODO 156 # self.addOutput(getOutputFromString(line)) 157 line = lines.readline().strip('\n').strip() 158 159 def processAlgorithm(self, parameters, context, feedback): 160 commands = list() 161 self.exportedLayers = {} 162 163 self.preProcessInputs() 164 extent = None 165 crs = None 166 167 # 1: Export rasters to sgrd and vectors to shp 168 # Tables must be in dbf format. We check that. 169 for param in self.parameterDefinitions(): 170 if isinstance(param, QgsProcessingParameterRasterLayer): 171 if param.name() not in parameters or parameters[param.name()] is None: 172 continue 173 174 if isinstance(parameters[param.name()], str): 175 if parameters[param.name()].lower().endswith('sdat'): 176 self.exportedLayers[param.name()] = parameters[param.name()][:-4] + 'sgrd' 177 elif parameters[param.name()].lower().endswith('sgrd'): 178 self.exportedLayers[param.name()] = parameters[param.name()] 179 else: 180 layer = self.parameterAsRasterLayer(parameters, param.name(), context) 181 exportCommand = self.exportRasterLayer(param.name(), layer) 182 if exportCommand is not None: 183 commands.append(exportCommand) 184 else: 185 if parameters[param.name()].source().lower().endswith('sdat'): 186 self.exportedLayers[param.name()] = parameters[param.name()].source()[:-4] + 'sgrd' 187 if parameters[param.name()].source().lower().endswith('sgrd'): 188 self.exportedLayers[param.name()] = parameters[param.name()].source() 189 else: 190 exportCommand = self.exportRasterLayer(param.name(), parameters[param.name()]) 191 if exportCommand is not None: 192 commands.append(exportCommand) 193 elif isinstance(param, QgsProcessingParameterFeatureSource): 194 if param.name() not in parameters or parameters[param.name()] is None: 195 continue 196 197 if not crs: 198 source = self.parameterAsSource(parameters, param.name(), context) 199 if source is None: 200 raise QgsProcessingException(self.invalidSourceError(parameters, param.name())) 201 202 crs = source.sourceCrs() 203 204 layer_path = self.parameterAsCompatibleSourceLayerPath(parameters, param.name(), context, ['shp'], 'shp', feedback=feedback) 205 if layer_path: 206 self.exportedLayers[param.name()] = layer_path 207 else: 208 raise QgsProcessingException( 209 self.tr('Unsupported file format')) 210 elif isinstance(param, QgsProcessingParameterMultipleLayers): 211 if param.name() not in parameters or parameters[param.name()] is None: 212 continue 213 214 layers = self.parameterAsLayerList(parameters, param.name(), context) 215 if layers is None or len(layers) == 0: 216 continue 217 if param.layerType() == QgsProcessing.TypeRaster: 218 files = [] 219 for i, layer in enumerate(layers): 220 if layer.source().lower().endswith('sdat'): 221 files.append(layer.source()[:-4] + 'sgrd') 222 elif layer.source().lower().endswith('sgrd'): 223 files.append(layer.source()) 224 else: 225 exportCommand = self.exportRasterLayer(param.name(), layer) 226 files.append(self.exportedLayers[param.name()]) 227 if exportCommand is not None: 228 commands.append(exportCommand) 229 230 self.exportedLayers[param.name()] = files 231 else: 232 for layer in layers: 233 temp_params = {} 234 temp_params[param.name()] = layer 235 236 if not crs: 237 source = self.parameterAsSource(temp_params, param.name(), context) 238 if source is None: 239 raise QgsProcessingException(self.invalidSourceError(parameters, param.name())) 240 241 crs = source.sourceCrs() 242 243 layer_path = self.parameterAsCompatibleSourceLayerPath(temp_params, param.name(), context, ['shp'], 'shp', 244 feedback=feedback) 245 if layer_path: 246 if param.name() in self.exportedLayers: 247 self.exportedLayers[param.name()].append(layer_path) 248 else: 249 self.exportedLayers[param.name()] = [layer_path] 250 else: 251 raise QgsProcessingException( 252 self.tr('Unsupported file format')) 253 254 # 2: Set parameters and outputs 255 command = self.undecorated_group + ' "' + self.cmdname + '"' 256 command += ' ' + ' '.join(self.hardcoded_strings) 257 258 for param in self.parameterDefinitions(): 259 if not param.name() in parameters or parameters[param.name()] is None: 260 continue 261 if param.isDestination(): 262 continue 263 264 if isinstance(param, (QgsProcessingParameterRasterLayer, QgsProcessingParameterFeatureSource)): 265 command += ' -{} "{}"'.format(param.name(), self.exportedLayers[param.name()]) 266 elif isinstance(param, QgsProcessingParameterMultipleLayers): 267 if parameters[param.name()]: # parameter may have been an empty list 268 command += ' -{} "{}"'.format(param.name(), ';'.join(self.exportedLayers[param.name()])) 269 elif isinstance(param, QgsProcessingParameterBoolean): 270 if self.parameterAsBoolean(parameters, param.name(), context): 271 command += ' -{} true'.format(param.name().strip()) 272 else: 273 command += ' -{} false'.format(param.name().strip()) 274 elif isinstance(param, QgsProcessingParameterMatrix): 275 tempTableFile = getTempFilename('txt') 276 with open(tempTableFile, 'w') as f: 277 f.write('\t'.join([col for col in param.headers()]) + '\n') 278 values = self.parameterAsMatrix(parameters, param.name(), context) 279 for i in range(0, len(values), 3): 280 s = '{}\t{}\t{}\n'.format(values[i], values[i + 1], values[i + 2]) 281 f.write(s) 282 command += ' -{} "{}"'.format(param.name(), tempTableFile) 283 elif isinstance(param, QgsProcessingParameterExtent): 284 # 'We have to subtract/add half cell size, since SAGA is 285 # center based, not corner based 286 halfcell = self.getOutputCellsize(parameters, context) / 2 287 offset = [halfcell, -halfcell, halfcell, -halfcell] 288 rect = self.parameterAsExtent(parameters, param.name(), context) 289 290 values = [] 291 values.append(rect.xMinimum()) 292 values.append(rect.xMaximum()) 293 values.append(rect.yMinimum()) 294 values.append(rect.yMaximum()) 295 296 for i in range(4): 297 command += ' -{} {}'.format(param.name().split(' ')[i], float(values[i]) + offset[i]) 298 elif isinstance(param, QgsProcessingParameterNumber): 299 if param.dataType() == QgsProcessingParameterNumber.Integer: 300 command += ' -{} {}'.format(param.name(), self.parameterAsInt(parameters, param.name(), context)) 301 else: 302 command += ' -{} {}'.format(param.name(), self.parameterAsDouble(parameters, param.name(), context)) 303 elif isinstance(param, QgsProcessingParameterEnum): 304 command += ' -{} {}'.format(param.name(), self.parameterAsEnum(parameters, param.name(), context)) 305 elif isinstance(param, (QgsProcessingParameterString, QgsProcessingParameterFile)): 306 command += ' -{} "{}"'.format(param.name(), self.parameterAsFile(parameters, param.name(), context)) 307 elif isinstance(param, (QgsProcessingParameterString, QgsProcessingParameterField)): 308 command += ' -{} "{}"'.format(param.name(), self.parameterAsString(parameters, param.name(), context)) 309 310 output_layers = [] 311 output_files = {} 312 # If the user has entered an output file that has non-ascii chars, we use a different path with only ascii chars 313 output_files_nonascii = {} 314 for out in self.destinationParameterDefinitions(): 315 filePath = self.parameterAsOutputLayer(parameters, out.name(), context) 316 if isinstance(out, (QgsProcessingParameterRasterDestination, QgsProcessingParameterVectorDestination)): 317 output_layers.append(filePath) 318 try: 319 filePath.encode('ascii') 320 except UnicodeEncodeError: 321 nonAsciiFilePath = filePath 322 filePath = QgsProcessingUtils.generateTempFilename(out.name() + os.path.splitext(filePath)[1]) 323 output_files_nonascii[filePath] = nonAsciiFilePath 324 325 output_files[out.name()] = filePath 326 command += ' -{} "{}"'.format(out.name(), filePath) 327 commands.append(command) 328 329 # special treatment for RGB algorithm 330 # TODO: improve this and put this code somewhere else 331 if self.cmdname == 'RGB Composite': 332 for out in self.destinationParameterDefinitions(): 333 if isinstance(out, QgsProcessingParameterRasterDestination): 334 filename = self.parameterAsOutputLayer(parameters, out.name(), context) 335 filename2 = os.path.splitext(filename)[0] + '.sgrd' 336 commands.append('io_grid_image 0 -COLOURING 4 -GRID:"{}" -FILE:"{}"'.format(filename2, filename)) 337 338 # 3: Run SAGA 339 commands = self.editCommands(commands) 340 SagaUtils.createSagaBatchJobFileFromSagaCommands(commands) 341 loglines = [] 342 loglines.append(self.tr('SAGA execution commands')) 343 for line in commands: 344 feedback.pushCommandInfo(line) 345 loglines.append(line) 346 if ProcessingConfig.getSetting(SagaUtils.SAGA_LOG_COMMANDS): 347 QgsMessageLog.logMessage('\n'.join(loglines), self.tr('Processing'), Qgis.Info) 348 SagaUtils.executeSaga(feedback) 349 350 if crs is not None: 351 for out in output_layers: 352 prjFile = os.path.splitext(out)[0] + '.prj' 353 with open(prjFile, 'w') as f: 354 f.write(crs.toWkt()) 355 356 for old, new in output_files_nonascii.items(): 357 oldFolder = os.path.dirname(old) 358 newFolder = os.path.dirname(new) 359 newName = os.path.splitext(os.path.basename(new))[0] 360 files = [f for f in os.listdir(oldFolder)] 361 for f in files: 362 ext = os.path.splitext(f)[1] 363 newPath = os.path.join(newFolder, newName + ext) 364 oldPath = os.path.join(oldFolder, f) 365 shutil.move(oldPath, newPath) 366 367 result = {} 368 for o in self.outputDefinitions(): 369 if o.name() in output_files: 370 result[o.name()] = output_files[o.name()] 371 return result 372 373 def preProcessInputs(self): 374 name = self.name().replace('.', '_') 375 try: 376 module = importlib.import_module('processing.algs.saga.ext.' + name) 377 except ImportError: 378 return 379 if hasattr(module, 'preProcessInputs'): 380 func = getattr(module, 'preProcessInputs') 381 func(self) 382 383 def editCommands(self, commands): 384 try: 385 module = importlib.import_module('processing.algs.saga.ext.' + self.name()) 386 except ImportError: 387 return commands 388 if hasattr(module, 'editCommands'): 389 func = getattr(module, 'editCommands') 390 return func(commands) 391 else: 392 return commands 393 394 def getOutputCellsize(self, parameters, context): 395 """Tries to guess the cell size of the output, searching for 396 a parameter with an appropriate name for it. 397 :param parameters: 398 """ 399 400 cellsize = 0 401 for param in self.parameterDefinitions(): 402 if param.name() in parameters and param.name() == 'USER_SIZE': 403 cellsize = self.parameterAsDouble(parameters, param.name(), context) 404 break 405 return cellsize 406 407 def exportRasterLayer(self, parameterName, layer): 408 global sessionExportedLayers 409 if layer.source() in sessionExportedLayers: 410 exportedLayer = sessionExportedLayers[layer.source()] 411 if os.path.exists(exportedLayer): 412 self.exportedLayers[parameterName] = exportedLayer 413 return None 414 else: 415 del sessionExportedLayers[layer.source()] 416 417 if layer: 418 filename = layer.name() 419 else: 420 filename = os.path.basename(layer.source()) 421 422 validChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789:' 423 filename = ''.join(c for c in filename if c in validChars) 424 425 if len(filename) == 0: 426 filename = 'layer' 427 428 destFilename = QgsProcessingUtils.generateTempFilename(filename + '.sgrd') 429 sessionExportedLayers[layer.source()] = destFilename 430 self.exportedLayers[parameterName] = destFilename 431 432 return 'io_gdal 0 -TRANSFORM 1 -RESAMPLING 3 -GRIDS "{}" -FILES "{}"'.format(destFilename, layer.source()) 433 434 def checkParameterValues(self, parameters, context): 435 """ 436 We check that there are no multiband layers, which are not 437 supported by SAGA, and that raster layers have the same grid extent 438 """ 439 extent = None 440 raster_layer_params = [] 441 for param in self.parameterDefinitions(): 442 if param not in parameters or parameters[param.name()] is None: 443 continue 444 445 if isinstance(param, QgsProcessingParameterRasterLayer): 446 raster_layer_params.append(param.name()) 447 elif (isinstance(param, QgsProcessingParameterMultipleLayers) 448 and param.layerType() == QgsProcessing.TypeRaster): 449 raster_layer_params.extend(param.name()) 450 451 for layer_param in raster_layer_params: 452 layer = self.parameterAsRasterLayer(parameters, layer_param, context) 453 454 if layer is None: 455 continue 456 if layer.bandCount() > 1: 457 return False, self.tr('Input layer {0} has more than one band.\n' 458 'Multiband layers are not supported by SAGA').format(layer.name()) 459 if not self.allow_nonmatching_grid_extents: 460 if extent is None: 461 extent = (layer.extent(), layer.height(), layer.width()) 462 else: 463 extent2 = (layer.extent(), layer.height(), layer.width()) 464 if extent != extent2: 465 return False, self.tr("Input layers do not have the same grid extent.") 466 return super(SagaAlgorithm, self).checkParameterValues(parameters, context) 467