1# -*- coding: utf-8 -*-
2
3"""
4***************************************************************************
5    SagaAlgorithm.py
6    ---------------------
7    Date                 : August 2012
8    Copyright            : (C) 2012 by Victor Olaya
9    Email                : volayaf at gmail dot com
10***************************************************************************
11*                                                                         *
12*   This program is free software; you can redistribute it and/or modify  *
13*   it under the terms of the GNU General Public License as published by  *
14*   the Free Software Foundation; either version 2 of the License, or     *
15*   (at your option) any later version.                                   *
16*                                                                         *
17***************************************************************************
18"""
19
20__author__ = 'Victor Olaya'
21__date__ = 'August 2012'
22__copyright__ = '(C) 2012, Victor Olaya'
23
24import os
25import shutil
26import importlib
27from qgis.core import (Qgis,
28                       QgsApplication,
29                       QgsProcessingUtils,
30                       QgsProcessingException,
31                       QgsMessageLog,
32                       QgsProcessing,
33                       QgsProcessingAlgorithm,
34                       QgsProcessingParameterRasterLayer,
35                       QgsProcessingParameterFeatureSource,
36                       QgsProcessingParameterBoolean,
37                       QgsProcessingParameterNumber,
38                       QgsProcessingParameterEnum,
39                       QgsProcessingParameterMultipleLayers,
40                       QgsProcessingParameterMatrix,
41                       QgsProcessingParameterString,
42                       QgsProcessingParameterField,
43                       QgsProcessingParameterFile,
44                       QgsProcessingParameterExtent,
45                       QgsProcessingParameterRasterDestination,
46                       QgsProcessingParameterVectorDestination)
47from processing.core.ProcessingConfig import ProcessingConfig
48from processing.algs.help import shortHelp
49from processing.tools.system import getTempFilename
50from processing.algs.saga.SagaNameDecorator import decoratedAlgorithmName, decoratedGroupName
51from processing.algs.saga.SagaParameters import Parameters
52from . import SagaUtils
53from .SagaAlgorithmBase import SagaAlgorithmBase
54
55pluginPath = os.path.normpath(os.path.join(
56    os.path.split(os.path.dirname(__file__))[0], os.pardir))
57
58sessionExportedLayers = {}
59
60
61class SagaAlgorithm(SagaAlgorithmBase):
62    OUTPUT_EXTENT = 'OUTPUT_EXTENT'
63
64    def __init__(self, descriptionfile):
65        super().__init__()
66        self.hardcoded_strings = []
67        self.allow_nonmatching_grid_extents = False
68        self.description_file = descriptionfile
69        self.undecorated_group = None
70        self._name = ''
71        self._display_name = ''
72        self._group = ''
73        self._groupId = ''
74        self.params = []
75        self.known_issues = False
76        self.defineCharacteristicsFromFile()
77
78    def createInstance(self):
79        return SagaAlgorithm(self.description_file)
80
81    def initAlgorithm(self, config=None):
82        for p in self.params:
83            self.addParameter(p)
84
85    def name(self):
86        return self._name
87
88    def displayName(self):
89        return self._display_name
90
91    def group(self):
92        return self._group
93
94    def groupId(self):
95        return self._groupId
96
97    def shortHelpString(self):
98        return shortHelp.get(self.id(), None)
99
100    def icon(self):
101        return QgsApplication.getThemeIcon("/providerSaga.svg")
102
103    def svgIconPath(self):
104        return QgsApplication.iconPath("providerSaga.svg")
105
106    def flags(self):
107        # TODO - maybe it's safe to background thread this?
108        f = super().flags() | QgsProcessingAlgorithm.FlagNoThreading
109        if self.known_issues:
110            f = f | QgsProcessingAlgorithm.FlagKnownIssues
111        return f
112
113    def defineCharacteristicsFromFile(self):
114        with open(self.description_file, encoding="utf-8") as lines:
115            line = lines.readline().strip('\n').strip()
116
117            self._name = line
118            if '|' in self._name:
119                tokens = self._name.split('|')
120                self._name = tokens[0]
121                # cmdname is the name of the algorithm in SAGA, that is, the name to use to call it in the console
122                self.cmdname = tokens[1]
123
124            else:
125                self.cmdname = self._name
126                self._display_name = self.tr(str(self._name))
127            self._name = decoratedAlgorithmName(self._name)
128            self._display_name = self.tr(str(self._name))
129
130            self._name = self._name.lower()
131            validChars = \
132                'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789:'
133            self._name = ''.join(c for c in self._name if c in validChars)
134
135            line = lines.readline().strip('\n').strip()
136            if line == '##known_issues':
137                self.known_issues = True
138                line = lines.readline().strip('\n').strip()
139
140            self.undecorated_group = line
141            self._group = self.tr(decoratedGroupName(self.undecorated_group))
142
143            validChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789:'
144            grpName = decoratedGroupName(self.undecorated_group).lower()
145            self._groupId = ''.join(c for c in grpName if c in validChars)
146            line = lines.readline().strip('\n').strip()
147            while line != '':
148                if line.startswith('Hardcoded'):
149                    self.hardcoded_strings.append(line[len('Hardcoded|'):])
150                elif Parameters.is_parameter_line(line):
151                    self.params.append(Parameters.create_parameter_from_line(line))
152                elif line.startswith('AllowUnmatching'):
153                    self.allow_nonmatching_grid_extents = True
154                else:
155                    pass  # TODO
156                    # self.addOutput(getOutputFromString(line))
157                line = lines.readline().strip('\n').strip()
158
159    def processAlgorithm(self, parameters, context, feedback):
160        commands = list()
161        self.exportedLayers = {}
162
163        self.preProcessInputs()
164        extent = None
165        crs = None
166
167        # 1: Export rasters to sgrd and vectors to shp
168        # Tables must be in dbf format. We check that.
169        for param in self.parameterDefinitions():
170            if isinstance(param, QgsProcessingParameterRasterLayer):
171                if param.name() not in parameters or parameters[param.name()] is None:
172                    continue
173
174                if isinstance(parameters[param.name()], str):
175                    if parameters[param.name()].lower().endswith('sdat'):
176                        self.exportedLayers[param.name()] = parameters[param.name()][:-4] + 'sgrd'
177                    elif parameters[param.name()].lower().endswith('sgrd'):
178                        self.exportedLayers[param.name()] = parameters[param.name()]
179                    else:
180                        layer = self.parameterAsRasterLayer(parameters, param.name(), context)
181                        exportCommand = self.exportRasterLayer(param.name(), layer)
182                        if exportCommand is not None:
183                            commands.append(exportCommand)
184                else:
185                    if parameters[param.name()].source().lower().endswith('sdat'):
186                        self.exportedLayers[param.name()] = parameters[param.name()].source()[:-4] + 'sgrd'
187                    if parameters[param.name()].source().lower().endswith('sgrd'):
188                        self.exportedLayers[param.name()] = parameters[param.name()].source()
189                    else:
190                        exportCommand = self.exportRasterLayer(param.name(), parameters[param.name()])
191                        if exportCommand is not None:
192                            commands.append(exportCommand)
193            elif isinstance(param, QgsProcessingParameterFeatureSource):
194                if param.name() not in parameters or parameters[param.name()] is None:
195                    continue
196
197                if not crs:
198                    source = self.parameterAsSource(parameters, param.name(), context)
199                    if source is None:
200                        raise QgsProcessingException(self.invalidSourceError(parameters, param.name()))
201
202                    crs = source.sourceCrs()
203
204                layer_path = self.parameterAsCompatibleSourceLayerPath(parameters, param.name(), context, ['shp'], 'shp', feedback=feedback)
205                if layer_path:
206                    self.exportedLayers[param.name()] = layer_path
207                else:
208                    raise QgsProcessingException(
209                        self.tr('Unsupported file format'))
210            elif isinstance(param, QgsProcessingParameterMultipleLayers):
211                if param.name() not in parameters or parameters[param.name()] is None:
212                    continue
213
214                layers = self.parameterAsLayerList(parameters, param.name(), context)
215                if layers is None or len(layers) == 0:
216                    continue
217                if param.layerType() == QgsProcessing.TypeRaster:
218                    files = []
219                    for i, layer in enumerate(layers):
220                        if layer.source().lower().endswith('sdat'):
221                            files.append(layer.source()[:-4] + 'sgrd')
222                        elif layer.source().lower().endswith('sgrd'):
223                            files.append(layer.source())
224                        else:
225                            exportCommand = self.exportRasterLayer(param.name(), layer)
226                            files.append(self.exportedLayers[param.name()])
227                            if exportCommand is not None:
228                                commands.append(exportCommand)
229
230                    self.exportedLayers[param.name()] = files
231                else:
232                    for layer in layers:
233                        temp_params = {}
234                        temp_params[param.name()] = layer
235
236                        if not crs:
237                            source = self.parameterAsSource(temp_params, param.name(), context)
238                            if source is None:
239                                raise QgsProcessingException(self.invalidSourceError(parameters, param.name()))
240
241                            crs = source.sourceCrs()
242
243                        layer_path = self.parameterAsCompatibleSourceLayerPath(temp_params, param.name(), context, ['shp'], 'shp',
244                                                                               feedback=feedback)
245                        if layer_path:
246                            if param.name() in self.exportedLayers:
247                                self.exportedLayers[param.name()].append(layer_path)
248                            else:
249                                self.exportedLayers[param.name()] = [layer_path]
250                        else:
251                            raise QgsProcessingException(
252                                self.tr('Unsupported file format'))
253
254        # 2: Set parameters and outputs
255        command = self.undecorated_group + ' "' + self.cmdname + '"'
256        command += ' ' + ' '.join(self.hardcoded_strings)
257
258        for param in self.parameterDefinitions():
259            if not param.name() in parameters or parameters[param.name()] is None:
260                continue
261            if param.isDestination():
262                continue
263
264            if isinstance(param, (QgsProcessingParameterRasterLayer, QgsProcessingParameterFeatureSource)):
265                command += ' -{} "{}"'.format(param.name(), self.exportedLayers[param.name()])
266            elif isinstance(param, QgsProcessingParameterMultipleLayers):
267                if parameters[param.name()]:  # parameter may have been an empty list
268                    command += ' -{} "{}"'.format(param.name(), ';'.join(self.exportedLayers[param.name()]))
269            elif isinstance(param, QgsProcessingParameterBoolean):
270                if self.parameterAsBoolean(parameters, param.name(), context):
271                    command += ' -{} true'.format(param.name().strip())
272                else:
273                    command += ' -{} false'.format(param.name().strip())
274            elif isinstance(param, QgsProcessingParameterMatrix):
275                tempTableFile = getTempFilename('txt')
276                with open(tempTableFile, 'w') as f:
277                    f.write('\t'.join([col for col in param.headers()]) + '\n')
278                    values = self.parameterAsMatrix(parameters, param.name(), context)
279                    for i in range(0, len(values), 3):
280                        s = '{}\t{}\t{}\n'.format(values[i], values[i + 1], values[i + 2])
281                        f.write(s)
282                command += ' -{} "{}"'.format(param.name(), tempTableFile)
283            elif isinstance(param, QgsProcessingParameterExtent):
284                # 'We have to subtract/add half cell size, since SAGA is
285                # center based, not corner based
286                halfcell = self.getOutputCellsize(parameters, context) / 2
287                offset = [halfcell, -halfcell, halfcell, -halfcell]
288                rect = self.parameterAsExtent(parameters, param.name(), context)
289
290                values = []
291                values.append(rect.xMinimum())
292                values.append(rect.xMaximum())
293                values.append(rect.yMinimum())
294                values.append(rect.yMaximum())
295
296                for i in range(4):
297                    command += ' -{} {}'.format(param.name().split(' ')[i], float(values[i]) + offset[i])
298            elif isinstance(param, QgsProcessingParameterNumber):
299                if param.dataType() == QgsProcessingParameterNumber.Integer:
300                    command += ' -{} {}'.format(param.name(), self.parameterAsInt(parameters, param.name(), context))
301                else:
302                    command += ' -{} {}'.format(param.name(), self.parameterAsDouble(parameters, param.name(), context))
303            elif isinstance(param, QgsProcessingParameterEnum):
304                command += ' -{} {}'.format(param.name(), self.parameterAsEnum(parameters, param.name(), context))
305            elif isinstance(param, (QgsProcessingParameterString, QgsProcessingParameterFile)):
306                command += ' -{} "{}"'.format(param.name(), self.parameterAsFile(parameters, param.name(), context))
307            elif isinstance(param, (QgsProcessingParameterString, QgsProcessingParameterField)):
308                command += ' -{} "{}"'.format(param.name(), self.parameterAsString(parameters, param.name(), context))
309
310        output_layers = []
311        output_files = {}
312        # If the user has entered an output file that has non-ascii chars, we use a different path with only ascii chars
313        output_files_nonascii = {}
314        for out in self.destinationParameterDefinitions():
315            filePath = self.parameterAsOutputLayer(parameters, out.name(), context)
316            if isinstance(out, (QgsProcessingParameterRasterDestination, QgsProcessingParameterVectorDestination)):
317                output_layers.append(filePath)
318                try:
319                    filePath.encode('ascii')
320                except UnicodeEncodeError:
321                    nonAsciiFilePath = filePath
322                    filePath = QgsProcessingUtils.generateTempFilename(out.name() + os.path.splitext(filePath)[1])
323                    output_files_nonascii[filePath] = nonAsciiFilePath
324
325            output_files[out.name()] = filePath
326            command += ' -{} "{}"'.format(out.name(), filePath)
327        commands.append(command)
328
329        # special treatment for RGB algorithm
330        # TODO: improve this and put this code somewhere else
331        if self.cmdname == 'RGB Composite':
332            for out in self.destinationParameterDefinitions():
333                if isinstance(out, QgsProcessingParameterRasterDestination):
334                    filename = self.parameterAsOutputLayer(parameters, out.name(), context)
335                    filename2 = os.path.splitext(filename)[0] + '.sgrd'
336                    commands.append('io_grid_image 0 -COLOURING 4 -GRID:"{}" -FILE:"{}"'.format(filename2, filename))
337
338        # 3: Run SAGA
339        commands = self.editCommands(commands)
340        SagaUtils.createSagaBatchJobFileFromSagaCommands(commands)
341        loglines = []
342        loglines.append(self.tr('SAGA execution commands'))
343        for line in commands:
344            feedback.pushCommandInfo(line)
345            loglines.append(line)
346        if ProcessingConfig.getSetting(SagaUtils.SAGA_LOG_COMMANDS):
347            QgsMessageLog.logMessage('\n'.join(loglines), self.tr('Processing'), Qgis.Info)
348        SagaUtils.executeSaga(feedback)
349
350        if crs is not None:
351            for out in output_layers:
352                prjFile = os.path.splitext(out)[0] + '.prj'
353                with open(prjFile, 'w') as f:
354                    f.write(crs.toWkt())
355
356        for old, new in output_files_nonascii.items():
357            oldFolder = os.path.dirname(old)
358            newFolder = os.path.dirname(new)
359            newName = os.path.splitext(os.path.basename(new))[0]
360            files = [f for f in os.listdir(oldFolder)]
361            for f in files:
362                ext = os.path.splitext(f)[1]
363                newPath = os.path.join(newFolder, newName + ext)
364                oldPath = os.path.join(oldFolder, f)
365                shutil.move(oldPath, newPath)
366
367        result = {}
368        for o in self.outputDefinitions():
369            if o.name() in output_files:
370                result[o.name()] = output_files[o.name()]
371        return result
372
373    def preProcessInputs(self):
374        name = self.name().replace('.', '_')
375        try:
376            module = importlib.import_module('processing.algs.saga.ext.' + name)
377        except ImportError:
378            return
379        if hasattr(module, 'preProcessInputs'):
380            func = getattr(module, 'preProcessInputs')
381            func(self)
382
383    def editCommands(self, commands):
384        try:
385            module = importlib.import_module('processing.algs.saga.ext.' + self.name())
386        except ImportError:
387            return commands
388        if hasattr(module, 'editCommands'):
389            func = getattr(module, 'editCommands')
390            return func(commands)
391        else:
392            return commands
393
394    def getOutputCellsize(self, parameters, context):
395        """Tries to guess the cell size of the output, searching for
396        a parameter with an appropriate name for it.
397        :param parameters:
398        """
399
400        cellsize = 0
401        for param in self.parameterDefinitions():
402            if param.name() in parameters and param.name() == 'USER_SIZE':
403                cellsize = self.parameterAsDouble(parameters, param.name(), context)
404                break
405        return cellsize
406
407    def exportRasterLayer(self, parameterName, layer):
408        global sessionExportedLayers
409        if layer.source() in sessionExportedLayers:
410            exportedLayer = sessionExportedLayers[layer.source()]
411            if os.path.exists(exportedLayer):
412                self.exportedLayers[parameterName] = exportedLayer
413                return None
414            else:
415                del sessionExportedLayers[layer.source()]
416
417        if layer:
418            filename = layer.name()
419        else:
420            filename = os.path.basename(layer.source())
421
422        validChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789:'
423        filename = ''.join(c for c in filename if c in validChars)
424
425        if len(filename) == 0:
426            filename = 'layer'
427
428        destFilename = QgsProcessingUtils.generateTempFilename(filename + '.sgrd')
429        sessionExportedLayers[layer.source()] = destFilename
430        self.exportedLayers[parameterName] = destFilename
431
432        return 'io_gdal 0 -TRANSFORM 1 -RESAMPLING 3 -GRIDS "{}" -FILES "{}"'.format(destFilename, layer.source())
433
434    def checkParameterValues(self, parameters, context):
435        """
436        We check that there are no multiband layers, which are not
437        supported by SAGA, and that raster layers have the same grid extent
438        """
439        extent = None
440        raster_layer_params = []
441        for param in self.parameterDefinitions():
442            if param not in parameters or parameters[param.name()] is None:
443                continue
444
445            if isinstance(param, QgsProcessingParameterRasterLayer):
446                raster_layer_params.append(param.name())
447            elif (isinstance(param, QgsProcessingParameterMultipleLayers)
448                    and param.layerType() == QgsProcessing.TypeRaster):
449                raster_layer_params.extend(param.name())
450
451        for layer_param in raster_layer_params:
452            layer = self.parameterAsRasterLayer(parameters, layer_param, context)
453
454            if layer is None:
455                continue
456            if layer.bandCount() > 1:
457                return False, self.tr('Input layer {0} has more than one band.\n'
458                                      'Multiband layers are not supported by SAGA').format(layer.name())
459            if not self.allow_nonmatching_grid_extents:
460                if extent is None:
461                    extent = (layer.extent(), layer.height(), layer.width())
462                else:
463                    extent2 = (layer.extent(), layer.height(), layer.width())
464                    if extent != extent2:
465                        return False, self.tr("Input layers do not have the same grid extent.")
466        return super(SagaAlgorithm, self).checkParameterValues(parameters, context)
467