1 /******************************************************************************
2  *
3  * Project:  Virtual GDAL Datasets
4  * Purpose:  Implementation of a sourced raster band that derives its raster
5  *           by applying an algorithm (GDALDerivedPixelFunc) to the sources.
6  * Author:   Pete Nagy
7  *
8  ******************************************************************************
9  * Copyright (c) 2005 Vexcel Corp.
10  * Copyright (c) 2008-2011, Even Rouault <even dot rouault at spatialys.com>
11  *
12  * Permission is hereby granted, free of charge, to any person obtaining a
13  * copy of this software and associated documentation files (the "Software"),
14  * to deal in the Software without restriction, including without limitation
15  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16  * and/or sell copies of the Software, and to permit persons to whom the
17  * Software is furnished to do so, subject to the following conditions:
18  *
19  * The above copyright notice and this permission notice shall be included
20  * in all copies or substantial portions of the Software.
21  *
22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28  * DEALINGS IN THE SOFTWARE.
29  *****************************************************************************/
30 
31 #include "cpl_minixml.h"
32 #include "cpl_string.h"
33 #include "vrtdataset.h"
34 #include "cpl_multiproc.h"
35 #include "gdalpython.h"
36 
37 #include <algorithm>
38 #include <map>
39 #include <vector>
40 #include <utility>
41 
42 /*! @cond Doxygen_Suppress */
43 
44 CPL_CVSID("$Id: vrtderivedrasterband.cpp 33f3d049c42d69081955817eedf05c72339ae2ca 2021-02-25 03:00:58 -0500 Elliott Sales de Andrade $")
45 
46 using namespace GDALPy;
47 
48 // #define GDAL_VRT_DISABLE_PYTHON
49 
50 #ifndef GDAL_VRT_ENABLE_PYTHON_DEFAULT
51 // Can be YES, NO or TRUSTED_MODULES
52 #define GDAL_VRT_ENABLE_PYTHON_DEFAULT "TRUSTED_MODULES"
53 #endif
54 
55 static std::map<CPLString, GDALDerivedPixelFunc> osMapPixelFunction;
56 
57 /* Flags for getting buffers */
58 #define PyBUF_WRITABLE 0x0001
59 #define PyBUF_FORMAT 0x0004
60 #define PyBUF_ND 0x0008
61 #define PyBUF_STRIDES (0x0010 | PyBUF_ND)
62 #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES)
63 #define PyBUF_FULL (PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT)
64 
65 /************************************************************************/
66 /*                        GDALCreateNumpyArray()                        */
67 /************************************************************************/
68 
GDALCreateNumpyArray(PyObject * pCreateArray,void * pBuffer,GDALDataType eType,int nHeight,int nWidth)69 static PyObject* GDALCreateNumpyArray(PyObject* pCreateArray,
70                                       void* pBuffer,
71                                       GDALDataType eType,
72                                       int nHeight,
73                                       int nWidth )
74 {
75     PyObject* poPyBuffer;
76     const size_t nSize = static_cast<size_t>(nHeight) * nWidth *
77                                     GDALGetDataTypeSizeBytes(eType);
78     Py_buffer pybuffer;
79     if( PyBuffer_FillInfo(&pybuffer, nullptr, static_cast<char*>(pBuffer),
80                           nSize,
81                           0, PyBUF_FULL) != 0)
82     {
83         return nullptr;
84     }
85     poPyBuffer = PyMemoryView_FromBuffer(&pybuffer);
86     PyObject* pArgsCreateArray = PyTuple_New(4);
87     PyTuple_SetItem(pArgsCreateArray, 0, poPyBuffer);
88     const char* pszDataType = nullptr;
89     switch( eType )
90     {
91         case GDT_Byte: pszDataType = "uint8"; break;
92         case GDT_UInt16: pszDataType = "uint16"; break;
93         case GDT_Int16: pszDataType = "int16"; break;
94         case GDT_UInt32: pszDataType = "uint32"; break;
95         case GDT_Int32: pszDataType = "int32"; break;
96         case GDT_Float32: pszDataType = "float32"; break;
97         case GDT_Float64: pszDataType = "float64"; break;
98         case GDT_CInt16:
99         case GDT_CInt32:
100             CPLAssert(FALSE);
101             break;
102         case GDT_CFloat32: pszDataType = "complex64"; break;
103         case GDT_CFloat64: pszDataType = "complex128"; break;
104         default:
105             CPLAssert(FALSE);
106             break;
107     }
108     PyTuple_SetItem(pArgsCreateArray, 1,
109                 PyBytes_FromStringAndSize(pszDataType, strlen(pszDataType)));
110     PyTuple_SetItem(pArgsCreateArray, 2, PyLong_FromLong(nHeight));
111     PyTuple_SetItem(pArgsCreateArray, 3, PyLong_FromLong(nWidth));
112     PyObject* poNumpyArray = PyObject_Call(pCreateArray, pArgsCreateArray, nullptr);
113     Py_DecRef(pArgsCreateArray);
114     if (PyErr_Occurred())
115         PyErr_Print();
116     return poNumpyArray;
117 }
118 
119 /************************************************************************/
120 /* ==================================================================== */
121 /*                     VRTDerivedRasterBandPrivateData                  */
122 /* ==================================================================== */
123 /************************************************************************/
124 
125 class VRTDerivedRasterBandPrivateData
126 {
127         VRTDerivedRasterBandPrivateData(const VRTDerivedRasterBandPrivateData&) = delete;
128         VRTDerivedRasterBandPrivateData& operator= (const VRTDerivedRasterBandPrivateData&) = delete;
129 
130     public:
131         CPLString m_osCode{};
132         CPLString m_osLanguage;
133         int       m_nBufferRadius;
134         PyObject* m_poGDALCreateNumpyArray;
135         PyObject* m_poUserFunction;
136         bool      m_bPythonInitializationDone;
137         bool      m_bPythonInitializationSuccess;
138         bool      m_bExclusiveLock;
139         bool      m_bFirstTime;
140         std::vector< std::pair<CPLString,CPLString> > m_oFunctionArgs{};
141 
VRTDerivedRasterBandPrivateData()142         VRTDerivedRasterBandPrivateData():
143             m_osLanguage("C"),
144             m_nBufferRadius(0),
145             m_poGDALCreateNumpyArray(nullptr),
146             m_poUserFunction(nullptr),
147             m_bPythonInitializationDone(false),
148             m_bPythonInitializationSuccess(false),
149             m_bExclusiveLock(false),
150             m_bFirstTime(true)
151         {
152         }
153 
~VRTDerivedRasterBandPrivateData()154         virtual ~VRTDerivedRasterBandPrivateData()
155         {
156             if( m_poGDALCreateNumpyArray )
157                 Py_DecRef(m_poGDALCreateNumpyArray);
158             if( m_poUserFunction )
159                 Py_DecRef(m_poUserFunction);
160         }
161 };
162 
163 /************************************************************************/
164 /* ==================================================================== */
165 /*                          VRTDerivedRasterBand                        */
166 /* ==================================================================== */
167 /************************************************************************/
168 
169 /************************************************************************/
170 /*                        VRTDerivedRasterBand()                        */
171 /************************************************************************/
172 
VRTDerivedRasterBand(GDALDataset * poDSIn,int nBandIn)173 VRTDerivedRasterBand::VRTDerivedRasterBand( GDALDataset *poDSIn, int nBandIn ) :
174     VRTSourcedRasterBand( poDSIn, nBandIn ),
175     m_poPrivate(nullptr),
176     pszFuncName(nullptr),
177     eSourceTransferType(GDT_Unknown)
178 {
179     m_poPrivate = new VRTDerivedRasterBandPrivateData;
180 }
181 
182 /************************************************************************/
183 /*                        VRTDerivedRasterBand()                        */
184 /************************************************************************/
185 
VRTDerivedRasterBand(GDALDataset * poDSIn,int nBandIn,GDALDataType eType,int nXSize,int nYSize)186 VRTDerivedRasterBand::VRTDerivedRasterBand( GDALDataset *poDSIn, int nBandIn,
187                                             GDALDataType eType,
188                                             int nXSize, int nYSize ) :
189     VRTSourcedRasterBand(poDSIn, nBandIn, eType, nXSize, nYSize),
190     m_poPrivate(nullptr),
191     pszFuncName(nullptr),
192     eSourceTransferType(GDT_Unknown)
193 {
194     m_poPrivate = new VRTDerivedRasterBandPrivateData;
195 }
196 
197 /************************************************************************/
198 /*                       ~VRTDerivedRasterBand()                        */
199 /************************************************************************/
200 
~VRTDerivedRasterBand()201 VRTDerivedRasterBand::~VRTDerivedRasterBand()
202 
203 {
204     CPLFree( pszFuncName );
205     delete m_poPrivate;
206 }
207 
208 /************************************************************************/
209 /*                               Cleanup()                              */
210 /************************************************************************/
211 
Cleanup()212 void VRTDerivedRasterBand::Cleanup()
213 {
214 }
215 
216 /************************************************************************/
217 /*                           AddPixelFunction()                         */
218 /************************************************************************/
219 
220 /*! @endcond */
221 
222 /**
223  * This adds a pixel function to the global list of available pixel
224  * functions for derived bands.  Pixel functions must be registered
225  * in this way before a derived band tries to access data.
226  *
227  * Derived bands are stored with only the name of the pixel function
228  * that it will apply, and if a pixel function matching the name is not
229  * found the IRasterIO() call will do nothing.
230  *
231  * @param pszFuncName Name used to access pixel function
232  * @param pfnNewFunction Pixel function associated with name.  An
233  *  existing pixel function registered with the same name will be
234  *  replaced with the new one.
235  *
236  * @return CE_None, invalid (NULL) parameters are currently ignored.
237  */
238 CPLErr CPL_STDCALL
GDALAddDerivedBandPixelFunc(const char * pszFuncName,GDALDerivedPixelFunc pfnNewFunction)239 GDALAddDerivedBandPixelFunc( const char *pszFuncName,
240                              GDALDerivedPixelFunc pfnNewFunction )
241 {
242     if( pszFuncName == nullptr || pszFuncName[0] == '\0' ||
243         pfnNewFunction == nullptr )
244     {
245       return CE_None;
246     }
247 
248     osMapPixelFunction[pszFuncName] = pfnNewFunction;
249 
250     return CE_None;
251 }
252 
253 /*! @cond Doxygen_Suppress */
254 
255 /**
256  * This adds a pixel function to the global list of available pixel
257  * functions for derived bands.
258  *
259  * This is the same as the c function GDALAddDerivedBandPixelFunc()
260  *
261  * @param pszFuncName Name used to access pixel function
262  * @param pfnNewFunction Pixel function associated with name.  An
263  *  existing pixel function registered with the same name will be
264  *  replaced with the new one.
265  *
266  * @return CE_None, invalid (NULL) parameters are currently ignored.
267  */
268 CPLErr
AddPixelFunction(const char * pszFuncName,GDALDerivedPixelFunc pfnNewFunction)269 VRTDerivedRasterBand::AddPixelFunction(
270     const char *pszFuncName, GDALDerivedPixelFunc pfnNewFunction )
271 {
272     return GDALAddDerivedBandPixelFunc(pszFuncName, pfnNewFunction);
273 }
274 
275 /************************************************************************/
276 /*                           GetPixelFunction()                         */
277 /************************************************************************/
278 
279 /**
280  * Get a pixel function previously registered using the global
281  * AddPixelFunction.
282  *
283  * @param pszFuncName The name associated with the pixel function.
284  *
285  * @return A derived band pixel function, or NULL if none have been
286  * registered for pszFuncName.
287  */
288 GDALDerivedPixelFunc
GetPixelFunction(const char * pszFuncName)289 VRTDerivedRasterBand::GetPixelFunction( const char *pszFuncName )
290 {
291     if( pszFuncName == nullptr || pszFuncName[0] == '\0' )
292     {
293         return nullptr;
294     }
295 
296     std::map<CPLString, GDALDerivedPixelFunc>::iterator oIter =
297         osMapPixelFunction.find(pszFuncName);
298 
299     if( oIter == osMapPixelFunction.end())
300         return nullptr;
301 
302     return oIter->second;
303 }
304 
305 /************************************************************************/
306 /*                         SetPixelFunctionName()                       */
307 /************************************************************************/
308 
309 /**
310  * Set the pixel function name to be applied to this derived band.  The
311  * name should match a pixel function registered using AddPixelFunction.
312  *
313  * @param pszFuncNameIn Name of pixel function to be applied to this derived
314  * band.
315  */
SetPixelFunctionName(const char * pszFuncNameIn)316 void VRTDerivedRasterBand::SetPixelFunctionName( const char *pszFuncNameIn )
317 {
318     pszFuncName = CPLStrdup( pszFuncNameIn );
319 }
320 
321 /************************************************************************/
322 /*                         SetPixelFunctionLanguage()                   */
323 /************************************************************************/
324 
325 /**
326  * Set the language of the pixel function.
327  *
328  * @param pszLanguage Language of the pixel function (only "C" and "Python"
329  * are supported currently)
330  * @since GDAL 2.3
331  */
SetPixelFunctionLanguage(const char * pszLanguage)332 void VRTDerivedRasterBand::SetPixelFunctionLanguage( const char* pszLanguage )
333 {
334     m_poPrivate->m_osLanguage = pszLanguage;
335 }
336 
337 /************************************************************************/
338 /*                         SetSourceTransferType()                      */
339 /************************************************************************/
340 
341 /**
342  * Set the transfer type to be used to obtain pixel information from
343  * all of the sources.  If unset, the transfer type used will be the
344  * same as the derived band data type.  This makes it possible, for
345  * example, to pass CFloat32 source pixels to the pixel function, even
346  * if the pixel function generates a raster for a derived band that
347  * is of type Byte.
348  *
349  * @param eDataTypeIn Data type to use to obtain pixel information from
350  * the sources to be passed to the derived band pixel function.
351  */
SetSourceTransferType(GDALDataType eDataTypeIn)352 void VRTDerivedRasterBand::SetSourceTransferType( GDALDataType eDataTypeIn )
353 {
354     eSourceTransferType = eDataTypeIn;
355 }
356 
357 /************************************************************************/
358 /*                           InitializePython()                         */
359 /************************************************************************/
360 
InitializePython()361 bool VRTDerivedRasterBand::InitializePython()
362 {
363     if( m_poPrivate->m_bPythonInitializationDone )
364         return m_poPrivate->m_bPythonInitializationSuccess;
365 
366     m_poPrivate->m_bPythonInitializationDone = true;
367     m_poPrivate->m_bPythonInitializationSuccess = false;
368 
369     const CPLString osPythonFullname( pszFuncName ? pszFuncName : "" );
370     const size_t nIdxDot = osPythonFullname.rfind(".");
371     CPLString osPythonModule;
372     CPLString osPythonFunction;
373     if( nIdxDot != std::string::npos )
374     {
375         osPythonModule = osPythonFullname.substr(0, nIdxDot);
376         osPythonFunction = osPythonFullname.substr(nIdxDot+1);
377     }
378     else
379     {
380         osPythonFunction = osPythonFullname;
381     }
382 
383 #ifndef GDAL_VRT_DISABLE_PYTHON
384     const char* pszPythonEnabled =
385                             CPLGetConfigOption("GDAL_VRT_ENABLE_PYTHON", nullptr);
386 #else
387     const char* pszPythonEnabled = "NO";
388 #endif
389     const CPLString osPythonEnabled(pszPythonEnabled ? pszPythonEnabled :
390                                             GDAL_VRT_ENABLE_PYTHON_DEFAULT);
391 
392     if( EQUAL(osPythonEnabled, "TRUSTED_MODULES") )
393     {
394         bool bIsTrustedModule = false;
395         const CPLString osVRTTrustedModules(
396                     CPLGetConfigOption( "GDAL_VRT_PYTHON_TRUSTED_MODULES", "") );
397         if( !osPythonModule.empty() )
398         {
399             char** papszTrustedModules = CSLTokenizeString2(
400                                                 osVRTTrustedModules, ",", 0 );
401             for( char** papszIter = papszTrustedModules;
402                 !bIsTrustedModule && papszIter && *papszIter;
403                 ++papszIter )
404             {
405                 const char* pszIterModule = *papszIter;
406                 size_t nIterModuleLen = strlen(pszIterModule);
407                 if( nIterModuleLen > 2 &&
408                     strncmp(pszIterModule + nIterModuleLen - 2, ".*", 2) == 0 )
409                 {
410                     bIsTrustedModule =
411                         (strncmp( osPythonModule, pszIterModule,
412                                                   nIterModuleLen - 2 ) == 0) &&
413                         (osPythonModule.size() == nIterModuleLen - 2 ||
414                          (osPythonModule.size() >= nIterModuleLen &&
415                           osPythonModule[nIterModuleLen-1] == '.') );
416                 }
417                 else if( nIterModuleLen >= 1 &&
418                         pszIterModule[nIterModuleLen-1] == '*' )
419                 {
420                     bIsTrustedModule = (strncmp( osPythonModule, pszIterModule,
421                                                 nIterModuleLen - 1 ) == 0);
422                 }
423                 else
424                 {
425                     bIsTrustedModule =
426                                 (strcmp(osPythonModule, pszIterModule) == 0);
427                 }
428             }
429             CSLDestroy(papszTrustedModules);
430         }
431 
432         if( !bIsTrustedModule )
433         {
434             if( osPythonModule.empty() )
435             {
436                 CPLError(CE_Failure, CPLE_AppDefined,
437                          "Python code needs to be executed, but it uses online code "
438                          "in the VRT whereas the current policy is to trust only "
439                          "code from external trusted modules (defined in the "
440                          "GDAL_VRT_PYTHON_TRUSTED_MODULES configuration option). "
441                          "If you trust the code in %s, you can set the "
442                          "GDAL_VRT_ENABLE_PYTHON configuration option to YES.",
443                          GetDataset() ? GetDataset()->GetDescription() :
444                                     "(unknown VRT)");
445             }
446             else if( osVRTTrustedModules.empty() )
447             {
448                 CPLError(CE_Failure, CPLE_AppDefined,
449                          "Python code needs to be executed, but it uses code "
450                          "from module '%s', whereas the current policy is to "
451                          "trust only code from modules defined in the "
452                          "GDAL_VRT_PYTHON_TRUSTED_MODULES configuration option, "
453                          "which is currently unset. "
454                          "If you trust the code in '%s', you can add module '%s' "
455                          "to GDAL_VRT_PYTHON_TRUSTED_MODULES (or set the "
456                          "GDAL_VRT_ENABLE_PYTHON configuration option to YES).",
457                          osPythonModule.c_str(),
458                          GetDataset() ? GetDataset()->GetDescription() :
459                                     "(unknown VRT)",
460                          osPythonModule.c_str());
461             }
462             else
463             {
464                 CPLError(CE_Failure, CPLE_AppDefined,
465                          "Python code needs to be executed, but it uses code "
466                          "from module '%s', whereas the current policy is to "
467                          "trust only code from modules '%s' (defined in the "
468                          "GDAL_VRT_PYTHON_TRUSTED_MODULES configuration option). "
469                          "If you trust the code in '%s', you can add module '%s' "
470                          "to GDAL_VRT_PYTHON_TRUSTED_MODULES (or set the "
471                          "GDAL_VRT_ENABLE_PYTHON configuration option to YES).",
472                          osPythonModule.c_str(),
473                          osVRTTrustedModules.c_str(),
474                          GetDataset() ? GetDataset()->GetDescription() :
475                                     "(unknown VRT)",
476                          osPythonModule.c_str());
477             }
478             return false;
479         }
480     }
481 
482 #ifdef disabled_because_this_is_probably_broken_by_design
483     // See https://lwn.net/Articles/574215/
484     // and http://nedbatchelder.com/blog/201206/eval_really_is_dangerous.html
485     else if( EQUAL(osPythonEnabled, "IF_SAFE") )
486     {
487         bool bSafe = true;
488         // If the function comes from another module, then we don't know
489         if( !osPythonModule.empty() )
490         {
491             CPLDebug("VRT", "Python function is from another module");
492             bSafe = false;
493         }
494 
495         CPLString osCode(m_poPrivate->m_osCode);
496 
497         // Reject all imports except a few trusted modules
498         const char* const apszTrustedImports[] = {
499                 "import math",
500                 "from math import",
501                 "import numpy", // caution: numpy has lots of I/O functions !
502                 "from numpy import",
503                 // TODO: not sure if importing arbitrary stuff from numba is OK
504                 // so let's just restrict to jit.
505                 "from numba import jit",
506 
507                 // Not imports but still whitelisted, whereas other __ is banned
508                 "__init__",
509                 "__call__",
510         };
511         for( size_t i = 0; i < CPL_ARRAYSIZE(apszTrustedImports); ++i )
512         {
513             osCode.replaceAll(CPLString(apszTrustedImports[i]), "");
514         }
515 
516         // Some dangerous built-in functions or numpy functions
517         const char* const apszUntrusted[] = { "import", // and __import__
518                                               "eval",
519                                               "compile",
520                                               "open",
521                                               "load", // reload, numpy.load
522                                               "file", // and exec_file, numpy.fromfile, numpy.tofile
523                                               "input", // and raw_input
524                                               "save", // numpy.save
525                                               "memmap", // numpy.memmap
526                                               "DataSource", // numpy.DataSource
527                                               "genfromtxt", // numpy.genfromtxt
528                                               "getattr",
529                                               "ctypeslib", // numpy.ctypeslib
530                                               "testing", // numpy.testing
531                                               "dump", // numpy.ndarray.dump
532                                               "fromregex", // numpy.fromregex
533                                               "__"
534                                              };
535         for( size_t i = 0; i < CPL_ARRAYSIZE(apszUntrusted); ++i )
536         {
537             if( osCode.find(apszUntrusted[i]) != std::string::npos )
538             {
539                 CPLDebug("VRT", "Found '%s' word in Python code",
540                          apszUntrusted[i]);
541                 bSafe = false;
542             }
543         }
544 
545         if( !bSafe )
546         {
547             CPLError(CE_Failure, CPLE_AppDefined,
548                      "Python code needs to be executed, but we cannot verify "
549                      "if it is safe, so this is disabled by default. "
550                      "If you trust the code in %s, you can set the "
551                      "GDAL_VRT_ENABLE_PYTHON configuration option to YES.",
552                      GetDataset() ? GetDataset()->GetDescription() :
553                                     "(unknown VRT)");
554             return false;
555         }
556     }
557 #endif //disabled_because_this_is_probably_broken_by_design
558 
559     else if( !EQUAL(osPythonEnabled, "YES") &&
560              !EQUAL(osPythonEnabled, "ON") &&
561              !EQUAL(osPythonEnabled, "TRUE") )
562     {
563         if( pszPythonEnabled == nullptr )
564         {
565             // Note: this is dead code with our current default policy
566             // GDAL_VRT_ENABLE_PYTHON == "TRUSTED_MODULES"
567             CPLError(CE_Failure, CPLE_AppDefined,
568                  "Python code needs to be executed, but this is "
569                  "disabled by default. If you trust the code in %s, "
570                  "you can set the GDAL_VRT_ENABLE_PYTHON configuration "
571                  "option to YES.",
572                 GetDataset() ? GetDataset()->GetDescription() :
573                                                     "(unknown VRT)");
574         }
575         else
576         {
577             CPLError(CE_Failure, CPLE_AppDefined,
578                     "Python code in %s needs to be executed, but this has been "
579                     "explicitly disabled.",
580                      GetDataset() ? GetDataset()->GetDescription() :
581                                                             "(unknown VRT)");
582         }
583         return false;
584     }
585 
586     if( !GDALPythonInitialize() )
587         return false;
588 
589     // Whether we should just use our own global mutex, in addition to Python
590     // GIL locking.
591     m_poPrivate->m_bExclusiveLock =
592         CPLTestBool(CPLGetConfigOption("GDAL_VRT_PYTHON_EXCLUSIVE_LOCK", "NO"));
593 
594     // numba jit'ification doesn't seem to be thread-safe, so force use of
595     // lock now and at first execution of function. Later executions seem to
596     // be thread-safe. This problem doesn't seem to appear for code in
597     // regular files
598     const bool bUseExclusiveLock = m_poPrivate->m_bExclusiveLock ||
599                     m_poPrivate->m_osCode.find("@jit") != std::string::npos;
600     GIL_Holder oHolder(bUseExclusiveLock);
601 
602     // As we don't want to depend on numpy C API/ABI, we use a trick to build
603     // a numpy array object. We define a Python function to which we pass a
604     // Python buffer object.
605 
606     // We need to build a unique module name, otherwise this will crash in
607     // multithreaded use cases.
608     CPLString osModuleName( CPLSPrintf("gdal_vrt_module_%p", this) );
609     PyObject* poCompiledString = Py_CompileString(
610         ("import numpy\n"
611         "def GDALCreateNumpyArray(buffer, dtype, height, width):\n"
612         "    return numpy.frombuffer(buffer, str(dtype.decode('ascii')))."
613                                                 "reshape([height, width])\n"
614         "\n" + m_poPrivate->m_osCode).c_str(),
615         osModuleName, Py_file_input);
616     if( poCompiledString == nullptr || PyErr_Occurred() )
617     {
618         CPLError(CE_Failure, CPLE_AppDefined,
619                  "Couldn't compile code:\n%s",
620                  GetPyExceptionString().c_str());
621         return false;
622     }
623     PyObject* poModule =
624         PyImport_ExecCodeModule(osModuleName, poCompiledString);
625     Py_DecRef(poCompiledString);
626 
627     if( poModule == nullptr || PyErr_Occurred() )
628     {
629         CPLError(CE_Failure, CPLE_AppDefined,
630                  "%s", GetPyExceptionString().c_str());
631         return false;
632     }
633 
634     // Fetch user computation function
635     if( !osPythonModule.empty() )
636     {
637         PyObject* poUserModule = PyImport_ImportModule(osPythonModule);
638         if (poUserModule == nullptr || PyErr_Occurred())
639         {
640             CPLString osException = GetPyExceptionString();
641             if( !osException.empty() && osException.back() == '\n' )
642             {
643                 osException.resize( osException.size() - 1 );
644             }
645             if( osException.find("ModuleNotFoundError") == 0 )
646             {
647                 osException += ". You may need to define PYTHONPATH";
648             }
649             CPLError(CE_Failure, CPLE_AppDefined,
650                  "%s", osException.c_str());
651             Py_DecRef(poModule);
652             return false;
653         }
654         m_poPrivate->m_poUserFunction = PyObject_GetAttrString(poUserModule,
655                                                             osPythonFunction );
656         Py_DecRef(poUserModule);
657     }
658     else
659     {
660         m_poPrivate->m_poUserFunction = PyObject_GetAttrString(poModule,
661                                             osPythonFunction );
662     }
663     if (m_poPrivate->m_poUserFunction == nullptr || PyErr_Occurred())
664     {
665         CPLError(CE_Failure, CPLE_AppDefined,
666                  "%s", GetPyExceptionString().c_str());
667         Py_DecRef(poModule);
668         return false;
669     }
670     if( !PyCallable_Check(m_poPrivate->m_poUserFunction) )
671     {
672         CPLError(CE_Failure, CPLE_AppDefined, "Object '%s' is not callable",
673                  osPythonFunction.c_str());
674         Py_DecRef(poModule);
675         return false;
676     }
677 
678     // Fetch our GDALCreateNumpyArray python function
679     m_poPrivate->m_poGDALCreateNumpyArray =
680         PyObject_GetAttrString(poModule, "GDALCreateNumpyArray" );
681     if (m_poPrivate->m_poGDALCreateNumpyArray == nullptr || PyErr_Occurred())
682     {
683         // Shouldn't happen normally...
684         CPLError(CE_Failure, CPLE_AppDefined,
685                  "%s", GetPyExceptionString().c_str());
686         Py_DecRef(poModule);
687         return false;
688     }
689     Py_DecRef(poModule);
690 
691     m_poPrivate->m_bPythonInitializationSuccess = true;
692     return true;
693 }
694 
695 /************************************************************************/
696 /*                             IRasterIO()                              */
697 /************************************************************************/
698 
699 /**
700  * Read/write a region of image data for this band.
701  *
702  * Each of the sources for this derived band will be read and passed to
703  * the derived band pixel function.  The pixel function is responsible
704  * for applying whatever algorithm is necessary to generate this band's
705  * pixels from the sources.
706  *
707  * The sources will be read using the transfer type specified for sources
708  * using SetSourceTransferType().  If no transfer type has been set for
709  * this derived band, the band's data type will be used as the transfer type.
710  *
711  * @see gdalrasterband
712  *
713  * @param eRWFlag Either GF_Read to read a region of data, or GT_Write to
714  * write a region of data.
715  *
716  * @param nXOff The pixel offset to the top left corner of the region
717  * of the band to be accessed.  This would be zero to start from the left side.
718  *
719  * @param nYOff The line offset to the top left corner of the region
720  * of the band to be accessed.  This would be zero to start from the top.
721  *
722  * @param nXSize The width of the region of the band to be accessed in pixels.
723  *
724  * @param nYSize The height of the region of the band to be accessed in lines.
725  *
726  * @param pData The buffer into which the data should be read, or from which
727  * it should be written.  This buffer must contain at least nBufXSize *
728  * nBufYSize words of type eBufType.  It is organized in left to right,
729  * top to bottom pixel order.  Spacing is controlled by the nPixelSpace,
730  * and nLineSpace parameters.
731  *
732  * @param nBufXSize The width of the buffer image into which the desired
733  * region is to be read, or from which it is to be written.
734  *
735  * @param nBufYSize The height of the buffer image into which the desired
736  * region is to be read, or from which it is to be written.
737  *
738  * @param eBufType The type of the pixel values in the pData data buffer.  The
739  * pixel values will automatically be translated to/from the GDALRasterBand
740  * data type as needed.
741  *
742  * @param nPixelSpace The byte offset from the start of one pixel value in
743  * pData to the start of the next pixel value within a scanline.  If defaulted
744  * (0) the size of the datatype eBufType is used.
745  *
746  * @param nLineSpace The byte offset from the start of one scanline in
747  * pData to the start of the next.  If defaulted the size of the datatype
748  * eBufType * nBufXSize is used.
749  *
750  * @return CE_Failure if the access fails, otherwise CE_None.
751  */
IRasterIO(GDALRWFlag eRWFlag,int nXOff,int nYOff,int nXSize,int nYSize,void * pData,int nBufXSize,int nBufYSize,GDALDataType eBufType,GSpacing nPixelSpace,GSpacing nLineSpace,GDALRasterIOExtraArg * psExtraArg)752 CPLErr VRTDerivedRasterBand::IRasterIO( GDALRWFlag eRWFlag,
753                                         int nXOff, int nYOff, int nXSize,
754                                         int nYSize, void * pData, int nBufXSize,
755                                         int nBufYSize, GDALDataType eBufType,
756                                         GSpacing nPixelSpace,
757                                         GSpacing nLineSpace,
758                                         GDALRasterIOExtraArg* psExtraArg )
759 {
760     if( eRWFlag == GF_Write )
761     {
762         CPLError( CE_Failure, CPLE_AppDefined,
763                   "Writing through VRTSourcedRasterBand is not supported." );
764         return CE_Failure;
765     }
766 
767     const int nBufTypeSize = GDALGetDataTypeSizeBytes(eBufType);
768     GDALDataType eSrcType = eSourceTransferType;
769     if( eSrcType == GDT_Unknown || eSrcType >= GDT_TypeCount ) {
770         eSrcType = eBufType;
771     }
772     const int nSrcTypeSize = GDALGetDataTypeSizeBytes(eSrcType);
773 
774 /* -------------------------------------------------------------------- */
775 /*      Initialize the buffer to some background value. Use the         */
776 /*      nodata value if available.                                      */
777 /* -------------------------------------------------------------------- */
778     if( bSkipBufferInitialization )
779     {
780         // Do nothing
781     }
782     else if( nPixelSpace == nBufTypeSize &&
783         (!m_bNoDataValueSet || m_dfNoDataValue == 0) ) {
784         memset( pData, 0,
785                 static_cast<size_t>(nBufXSize * nBufYSize * nPixelSpace) );
786     }
787     else if( m_bNoDataValueSet )
788     {
789         double dfWriteValue = m_dfNoDataValue;
790 
791         for( int iLine = 0; iLine < nBufYSize; iLine++ )
792         {
793             GDALCopyWords(
794                 &dfWriteValue, GDT_Float64, 0,
795                 static_cast<GByte *>( pData ) + nLineSpace * iLine,
796                 eBufType, static_cast<int>(nPixelSpace), nBufXSize );
797         }
798     }
799 
800 /* -------------------------------------------------------------------- */
801 /*      Do we have overviews that would be appropriate to satisfy       */
802 /*      this request?                                                   */
803 /* -------------------------------------------------------------------- */
804     if( (nBufXSize < nXSize || nBufYSize < nYSize)
805         && GetOverviewCount() > 0 )
806     {
807         if( OverviewRasterIO(
808                eRWFlag, nXOff, nYOff, nXSize, nYSize,
809                pData, nBufXSize, nBufYSize,
810                eBufType, nPixelSpace, nLineSpace, psExtraArg ) == CE_None )
811             return CE_None;
812     }
813 
814     /* ---- Get pixel function for band ---- */
815     GDALDerivedPixelFunc pfnPixelFunc = nullptr;
816 
817     if( EQUAL(m_poPrivate->m_osLanguage, "C") )
818     {
819         pfnPixelFunc = VRTDerivedRasterBand::GetPixelFunction(pszFuncName);
820         if( pfnPixelFunc == nullptr )
821         {
822             CPLError( CE_Failure, CPLE_IllegalArg,
823                     "VRTDerivedRasterBand::IRasterIO:"
824                     "Derived band pixel function '%s' not registered.",
825                     this->pszFuncName) ;
826             return CE_Failure;
827         }
828     }
829 
830     /* TODO: It would be nice to use a MallocBlock function for each
831        individual buffer that would recycle blocks of memory from a
832        cache by reassigning blocks that are nearly the same size.
833        A corresponding FreeBlock might only truly free if the total size
834        of freed blocks gets to be too great of a percentage of the size
835        of the allocated blocks. */
836 
837     // Get buffers for each source.
838     const int nBufferRadius = m_poPrivate->m_nBufferRadius;
839     if( nBufferRadius > (INT_MAX - nBufXSize) / 2 ||
840         nBufferRadius > (INT_MAX - nBufYSize) / 2 )
841     {
842         return CE_Failure;
843     }
844     const int nExtBufXSize = nBufXSize + 2 * nBufferRadius;
845     const int nExtBufYSize = nBufYSize + 2 * nBufferRadius;
846     void **pBuffers
847         = static_cast<void **>( CPLMalloc(sizeof(void *) * nSources) );
848     for( int iSource = 0; iSource < nSources; iSource++ ) {
849         pBuffers[iSource] =
850             VSI_MALLOC3_VERBOSE(nSrcTypeSize, nExtBufXSize, nExtBufYSize);
851         if( pBuffers[iSource] == nullptr )
852         {
853             for (int i = 0; i < iSource; i++) {
854                 VSIFree(pBuffers[i]);
855             }
856             CPLFree(pBuffers);
857             return CE_Failure;
858         }
859 
860         /* ------------------------------------------------------------ */
861         /* #4045: Initialize the newly allocated buffers before handing */
862         /* them off to the sources. These buffers are packed, so we     */
863         /* don't need any special line-by-line handling when a nonzero  */
864         /* nodata value is set.                                         */
865         /* ------------------------------------------------------------ */
866         if( !m_bNoDataValueSet || m_dfNoDataValue == 0 )
867         {
868             memset( pBuffers[iSource], 0, static_cast<size_t>(nSrcTypeSize) *
869                     nExtBufXSize * nExtBufYSize );
870         }
871         else
872         {
873             GDALCopyWords( &m_dfNoDataValue, GDT_Float64, 0,
874                            static_cast<GByte *>( pBuffers[iSource] ),
875                            eSrcType, nSrcTypeSize,
876                            nExtBufXSize * nExtBufYSize );
877         }
878     }
879 
880     GDALRasterIOExtraArg sExtraArg;
881     GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg);
882 
883     int nXShiftInBuffer = 0;
884     int nYShiftInBuffer = 0;
885     int nExtBufXSizeReq = nExtBufXSize;
886     int nExtBufYSizeReq = nExtBufYSize;
887 
888     int nXOffExt = nXOff;
889     int nYOffExt = nYOff;
890     int nXSizeExt = nXSize;
891     int nYSizeExt = nYSize;
892 
893     if( nBufferRadius )
894     {
895         double dfXRatio = static_cast<double>(nXSize) / nBufXSize;
896         double dfYRatio = static_cast<double>(nYSize) / nBufYSize;
897 
898         if( !sExtraArg.bFloatingPointWindowValidity )
899         {
900             sExtraArg.dfXOff = nXOff;
901             sExtraArg.dfYOff = nYOff;
902             sExtraArg.dfXSize = nXSize;
903             sExtraArg.dfYSize = nYSize;
904         }
905 
906         sExtraArg.dfXOff -= dfXRatio * nBufferRadius;
907         sExtraArg.dfYOff -= dfYRatio * nBufferRadius;
908         sExtraArg.dfXSize += 2 * dfXRatio * nBufferRadius;
909         sExtraArg.dfYSize += 2 * dfYRatio * nBufferRadius;
910         if( sExtraArg.dfXOff < 0 )
911         {
912             nXShiftInBuffer = -static_cast<int>(sExtraArg.dfXOff / dfXRatio);
913             nExtBufXSizeReq -= nXShiftInBuffer;
914             sExtraArg.dfXSize += sExtraArg.dfXOff;
915             sExtraArg.dfXOff = 0;
916         }
917         if( sExtraArg.dfYOff < 0 )
918         {
919             nYShiftInBuffer = -static_cast<int>(sExtraArg.dfYOff / dfYRatio);
920             nExtBufYSizeReq -= nYShiftInBuffer;
921             sExtraArg.dfYSize += sExtraArg.dfYOff;
922             sExtraArg.dfYOff = 0;
923         }
924         if( sExtraArg.dfXOff + sExtraArg.dfXSize > nRasterXSize )
925         {
926             nExtBufXSizeReq -= static_cast<int>((sExtraArg.dfXOff +
927                         sExtraArg.dfXSize - nRasterXSize) / dfXRatio);
928             sExtraArg.dfXSize = nRasterXSize - sExtraArg.dfXOff;
929         }
930         if( sExtraArg.dfYOff + sExtraArg.dfYSize > nRasterYSize )
931         {
932             nExtBufYSizeReq -= static_cast<int>((sExtraArg.dfYOff +
933                         sExtraArg.dfYSize - nRasterYSize) / dfYRatio);
934             sExtraArg.dfYSize = nRasterYSize - sExtraArg.dfYOff;
935         }
936 
937         nXOffExt = static_cast<int>(sExtraArg.dfXOff);
938         nYOffExt = static_cast<int>(sExtraArg.dfYOff);
939         nXSizeExt = std::min(static_cast<int>(sExtraArg.dfXSize + 0.5),
940                              nRasterXSize - nXOffExt);
941         nYSizeExt = std::min(static_cast<int>(sExtraArg.dfYSize + 0.5),
942                              nRasterYSize - nYOffExt);
943     }
944 
945     // Load values for sources into packed buffers.
946     CPLErr eErr = CE_None;
947     for( int iSource = 0; iSource < nSources && eErr == CE_None; iSource++ ) {
948         GByte* pabyBuffer = static_cast<GByte*>(pBuffers[iSource]);
949         eErr = static_cast<VRTSource *>( papoSources[iSource] )->RasterIO(
950             eSrcType,
951             nXOffExt, nYOffExt, nXSizeExt, nYSizeExt,
952             pabyBuffer + (nYShiftInBuffer * nExtBufXSize +
953                                             nXShiftInBuffer) * nSrcTypeSize,
954             nExtBufXSizeReq, nExtBufYSizeReq,
955             eSrcType,
956             nSrcTypeSize,
957             nSrcTypeSize * nExtBufXSize,
958             &sExtraArg );
959 
960         // Extend first lines
961         for( int iY = 0; iY < nYShiftInBuffer; iY++ )
962         {
963             memcpy( pabyBuffer + iY * nExtBufXSize * nSrcTypeSize,
964                     pabyBuffer + nYShiftInBuffer * nExtBufXSize * nSrcTypeSize,
965                     nExtBufXSize * nSrcTypeSize );
966         }
967         // Extend last lines
968         for( int iY = nYShiftInBuffer + nExtBufYSizeReq; iY < nExtBufYSize; iY++ )
969         {
970             memcpy( pabyBuffer + iY * nExtBufXSize * nSrcTypeSize,
971                     pabyBuffer + (nYShiftInBuffer + nExtBufYSizeReq - 1) *
972                                                     nExtBufXSize * nSrcTypeSize,
973                     nExtBufXSize * nSrcTypeSize );
974         }
975         // Extend first cols
976         if( nXShiftInBuffer )
977         {
978             for( int iY = 0; iY < nExtBufYSize; iY ++ )
979             {
980                 for( int iX = 0; iX < nXShiftInBuffer; iX++ )
981                 {
982                     memcpy( pabyBuffer + (iY * nExtBufXSize + iX) * nSrcTypeSize,
983                             pabyBuffer + (iY * nExtBufXSize +
984                                                 nXShiftInBuffer) * nSrcTypeSize,
985                             nSrcTypeSize );
986                 }
987             }
988         }
989         // Extent last cols
990         if( nXShiftInBuffer + nExtBufXSizeReq < nExtBufXSize )
991         {
992             for( int iY = 0; iY < nExtBufYSize; iY ++ )
993             {
994                 for( int iX = nXShiftInBuffer + nExtBufXSizeReq;
995                          iX < nExtBufXSize; iX++ )
996                 {
997                     memcpy( pabyBuffer + (iY * nExtBufXSize + iX) * nSrcTypeSize,
998                             pabyBuffer + (iY * nExtBufXSize + nXShiftInBuffer +
999                                             nExtBufXSizeReq - 1) * nSrcTypeSize,
1000                             nSrcTypeSize );
1001                 }
1002             }
1003         }
1004     }
1005 
1006     // Apply pixel function.
1007     if( eErr == CE_None && EQUAL(m_poPrivate->m_osLanguage, "Python") )
1008     {
1009         eErr = CE_Failure;
1010 
1011         // numpy doesn't have native cint16/cint32
1012         if( eSrcType == GDT_CInt16 || eSrcType == GDT_CInt32 )
1013         {
1014             CPLError(CE_Failure, CPLE_AppDefined,
1015                      "CInt16/CInt32 data type not supported for SourceTransferType");
1016             goto end;
1017         }
1018         if( eDataType == GDT_CInt16 || eDataType == GDT_CInt32 )
1019         {
1020             CPLError(CE_Failure, CPLE_AppDefined,
1021                      "CInt16/CInt32 data type not supported for data type");
1022             goto end;
1023         }
1024 
1025         if( !InitializePython() )
1026             goto end;
1027 
1028         GByte* pabyTmpBuffer = nullptr;
1029         // Do we need a temporary buffer or can we use directly the output
1030         // buffer ?
1031         if( nBufferRadius != 0 ||
1032             eDataType != eBufType ||
1033             nPixelSpace != nBufTypeSize ||
1034             nLineSpace != static_cast<GSpacing>(nBufTypeSize) * nBufXSize )
1035         {
1036             pabyTmpBuffer = static_cast<GByte*>(VSI_CALLOC_VERBOSE(
1037                             static_cast<size_t>(nExtBufXSize) * nExtBufYSize,
1038                             GDALGetDataTypeSizeBytes(eDataType)));
1039             if( !pabyTmpBuffer )
1040                 goto end;
1041         }
1042 
1043         {
1044         const bool bUseExclusiveLock = m_poPrivate->m_bExclusiveLock ||
1045                     ( m_poPrivate->m_bFirstTime &&
1046                     m_poPrivate->m_osCode.find("@jit") != std::string::npos);
1047         m_poPrivate->m_bFirstTime = false;
1048         GIL_Holder oHolder(bUseExclusiveLock);
1049 
1050         // Prepare target numpy array
1051         PyObject* poPyDstArray = GDALCreateNumpyArray(
1052                                     m_poPrivate->m_poGDALCreateNumpyArray,
1053                                     pabyTmpBuffer ? pabyTmpBuffer : pData,
1054                                     eDataType,
1055                                     nExtBufYSize,
1056                                     nExtBufXSize);
1057         if( !poPyDstArray )
1058         {
1059             VSIFree(pabyTmpBuffer);
1060             goto end;
1061         }
1062 
1063         // Wrap source buffers as input numpy arrays
1064         PyObject* pyArgInputArray = PyTuple_New(nSources);
1065         for( int i = 0; i < nSources; i++ )
1066         {
1067             GByte* pabyBuffer = static_cast<GByte*>(pBuffers[i]);
1068             PyObject* poPySrcArray = GDALCreateNumpyArray(
1069                         m_poPrivate->m_poGDALCreateNumpyArray,
1070                         pabyBuffer,
1071                         eSrcType,
1072                         nExtBufYSize,
1073                         nExtBufXSize);
1074             CPLAssert(poPySrcArray);
1075             PyTuple_SetItem(pyArgInputArray, i, poPySrcArray);
1076         }
1077 
1078         // Create arguments
1079         PyObject* pyArgs = PyTuple_New(10);
1080         PyTuple_SetItem(pyArgs, 0, pyArgInputArray);
1081         PyTuple_SetItem(pyArgs, 1, poPyDstArray);
1082         PyTuple_SetItem(pyArgs, 2, PyLong_FromLong(nXOff));
1083         PyTuple_SetItem(pyArgs, 3, PyLong_FromLong(nYOff));
1084         PyTuple_SetItem(pyArgs, 4, PyLong_FromLong(nXSize));
1085         PyTuple_SetItem(pyArgs, 5, PyLong_FromLong(nYSize));
1086         PyTuple_SetItem(pyArgs, 6, PyLong_FromLong(nRasterXSize));
1087         PyTuple_SetItem(pyArgs, 7, PyLong_FromLong(nRasterYSize));
1088         PyTuple_SetItem(pyArgs, 8, PyLong_FromLong(nBufferRadius));
1089 
1090         double adfGeoTransform[6];
1091         adfGeoTransform[0] = 0;
1092         adfGeoTransform[1] = 1;
1093         adfGeoTransform[2] = 0;
1094         adfGeoTransform[3] = 0;
1095         adfGeoTransform[4] = 0;
1096         adfGeoTransform[5] = 1;
1097         if( GetDataset() )
1098             GetDataset()->GetGeoTransform(adfGeoTransform);
1099         PyObject* pyGT = PyTuple_New(6);
1100         for(int i = 0; i < 6; i++ )
1101             PyTuple_SetItem(pyGT, i, PyFloat_FromDouble(adfGeoTransform[i]));
1102         PyTuple_SetItem(pyArgs, 9, pyGT);
1103 
1104         // Prepare kwargs
1105         PyObject* pyKwargs = PyDict_New();
1106         for( size_t i = 0; i < m_poPrivate->m_oFunctionArgs.size(); ++i )
1107         {
1108             const char* pszKey =
1109                 m_poPrivate->m_oFunctionArgs[i].first.c_str();
1110             const char* pszValue =
1111                 m_poPrivate->m_oFunctionArgs[i].second.c_str();
1112             PyDict_SetItemString(pyKwargs, pszKey,
1113                 PyBytes_FromStringAndSize(pszValue, strlen(pszValue)));
1114         }
1115 
1116         // Call user function
1117         PyObject* pRetValue = PyObject_Call(
1118                                         m_poPrivate->m_poUserFunction,
1119                                         pyArgs, pyKwargs);
1120 
1121         Py_DecRef(pyArgs);
1122         Py_DecRef(pyKwargs);
1123 
1124         if( ErrOccurredEmitCPLError() )
1125         {
1126             // do nothing
1127         }
1128         else
1129         {
1130             eErr = CE_None;
1131         }
1132         if( pRetValue )
1133             Py_DecRef(pRetValue);
1134         } // End of GIL section
1135 
1136         if( pabyTmpBuffer )
1137         {
1138             // Copy numpy destination array to user buffer
1139             for( int iY = 0; iY < nBufYSize; iY++ )
1140             {
1141                 size_t nSrcOffset = (static_cast<size_t>(iY + nBufferRadius) *
1142                     nExtBufXSize + nBufferRadius) *
1143                     GDALGetDataTypeSizeBytes(eDataType);
1144                 GDALCopyWords(pabyTmpBuffer + nSrcOffset,
1145                               eDataType,
1146                               GDALGetDataTypeSizeBytes(eDataType),
1147                               static_cast<GByte*>(pData) + iY * nLineSpace,
1148                               eBufType,
1149                               static_cast<int>(nPixelSpace),
1150                               nBufXSize);
1151             }
1152 
1153             VSIFree(pabyTmpBuffer);
1154         }
1155     }
1156     else if( eErr == CE_None && pfnPixelFunc != nullptr ) {
1157         eErr = pfnPixelFunc( static_cast<void **>( pBuffers ), nSources,
1158                              pData, nBufXSize, nBufYSize,
1159                              eSrcType, eBufType, static_cast<int>(nPixelSpace),
1160                              static_cast<int>(nLineSpace) );
1161     }
1162 end:
1163     // Release buffers.
1164     for ( int iSource = 0; iSource < nSources; iSource++ ) {
1165         VSIFree(pBuffers[iSource]);
1166     }
1167     CPLFree(pBuffers);
1168 
1169     return eErr;
1170 }
1171 
1172 /************************************************************************/
1173 /*                         IGetDataCoverageStatus()                     */
1174 /************************************************************************/
1175 
IGetDataCoverageStatus(int,int,int,int,int,double * pdfDataPct)1176 int  VRTDerivedRasterBand::IGetDataCoverageStatus( int /* nXOff */,
1177                                                    int /* nYOff */,
1178                                                    int /* nXSize */,
1179                                                    int /* nYSize */,
1180                                                    int /* nMaskFlagStop */,
1181                                                    double* pdfDataPct)
1182 {
1183     if( pdfDataPct != nullptr )
1184         *pdfDataPct = -1.0;
1185     return GDAL_DATA_COVERAGE_STATUS_UNIMPLEMENTED | GDAL_DATA_COVERAGE_STATUS_DATA;
1186 }
1187 
1188 /************************************************************************/
1189 /*                              XMLInit()                               */
1190 /************************************************************************/
1191 
XMLInit(CPLXMLNode * psTree,const char * pszVRTPath,void * pUniqueHandle,std::map<CPLString,GDALDataset * > & oMapSharedSources)1192 CPLErr VRTDerivedRasterBand::XMLInit( CPLXMLNode *psTree,
1193                                       const char *pszVRTPath,
1194                                       void* pUniqueHandle,
1195                                       std::map<CPLString, GDALDataset*>& oMapSharedSources )
1196 
1197 {
1198     const CPLErr eErr = VRTSourcedRasterBand::XMLInit( psTree, pszVRTPath,
1199                                                        pUniqueHandle,
1200                                                        oMapSharedSources );
1201     if( eErr != CE_None )
1202         return eErr;
1203 
1204     // Read derived pixel function type.
1205     SetPixelFunctionName( CPLGetXMLValue( psTree, "PixelFunctionType", nullptr ) );
1206     if( pszFuncName == nullptr || EQUAL(pszFuncName, "") )
1207     {
1208         CPLError(CE_Failure, CPLE_AppDefined,
1209                  "PixelFunctionType missing");
1210         return CE_Failure;
1211     }
1212 
1213     m_poPrivate->m_osLanguage = CPLGetXMLValue( psTree,
1214                                                 "PixelFunctionLanguage", "C" );
1215     if( !EQUAL(m_poPrivate->m_osLanguage, "C") &&
1216         !EQUAL(m_poPrivate->m_osLanguage, "Python") )
1217     {
1218         CPLError(CE_Failure, CPLE_NotSupported,
1219                  "Unsupported PixelFunctionLanguage");
1220         return CE_Failure;
1221     }
1222 
1223     m_poPrivate->m_osCode =
1224                         CPLGetXMLValue( psTree, "PixelFunctionCode", "" );
1225     if( !m_poPrivate->m_osCode.empty() &&
1226         !EQUAL(m_poPrivate->m_osLanguage, "Python") )
1227     {
1228         CPLError(CE_Failure, CPLE_NotSupported,
1229                  "PixelFunctionCode can only be used with Python");
1230         return CE_Failure;
1231     }
1232 
1233     m_poPrivate->m_nBufferRadius =
1234                         atoi(CPLGetXMLValue( psTree, "BufferRadius", "0" ));
1235     if( m_poPrivate->m_nBufferRadius < 0 ||
1236         m_poPrivate->m_nBufferRadius > 1024 )
1237     {
1238         CPLError(CE_Failure, CPLE_AppDefined, "Invalid value for BufferRadius");
1239         return CE_Failure;
1240     }
1241     if( m_poPrivate->m_nBufferRadius != 0 &&
1242         !EQUAL(m_poPrivate->m_osLanguage, "Python") )
1243     {
1244         CPLError(CE_Failure, CPLE_NotSupported,
1245                  "BufferRadius can only be used with Python");
1246         return CE_Failure;
1247     }
1248 
1249     CPLXMLNode* psArgs = CPLGetXMLNode( psTree, "PixelFunctionArguments" );
1250     if( psArgs != nullptr )
1251     {
1252         if( !EQUAL(m_poPrivate->m_osLanguage, "Python") )
1253         {
1254             CPLError(CE_Failure, CPLE_NotSupported,
1255                      "PixelFunctionArguments can only be used with Python");
1256             return CE_Failure;
1257         }
1258         for( CPLXMLNode* psIter = psArgs->psChild;
1259                          psIter != nullptr;
1260                          psIter = psIter->psNext )
1261         {
1262             if( psIter->eType == CXT_Attribute )
1263             {
1264                 m_poPrivate->m_oFunctionArgs.push_back(
1265                     std::pair<CPLString,CPLString>(psIter->pszValue,
1266                                                    psIter->psChild->pszValue));
1267             }
1268         }
1269     }
1270 
1271     // Read optional source transfer data type.
1272     const char *pszTypeName = CPLGetXMLValue(psTree, "SourceTransferType", nullptr);
1273     if( pszTypeName != nullptr )
1274     {
1275         eSourceTransferType = GDALGetDataTypeByName( pszTypeName );
1276     }
1277 
1278     return CE_None;
1279 }
1280 
1281 /************************************************************************/
1282 /*                           SerializeToXML()                           */
1283 /************************************************************************/
1284 
SerializeToXML(const char * pszVRTPath)1285 CPLXMLNode *VRTDerivedRasterBand::SerializeToXML( const char *pszVRTPath )
1286 {
1287     CPLXMLNode *psTree = VRTSourcedRasterBand::SerializeToXML( pszVRTPath );
1288 
1289 /* -------------------------------------------------------------------- */
1290 /*      Set subclass.                                                   */
1291 /* -------------------------------------------------------------------- */
1292     CPLCreateXMLNode(
1293         CPLCreateXMLNode( psTree, CXT_Attribute, "subClass" ),
1294         CXT_Text, "VRTDerivedRasterBand" );
1295 
1296     /* ---- Encode DerivedBand-specific fields ---- */
1297     if( !EQUAL( m_poPrivate->m_osLanguage, "C" ) )
1298     {
1299         CPLSetXMLValue( psTree, "PixelFunctionLanguage",
1300                         m_poPrivate->m_osLanguage );
1301     }
1302     if( pszFuncName != nullptr && strlen(pszFuncName) > 0 )
1303         CPLSetXMLValue( psTree, "PixelFunctionType", pszFuncName );
1304     if( !m_poPrivate->m_oFunctionArgs.empty() )
1305     {
1306         CPLXMLNode* psArgs = CPLCreateXMLNode( psTree, CXT_Element,
1307                                                "PixelFunctionArguments" );
1308         for( size_t i = 0; i < m_poPrivate->m_oFunctionArgs.size(); ++i )
1309         {
1310             const char* pszKey =
1311                 m_poPrivate->m_oFunctionArgs[i].first.c_str();
1312             const char* pszValue =
1313                 m_poPrivate->m_oFunctionArgs[i].second.c_str();
1314             CPLCreateXMLNode(
1315                 CPLCreateXMLNode( psArgs, CXT_Attribute, pszKey ),
1316                                   CXT_Text, pszValue );
1317         }
1318     }
1319     if( !m_poPrivate->m_osCode.empty() )
1320     {
1321         if( m_poPrivate->m_osCode.find("<![CDATA[") == std::string::npos )
1322         {
1323             CPLCreateXMLNode(
1324                 CPLCreateXMLNode( psTree,
1325                                   CXT_Element, "PixelFunctionCode" ),
1326                  CXT_Literal,
1327                  ("<![CDATA[" + m_poPrivate->m_osCode + "]]>").c_str() );
1328         }
1329         else
1330         {
1331             CPLSetXMLValue( psTree, "PixelFunctionCode",
1332                             m_poPrivate->m_osCode );
1333         }
1334     }
1335     if( m_poPrivate->m_nBufferRadius != 0 )
1336         CPLSetXMLValue( psTree, "BufferRadius",
1337                         CPLSPrintf("%d",m_poPrivate->m_nBufferRadius) );
1338     if( this->eSourceTransferType != GDT_Unknown)
1339         CPLSetXMLValue( psTree, "SourceTransferType",
1340                         GDALGetDataTypeName( eSourceTransferType ) );
1341 
1342     return psTree;
1343 }
1344 
1345 /************************************************************************/
1346 /*                             GetMinimum()                             */
1347 /************************************************************************/
1348 
GetMinimum(int * pbSuccess)1349 double VRTDerivedRasterBand::GetMinimum( int *pbSuccess )
1350 {
1351     return GDALRasterBand::GetMinimum(pbSuccess);
1352 }
1353 
1354 /************************************************************************/
1355 /*                             GetMaximum()                             */
1356 /************************************************************************/
1357 
GetMaximum(int * pbSuccess)1358 double VRTDerivedRasterBand::GetMaximum( int *pbSuccess )
1359 {
1360     return GDALRasterBand::GetMaximum(pbSuccess);
1361 }
1362 
1363 /************************************************************************/
1364 /*                       ComputeRasterMinMax()                          */
1365 /************************************************************************/
1366 
ComputeRasterMinMax(int bApproxOK,double * adfMinMax)1367 CPLErr VRTDerivedRasterBand::ComputeRasterMinMax( int bApproxOK, double* adfMinMax )
1368 {
1369     return GDALRasterBand::ComputeRasterMinMax( bApproxOK, adfMinMax );
1370 }
1371 
1372 /************************************************************************/
1373 /*                         ComputeStatistics()                          */
1374 /************************************************************************/
1375 
1376 CPLErr
ComputeStatistics(int bApproxOK,double * pdfMin,double * pdfMax,double * pdfMean,double * pdfStdDev,GDALProgressFunc pfnProgress,void * pProgressData)1377 VRTDerivedRasterBand::ComputeStatistics( int bApproxOK,
1378                                          double *pdfMin, double *pdfMax,
1379                                          double *pdfMean, double *pdfStdDev,
1380                                          GDALProgressFunc pfnProgress,
1381                                          void *pProgressData )
1382 
1383 {
1384     return GDALRasterBand::ComputeStatistics(  bApproxOK,
1385                                             pdfMin, pdfMax,
1386                                             pdfMean, pdfStdDev,
1387                                             pfnProgress, pProgressData );
1388 }
1389 
1390 /************************************************************************/
1391 /*                            GetHistogram()                            */
1392 /************************************************************************/
1393 
GetHistogram(double dfMin,double dfMax,int nBuckets,GUIntBig * panHistogram,int bIncludeOutOfRange,int bApproxOK,GDALProgressFunc pfnProgress,void * pProgressData)1394 CPLErr VRTDerivedRasterBand::GetHistogram( double dfMin, double dfMax,
1395                                            int nBuckets, GUIntBig *panHistogram,
1396                                            int bIncludeOutOfRange, int bApproxOK,
1397                                            GDALProgressFunc pfnProgress,
1398                                            void *pProgressData )
1399 
1400 {
1401     return VRTRasterBand::GetHistogram( dfMin, dfMax,
1402                                             nBuckets, panHistogram,
1403                                             bIncludeOutOfRange, bApproxOK,
1404                                             pfnProgress, pProgressData );
1405 }
1406 
1407 /*! @endcond */
1408