1 /******************************************************************************
2 *
3 * Project: GDAL Core
4 * Purpose: Python interface
5 * Author: Even Rouault, <even dot rouault at spatialys dot com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2017-2019, Even Rouault, <even dot rouault at spatialys dot com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29 #include "cpl_conv.h"
30 #include "cpl_error.h"
31 #include "cpl_string.h"
32 #include "cpl_spawn.h"
33 #include "gdalpython.h"
34
35 #include <algorithm>
36 #include <mutex>
37 #include <set>
38 #include <vector>
39
40 using namespace GDALPy;
41
42 typedef struct PyThreadState_t PyThreadState;
43
44 static PyThreadState* (*PyEval_SaveThread)(void) = nullptr;
45 static void (*PyEval_RestoreThread)(PyThreadState*) = nullptr;
46 static void (*Py_Finalize)(void) = nullptr;
47 static void (*Py_InitializeEx)(int) = nullptr;
48 static void (*PyEval_InitThreads)(void) = nullptr;
49 static PyObject* (*Py_CompileStringExFlags)(const char*, const char*, int, void*, int) = nullptr;
50
51 static std::mutex gMutex;
52 static bool gbHasInitializedPython = false;
53 static PyThreadState* gphThreadState = nullptr;
54
55 // Emulate Py_CompileString with Py_CompileStringExFlags
56 // Probably just a temporary measure for a bug of Python 3.8.0 on Windows
57 // https://bugs.python.org/issue37633
GDAL_Py_CompileString(const char * str,const char * filename,int start)58 static PyObject* GDAL_Py_CompileString(const char *str, const char *filename, int start)
59 {
60 return Py_CompileStringExFlags(str, filename, start, nullptr, -1);
61 }
62
63 namespace GDALPy
64 {
65 int (*Py_IsInitialized)(void) = nullptr;
66 PyGILState_STATE (*PyGILState_Ensure)(void) = nullptr;
67 void (*PyGILState_Release)(PyGILState_STATE) = nullptr;
68 void (*Py_SetProgramName)(const char*) = nullptr;
69 PyObject* (*PyObject_Type)(PyObject*) = nullptr;
70 int (*PyObject_IsInstance)(PyObject*, PyObject*) = nullptr;
71 PyObject* (*PyTuple_New)(size_t) = nullptr;
72 PyObject* (*PyBool_FromLong)(long) = nullptr;
73 PyObject* (*PyLong_FromLong)(long) = nullptr;
74 long (*PyLong_AsLong)(PyObject *) = nullptr;
75 PyObject* (*PyLong_FromLongLong)(GIntBig) = nullptr;
76 GIntBig (*PyLong_AsLongLong)(PyObject *) = nullptr;
77 PyObject* (*PyFloat_FromDouble)(double) = nullptr;
78 double (*PyFloat_AsDouble)(PyObject*) = nullptr;
79 PyObject* (*PyObject_Call)(PyObject*, PyObject*, PyObject*) = nullptr;
80 PyObject* (*PyObject_GetIter)(PyObject*) = nullptr;
81 PyObject* (*PyIter_Next)(PyObject*) = nullptr;
82 void (*Py_IncRef)(PyObject*) = nullptr;
83 void (*Py_DecRef)(PyObject*) = nullptr;
84 PyObject* (*PyErr_Occurred)(void) = nullptr;
85 void (*PyErr_Print)(void) = nullptr;
86
87 PyObject* (*Py_CompileString)(const char*, const char*, int) = nullptr;
88 PyObject* (*PyImport_ExecCodeModule)(const char*, PyObject*) = nullptr;
89 int (*PyObject_HasAttrString)(PyObject*, const char*) = nullptr;
90 PyObject* (*PyObject_GetAttrString)(PyObject*, const char*) = nullptr;
91 int (*PyObject_SetAttrString)(PyObject*, const char*, PyObject*) = nullptr;
92 int (*PyTuple_SetItem)(PyObject *, size_t, PyObject *) = nullptr;
93 void (*PyObject_Print)(PyObject*,FILE*,int) = nullptr;
94 Py_ssize_t (*PyBytes_Size)(PyObject *) = nullptr;
95 const char* (*PyBytes_AsString)(PyObject*) = nullptr;
96 PyObject* (*PyBytes_FromStringAndSize)(const void*, size_t) = nullptr;
97 PyObject* (*PyUnicode_FromString)(const char*) = nullptr;
98 PyObject* (*PyUnicode_AsUTF8String)(PyObject *) = nullptr;
99 PyObject* (*PyImport_ImportModule)(const char*) = nullptr;
100 int (*PyCallable_Check)(PyObject*) = nullptr;
101 PyObject* (*PyDict_New)(void) = nullptr;
102 int (*PyDict_SetItemString)(PyObject *p, const char *key,
103 PyObject *val) = nullptr;
104 int (*PyDict_Next)(PyObject *p, size_t *, PyObject **, PyObject **) = nullptr;
105 PyObject* (*PyDict_GetItemString)(PyObject *p, const char *key) = nullptr;
106 PyObject* (*PyList_New)(Py_ssize_t) = nullptr;
107 int (*PyList_SetItem)(PyObject *, Py_ssize_t , PyObject *) = nullptr;
108 int (*PyArg_ParseTuple)(PyObject *, const char *, ...) = nullptr;
109
110 int (*PySequence_Check)(PyObject *o) = nullptr;
111 Py_ssize_t (*PySequence_Size)(PyObject *o) = nullptr;
112 PyObject* (*PySequence_GetItem)(PyObject *o, Py_ssize_t i) = nullptr;
113
114 void (*PyErr_Fetch)(PyObject **poPyType, PyObject **poPyValue,
115 PyObject **poPyTraceback) = nullptr;
116 void (*PyErr_Clear)(void) = nullptr;
117 const char* (*Py_GetVersion)(void) = nullptr;
118
119 int (*PyBuffer_FillInfo)(Py_buffer *view, PyObject *obj, void *buf,
120 size_t len, int readonly, int infoflags) = nullptr;
121 PyObject* (*PyMemoryView_FromBuffer)(Py_buffer *view) = nullptr;
122
123 PyObject * (*PyModule_Create2)(struct PyModuleDef*, int) = nullptr;
124 }
125
126 /* MinGW32 might define HAVE_DLFCN_H, so skip the unix implementation */
127 #if defined(HAVE_DLFCN_H) && !defined(WIN32)
128
129 #include <dlfcn.h>
130
131 typedef void* LibraryHandle;
132
133 #define LOAD_NOCHECK_WITH_NAME(libHandle, x, name) \
134 do { \
135 void* ptr = dlsym(libHandle, name); \
136 memcpy(&x, &ptr, sizeof(void*)); \
137 } while(0)
138
139 #elif defined(WIN32)
140
141 #include <windows.h>
142 #include <psapi.h>
143
144 typedef HMODULE LibraryHandle;
145
146 #define LOAD_NOCHECK_WITH_NAME(libHandle, x, name) \
147 do { \
148 FARPROC ptr = GetProcAddress(libHandle, name); \
149 memcpy(&x, &ptr, sizeof(void*)); \
150 } while(0)
151
152 #endif
153
154 #define STRINGIFY(x) #x
155
156 #define LOAD_NOCHECK(libHandle, x) LOAD_NOCHECK_WITH_NAME(libHandle, x, STRINGIFY(x))
157 #define LOAD_WITH_NAME(libHandle, x, name) \
158 do { \
159 LOAD_NOCHECK_WITH_NAME(libHandle, x, name); \
160 if (!x) \
161 { \
162 CPLError(CE_Failure, CPLE_AppDefined, "Cannot find %s", name); \
163 return false; \
164 } \
165 } while(0)
166 #define LOAD(libHandle, x) LOAD_WITH_NAME(libHandle, x, STRINGIFY(x))
167
168 /************************************************************************/
169 /* LoadPythonAPI() */
170 /************************************************************************/
171
172 #if defined(LOAD_NOCHECK_WITH_NAME) && defined(HAVE_DLFCN_H) && !defined(WIN32)
173 static LibraryHandle libHandleStatic = nullptr;
174 #endif
175
176 /** Load the subset of the Python C API that we need */
LoadPythonAPI()177 static bool LoadPythonAPI()
178 {
179 static bool bInit = false;
180 if( bInit )
181 return true;
182
183 #ifdef LOAD_NOCHECK_WITH_NAME
184 // The static here is just to avoid Coverity warning about resource leak.
185 LibraryHandle libHandle = nullptr;
186
187 const char* pszPythonSO = CPLGetConfigOption("PYTHONSO", nullptr);
188 #if defined(HAVE_DLFCN_H) && !defined(WIN32)
189
190 // First try in the current process in case the python symbols would
191 // be already loaded
192 (void) libHandle;
193 libHandle = dlopen(nullptr, RTLD_LAZY);
194 libHandleStatic = libHandle;
195 if( libHandle != nullptr &&
196 dlsym(libHandle, "Py_SetProgramName") != nullptr )
197 {
198 CPLDebug("GDAL", "Current process has python symbols loaded");
199 }
200 else
201 {
202 libHandle = nullptr;
203 }
204
205 // Then try the user provided shared object name
206 if( libHandle == nullptr && pszPythonSO != nullptr )
207 {
208 // coverity[tainted_string]
209 libHandle = dlopen(pszPythonSO, RTLD_NOW | RTLD_GLOBAL);
210 if( libHandle == nullptr )
211 {
212 CPLError(CE_Failure, CPLE_AppDefined,
213 "Cannot load %s",
214 pszPythonSO);
215 return false;
216 }
217 if( dlsym(libHandle, "Py_SetProgramName") == nullptr )
218 {
219 CPLError(CE_Failure, CPLE_AppDefined,
220 "Cannot find Py_SetProgramName symbol in %s",
221 pszPythonSO);
222 return false;
223 }
224 }
225
226 // Then try the PYTHONSO_DEFAULT if defined at compile time
227 #ifdef PYTHONSO_DEFAULT
228 if( libHandle == nullptr )
229 {
230 libHandle = dlopen(PYTHONSO_DEFAULT, RTLD_NOW | RTLD_GLOBAL);
231 if( !libHandle )
232 {
233 CPLDebug("GDAL", "%s found", PYTHONSO_DEFAULT);
234 }
235 }
236 #endif
237
238 #if defined(__MACH__) && defined(__APPLE__)
239 #define SO_EXT "dylib"
240 #else
241 #define IS_SO_EXT
242 #define SO_EXT "so"
243 #endif
244
245 const auto tryDlopen = [](CPLString osPythonSO)
246 {
247 CPLDebug("GDAL", "Trying %s", osPythonSO.c_str());
248 auto l_libHandle = dlopen(osPythonSO.c_str(), RTLD_NOW | RTLD_GLOBAL);
249 #ifdef IS_SO_EXT
250 if( l_libHandle == nullptr )
251 {
252 osPythonSO += ".1.0";
253 CPLDebug("GDAL", "Trying %s", osPythonSO.c_str());
254 l_libHandle = dlopen(osPythonSO.c_str(), RTLD_NOW | RTLD_GLOBAL);
255 }
256 #endif
257 return l_libHandle;
258 };
259
260 // Then try to find the libpython that corresponds to the python binary
261 // in the PATH
262 if( libHandle == nullptr )
263 {
264 CPLString osVersion;
265 char* pszPath = getenv("PATH");
266 if( pszPath != nullptr
267 #ifdef DEBUG
268 // For testing purposes
269 && CPLTestBool( CPLGetConfigOption(
270 "GDAL_ENABLE_PYTHON_PATH", "YES") )
271 #endif
272 )
273 {
274 char** papszTokens = CSLTokenizeString2(pszPath, ":", 0);
275 for( int iTry = 0; iTry < 2; ++iTry )
276 {
277 for( char** papszIter = papszTokens;
278 papszIter != nullptr && *papszIter != nullptr;
279 ++papszIter )
280 {
281 struct stat sStat;
282 CPLString osPythonBinary(
283 CPLFormFilename(*papszIter, "python", nullptr));
284 if( iTry == 1 )
285 osPythonBinary += "3";
286 if( lstat(osPythonBinary, &sStat) != 0 )
287 continue;
288
289 CPLDebug("GDAL", "Found %s", osPythonBinary.c_str());
290
291 if( S_ISLNK(sStat.st_mode)
292 #ifdef DEBUG
293 // For testing purposes
294 && CPLTestBool( CPLGetConfigOption(
295 "GDAL_ENABLE_PYTHON_SYMLINK", "YES") )
296 #endif
297 )
298 {
299 std::set<std::string> oSetAlreadyTriedLinks;
300 while( true )
301 {
302 oSetAlreadyTriedLinks.insert(osPythonBinary);
303
304 // If this is a symlink, hopefully the resolved
305 // name will be like "python3.6"
306 const int nBufSize = 2048;
307 std::vector<char> oFilename(nBufSize);
308 char *szPointerFilename = &oFilename[0];
309 int nBytes = static_cast<int>(
310 readlink( osPythonBinary, szPointerFilename,
311 nBufSize ) );
312 if (nBytes != -1)
313 {
314 szPointerFilename[std::min(nBytes,
315 nBufSize - 1)] = 0;
316 CPLString osFilename(
317 CPLGetFilename(szPointerFilename));
318 CPLDebug("GDAL", "Which is an alias to: %s",
319 szPointerFilename);
320
321 if( STARTS_WITH(osFilename, "python") )
322 {
323 CPLString osResolvedFullLink;
324 // If the filename is again a symlink,
325 // resolve it
326 if( CPLIsFilenameRelative(osFilename) )
327 {
328 osResolvedFullLink = CPLFormFilename(
329 CPLGetPath(osPythonBinary), osFilename, nullptr );
330 }
331 else
332 {
333 osResolvedFullLink = osFilename;
334 }
335 if( oSetAlreadyTriedLinks.find(osResolvedFullLink) ==
336 oSetAlreadyTriedLinks.end() &&
337 lstat(osResolvedFullLink, &sStat) == 0 &&
338 S_ISLNK(sStat.st_mode) )
339 {
340 osPythonBinary = osResolvedFullLink;
341 continue;
342 }
343
344 osVersion = osFilename.substr(strlen("python"));
345 CPLDebug("GDAL",
346 "Python version from binary name: %s",
347 osVersion.c_str());
348 }
349 }
350 else
351 {
352 CPLDebug("GDAL", "realink(%s) failed",
353 osPythonBinary.c_str());
354 }
355 break;
356 }
357 }
358
359 // Otherwise, expensive way: start the binary and ask
360 // it for its version...
361 if( osVersion.empty() )
362 {
363 const char* pszPrintVersion =
364 "import sys; print(str(sys.version_info[0]) +"
365 "'.' + str(sys.version_info[1]))";
366 const char* const apszArgv[] = {
367 osPythonBinary.c_str(), "-c",
368 pszPrintVersion,
369 nullptr };
370 const CPLString osTmpFilename(
371 "/vsimem/LoadPythonAPI/out.txt");
372 VSILFILE* fout = VSIFOpenL( osTmpFilename, "wb+");
373 if( CPLSpawn( apszArgv, nullptr, fout, FALSE ) == 0 )
374 {
375 char* pszStr = reinterpret_cast<char*>(
376 VSIGetMemFileBuffer( osTmpFilename,
377 nullptr, FALSE ));
378 osVersion = pszStr;
379 if( !osVersion.empty() &&
380 osVersion.back() == '\n' )
381 {
382 osVersion.resize(osVersion.size() - 1);
383 }
384 CPLDebug("GDAL", "Python version from binary: %s",
385 osVersion.c_str());
386 }
387 VSIFCloseL(fout);
388 VSIUnlink(osTmpFilename);
389 }
390 break;
391 }
392 if( !osVersion.empty() )
393 break;
394 }
395 CSLDestroy(papszTokens);
396 }
397
398 if( !osVersion.empty() )
399 {
400 libHandle = tryDlopen("libpython" + osVersion + "." SO_EXT);
401 if( libHandle != nullptr )
402 {
403 CPLDebug("GDAL", "... success");
404 }
405 else if( osVersion[0] == '3' )
406 {
407 libHandle = tryDlopen("libpython" + osVersion + "m." SO_EXT);
408 if( libHandle != nullptr )
409 {
410 CPLDebug("GDAL", "... success");
411 }
412 }
413 }
414 }
415
416 // Otherwise probe a few known objects.
417 // Note: update doc/source/drivers/raster/vrt.rst if change
418 if( libHandle == nullptr )
419 {
420 const char* const apszPythonSO[] = {
421 "libpython3.6m." SO_EXT,
422 "libpython3.7m." SO_EXT,
423 "libpython3.8m." SO_EXT,
424 "libpython3.9m." SO_EXT,
425 "libpython3.5m." SO_EXT,
426 "libpython3.4m." SO_EXT,
427 "libpython3.3." SO_EXT,
428 "libpython3.2." SO_EXT };
429 for( size_t i = 0; libHandle == nullptr &&
430 i < CPL_ARRAYSIZE(apszPythonSO); ++i )
431 {
432 libHandle = tryDlopen(apszPythonSO[i]);
433 if( libHandle != nullptr )
434 CPLDebug("GDAL", "... success");
435 }
436 }
437
438 #elif defined(WIN32)
439
440 // First try in the current process in case the python symbols would
441 // be already loaded
442 HANDLE hProcess = GetCurrentProcess();
443 std::vector<HMODULE> ahModules;
444
445 // 100 is not large enough when GDAL is loaded from QGIS for example
446 ahModules.resize(1000);
447 for( int i = 0; i < 2; i++ )
448 {
449 DWORD nSizeNeeded = 0;
450 const DWORD nSizeIn = static_cast<DWORD>(
451 ahModules.size() * sizeof(HMODULE));
452 EnumProcessModules(hProcess, &ahModules[0], nSizeIn, &nSizeNeeded);
453 ahModules.resize(static_cast<size_t>(nSizeNeeded) / sizeof(HMODULE));
454 if( nSizeNeeded <= nSizeIn )
455 {
456 break;
457 }
458 }
459
460 for( size_t i = 0; i < ahModules.size(); i++ )
461 {
462 if( GetProcAddress(ahModules[i], "Py_SetProgramName") )
463 {
464 libHandle = ahModules[i];
465 CPLDebug("GDAL", "Current process has python symbols loaded");
466 break;
467 }
468 }
469
470 // Then try the user provided shared object name
471 if( libHandle == nullptr && pszPythonSO != nullptr )
472 {
473 UINT uOldErrorMode;
474 /* Avoid error boxes to pop up (#5211, #5525) */
475 uOldErrorMode = SetErrorMode(SEM_NOOPENFILEERRORBOX |
476 SEM_FAILCRITICALERRORS);
477
478 #if (defined(WIN32) && _MSC_VER >= 1310) || __MSVCRT_VERSION__ >= 0x0601
479 if( CPLTestBool( CPLGetConfigOption( "GDAL_FILENAME_IS_UTF8", "YES" ) ) )
480 {
481 wchar_t *pwszFilename =
482 CPLRecodeToWChar( pszPythonSO, CPL_ENC_UTF8, CPL_ENC_UCS2 );
483 libHandle = LoadLibraryW(pwszFilename);
484 CPLFree( pwszFilename );
485 }
486 else
487 #endif
488 {
489 libHandle = LoadLibrary(pszPythonSO);
490 }
491
492 SetErrorMode(uOldErrorMode);
493
494 if( libHandle == nullptr )
495 {
496 CPLError(CE_Failure, CPLE_AppDefined,
497 "Cannot load %s",
498 pszPythonSO);
499 return false;
500 }
501 if( GetProcAddress(libHandle, "Py_SetProgramName") == nullptr )
502 {
503 CPLError(CE_Failure, CPLE_AppDefined,
504 "Cannot find Py_SetProgramName symbol in %s",
505 pszPythonSO);
506 return false;
507 }
508 }
509
510 // Then try the PYTHONSO_DEFAULT if defined at compile time
511 #ifdef PYTHONSO_DEFAULT
512 if( libHandle == nullptr )
513 {
514 UINT uOldErrorMode;
515 uOldErrorMode = SetErrorMode(SEM_NOOPENFILEERRORBOX |
516 SEM_FAILCRITICALERRORS);
517
518 libHandle = LoadLibrary(PYTHONSO_DEFAULT);
519 SetErrorMode(uOldErrorMode);
520 if( !libHandle )
521 {
522 CPLDebug("GDAL", "%s found", PYTHONSO_DEFAULT);
523 }
524 }
525 #endif
526
527 // Then try to find the pythonXY.dll that corresponds to the python binary
528 // in the PATH
529 if( libHandle == nullptr )
530 {
531 CPLString osDLLName;
532 char* pszPath = getenv("PATH");
533 if( pszPath != nullptr
534 #ifdef DEBUG
535 // For testing purposes
536 && CPLTestBool( CPLGetConfigOption(
537 "GDAL_ENABLE_PYTHON_PATH", "YES") )
538 #endif
539 )
540 {
541 char** papszTokens = CSLTokenizeString2(pszPath, ";", 0);
542 for( int iTry = 0; iTry < 2; ++iTry )
543 {
544 for( char** papszIter = papszTokens;
545 papszIter != nullptr && *papszIter != nullptr;
546 ++papszIter )
547 {
548 VSIStatBufL sStat;
549 CPLString osPythonBinary(
550 CPLFormFilename(*papszIter, "python.exe", nullptr));
551 if( iTry == 1 )
552 osPythonBinary += "3";
553 if( VSIStatL(osPythonBinary, &sStat) != 0 )
554 continue;
555
556 CPLDebug("GDAL", "Found %s", osPythonBinary.c_str());
557
558 // In python2.7, the dll is in the same directory as the exe
559 char** papszFiles = VSIReadDir(*papszIter);
560 for( char** papszFileIter = papszFiles;
561 papszFileIter != nullptr && *papszFileIter != nullptr;
562 ++papszFileIter )
563 {
564 if( STARTS_WITH_CI(*papszFileIter, "python") &&
565 !EQUAL(*papszFileIter, "python3.dll") &&
566 EQUAL(CPLGetExtension(*papszFileIter), "dll") )
567 {
568 osDLLName = CPLFormFilename(*papszIter,
569 *papszFileIter,
570 nullptr);
571 break;
572 }
573 }
574 CSLDestroy(papszFiles);
575
576 // In python3.2, the dll is in the DLLs subdirectory
577 if( osDLLName.empty() )
578 {
579 CPLString osDLLsDir(
580 CPLFormFilename(*papszIter, "DLLs", nullptr));
581 papszFiles = VSIReadDir( osDLLsDir );
582 for( char** papszFileIter = papszFiles;
583 papszFileIter != nullptr && *papszFileIter != nullptr;
584 ++papszFileIter )
585 {
586 if( STARTS_WITH_CI(*papszFileIter, "python") &&
587 EQUAL(CPLGetExtension(*papszFileIter), "dll") )
588 {
589 osDLLName = CPLFormFilename(osDLLsDir,
590 *papszFileIter,
591 nullptr);
592 break;
593 }
594 }
595 CSLDestroy(papszFiles);
596 }
597
598 break;
599 }
600 if( !osDLLName.empty() )
601 break;
602 }
603 CSLDestroy(papszTokens);
604 }
605
606 if( !osDLLName.empty() )
607 {
608 //CPLDebug("GDAL", "Trying %s", osDLLName.c_str());
609 UINT uOldErrorMode;
610 uOldErrorMode = SetErrorMode(SEM_NOOPENFILEERRORBOX |
611 SEM_FAILCRITICALERRORS);
612 libHandle = LoadLibrary(osDLLName);
613 SetErrorMode(uOldErrorMode);
614 if( libHandle != nullptr )
615 {
616 CPLDebug("GDAL", "%s loaded", osDLLName.c_str());
617 }
618 }
619 }
620
621 // Otherwise probe a few known objects
622 // Note: update doc/source/drivers/raster/vrt.rst if change
623 if( libHandle == nullptr )
624 {
625 const char* const apszPythonSO[] = {"python36.dll",
626 "python37.dll",
627 "python38.dll",
628 "python39.dll",
629 "python35.dll",
630 "python34.dll",
631 "python33.dll",
632 "python32.dll" };
633 UINT uOldErrorMode;
634 uOldErrorMode = SetErrorMode(SEM_NOOPENFILEERRORBOX |
635 SEM_FAILCRITICALERRORS);
636
637 for( size_t i = 0; libHandle == nullptr &&
638 i < CPL_ARRAYSIZE(apszPythonSO); ++i )
639 {
640 CPLDebug("GAL", "Trying %s", apszPythonSO[i]);
641 libHandle = LoadLibrary(apszPythonSO[i]);
642 if( libHandle != nullptr )
643 CPLDebug("GDAL", "... success");
644 }
645 SetErrorMode(uOldErrorMode);
646 }
647 #endif
648 if( !libHandle )
649 {
650 CPLError(CE_Failure, CPLE_AppDefined,
651 "Cannot find python/libpython. You can set the PYTHONSO "
652 "configuration option to point to the a python .so/.dll/.dylib");
653 return false;
654 }
655
656 LOAD(libHandle, Py_SetProgramName);
657 LOAD(libHandle, PyBuffer_FillInfo);
658 LOAD(libHandle, PyMemoryView_FromBuffer);
659 LOAD(libHandle, PyObject_Type);
660 LOAD(libHandle, PyObject_IsInstance);
661 LOAD(libHandle, PyTuple_New);
662 LOAD(libHandle, PyBool_FromLong);
663 LOAD(libHandle, PyLong_FromLong);
664 LOAD(libHandle, PyLong_AsLong);
665 LOAD(libHandle, PyLong_FromLongLong);
666 LOAD(libHandle, PyLong_AsLongLong);
667 LOAD(libHandle, PyBytes_Size);
668 LOAD(libHandle, PyBytes_AsString);
669 LOAD(libHandle, PyBytes_FromStringAndSize);
670
671 LOAD(libHandle, PyModule_Create2);
672
673 LOAD_NOCHECK_WITH_NAME(libHandle, PyUnicode_FromString,
674 "PyUnicode_FromString");
675 if( PyUnicode_FromString == nullptr )
676 {
677 LOAD_NOCHECK_WITH_NAME(libHandle, PyUnicode_FromString,
678 "PyUnicodeUCS2_FromString");
679 }
680 if( PyUnicode_FromString == nullptr )
681 {
682 LOAD_WITH_NAME(libHandle, PyUnicode_FromString,
683 "PyUnicodeUCS4_FromString");
684 }
685 LOAD_NOCHECK_WITH_NAME(libHandle, PyUnicode_AsUTF8String,
686 "PyUnicode_AsUTF8String");
687 if( PyUnicode_AsUTF8String == nullptr )
688 {
689 LOAD_NOCHECK_WITH_NAME(libHandle, PyUnicode_AsUTF8String,
690 "PyUnicodeUCS2_AsUTF8String");
691 }
692 if( PyUnicode_AsUTF8String == nullptr )
693 {
694 LOAD_WITH_NAME(libHandle, PyUnicode_AsUTF8String,
695 "PyUnicodeUCS4_AsUTF8String");
696 }
697
698 LOAD(libHandle, PyFloat_FromDouble);
699 LOAD(libHandle, PyFloat_AsDouble);
700 LOAD(libHandle, PyObject_Call);
701 LOAD(libHandle, PyObject_GetIter);
702 LOAD(libHandle, PyIter_Next);
703 LOAD(libHandle, Py_IncRef);
704 LOAD(libHandle, Py_DecRef);
705 LOAD(libHandle, PyErr_Occurred);
706 LOAD(libHandle, PyErr_Print);
707 LOAD(libHandle, Py_IsInitialized);
708 LOAD(libHandle, Py_InitializeEx);
709 LOAD(libHandle, PyEval_InitThreads);
710 LOAD(libHandle, PyEval_SaveThread);
711 LOAD(libHandle, PyEval_RestoreThread);
712 LOAD(libHandle, Py_Finalize);
713 LOAD_NOCHECK(libHandle, Py_CompileString);
714 if( Py_CompileString == nullptr )
715 {
716 // Probably just a temporary measure for a bug of Python 3.8.0 on Windows
717 // https://bugs.python.org/issue37633
718 LOAD(libHandle, Py_CompileStringExFlags);
719 Py_CompileString = GDAL_Py_CompileString;
720 }
721 LOAD(libHandle, PyImport_ExecCodeModule);
722 LOAD(libHandle, PyObject_HasAttrString);
723 LOAD(libHandle, PyObject_GetAttrString);
724 LOAD(libHandle, PyObject_SetAttrString);
725 LOAD(libHandle, PyTuple_SetItem);
726 LOAD(libHandle, PyObject_Print);
727 LOAD(libHandle, PyImport_ImportModule);
728 LOAD(libHandle, PyCallable_Check);
729 LOAD(libHandle, PyDict_New);
730 LOAD(libHandle, PyDict_SetItemString);
731 LOAD(libHandle, PyDict_Next);
732 LOAD(libHandle, PyDict_GetItemString);
733 LOAD(libHandle, PyList_New);
734 LOAD(libHandle, PyList_SetItem);
735 LOAD(libHandle, PySequence_Check);
736 LOAD(libHandle, PySequence_Size);
737 LOAD(libHandle, PySequence_GetItem);
738 LOAD(libHandle, PyArg_ParseTuple);
739 LOAD(libHandle, PyGILState_Ensure);
740 LOAD(libHandle, PyGILState_Release);
741 LOAD(libHandle, PyErr_Fetch);
742 LOAD(libHandle, PyErr_Clear);
743 LOAD(libHandle, Py_GetVersion);
744
745 CPLString osPythonVersion(Py_GetVersion());
746 osPythonVersion.replaceAll("\r\n", ' ');
747 osPythonVersion.replaceAll('\n', ' ');
748 CPLDebug("GDAL", "Python version used: %s", osPythonVersion.c_str());
749
750 #else // LOAD_NOCHECK_WITH_NAME
751 CPLError(CE_Failure, CPLE_AppDefined,
752 "This platform doesn't support dynamic loading of libraries")
753 return false;
754 #endif // LOAD_NOCHECK_WITH_NAME
755
756 bInit = true;
757 return bInit;
758 }
759
760 //! @cond Doxygen_Suppress
761
762 /************************************************************************/
763 /* GDALPythonInitialize() */
764 /************************************************************************/
765
766 /** Call this to initialize the Python environment.
767 */
GDALPythonInitialize()768 bool GDALPythonInitialize()
769 {
770 std::lock_guard<std::mutex> guard(gMutex);
771
772 if( !LoadPythonAPI() )
773 return false;
774
775 int bIsInitialized = Py_IsInitialized();
776 if( !bIsInitialized)
777 {
778 gbHasInitializedPython = true;
779 Py_InitializeEx(0);
780 CPLDebug("GDAL", "Py_Initialize()");
781 PyEval_InitThreads();
782 gphThreadState = PyEval_SaveThread();
783 }
784
785 return true;
786 }
787
788 /************************************************************************/
789 /* GDALPythonFinalize() */
790 /************************************************************************/
791
792 /** To be called by GDALDestroy() */
GDALPythonFinalize()793 void GDALPythonFinalize()
794 {
795 if( gbHasInitializedPython )
796 {
797 CPLDebug("GDAL", "Py_Finalize() = %p", Py_Finalize);
798 PyEval_RestoreThread(gphThreadState);
799 Py_Finalize();
800 gbHasInitializedPython = false;
801 gphThreadState = nullptr;
802 }
803 }
804
805 namespace GDALPy
806 {
807
808 /************************************************************************/
809 /* GIL_Holder() */
810 /************************************************************************/
811
GIL_Holder(bool bExclusiveLock)812 GIL_Holder::GIL_Holder(bool bExclusiveLock):
813 m_bExclusiveLock(bExclusiveLock)
814 {
815 if( bExclusiveLock )
816 {
817 gMutex.lock();
818 }
819 m_eState = PyGILState_Ensure();
820 }
821
822 /************************************************************************/
823 /* ~GIL_Holder() */
824 /************************************************************************/
825
~GIL_Holder()826 GIL_Holder::~GIL_Holder()
827 {
828 PyGILState_Release(m_eState);
829 if( m_bExclusiveLock )
830 {
831 gMutex.unlock();
832 }
833 else
834 {
835 }
836 }
837
838 /************************************************************************/
839 /* GetString() */
840 /************************************************************************/
841
GetString(PyObject * obj,bool bEmitError)842 CPLString GetString(PyObject* obj, bool bEmitError)
843 {
844 PyObject* unicode = PyUnicode_AsUTF8String(obj);
845 if( PyErr_Occurred() )
846 {
847 if( bEmitError)
848 {
849 CPLError(CE_Failure, CPLE_AppDefined,
850 "%s", GetPyExceptionString().c_str());
851 }
852 return CPLString();
853 }
854
855 const char* pszRet = PyBytes_AsString(unicode);
856 CPLString osRet = pszRet ? pszRet : "";
857 Py_DecRef(unicode);
858 return osRet;
859 }
860
861 /************************************************************************/
862 /* GetPyExceptionString() */
863 /************************************************************************/
864
GetPyExceptionString()865 CPLString GetPyExceptionString()
866 {
867 PyObject *poPyType = nullptr;
868 PyObject *poPyValue = nullptr;
869 PyObject *poPyTraceback = nullptr;
870
871 PyErr_Fetch(&poPyType, &poPyValue, &poPyTraceback);
872 if( poPyType )
873 Py_IncRef(poPyType);
874 if( poPyValue )
875 Py_IncRef(poPyValue);
876 if( poPyTraceback )
877 Py_IncRef(poPyTraceback);
878
879 // This is a mess. traceback.format_exception/format_exception_only
880 // sometimes throw exceptions themselves !
881 CPLString osPythonCode(
882 "import traceback\n"
883 "\n"
884 "def GDALFormatException2(etype, value):\n"
885 " try:\n"
886 " return ''.join(traceback.format_exception_only(etype, value))\n"
887 " except:\n"
888 " return (str(etype) + ', ' + str(value))\n"
889 "\n"
890 "def GDALFormatException3(etype, value, tb):\n"
891 //" print(etype, value, tb)\n"
892 " try:\n"
893 " return ''.join(traceback.format_exception(etype, value, tb))\n"
894 " except:\n"
895 " return (str(etype) + ', ' + str(value))\n");
896
897 CPLString osRet("An exception occurred in exception formatting code...");
898
899 static int nCounter = 0;
900 CPLString osModuleName( CPLSPrintf("gdal_exception_%d", nCounter));
901 PyObject* poCompiledString = Py_CompileString(osPythonCode,
902 osModuleName, Py_file_input);
903 if( poCompiledString == nullptr || PyErr_Occurred() )
904 {
905 PyErr_Print();
906 }
907 else
908 {
909 PyObject* poModule =
910 PyImport_ExecCodeModule(osModuleName, poCompiledString);
911 CPLAssert(poModule);
912
913 Py_DecRef(poCompiledString);
914
915 PyObject* poPyGDALFormatException2 = PyObject_GetAttrString(poModule,
916 "GDALFormatException2" );
917 CPLAssert(poPyGDALFormatException2);
918
919 PyObject* poPyGDALFormatException3 = PyObject_GetAttrString(poModule,
920 "GDALFormatException3" );
921 CPLAssert(poPyGDALFormatException3);
922
923 Py_DecRef(poModule);
924
925 PyObject* pyArgs = PyTuple_New( poPyTraceback ? 3 : 2);
926 PyTuple_SetItem(pyArgs, 0, poPyType);
927 PyTuple_SetItem(pyArgs, 1, poPyValue);
928 if( poPyTraceback )
929 PyTuple_SetItem(pyArgs, 2, poPyTraceback );
930 PyObject* poPyRet = PyObject_Call(
931 poPyTraceback ? poPyGDALFormatException3 : poPyGDALFormatException2,
932 pyArgs, nullptr );
933 Py_DecRef(pyArgs);
934
935 if( PyErr_Occurred() )
936 {
937 osRet = "An exception occurred in exception formatting code...";
938 PyErr_Print();
939 }
940 else
941 {
942 osRet = GetString(poPyRet, false);
943 Py_DecRef(poPyRet);
944 }
945
946 Py_DecRef(poPyGDALFormatException2);
947 Py_DecRef(poPyGDALFormatException3);
948 }
949
950 if( poPyType )
951 Py_DecRef(poPyType);
952 if( poPyValue )
953 Py_DecRef(poPyValue);
954 if( poPyTraceback )
955 Py_DecRef(poPyTraceback);
956
957 return osRet;
958 }
959
960 /************************************************************************/
961 /* ErrOccurredEmitCPLError() */
962 /************************************************************************/
963
ErrOccurredEmitCPLError()964 bool ErrOccurredEmitCPLError()
965 {
966 if (PyErr_Occurred())
967 {
968 CPLError(CE_Failure, CPLE_AppDefined,
969 "%s", GetPyExceptionString().c_str());
970 return true;
971 }
972 return false;
973 }
974
975 } // namespace GDALPy
976
977 //! @endcond
978
979