1 /************************************************************************
2 **
3 ** Copyright (C) 2015-2021 Kevin Hendricks
4 ** Copyright (C) 2015 John Schember <john@nachtimwald.com>
5 **
6 ** This file is part of Sigil.
7 **
8 ** Sigil is free software: you can redistribute it and/or modify
9 ** it under the terms of the GNU General Public License as published by
10 ** the Free Software Foundation, either version 3 of the License, or
11 ** (at your option) any later version.
12 **
13 ** Sigil is distributed in the hope that it will be useful,
14 ** but WITHOUT ANY WARRANTY; without even the implied warranty of
15 ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 ** GNU General Public License for more details.
17 **
18 ** You should have received a copy of the GNU General Public License
19 ** along with Sigil. If not, see <http://www.gnu.org/licenses/>.
20 **
21 *************************************************************************/
22
23 #include "EmbedPython/EmbeddedPython.h"
24 #include <QString>
25 #include <QByteArray>
26 #include <QList>
27 #include <QStringList>
28 #include <QVariant>
29 #include <QMetaType>
30 #include <QStandardPaths>
31 #include <QDir>
32 #include "Misc/Utility.h"
33 #include "sigil_constants.h"
34
35 // IMPORTANT NOTE: This interface does NOT support passing/converting the type bool
36 // All bool values should be converted to integeres with values of 0 for false and 1 for true
37
38 /**
39 * Possibly Useful QMetaTypes::Type types
40 *
41 * QMetaType::Bool 1 bool
42 * QMetaType::Int 2 int
43 * QMetaType::UInt 3 unsigned int
44 * QMetaType::Double 6 double
45 * QMetaType::QChar 7 QChar
46 * QMetaType::QString 10 QString
47 * QMetaType::QByteArray 12 QByteArray
48 * QMetaType::Long 32 long
49 * QMetaType::LongLong 4 LongLong
50 * QMetaType::Short 33 short
51 * QMetaType::Char 34 char
52 * QMetaType::ULong 35 unsigned long
53 * QMetaType::ULongLong 5 ULongLong
54 * QMetaType::UShort 36 unsigned short
55 * QMetaType::SChar 40 signed char
56 * QMetaType::UChar 37 unsigned char
57 * QMetaType::Float 38 float
58 * QMetaType::QVariant 41 QVariant
59 * QMetaType::QVariantList 9 QVariantList
60 * QMetaType::QStringList 11 QStringList
61 * QMetaType::QVariantMap 8 QVariantMap
62 * QMetaType::QVariantHash 28 QVariantHash
63 * QMetaType::User 1024 Base value for User registered Type
64 * QMetaType::UnknownType 0 This is an invalid type id. It is returned from QMetaType for types that are not registered
65 */
66
67
68 /**
69 * // example of how to run a python function inside a specific module
70 *
71 * void EmbeddedPython::multiply_pushed(int val1, int val2)
72 * {
73 * int rv = 0;
74 * QString error_traceback;
75 * QList<QVariant> args;
76 * args.append(QVariant(val1));
77 * args.append(QVariant(val2));
78 * QVariant res = runInPython(QString("multiply"),
79 * QString("multiply"),
80 * args,
81 * &rv,
82 * error_traceback);
83 * if (rv == 0) {
84 * // no errors
85 * } else {
86 * // error occured
87 * }
88 * }
89 *
90 * // Where multiply.py is:
91 *
92 * #!/usr/bin/env python3
93 *
94 * def multiply(a,b):
95 * print("Will compute", a, "times", b)
96 * c = a * b
97 * return c
98 */
99
100 /**
101 * // example of how to use the callPyObjMethod
102 *
103 * // First invoke module function to get the Python object
104 *
105 * PyObjectPtr MyClass::get_object()
106 * {
107 * int rv = 0;
108 * QString traceback;
109 * QString v1 = QString("Hello");
110 * QList<QVariant> args;
111 * args.append(QVariant(v1));
112 *
113 * QVariant res = m_epp->runInPython( QString("multiply"),
114 * QString("get_object"),
115 * args,
116 * &rv,
117 * traceback,
118 * true);
119 * if (rv == 0) {
120 * return PyObjectPtr(res);
121 * } else {
122 * return PyObjectPtr();
123 * }
124 * }
125 *
126 * // Now invoke its "get_len" method
127 *
128 * QString MyClass:use_object(PyObjectPtr v)
129 * {
130 * int rv = 0;
131 * QString traceback;
132 * QList<QVariant> args;
133 * QVariant res = m_epp->callPyObjMethod(v,
134 * QString("get_len"),
135 * args,
136 * &rv,
137 * traceback);
138 * if (rv == 0) {
139 * return res.toString();
140 * } else {
141 * return QString("Error: ") + QString::number(rv);
142 * }
143 * }
144 *
145 *
146 * # With the following python code inside of multiply.py
147 *
148 * class TestMe:
149 * def __init__(self, storeme):
150 * self.storeme = storeme
151 * self.mylen = len(self.storeme)
152 *
153 * def get_me(self):
154 * return self.storeme
155 *
156 * def get_len(self):
157 * return self.mylen
158 *
159 * def get_object(v1):
160 * tme = TestMe(v1)
161 * return tme
162 */
163
164
165 QMutex EmbeddedPython::m_mutex;
166
167 EmbeddedPython* EmbeddedPython::m_instance = 0;
168 int EmbeddedPython::m_pyobjmetaid = 0;
169 int EmbeddedPython::m_listintmetaid = 0;
170 PyThreadState * EmbeddedPython::m_threadstate = NULL;
171
instance()172 EmbeddedPython* EmbeddedPython::instance()
173 {
174 if (m_instance == 0) {
175 m_instance = new EmbeddedPython();
176 }
177 return m_instance;
178 }
179
EmbeddedPython()180 EmbeddedPython::EmbeddedPython()
181 {
182 // Build string list of paths that will
183 // comprise the embedded Python's sys.path
184 #if defined(BUNDLING_PYTHON)
185 // Apple for Python 3.8 does need to set these now
186 #if defined(__APPLE__)
187 QDir exedir(QCoreApplication::applicationDirPath());
188 exedir.cdUp();
189 QString pyhomepath = exedir.absolutePath() + PYTHON_MAIN_PREFIX;
190 QString pylibpath = pyhomepath + PYTHON_LIB_PATH;
191 wchar_t *hpath = new wchar_t[pyhomepath.size()+1];
192 pyhomepath.toWCharArray(hpath);
193 hpath[pyhomepath.size()]=L'\0';
194 QString pysyspath = pylibpath;
195 foreach (const QString &src_path, PYTHON_SYS_PATHS) {
196 pysyspath = pysyspath + PATH_LIST_DELIM + pylibpath + src_path;
197 }
198 wchar_t *mpath = new wchar_t[pysyspath.size()+1];
199 pysyspath.toWCharArray(mpath);
200 mpath[pysyspath.size()]=L'\0';
201 delete[] hpath;
202
203 // Py_OptimizeFlag = 2;
204 // Py_NoSiteFlag = 1;
205
206 #else // Windows
207 QString pyhomepath = QCoreApplication::applicationDirPath();
208 wchar_t *hpath = new wchar_t[pyhomepath.size()+1];
209 pyhomepath.toWCharArray(hpath);
210 hpath[pyhomepath.size()]=L'\0';
211
212 QString pysyspath = pyhomepath + PYTHON_MAIN_PATH;
213 foreach (const QString &src_path, PYTHON_SYS_PATHS) {
214 pysyspath = pysyspath + PATH_LIST_DELIM + pyhomepath + PYTHON_MAIN_PATH + src_path;
215 }
216 wchar_t *mpath = new wchar_t[pysyspath.size()+1];
217 pysyspath.toWCharArray(mpath);
218 mpath[pysyspath.size()]=L'\0';
219 delete[] hpath;
220
221 Py_OptimizeFlag = 2;
222 Py_NoSiteFlag = 1;
223 #endif // defined(__APPLE__)
224 // Everyone uses these flags when python is bundled.
225 Py_DontWriteBytecodeFlag = 1;
226 Py_IgnoreEnvironmentFlag = 1;
227 Py_NoUserSiteDirectory = 1;
228 //Py_DebugFlag = 0;
229 //Py_VerboseFlag = 0;
230 // Set before Py_Initialize to ensure isolation from system python
231 Py_SetPath(mpath);
232 delete[] mpath;
233 #endif // defined(BUNDLING_PYTHON)
234
235 Py_Initialize();
236 PyEval_InitThreads();
237 m_threadstate = PyEval_SaveThread();
238 m_pyobjmetaid = qMetaTypeId<PyObjectPtr>();
239 m_listintmetaid = qMetaTypeId<QList<int> >();
240 }
241
242
~EmbeddedPython()243 EmbeddedPython::~EmbeddedPython()
244 {
245 if (m_instance) {
246 delete m_instance;
247 m_instance = 0;
248 }
249 m_pyobjmetaid = 0;
250 m_listintmetaid = 0;
251 PyEval_RestoreThread(m_threadstate);
252 Py_Finalize();
253 }
254
embeddedRoot()255 QString EmbeddedPython::embeddedRoot()
256 {
257 QString embedded_root;
258 QStringList embedded_roots;
259 QDir d;
260
261 #ifdef Q_OS_MAC
262 embedded_roots.append(QCoreApplication::applicationDirPath() + "/../python3lib/");
263 #elif defined(Q_OS_WIN32)
264 embedded_roots.append(QCoreApplication::applicationDirPath() + "/python3lib/");
265 #elif !defined(Q_OS_WIN32) && !defined(Q_OS_MAC)
266 // all flavours of linux / unix
267 // user supplied environment variable to 'share/sigil' directory will overrides everything
268 if (!sigil_extra_root.isEmpty()) {
269 embedded_roots.append(sigil_extra_root + "/python3lib/");
270 } else {
271 embedded_roots.append(sigil_share_root + "/python3lib/");
272 }
273 #endif
274
275 Q_FOREACH (QString s, embedded_roots) {
276 if (d.exists(s)) {
277 embedded_root = s;
278 break;
279 }
280 }
281
282 QDir base(embedded_root);
283 return base.absolutePath();
284 }
285
addToPythonSysPath(const QString & mpath)286 bool EmbeddedPython::addToPythonSysPath(const QString &mpath)
287 {
288 EmbeddedPython::m_mutex.lock();
289 PyGILState_STATE gstate = PyGILState_Ensure();
290
291 PyObject* sysPath = NULL;
292 PyObject* aPath = NULL;
293 bool success = false;
294
295 // PySys_GetObject borrows a reference */
296 sysPath = PySys_GetObject((char*)"path");
297 if (sysPath != NULL) {
298 aPath = PyUnicode_FromString(mpath.toUtf8().constData());
299 if (aPath != NULL) {
300 PyList_Append(sysPath, aPath);
301 success = true;
302 }
303 }
304 Py_XDECREF(aPath);
305 PyGILState_Release(gstate);
306 EmbeddedPython::m_mutex.unlock();
307 return success;
308 }
309
310 // run interpreter without initiating/locking/unlocking GIL
311 // in a single thread at a time
runInPython(const QString & mname,const QString & fname,const QVariantList & args,int * rv,QString & tb,bool ret_python_object)312 QVariant EmbeddedPython::runInPython(const QString &mname,
313 const QString &fname,
314 const QVariantList &args,
315 int *rv,
316 QString &tb,
317 bool ret_python_object)
318 {
319 EmbeddedPython::m_mutex.lock();
320 PyGILState_STATE gstate = PyGILState_Ensure();
321 QVariant res = QVariant(QString());
322 PyObject *moduleName = NULL;
323 PyObject *module = NULL;
324 PyObject *func = NULL;
325 PyObject *pyargs = NULL;
326 PyObject *pyres = NULL;
327 int idx = 0;
328
329 moduleName = PyUnicode_FromString(mname.toUtf8().constData());
330 if (moduleName == NULL) {
331 *rv = -1;
332 goto cleanup;
333 }
334
335 module = PyImport_Import(moduleName);
336 if (module == NULL) {
337 *rv = -2;
338 goto cleanup;
339 }
340
341 func = PyObject_GetAttrString(module,fname.toUtf8().constData());
342 if (func == NULL) {
343 *rv = -3;
344 goto cleanup;
345 }
346
347 if (!PyCallable_Check(func)) {
348 *rv = -4;
349 goto cleanup;
350 }
351
352 // Build up Python argument List from args
353 pyargs = PyTuple_New(args.size());
354 idx = 0;
355 foreach(QVariant arg, args) {
356 PyTuple_SetItem(pyargs, idx, QVariantToPyObject(arg));
357 idx++;
358 }
359
360 pyres = PyObject_CallObject(func, pyargs);
361 if (pyres == NULL) {
362 *rv = -5;
363 goto cleanup;
364 }
365
366 *rv = 0;
367
368 res = PyObjectToQVariant(pyres, ret_python_object);
369
370 cleanup:
371 if (PyErr_Occurred() != NULL) {
372 QString default_error = "Module Error: " + mname + " " + fname;
373 tb = getPythonErrorTraceback(default_error);
374 }
375 Py_XDECREF(pyres);
376 Py_XDECREF(pyargs);
377 Py_XDECREF(func);
378 Py_XDECREF(module);
379 Py_XDECREF(moduleName);
380
381 PyGILState_Release(gstate);
382 EmbeddedPython::m_mutex.unlock();
383 return res;
384 }
385
386
387 // given an existing python object instance, invoke one of its methods
388 // grabs mutex to prevent need for Python GIL
callPyObjMethod(PyObjectPtr & pyobj,const QString & methname,const QVariantList & args,int * rv,QString & tb,bool ret_python_object)389 QVariant EmbeddedPython::callPyObjMethod(PyObjectPtr &pyobj,
390 const QString &methname,
391 const QVariantList &args,
392 int *rv,
393 QString &tb,
394 bool ret_python_object)
395 {
396 EmbeddedPython::m_mutex.lock();
397 PyGILState_STATE gstate = PyGILState_Ensure();
398
399 QVariant res = QVariant(QString());
400 PyObject* obj = pyobj.object();
401 PyObject* func = NULL;
402 PyObject* pyargs = NULL;
403 PyObject* pyres = NULL;
404 int idx = 0;
405
406 func = PyObject_GetAttrString(obj,methname.toUtf8().constData());
407 if (func == NULL) {
408 *rv = -1;
409 goto cleanup;
410 }
411
412 if (!PyCallable_Check(func)) {
413 *rv = -2;
414 goto cleanup;
415 }
416
417 // Build up Python argument List from args
418 pyargs = PyTuple_New(args.size());
419 idx = 0;
420 foreach(QVariant arg, args) {
421 PyTuple_SetItem(pyargs, idx, QVariantToPyObject(arg));
422 idx++;
423 }
424
425 pyres = PyObject_CallObject(func, pyargs);
426 if (pyres == NULL) {
427 *rv = -3;
428 goto cleanup;
429 }
430
431 *rv = 0;
432
433 res = PyObjectToQVariant(pyres, ret_python_object);
434
435 cleanup:
436 if (PyErr_Occurred() != NULL) {
437 QString default_error = "Python Object Method Invocation Error: " + methname;
438 tb = getPythonErrorTraceback(default_error);
439 }
440 Py_XDECREF(pyres);
441 Py_XDECREF(pyargs);
442 Py_XDECREF(func);
443
444 PyGILState_Release(gstate);
445 EmbeddedPython::m_mutex.unlock();
446 return res;
447 }
448
449
450 // *** below here all routines are private and only invoked
451 // *** from runInPython and callPyObjMethod with lock held
452
453
454 // Convert PyObject types to their QVariant equivalents
455 // call recursively to allow populating QVariant lists and lists of lists
PyObjectToQVariant(PyObject * po,bool ret_python_object)456 QVariant EmbeddedPython::PyObjectToQVariant(PyObject *po, bool ret_python_object)
457 {
458 QVariant res = QVariant(QString());
459
460 if ((po) == NULL)
461 return res;
462
463 if (PyLong_Check(po)) {
464 res = QVariant(PyLong_AsLongLong(po));
465
466 } else if (PyFloat_Check(po)) {
467 res = QVariant(PyFloat_AsDouble(po));
468
469 } else if (PyBytes_Check(po)) {
470 res = QVariant(QByteArray(PyBytes_AsString(po)));
471
472 } else if (PyUnicode_Check(po)) {
473
474 int kind = PyUnicode_KIND(po);
475
476 if (PyUnicode_READY(po) != 0)
477 return res;
478
479 if (kind == PyUnicode_1BYTE_KIND) {
480 // latin 1 according to PEP 393
481 res = QVariant(QString::fromLatin1(reinterpret_cast<const char *>PyUnicode_1BYTE_DATA(po), -1));
482
483 } else if (kind == PyUnicode_2BYTE_KIND) {
484 res = QVariant(QString::fromUtf16(PyUnicode_2BYTE_DATA(po), -1));
485
486 } else if (kind == PyUnicode_4BYTE_KIND) {
487 // PyUnicode_4BYTE_KIND
488 res = QVariant(QString::fromUcs4(PyUnicode_4BYTE_DATA(po), -1));
489
490 } else {
491 // convert to utf8 since not a known
492 res = QVariant(QString::fromUtf8(PyUnicode_AsUTF8(po),-1));
493 }
494
495 } else if (PyTuple_Check(po)) {
496 QVariantList vlist;
497 int n = PyTuple_Size(po);
498 for (int i=0; i< n; i++) {
499 vlist.append(PyObjectToQVariant(PyTuple_GetItem(po,i)));
500 }
501 res = QVariant(vlist);
502
503 } else if (PyList_Check(po)) {
504 QVariantList vlist;
505 int n = PyList_Size(po);
506 for (int i=0; i< n; i++) {
507 vlist.append(PyObjectToQVariant(PyList_GetItem(po,i)));
508 }
509 res = QVariant(vlist);
510
511 } else if (ret_python_object) {
512 QVariant var;
513 var.setValue(PyObjectPtr(po));
514 res = var;
515 } else {
516 // do nothing here to return null value
517 }
518 return res;
519 }
520
521 // Convert QVariant to a Python Equivalent Type
522 // call recursively to allow populating tuples/lists and lists of lists
QVariantToPyObject(const QVariant & v)523 PyObject* EmbeddedPython::QVariantToPyObject(const QVariant &v)
524 {
525 PyObject* value = NULL;
526 bool ok;
527 switch ((QMetaType::Type)v.type()) {
528 case QMetaType::Double:
529 value = Py_BuildValue("d", v.toDouble(&ok));
530 break;
531 case QMetaType::Float:
532 value = Py_BuildValue("f", v.toFloat(&ok));
533 break;
534 case QMetaType::Int:
535 value = Py_BuildValue("i", v.toInt(&ok));
536 break;
537 case QMetaType::UInt:
538 value = Py_BuildValue("I", v.toUInt(&ok));
539 break;
540 case QMetaType::LongLong:
541 value = Py_BuildValue("L", v.toLongLong(&ok));
542 break;
543 case QMetaType::ULongLong:
544 value = Py_BuildValue("K", v.toULongLong(&ok));
545 break;
546 case QMetaType::QString:
547 // since QString's utf-16 may contain surrogates or may only be pure ascii we have no easy
548 // to know the proper string storage type to use internal to python (latin1, ucs2, ucs4)
549 // so punt and create utf-8 and let python handle the conversion internally via its c-api
550 value = Py_BuildValue("s", v.toString().toUtf8().constData());
551 break;
552 case QMetaType::QByteArray:
553 value = Py_BuildValue("y", v.toByteArray().constData());
554 break;
555 case QMetaType::QStringList:
556 {
557 QStringList vlist = v.toStringList();
558 value = PyList_New(vlist.size());
559 int pos = 0;
560 foreach(QString av, vlist) {
561 // since QString's utf-16 may contain surrogates or may only be pure ascii we have no easy
562 // to know the proper string storage type to use internal to python (latin1, ucs2, ucs4)
563 // so punt and create utf-8 and let python handle the conversion internally via its c-api
564 PyObject* strval = Py_BuildValue("s", av.toUtf8().constData());
565 PyList_SetItem(value, pos, strval);
566 pos++;
567 }
568 }
569 break;
570 case QMetaType::QVariantList:
571 {
572 QVariantList vlist = v.toList();
573 value = PyList_New(vlist.size());
574 int pos = 0;
575 foreach(QVariant av, vlist) {
576 PyList_SetItem(value, pos, QVariantToPyObject(av));
577 pos++;
578 }
579 }
580 break;
581 default:
582 {
583 if ((QMetaType::Type)v.type() >= QMetaType::User && (v.userType() == m_pyobjmetaid))
584 {
585 PyObjectPtr op = v.value<PyObjectPtr>();
586 value = op.object();
587 // Need to increment object count otherwise will go away when Py_XDECREF used on pyargs
588 Py_XINCREF(value);
589
590 } else if ((QMetaType::Type)v.type() >= QMetaType::User && (v.userType() == m_listintmetaid))
591 {
592 QList<int> alist = v.value<QList<int> >();
593 value = PyList_New(alist.size());
594 int pos = 0;
595 foreach(int i, alist) {
596 PyList_SetItem(value, pos, Py_BuildValue("i", i));
597 pos++;
598 }
599 } else {
600
601 // Ensure we don't have any holes.
602 value = Py_BuildValue("u", "");
603 }
604 }
605 break;
606 }
607 return value;
608 }
609
610
611 // get traceback from inside interpreter upon error
getPythonErrorTraceback(const QString & default_message,bool useMsgBox)612 QString EmbeddedPython::getPythonErrorTraceback(const QString& default_message, bool useMsgBox)
613 {
614 PyObject *etype = NULL;
615 PyObject *evalue = NULL;
616 PyObject *etraceback = NULL;
617 PyObject *mod = NULL;
618 PyObject *elist = NULL;
619 QStringList tblist;
620
621 PyErr_Fetch(&etype, &evalue, &etraceback);
622 PyErr_NormalizeException(&etype, &evalue, &etraceback);
623
624 mod = PyImport_ImportModule("traceback");
625
626 if (mod) {
627 elist = PyObject_CallMethod(mod, "format_exception", "OOO", etype, evalue, etraceback);
628 if (elist != NULL) {
629 tblist = PyObjectToQVariant(elist).toStringList();
630 } else {
631 tblist.append(default_message);
632 }
633 } else {
634 tblist.append(QString("Error: traceback module failed to load"));
635 }
636
637 Py_XDECREF(elist);
638 Py_XDECREF(mod);
639 Py_XDECREF(etraceback);
640 Py_XDECREF(evalue);
641 Py_XDECREF(etype);
642
643 PyErr_Clear();
644
645 QString tb = tblist.join(QString("\n"));
646 if (useMsgBox) {
647 QString message = QString(tr("Embedded Python Error"));
648 Utility::DisplayStdErrorDialog(message, tb);
649 }
650 return tb;
651 }
652