1 #define PY_SSIZE_T_CLEAN
2 #include <Python.h>
3 
4 #define NPY_NO_DEPRECATED_API NPY_API_VERSION
5 #define _MULTIARRAYMODULE
6 #include "numpy/arrayobject.h"
7 
8 #include "npy_config.h"
9 #include "npy_pycompat.h"
10 #include "common.h"
11 
12 #include "abstractdtypes.h"
13 #include "usertypes.h"
14 
15 #include "npy_buffer.h"
16 
17 #include "get_attr_string.h"
18 #include "mem_overlap.h"
19 #include "array_coercion.h"
20 
21 /*
22  * The casting to use for implicit assignment operations resulting from
23  * in-place operations (like +=) and out= arguments. (Notice that this
24  * variable is misnamed, but it's part of the public API so I'm not sure we
25  * can just change it. Maybe someone should try and see if anyone notices.
26  */
27 /*
28  * In numpy 1.6 and earlier, this was NPY_UNSAFE_CASTING. In a future
29  * release, it will become NPY_SAME_KIND_CASTING.  Right now, during the
30  * transitional period, we continue to follow the NPY_UNSAFE_CASTING rules (to
31  * avoid breaking people's code), but we also check for whether the cast would
32  * be allowed under the NPY_SAME_KIND_CASTING rules, and if not we issue a
33  * warning (that people's code will be broken in a future release.)
34  */
35 
36 NPY_NO_EXPORT NPY_CASTING NPY_DEFAULT_ASSIGN_CASTING = NPY_SAME_KIND_CASTING;
37 
38 
39 NPY_NO_EXPORT PyArray_Descr *
_array_find_python_scalar_type(PyObject * op)40 _array_find_python_scalar_type(PyObject *op)
41 {
42     if (PyFloat_Check(op)) {
43         return PyArray_DescrFromType(NPY_DOUBLE);
44     }
45     else if (PyComplex_Check(op)) {
46         return PyArray_DescrFromType(NPY_CDOUBLE);
47     }
48     else if (PyLong_Check(op)) {
49         return PyArray_PyIntAbstractDType.discover_descr_from_pyobject(
50                     &PyArray_PyIntAbstractDType, op);
51     }
52     return NULL;
53 }
54 
55 
56 /*
57  * Get a suitable string dtype by calling `__str__`.
58  * For `np.bytes_`, this assumes an ASCII encoding.
59  */
60 NPY_NO_EXPORT PyArray_Descr *
PyArray_DTypeFromObjectStringDiscovery(PyObject * obj,PyArray_Descr * last_dtype,int string_type)61 PyArray_DTypeFromObjectStringDiscovery(
62         PyObject *obj, PyArray_Descr *last_dtype, int string_type)
63 {
64     int itemsize;
65 
66     if (string_type == NPY_STRING) {
67         PyObject *temp = PyObject_Str(obj);
68         if (temp == NULL) {
69             return NULL;
70         }
71         /* assume that when we do the encoding elsewhere we'll use ASCII */
72         itemsize = PyUnicode_GetLength(temp);
73         Py_DECREF(temp);
74         if (itemsize < 0) {
75             return NULL;
76         }
77     }
78     else if (string_type == NPY_UNICODE) {
79         PyObject *temp = PyObject_Str(obj);
80         if (temp == NULL) {
81             return NULL;
82         }
83         itemsize = PyUnicode_GetLength(temp);
84         Py_DECREF(temp);
85         if (itemsize < 0) {
86             return NULL;
87         }
88         itemsize *= 4;  /* convert UCS4 codepoints to bytes */
89     }
90     else {
91         return NULL;
92     }
93     if (last_dtype != NULL &&
94         last_dtype->type_num == string_type &&
95         last_dtype->elsize >= itemsize) {
96         Py_INCREF(last_dtype);
97         return last_dtype;
98     }
99     PyArray_Descr *dtype = PyArray_DescrNewFromType(string_type);
100     if (dtype == NULL) {
101         return NULL;
102     }
103     dtype->elsize = itemsize;
104     return dtype;
105 }
106 
107 
108 /*
109  * This function is now identical to the new PyArray_DiscoverDTypeAndShape
110  * but only returns the the dtype. It should in most cases be slowly phased
111  * out. (Which may need some refactoring to PyArray_FromAny to make it simpler)
112  */
113 NPY_NO_EXPORT int
PyArray_DTypeFromObject(PyObject * obj,int maxdims,PyArray_Descr ** out_dtype)114 PyArray_DTypeFromObject(PyObject *obj, int maxdims, PyArray_Descr **out_dtype)
115 {
116     coercion_cache_obj *cache = NULL;
117     npy_intp shape[NPY_MAXDIMS];
118     int ndim;
119 
120     ndim = PyArray_DiscoverDTypeAndShape(
121             obj, maxdims, shape, &cache, NULL, NULL, out_dtype);
122     if (ndim < 0) {
123         return -1;
124     }
125     npy_free_coercion_cache(cache);
126     return 0;
127 }
128 
129 NPY_NO_EXPORT char *
index2ptr(PyArrayObject * mp,npy_intp i)130 index2ptr(PyArrayObject *mp, npy_intp i)
131 {
132     npy_intp dim0;
133 
134     if (PyArray_NDIM(mp) == 0) {
135         PyErr_SetString(PyExc_IndexError, "0-d arrays can't be indexed");
136         return NULL;
137     }
138     dim0 = PyArray_DIMS(mp)[0];
139     if (check_and_adjust_index(&i, dim0, 0, NULL) < 0)
140         return NULL;
141     if (i == 0) {
142         return PyArray_DATA(mp);
143     }
144     return PyArray_BYTES(mp)+i*PyArray_STRIDES(mp)[0];
145 }
146 
147 NPY_NO_EXPORT int
_zerofill(PyArrayObject * ret)148 _zerofill(PyArrayObject *ret)
149 {
150     if (PyDataType_REFCHK(PyArray_DESCR(ret))) {
151         PyObject *zero = PyLong_FromLong(0);
152         PyArray_FillObjectArray(ret, zero);
153         Py_DECREF(zero);
154         if (PyErr_Occurred()) {
155             Py_DECREF(ret);
156             return -1;
157         }
158     }
159     else {
160         npy_intp n = PyArray_NBYTES(ret);
161         memset(PyArray_DATA(ret), 0, n);
162     }
163     return 0;
164 }
165 
166 NPY_NO_EXPORT npy_bool
_IsWriteable(PyArrayObject * ap)167 _IsWriteable(PyArrayObject *ap)
168 {
169     PyObject *base = PyArray_BASE(ap);
170     Py_buffer view;
171 
172     /*
173      * C-data wrapping arrays may not own their data while not having a base;
174      * WRITEBACKIFCOPY arrays have a base, but do own their data.
175      */
176     if (base == NULL || PyArray_CHKFLAGS(ap, NPY_ARRAY_OWNDATA)) {
177         /*
178          * This is somewhat unsafe for directly wrapped non-writable C-arrays,
179          * which do not know whether the memory area is writable or not and
180          * do not own their data (but have no base).
181          * It would be better if this returned PyArray_ISWRITEABLE(ap).
182          * Since it is hard to deprecate, this is deprecated only on the Python
183          * side, but not on in PyArray_UpdateFlags.
184          */
185         return NPY_TRUE;
186     }
187 
188     /*
189      * Get to the final base object.
190      * If it is a writeable array, then return True if we can
191      * find an array object or a writeable buffer object as
192      * the final base object.
193      */
194     while (PyArray_Check(base)) {
195         ap = (PyArrayObject *)base;
196         base = PyArray_BASE(ap);
197 
198         if (PyArray_ISWRITEABLE(ap)) {
199             /*
200              * If any base is writeable, it must be OK to switch, note that
201              * bases are typically collapsed to always point to the most
202              * general one.
203              */
204             return NPY_TRUE;
205         }
206 
207         if (base == NULL || PyArray_CHKFLAGS(ap, NPY_ARRAY_OWNDATA)) {
208             /* there is no further base to test the writeable flag for */
209             return NPY_FALSE;
210         }
211         assert(!PyArray_CHKFLAGS(ap, NPY_ARRAY_OWNDATA));
212     }
213 
214     if (PyObject_GetBuffer(base, &view, PyBUF_WRITABLE|PyBUF_SIMPLE) < 0) {
215         PyErr_Clear();
216         return NPY_FALSE;
217     }
218     PyBuffer_Release(&view);
219     return NPY_TRUE;
220 }
221 
222 
223 /**
224  * Convert an array shape to a string such as "(1, 2)".
225  *
226  * @param Dimensionality of the shape
227  * @param npy_intp pointer to shape array
228  * @param String to append after the shape `(1, 2)%s`.
229  *
230  * @return Python unicode string
231  */
232 NPY_NO_EXPORT PyObject *
convert_shape_to_string(npy_intp n,npy_intp const * vals,char * ending)233 convert_shape_to_string(npy_intp n, npy_intp const *vals, char *ending)
234 {
235     npy_intp i;
236 
237     /*
238      * Negative dimension indicates "newaxis", which can
239      * be discarded for printing if it's a leading dimension.
240      * Find the first non-"newaxis" dimension.
241      */
242     for (i = 0; i < n && vals[i] < 0; i++);
243 
244     if (i == n) {
245         return PyUnicode_FromFormat("()%s", ending);
246     }
247 
248     PyObject *ret = PyUnicode_FromFormat("%" NPY_INTP_FMT, vals[i++]);
249     if (ret == NULL) {
250         return NULL;
251     }
252     for (; i < n; ++i) {
253         PyObject *tmp;
254 
255         if (vals[i] < 0) {
256             tmp = PyUnicode_FromString(",newaxis");
257         }
258         else {
259             tmp = PyUnicode_FromFormat(",%" NPY_INTP_FMT, vals[i]);
260         }
261         if (tmp == NULL) {
262             Py_DECREF(ret);
263             return NULL;
264         }
265 
266         Py_SETREF(ret, PyUnicode_Concat(ret, tmp));
267         Py_DECREF(tmp);
268         if (ret == NULL) {
269             return NULL;
270         }
271     }
272 
273     if (i == 1) {
274         Py_SETREF(ret, PyUnicode_FromFormat("(%S,)%s", ret, ending));
275     }
276     else {
277         Py_SETREF(ret, PyUnicode_FromFormat("(%S)%s", ret, ending));
278     }
279     return ret;
280 }
281 
282 
283 NPY_NO_EXPORT void
dot_alignment_error(PyArrayObject * a,int i,PyArrayObject * b,int j)284 dot_alignment_error(PyArrayObject *a, int i, PyArrayObject *b, int j)
285 {
286     PyObject *errmsg = NULL, *format = NULL, *fmt_args = NULL,
287              *i_obj = NULL, *j_obj = NULL,
288              *shape1 = NULL, *shape2 = NULL,
289              *shape1_i = NULL, *shape2_j = NULL;
290 
291     format = PyUnicode_FromString("shapes %s and %s not aligned:"
292                                   " %d (dim %d) != %d (dim %d)");
293 
294     shape1 = convert_shape_to_string(PyArray_NDIM(a), PyArray_DIMS(a), "");
295     shape2 = convert_shape_to_string(PyArray_NDIM(b), PyArray_DIMS(b), "");
296 
297     i_obj = PyLong_FromLong(i);
298     j_obj = PyLong_FromLong(j);
299 
300     shape1_i = PyLong_FromSsize_t(PyArray_DIM(a, i));
301     shape2_j = PyLong_FromSsize_t(PyArray_DIM(b, j));
302 
303     if (!format || !shape1 || !shape2 || !i_obj || !j_obj ||
304             !shape1_i || !shape2_j) {
305         goto end;
306     }
307 
308     fmt_args = PyTuple_Pack(6, shape1, shape2,
309                             shape1_i, i_obj, shape2_j, j_obj);
310     if (fmt_args == NULL) {
311         goto end;
312     }
313 
314     errmsg = PyUnicode_Format(format, fmt_args);
315     if (errmsg != NULL) {
316         PyErr_SetObject(PyExc_ValueError, errmsg);
317     }
318     else {
319         PyErr_SetString(PyExc_ValueError, "shapes are not aligned");
320     }
321 
322 end:
323     Py_XDECREF(errmsg);
324     Py_XDECREF(fmt_args);
325     Py_XDECREF(format);
326     Py_XDECREF(i_obj);
327     Py_XDECREF(j_obj);
328     Py_XDECREF(shape1);
329     Py_XDECREF(shape2);
330     Py_XDECREF(shape1_i);
331     Py_XDECREF(shape2_j);
332 }
333 
334 /**
335  * unpack tuple of dtype->fields (descr, offset, title[not-needed])
336  *
337  * @param "value" should be the tuple.
338  *
339  * @return "descr" will be set to the field's dtype
340  * @return "offset" will be set to the field's offset
341  *
342  * returns -1 on failure, 0 on success.
343  */
344 NPY_NO_EXPORT int
_unpack_field(PyObject * value,PyArray_Descr ** descr,npy_intp * offset)345 _unpack_field(PyObject *value, PyArray_Descr **descr, npy_intp *offset)
346 {
347     PyObject * off;
348     if (PyTuple_GET_SIZE(value) < 2) {
349         return -1;
350     }
351     *descr = (PyArray_Descr *)PyTuple_GET_ITEM(value, 0);
352     off  = PyTuple_GET_ITEM(value, 1);
353 
354     if (PyLong_Check(off)) {
355         *offset = PyLong_AsSsize_t(off);
356     }
357     else {
358         PyErr_SetString(PyExc_IndexError, "can't convert offset");
359         return -1;
360     }
361 
362     return 0;
363 }
364 
365 /*
366  * check whether arrays with datatype dtype might have object fields. This will
367  * only happen for structured dtypes (which may have hidden objects even if the
368  * HASOBJECT flag is false), object dtypes, or subarray dtypes whose base type
369  * is either of these.
370  */
371 NPY_NO_EXPORT int
_may_have_objects(PyArray_Descr * dtype)372 _may_have_objects(PyArray_Descr *dtype)
373 {
374     PyArray_Descr *base = dtype;
375     if (PyDataType_HASSUBARRAY(dtype)) {
376         base = dtype->subarray->base;
377     }
378 
379     return (PyDataType_HASFIELDS(base) ||
380             PyDataType_FLAGCHK(base, NPY_ITEM_HASOBJECT) );
381 }
382 
383 /*
384  * Make a new empty array, of the passed size, of a type that takes the
385  * priority of ap1 and ap2 into account.
386  *
387  * If `out` is non-NULL, memory overlap is checked with ap1 and ap2, and an
388  * updateifcopy temporary array may be returned. If `result` is non-NULL, the
389  * output array to be returned (`out` if non-NULL and the newly allocated array
390  * otherwise) is incref'd and put to *result.
391  */
392 NPY_NO_EXPORT PyArrayObject *
new_array_for_sum(PyArrayObject * ap1,PyArrayObject * ap2,PyArrayObject * out,int nd,npy_intp dimensions[],int typenum,PyArrayObject ** result)393 new_array_for_sum(PyArrayObject *ap1, PyArrayObject *ap2, PyArrayObject* out,
394                   int nd, npy_intp dimensions[], int typenum, PyArrayObject **result)
395 {
396     PyArrayObject *out_buf;
397 
398     if (out) {
399         int d;
400 
401         /* verify that out is usable */
402         if (PyArray_NDIM(out) != nd ||
403             PyArray_TYPE(out) != typenum ||
404             !PyArray_ISCARRAY(out)) {
405             PyErr_SetString(PyExc_ValueError,
406                 "output array is not acceptable (must have the right datatype, "
407                 "number of dimensions, and be a C-Array)");
408             return 0;
409         }
410         for (d = 0; d < nd; ++d) {
411             if (dimensions[d] != PyArray_DIM(out, d)) {
412                 PyErr_SetString(PyExc_ValueError,
413                     "output array has wrong dimensions");
414                 return 0;
415             }
416         }
417 
418         /* check for memory overlap */
419         if (!(solve_may_share_memory(out, ap1, 1) == 0 &&
420               solve_may_share_memory(out, ap2, 1) == 0)) {
421             /* allocate temporary output array */
422             out_buf = (PyArrayObject *)PyArray_NewLikeArray(out, NPY_CORDER,
423                                                             NULL, 0);
424             if (out_buf == NULL) {
425                 return NULL;
426             }
427 
428             /* set copy-back */
429             Py_INCREF(out);
430             if (PyArray_SetWritebackIfCopyBase(out_buf, out) < 0) {
431                 Py_DECREF(out);
432                 Py_DECREF(out_buf);
433                 return NULL;
434             }
435         }
436         else {
437             Py_INCREF(out);
438             out_buf = out;
439         }
440 
441         if (result) {
442             Py_INCREF(out);
443             *result = out;
444         }
445 
446         return out_buf;
447     }
448     else {
449         PyTypeObject *subtype;
450         double prior1, prior2;
451         /*
452          * Need to choose an output array that can hold a sum
453          * -- use priority to determine which subtype.
454          */
455         if (Py_TYPE(ap2) != Py_TYPE(ap1)) {
456             prior2 = PyArray_GetPriority((PyObject *)ap2, 0.0);
457             prior1 = PyArray_GetPriority((PyObject *)ap1, 0.0);
458             subtype = (prior2 > prior1 ? Py_TYPE(ap2) : Py_TYPE(ap1));
459         }
460         else {
461             prior1 = prior2 = 0.0;
462             subtype = Py_TYPE(ap1);
463         }
464 
465         out_buf = (PyArrayObject *)PyArray_New(subtype, nd, dimensions,
466                                                typenum, NULL, NULL, 0, 0,
467                                                (PyObject *)
468                                                (prior2 > prior1 ? ap2 : ap1));
469 
470         if (out_buf != NULL && result) {
471             Py_INCREF(out_buf);
472             *result = out_buf;
473         }
474 
475         return out_buf;
476     }
477 }
478 
479