1 #include <Python.h>
2 #include <structmember.h>
3
4 /*
5 Persistent/Immutable/Functional vector and helper types.
6
7 Please note that they are anything but immutable at this level since
8 there is a whole lot of reference counting going on. That's the way
9 CPython works though and the GIL makes them appear immutable.
10
11 To the programmer using them from Python they appear immutable and
12 behave immutably at least.
13
14 Naming conventions
15 ------------------
16 initpyrsistentc - This is the method that initializes the whole module
17 pyrsistent_* - Methods part of the interface
18 <typename>_* - Instance methods of types. For examle PVector_append(...)
19
20 All other methods are camel cased without prefix. All methods are static, none should
21 require to be exposed outside of this module.
22 */
23
24 #define SHIFT 5
25 #define BRANCH_FACTOR (1 << SHIFT)
26 #define BIT_MASK (BRANCH_FACTOR - 1)
27
28 static PyTypeObject PVectorType;
29 static PyTypeObject PVectorEvolverType;
30
31 typedef struct {
32 void *items[BRANCH_FACTOR];
33 unsigned int refCount;
34 } VNode;
35
36 #define NODE_CACHE_MAX_SIZE 1024
37
38 typedef struct {
39 unsigned int size;
40 VNode* nodes[NODE_CACHE_MAX_SIZE];
41 } vNodeCache;
42
43 static vNodeCache nodeCache;
44
45 typedef struct {
46 PyObject_HEAD
47 unsigned int count; // Perhaps ditch this one in favor of ob_size/Py_SIZE()
48 unsigned int shift;
49 VNode *root;
50 VNode *tail;
51 PyObject *in_weakreflist; /* List of weak references */
52 } PVector;
53
54 typedef struct {
55 PyObject_HEAD
56 PVector* originalVector;
57 PVector* newVector;
58 PyObject* appendList;
59 } PVectorEvolver;
60
61
62 static PVector* EMPTY_VECTOR = NULL;
63 static PyObject* transform_fn = NULL;
64
transform(PVector * self,PyObject * args)65 static PyObject* transform(PVector* self, PyObject* args) {
66 if(transform_fn == NULL) {
67 // transform to avoid circular import problems
68 transform_fn = PyObject_GetAttrString(PyImport_ImportModule("pyrsistent._transformations"), "transform");
69 }
70
71 return PyObject_CallFunctionObjArgs(transform_fn, self, args, NULL);
72 }
73
74
75 // No access to internal members
76 static PyMemberDef PVector_members[] = {
77 {NULL} /* Sentinel */
78 };
79
80 #define debug(...)
81 // #define debug printf
82
83 #define NODE_REF_COUNT(n) ((n)->refCount)
84 #define SET_NODE_REF_COUNT(n, c) (NODE_REF_COUNT(n) = (c))
85 #define INC_NODE_REF_COUNT(n) (NODE_REF_COUNT(n)++)
86 #define DEC_NODE_REF_COUNT(n) (NODE_REF_COUNT(n)--)
87
allocNode(void)88 static VNode* allocNode(void) {
89 if(nodeCache.size > 0) {
90 nodeCache.size--;
91 return nodeCache.nodes[nodeCache.size];
92 }
93
94 return PyMem_Malloc(sizeof(VNode));
95 }
96
freeNode(VNode * node)97 static void freeNode(VNode *node) {
98 if(nodeCache.size < NODE_CACHE_MAX_SIZE) {
99 nodeCache.nodes[nodeCache.size] = node;
100 nodeCache.size++;
101 } else {
102 PyMem_Free(node);
103 }
104 }
105
newNode(void)106 static VNode* newNode(void) {
107 VNode* result = allocNode();
108 memset(result, 0x0, sizeof(VNode));
109 SET_NODE_REF_COUNT(result, 1);
110 debug("newNode() %p\n", result);
111 return result;
112 }
113
copyNode(VNode * source)114 static VNode* copyNode(VNode* source) {
115 /* NB: Only to be used for internal nodes, eg. nodes that do not
116 hold direct references to python objects but only to other nodes. */
117 int i;
118 VNode* result = allocNode();
119 debug("copyNode() %p\n", result);
120 memcpy(result->items, source->items, sizeof(source->items));
121
122 for(i = 0; i < BRANCH_FACTOR; i++) {
123 // TODO-OPT: Any need to go on when the first NULL has been found?
124 if(result->items[i] != NULL) {
125 INC_NODE_REF_COUNT((VNode*)result->items[i]);
126 }
127 }
128
129 SET_NODE_REF_COUNT(result, 1);
130 return result;
131 }
132
133 static PVector* emptyNewPvec(void);
134 static PVector* copyPVector(PVector *original);
135 static void extendWithItem(PVector *newVec, PyObject *item);
136
137 static PyObject *PVectorEvolver_persistent(PVectorEvolver *);
138 static int PVectorEvolver_set_item(PVectorEvolver *, PyObject*, PyObject*);
139
PVector_len(PVector * self)140 static Py_ssize_t PVector_len(PVector *self) {
141 return self->count;
142 }
143
144 /* Convenience macros */
145 #define ROOT_NODE_FULL(vec) ((vec->count >> SHIFT) > (1 << vec->shift))
146 #define TAIL_OFF(vec) ((vec->count < BRANCH_FACTOR) ? 0 : (((vec->count - 1) >> SHIFT) << SHIFT))
147 #define TAIL_SIZE(vec) (vec->count - TAIL_OFF(vec))
148 #define PVector_CheckExact(op) (Py_TYPE(op) == &PVectorType)
149
nodeFor(PVector * self,int i)150 static VNode* nodeFor(PVector *self, int i){
151 int level;
152 if((i >= 0) && (i < self->count)) {
153 if(i >= TAIL_OFF(self)) {
154 return self->tail;
155 }
156
157 VNode* node = self->root;
158 for(level = self->shift; level > 0; level -= SHIFT) {
159 node = (VNode*) node->items[(i >> level) & BIT_MASK];
160 }
161
162 return node;
163 }
164
165 PyErr_Format(PyExc_IndexError, "Index out of range: %i", i);
166 return NULL;
167 }
168
_get_item(PVector * self,Py_ssize_t pos)169 static PyObject* _get_item(PVector *self, Py_ssize_t pos) {
170 VNode* node = nodeFor((PVector*)self, pos);
171 PyObject *result = NULL;
172 if(node != NULL) {
173 result = node->items[pos & BIT_MASK];
174 }
175 return result;
176 }
177
178 /*
179 Returns a new reference as specified by the PySequence_GetItem function.
180 */
PVector_get_item(PVector * self,Py_ssize_t pos)181 static PyObject* PVector_get_item(PVector *self, Py_ssize_t pos) {
182 if (pos < 0) {
183 pos += self->count;
184 }
185
186 PyObject* obj = _get_item(self, pos);
187 Py_XINCREF(obj);
188 return obj;
189 }
190
releaseNode(int level,VNode * node)191 static void releaseNode(int level, VNode *node) {
192 if(node == NULL) {
193 return;
194 }
195
196 debug("releaseNode(): node=%p, level=%i, refCount=%i\n", node, level, NODE_REF_COUNT(node));
197
198 int i;
199
200 DEC_NODE_REF_COUNT(node);
201 debug("Refcount when trying to release: %u\n", NODE_REF_COUNT(node));
202 if(NODE_REF_COUNT(node) == 0) {
203 if(level > 0) {
204 for(i = 0; i < BRANCH_FACTOR; i++) {
205 if(node->items[i] != NULL) {
206 releaseNode(level - SHIFT, node->items[i]);
207 }
208 }
209 freeNode(node);
210 } else {
211 for(i = 0; i < BRANCH_FACTOR; i++) {
212 Py_XDECREF(node->items[i]);
213 }
214 freeNode(node);
215 }
216 }
217
218 debug("releaseNode(): Done! node=%p!\n", node);
219 }
220
221 /*
222 Returns all references to PyObjects that have been stolen. Also decrements
223 the internal reference counts used for shared memory structures and deallocates
224 those if needed.
225 */
PVector_dealloc(PVector * self)226 static void PVector_dealloc(PVector *self) {
227 debug("Dealloc(): self=%p, self->count=%u, tail->refCount=%u, root->refCount=%u, self->shift=%u, self->tail=%p, self->root=%p\n",
228 self, self->count, NODE_REF_COUNT(self->tail), NODE_REF_COUNT(self->root), self->shift, self->tail, self->root);
229
230 if (self->in_weakreflist != NULL) {
231 PyObject_ClearWeakRefs((PyObject *) self);
232 }
233
234 PyObject_GC_UnTrack((PyObject*)self);
235 Py_TRASHCAN_SAFE_BEGIN(self);
236
237 releaseNode(0, self->tail);
238 releaseNode(self->shift, self->root);
239
240 PyObject_GC_Del(self);
241 Py_TRASHCAN_SAFE_END(self);
242 }
243
PVector_toList(PVector * self)244 static PyObject *PVector_toList(PVector *self) {
245 Py_ssize_t i;
246 PyObject *list = PyList_New(self->count);
247 for (i = 0; i < self->count; ++i) {
248 PyObject *o = _get_item(self, i);
249 Py_INCREF(o);
250 PyList_SET_ITEM(list, i, o);
251 }
252
253 return list;
254 }
255
256
PVector_repr(PVector * self)257 static PyObject *PVector_repr(PVector *self) {
258 // Reuse the list repr code, a bit less efficient but saves some code
259 PyObject *list = PVector_toList(self);
260 PyObject *list_repr = PyObject_Repr(list);
261 Py_DECREF(list);
262
263 if(list_repr == NULL) {
264 // Exception raised during call to repr
265 return NULL;
266 }
267
268 // Repr for list implemented differently in python 2 and 3. Need to
269 // handle this or core dump will occur.
270 #if PY_MAJOR_VERSION >= 3
271 PyObject *s = PyUnicode_FromFormat("%s%U%s", "pvector(", list_repr, ")");
272 Py_DECREF(list_repr);
273 #else
274 PyObject *s = PyString_FromString("pvector(");
275 PyString_ConcatAndDel(&s, list_repr);
276 PyString_ConcatAndDel(&s, PyString_FromString(")"));
277 #endif
278
279 return s;
280 }
281
282
PVector_hash(PVector * self)283 static long PVector_hash(PVector *self) {
284 // Follows the pattern of the tuple hash
285 long x, y;
286 Py_ssize_t i;
287 long mult = 1000003L;
288 x = 0x456789L;
289 for(i=0; i<self->count; i++) {
290 y = PyObject_Hash(_get_item(self, i));
291 if (y == -1) {
292 return -1;
293 }
294 x = (x ^ y) * mult;
295 mult += (long)(82520L + i + i);
296 }
297
298 x += 97531L;
299 if(x == -1) {
300 x = -2;
301 }
302
303 return x;
304 }
305
compareSizes(long vlen,long wlen,int op)306 static PyObject* compareSizes(long vlen, long wlen, int op) {
307 int cmp;
308 PyObject *res;
309 switch (op) {
310 case Py_LT: cmp = vlen < wlen; break;
311 case Py_LE: cmp = vlen <= wlen; break;
312 case Py_EQ: cmp = vlen == wlen; break;
313 case Py_NE: cmp = vlen != wlen; break;
314 case Py_GT: cmp = vlen > wlen; break;
315 case Py_GE: cmp = vlen >= wlen; break;
316 default: return NULL; /* cannot happen */
317 }
318
319 if (cmp) {
320 res = Py_True;
321 } else {
322 res = Py_False;
323 }
324
325 Py_INCREF(res);
326 return res;
327 }
328
PVector_richcompare(PyObject * v,PyObject * w,int op)329 static PyObject* PVector_richcompare(PyObject *v, PyObject *w, int op) {
330 // Follows the principles of the tuple comparison
331 PVector *vt, *wt;
332 Py_ssize_t i;
333 Py_ssize_t vlen, wlen;
334 PyObject *list;
335 PyObject *result;
336
337 if(!PVector_CheckExact(v) || !PVector_CheckExact(w)) {
338 if(PVector_CheckExact(v)) {
339 list = PVector_toList((PVector*)v);
340 result = PyObject_RichCompare(list , w, op);
341 Py_DECREF(list);
342 return result;
343 }
344
345 if(PVector_CheckExact(w)) {
346 list = PVector_toList((PVector*)w);
347 result = PyObject_RichCompare(v, list, op);
348 Py_DECREF(list);
349 return result;
350 }
351
352 Py_INCREF(Py_NotImplemented);
353 return Py_NotImplemented;
354 }
355
356 if((op == Py_EQ) && (v == w)) {
357 Py_INCREF(Py_True);
358 return Py_True;
359 }
360
361 vt = (PVector *)v;
362 wt = (PVector *)w;
363
364 vlen = vt->count;
365 wlen = wt->count;
366
367 if (vlen != wlen) {
368 if (op == Py_EQ) {
369 Py_INCREF(Py_False);
370 return Py_False;
371 } else if (op == Py_NE) {
372 Py_INCREF(Py_True);
373 return Py_True;
374 }
375 }
376
377 /* Search for the first index where items are different. */
378 PyObject *left = NULL;
379 PyObject *right = NULL;
380 for (i = 0; i < vlen && i < wlen; i++) {
381 left = _get_item(vt, i);
382 right = _get_item(wt, i);
383 int k = PyObject_RichCompareBool(left, right, Py_EQ);
384 if (k < 0) {
385 return NULL;
386 }
387 if (!k) {
388 break;
389 }
390 }
391
392 if (i >= vlen || i >= wlen) {
393 /* No more items to compare -- compare sizes */
394 return compareSizes(vlen, wlen, op);
395 }
396
397 /* We have an item that differs -- shortcuts for EQ/NE */
398 if (op == Py_EQ) {
399 Py_INCREF(Py_False);
400 return Py_False;
401 } else if (op == Py_NE) {
402 Py_INCREF(Py_True);
403 return Py_True;
404 } else {
405 /* Compare the final item again using the proper operator */
406 return PyObject_RichCompare(left, right, op);
407 }
408 }
409
410
PVector_repeat(PVector * self,Py_ssize_t n)411 static PyObject* PVector_repeat(PVector *self, Py_ssize_t n) {
412 if (n < 0) {
413 n = 0;
414 }
415
416 if ((n == 0) || (self->count == 0)) {
417 Py_INCREF(EMPTY_VECTOR);
418 return (PyObject *)EMPTY_VECTOR;
419 } else if (n == 1) {
420 Py_INCREF(self);
421 return (PyObject *)self;
422 } else if ((self->count * n)/self->count != n) {
423 return PyErr_NoMemory();
424 } else {
425 int i, j;
426 PVector *newVec = copyPVector(self);
427 for(i=0; i<(n-1); i++) {
428 for(j=0; j<self->count; j++) {
429 extendWithItem(newVec, PVector_get_item(self, j));
430 }
431 }
432 return (PyObject*)newVec;
433 }
434 }
435
PVector_traverse(PVector * o,visitproc visit,void * arg)436 static int PVector_traverse(PVector *o, visitproc visit, void *arg) {
437 // Naive traverse
438 Py_ssize_t i;
439 for (i = o->count; --i >= 0; ) {
440 Py_VISIT(_get_item(o, i));
441 }
442
443 return 0;
444 }
445
446
PVector_index(PVector * self,PyObject * args)447 static PyObject* PVector_index(PVector *self, PyObject *args) {
448 // A direct rip-off of the tuple version
449 Py_ssize_t i, start=0, stop=self->count;
450 PyObject *value;
451
452 if (!PyArg_ParseTuple(args, "O|O&O&:index", &value,
453 _PyEval_SliceIndex, &start,
454 _PyEval_SliceIndex, &stop)) {
455 return NULL;
456 }
457
458 if (start < 0) {
459 start += self->count;
460 if (start < 0) {
461 start = 0;
462 }
463 }
464
465 if (stop < 0) {
466 stop += self->count;
467 if (stop < 0) {
468 stop = 0;
469 }
470 }
471
472 for (i = start; i < stop && i < self->count; i++) {
473 int cmp = PyObject_RichCompareBool(_get_item(self, i), value, Py_EQ);
474 if (cmp > 0) {
475 #if PY_MAJOR_VERSION >= 3
476 return PyLong_FromSsize_t(i);
477 #else
478 return PyInt_FromSsize_t(i);
479 #endif
480 } else if (cmp < 0) {
481 return NULL;
482 }
483 }
484
485 PyErr_SetString(PyExc_ValueError, "PVector.index(x): x not in vector");
486 return NULL;
487 }
488
PVector_count(PVector * self,PyObject * value)489 static PyObject* PVector_count(PVector *self, PyObject *value) {
490 Py_ssize_t count = 0;
491 Py_ssize_t i;
492
493 for (i = 0; i < self->count; i++) {
494 int cmp = PyObject_RichCompareBool(_get_item(self, i), value, Py_EQ);
495 if (cmp > 0) {
496 count++;
497 } else if (cmp < 0) {
498 return NULL;
499 }
500 }
501
502 #if PY_MAJOR_VERSION >= 3
503 return PyLong_FromSsize_t(count);
504 #else
505 return PyInt_FromSsize_t(count);
506 #endif
507 }
508
PVector_pickle_reduce(PVector * self)509 static PyObject* PVector_pickle_reduce(PVector *self) {
510
511 PyObject* module = PyImport_ImportModule("pvectorc");
512 PyObject* pvector_fn = PyObject_GetAttrString(module, "pvector");
513 Py_DECREF(module);
514
515 PyObject *list = PVector_toList(self);
516 PyObject *arg_tuple = PyTuple_New(1);
517 PyTuple_SET_ITEM(arg_tuple, 0, list);
518
519 PyObject *result_tuple = PyTuple_New(2);
520 PyTuple_SET_ITEM(result_tuple, 0, pvector_fn);
521 PyTuple_SET_ITEM(result_tuple, 1, arg_tuple);
522
523 return result_tuple;
524 }
525
rawCopyPVector(PVector * vector)526 static PVector* rawCopyPVector(PVector* vector) {
527 PVector* newVector = PyObject_GC_New(PVector, &PVectorType);
528 newVector->count = vector->count;
529 newVector->shift = vector->shift;
530 newVector->root = vector->root;
531 newVector->tail = vector->tail;
532 newVector->in_weakreflist = NULL;
533 PyObject_GC_Track((PyObject*)newVector);
534 return newVector;
535 }
536
initializeEvolver(PVectorEvolver * evolver,PVector * vector,PyObject * appendList)537 static void initializeEvolver(PVectorEvolver* evolver, PVector* vector, PyObject* appendList) {
538 // Need to hold a reference to the underlying vector to manage
539 // the ref counting properly.
540 evolver->originalVector = vector;
541 evolver->newVector = vector;
542
543 if(appendList == NULL) {
544 evolver->appendList = PyList_New(0);
545 } else {
546 evolver->appendList = appendList;
547 }
548 }
549
PVector_evolver(PVector * self)550 static PyObject * PVector_evolver(PVector *self) {
551 PVectorEvolver *evolver = PyObject_GC_New(PVectorEvolver, &PVectorEvolverType);
552 if (evolver == NULL) {
553 return NULL;
554 }
555 initializeEvolver(evolver, self, NULL);
556 PyObject_GC_Track(evolver);
557 Py_INCREF(self);
558 return (PyObject *)evolver;
559 }
560
561
copyInsert(void ** dest,void ** src,Py_ssize_t pos,void * obj)562 static void copyInsert(void** dest, void** src, Py_ssize_t pos, void *obj) {
563 memcpy(dest, src, BRANCH_FACTOR * sizeof(void*));
564 dest[pos] = obj;
565 }
566
567 static PyObject* PVector_append(PVector *self, PyObject *obj);
568
569 static PyObject* PVector_transform(PVector *self, PyObject *obj);
570
571 static PyObject* PVector_set(PVector *self, PyObject *obj);
572
573 static PyObject* PVector_mset(PVector *self, PyObject *args);
574
575 static PyObject* PVector_subscript(PVector* self, PyObject* item);
576
577 static PyObject* PVector_extend(PVector *self, PyObject *args);
578
579 static PyObject* PVector_delete(PVector *self, PyObject *args);
580
581 static PyObject* PVector_remove(PVector *self, PyObject *args);
582
583 static PySequenceMethods PVector_sequence_methods = {
584 (lenfunc)PVector_len, /* sq_length */
585 (binaryfunc)PVector_extend, /* sq_concat */
586 (ssizeargfunc)PVector_repeat, /* sq_repeat */
587 (ssizeargfunc)PVector_get_item, /* sq_item */
588 // TODO might want to move the slice function to here
589 NULL, /* sq_slice */
590 NULL, /* sq_ass_item */
591 NULL, /* sq_ass_slice */
592 NULL, /* sq_contains */
593 NULL, /* sq_inplace_concat */
594 NULL, /* sq_inplace_repeat */
595 };
596
597 static PyMappingMethods PVector_mapping_methods = {
598 (lenfunc)PVector_len,
599 (binaryfunc)PVector_subscript,
600 NULL
601 };
602
603
604 static PyMethodDef PVector_methods[] = {
605 {"append", (PyCFunction)PVector_append, METH_O, "Appends an element"},
606 {"set", (PyCFunction)PVector_set, METH_VARARGS, "Inserts an element at the specified position"},
607 {"extend", (PyCFunction)PVector_extend, METH_O|METH_COEXIST, "Extend"},
608 {"transform", (PyCFunction)PVector_transform, METH_VARARGS, "Apply one or more transformations"},
609 {"index", (PyCFunction)PVector_index, METH_VARARGS, "Return first index of value"},
610 {"count", (PyCFunction)PVector_count, METH_O, "Return number of occurrences of value"},
611 {"__reduce__", (PyCFunction)PVector_pickle_reduce, METH_NOARGS, "Pickle support method"},
612 {"evolver", (PyCFunction)PVector_evolver, METH_NOARGS, "Return new evolver for pvector"},
613 {"mset", (PyCFunction)PVector_mset, METH_VARARGS, "Inserts multiple elements at the specified positions"},
614 {"tolist", (PyCFunction)PVector_toList, METH_NOARGS, "Convert to list"},
615 {"delete", (PyCFunction)PVector_delete, METH_VARARGS, "Delete element(s) by index"},
616 {"remove", (PyCFunction)PVector_remove, METH_VARARGS, "Remove element(s) by equality"},
617 {NULL}
618 };
619
620 static PyObject * PVectorIter_iter(PyObject *seq);
621
622 static PyTypeObject PVectorType = {
623 PyVarObject_HEAD_INIT(NULL, 0)
624 "pvectorc.PVector", /* tp_name */
625 sizeof(PVector), /* tp_basicsize */
626 0, /* tp_itemsize */
627 (destructor)PVector_dealloc, /* tp_dealloc */
628 0, /* tp_print */
629 0, /* tp_getattr */
630 0, /* tp_setattr */
631 0, /* tp_compare */
632 (reprfunc)PVector_repr, /* tp_repr */
633 0, /* tp_as_number */
634 &PVector_sequence_methods, /* tp_as_sequence */
635 &PVector_mapping_methods, /* tp_as_mapping */
636 (hashfunc)PVector_hash, /* tp_hash */
637 0, /* tp_call */
638 0, /* tp_str */
639 0, /* tp_getattro */
640 0, /* tp_setattro */
641 0, /* tp_as_buffer */
642 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
643 "Persistent vector", /* tp_doc */
644 (traverseproc)PVector_traverse, /* tp_traverse */
645 0, /* tp_clear */
646 PVector_richcompare, /* tp_richcompare */
647 offsetof(PVector, in_weakreflist), /* tp_weaklistoffset */
648 PVectorIter_iter, /* tp_iter */
649 0, /* tp_iternext */
650 PVector_methods, /* tp_methods */
651 PVector_members, /* tp_members */
652 0, /* tp_getset */
653 0, /* tp_base */
654 0, /* tp_dict */
655 0, /* tp_descr_get */
656 0, /* tp_descr_set */
657 0, /* tp_dictoffset */
658 };
659
pyrsistent_pvec(PyObject * self,PyObject * args)660 static PyObject* pyrsistent_pvec(PyObject *self, PyObject *args) {
661 debug("pyrsistent_pvec(): %x\n", args);
662
663 PyObject *argObj = NULL; /* list of arguments */
664
665 if(!PyArg_ParseTuple(args, "|O", &argObj)) {
666 return NULL;
667 }
668
669 if(argObj == NULL) {
670 Py_INCREF(EMPTY_VECTOR);
671 return (PyObject*)EMPTY_VECTOR;
672 }
673
674 return PVector_extend(EMPTY_VECTOR, argObj);
675 }
676
emptyNewPvec(void)677 static PVector* emptyNewPvec(void) {
678 PVector *pvec = PyObject_GC_New(PVector, &PVectorType);
679 debug("pymem alloc_new %x, ref cnt: %u\n", pvec, pvec->ob_refcnt);
680 pvec->count = (Py_ssize_t)0;
681 pvec->shift = SHIFT;
682 pvec->root = newNode();
683 pvec->tail = newNode();
684 pvec->in_weakreflist = NULL;
685 PyObject_GC_Track((PyObject*)pvec);
686 return pvec;
687 }
688
incRefs(PyObject ** obj)689 static void incRefs(PyObject **obj) {
690 // TODO-OPT: Would it be OK to exit on first NULL? Should not be any
691 // non NULLs beyond a NULL.
692 int i;
693 for(i = 0; i < BRANCH_FACTOR; i++) {
694 Py_XINCREF(obj[i]);
695 }
696 }
697
698
newPvec(unsigned int count,unsigned int shift,VNode * root)699 static PVector* newPvec(unsigned int count, unsigned int shift, VNode *root) {
700 // TODO-OPT: Introduce object cache
701 PVector *pvec = PyObject_GC_New(PVector, &PVectorType);
702 debug("pymem alloc_copy %x, ref cnt: %u\n", pvec, pvec->ob_refcnt);
703 pvec->count = count;
704 pvec->shift = shift;
705 pvec->root = root;
706 pvec->tail = newNode();
707 pvec->in_weakreflist = NULL;
708 PyObject_GC_Track((PyObject*)pvec);
709 return pvec;
710 }
711
newPath(unsigned int level,VNode * node)712 static VNode* newPath(unsigned int level, VNode* node){
713 if(level == 0) {
714 INC_NODE_REF_COUNT(node);
715 return node;
716 }
717
718 VNode* result = newNode();
719 result->items[0] = newPath(level - SHIFT, node);
720 return result;
721 }
722
pushTail(unsigned int level,unsigned int count,VNode * parent,VNode * tail)723 static VNode* pushTail(unsigned int level, unsigned int count, VNode* parent, VNode* tail) {
724 int subIndex = ((count - 1) >> level) & BIT_MASK;
725 VNode* result = copyNode(parent);
726 VNode* nodeToInsert;
727 VNode* child;
728 debug("pushTail(): count = %i, subIndex = %i\n", count, subIndex);
729
730 if(level == SHIFT) {
731 // We're at the bottom
732 INC_NODE_REF_COUNT(tail);
733 nodeToInsert = tail;
734 } else {
735 // More levels available in the tree
736 child = parent->items[subIndex];
737
738 if(child != NULL) {
739 nodeToInsert = pushTail(level - SHIFT, count, child, tail);
740
741 // Need to make an adjustment of the ref COUNT for the child node here since
742 // it was incremented in an earlier stage when the node was copied. Now the child
743 // node will be part of the path copy so the number of references to the original
744 // child will not increase at all.
745 DEC_NODE_REF_COUNT(child);
746 } else {
747 nodeToInsert = newPath(level - SHIFT, tail);
748 }
749 }
750
751 result->items[subIndex] = nodeToInsert;
752 return result;
753 }
754
copyPVector(PVector * original)755 static PVector* copyPVector(PVector *original) {
756 PVector *newVec = newPvec(original->count, original->shift, original->root);
757 INC_NODE_REF_COUNT(original->root);
758 memcpy(newVec->tail->items, original->tail->items, TAIL_SIZE(original) * sizeof(void*));
759 incRefs((PyObject**)newVec->tail->items);
760 return newVec;
761 }
762
763 /* Does not steal a reference, this must be managed outside of this function */
extendWithItem(PVector * newVec,PyObject * item)764 static void extendWithItem(PVector *newVec, PyObject *item) {
765 unsigned int tail_size = TAIL_SIZE(newVec);
766
767 if(tail_size >= BRANCH_FACTOR) {
768 VNode* new_root;
769 if(ROOT_NODE_FULL(newVec)) {
770 new_root = newNode();
771 new_root->items[0] = newVec->root;
772 new_root->items[1] = newPath(newVec->shift, newVec->tail);
773 newVec->shift += SHIFT;
774 } else {
775 new_root = pushTail(newVec->shift, newVec->count, newVec->root, newVec->tail);
776 releaseNode(newVec->shift, newVec->root);
777 }
778
779 newVec->root = new_root;
780
781 // Need to adjust the ref count of the old tail here since no new references were
782 // actually created, we just moved the tail.
783 DEC_NODE_REF_COUNT(newVec->tail);
784 newVec->tail = newNode();
785 tail_size = 0;
786 }
787
788 newVec->tail->items[tail_size] = item;
789 newVec->count++;
790 }
791
792
793 #if PY_MAJOR_VERSION >= 3
794 // This was changed in 3.2 but we do not claim compatibility with any older version of python 3.
795 #define SLICE_CAST
796 #else
797 #define SLICE_CAST (PySliceObject *)
798 #endif
799
PVector_subscript(PVector * self,PyObject * item)800 static PyObject *PVector_subscript(PVector* self, PyObject* item) {
801 if (PyIndex_Check(item)) {
802 Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
803 if (i == -1 && PyErr_Occurred()) {
804 return NULL;
805 }
806
807 return PVector_get_item(self, i);
808 } else if (PySlice_Check(item)) {
809 Py_ssize_t start, stop, step, slicelength, cur, i;
810 if (PySlice_GetIndicesEx(SLICE_CAST item, self->count,
811 &start, &stop, &step, &slicelength) < 0) {
812 return NULL;
813 }
814
815 debug("start=%i, stop=%i, step=%i\n", start, stop, step);
816
817 if (slicelength <= 0) {
818 Py_INCREF(EMPTY_VECTOR);
819 return (PyObject*)EMPTY_VECTOR;
820 } else if((slicelength == self->count) && (step > 0)) {
821 Py_INCREF(self);
822 return (PyObject*)self;
823 } else {
824 PVector *newVec = copyPVector(EMPTY_VECTOR);
825 for (cur=start, i=0; i<slicelength; cur += (size_t)step, i++) {
826 extendWithItem(newVec, PVector_get_item(self, cur));
827 }
828
829 return (PyObject*)newVec;
830 }
831 } else {
832 PyErr_Format(PyExc_TypeError, "pvector indices must be integers, not %.200s", Py_TYPE(item)->tp_name);
833 return NULL;
834 }
835 }
836
837 /* A hack to get some of the error handling code away from the function
838 doing the actual work */
839 #define HANDLE_ITERATION_ERROR() \
840 if (PyErr_Occurred()) { \
841 if (PyErr_ExceptionMatches(PyExc_StopIteration)) { \
842 PyErr_Clear(); \
843 } else { \
844 return NULL; \
845 } \
846 }
847
848
849 /* Returns a new vector that is extended with the iterable b.
850 Takes a copy of the original vector and performs the extension in place on this
851 one for efficiency.
852
853 These are some optimizations that could be done to this function,
854 these are not considered important enough yet though.
855 - Use the PySequence_Fast ops if the iterable is a list or a tuple (which it
856 whould probably often be)
857 - Only copy the original tail if it is not full
858 - No need to try to increment ref count in tail for the whole tail
859 */
PVector_extend(PVector * self,PyObject * iterable)860 static PyObject* PVector_extend(PVector *self, PyObject *iterable) {
861 PyObject *it;
862 PyObject *(*iternext)(PyObject *);
863
864 it = PyObject_GetIter(iterable);
865 if (it == NULL) {
866 return NULL;
867 }
868
869 // TODO-OPT: Use special fast iterator if available
870 iternext = *Py_TYPE(it)->tp_iternext;
871 PyObject *item = iternext(it);
872 if (item == NULL) {
873 Py_DECREF(it);
874 HANDLE_ITERATION_ERROR()
875 Py_INCREF(self);
876 return (PyObject *)self;
877 } else {
878 PVector *newVec = copyPVector(self);
879 // TODO-OPT test using special case code here for extension to
880 // avoid recalculating tail length all the time.
881 while(item != NULL) {
882 extendWithItem(newVec, item);
883 item = iternext(it);
884 }
885
886 Py_DECREF(it);
887 HANDLE_ITERATION_ERROR()
888 return (PyObject*)newVec;
889 }
890 }
891
892 /*
893 Steals a reference to the object that is appended to the list.
894 */
PVector_append(PVector * self,PyObject * obj)895 static PyObject* PVector_append(PVector *self, PyObject *obj) {
896 assert (obj != NULL);
897
898 unsigned int tail_size = TAIL_SIZE(self);
899 debug("append(): count = %u, tail_size = %u\n", self->count, tail_size);
900
901 // Does the new object fit in the tail? If so, take a copy of the tail and
902 // insert the new element in that.
903 if(tail_size < BRANCH_FACTOR) {
904 INC_NODE_REF_COUNT(self->root);
905 PVector *new_pvec = newPvec(self->count + 1, self->shift, self->root);
906 // TODO-OPT No need to copy more than the current tail length
907 // TODO-OPT No need to incRefs for all elements all the time
908 copyInsert(new_pvec->tail->items, self->tail->items, tail_size, obj);
909 incRefs((PyObject**)new_pvec->tail->items);
910 debug("append(): new_pvec=%p, new_pvec->tail=%p, new_pvec->root=%p\n",
911 new_pvec, new_pvec->tail, new_pvec->root);
912
913 return (PyObject*)new_pvec;
914 }
915
916 // Tail is full, need to push it into the tree
917 VNode* new_root;
918 unsigned int new_shift;
919 if(ROOT_NODE_FULL(self)) {
920 new_root = newNode();
921 new_root->items[0] = self->root;
922 INC_NODE_REF_COUNT(self->root);
923 new_root->items[1] = newPath(self->shift, self->tail);
924 new_shift = self->shift + SHIFT;
925 } else {
926 new_root = pushTail(self->shift, self->count, self->root, self->tail);
927 new_shift = self->shift;
928 }
929
930 PVector* pvec = newPvec(self->count + 1, new_shift, new_root);
931 pvec->tail->items[0] = obj;
932 Py_XINCREF(obj);
933 debug("append_push(): pvec=%p, pvec->tail=%p, pvec->root=%p\n", pvec, pvec->tail, pvec->root);
934 return (PyObject*)pvec;
935 }
936
doSet(VNode * node,unsigned int level,unsigned int position,PyObject * value)937 static VNode* doSet(VNode* node, unsigned int level, unsigned int position, PyObject* value) {
938 debug("doSet(): level == %i\n", level);
939 if(level == 0) {
940 // TODO-OPT: Perhaps an alloc followed by a reset of reference
941 // count is enough here since we overwrite all subnodes below.
942 VNode* theNewNode = newNode();
943 copyInsert(theNewNode->items, node->items, position & BIT_MASK, value);
944 incRefs((PyObject**)theNewNode->items);
945 return theNewNode;
946 } else {
947 VNode* theNewNode = copyNode(node);
948 Py_ssize_t index = (position >> level) & BIT_MASK;
949
950 // Drop reference to this node since we're about to replace it
951 DEC_NODE_REF_COUNT((VNode*)theNewNode->items[index]);
952 theNewNode->items[index] = doSet(node->items[index], level - SHIFT, position, value);
953 return theNewNode;
954 }
955 }
956
957
internalSet(PVector * self,Py_ssize_t position,PyObject * argObj)958 static PyObject* internalSet(PVector *self, Py_ssize_t position, PyObject *argObj) {
959 if(position < 0) {
960 position += self->count;
961 }
962
963 if((0 <= position) && (position < self->count)) {
964 if(position >= TAIL_OFF(self)) {
965 // Reuse the root, replace the tail
966 INC_NODE_REF_COUNT(self->root);
967 PVector *new_pvec = newPvec(self->count, self->shift, self->root);
968 copyInsert(new_pvec->tail->items, self->tail->items, position & BIT_MASK, argObj);
969 incRefs((PyObject**)new_pvec->tail->items);
970 return (PyObject*)new_pvec;
971 } else {
972 // Keep the tail, replace the root
973 VNode *newRoot = doSet(self->root, self->shift, position, argObj);
974 PVector *new_pvec = newPvec(self->count, self->shift, newRoot);
975
976 // Free the tail and replace it with a reference to the tail of the original vector
977 freeNode(new_pvec->tail);
978 new_pvec->tail = self->tail;
979 INC_NODE_REF_COUNT(self->tail);
980 return (PyObject*)new_pvec;
981 }
982 } else if (position == self->count) {
983 // TODO Remove this case?
984 return PVector_append(self, argObj);
985 } else {
986 PyErr_Format(PyExc_IndexError, "Index out of range: %zd", position);
987 return NULL;
988 }
989 }
990
PVector_transform(PVector * self,PyObject * obj)991 static PyObject* PVector_transform(PVector *self, PyObject *obj) {
992 return transform(self, obj);
993 }
994
995 /*
996 Steals a reference to the object that is inserted in the vector.
997 */
PVector_set(PVector * self,PyObject * args)998 static PyObject* PVector_set(PVector *self, PyObject *args) {
999 PyObject *argObj = NULL; /* argument to insert */
1000 Py_ssize_t position;
1001
1002 /* The n parses for size, the O parses for a Python object */
1003 if(!PyArg_ParseTuple(args, "nO", &position, &argObj)) {
1004 return NULL;
1005 }
1006
1007 return internalSet(self, position, argObj);
1008 }
1009
1010
PVector_mset(PVector * self,PyObject * args)1011 static PyObject* PVector_mset(PVector *self, PyObject *args) {
1012 Py_ssize_t size = PyTuple_Size(args);
1013 if(size % 2) {
1014 PyErr_SetString(PyExc_TypeError, "mset expected an even number of arguments");
1015 return NULL;
1016 }
1017
1018 PVectorEvolver* evolver = (PVectorEvolver*)PVector_evolver(self);
1019 Py_ssize_t i;
1020 for(i=0; i<size; i+=2) {
1021 if(PVectorEvolver_set_item(evolver, PyTuple_GetItem(args, i), PyTuple_GetItem(args, i + 1)) < 0) {
1022 Py_DECREF(evolver);
1023 return NULL;
1024 }
1025 }
1026
1027 PyObject* vector = PVectorEvolver_persistent(evolver);
1028 Py_DECREF(evolver);
1029 return vector;
1030 }
1031
1032
internalDelete(PVector * self,Py_ssize_t index,PyObject * stop_obj)1033 static PyObject* internalDelete(PVector *self, Py_ssize_t index, PyObject *stop_obj) {
1034 Py_ssize_t stop;
1035 PyObject *list;
1036 PyObject *result;
1037
1038 if (index < 0) {
1039 index += self->count;
1040 }
1041
1042 if (stop_obj != NULL) {
1043 if (PyIndex_Check(stop_obj)) {
1044 stop = PyNumber_AsSsize_t(stop_obj, PyExc_IndexError);
1045 if (stop == -1 && PyErr_Occurred()) {
1046 return NULL;
1047 }
1048 } else {
1049 PyErr_Format(PyExc_TypeError, "Stop index must be integer, not %.200s", Py_TYPE(stop_obj)->tp_name);
1050 return NULL;
1051 }
1052
1053 if (stop < 0) {
1054 stop += self->count;
1055 }
1056 } else {
1057 if (index < 0 || index >= self->count) {
1058 PyErr_SetString(PyExc_IndexError, "delete index out of range");
1059 return NULL;
1060 }
1061
1062 stop = index + 1;
1063 }
1064
1065 list = PVector_toList(self);
1066 if(PyList_SetSlice(list, index, stop, NULL) < 0) {
1067 return NULL;
1068 }
1069
1070 result = PVector_extend(EMPTY_VECTOR, list);
1071 Py_DECREF(list);
1072 return result;
1073 }
1074
PVector_delete(PVector * self,PyObject * args)1075 static PyObject* PVector_delete(PVector *self, PyObject *args) {
1076 Py_ssize_t index;
1077 PyObject *stop_obj = NULL;
1078
1079 if(!PyArg_ParseTuple(args, "n|O:delete", &index, &stop_obj)) {
1080 return NULL;
1081 }
1082
1083 return internalDelete(self, index, stop_obj);
1084 }
1085
PVector_remove(PVector * self,PyObject * args)1086 static PyObject* PVector_remove(PVector *self, PyObject *args) {
1087 Py_ssize_t index;
1088 PyObject* py_index = PVector_index(self, args);
1089
1090 if(py_index != NULL) {
1091 #if PY_MAJOR_VERSION >= 3
1092 index = PyLong_AsSsize_t(py_index);
1093 #else
1094 index = PyInt_AsSsize_t(py_index);
1095 #endif
1096 Py_DECREF(py_index);
1097 return internalDelete(self, index, NULL);
1098 }
1099
1100 PyErr_SetString(PyExc_ValueError, "PVector.remove(x): x not in vector");
1101 return NULL;
1102 }
1103
1104
1105 /*********************** PVector Iterator **************************/
1106
1107 /*
1108 The Sequence class provides us with a default iterator but the runtime
1109 overhead of using that compared to the iterator below is huge.
1110 */
1111
1112 typedef struct {
1113 PyObject_HEAD
1114 Py_ssize_t it_index;
1115 PVector *it_seq; /* Set to NULL when iterator is exhausted */
1116 } PVectorIter;
1117
1118 static void PVectorIter_dealloc(PVectorIter *);
1119 static int PVectorIter_traverse(PVectorIter *, visitproc, void *);
1120 static PyObject *PVectorIter_next(PVectorIter *);
1121
1122 static PyMethodDef PVectorIter_methods[] = {
1123 {NULL, NULL} /* sentinel */
1124 };
1125
1126 static PyTypeObject PVectorIterType = {
1127 PyVarObject_HEAD_INIT(NULL, 0)
1128 "pvector_iterator", /* tp_name */
1129 sizeof(PVectorIter), /* tp_basicsize */
1130 0, /* tp_itemsize */
1131 /* methods */
1132 (destructor)PVectorIter_dealloc, /* tp_dealloc */
1133 0, /* tp_print */
1134 0, /* tp_getattr */
1135 0, /* tp_setattr */
1136 0, /* tp_compare */
1137 0, /* tp_repr */
1138 0, /* tp_as_number */
1139 0, /* tp_as_sequence */
1140 0, /* tp_as_mapping */
1141 0, /* tp_hash */
1142 0, /* tp_call */
1143 0, /* tp_str */
1144 PyObject_GenericGetAttr, /* tp_getattro */
1145 0, /* tp_setattro */
1146 0, /* tp_as_buffer */
1147 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
1148 0, /* tp_doc */
1149 (traverseproc)PVectorIter_traverse, /* tp_traverse */
1150 0, /* tp_clear */
1151 0, /* tp_richcompare */
1152 0, /* tp_weaklistoffset */
1153 PyObject_SelfIter, /* tp_iter */
1154 (iternextfunc)PVectorIter_next, /* tp_iternext */
1155 PVectorIter_methods, /* tp_methods */
1156 0, /* tp_members */
1157 };
1158
PVectorIter_iter(PyObject * seq)1159 static PyObject *PVectorIter_iter(PyObject *seq) {
1160 PVectorIter *it = PyObject_GC_New(PVectorIter, &PVectorIterType);
1161 if (it == NULL) {
1162 return NULL;
1163 }
1164
1165 it->it_index = 0;
1166 Py_INCREF(seq);
1167 it->it_seq = (PVector *)seq;
1168 PyObject_GC_Track(it);
1169 return (PyObject *)it;
1170 }
1171
PVectorIter_dealloc(PVectorIter * it)1172 static void PVectorIter_dealloc(PVectorIter *it) {
1173 PyObject_GC_UnTrack(it);
1174 Py_XDECREF(it->it_seq);
1175 PyObject_GC_Del(it);
1176 }
1177
PVectorIter_traverse(PVectorIter * it,visitproc visit,void * arg)1178 static int PVectorIter_traverse(PVectorIter *it, visitproc visit, void *arg) {
1179 Py_VISIT(it->it_seq);
1180 return 0;
1181 }
1182
PVectorIter_next(PVectorIter * it)1183 static PyObject *PVectorIter_next(PVectorIter *it) {
1184 assert(it != NULL);
1185 PVector *seq = it->it_seq;
1186 if (seq == NULL) {
1187 return NULL;
1188 }
1189
1190 if (it->it_index < seq->count) {
1191 PyObject *item = _get_item(seq, it->it_index);
1192 ++it->it_index;
1193 Py_INCREF(item);
1194 return item;
1195 }
1196
1197 Py_DECREF(seq);
1198 it->it_seq = NULL;
1199 return NULL;
1200 }
1201
1202
1203 /*********************** PVector Evolver **************************/
1204
1205 /*
1206 Evolver to make multi updates easier to work with and more efficient.
1207 */
1208
1209 static void PVectorEvolver_dealloc(PVectorEvolver *);
1210 static PyObject *PVectorEvolver_append(PVectorEvolver *, PyObject *);
1211 static PyObject *PVectorEvolver_extend(PVectorEvolver *, PyObject *);
1212 static PyObject *PVectorEvolver_set(PVectorEvolver *, PyObject *);
1213 static PyObject *PVectorEvolver_delete(PVectorEvolver *self, PyObject *args);
1214 static PyObject *PVectorEvolver_subscript(PVectorEvolver *, PyObject *);
1215 static PyObject *PVectorEvolver_persistent(PVectorEvolver *);
1216 static Py_ssize_t PVectorEvolver_len(PVectorEvolver *);
1217 static PyObject *PVectorEvolver_is_dirty(PVectorEvolver *);
1218 static int PVectorEvolver_traverse(PVectorEvolver *self, visitproc visit, void *arg);
1219
1220 static PyMappingMethods PVectorEvolver_mapping_methods = {
1221 (lenfunc)PVectorEvolver_len,
1222 (binaryfunc)PVectorEvolver_subscript,
1223 (objobjargproc)PVectorEvolver_set_item,
1224 };
1225
1226
1227 static PyMethodDef PVectorEvolver_methods[] = {
1228 {"append", (PyCFunction)PVectorEvolver_append, METH_O, "Appends an element"},
1229 {"extend", (PyCFunction)PVectorEvolver_extend, METH_O|METH_COEXIST, "Extend"},
1230 {"set", (PyCFunction)PVectorEvolver_set, METH_VARARGS, "Set item"},
1231 {"delete", (PyCFunction)PVectorEvolver_delete, METH_VARARGS, "Delete item"},
1232 {"persistent", (PyCFunction)PVectorEvolver_persistent, METH_NOARGS, "Create PVector from evolver"},
1233 {"is_dirty", (PyCFunction)PVectorEvolver_is_dirty, METH_NOARGS, "Check if evolver contains modifications"},
1234 {NULL, NULL} /* sentinel */
1235 };
1236
1237 static PyTypeObject PVectorEvolverType = {
1238 PyVarObject_HEAD_INIT(NULL, 0)
1239 "pvector_evolver", /* tp_name */
1240 sizeof(PVectorEvolver), /* tp_basicsize */
1241 0, /* tp_itemsize */
1242 /* methods */
1243 (destructor)PVectorEvolver_dealloc, /* tp_dealloc */
1244 0, /* tp_print */
1245 0, /* tp_getattr */
1246 0, /* tp_setattr */
1247 0, /* tp_compare */
1248 0, /* tp_repr */
1249 0, /* tp_as_number */
1250 0, /* tp_as_sequence */
1251 &PVectorEvolver_mapping_methods, /* tp_as_mapping */
1252 0, /* tp_hash */
1253 0, /* tp_call */
1254 0, /* tp_str */
1255 PyObject_GenericGetAttr, /* tp_getattro */
1256 0, /* tp_setattro */
1257 0, /* tp_as_buffer */
1258 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
1259 0, /* tp_doc */
1260 (traverseproc)PVectorEvolver_traverse, /* tp_traverse */
1261 0, /* tp_clear */
1262 0, /* tp_richcompare */
1263 0, /* tp_weaklistoffset */
1264 0, /* tp_iter */
1265 0, /* tp_iternext */
1266 PVectorEvolver_methods, /* tp_methods */
1267 0, /* tp_members */
1268 };
1269
1270
1271 // Indicate that a node is "dirty" (has been updated by the evolver)
1272 // by setting the MSB of the refCount. This will be cleared when
1273 // creating a pvector from the evolver (cleaning it).
1274 #define DIRTY_BIT 0x80000000
1275 #define REF_COUNT_MASK (~DIRTY_BIT)
1276 #define IS_DIRTY(node) ((node)->refCount & DIRTY_BIT)
1277 #define SET_DIRTY(node) ((node)->refCount |= DIRTY_BIT)
1278 #define CLEAR_DIRTY(node) ((node)->refCount &= REF_COUNT_MASK)
1279
1280
cleanNodeRecursively(VNode * node,int level)1281 static void cleanNodeRecursively(VNode *node, int level) {
1282 debug("Cleaning recursively node=%p, level=%u\n", node, level);
1283
1284 int i;
1285 CLEAR_DIRTY(node);
1286 SET_NODE_REF_COUNT(node, 1);
1287 if(level > 0) {
1288 for(i = 0; i < BRANCH_FACTOR; i++) {
1289 VNode *nextNode = (VNode*)node->items[i];
1290 if((nextNode != NULL) && IS_DIRTY(nextNode)) {
1291 cleanNodeRecursively(nextNode, level - SHIFT);
1292 }
1293 }
1294 }
1295 }
1296
cleanVector(PVector * vector)1297 static void cleanVector(PVector *vector) {
1298 // Cleaning the vector means that all dirty indications are cleared
1299 // and that the nodes that were dirty get a ref count of 1 since
1300 // they are brand new. Once cleaned the vector can be released into
1301 // the wild.
1302 if(IS_DIRTY(vector->tail)) {
1303 cleanNodeRecursively(vector->tail, 0);
1304 } else {
1305 INC_NODE_REF_COUNT(vector->tail);
1306 }
1307
1308 if(IS_DIRTY(vector->root)) {
1309 cleanNodeRecursively(vector->root, vector->shift);
1310 } else {
1311 INC_NODE_REF_COUNT(vector->root);
1312 }
1313 }
1314
PVectorEvolver_dealloc(PVectorEvolver * self)1315 static void PVectorEvolver_dealloc(PVectorEvolver *self) {
1316 PyObject_GC_UnTrack(self);
1317 Py_TRASHCAN_SAFE_BEGIN(self);
1318
1319 if(self->originalVector != self->newVector) {
1320 cleanVector(self->newVector);
1321 Py_DECREF(self->newVector);
1322 }
1323
1324 Py_DECREF(self->originalVector);
1325 Py_DECREF(self->appendList);
1326
1327 PyObject_GC_Del(self);
1328 Py_TRASHCAN_SAFE_END(self);
1329 }
1330
PVectorEvolver_append(PVectorEvolver * self,PyObject * args)1331 static PyObject *PVectorEvolver_append(PVectorEvolver *self, PyObject *args) {
1332 if (PyList_Append(self->appendList, args) == 0) {
1333 Py_INCREF(self);
1334 return (PyObject*)self;
1335 }
1336
1337 return NULL;
1338 }
1339
PVectorEvolver_extend(PVectorEvolver * self,PyObject * args)1340 static PyObject *PVectorEvolver_extend(PVectorEvolver *self, PyObject *args) {
1341 PyObject *retVal = _PyList_Extend((PyListObject *)self->appendList, args);
1342 if (retVal == NULL) {
1343 return NULL;
1344 }
1345
1346 Py_DECREF(retVal);
1347 Py_INCREF(self);
1348 return (PyObject*)self;
1349 }
1350
PVectorEvolver_subscript(PVectorEvolver * self,PyObject * item)1351 static PyObject *PVectorEvolver_subscript(PVectorEvolver *self, PyObject *item) {
1352 if (PyIndex_Check(item)) {
1353 Py_ssize_t position = PyNumber_AsSsize_t(item, PyExc_IndexError);
1354 if (position == -1 && PyErr_Occurred()) {
1355 return NULL;
1356 }
1357
1358 if (position < 0) {
1359 position += self->newVector->count + PyList_GET_SIZE(self->appendList);
1360 }
1361
1362 if(0 <= position && position < self->newVector->count) {
1363 PyObject *result = _get_item(self->newVector, position);
1364 Py_XINCREF(result);
1365 return result;
1366 } else if (0 <= position && position < (self->newVector->count + PyList_GET_SIZE(self->appendList))) {
1367 PyObject *result = PyList_GetItem(self->appendList, position - self->newVector->count);
1368 Py_INCREF(result);
1369 return result;
1370 } else {
1371 PyErr_SetString(PyExc_IndexError, "Index out of range");
1372 }
1373 } else {
1374 PyErr_Format(PyExc_TypeError, "Indices must be integers, not %.200s", item->ob_type->tp_name);
1375 }
1376
1377 return NULL;
1378 }
1379
doSetWithDirty(VNode * node,unsigned int level,unsigned int position,PyObject * value)1380 static VNode* doSetWithDirty(VNode* node, unsigned int level, unsigned int position, PyObject* value) {
1381 VNode* resultNode;
1382 debug("doSetWithDirty(): level == %i\n", level);
1383 if(level == 0) {
1384 if(!IS_DIRTY(node)) {
1385 resultNode = allocNode();
1386 copyInsert(resultNode->items, node->items, position & BIT_MASK, value);
1387 incRefs((PyObject**)resultNode->items);
1388 SET_DIRTY(resultNode);
1389 } else {
1390 resultNode = node;
1391 Py_INCREF(value);
1392 Py_DECREF(resultNode->items[position & BIT_MASK]);
1393 resultNode->items[position & BIT_MASK] = value;
1394 }
1395 } else {
1396 if(!IS_DIRTY(node)) {
1397 resultNode = copyNode(node);
1398 SET_DIRTY(resultNode);
1399 } else {
1400 resultNode = node;
1401 }
1402
1403 Py_ssize_t index = (position >> level) & BIT_MASK;
1404 VNode* oldNode = (VNode*)resultNode->items[index];
1405 resultNode->items[index] = doSetWithDirty(resultNode->items[index], level - SHIFT, position, value);
1406
1407 if(resultNode->items[index] != oldNode) {
1408 // Node replaced, drop references to old node
1409 DEC_NODE_REF_COUNT(oldNode);
1410 }
1411 }
1412
1413 return resultNode;
1414 }
1415
1416 /*
1417 Steals a reference to the object that is inserted in the vector.
1418 */
PVectorEvolver_set(PVectorEvolver * self,PyObject * args)1419 static PyObject *PVectorEvolver_set(PVectorEvolver *self, PyObject *args) {
1420 PyObject *argObj = NULL; /* argument to insert */
1421 PyObject *position = NULL;
1422
1423 /* The n parses for size, the O parses for a Python object */
1424 if(!PyArg_ParseTuple(args, "OO", &position, &argObj)) {
1425 return NULL;
1426 }
1427
1428 if(PVectorEvolver_set_item(self, position, argObj) < 0) {
1429 return NULL;
1430 }
1431
1432 Py_INCREF(self);
1433 return (PyObject*)self;
1434 }
1435
PVectorEvolver_delete(PVectorEvolver * self,PyObject * args)1436 static PyObject *PVectorEvolver_delete(PVectorEvolver *self, PyObject *args) {
1437 PyObject *position = NULL;
1438
1439 /* The n parses for size, the O parses for a Python object */
1440 if(!PyArg_ParseTuple(args, "O", &position)) {
1441 return NULL;
1442 }
1443
1444 if(PVectorEvolver_set_item(self, position, NULL) < 0) {
1445 return NULL;
1446 }
1447
1448 Py_INCREF(self);
1449 return (PyObject*)self;
1450 }
1451
1452
internalPVectorDelete(PVectorEvolver * self,Py_ssize_t position)1453 static int internalPVectorDelete(PVectorEvolver *self, Py_ssize_t position) {
1454 // Delete element. Should be unusual. Simple but expensive operation
1455 // that reuses the delete code for the vector. Realize the vector, delete on it and
1456 // then reset the evolver to work on the new vector.
1457 PVector *temp = (PVector*)PVectorEvolver_persistent(self);
1458 PVector *temp2 = (PVector*)internalDelete(temp, position, NULL);
1459 Py_DECREF(temp);
1460
1461 if(temp2 == NULL) {
1462 return -1;
1463 }
1464
1465 Py_DECREF(self->originalVector);
1466 self->originalVector = temp2;
1467 self->newVector = self->originalVector;
1468 return 0;
1469 }
1470
PVectorEvolver_set_item(PVectorEvolver * self,PyObject * item,PyObject * value)1471 static int PVectorEvolver_set_item(PVectorEvolver *self, PyObject* item, PyObject* value) {
1472 if (PyIndex_Check(item)) {
1473 Py_ssize_t position = PyNumber_AsSsize_t(item, PyExc_IndexError);
1474 if (position == -1 && PyErr_Occurred()) {
1475 return -1;
1476 }
1477
1478 if (position < 0) {
1479 position += self->newVector->count + PyList_GET_SIZE(self->appendList);
1480 }
1481
1482 if((0 <= position) && (position < self->newVector->count)) {
1483 if(self->originalVector == self->newVector) {
1484 // Create new vector since we're about to modify the original
1485 self->newVector = rawCopyPVector(self->originalVector);
1486 }
1487
1488 if(value != NULL) {
1489 if(position < TAIL_OFF(self->newVector)) {
1490 self->newVector->root = doSetWithDirty(self->newVector->root, self->newVector->shift, position, value);
1491 } else {
1492 self->newVector->tail = doSetWithDirty(self->newVector->tail, 0, position, value);
1493 }
1494
1495 return 0;
1496 }
1497
1498 return internalPVectorDelete(self, position);
1499 } else if((0 <= position) && (position < (self->newVector->count + PyList_GET_SIZE(self->appendList)))) {
1500 if (value != NULL) {
1501 int result = PyList_SetItem(self->appendList, position - self->newVector->count, value);
1502 if(result == 0) {
1503 Py_INCREF(value);
1504 }
1505 return result;
1506 }
1507
1508 return internalPVectorDelete(self, position);
1509 } else if((0 <= position)
1510 && (position < (self->newVector->count + PyList_GET_SIZE(self->appendList) + 1))
1511 && (value != NULL)) {
1512 return PyList_Append(self->appendList, value);
1513 } else {
1514 PyErr_Format(PyExc_IndexError, "Index out of range: %zd", position);
1515 }
1516 } else {
1517 PyErr_Format(PyExc_TypeError, "Indices must be integers, not %.200s", item->ob_type->tp_name);
1518 }
1519 return -1;
1520 }
1521
PVectorEvolver_persistent(PVectorEvolver * self)1522 static PyObject *PVectorEvolver_persistent(PVectorEvolver *self) {
1523 PVector *resultVector;
1524 if(self->newVector != self->originalVector) {
1525 cleanVector(self->newVector);
1526 Py_DECREF(self->originalVector);
1527 }
1528
1529 resultVector = self->newVector;
1530
1531 if(PyList_GET_SIZE(self->appendList)) {
1532 PVector *oldVector = resultVector;
1533 resultVector = (PVector*)PVector_extend(resultVector, self->appendList);
1534 Py_DECREF(oldVector);
1535 Py_DECREF(self->appendList);
1536 self->appendList = NULL;
1537 }
1538
1539 initializeEvolver(self, resultVector, self->appendList);
1540 Py_INCREF(resultVector);
1541 return (PyObject*)resultVector;
1542 }
1543
PVectorEvolver_len(PVectorEvolver * self)1544 static Py_ssize_t PVectorEvolver_len(PVectorEvolver *self) {
1545 return self->newVector->count + PyList_GET_SIZE(self->appendList);
1546 }
1547
PVectorEvolver_is_dirty(PVectorEvolver * self)1548 static PyObject* PVectorEvolver_is_dirty(PVectorEvolver *self) {
1549 if((self->newVector != self->originalVector) || (PyList_GET_SIZE(self->appendList) > 0)) {
1550 Py_INCREF(Py_True);
1551 return Py_True;
1552 }
1553
1554 Py_INCREF(Py_False);
1555 return Py_False;
1556 }
1557
PVectorEvolver_traverse(PVectorEvolver * self,visitproc visit,void * arg)1558 static int PVectorEvolver_traverse(PVectorEvolver *self, visitproc visit, void *arg) {
1559 Py_VISIT(self->newVector);
1560 if (self->newVector != self->originalVector) {
1561 Py_VISIT(self->originalVector);
1562 }
1563 Py_VISIT(self->appendList);
1564 return 0;
1565 }
1566
1567 static PyMethodDef PyrsistentMethods[] = {
1568 {"pvector", pyrsistent_pvec, METH_VARARGS,
1569 "pvector([iterable])\n"
1570 "Create a new persistent vector containing the elements in iterable.\n\n"
1571 ">>> v1 = pvector([1, 2, 3])\n"
1572 ">>> v1\n"
1573 "pvector([1, 2, 3])"},
1574 {NULL, NULL, 0, NULL}
1575 };
1576
1577
1578 /********************* Python module initialization ************************/
1579
1580 #if PY_MAJOR_VERSION >= 3
1581 static struct PyModuleDef moduledef = {
1582 PyModuleDef_HEAD_INIT,
1583 "pvectorc", /* m_name */
1584 "Persistent vector", /* m_doc */
1585 -1, /* m_size */
1586 PyrsistentMethods, /* m_methods */
1587 NULL, /* m_reload */
1588 NULL, /* m_traverse */
1589 NULL, /* m_clear */
1590 NULL, /* m_free */
1591 };
1592 #endif
1593
pyrsistent_pvectorc_moduleinit(void)1594 static PyObject* pyrsistent_pvectorc_moduleinit(void) {
1595 PyObject* m;
1596
1597 // Only allow creation/initialization through factory method pvec
1598 PVectorType.tp_init = NULL;
1599 PVectorType.tp_new = NULL;
1600
1601 if (PyType_Ready(&PVectorType) < 0) {
1602 return NULL;
1603 }
1604 if (PyType_Ready(&PVectorIterType) < 0) {
1605 return NULL;
1606 }
1607 if (PyType_Ready(&PVectorEvolverType) < 0) {
1608 return NULL;
1609 }
1610
1611
1612 #if PY_MAJOR_VERSION >= 3
1613 m = PyModule_Create(&moduledef);
1614 #else
1615 m = Py_InitModule3("pvectorc", PyrsistentMethods, "Persistent vector");
1616 #endif
1617
1618 if (m == NULL) {
1619 return NULL;
1620 }
1621
1622 if(EMPTY_VECTOR == NULL) {
1623 EMPTY_VECTOR = emptyNewPvec();
1624 }
1625
1626 nodeCache.size = 0;
1627
1628 Py_INCREF(&PVectorType);
1629 PyModule_AddObject(m, "PVector", (PyObject *)&PVectorType);
1630
1631 return m;
1632 }
1633
1634 #if PY_MAJOR_VERSION >= 3
PyInit_pvectorc(void)1635 PyMODINIT_FUNC PyInit_pvectorc(void) {
1636 return pyrsistent_pvectorc_moduleinit();
1637 }
1638 #else
initpvectorc(void)1639 PyMODINIT_FUNC initpvectorc(void) {
1640 pyrsistent_pvectorc_moduleinit();
1641 }
1642 #endif
1643