1# Eclipse SUMO, Simulation of Urban MObility; see https://eclipse.org/sumo 2# Copyright (C) 2016-2019 German Aerospace Center (DLR) and others. 3# SUMOPy module 4# Copyright (C) 2012-2017 University of Bologna - DICAM 5# This program and the accompanying materials 6# are made available under the terms of the Eclipse Public License v2.0 7# which accompanies this distribution, and is available at 8# http://www.eclipse.org/legal/epl-v20.html 9# SPDX-License-Identifier: EPL-2.0 10 11# @file arrayman.py 12# @author Joerg Schweizer 13# @date 14# @version $Id$ 15 16from classman import * 17import numpy as np 18 19 20class ArrayConfMixin: 21 def __init__(self, attrname, default, dtype=None, is_index=False, **attrs): 22 self._is_index = is_index 23 self._dtype = dtype 24 AttrConf.__init__(self, attrname, default, 25 struct='array', 26 **attrs) 27 28 if is_index: 29 self._init_indexing() 30 31 def get_dtype(self): 32 return self._dtype 33 34 def convert_type(self, array): 35 return np.array(array, dtype=self._dtype) 36 37 def get_defaults(self, ids): 38 # create a list, should work for all types and dimensions 39 # default can be scalar or an array of any dimension 40 # print '\n\nget_defaults',self._name,ids 41 default = self.get_default() 42 43 if hasattr(default, '__iter__'): 44 default = np.asarray(default) 45 if self._dtype is not None: 46 dtype = self._dtype 47 else: 48 dtype = type(default.flatten()[0]) 49 # print ' default=',default,len(default) 50 if len(ids) > 0: 51 defaults = np.array(len(ids)*[default], dtype) 52 # print ' size,type',len(ids)*[default], type(default.flatten()[0]) 53 else: 54 #defaults = np.zeros( (0,len(default)),type(default.flatten()[0]) ) 55 defaults = np.zeros((0,)+default.shape, dtype) 56 # print ' return',defaults,defaults.shape,defaults.dtype 57 return defaults 58 else: 59 if self._dtype is not None: 60 dtype = self._dtype 61 else: 62 dtype = type(default) 63 #defaults= np.array( len(ids)*[default], dtype ) 64 # print ' return 1D',defaults,defaults.shape,defaults.dtype 65 return np.array(len(ids)*[default], dtype) 66 67 def get_init(self): 68 """ 69 Returns initialization of attribute. 70 Usually same as get_default for scalars. 71 Overridden by table configuration classes 72 """ 73 ids = self._manager.get_ids() 74 75 # print '\n\nget_init',self.attrname,ids,self._is_localvalue 76 values = self.get_defaults(ids) 77 78 # store locally if required 79 if self._is_localvalue: 80 self.value = values 81 # pass on to calling instance 82 # in this cas the data is stored under self._obj 83 return values 84 85 def get_ids_sorted(self): 86 inds = self._manager.get_inds() 87 return self._manager.get_ids(inds[np.argsort(self.get_value()[inds])]) 88 89 #sortarray = np.concatenate((self.get_value()[inds],inds)) 90 91 def delete_ind(self, i): 92 # called from del_rows 93 if self._is_index: 94 _id = self._manager._ids[i] 95 self.del_index(_id) 96 arr = self.get_value() 97 self.set_value(np.concatenate((arr[:i], arr[i+1:]))) 98 99 def __delitem__(self, ids): 100 # print ' before=\n',self.__dict__[attr] 101 #attr = self.attrconf.get_attr() 102 if hasattr(ids, '__iter__'): 103 for i in self._manager._inds[ids]: 104 self.delete_ind[i] 105 else: 106 self.delete_ind(self._manager._inds[ids]) 107 108 def __getitem__(self, ids): 109 # print '__getitem__',key 110 return self.get_value()[self._manager._inds[ids]] 111 112 def __setitem__(self, ids, values): 113 # print '__setitem__',ids,values,type(self.get_value()),self.get_value().dtype 114 115 if self._is_index: 116 if hasattr(ids, '__iter__'): 117 self.set_indices(ids, values) 118 119 else: 120 self.set_index(ids, values) 121 122 self.get_value()[self._manager._inds[ids]] = values 123 124 def set(self, ids, values): 125 if values is None: 126 return 127 128 if not hasattr(ids, '__iter__'): 129 _ids = [ids] 130 _values = np.array([values], self._dtype) 131 132 else: 133 _ids = ids 134 _values = np.array(values, self._dtype) 135 # print 'set', _ids ,_values 136 self[_ids] = _values 137 if self._is_index: 138 self.set_indices(_ids, _values) 139 self._is_modified = True 140 141 def set_plugin(self, ids, values): 142 if not hasattr(ids, '__iter__'): 143 _ids = [ids] 144 _values = np.array([values], self._dtype) 145 146 else: 147 _ids = ids 148 _values = np.array(values, self._dtype) 149 150 self[ids] = _values 151 if self._is_index: 152 self.set_indices(_ids, _values) 153 self._is_modified = True 154 self.plugin.exec_events_ids(EVTSETITEM, _ids) 155 156 def add(self, ids, values=None): 157 if not hasattr(ids, '__iter__'): 158 _ids = [ids] 159 if values is not None: 160 _values = np.array([values], self._dtype) 161 else: 162 _values = self.get_defaults(_ids) 163 164 else: 165 # if values is None: 166 # print 'WARNING:',self.attrname, ids,self._dtype 167 _ids = ids 168 if values is not None: 169 _values = np.array(values, self._dtype) 170 else: 171 _values = self.get_defaults(_ids) 172 # if values is None: 173 # _values = self.get_defaults(_ids) 174 175 # print 'add ids',self.attrname,ids,_ids,self._is_modified 176 # print ' values',values 177 # print ' _values',_values 178 # print ' self.get_value()',self.get_value() 179 # print ' type(_values),type(self.get_value())',type(_values),type(self.get_value()) 180 # print ' _values.shape,self.get_value().shape',_values.shape,self.get_value().shape 181 182 #newvalue = np.concatenate((self.get_value(),_values)) 183 # print ' ', type(newvalue),newvalue.dtype 184 self.set_value(np.concatenate((self.get_value(), _values))) 185 # print ' done:',self.attrname,self.get_value() 186 if self._is_index: 187 self.add_indices(_ids, _values) 188 self._is_modified = True 189 190 def add_plugin(self, ids, values=None): 191 # print 'add_plugin',self.attrname 192 if not hasattr(ids, '__iter__'): 193 _ids = [ids] 194 if values is not None: 195 _values = np.array([values], self._dtype) 196 197 else: 198 _ids = ids 199 if values is not None: 200 _values = np.array(values, self._dtype) 201 202 if values is None: 203 _values = self.get_defaults(_ids) 204 self._is_modified = True 205 # print 'add ids',self.attrname,ids,_ids,self._is_modified 206 # print ' values',values 207 # print ' _values',_values 208 # print ' self.get_value()',self.get_value() 209 # print ' type(_values),type(self.get_value())',type(_values),type(self.get_value()) 210 # print ' _values.shape,self.get_value().shape',_values.shape,self.get_value().shape 211 212 #newvalue = np.concatenate((self.get_value(),_values)) 213 # print ' ', type(newvalue),newvalue.dtype 214 self.set_value(np.concatenate((self.get_value(), _values))) 215 # print ' done:',self.attrname,self.get_value() 216 217 if self._is_index: 218 self.add_indices(_ids, _values) 219 220 self.plugin.exec_events_ids(EVTADDITEM, _ids) 221 222 # use original one from AttrConfig 223 # def _write_xml_value(self,val,fd): 224 # #print 'write_xml',self.xmltag,type(val),hasattr(val, '__iter__') 225 # if hasattr(val, '__iter__'): 226 # if len(val)>0: 227 # if hasattr(val[0], '__iter__'): 228 # # matrix 229 # fd.write(xm.mat(self.xmltag,val)) 230 # else: 231 # if type(val)==np.ndarray: 232 # # vector 233 # fd.write(xm.arr(self.xmltag,val,sep=',')) 234 # else: 235 # # list 236 # fd.write(xm.arr(self.xmltag,val)) 237 # else: 238 # # empty list 239 # fd.write(xm.arr(self.xmltag,val)) 240 # else: 241 # # scalar number or string 242 # fd.write(xm.num(self.xmltag,val)) 243 244 def format_value(self, _id, show_unit=False, show_parentesis=False): 245 if show_unit: 246 unit = ' '+self.format_unit(show_parentesis) 247 else: 248 unit = '' 249 # return repr(self[_id])+unit 250 251 #self.min = minval 252 #self.max = maxval 253 #self.digits_integer = digits_integer 254 #self.digits_fraction = digits_fraction 255 val = self[_id] 256 tt = type(val) 257 258 if tt in (np.int, np.int32, np.float64): 259 return str(val)+unit 260 261 elif tt in (np.float, np.float32, np.float64): 262 if hasattr(self, 'digits_fraction'): 263 digits_fraction = self.digits_fraction 264 else: 265 digits_fraction = 3 266 s = "%."+str(digits_fraction)+"f" 267 return s % (val)+unit 268 269 else: 270 return str(val)+unit 271 272 273class ArrayConf(ArrayConfMixin, ColConf): 274 """ 275 Column made of numeric array. 276 277 """ 278 pass 279 280 281class ListArrayConf(ArrayConfMixin, ColConf): 282 """ 283 Column made of an array of lists. 284 285 """ 286 287 def __init__(self, attrname, dtype=None, **attrs): 288 ArrayConfMixin.__init__(self, attrname, None, dtype=np.object, **attrs) 289 290 def add(self, ids, values=None): 291 if not hasattr(ids, '__iter__'): 292 _ids = [ids] 293 if values is not None: 294 _values = np.zeros(1, self._dtype) 295 _values[0] = values 296 297 else: 298 299 _ids = ids 300 if values is not None: 301 _values = np.zeros(len(ids), self._dtype) 302 _values[:] = values 303 304 if values is None: 305 _values = self.get_defaults(_ids) 306 307 # print 'add ids, _values',self.attrname,ids 308 # print ' values',values 309 # print ' _values',_values 310 # print ' self.get_value()',self.get_value() 311 # print ' type(_values),type(self.get_value())',type(_values),type(self.get_value()) 312 # print ' _values.shape,self.get_value().shape',_values.shape,self.get_value().shape 313 314 newvalue = np.concatenate((self.get_value(), _values)) 315 # print ' ', type(newvalue),newvalue.dtype 316 self.set_value(np.concatenate((self.get_value(), _values))) 317 318 if self._is_index: 319 self.add_indices(_ids, _values) 320 321 def add_plugin(self, ids, values=None): 322 # print 'add_plugin',self.attrname,ids 323 if not hasattr(ids, '__iter__'): 324 _ids = [ids] 325 if values is not None: 326 _values = np.zeros(1, self._dtype) 327 _values[0] = values 328 329 else: 330 331 _ids = ids 332 if values is not None: 333 _values = np.zeros(len(ids), self._dtype) 334 _values[:] = values 335 336 if values is None: 337 _values = self.get_defaults(_ids) 338 339 # print 'add ids, _values',self.attrname,ids 340 # print ' values',values 341 # print ' _values',_values 342 # print ' self.get_value()',self.get_value() 343 # print ' type(_values),type(self.get_value())',type(_values),type(self.get_value()) 344 # print ' _values.shape,self.get_value().shape',_values.shape,self.get_value().shape 345 346 newvalue = np.concatenate((self.get_value(), _values)) 347 # print ' ', type(newvalue),newvalue.dtype 348 self.set_value(np.concatenate((self.get_value(), _values))) 349 350 if self._is_index: 351 self.add_indices(_ids, _values) 352 353 if self.plugin: 354 self.plugin.exec_events_ids(EVTADDITEM, _ids) 355 356 357class NumArrayConf(ArrayConfMixin, ColConf): 358 """ 359 Column made of numeric array. 360 361 """ 362 # def __init__(self, **attrs): 363 # print 'ColConf',attrs 364 365 def __init__(self, attrname, default, 366 digits_integer=None, digits_fraction=None, 367 minval=None, maxval=None, 368 **attrs): 369 370 self.min = minval 371 self.max = maxval 372 self.digits_integer = digits_integer 373 self.digits_fraction = digits_fraction 374 375 ArrayConfMixin.__init__(self, attrname, default, metatype='number', **attrs) 376 377 378class IdsArrayConf(ArrayConfMixin, ColConf): 379 """ 380 Column, where each entry is the id of a single Table. 381 """ 382 383 def __init__(self, attrname, tab, is_index=False, id_default=-1, perm='r', **kwargs): 384 self._tab = tab 385 ArrayConfMixin.__init__(self, attrname, 386 id_default, # default id 387 dtype=np.int32, 388 metatype='id', 389 perm=perm, 390 is_index=is_index, 391 **kwargs 392 ) 393 self.init_xml() 394 # print 'IdsConf.__init__',attrname 395 # print ' ',self._tab.xmltag,self._attrconfig_id_tab 396 397 def get_defaults(self, ids): 398 # create a list, should work for all types and dimensions 399 # default can be scalar or an array of any dimension 400 # print '\n\nget_defaults',self.attrname,ids,self.get_default() 401 return self.get_default()*np.ones(len(ids), dtype=self._dtype) 402 403 404# ------------------------------------------------------------------------------- 405 # copied from IdsConf!!! 406 def set_linktab(self, tab): 407 self._tab = tab 408 409 def get_linktab(self): 410 return self._tab 411 412 def init_xml(self): 413 # print 'init_xml',self.attrname,self._tab 414 if self._tab.xmltag is not None: 415 xmltag_tab, xmltag_item_tab, attrname_id_tab = self._tab.xmltag 416 if (attrname_id_tab is None) | (attrname_id_tab is ''): 417 self._attrconfig_id_tab = None 418 else: 419 self._attrconfig_id_tab = getattr(self._tab, attrname_id_tab) # tab = tabman ! 420 421 if not hasattr(self, 'is_xml_include_tab'): 422 # this means that entire table rows will be included 423 self.is_xml_include_tab = False 424 # print ' xmltag_tab, xmltag_item_tab, attrname_id_tab',xmltag_tab, xmltag_item_tab, attrname_id_tab,self.is_xml_include_tab 425 426 else: 427 self._attrconfig_id_tab = None 428 self.is_xml_include_tab = False 429 430 def write_xml(self, fd, _id, indent=0): 431 # print 'write_xml',self.attrname 432 if (self.xmltag is not None) & (np.all(self[_id] >= 0)): 433 # print 'write_xml',self.attrname, _id,'value',self[_id] 434 if self._attrconfig_id_tab is None: 435 self._write_xml_value(self[_id], fd) 436 elif self.is_xml_include_tab: 437 # print ' write table row(s)',self[_id] 438 self._tab.write_xml(fd, indent, ids=self[_id], 439 is_print_begin_end=False) 440 else: 441 # print ' write id(s)',self[_id] 442 self._write_xml_value(self._attrconfig_id_tab[self[_id]], fd) 443 444 def _write_xml_value(self, val, fd): 445 # print 'write_xml',self.xmltag,hasattr(val, '__iter__') 446 if hasattr(val, '__iter__'): 447 if len(val) > 0: 448 if hasattr(val[0], '__iter__'): 449 # matrix 450 fd.write(xm.mat(self.xmltag, val)) 451 else: 452 # list 453 fd.write(xm.arr(self.xmltag, val, self.xmlsep)) 454 else: 455 # empty list 456 # fd.write(xm.arr(self.xmltag,val)) 457 # don't even write empty lists 458 pass 459 460 elif type(self._default) in (types.UnicodeType, types.StringType): 461 if len(val) > 0: 462 fd.write(xm.num(self.xmltag, val)) 463 464 else: 465 # scalar number or string 466 fd.write(xm.num(self.xmltag, val)) 467 468 def _getstate_specific(self, state): 469 """ 470 Called by __getstate__ to add/change specific states, 471 before returning states. 472 To be overridden. 473 """ 474 if self._is_save: 475 # if self._is_child: 476 # # OK self.value already set in 477 # pass 478 # else: 479 # # remove table reference and create ident 480 # print '_getstate_specific',self.attrname 481 # print ' self._tab',self._tab 482 # print '_getstate_specific',self._tab.ident, self._tab.get_ident_abs() 483 state['_tab'] = None 484 # try: 485 state['_ident_tab'] = self._tab.get_ident_abs() 486 # except: 487 # print 'WARNING:_getstate_specific',self._tab,self._tab.attrname 488 489 def init_postload_internal(self, man, obj): 490 # print 'IdsConf.init_postload_internal',self.attrname,hasattr(self,'value'),self._is_save,self._is_localvalue,'obj:',obj.ident 491 492 AttrConf.init_postload_internal(self, man, obj) 493 # print 'IdsConf.init_postload_internal',self.attrname,self.get_value().dtype,self.get_value().dtype == np.int64 494 if self.get_value().dtype == np.int64: 495 print 'WARNING in init_postload_internal: convert ids array to 32 bit' 496 self.set_value(np.array(self.get_value(), dtype=np.int32)) 497 # if self._is_child: 498 # print ' make sure children get initialized' 499 # print ' call init_postload_internal of',self._tab.ident 500 # self._tab.init_postload_internal(obj) 501 502 def init_postload_external(self): 503 # if self._is_child: 504 # # restore normally 505 # AttrConf.init_postload_external(self) 506 # self._tab.init_postload_external() 507 # else: 508 509 # Substitute absolute ident with link object. 510 # Called from init_postload_external of attrsman during load_obj 511 # 512 ident_abs = self._ident_tab 513 # print 'init_postload_external',self.attrname,ident_abs 514 obj = self.get_obj() 515 rootobj = obj.get_root() 516 # print ' obj,rootobj',obj,rootobj 517 linkobj = rootobj.get_obj_from_ident(ident_abs) 518 # print ' linkobj',linkobj.ident 519 self._tab = linkobj 520 self.init_xml() 521 522 def is_modified(self): 523 return False 524 525 def set_modified(self, is_modified): 526 pass 527 528 529class IdlistsArrayConf(IdsArrayConf): 530 """ 531 Column, where each entry is a list of ids of a single Table. 532 """ 533 534 def __init__(self, attrname, tab, metatype=None, perm='r', **kwargs): 535 self._is_index = False 536 self._tab = tab 537 ArrayConfMixin.__init__(self, attrname, 538 None, # default, will be substituted by id list 539 dtype='object', 540 metatype='ids', 541 perm=perm, 542 **kwargs 543 ) 544 self.init_xml() 545 546 def get_defaults(self, ids): 547 # here we initialize with None for reach element 548 return np.array(len(ids)*[None, ], self._dtype) 549 550 def add(self, ids, values=None): 551 if not hasattr(ids, '__iter__'): 552 _ids = [ids] 553 if values is not None: 554 _values = np.zeros(1, self._dtype) 555 _values[0] = values 556 557 else: 558 559 _ids = ids 560 _values = np.zeros(len(ids), self._dtype) 561 _values[:] = values 562 563 if values is None: 564 _values = self.get_defaults(_ids) 565 566 # print 'add ids, _values',self.attrname,ids 567 # print ' values',values 568 # print ' _values',_values 569 # print ' self.get_value()',self.get_value() 570 # print ' type(_values),type(self.get_value())',type(_values),type(self.get_value()) 571 # print ' _values.shape,self.get_value().shape',_values.shape,self.get_value().shape 572 573 newvalue = np.concatenate((self.get_value(), _values)) 574 # print ' ', type(newvalue),newvalue.dtype 575 self.set_value(np.concatenate((self.get_value(), _values))) 576 577 if self._is_index: 578 self.add_indices(_ids, _values) 579 580 581class TabIdListArrayConf(ArrayConfMixin, ColConf): 582 """ 583 Column made of an array of lists with (table,id) tupels. 584 The tables are linked, and will not be saved. 585 """ 586 587 def __init__(self, attrname, dtype=None, perm='r', **attrs): 588 ArrayConfMixin.__init__(self, attrname, None, # default, will be substituted by (table,id) list 589 dtype='object', 590 metatype='tabidlist', 591 perm=perm, **attrs) 592 593 def add(self, ids, values=None): 594 if not hasattr(ids, '__iter__'): 595 _ids = [ids] 596 if values is not None: 597 _values = np.zeros(1, self._dtype) 598 _values[0] = values 599 600 else: 601 602 _ids = ids 603 _values = np.zeros(len(ids), self._dtype) 604 _values[:] = values 605 606 if values is None: 607 _values = self.get_defaults(_ids) 608 609 # print 'add ids, _values',self.attrname,ids 610 # print ' values',values 611 # print ' _values',_values 612 # print ' self.get_value()',self.get_value() 613 # print ' type(_values),type(self.get_value())',type(_values),type(self.get_value()) 614 # print ' _values.shape,self.get_value().shape',_values.shape,self.get_value().shape 615 616 newvalue = np.concatenate((self.get_value(), _values)) 617 # print ' ', type(newvalue),newvalue.dtype 618 self.set_value(np.concatenate((self.get_value(), _values))) 619 620 if self._is_index: 621 self.add_indices(_ids, _values) 622 623 def add_plugin(self, ids, values=None): 624 if not hasattr(ids, '__iter__'): 625 _ids = [ids] 626 if values is not None: 627 _values = np.zeros(1, self._dtype) 628 _values[0] = values 629 630 else: 631 632 _ids = ids 633 _values = np.zeros(len(ids), self._dtype) 634 _values[:] = values 635 636 if values is None: 637 _values = self.get_defaults(_ids) 638 639 ### 640 641 def format_value(self, _id, show_unit=False, show_parentesis=False): 642 s = '' 643 rowlist = self[_id] 644 if rowlist is None: 645 return s 646 # elif (type(rowlist)in STRINGTYPES): 647 # return rowlist 648 elif len(rowlist) == 0: 649 return s 650 elif len(rowlist) == 1: 651 tab, ids = rowlist[0] 652 return str(tab)+'['+str(ids)+']' 653 elif len(rowlist) > 1: 654 tab, ids = rowlist[0] 655 s = str(tab)+'['+str(ids)+']' 656 for tab, ids in rowlist[1:]: 657 s += ','+str(tab)+'['+str(ids)+']' 658 return s 659 660 def _getstate_specific(self, state): 661 """ 662 Called by __getstate__ to add/change specific states, 663 before returning states. 664 To be overridden. 665 """ 666 # print '_getstate_specific',self.attrname, self._is_save 667 # print ' self.get_value',self.get_value() 668 # print len(self.get_value()) 669 if self._is_save: 670 n = len(state['value']) 671 state['value'] = None 672 _tabidlists_save = n*[None] 673 i = 0 674 for rowlist in self.get_value(): 675 if rowlist is not None: 676 rowlist_save = [] 677 for tab, ids in rowlist: 678 rowlist_save.append([tab.get_ident_abs(), ids]) 679 # print ' tab.get_ident'.get_ident() 680 # print ' appended',[tab.get_ident_abs(), ids] 681 _tabidlists_save[i] = rowlist_save 682 # print ' ',i,rowlist_save 683 i += 1 684 state['_tabidlists_save'] = _tabidlists_save 685 686 def init_postload_external(self): 687 # Substitute absolute ident with link object. 688 # Called from init_postload_external of attrsman during load_obj 689 # 690 # print 'init_postload_external',self.attrname, len(self._tabidlists_save) 691 #obj = self.get_obj() 692 #rootobj = obj.get_root() 693 # print ' rootobj',rootobj.ident 694 #linkobj = rootobj.get_obj_from_ident(ident_abs) 695 # print ' linkobj',linkobj.ident 696 #self._tab = linkobj 697 698 # Substitute absolute ident with link object. 699 # Called from init_postload_external of attrsman during load_obj 700 # 701 _tabidlists_save = self._tabidlists_save 702 #ident_abs = self._ident_value 703 # print 'init_postload_external',self.attrname,_tabids_save 704 obj = self.get_obj() 705 rootobj = obj.get_root() 706 # print ' rootobj',rootobj.ident 707 tabidlists = np.zeros(len(_tabidlists_save), dtype=self._dtype) 708 709 i = 0 710 for rowlist_save in _tabidlists_save: 711 rowlist = [] 712 # print ' rowlist_save',rowlist_save 713 if rowlist_save is not None: 714 for tabident, ids in rowlist_save: 715 tab = rootobj.get_obj_from_ident(tabident) 716 # print ' recovered tab',tab.get_ident_abs(), ids 717 rowlist.append([tab, ids]) 718 tabidlists[i] = rowlist 719 else: 720 tabidlists[i] = None 721 i += 1 722 # print ' tabidlists', tabidlists 723 self.set_value(tabidlists) 724 725 def is_modified(self): 726 return False 727 728 def set_modified(self, is_modified): 729 pass 730 731 732class TabIdsArrayConf(ArrayConfMixin, ColConf): 733 """ 734 Column, where each entry contains a tuple with table object and id. 735 The tables are linked, and will not be saved. 736 """ 737 738 def __init__(self, attrname, is_index=False, perm='r', **kwargs): 739 self._is_index = is_index 740 ArrayConfMixin.__init__(self, attrname, 741 (None, -1), # default id 742 dtype=[('ob', object), ('id', np.int)], 743 metatype='tabid', 744 perm=perm, 745 **kwargs 746 ) 747 748 def get_defaults(self, ids): 749 # create a list, should work for all types and dimensions 750 # default can be scalar or an array of any dimension 751 # print '\n\nget_defaults',self.attrname,ids,self.get_default() 752 return np.zeros(len(ids), dtype=self._dtype) 753 754 def _getstate_specific(self, state): 755 """ 756 Called by __getstate__ to add/change specific states, 757 before returning states. 758 To be overridden. 759 """ 760 if self._is_save: 761 n = len(state['value']) 762 state['value'] = None 763 _tabids_save = n*[None] 764 i = 0 765 for tab, ids in self.get_value(): 766 _tabids_save[i] = [tab.get_ident_abs(), ids] 767 i += 1 768 state['_tabids_save'] = _tabids_save 769 770 def init_postload_external(self): 771 # if self._is_child: 772 # # restore normally 773 # AttrConf.init_postload_external(self) 774 # self._tab.init_postload_external() 775 # else: 776 777 # Substitute absolute ident with link object. 778 # Called from init_postload_external of attrsman during load_obj 779 # 780 #ident_abs = self._ident_tab 781 # print 'reset_linkobj',self.attrname,ident_abs 782 #obj = self.get_obj() 783 #rootobj = obj.get_root() 784 # print ' rootobj',rootobj.ident 785 #linkobj = rootobj.get_obj_from_ident(ident_abs) 786 # print ' linkobj',linkobj.ident 787 #self._tab = linkobj 788 789 # Substitute absolute ident with link object. 790 # Called from init_postload_external of attrsman during load_obj 791 # 792 _tabids_save = self._tabids_save 793 #ident_abs = self._ident_value 794 # print 'init_postload_external',self.attrname,_tabids_save 795 obj = self.get_obj() 796 rootobj = obj.get_root() 797 # print ' rootobj',rootobj.ident 798 tabids = np.zeros(len(self._tabids_save), dtype=self._dtype) 799 800 i = 0 801 for tabident, ids in self._tabids_save: 802 tab = rootobj.get_obj_from_ident(tabident) 803 # print ' ',tab.get_ident_abs(), ids 804 tabids[i] = (tab, ids) 805 i += 1 806 807 self.set_value(tabids) 808 809 def is_modified(self): 810 return False 811 812 def set_modified(self, is_modified): 813 pass 814 815 816class Arrayman(Tabman): 817 """ 818 Manages all table attributes of an object. 819 820 if argument obj is specified with an instance 821 then column attributes are stored under this instance. 822 The values of attrname is then directly accessible with 823 824 obj.attrname 825 826 If nothing is specified, then column attribute will be stored under 827 the respective config instance of this tab man (self). 828 The values of attrname is then directly accessible with 829 830 self.attrname.value 831 832 """ 833 834 def __init__(self, obj=None, **kwargs): 835 836 Attrsman.__init__(self, obj, **kwargs) 837 self._colconfigs = [] 838 self._inds = np.zeros((0,), dtype=np.int32) 839 self._ids = np.zeros((0,), dtype=np.int32) 840 841 def get_inds(self, ids=None): 842 if ids is not None: 843 return self._inds[ids] 844 else: 845 return self._inds[self._ids] 846 847 def get_ids(self, inds=None): 848 if inds is not None: 849 return self._ids[inds] 850 else: 851 return self._ids 852 853 return 854 855 def get_ind(self, id): 856 return self._inds[id] 857 858 def select_ids(self, mask): 859 # print 'select_ids' 860 # print ' mask\n=',mask 861 # print ' self._ids\n=',self._ids 862 # if len(self)>0: 863 return np.take(self._ids, np.flatnonzero(mask)) 864 # else: 865 # return np.zeros(0,int) 866 867 def suggest_id(self, is_zeroid=False): 868 """ 869 Returns a an availlable id. 870 871 Options: 872 is_zeroid=True allows id to be zero. 873 874 """ 875 return self.suggest_ids(1, is_zeroid)[0] 876 877 def format_id(self, id): 878 return self.format_ids([id]) 879 880 def format_ids(self, ids): 881 return ', '.join(np.array(ids, dtype='|S24')) 882 883 def get_id_from_formatted(self, idstr): 884 return int(idstr) 885 886 def get_ids_from_formatted(self, idstrs): 887 idstrs_arr = idstrs.split(',') 888 ids = np.zeros(len(idstrs_arr), dtype=np.float32) 889 i = 0 890 for idstr in idstrs_arr: 891 ids[i] = int[idstr] 892 893 return ids 894 895 def suggest_ids(self, n, is_zeroid=False): 896 """ 897 Returns a list of n availlable ids. 898 It returns even a list for n=1. 899 900 Options: 901 is_zeroid=True allows id to be zero. 902 """ 903 # TODO: does always return 1 if is_index is True ????? 904 # print 'suggest_ids',n,is_zeroid,self._inds,len(self._inds),self._inds.dtype 905 ids_unused_orig = np.flatnonzero(np.less(self._inds, 0)) 906 907 if not is_zeroid: 908 if len(self._inds) == 0: 909 ids_unused = np.zeros(0, dtype=np.int32) 910 else: 911 # avoid 0 as id: 912 # ids_unused=take(ids_unused,flatnonzero(greater(ids_unused,0))) 913 # print ' ids_unused_orig',ids_unused_orig,type(ids_unused_orig) 914 # print ' len(ids_unused_orig)',len(ids_unused_orig),ids_unused_orig.shape 915 # print ' greater(ids_unused_orig,0)',greater(ids_unused_orig,0) 916 # print ' len(greater(ids_unused_orig,0))',len(greater(ids_unused_orig,0)) 917 # print ' flatnonzero(greater(ids_unused_orig,0))',flatnonzero(greater(ids_unused_orig,0)) 918 # print ' len(flatnonzero(greater(ids_unused_orig,0)))=',len(flatnonzero(greater(ids_unused_orig,0)) ) 919 ids_unused = ids_unused_orig[np.flatnonzero(np.greater(ids_unused_orig, 0))] 920 zid = 1 921 else: 922 if len(self._inds) == 0: 923 ids_unused = np.zeros(0, dtype=np.int32) 924 else: 925 ids_unused = ids_unused_orig.copy() 926 927 zid = 0 928 929 n_unused = len(ids_unused) 930 n_max = len(self._inds)-1 931 # print ' ids_unused',ids_unused 932 # print ' ids_unused.shape',ids_unused.shape 933 # print ' len(ids_unused)',len(ids_unused) 934 # print ' n_unused,n_max,zid=',n_unused,n_max,zid 935 936 if n_max < zid: 937 # first id generation 938 ids = np.arange(zid, n+zid, dtype=np.int32) 939 940 elif n_unused > 0: 941 if n_unused >= n: 942 ids = ids_unused[:n] 943 else: 944 # print ' ids_unused',ids_unused 945 # print ' from to',n_max+1,n_max+1+n-n_unused 946 # print ' arange=',arange(n_max+1,n_max+1+n-n_unused) 947 # print ' type(ids_unused)',type(ids_unused) 948 # print ' dtype(ids_unused)',ids_unused.dtype 949 ids = np.concatenate((ids_unused, np.arange(n_max+1, n_max+1+n-n_unused))) 950 951 else: 952 ids = np.arange(n_max+1, n_max+1+n, dtype=np.int32) 953 954 return ids 955 956 def _add_ids(self, ids): 957 n = len(ids) 958 if n == 0: 959 return 960 961 id_max = max(ids) 962 id_max_old = len(self._inds)-1 963 n_array_old = len(self) 964 965 ids_existing = np.take(ids, np.flatnonzero(np.less(ids, id_max_old))) 966 # print ' ids',ids,'id_max_old',id_max_old,'ids_existing',ids_existing 967 968 # check here if ids are still available 969 # if np.sometrue( np.not_equal( np.take(self._inds, ids_existing), -1) ): 970 # print 'WARNING in create_ids: some ids already in use',ids_existing 971 # return np.zeros(0,int) 972 973 # extend index map with -1 as necessary 974 if id_max > id_max_old: 975 # print 'ext',-1*ones(id_max-id_max_old) 976 self._inds = np.concatenate((self._inds, -1*np.ones(id_max-id_max_old, int))) 977 978 # assign n new indexes to new ids 979 ind_new = np.arange(n_array_old, n_array_old+n, dtype=np.int32) 980 981 # print 'ind_new',ind_new 982 np.put(self._inds, ids, ind_new) 983 984 # print ' concat ids..',self._ids,ids 985 self._ids = np.concatenate((self._ids, ids)) 986 987 def add_rows(self, n=None, ids=[], **attrs): 988 989 if n is not None: 990 ids = self.suggest_ids(n) 991 elif (len(ids) == 0) & (len(attrs) > 0): 992 # get number of rows from any valye vector provided 993 ids = self.suggest_ids(len(attrs.values()[0])) 994 elif (n is None) & (len(ids) == 0) & (len(attrs) > 0): 995 # nothing given really-> do nothing 996 return np.zeros((0), np.int) 997 998 else: 999 # ids already given , no ids to create 1000 pass 1001 1002 # print 'add_rows ids', ids, type(ids) 1003 self._add_ids(ids) 1004 1005 for colconfig in self._colconfigs: 1006 colconfig.add(ids, values=attrs.get(colconfig.attrname, None)) 1007 if self.plugin: 1008 self.plugin.exec_events_ids(EVTADDITEM, ids) 1009 return ids 1010 1011 def copy_cols(self, attrman2, ids=None): 1012 # print 'copy_cols' 1013 if ids is None: 1014 ids2 = attrman2.get_ids() 1015 else: 1016 ids2 = ids 1017 #ids_new = self.suggest_ids(ids2) 1018 ids_new = self.add_rows(n=len(ids2)) 1019 1020 for colconfig2 in attrman2._colconfigs: 1021 if hasattr(self, colconfig2.attrname): 1022 colconfig = getattr(self, colconfig2.attrname) 1023 colconfig.set(ids_new, values=colconfig2[ids2].copy()) 1024 1025 return ids_new 1026 1027 def set_rows(self, ids, **attrs): 1028 1029 for colconfig in self._colconfigs: 1030 colconfig.set(ids, values=attrs.get(colconfig.attrname, None)) 1031 if self.plugin: 1032 self.plugin.exec_events_ids(EVTSETITEM, ids) 1033 1034 def add_row(self, _id=None, **attrs): 1035 if _id is None: 1036 _id = self.suggest_id() 1037 1038 self._add_ids([_id]) 1039 #ids = self.add_rows(1, **attrs) 1040 for colconfig in self._colconfigs: 1041 # print ' add_row',colconfig.attrname,attrs.get(colconfig.attrname, None ) 1042 colconfig.add(_id, values=attrs.get(colconfig.attrname, None)) 1043 1044 if self.plugin: 1045 self.plugin.exec_events_ids(EVTADDITEM, [id]) 1046 return _id 1047 1048 def set_row(self, _id, **attrs): 1049 # if _id is None: 1050 # print ' set_row ',self.get_ident(),attrs 1051 for colconfig in self._colconfigs: # TODO: run through keys!!!! 1052 # print ' add_row',_id,colconfig.attrname,attrs.get(colconfig.attrname, None ) 1053 # if attrs.has_key(colconfig.attrname): 1054 #colconfig.set(_id, values = attrs[colconfig.attrname]) 1055 colconfig.set(_id, values=attrs.get(colconfig.attrname, None)) 1056 1057 if self.plugin: 1058 self.plugin.exec_events_ids(EVTSETITEM, [id]) 1059 1060 def del_row(self, _id): 1061 # print 'del_row',id 1062 self.del_rows([_id]) 1063 1064 def del_rows(self, ids): 1065 # print '\n\ndel_rows',self.ident,ids 1066 # print ' self._ids',self._ids 1067 # print ' self._inds',self._inds 1068 # TODO: this could be done in with array methods 1069 1070 for _id in ids: 1071 i = self._inds[_id] 1072 # print ' id to eliminate _id=',_id 1073 # print ' index to eliminate i=',i 1074 for colconfig in self._colconfigs: 1075 # print ' colconfig',colconfig.attrname,i 1076 # colconfig.delete_ind(i) 1077 del colconfig[_id] # this is universal, also for cm.ColConfigs 1078 1079 # print ' del from id lookup' 1080 self._ids = np.concatenate((self._ids[:i], self._ids[i+1:])) 1081 # print ' ids after cut',self._ids 1082 1083 # print ' free index',id 1084 if _id == len(self._inds)-1: 1085 # id is highest, let's shrink index array by 1 1086 self._inds = self._inds[:-1] 1087 else: 1088 self._inds[_id] = -1 1089 1090 # get ids of all indexes which are above i 1091 ids_above = np.flatnonzero(self._inds > i) 1092 1093 # decrease index from those wich are above the deleted one 1094 #put(self._inds, ids_above,take(self._inds,ids_above)-1) 1095 self._inds[ids_above] -= 1 1096 1097 # print ' inds after cut',self._inds 1098 1099 # print ' self._inds',self._inds 1100 1101 if self.plugin: 1102 self.plugin.exec_events_ids(EVTDELITEM, ids) 1103 1104 # print ' del',ids,' done.' 1105 1106 def clear_rows(self): 1107 # print 'clear_rows',self.ident 1108 1109 if self.plugin: 1110 self.plugin.exec_events_ids(EVTDELITEM, self.get_ids()) 1111 self._ids = [] 1112 self._inds = np.zeros((0,), int) 1113 self._ids = np.zeros((0,), int) 1114 1115 for colconfig in self._colconfigs: 1116 # print 'clear_rows',colconfig.attrname,len(colconfig.get_value()) 1117 colconfig.clear() 1118 # print ' done',len(colconfig.get_value()) 1119 1120 1121class ArrayObjman(Arrayman, TableMixin): 1122 """ 1123 Array Object management manages objects with numeric Python arrays 1124 based columns. Can also handle list and dict based columns. 1125 """ 1126 1127 def __init__(self, ident, **kwargs): 1128 self._init_objman(ident, **kwargs) 1129 1130 def _init_objman(self, ident, is_plugin=False, **kwargs): 1131 BaseObjman._init_objman(self, ident, managertype='table', **kwargs) 1132 Arrayman.__init__(self, is_plugin=is_plugin) 1133 self.set_attrsman(self) 1134 1135 def init_postload_internal(self, parent): 1136 """ 1137 Called after set state. 1138 Link internal states. 1139 """ 1140 TableMixin.init_postload_internal(self, parent) 1141 attrman = self.get_attrsman() 1142 1143 # this should no longer happen in the future as ind and ids 1144 # have been formatted propperly 1145 if attrman._inds.dtype != np.int32: 1146 print 'WARNING: 64 bit ids and inds...will adjust to 32 bit' 1147 attrman._inds = np.array(attrman._inds, dtype=np.int32) 1148 attrman._ids = np.array(attrman._ids, dtype=np.int32) 1149 1150 def init_postload_external(self): 1151 """ 1152 Called after set state. 1153 Link internal states. 1154 """ 1155 TableMixin.init_postload_external(self) 1156 self._init_attributes() 1157 self._init_constants() 1158 1159 def clear_rows(self): 1160 if self.plugin: 1161 self.plugin.exec_events_ids(EVTDELITEM, self.get_ids()) 1162 self._inds = np.zeros((0,), dtype=np.int32) 1163 self._ids = np.zeros((0,), dtype=np.int32) 1164 for colconfig in self.get_attrsman()._colconfigs: 1165 # print 'ArrayObjman.clear_rows',colconfig.attrname,len(colconfig.get_value()) 1166 colconfig.clear() 1167 # print ' done',len(colconfig.get_value()) 1168 1169 def clear(self): 1170 # print 'ArrayObjman.clear',self.ident 1171 # clear/reset scalars 1172 for attrconfig in self.get_attrsman().get_configs(structs=STRUCTS_SCALAR): 1173 attrconfig.clear() 1174 self.clear_rows() 1175 self._init_constants() 1176 self.set_modified() 1177