1# 2# Gramps - a GTK+/GNOME based genealogy program 3# 4# Copyright (C) 2007 Douglas S. Blank 5# Copyright (C) 2000-2007 Donald N. Allingham 6# Copyright (C) 2008 Raphael Ackerman 7# Copyright (C) 2008 Brian G. Matherly 8# Copyright (C) 2011 Tim G L Lyons 9# 10# This program is free software; you can redistribute it and/or modify 11# it under the terms of the GNU General Public License as published by 12# the Free Software Foundation; either version 2 of the License, or 13# (at your option) any later version. 14# 15# This program is distributed in the hope that it will be useful, 16# but WITHOUT ANY WARRANTY; without even the implied warranty of 17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18# GNU General Public License for more details. 19# 20# You should have received a copy of the GNU General Public License 21# along with this program; if not, write to the Free Software 22# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 23# 24 25"Import from CSV Spreadsheet" 26 27#------------------------------------------------------------------------- 28# 29# Standard Python Modules 30# 31#------------------------------------------------------------------------- 32import time 33import csv 34import codecs 35from io import TextIOWrapper 36 37#------------------------------------------------------------------------ 38# 39# Set up logging 40# 41#------------------------------------------------------------------------ 42import logging 43LOG = logging.getLogger(".ImportCSV") 44 45#------------------------------------------------------------------------- 46# 47# Gramps modules 48# 49#------------------------------------------------------------------------- 50from gramps.gen.const import GRAMPS_LOCALE as glocale 51_ = glocale.translation.sgettext 52ngettext = glocale.translation.ngettext # else "nearby" comments are ignored 53from gramps.gen.lib import (ChildRef, Citation, Event, EventRef, EventType, 54 Family, FamilyRelType, Name, NameType, Note, 55 NoteType, Person, Place, Source, Surname, Tag, 56 PlaceName, PlaceType, PlaceRef, 57 Attribute, AttributeType) 58from gramps.gen.db import DbTxn 59from gramps.gen.datehandler import parser as _dp 60from gramps.gen.utils.string import gender as gender_map 61from gramps.gen.utils.id import create_id 62from gramps.gen.utils.location import located_in 63from gramps.gen.lib.eventroletype import EventRoleType 64from gramps.gen.config import config 65from gramps.gen.display.place import displayer as place_displayer 66from gramps.gen.utils.libformatting import ImportInfo 67from gramps.gen.errors import GrampsImportError as Error 68 69#------------------------------------------------------------------------- 70# 71# Support Functions 72# 73#------------------------------------------------------------------------- 74def get_primary_event_ref_from_type(dbase, person, event_name): 75 """ 76 >>> get_primary_event_ref_from_type(dbase, Person(), "Baptism"): 77 """ 78 for ref in person.event_ref_list: 79 if ref.get_role() == EventRoleType.PRIMARY: 80 event = dbase.get_event_from_handle(ref.ref) 81 if event and event.type.is_type(event_name): 82 return ref 83 return None 84 85#------------------------------------------------------------------------- 86# 87# Support and main functions 88# 89#------------------------------------------------------------------------- 90def rd(line_number, row, col, key, default = None): 91 """ Return Row data by column name """ 92 if key in col: 93 if col[key] >= len(row): 94 LOG.warning("missing '%s, on line %d" % (key, line_number)) 95 return default 96 retval = row[col[key]].strip() 97 if retval == "": 98 return default 99 else: 100 return retval 101 else: 102 return default 103 104def importData(dbase, filename, user): 105 """Function called by Gramps to import data on persons in CSV format.""" 106 if dbase.get_feature("skip-import-additions"): # don't add source or tags 107 parser = CSVParser(dbase, user, None) 108 else: 109 parser = CSVParser(dbase, user, (config.get('preferences.tag-on-import-format') if 110 config.get('preferences.tag-on-import') else None)) 111 try: 112 with open(filename, 'rb') as filehandle: 113 line = filehandle.read(3) 114 if line == codecs.BOM_UTF8: 115 filehandle.seek(0) 116 filehandle = TextIOWrapper(filehandle, encoding='utf_8_sig', 117 errors='replace', newline='') 118 else: # just open with OS encoding 119 filehandle.seek(0) 120 filehandle = TextIOWrapper(filehandle, 121 errors='replace', newline='') 122 parser.parse(filehandle) 123 except EnvironmentError as err: 124 user.notify_error(_("%s could not be opened\n") % filename, str(err)) 125 return 126 return ImportInfo({_("Results"): _("done")}) 127 128#------------------------------------------------------------------------- 129# 130# CSV Parser 131# 132#------------------------------------------------------------------------- 133class CSVParser: 134 """Class to read data in CSV format from a file object.""" 135 def __init__(self, dbase, user, default_tag_format=None): 136 self.db = dbase 137 self.user = user 138 self.trans = None 139 self.lineno = 0 140 self.index = 0 141 self.fam_count = 0 142 self.indi_count = 0 143 self.place_count = 0 144 self.pref = {} # person ref, internal to this sheet 145 self.fref = {} # family ref, internal to this sheet 146 self.placeref = {} 147 self.place_types = {} 148 # Build reverse dictionary, name to type number 149 for items in PlaceType().get_map().items(): # (0, 'Custom') 150 self.place_types[items[1]] = items[0] 151 self.place_types[items[1].lower()] = items[0] 152 if _(items[1]) != items[1]: 153 self.place_types[_(items[1])] = items[0] 154 # Add custom types: 155 for custom_type in self.db.get_place_types(): 156 self.place_types[custom_type] = 0 157 self.place_types[custom_type.lower()] = 0 158 column2label = { 159 "surname": ("lastname", "last_name", "surname", _("surname"), 160 _("Surname")), 161 "firstname": ("firstname", "first_name", "given_name", "given", 162 "given name", _("given name"), _("given"), 163 _("Given"), _("Given name")), 164 "callname": ("call name", _("Call name"), "callname", "call_name", 165 "call", _("Call"), _("call")), 166 "title": ("title", _("title"), _("Person or Place|title")), 167 "prefix": ("prefix", _("prefix"), _("Prefix")), 168 "suffix": ("suffix", _("suffix"), _("Suffix")), 169 "gender": ("gender", _("gender"), _("Gender")), 170 "source": ("source", _("source"), _("Source")), 171 "note": ("note", _("note"), _("Note")), 172 "birthplace": ("birthplace", "birth_place", "birth place", 173 _("birth place"), _("Birth place")), 174 "birthplace_id": ("birthplaceid", "birth_place_id", 175 "birth place id", _("birth place id"), 176 "birthplace_id"), 177 "birthdate": ("birthdate", "birth_date", "birth date", 178 _("birth date")), 179 "birthsource": ("birthsource", "birth_source", "birth source", 180 _("birth source")), 181 "baptismplace": ("baptismplace", "baptism place", 182 _("baptism place")), 183 "baptismplace_id": ("baptismplaceid", "baptism place id", 184 _("baptism place id"), "baptism_place_id", 185 "baptismplace_id"), 186 "baptismdate": ("baptismdate", "baptism date", _("baptism date")), 187 "baptismsource": ("baptismsource", "baptism source", 188 _("baptism source")), 189 "burialplace": ("burialplace", "burial place", _("burial place")), 190 "burialplace_id": ("burialplaceid", "burial place id", 191 _("burial place id"), "burial_place_id", 192 "burialplace_id"), 193 "burialdate": ("burialdate", "burial date", _("burial date")), 194 "burialsource": ("burialsource", "burial source", 195 _("burial source")), 196 "deathplace": ("deathplace", "death_place", "death place", 197 _("death place")), 198 "deathplace_id": ("deathplaceid", "death place id", 199 _("death place id"), "death_place_id", 200 "deathplace_id"), 201 "deathdate": ("deathdate", "death_date", "death date", 202 _("death date")), 203 "deathsource": ("deathsource", "death_source", "death source", 204 _("death source")), 205 "deathcause": ("deathcause", "death_cause", "death cause", 206 _("death cause")), 207 "grampsid": (_("Gramps ID"), "grampsid", "id", "gramps_id", 208 "gramps id"), 209 "person": ("person", _("person"), _("Person")), 210 "occupationdescr": ("occupationdescr", _("occupationdescr"), _("Occupation description")), 211 "occupationdate": ("occupationdate", _("occupationdate"), _("Occupation date")), 212 "occupationplace": ("occupationplace", _("occupationplace"), _("Occupation place")), 213 "occupationplace_id": ("occupationplace_id", _("occupationplace_id"), _("Occupation place id")), 214 "occupationsource": ("occupationsource", _("occupationsource"), _("Occupation source")), 215 216 "residencedate": ("residencedate", _("residencedate"), _("residence date")), 217 "residenceplace": ("residenceplace", _("residenceplace"), _("residence place")), 218 "residenceplace_id": ("residenceplace_id", _("residenceplace_id"), _("residence place id")), 219 "residencesource": ("residencesource", _("residencesource"), _("residence source")), 220 221 "attributetype": ("attributetype", _("attributetype"), _("attribute type")), 222 "attributevalue": ("attributevalue", _("attributevalue"), _("attribute value")), 223 "attributesource": ("attributesource", _("attributesource"), _("attribute source")), 224 225 # ---------------------------------- 226 "child": ("child", _("child"), _("Child")), 227 "family": ("family", _("family"), _("Family")), 228 # ---------------------------------- 229 "wife": ("mother", _("mother"), _("Mother"), 230 "wife", _("wife"), _("Wife"), 231 "parent2", _("parent2")), 232 "husband": ("father", _("father"), _("Father"), 233 "husband", _("husband"), _("Husband"), 234 "parent1", _("parent1")), 235 "marriage": ("marriage", _("marriage"), _("Marriage")), 236 "date": ("date", _("date"), _("Date")), 237 "place": ("place", _("place"), _("Place")), 238 "place_id": ("place id", "place_id", "placeid", _("place id")), 239 "name": ("name", _("name"), _("Name")), 240 "type": ("type", _("type"), _("Type")), 241 "latitude": ("latitude", _("latitude")), 242 "longitude": ("longitude", _("longitude")), 243 "code": ("code", _("code"), _("Code")), 244 "enclosed_by": ("enclosed by", _("enclosed by"), 245 "enclosed_by", _("enclosed_by"), "enclosedby") 246 } 247 lab2col_dict = [] 248 for key in list(column2label.keys()): 249 for val in column2label[key]: 250 lab2col_dict.append((val.lower(), key)) 251 self.label2column = dict(lab2col_dict) 252 if default_tag_format: 253 name = time.strftime(default_tag_format) 254 tag = self.db.get_tag_from_name(name) 255 if tag: 256 self.default_tag = tag 257 else: 258 self.default_tag = Tag() 259 self.default_tag.set_name(name) 260 else: 261 self.default_tag = None 262 263 def cleanup_column_name(self, column): 264 """Handle column aliases for CSV spreadsheet import and SQL.""" 265 return self.label2column.get(column, column) 266 267 def read_csv(self, filehandle): 268 "Read the data from the file and return it as a list." 269 try: 270 data = [[r.strip() for r in row] for row in csv.reader(filehandle)] 271 except csv.Error as err: 272 self.user.notify_error(_('format error: line %(line)d: %(zero)s') % { 273 'line' : reader.line_num, 'zero' : err } ) 274 return None 275 return data 276 277 def lookup(self, type_, id_): 278 """ 279 Return the object of type type_ with id id_ from db or previously 280 stored value. 281 """ 282 if id_ is None: 283 return None 284 if type_ == "family": 285 if id_.startswith("[") and id_.endswith("]"): 286 id_ = self.db.fid2user_format(id_[1:-1]) 287 db_lookup = self.db.get_family_from_gramps_id(id_) 288 if db_lookup is None: 289 return self.lookup(type_, id_) 290 else: 291 return db_lookup 292 else: 293 id_ = self.db.fid2user_format(id_) 294 if id_.lower() in self.fref: 295 return self.fref[id_.lower()] 296 else: 297 return None 298 elif type_ == "person": 299 if id_.startswith("[") and id_.endswith("]"): 300 id_ = self.db.id2user_format(id_[1:-1]) 301 db_lookup = self.db.get_person_from_gramps_id(id_) 302 if db_lookup is None: 303 return self.lookup(type_, id_) 304 else: 305 return db_lookup 306 else: 307 id_ = self.db.id2user_format(id_) 308 if id_.lower() in self.pref: 309 return self.pref[id_.lower()] 310 else: 311 return None 312 elif type_ == "place": 313 if id_.startswith("[") and id_.endswith("]"): 314 id_ = self.db.pid2user_format(id_[1:-1]) 315 db_lookup = self.db.get_place_from_gramps_id(id_) 316 if db_lookup is None: 317 return self.lookup(type_, id_) 318 else: 319 return db_lookup 320 else: 321 id_ = self.db.pid2user_format(id_) 322 if id_.lower() in self.placeref: 323 return self.placeref[id_.lower()] 324 else: 325 return None 326 else: 327 LOG.warning("invalid lookup type in CSV import: '%s'" % type_) 328 return None 329 330 def storeup(self, type_, id_, object_): 331 "Store object object_ of type type_ in a dictionary under key id_." 332 if id_.startswith("[") and id_.endswith("]"): 333 id_ = id_[1:-1] 334 #return # do not store gramps people; go look them up 335 if type_ == "person": 336 id_ = self.db.id2user_format(id_) 337 self.pref[id_.lower()] = object_ 338 elif type_ == "family": 339 id_ = self.db.fid2user_format(id_) 340 self.fref[id_.lower()] = object_ 341 elif type_ == "place": 342 id_ = self.db.pid2user_format(id_) 343 self.placeref[id_.lower()] = object_ 344 else: 345 LOG.warning("invalid storeup type in CSV import: '%s'" % type_) 346 347 def parse(self, filehandle): 348 """ 349 Prepare the database and parse the input file. 350 351 :param filehandle: open file handle positioned at start of the file 352 """ 353 progress_title = _('CSV Import') 354 with self.user.progress(progress_title, 355 _('Reading data...'), 1) as step: 356 data = self.read_csv(filehandle) 357 358 with self.user.progress(progress_title, 359 _('Importing data...'), len(data)) as step: 360 tym = time.time() 361 self.db.disable_signals() 362 with DbTxn(_("CSV import"), self.db, batch=True) as self.trans: 363 if self.default_tag and self.default_tag.handle is None: 364 self.db.add_tag(self.default_tag, self.trans) 365 self._parse_csv_data(data, step) 366 self.db.enable_signals() 367 self.db.request_rebuild() 368 tym = time.time() - tym 369 # translators: leave all/any {...} untranslated 370 msg = ngettext('Import Complete: {number_of} second', 371 'Import Complete: {number_of} seconds', tym 372 ).format(number_of=tym) 373 LOG.debug(msg) 374 LOG.debug("New Families: %d" % self.fam_count) 375 LOG.debug("New Individuals: %d" % self.indi_count) 376 377 def _parse_csv_data(self, data, step): 378 """Parse each line of the input data and act accordingly.""" 379 self.lineno = 0 380 self.index = 0 381 self.fam_count = 0 382 self.indi_count = 0 383 self.place_count = 0 384 self.pref = {} # person ref, internal to this sheet 385 self.fref = {} # family ref, internal to this sheet 386 self.placeref = {} 387 header = None 388 line_number = 0 389 for row in data: 390 step() 391 line_number += 1 392 if "".join(row) == "": # no blanks are allowed inside a table 393 header = None # clear headers, ready for next "table" 394 continue 395 ###################################### 396 if header is None: 397 header = [self.cleanup_column_name(r.lower()) for r in row] 398 col = {} 399 count = 0 400 for key in header: 401 col[key] = count 402 count += 1 403 continue 404 # four different kinds of data: person, family, and marriage 405 if (("marriage" in header) or 406 ("husband" in header) or 407 ("wife" in header)): 408 self._parse_marriage(line_number, row, col) 409 elif "family" in header: 410 self._parse_family(line_number, row, col) 411 elif "surname" in header: 412 self._parse_person(line_number, row, col) 413 elif "place" in header: 414 self._parse_place(line_number, row, col) 415 else: 416 LOG.warning("ignoring line %d" % line_number) 417 return None 418 419 def _parse_marriage(self, line_number, row, col): 420 "Parse the content of a Marriage,Husband,Wife line." 421 marriage_ref = rd(line_number, row, col, "marriage") 422 husband = rd(line_number, row, col, "husband") 423 wife = rd(line_number, row, col, "wife") 424 marriagedate = rd(line_number, row, col, "date") 425 marriageplace = rd(line_number, row, col, "place") 426 marriageplace_id = rd(line_number, row, col, "place_id") 427 marriagesource = rd(line_number, row, col, "source") 428 note = rd(line_number, row, col, "note") 429 wife = self.lookup("person", wife) 430 husband = self.lookup("person", husband) 431 if husband is None and wife is None: 432 # might have children, so go ahead and add 433 LOG.warning("no parents on line %d; adding family anyway" % 434 line_number) 435 family = self.get_or_create_family(marriage_ref, husband, wife) 436 # adjust gender, if not already provided 437 if husband: 438 # this is just a guess, if unknown 439 if husband.get_gender() == Person.UNKNOWN: 440 husband.set_gender(Person.MALE) 441 self.db.commit_person(husband, self.trans) 442 if wife: 443 # this is just a guess, if unknown 444 if wife.get_gender() == Person.UNKNOWN: 445 wife.set_gender(Person.FEMALE) 446 self.db.commit_person(wife, self.trans) 447 if marriage_ref: 448 self.storeup("family", marriage_ref, family) 449 if marriagesource: 450 # add, if new 451 new, marriagesource = self.get_or_create_source(marriagesource) 452 if marriageplace and marriageplace_id: 453 raise Error("Error in marriage: can't have a place and place_id") 454 if marriageplace: 455 # add, if new 456 new, marriageplace = self.get_or_create_place(marriageplace) 457 elif marriageplace_id: 458 # better exist already, locally or in database: 459 marriageplace = self.lookup("place", marriageplace_id) 460 if marriagedate: 461 marriagedate = _dp.parse(marriagedate) 462 if marriagedate or marriageplace or marriagesource or note: 463 # add, if new; replace, if different 464 new, marriage = self.get_or_create_event(family, 465 EventType.MARRIAGE, marriagedate, 466 marriageplace, marriagesource) 467 if new: 468 mar_ref = EventRef() 469 mar_ref.set_reference_handle(marriage.get_handle()) 470 mar_ref.set_role(EventRoleType(EventRoleType.FAMILY)) 471 family.add_event_ref(mar_ref) 472 self.db.commit_family(family, self.trans) 473 # only add note to event: 474 # append notes, if previous notes 475 if note: 476 previous_notes_list = marriage.get_note_list() 477 updated_note = False 478 for note_handle in previous_notes_list: 479 previous_note = self.db.get_note_from_handle( 480 note_handle) 481 if previous_note.type == NoteType.EVENT: 482 previous_text = previous_note.get() 483 if note not in previous_text: 484 note = previous_text + "\n" + note 485 previous_note.set(note) 486 self.db.commit_note(previous_note, self.trans) 487 updated_note = True 488 break 489 if not updated_note: 490 # add new note here 491 new_note = Note() 492 new_note.handle = create_id() 493 new_note.type.set(NoteType.EVENT) 494 new_note.set(note) 495 if self.default_tag: 496 new_note.add_tag(self.default_tag.handle) 497 self.db.add_note(new_note, self.trans) 498 marriage.add_note(new_note.handle) 499 self.db.commit_event(marriage, self.trans) 500 501 def _parse_family(self, line_number, row, col): 502 "Parse the content of a family line" 503 family_ref = rd(line_number, row, col, "family") 504 if family_ref is None: 505 LOG.warning("no family reference found for family on line %d" % 506 line_number) 507 return # required 508 child = rd(line_number, row, col, "child") 509 source = rd(line_number, row, col, "source") 510 note = rd(line_number, row, col, "note") 511 gender = rd(line_number, row, col, "gender") 512 child = self.lookup("person", child) 513 family = self.lookup("family", family_ref) 514 if family is None: 515 LOG.warning("no matching family reference found for family " 516 "on line %d" % line_number) 517 return 518 if child is None: 519 LOG.warning("no matching child reference found for family " 520 "on line %d" % line_number) 521 return 522 # is this child already in this family? If so, don't add 523 LOG.debug("children: %s", [ref.ref for ref in 524 family.get_child_ref_list()]) 525 LOG.debug("looking for: %s", child.get_handle()) 526 if child.get_handle() not in [ref.ref for ref in 527 family.get_child_ref_list()]: 528 # add child to family 529 LOG.debug(" adding child [%s] to family [%s]", 530 child.get_gramps_id(), family.get_gramps_id()) 531 childref = ChildRef() 532 childref.set_reference_handle(child.get_handle()) 533 family.add_child_ref( childref) 534 self.db.commit_family(family, self.trans) 535 child.add_parent_family_handle(family.get_handle()) 536 if gender: 537 # replace 538 gender = gender.lower() 539 if gender == gender_map[Person.MALE].lower(): 540 gender = Person.MALE 541 elif gender == gender_map[Person.FEMALE].lower(): 542 gender = Person.FEMALE 543 else: 544 gender = Person.UNKNOWN 545 child.set_gender(gender) 546 if source: 547 # add, if new 548 dummy_new, source = self.get_or_create_source(source) 549 self.find_and_set_citation(child, source) 550 # put note on child 551 if note: 552 # append notes, if previous notes 553 previous_notes_list = child.get_note_list() 554 updated_note = False 555 for note_handle in previous_notes_list: 556 previous_note = self.db.get_note_from_handle(note_handle) 557 if previous_note.type == NoteType.PERSON: 558 previous_text = previous_note.get() 559 if note not in previous_text: 560 note = previous_text + "\n" + note 561 previous_note.set(note) 562 self.db.commit_note(previous_note, self.trans) 563 updated_note = True 564 break 565 if not updated_note: 566 # add new note here 567 new_note = Note() 568 new_note.handle = create_id() 569 new_note.type.set(NoteType.PERSON) 570 new_note.set(note) 571 if self.default_tag: 572 new_note.add_tag(self.default_tag.handle) 573 self.db.add_note(new_note, self.trans) 574 child.add_note(new_note.handle) 575 self.db.commit_person(child, self.trans) 576 577 def _parse_person(self, line_number, row, col): 578 "Parse the content of a Person line." 579 surname = rd(line_number, row, col, "surname") 580 firstname = rd(line_number, row, col, "firstname", "") 581 callname = rd(line_number, row, col, "callname") 582 title = rd(line_number, row, col, "title") 583 prefix = rd(line_number, row, col, "prefix") 584 suffix = rd(line_number, row, col, "suffix") 585 gender = rd(line_number, row, col, "gender") 586 source = rd(line_number, row, col, "source") 587 note = rd(line_number, row, col, "note") 588 birthplace = rd(line_number, row, col, "birthplace") 589 birthplace_id = rd(line_number, row, col, "birthplace_id") 590 birthdate = rd(line_number, row, col, "birthdate") 591 birthsource = rd(line_number, row, col, "birthsource") 592 baptismplace = rd(line_number, row, col, "baptismplace") 593 baptismplace_id = rd(line_number, row, col, "baptismplace_id") 594 baptismdate = rd(line_number, row, col, "baptismdate") 595 baptismsource = rd(line_number, row, col, "baptismsource") 596 burialplace = rd(line_number, row, col, "burialplace") 597 burialplace_id = rd(line_number, row, col, "burialplace_id") 598 burialdate = rd(line_number, row, col, "burialdate") 599 burialsource = rd(line_number, row, col, "burialsource") 600 deathplace = rd(line_number, row, col, "deathplace") 601 deathplace_id = rd(line_number, row, col, "deathplace_id") 602 deathdate = rd(line_number, row, col, "deathdate") 603 deathsource = rd(line_number, row, col, "deathsource") 604 deathcause = rd(line_number, row, col, "deathcause") 605 grampsid = rd(line_number, row, col, "grampsid") 606 person_ref = rd(line_number, row, col, "person") 607 occupationdescr = rd(line_number, row, col, "occupationdescr") 608 occupationplace = rd(line_number, row, col, "occupationplace") 609 occupationplace_id = rd(line_number, row, col, "occupationplace_id") 610 occupationsource = rd(line_number, row, col, "occupationsource") 611 occupationdate = rd(line_number, row, col, "occupationdate") 612 residencedate = rd(line_number, row, col, "residencedate") 613 residenceplace = rd(line_number, row, col, "residenceplace") 614 residenceplace_id = rd(line_number, row, col, "residenceplace_id") 615 residencesource = rd(line_number, row, col, "residencesource") 616 attributetype = rd(line_number, row, col, "attributetype") 617 attributevalue = rd(line_number, row, col, "attributevalue") 618 attributesource = rd(line_number, row, col, "attributesource") 619 620 ######################################################### 621 # if this person already exists, don't create them 622 person = self.lookup("person", person_ref) 623 if person is None: 624 if surname is None: 625 LOG.warning("empty surname for new person on line %d" % 626 line_number) 627 surname = "" 628 # new person 629 person = self.create_person() 630 name = Name() 631 name.set_type(NameType(NameType.BIRTH)) 632 name.set_first_name(firstname) 633 surname_obj = Surname() 634 surname_obj.set_surname(surname) 635 name.add_surname(surname_obj) 636 person.set_primary_name(name) 637 else: 638 name = person.get_primary_name() 639 ######################################################### 640 if person_ref is not None: 641 self.storeup("person", person_ref, person) 642 # replace 643 if surname is not None: 644 name.get_primary_surname().set_surname(surname) 645 if firstname is not None: 646 name.set_first_name(firstname) 647 if callname is not None: 648 name.set_call_name(callname) 649 if title is not None: 650 name.set_title(title) 651 if prefix is not None: 652 name.get_primary_surname().set_prefix(prefix) 653 name.group_as = '' # HELP? what should I do here? 654 if suffix is not None: 655 name.set_suffix(suffix) 656 if note is not None: 657 # append notes, if previous notes 658 previous_notes_list = person.get_note_list() 659 updated_note = False 660 for note_handle in previous_notes_list: 661 previous_note = self.db.get_note_from_handle(note_handle) 662 if previous_note.type == NoteType.PERSON: 663 previous_text = previous_note.get() 664 if note not in previous_text: 665 note = previous_text + "\n" + note 666 previous_note.set(note) 667 self.db.commit_note(previous_note, self.trans) 668 updated_note = True 669 break 670 if not updated_note: 671 # add new note here 672 new_note = Note() 673 new_note.handle = create_id() 674 new_note.type.set(NoteType.PERSON) 675 new_note.set(note) 676 if self.default_tag: 677 new_note.add_tag(self.default_tag.handle) 678 self.db.add_note(new_note, self.trans) 679 person.add_note(new_note.handle) 680 if grampsid is not None: 681 person.gramps_id = grampsid 682 elif person_ref is not None: 683 if person_ref.startswith("[") and person_ref.endswith("]"): 684 person.gramps_id = self.db.id2user_format(person_ref[1:-1]) 685 if (person.get_gender() == Person.UNKNOWN and 686 gender is not None): 687 gender = gender.lower() 688 if gender == gender_map[Person.MALE].lower(): 689 gender = Person.MALE 690 elif gender == gender_map[Person.FEMALE].lower(): 691 gender = Person.FEMALE 692 else: 693 gender = Person.UNKNOWN 694 person.set_gender(gender) 695 ######################################################### 696 # add if new, replace if different 697 # Birth: 698 if birthdate is not None: 699 birthdate = _dp.parse(birthdate) 700 if birthplace and birthplace_id: 701 raise Error("Error in person: can't have a birthplace and birthplace_id") 702 if birthplace is not None: 703 new, birthplace = self.get_or_create_place(birthplace) 704 elif birthplace_id: 705 # better exist already, locally or in database: 706 birthplace = self.lookup("place", birthplace_id) 707 if birthsource is not None: 708 new, birthsource = self.get_or_create_source(birthsource) 709 if birthdate or birthplace or birthsource: 710 new, birth = self.get_or_create_event(person, 711 EventType.BIRTH, birthdate, 712 birthplace, birthsource) 713 birth_ref = person.get_birth_ref() 714 if birth_ref is None: 715 # new 716 birth_ref = EventRef() 717 birth_ref.set_reference_handle( birth.get_handle()) 718 person.set_birth_ref( birth_ref) 719 # Baptism: 720 if baptismdate is not None: 721 baptismdate = _dp.parse(baptismdate) 722 if baptismplace and baptismplace_id: 723 raise Error("Error in person: can't have a baptismplace and baptismplace_id") 724 if baptismplace is not None: 725 new, baptismplace = self.get_or_create_place(baptismplace) 726 elif baptismplace_id: 727 # better exist already, locally or in database: 728 baptismplace = self.lookup("place", baptismplace_id) 729 if baptismsource is not None: 730 new, baptismsource = self.get_or_create_source(baptismsource) 731 if baptismdate or baptismplace or baptismsource: 732 new, baptism = self.get_or_create_event(person, 733 EventType.BAPTISM, baptismdate, 734 baptismplace, baptismsource) 735 baptism_ref = get_primary_event_ref_from_type(self.db, person, 736 "Baptism") 737 if baptism_ref is None: 738 # new 739 baptism_ref = EventRef() 740 baptism_ref.set_reference_handle( baptism.get_handle()) 741 person.add_event_ref( baptism_ref) 742 # Death: 743 if deathdate is not None: 744 deathdate = _dp.parse(deathdate) 745 if deathplace and deathplace_id: 746 raise Error("Error in person: can't have a deathplace and deathplace_id") 747 if deathplace is not None: 748 new, deathplace = self.get_or_create_place(deathplace) 749 elif deathplace_id: 750 # better exist already, locally or in database: 751 deathplace = self.lookup("place", deathplace_id) 752 if deathsource is not None: 753 new, deathsource = self.get_or_create_source(deathsource) 754 if deathdate or deathplace or deathsource or deathcause: 755 new, death = self.get_or_create_event(person, 756 EventType.DEATH, deathdate, deathplace, 757 deathsource) 758 if deathcause: 759 death.set_description(deathcause) 760 self.db.commit_event(death, self.trans) 761 death_ref = person.get_death_ref() 762 if death_ref is None: 763 # new 764 death_ref = EventRef() 765 death_ref.set_reference_handle(death.get_handle()) 766 person.set_death_ref(death_ref) 767 # Burial: 768 if burialdate is not None: 769 burialdate = _dp.parse(burialdate) 770 if burialplace and burialplace_id: 771 raise Error("Error in person: can't have a burialplace and burialplace_id") 772 if burialplace is not None: 773 new, burialplace = self.get_or_create_place(burialplace) 774 elif burialplace_id: 775 # better exist already, locally or in database: 776 burialplace = self.lookup("place", burialplace_id) 777 if burialsource is not None: 778 new, burialsource = self.get_or_create_source(burialsource) 779 if burialdate or burialplace or burialsource: 780 new, burial = self.get_or_create_event(person, 781 EventType.BURIAL, burialdate, 782 burialplace, burialsource) 783 burial_ref = get_primary_event_ref_from_type(self.db, person, 784 "Burial") 785 if burial_ref is None: 786 # new 787 burial_ref = EventRef() 788 burial_ref.set_reference_handle( burial.get_handle()) 789 person.add_event_ref( burial_ref) 790 if source: 791 # add, if new 792 new, source = self.get_or_create_source(source) 793 self.find_and_set_citation(person, source) 794 795 # Attribute 796 # update existing custom attribute or create it 797 if attributevalue is not None: 798 new, attr = self.get_or_create_attribute(person, attributetype, 799 attributevalue, attributesource) 800 801 # Occupation: 802 # Contrary to the fields above, 803 # each line in the csv will add a new occupation event 804 if occupationdescr is not None: # if no description we have no info to add 805 if occupationdate is not None: 806 occupationdate = _dp.parse(occupationdate) 807 # occupation place takes precedence over place id if both are set 808 if occupationplace is not None: 809 new, occupationplace = self.get_or_create_place(occupationplace) 810 elif occupationplace_id: 811 occupationplace = self.lookup("place", occupationplace_id) 812 if occupationsource is not None: 813 new, occupationsource = self.get_or_create_source(occupationsource) 814 new, occupation = self.get_or_create_event(person, 815 EventType.OCCUPATION, occupationdate, 816 occupationplace, occupationsource, occupationdescr, True) 817 occupation_ref = EventRef() 818 occupation_ref.set_reference_handle( occupation.get_handle()) 819 person.add_event_ref( occupation_ref) 820 821 # Residence: 822 # Contrary to the fields above occupation, 823 # each line in the csv will add a new residence event 824 if residencedate is not None: 825 residencedate = _dp.parse(residencedate) 826 # residence place takes precedence over place id if both are set 827 if residenceplace is not None: 828 new, residenceplace = self.get_or_create_place(residenceplace) 829 elif residenceplace_id: 830 residenceplace = self.lookup("place", residenceplace_id) 831 if residencesource is not None: 832 new, residencesource = self.get_or_create_source(residencesource) 833 if residencedate or residenceplace or residencesource: 834 new, residence = self.get_or_create_event(person, 835 EventType.RESIDENCE, residencedate, 836 residenceplace, residencesource, None, True) 837 residence_ref = EventRef() 838 residence_ref.set_reference_handle( residence.get_handle()) 839 person.add_event_ref( residence_ref) 840 841 842 self.db.commit_person(person, self.trans) 843 844 def _parse_place(self, line_number, row, col): 845 "Parse the content of a Place line." 846 place_id = rd(line_number, row, col, "place") 847 place_title = rd(line_number, row, col, "title") 848 place_name = rd(line_number, row, col, "name") 849 place_type_str = rd(line_number, row, col, "type") 850 place_latitude = rd(line_number, row, col, "latitude") 851 place_longitude = rd(line_number, row, col, "longitude") 852 place_code = rd(line_number, row, col, "code") 853 place_enclosed_by_id = rd(line_number, row, col, "enclosed_by") 854 place_date = rd(line_number, row, col, "date") 855 ######################################################### 856 # if this place already exists, don't create it 857 place = self.lookup("place", place_id) 858 if place is None: 859 # new place 860 place = self.create_place() 861 if place_id is not None: 862 if place_id.startswith("[") and place_id.endswith("]"): 863 place.gramps_id = self.db.pid2user_format(place_id[1:-1]) 864 self.storeup("place", place_id, place) 865 if place_title is not None: 866 place.title = place_title 867 if place_name is not None: 868 place.name = PlaceName(value=place_name) 869 if place_type_str is not None: 870 place.place_type = self.get_place_type(place_type_str) 871 if place_latitude is not None: 872 place.lat = place_latitude 873 if place_longitude is not None: 874 place.long = place_longitude 875 if place_code is not None: 876 place.code = place_code 877 if place_enclosed_by_id is not None: 878 place_enclosed_by = self.lookup("place", place_enclosed_by_id) 879 if place_enclosed_by is None: 880 raise Exception("cannot enclose %s in %s as it doesn't exist" % 881 (place.gramps_id, place_enclosed_by_id)) 882 for placeref in place.placeref_list: 883 if place_enclosed_by.handle == placeref.ref: 884 break 885 else: 886 placeref = PlaceRef() 887 placeref.ref = place_enclosed_by.handle 888 place.placeref_list.append(placeref) 889 if place_date: 890 placeref.date = _dp.parse(place_date) 891 ######################################################### 892 self.db.commit_place(place, self.trans) 893 894 def get_place_type(self, place_type_str): 895 if place_type_str in self.place_types: 896 return PlaceType((self.place_types[place_type_str], place_type_str)) 897 else: 898 # New custom type: 899 return PlaceType((0, place_type_str)) 900 901 def get_or_create_family(self, family_ref, husband, wife): 902 "Return the family object for the give family ID." 903 # if a gramps_id and exists: 904 LOG.debug("get_or_create_family") 905 if family_ref.startswith("[") and family_ref.endswith("]"): 906 id_ = self.db.fid2user_format(family_ref[1:-1]) 907 family = self.db.get_family_from_gramps_id(id_) 908 if family: 909 # don't delete, only add 910 fam_husband_handle = family.get_father_handle() 911 fam_wife_handle = family.get_mother_handle() 912 if husband: 913 if husband.get_handle() != fam_husband_handle: 914 # this husband is not the same old one! Add him! 915 family.set_father_handle(husband.get_handle()) 916 if wife: 917 if wife.get_handle() != fam_wife_handle: 918 # this wife is not the same old one! Add her! 919 family.set_mother_handle(wife.get_handle()) 920 LOG.debug(" returning existing family") 921 return family 922 # if not, create one: 923 family = Family() 924 # was marked with a gramps_id, but didn't exist, so we'll use it: 925 if family_ref.startswith("[") and family_ref.endswith("]"): 926 id_ = self.db.fid2user_format(family_ref[1:-1]) 927 family.set_gramps_id(id_) 928 # add it: 929 family.set_handle(create_id()) 930 if self.default_tag: 931 family.add_tag(self.default_tag.handle) 932 if husband: 933 family.set_father_handle(husband.get_handle()) 934 husband.add_family_handle(family.get_handle()) 935 if wife: 936 family.set_mother_handle(wife.get_handle()) 937 wife.add_family_handle(family.get_handle()) 938 if husband and wife: 939 family.set_relationship(FamilyRelType.MARRIED) 940 self.db.add_family(family, self.trans) 941 if husband: 942 self.db.commit_person(husband, self.trans) 943 if wife: 944 self.db.commit_person(wife, self.trans) 945 self.fam_count += 1 946 return family 947 948 def get_or_create_event(self, object_, type_, date=None, place=None, 949 source=None, descr=None, create_only=False): 950 # first, see if it exists 951 LOG.debug("get_or_create_event") 952 ref_list = object_.get_event_ref_list() 953 LOG.debug("refs: %s", ref_list) 954 # look for a match, and possible correction 955 # except if create_only is true (for events that 956 # can have several occurrences like occupations, residences) 957 if not create_only : 958 for ref in ref_list: 959 event = self.db.get_event_from_handle(ref.ref) 960 LOG.debug(" compare event type %s == %s", int(event.get_type()), 961 type_) 962 if int(event.get_type()) == type_: 963 # Match! Let's update 964 if date: 965 event.set_date_object(date) 966 if place: 967 event.set_place_handle(place.get_handle()) 968 if source: 969 self.find_and_set_citation(event, source) 970 if descr: 971 event.set_description(descr) 972 self.db.commit_event(event, self.trans) 973 LOG.debug(" returning existing event") 974 return (0, event) 975 # else create it: 976 LOG.debug(" creating event") 977 event = Event() 978 if type_: 979 event.set_type(EventType(type_)) 980 if date: 981 event.set_date_object(date) 982 if place: 983 event.set_place_handle(place.get_handle()) 984 if source: 985 self.find_and_set_citation(event, source) 986 if descr: 987 event.set_description(descr) 988 if self.default_tag: 989 event.add_tag(self.default_tag.handle) 990 self.db.add_event(event, self.trans) 991 return (1, event) 992 993 def get_or_create_attribute(self, object_, type_, value_, source=None): 994 "Replaces existing attribute or create it" 995 LOG.debug("get_or_create_attribute") 996 attr_list = object_.get_attribute_list() 997 LOG.debug("refs: %s", attr_list) 998 # remove attributes if it already exists 999 if type_ is None: 1000 type_ = "UNKNOWN" 1001 for attr in attr_list: 1002 if attr.get_type() == type_: 1003 object_.remove_attribute(attr) 1004 # then add it 1005 LOG.debug("adding attribute") 1006 attr = Attribute() 1007 attr.set_type(type_) 1008 attr.set_value(value_) 1009 if source is not None: 1010 new, source = self.get_or_create_source(source) 1011 self.find_and_set_citation(attr, source) 1012 object_.add_attribute(attr) 1013 return (1, attr) 1014 1015 def create_person(self): 1016 """ Used to create a new person we know doesn't exist """ 1017 person = Person() 1018 if self.default_tag: 1019 person.add_tag(self.default_tag.handle) 1020 self.db.add_person(person, self.trans) 1021 self.indi_count += 1 1022 return person 1023 1024 def create_place(self): 1025 """ Used to create a new person we know doesn't exist """ 1026 place = Place() 1027 if self.default_tag: 1028 place.add_tag(self.default_tag.handle) 1029 self.db.add_place(place, self.trans) 1030 self.place_count += 1 1031 return place 1032 1033 def get_or_create_place(self, place_name): 1034 "Return the requested place object tuple-packed with a new indicator." 1035 if place_name.startswith("[") and place_name.endswith("]"): 1036 place = self.lookup("place", place_name) 1037 return (0, place) 1038 LOG.debug("get_or_create_place: looking for: %s", place_name) 1039 for place_handle in self.db.iter_place_handles(): 1040 place = self.db.get_place_from_handle(place_handle) 1041 place_title = place_displayer.display(self.db, place) 1042 if place_title == place_name: 1043 return (0, place) 1044 place = Place() 1045 place.set_title(place_name) 1046 place.name = PlaceName(value=place_name) 1047 self.db.add_place(place, self.trans) 1048 return (1, place) 1049 1050 def get_or_create_source(self, source_text): 1051 "Return the requested source object tuple-packed with a new indicator." 1052 source_list = self.db.get_source_handles(sort_handles=False) 1053 LOG.debug("get_or_create_source: list: %s", source_list) 1054 LOG.debug("get_or_create_source: looking for: %s", source_text) 1055 for source_handle in source_list: 1056 source = self.db.get_source_from_handle(source_handle) 1057 if source.get_title() == source_text: 1058 LOG.debug(" returning existing source") 1059 return (0, source) 1060 LOG.debug(" creating source") 1061 source = Source() 1062 source.set_title(source_text) 1063 self.db.add_source(source, self.trans) 1064 return (1, source) 1065 1066 def find_and_set_citation(self, obj, source): 1067 # look for the source in the existing citations for the object 1068 LOG.debug("find_and_set_citation: looking for source: %s", 1069 source.get_gramps_id()) 1070 for citation_handle in obj.get_citation_list(): 1071 citation = self.db.get_citation_from_handle(citation_handle) 1072 LOG.debug("find_and_set_citation: existing citation: %s", 1073 citation.get_gramps_id()) 1074 poss_source = self.db.get_source_from_handle( 1075 citation.get_reference_handle()) 1076 LOG.debug(" compare source %s == %s", source.get_gramps_id(), 1077 poss_source.get_gramps_id()) 1078 if poss_source.get_handle() == source.get_handle(): 1079 # The source is already cited 1080 LOG.debug(" source already cited") 1081 return 1082 # we couldn't find an appropriate citation, so we have to create one. 1083 citation = Citation() 1084 LOG.debug(" creating citation") 1085 citation.set_reference_handle(source.get_handle()) 1086 self.db.add_citation(citation, self.trans) 1087 LOG.debug(" created citation, citation %s %s" % 1088 (citation, citation.get_gramps_id())) 1089 obj.add_citation(citation.get_handle()) 1090