1# -*- coding: utf-8 -*- 2# Part of Odoo. See LICENSE file for full copyright and licensing details. 3 4import functools 5import itertools 6 7import psycopg2 8import pytz 9 10from odoo import api, fields, models, _ 11from odoo.tools import ustr 12 13REFERENCING_FIELDS = {None, 'id', '.id'} 14def only_ref_fields(record): 15 return {k: v for k, v in record.items() if k in REFERENCING_FIELDS} 16def exclude_ref_fields(record): 17 return {k: v for k, v in record.items() if k not in REFERENCING_FIELDS} 18 19CREATE = lambda values: (0, False, values) 20UPDATE = lambda id, values: (1, id, values) 21DELETE = lambda id: (2, id, False) 22FORGET = lambda id: (3, id, False) 23LINK_TO = lambda id: (4, id, False) 24DELETE_ALL = lambda: (5, False, False) 25REPLACE_WITH = lambda ids: (6, False, ids) 26 27class ImportWarning(Warning): 28 """ Used to send warnings upwards the stack during the import process """ 29 pass 30 31class ConversionNotFound(ValueError): 32 pass 33 34class IrFieldsConverter(models.AbstractModel): 35 _name = 'ir.fields.converter' 36 _description = 'Fields Converter' 37 38 @api.model 39 def _format_import_error(self, error_type, error_msg, error_params=(), error_args=None): 40 # sanitize error params for later formatting by the import system 41 sanitize = lambda p: p.replace('%', '%%') if isinstance(p, str) else p 42 if error_params: 43 if isinstance(error_params, str): 44 error_params = sanitize(error_params) 45 elif isinstance(error_params, dict): 46 error_params = {k: sanitize(v) for k, v in error_params.items()} 47 elif isinstance(error_params, tuple): 48 error_params = tuple(sanitize(v) for v in error_params) 49 return error_type(error_msg % error_params, error_args) 50 51 @api.model 52 def for_model(self, model, fromtype=str): 53 """ Returns a converter object for the model. A converter is a 54 callable taking a record-ish (a dictionary representing an odoo 55 record with values of typetag ``fromtype``) and returning a converted 56 records matching what :meth:`odoo.osv.orm.Model.write` expects. 57 58 :param model: :class:`odoo.osv.orm.Model` for the conversion base 59 :returns: a converter callable 60 :rtype: (record: dict, logger: (field, error) -> None) -> dict 61 """ 62 # make sure model is new api 63 model = self.env[model._name] 64 65 converters = { 66 name: self.to_field(model, field, fromtype) 67 for name, field in model._fields.items() 68 } 69 70 def fn(record, log): 71 converted = {} 72 for field, value in record.items(): 73 if field in REFERENCING_FIELDS: 74 continue 75 if not value: 76 converted[field] = False 77 continue 78 try: 79 converted[field], ws = converters[field](value) 80 for w in ws: 81 if isinstance(w, str): 82 # wrap warning string in an ImportWarning for 83 # uniform handling 84 w = ImportWarning(w) 85 log(field, w) 86 except (UnicodeEncodeError, UnicodeDecodeError) as e: 87 log(field, ValueError(str(e))) 88 except ValueError as e: 89 log(field, e) 90 return converted 91 92 return fn 93 94 @api.model 95 def to_field(self, model, field, fromtype=str): 96 """ Fetches a converter for the provided field object, from the 97 specified type. 98 99 A converter is simply a callable taking a value of type ``fromtype`` 100 (or a composite of ``fromtype``, e.g. list or dict) and returning a 101 value acceptable for a write() on the field ``field``. 102 103 By default, tries to get a method on itself with a name matching the 104 pattern ``_$fromtype_to_$field.type`` and returns it. 105 106 Converter callables can either return a value and a list of warnings 107 to their caller or raise ``ValueError``, which will be interpreted as a 108 validation & conversion failure. 109 110 ValueError can have either one or two parameters. The first parameter 111 is mandatory, **must** be a unicode string and will be used as the 112 user-visible message for the error (it should be translatable and 113 translated). It can contain a ``field`` named format placeholder so the 114 caller can inject the field's translated, user-facing name (@string). 115 116 The second parameter is optional and, if provided, must be a mapping. 117 This mapping will be merged into the error dictionary returned to the 118 client. 119 120 If a converter can perform its function but has to make assumptions 121 about the data, it can send a warning to the user through adding an 122 instance of :class:`~.ImportWarning` to the second value 123 it returns. The handling of a warning at the upper levels is the same 124 as ``ValueError`` above. 125 126 :param field: field object to generate a value for 127 :type field: :class:`odoo.fields.Field` 128 :param fromtype: type to convert to something fitting for ``field`` 129 :type fromtype: type | str 130 :param context: odoo request context 131 :return: a function (fromtype -> field.write_type), if a converter is found 132 :rtype: Callable | None 133 """ 134 assert isinstance(fromtype, (type, str)) 135 # FIXME: return None 136 typename = fromtype.__name__ if isinstance(fromtype, type) else fromtype 137 converter = getattr(self, '_%s_to_%s' % (typename, field.type), None) 138 if not converter: 139 return None 140 return functools.partial(converter, model, field) 141 142 @api.model 143 def _str_to_boolean(self, model, field, value): 144 # all translatables used for booleans 145 true, yes, false, no = _(u"true"), _(u"yes"), _(u"false"), _(u"no") 146 # potentially broken casefolding? What about locales? 147 trues = set(word.lower() for word in itertools.chain( 148 [u'1', u"true", u"yes"], # don't use potentially translated values 149 self._get_translations(['code'], u"true"), 150 self._get_translations(['code'], u"yes"), 151 )) 152 if value.lower() in trues: 153 return True, [] 154 155 # potentially broken casefolding? What about locales? 156 falses = set(word.lower() for word in itertools.chain( 157 [u'', u"0", u"false", u"no"], 158 self._get_translations(['code'], u"false"), 159 self._get_translations(['code'], u"no"), 160 )) 161 if value.lower() in falses: 162 return False, [] 163 164 return True, [self._format_import_error( 165 ImportWarning, 166 _(u"Unknown value '%s' for boolean field '%%(field)s', assuming '%s'"), 167 (value, yes), 168 {'moreinfo': _(u"Use '1' for yes and '0' for no")} 169 )] 170 171 @api.model 172 def _str_to_integer(self, model, field, value): 173 try: 174 return int(value), [] 175 except ValueError: 176 raise self._format_import_error( 177 ValueError, 178 _(u"'%s' does not seem to be an integer for field '%%(field)s'"), 179 value 180 ) 181 182 @api.model 183 def _str_to_float(self, model, field, value): 184 try: 185 return float(value), [] 186 except ValueError: 187 raise self._format_import_error( 188 ValueError, 189 _(u"'%s' does not seem to be a number for field '%%(field)s'"), 190 value 191 ) 192 193 _str_to_monetary = _str_to_float 194 195 @api.model 196 def _str_id(self, model, field, value): 197 return value, [] 198 199 _str_to_reference = _str_to_char = _str_to_text = _str_to_binary = _str_to_html = _str_id 200 201 @api.model 202 def _str_to_date(self, model, field, value): 203 try: 204 parsed_value = fields.Date.from_string(value) 205 return fields.Date.to_string(parsed_value), [] 206 except ValueError: 207 raise self._format_import_error( 208 ValueError, 209 _(u"'%s' does not seem to be a valid date for field '%%(field)s'"), 210 value, 211 {'moreinfo': _(u"Use the format '%s'", u"2012-12-31")} 212 ) 213 214 @api.model 215 def _input_tz(self): 216 # if there's a tz in context, try to use that 217 if self._context.get('tz'): 218 try: 219 return pytz.timezone(self._context['tz']) 220 except pytz.UnknownTimeZoneError: 221 pass 222 223 # if the current user has a tz set, try to use that 224 user = self.env.user 225 if user.tz: 226 try: 227 return pytz.timezone(user.tz) 228 except pytz.UnknownTimeZoneError: 229 pass 230 231 # fallback if no tz in context or on user: UTC 232 return pytz.UTC 233 234 @api.model 235 def _str_to_datetime(self, model, field, value): 236 try: 237 parsed_value = fields.Datetime.from_string(value) 238 except ValueError: 239 raise self._format_import_error( 240 ValueError, 241 _(u"'%s' does not seem to be a valid datetime for field '%%(field)s'"), 242 value, 243 {'moreinfo': _(u"Use the format '%s'", u"2012-12-31 23:59:59")} 244 ) 245 246 input_tz = self._input_tz()# Apply input tz to the parsed naive datetime 247 dt = input_tz.localize(parsed_value, is_dst=False) 248 # And convert to UTC before reformatting for writing 249 return fields.Datetime.to_string(dt.astimezone(pytz.UTC)), [] 250 251 @api.model 252 def _get_translations(self, types, src): 253 types = tuple(types) 254 # Cache translations so they don't have to be reloaded from scratch on 255 # every row of the file 256 tnx_cache = self._cr.cache.setdefault(self._name, {}) 257 if tnx_cache.setdefault(types, {}) and src in tnx_cache[types]: 258 return tnx_cache[types][src] 259 260 Translations = self.env['ir.translation'] 261 tnx = Translations.search([('type', 'in', types), ('src', '=', src)]) 262 result = tnx_cache[types][src] = [t.value for t in tnx if t.value is not False] 263 return result 264 265 @api.model 266 def _str_to_selection(self, model, field, value): 267 # get untranslated values 268 env = self.with_context(lang=None).env 269 selection = field.get_description(env)['selection'] 270 271 for item, label in selection: 272 label = ustr(label) 273 labels = [label] + self._get_translations(('selection', 'model', 'code'), label) 274 if value == str(item) or value in labels: 275 return item, [] 276 277 raise self._format_import_error( 278 ValueError, 279 _(u"Value '%s' not found in selection field '%%(field)s'"), 280 value, 281 {'moreinfo': [_label or str(item) for item, _label in selection if _label or item]} 282 ) 283 284 @api.model 285 def db_id_for(self, model, field, subfield, value): 286 """ Finds a database id for the reference ``value`` in the referencing 287 subfield ``subfield`` of the provided field of the provided model. 288 289 :param model: model to which the field belongs 290 :param field: relational field for which references are provided 291 :param subfield: a relational subfield allowing building of refs to 292 existing records: ``None`` for a name_get/name_search, 293 ``id`` for an external id and ``.id`` for a database 294 id 295 :param value: value of the reference to match to an actual record 296 :param context: OpenERP request context 297 :return: a pair of the matched database identifier (if any), the 298 translated user-readable name for the field and the list of 299 warnings 300 :rtype: (ID|None, unicode, list) 301 """ 302 # the function 'flush' comes from BaseModel.load(), and forces the 303 # creation/update of former records (batch creation) 304 flush = self._context.get('import_flush', lambda **kw: None) 305 306 id = None 307 warnings = [] 308 error_msg = '' 309 action = { 310 'name': 'Possible Values', 311 'type': 'ir.actions.act_window', 'target': 'new', 312 'view_mode': 'tree,form', 313 'views': [(False, 'list'), (False, 'form')], 314 'context': {'create': False}, 315 'help': _(u"See all possible values")} 316 if subfield is None: 317 action['res_model'] = field.comodel_name 318 elif subfield in ('id', '.id'): 319 action['res_model'] = 'ir.model.data' 320 action['domain'] = [('model', '=', field.comodel_name)] 321 322 RelatedModel = self.env[field.comodel_name] 323 if subfield == '.id': 324 field_type = _(u"database id") 325 if isinstance(value, str) and not self._str_to_boolean(model, field, value)[0]: 326 return False, field_type, warnings 327 try: tentative_id = int(value) 328 except ValueError: tentative_id = value 329 try: 330 if RelatedModel.search([('id', '=', tentative_id)]): 331 id = tentative_id 332 except psycopg2.DataError: 333 # type error 334 raise self._format_import_error( 335 ValueError, 336 _(u"Invalid database id '%s' for the field '%%(field)s'"), 337 value, 338 {'moreinfo': action}) 339 elif subfield == 'id': 340 field_type = _(u"external id") 341 if not self._str_to_boolean(model, field, value)[0]: 342 return False, field_type, warnings 343 if '.' in value: 344 xmlid = value 345 else: 346 xmlid = "%s.%s" % (self._context.get('_import_current_module', ''), value) 347 flush(xml_id=xmlid) 348 id = self._xmlid_to_record_id(xmlid, RelatedModel) 349 elif subfield is None: 350 field_type = _(u"name") 351 if value == '': 352 return False, field_type, warnings 353 flush(model=field.comodel_name) 354 ids = RelatedModel.name_search(name=value, operator='=') 355 if ids: 356 if len(ids) > 1: 357 warnings.append(ImportWarning( 358 _(u"Found multiple matches for field '%%(field)s' (%d matches)") 359 % (len(ids)))) 360 id, _name = ids[0] 361 else: 362 name_create_enabled_fields = self.env.context.get('name_create_enabled_fields') or {} 363 if name_create_enabled_fields.get(field.name): 364 try: 365 id, _name = RelatedModel.name_create(name=value) 366 except (Exception, psycopg2.IntegrityError): 367 error_msg = _(u"Cannot create new '%s' records from their name alone. Please create those records manually and try importing again.", RelatedModel._description) 368 else: 369 raise self._format_import_error( 370 Exception, 371 _(u"Unknown sub-field '%s'"), 372 subfield 373 ) 374 375 if id is None: 376 if error_msg: 377 message = _("No matching record found for %(field_type)s '%(value)s' in field '%%(field)s' and the following error was encountered when we attempted to create one: %(error_message)s") 378 else: 379 message = _("No matching record found for %(field_type)s '%(value)s' in field '%%(field)s'") 380 raise self._format_import_error( 381 ValueError, 382 message, 383 {'field_type': field_type, 'value': value, 'error_message': error_msg}, 384 {'moreinfo': action}) 385 return id, field_type, warnings 386 387 def _xmlid_to_record_id(self, xmlid, model): 388 """ Return the record id corresponding to the given external id, 389 provided that the record actually exists; otherwise return ``None``. 390 """ 391 import_cache = self.env.context.get('import_cache', {}) 392 result = import_cache.get(xmlid) 393 394 if not result: 395 module, name = xmlid.split('.', 1) 396 query = """ 397 SELECT d.model, d.res_id 398 FROM ir_model_data d 399 JOIN "{}" r ON d.res_id = r.id 400 WHERE d.module = %s AND d.name = %s 401 """.format(model._table) 402 self.env.cr.execute(query, [module, name]) 403 result = self.env.cr.fetchone() 404 405 if result: 406 res_model, res_id = import_cache[xmlid] = result 407 if res_model != model._name: 408 MSG = "Invalid external ID %s: expected model %r, found %r" 409 raise ValueError(MSG % (xmlid, model._name, res_model)) 410 return res_id 411 412 def _referencing_subfield(self, record): 413 """ Checks the record for the subfields allowing referencing (an 414 existing record in an other table), errors out if it finds potential 415 conflicts (multiple referencing subfields) or non-referencing subfields 416 returns the name of the correct subfield. 417 418 :param record: 419 :return: the record subfield to use for referencing and a list of warnings 420 :rtype: str, list 421 """ 422 # Can import by name_get, external id or database id 423 fieldset = set(record) 424 if fieldset - REFERENCING_FIELDS: 425 raise ValueError( 426 _(u"Can not create Many-To-One records indirectly, import the field separately")) 427 if len(fieldset) > 1: 428 raise ValueError( 429 _(u"Ambiguous specification for field '%(field)s', only provide one of name, external id or database id")) 430 431 # only one field left possible, unpack 432 [subfield] = fieldset 433 return subfield, [] 434 435 @api.model 436 def _str_to_many2one(self, model, field, values): 437 # Should only be one record, unpack 438 [record] = values 439 440 subfield, w1 = self._referencing_subfield(record) 441 442 id, _, w2 = self.db_id_for(model, field, subfield, record[subfield]) 443 return id, w1 + w2 444 445 @api.model 446 def _str_to_many2one_reference(self, model, field, value): 447 return self._str_to_integer(model, field, value) 448 449 @api.model 450 def _str_to_many2many(self, model, field, value): 451 [record] = value 452 453 subfield, warnings = self._referencing_subfield(record) 454 455 ids = [] 456 for reference in record[subfield].split(','): 457 id, _, ws = self.db_id_for(model, field, subfield, reference) 458 ids.append(id) 459 warnings.extend(ws) 460 461 if self._context.get('update_many2many'): 462 return [LINK_TO(id) for id in ids], warnings 463 else: 464 return [REPLACE_WITH(ids)], warnings 465 466 @api.model 467 def _str_to_one2many(self, model, field, records): 468 name_create_enabled_fields = self._context.get('name_create_enabled_fields') or {} 469 prefix = field.name + '/' 470 relative_name_create_enabled_fields = { 471 k[len(prefix):]: v 472 for k, v in name_create_enabled_fields.items() 473 if k.startswith(prefix) 474 } 475 commands = [] 476 warnings = [] 477 478 if len(records) == 1 and exclude_ref_fields(records[0]) == {}: 479 # only one row with only ref field, field=ref1,ref2,ref3 as in 480 # m2o/m2m 481 record = records[0] 482 subfield, ws = self._referencing_subfield(record) 483 warnings.extend(ws) 484 # transform [{subfield:ref1,ref2,ref3}] into 485 # [{subfield:ref1},{subfield:ref2},{subfield:ref3}] 486 records = ({subfield:item} for item in record[subfield].split(',')) 487 488 def log(f, exception): 489 if not isinstance(exception, Warning): 490 current_field_name = self.env[field.comodel_name]._fields[f].string 491 arg0 = exception.args[0] % {'field': '%(field)s/' + current_field_name} 492 exception.args = (arg0, *exception.args[1:]) 493 raise exception 494 warnings.append(exception) 495 496 convert = self.with_context(name_create_enabled_fields=relative_name_create_enabled_fields).for_model(self.env[field.comodel_name]) 497 498 for record in records: 499 id = None 500 refs = only_ref_fields(record) 501 writable = convert(exclude_ref_fields(record), log) 502 if refs: 503 subfield, w1 = self._referencing_subfield(refs) 504 warnings.extend(w1) 505 try: 506 id, _, w2 = self.db_id_for(model, field, subfield, record[subfield]) 507 warnings.extend(w2) 508 except ValueError: 509 if subfield != 'id': 510 raise 511 writable['id'] = record['id'] 512 513 if id: 514 commands.append(LINK_TO(id)) 515 commands.append(UPDATE(id, writable)) 516 else: 517 commands.append(CREATE(writable)) 518 519 return commands, warnings 520 521class O2MIdMapper(models.AbstractModel): 522 """ 523 Updates the base class to support setting xids directly in create by 524 providing an "id" key (otherwise stripped by create) during an import 525 (which should strip 'id' from the input data anyway) 526 """ 527 _inherit = 'base' 528 529 # sadly _load_records_create is only called for the toplevel record so we 530 # can't hook into that 531 @api.model_create_multi 532 @api.returns('self', lambda value: value.id) 533 def create(self, vals_list): 534 recs = super().create(vals_list) 535 536 import_module = self.env.context.get('_import_current_module') 537 if not import_module: # not an import -> bail 538 return recs 539 noupdate = self.env.context.get('noupdate', False) 540 541 xids = (v.get('id') for v in vals_list) 542 self.env['ir.model.data']._update_xmlids([ 543 { 544 'xml_id': xid if '.' in xid else ('%s.%s' % (import_module, xid)), 545 'record': rec, 546 # note: this is not used when updating o2ms above... 547 'noupdate': noupdate, 548 } 549 for rec, xid in zip(recs, xids) 550 if xid and isinstance(xid, str) 551 ]) 552 553 return recs 554