1# -*- coding: utf-8 -*- 2# Part of Odoo. See LICENSE file for full copyright and licensing details. 3 4""" Models registries. 5 6""" 7from collections import defaultdict, deque 8from collections.abc import Mapping 9from contextlib import closing, contextmanager 10from functools import partial 11from operator import attrgetter 12from weakref import WeakValueDictionary 13import logging 14import os 15import threading 16 17import psycopg2 18 19import odoo 20from .. import SUPERUSER_ID 21from odoo.sql_db import TestCursor 22from odoo.tools import (config, existing_tables, ignore, 23 lazy_classproperty, lazy_property, sql, OrderedSet) 24from odoo.tools.lru import LRU 25 26_logger = logging.getLogger(__name__) 27_schema = logging.getLogger('odoo.schema') 28 29 30class Registry(Mapping): 31 """ Model registry for a particular database. 32 33 The registry is essentially a mapping between model names and model classes. 34 There is one registry instance per database. 35 36 """ 37 _lock = threading.RLock() 38 _saved_lock = None 39 40 # a cache for model classes, indexed by their base classes 41 model_cache = WeakValueDictionary() 42 43 @lazy_classproperty 44 def registries(cls): 45 """ A mapping from database names to registries. """ 46 size = config.get('registry_lru_size', None) 47 if not size: 48 # Size the LRU depending of the memory limits 49 if os.name != 'posix': 50 # cannot specify the memory limit soft on windows... 51 size = 42 52 else: 53 # A registry takes 10MB of memory on average, so we reserve 54 # 10Mb (registry) + 5Mb (working memory) per registry 55 avgsz = 15 * 1024 * 1024 56 size = int(config['limit_memory_soft'] / avgsz) 57 return LRU(size) 58 59 def __new__(cls, db_name): 60 """ Return the registry for the given database name.""" 61 with cls._lock: 62 try: 63 return cls.registries[db_name] 64 except KeyError: 65 return cls.new(db_name) 66 finally: 67 # set db tracker - cleaned up at the WSGI dispatching phase in 68 # odoo.service.wsgi_server.application 69 threading.current_thread().dbname = db_name 70 71 @classmethod 72 def new(cls, db_name, force_demo=False, status=None, update_module=False): 73 """ Create and return a new registry for the given database name. """ 74 with cls._lock: 75 with odoo.api.Environment.manage(): 76 registry = object.__new__(cls) 77 registry.init(db_name) 78 79 # Initializing a registry will call general code which will in 80 # turn call Registry() to obtain the registry being initialized. 81 # Make it available in the registries dictionary then remove it 82 # if an exception is raised. 83 cls.delete(db_name) 84 cls.registries[db_name] = registry 85 try: 86 registry.setup_signaling() 87 # This should be a method on Registry 88 try: 89 odoo.modules.load_modules(registry._db, force_demo, status, update_module) 90 except Exception: 91 odoo.modules.reset_modules_state(db_name) 92 raise 93 except Exception: 94 _logger.error('Failed to load registry') 95 del cls.registries[db_name] 96 raise 97 98 # load_modules() above can replace the registry by calling 99 # indirectly new() again (when modules have to be uninstalled). 100 # Yeah, crazy. 101 registry = cls.registries[db_name] 102 103 registry._init = False 104 registry.ready = True 105 registry.registry_invalidated = bool(update_module) 106 107 return registry 108 109 def init(self, db_name): 110 self.models = {} # model name/model instance mapping 111 self._sql_constraints = set() 112 self._init = True 113 self._assertion_report = odoo.tests.runner.OdooTestResult() 114 self._fields_by_model = None 115 self._ordinary_tables = None 116 self._constraint_queue = deque() 117 self.__cache = LRU(8192) 118 119 # modules fully loaded (maintained during init phase by `loading` module) 120 self._init_modules = set() 121 self.updated_modules = [] # installed/updated modules 122 self.loaded_xmlids = set() 123 124 self.db_name = db_name 125 self._db = odoo.sql_db.db_connect(db_name) 126 127 # cursor for test mode; None means "normal" mode 128 self.test_cr = None 129 self.test_lock = None 130 131 # Indicates that the registry is 132 self.loaded = False # whether all modules are loaded 133 self.ready = False # whether everything is set up 134 135 # Inter-process signaling: 136 # The `base_registry_signaling` sequence indicates the whole registry 137 # must be reloaded. 138 # The `base_cache_signaling sequence` indicates all caches must be 139 # invalidated (i.e. cleared). 140 self.registry_sequence = None 141 self.cache_sequence = None 142 143 # Flags indicating invalidation of the registry or the cache. 144 self.registry_invalidated = False 145 self.cache_invalidated = False 146 147 with closing(self.cursor()) as cr: 148 self.has_unaccent = odoo.modules.db.has_unaccent(cr) 149 150 @classmethod 151 def delete(cls, db_name): 152 """ Delete the registry linked to a given database. """ 153 with cls._lock: 154 if db_name in cls.registries: 155 del cls.registries[db_name] 156 157 @classmethod 158 def delete_all(cls): 159 """ Delete all the registries. """ 160 with cls._lock: 161 cls.registries.clear() 162 163 # 164 # Mapping abstract methods implementation 165 # => mixin provides methods keys, items, values, get, __eq__, and __ne__ 166 # 167 def __len__(self): 168 """ Return the size of the registry. """ 169 return len(self.models) 170 171 def __iter__(self): 172 """ Return an iterator over all model names. """ 173 return iter(self.models) 174 175 def __getitem__(self, model_name): 176 """ Return the model with the given name or raise KeyError if it doesn't exist.""" 177 return self.models[model_name] 178 179 def __call__(self, model_name): 180 """ Same as ``self[model_name]``. """ 181 return self.models[model_name] 182 183 def __setitem__(self, model_name, model): 184 """ Add or replace a model in the registry.""" 185 self.models[model_name] = model 186 187 def __delitem__(self, model_name): 188 """ Remove a (custom) model from the registry. """ 189 del self.models[model_name] 190 # the custom model can inherit from mixins ('mail.thread', ...) 191 for Model in self.models.values(): 192 Model._inherit_children.discard(model_name) 193 194 def descendants(self, model_names, *kinds): 195 """ Return the models corresponding to ``model_names`` and all those 196 that inherit/inherits from them. 197 """ 198 assert all(kind in ('_inherit', '_inherits') for kind in kinds) 199 funcs = [attrgetter(kind + '_children') for kind in kinds] 200 201 models = OrderedSet() 202 queue = deque(model_names) 203 while queue: 204 model = self[queue.popleft()] 205 models.add(model._name) 206 for func in funcs: 207 queue.extend(func(model)) 208 return models 209 210 def load(self, cr, module): 211 """ Load a given module in the registry, and return the names of the 212 modified models. 213 214 At the Python level, the modules are already loaded, but not yet on a 215 per-registry level. This method populates a registry with the given 216 modules, i.e. it instanciates all the classes of a the given module 217 and registers them in the registry. 218 219 """ 220 from .. import models 221 222 # clear cache to ensure consistency, but do not signal it 223 self.__cache.clear() 224 225 lazy_property.reset_all(self) 226 227 # Instantiate registered classes (via the MetaModel automatic discovery 228 # or via explicit constructor call), and add them to the pool. 229 model_names = [] 230 for cls in models.MetaModel.module_to_models.get(module.name, []): 231 # models register themselves in self.models 232 model = cls._build_model(self, cr) 233 model_names.append(model._name) 234 235 return self.descendants(model_names, '_inherit', '_inherits') 236 237 def setup_models(self, cr): 238 """ Complete the setup of models. 239 This must be called after loading modules and before using the ORM. 240 """ 241 env = odoo.api.Environment(cr, SUPERUSER_ID, {}) 242 243 # Uninstall registry hooks. Because of the condition, this only happens 244 # on a fully loaded registry, and not on a registry being loaded. 245 if self.ready: 246 for model in env.values(): 247 model._unregister_hook() 248 249 # clear cache to ensure consistency, but do not signal it 250 self.__cache.clear() 251 252 lazy_property.reset_all(self) 253 self.registry_invalidated = True 254 255 if env.all.tocompute: 256 _logger.error( 257 "Remaining fields to compute before setting up registry: %s", 258 env.all.tocompute, stack_info=True, 259 ) 260 261 # add manual models 262 if self._init_modules: 263 env['ir.model']._add_manual_models() 264 265 # prepare the setup on all models 266 models = list(env.values()) 267 for model in models: 268 model._prepare_setup() 269 270 # do the actual setup from a clean state 271 self._m2m = defaultdict(list) 272 for model in models: 273 model._setup_base() 274 275 for model in models: 276 model._setup_fields() 277 278 for model in models: 279 model._setup_complete() 280 281 # Reinstall registry hooks. Because of the condition, this only happens 282 # on a fully loaded registry, and not on a registry being loaded. 283 if self.ready: 284 for model in env.values(): 285 model._register_hook() 286 env['base'].flush() 287 288 @lazy_property 289 def field_computed(self): 290 """ Return a dict mapping each field to the fields computed by the same method. """ 291 computed = {} 292 for model_name, Model in self.models.items(): 293 groups = defaultdict(list) 294 for field in Model._fields.values(): 295 if field.compute: 296 computed[field] = group = groups[field.compute] 297 group.append(field) 298 for fields in groups.values(): 299 if len({field.compute_sudo for field in fields}) > 1: 300 _logger.warning("%s: inconsistent 'compute_sudo' for computed fields: %s", 301 model_name, ", ".join(field.name for field in fields)) 302 return computed 303 304 @lazy_property 305 def field_triggers(self): 306 # determine field dependencies 307 dependencies = {} 308 for Model in self.models.values(): 309 if Model._abstract: 310 continue 311 for field in Model._fields.values(): 312 # dependencies of custom fields may not exist; ignore that case 313 exceptions = (Exception,) if field.base_field.manual else () 314 with ignore(*exceptions): 315 dependencies[field] = set(field.resolve_depends(self)) 316 317 # determine transitive dependencies 318 def transitive_dependencies(field, seen=[]): 319 if field in seen: 320 return 321 for seq1 in dependencies.get(field, ()): 322 yield seq1 323 for seq2 in transitive_dependencies(seq1[-1], seen + [field]): 324 yield concat(seq1[:-1], seq2) 325 326 def concat(seq1, seq2): 327 if seq1 and seq2: 328 f1, f2 = seq1[-1], seq2[0] 329 if f1.type == 'one2many' and f2.type == 'many2one' and \ 330 f1.model_name == f2.comodel_name and f1.inverse_name == f2.name: 331 return concat(seq1[:-1], seq2[1:]) 332 return seq1 + seq2 333 334 # determine triggers based on transitive dependencies 335 triggers = {} 336 for field in dependencies: 337 for path in transitive_dependencies(field): 338 if path: 339 tree = triggers 340 for label in reversed(path): 341 tree = tree.setdefault(label, {}) 342 tree.setdefault(None, set()).add(field) 343 344 return triggers 345 346 def post_init(self, func, *args, **kwargs): 347 """ Register a function to call at the end of :meth:`~.init_models`. """ 348 self._post_init_queue.append(partial(func, *args, **kwargs)) 349 350 def post_constraint(self, func, *args, **kwargs): 351 """ Call the given function, and delay it if it fails during an upgrade. """ 352 try: 353 if (func, args, kwargs) not in self._constraint_queue: 354 # Module A may try to apply a constraint and fail but another module B inheriting 355 # from Module A may try to reapply the same constraint and succeed, however the 356 # constraint would already be in the _constraint_queue and would be executed again 357 # at the end of the registry cycle, this would fail (already-existing constraint) 358 # and generate an error, therefore a constraint should only be applied if it's 359 # not already marked as "to be applied". 360 func(*args, **kwargs) 361 except Exception as e: 362 if self._is_install: 363 _schema.error(*e.args) 364 else: 365 _schema.info(*e.args) 366 self._constraint_queue.append((func, args, kwargs)) 367 368 def finalize_constraints(self): 369 """ Call the delayed functions from above. """ 370 while self._constraint_queue: 371 func, args, kwargs = self._constraint_queue.popleft() 372 try: 373 func(*args, **kwargs) 374 except Exception as e: 375 _schema.error(*e.args) 376 377 def init_models(self, cr, model_names, context, install=True): 378 """ Initialize a list of models (given by their name). Call methods 379 ``_auto_init`` and ``init`` on each model to create or update the 380 database tables supporting the models. 381 382 The ``context`` may contain the following items: 383 - ``module``: the name of the module being installed/updated, if any; 384 - ``update_custom_fields``: whether custom fields should be updated. 385 """ 386 if not model_names: 387 return 388 389 if 'module' in context: 390 _logger.info('module %s: creating or updating database tables', context['module']) 391 elif context.get('models_to_check', False): 392 _logger.info("verifying fields for every extended model") 393 394 env = odoo.api.Environment(cr, SUPERUSER_ID, context) 395 models = [env[model_name] for model_name in model_names] 396 397 try: 398 self._post_init_queue = deque() 399 self._foreign_keys = {} 400 self._is_install = install 401 402 for model in models: 403 model._auto_init() 404 model.init() 405 406 env['ir.model']._reflect_models(model_names) 407 env['ir.model.fields']._reflect_fields(model_names) 408 env['ir.model.fields.selection']._reflect_selections(model_names) 409 env['ir.model.constraint']._reflect_constraints(model_names) 410 411 self._ordinary_tables = None 412 413 while self._post_init_queue: 414 func = self._post_init_queue.popleft() 415 func() 416 417 self.check_indexes(cr, model_names) 418 self.check_foreign_keys(cr) 419 420 env['base'].flush() 421 422 # make sure all tables are present 423 self.check_tables_exist(cr) 424 425 finally: 426 del self._post_init_queue 427 del self._foreign_keys 428 del self._is_install 429 430 def check_indexes(self, cr, model_names): 431 """ Create or drop column indexes for the given models. """ 432 expected = [ 433 ("%s_%s_index" % (Model._table, field.name), Model._table, field.name, field.index) 434 for model_name in model_names 435 for Model in [self.models[model_name]] 436 if Model._auto and not Model._abstract 437 for field in Model._fields.values() 438 if field.column_type and field.store 439 ] 440 if not expected: 441 return 442 443 cr.execute("SELECT indexname FROM pg_indexes WHERE indexname IN %s", 444 [tuple(row[0] for row in expected)]) 445 existing = {row[0] for row in cr.fetchall()} 446 447 for indexname, tablename, columnname, index in expected: 448 if index and indexname not in existing: 449 try: 450 with cr.savepoint(flush=False): 451 sql.create_index(cr, indexname, tablename, ['"%s"' % columnname]) 452 except psycopg2.OperationalError: 453 _schema.error("Unable to add index for %s", self) 454 elif not index and indexname in existing: 455 _schema.info("Keep unexpected index %s on table %s", indexname, tablename) 456 457 def add_foreign_key(self, table1, column1, table2, column2, ondelete, 458 model, module, force=True): 459 """ Specify an expected foreign key. """ 460 key = (table1, column1) 461 val = (table2, column2, ondelete, model, module) 462 if force: 463 self._foreign_keys[key] = val 464 else: 465 self._foreign_keys.setdefault(key, val) 466 467 def check_foreign_keys(self, cr): 468 """ Create or update the expected foreign keys. """ 469 if not self._foreign_keys: 470 return 471 472 # determine existing foreign keys on the tables 473 query = """ 474 SELECT fk.conname, c1.relname, a1.attname, c2.relname, a2.attname, fk.confdeltype 475 FROM pg_constraint AS fk 476 JOIN pg_class AS c1 ON fk.conrelid = c1.oid 477 JOIN pg_class AS c2 ON fk.confrelid = c2.oid 478 JOIN pg_attribute AS a1 ON a1.attrelid = c1.oid AND fk.conkey[1] = a1.attnum 479 JOIN pg_attribute AS a2 ON a2.attrelid = c2.oid AND fk.confkey[1] = a2.attnum 480 WHERE fk.contype = 'f' AND c1.relname IN %s 481 """ 482 cr.execute(query, [tuple({table for table, column in self._foreign_keys})]) 483 existing = { 484 (table1, column1): (name, table2, column2, deltype) 485 for name, table1, column1, table2, column2, deltype in cr.fetchall() 486 } 487 488 # create or update foreign keys 489 for key, val in self._foreign_keys.items(): 490 table1, column1 = key 491 table2, column2, ondelete, model, module = val 492 deltype = sql._CONFDELTYPES[ondelete.upper()] 493 spec = existing.get(key) 494 if spec is None: 495 sql.add_foreign_key(cr, table1, column1, table2, column2, ondelete) 496 conname = sql.get_foreign_keys(cr, table1, column1, table2, column2, ondelete)[0] 497 model.env['ir.model.constraint']._reflect_constraint(model, conname, 'f', None, module) 498 elif spec[1:] != (table2, column2, deltype): 499 sql.drop_constraint(cr, table1, spec[0]) 500 sql.add_foreign_key(cr, table1, column1, table2, column2, ondelete) 501 conname = sql.get_foreign_keys(cr, table1, column1, table2, column2, ondelete)[0] 502 model.env['ir.model.constraint']._reflect_constraint(model, conname, 'f', None, module) 503 504 def check_tables_exist(self, cr): 505 """ 506 Verify that all tables are present and try to initialize those that are missing. 507 """ 508 env = odoo.api.Environment(cr, SUPERUSER_ID, {}) 509 table2model = { 510 model._table: name 511 for name, model in env.items() 512 if not model._abstract and model.__class__._table_query is None 513 } 514 missing_tables = set(table2model).difference(existing_tables(cr, table2model)) 515 516 if missing_tables: 517 missing = {table2model[table] for table in missing_tables} 518 _logger.info("Models have no table: %s.", ", ".join(missing)) 519 # recreate missing tables 520 for name in missing: 521 _logger.info("Recreate table of model %s.", name) 522 env[name].init() 523 env['base'].flush() 524 # check again, and log errors if tables are still missing 525 missing_tables = set(table2model).difference(existing_tables(cr, table2model)) 526 for table in missing_tables: 527 _logger.error("Model %s has no table.", table2model[table]) 528 529 def _clear_cache(self): 530 """ Clear the cache and mark it as invalidated. """ 531 self.__cache.clear() 532 self.cache_invalidated = True 533 534 def clear_caches(self): 535 """ Clear the caches associated to methods decorated with 536 ``tools.ormcache`` or ``tools.ormcache_multi`` for all the models. 537 """ 538 for model in self.models.values(): 539 model.clear_caches() 540 541 def is_an_ordinary_table(self, model): 542 """ Return whether the given model has an ordinary table. """ 543 if self._ordinary_tables is None: 544 cr = model.env.cr 545 query = """ 546 SELECT c.relname 547 FROM pg_class c 548 JOIN pg_namespace n ON (n.oid = c.relnamespace) 549 WHERE c.relname IN %s 550 AND c.relkind = 'r' 551 AND n.nspname = 'public' 552 """ 553 tables = tuple(m._table for m in self.models.values()) 554 cr.execute(query, [tables]) 555 self._ordinary_tables = {row[0] for row in cr.fetchall()} 556 557 return model._table in self._ordinary_tables 558 559 def setup_signaling(self): 560 """ Setup the inter-process signaling on this registry. """ 561 if self.in_test_mode(): 562 return 563 564 with self.cursor() as cr: 565 # The `base_registry_signaling` sequence indicates when the registry 566 # must be reloaded. 567 # The `base_cache_signaling` sequence indicates when all caches must 568 # be invalidated (i.e. cleared). 569 cr.execute("SELECT sequence_name FROM information_schema.sequences WHERE sequence_name='base_registry_signaling'") 570 if not cr.fetchall(): 571 cr.execute("CREATE SEQUENCE base_registry_signaling INCREMENT BY 1 START WITH 1") 572 cr.execute("SELECT nextval('base_registry_signaling')") 573 cr.execute("CREATE SEQUENCE base_cache_signaling INCREMENT BY 1 START WITH 1") 574 cr.execute("SELECT nextval('base_cache_signaling')") 575 576 cr.execute(""" SELECT base_registry_signaling.last_value, 577 base_cache_signaling.last_value 578 FROM base_registry_signaling, base_cache_signaling""") 579 self.registry_sequence, self.cache_sequence = cr.fetchone() 580 _logger.debug("Multiprocess load registry signaling: [Registry: %s] [Cache: %s]", 581 self.registry_sequence, self.cache_sequence) 582 583 def check_signaling(self): 584 """ Check whether the registry has changed, and performs all necessary 585 operations to update the registry. Return an up-to-date registry. 586 """ 587 if self.in_test_mode(): 588 return self 589 590 with closing(self.cursor()) as cr: 591 cr.execute(""" SELECT base_registry_signaling.last_value, 592 base_cache_signaling.last_value 593 FROM base_registry_signaling, base_cache_signaling""") 594 r, c = cr.fetchone() 595 _logger.debug("Multiprocess signaling check: [Registry - %s -> %s] [Cache - %s -> %s]", 596 self.registry_sequence, r, self.cache_sequence, c) 597 # Check if the model registry must be reloaded 598 if self.registry_sequence != r: 599 _logger.info("Reloading the model registry after database signaling.") 600 self = Registry.new(self.db_name) 601 # Check if the model caches must be invalidated. 602 elif self.cache_sequence != c: 603 _logger.info("Invalidating all model caches after database signaling.") 604 self.clear_caches() 605 606 # prevent re-signaling the clear_caches() above, or any residual one that 607 # would be inherited from the master process (first request in pre-fork mode) 608 self.cache_invalidated = False 609 610 self.registry_sequence = r 611 self.cache_sequence = c 612 613 return self 614 615 def signal_changes(self): 616 """ Notifies other processes if registry or cache has been invalidated. """ 617 if self.registry_invalidated and not self.in_test_mode(): 618 _logger.info("Registry changed, signaling through the database") 619 with closing(self.cursor()) as cr: 620 cr.execute("select nextval('base_registry_signaling')") 621 self.registry_sequence = cr.fetchone()[0] 622 623 # no need to notify cache invalidation in case of registry invalidation, 624 # because reloading the registry implies starting with an empty cache 625 elif self.cache_invalidated and not self.in_test_mode(): 626 _logger.info("At least one model cache has been invalidated, signaling through the database.") 627 with closing(self.cursor()) as cr: 628 cr.execute("select nextval('base_cache_signaling')") 629 self.cache_sequence = cr.fetchone()[0] 630 631 self.registry_invalidated = False 632 self.cache_invalidated = False 633 634 def reset_changes(self): 635 """ Reset the registry and cancel all invalidations. """ 636 if self.registry_invalidated: 637 with closing(self.cursor()) as cr: 638 self.setup_models(cr) 639 self.registry_invalidated = False 640 if self.cache_invalidated: 641 self.__cache.clear() 642 self.cache_invalidated = False 643 644 @contextmanager 645 def manage_changes(self): 646 """ Context manager to signal/discard registry and cache invalidations. """ 647 try: 648 yield self 649 self.signal_changes() 650 except Exception: 651 self.reset_changes() 652 raise 653 654 def in_test_mode(self): 655 """ Test whether the registry is in 'test' mode. """ 656 return self.test_cr is not None 657 658 def enter_test_mode(self, cr): 659 """ Enter the 'test' mode, where one cursor serves several requests. """ 660 assert self.test_cr is None 661 self.test_cr = cr 662 self.test_lock = threading.RLock() 663 assert Registry._saved_lock is None 664 Registry._saved_lock = Registry._lock 665 Registry._lock = DummyRLock() 666 667 def leave_test_mode(self): 668 """ Leave the test mode. """ 669 assert self.test_cr is not None 670 self.test_cr = None 671 self.test_lock = None 672 assert Registry._saved_lock is not None 673 Registry._lock = Registry._saved_lock 674 Registry._saved_lock = None 675 676 def cursor(self): 677 """ Return a new cursor for the database. The cursor itself may be used 678 as a context manager to commit/rollback and close automatically. 679 """ 680 if self.test_cr is not None: 681 # When in test mode, we use a proxy object that uses 'self.test_cr' 682 # underneath. 683 return TestCursor(self.test_cr, self.test_lock) 684 return self._db.cursor() 685 686 687class DummyRLock(object): 688 """ Dummy reentrant lock, to be used while running rpc and js tests """ 689 def acquire(self): 690 pass 691 def release(self): 692 pass 693 def __enter__(self): 694 self.acquire() 695 def __exit__(self, type, value, traceback): 696 self.release() 697