From 8db67a09a0ccfe77e193d2ab0d251ee7d1d0a00e Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Sat, 7 Nov 2020 15:40:24 +0100 Subject: [PATCH] [MOVE] Port patches and framework from openupgrade project Co-authored-by: Stefan Rijnhart --- upgrade_analysis/odoo_patch/__init__.py | 3 + .../odoo_patch/addons/__init__.py | 3 + .../odoo_patch/addons/mrp/__init__.py | 20 + .../addons/point_of_sale/__init__.py | 1 + .../addons/point_of_sale/models/__init__.py | 1 + .../addons/point_of_sale/models/pos_config.py | 21 + .../odoo_patch/addons/stock/__init__.py | 17 + upgrade_analysis/odoo_patch/odoo/__init__.py | 5 + upgrade_analysis/odoo_patch/odoo/http.py | 32 + upgrade_analysis/odoo_patch/odoo/models.py | 179 ++++++ .../odoo_patch/odoo/modules/__init__.py | 12 + .../odoo_patch/odoo/modules/graph.py | 108 ++++ .../odoo_patch/odoo/modules/loading.py | 556 ++++++++++++++++++ .../odoo_patch/odoo/modules/migration.py | 118 ++++ .../odoo_patch/odoo/modules/registry.py | 58 ++ .../odoo_patch/odoo/service/__init__.py | 4 + .../odoo_patch/odoo/service/server.py | 71 +++ .../odoo_patch/odoo/tools/__init__.py | 2 + .../odoo_patch/odoo/tools/convert.py | 23 + .../odoo_patch/odoo/tools/view_validation.py | 29 + upgrade_analysis/openupgrade_loading.py | 318 ++++++++++ upgrade_analysis/openupgrade_log.py | 60 ++ 22 files changed, 1641 insertions(+) create mode 100644 upgrade_analysis/odoo_patch/__init__.py create mode 100644 upgrade_analysis/odoo_patch/addons/__init__.py create mode 100644 upgrade_analysis/odoo_patch/addons/mrp/__init__.py create mode 100644 upgrade_analysis/odoo_patch/addons/point_of_sale/__init__.py create mode 100644 upgrade_analysis/odoo_patch/addons/point_of_sale/models/__init__.py create mode 100644 upgrade_analysis/odoo_patch/addons/point_of_sale/models/pos_config.py create mode 100644 upgrade_analysis/odoo_patch/addons/stock/__init__.py create mode 100644 upgrade_analysis/odoo_patch/odoo/__init__.py create mode 100644 upgrade_analysis/odoo_patch/odoo/http.py create mode 100644 upgrade_analysis/odoo_patch/odoo/models.py create mode 100644 upgrade_analysis/odoo_patch/odoo/modules/__init__.py create mode 100644 upgrade_analysis/odoo_patch/odoo/modules/graph.py create mode 100644 upgrade_analysis/odoo_patch/odoo/modules/loading.py create mode 100644 upgrade_analysis/odoo_patch/odoo/modules/migration.py create mode 100644 upgrade_analysis/odoo_patch/odoo/modules/registry.py create mode 100644 upgrade_analysis/odoo_patch/odoo/service/__init__.py create mode 100644 upgrade_analysis/odoo_patch/odoo/service/server.py create mode 100644 upgrade_analysis/odoo_patch/odoo/tools/__init__.py create mode 100644 upgrade_analysis/odoo_patch/odoo/tools/convert.py create mode 100644 upgrade_analysis/odoo_patch/odoo/tools/view_validation.py create mode 100644 upgrade_analysis/openupgrade_loading.py create mode 100644 upgrade_analysis/openupgrade_log.py diff --git a/upgrade_analysis/odoo_patch/__init__.py b/upgrade_analysis/odoo_patch/__init__.py new file mode 100644 index 000000000..1fd6e167c --- /dev/null +++ b/upgrade_analysis/odoo_patch/__init__.py @@ -0,0 +1,3 @@ +from . import odoo +from . import addons + diff --git a/upgrade_analysis/odoo_patch/addons/__init__.py b/upgrade_analysis/odoo_patch/addons/__init__.py new file mode 100644 index 000000000..e5aa886ba --- /dev/null +++ b/upgrade_analysis/odoo_patch/addons/__init__.py @@ -0,0 +1,3 @@ +from . import mrp +from . import stock +from . import point_of_sale diff --git a/upgrade_analysis/odoo_patch/addons/mrp/__init__.py b/upgrade_analysis/odoo_patch/addons/mrp/__init__.py new file mode 100644 index 000000000..f7b8b8694 --- /dev/null +++ b/upgrade_analysis/odoo_patch/addons/mrp/__init__.py @@ -0,0 +1,20 @@ +from odoo.addons import mrp + + +def _pre_init_mrp(cr): + """ Allow installing MRP in databases with large stock.move table (>1M records) + - Creating the computed+stored field stock_move.is_done is terribly slow with the ORM and + leads to "Out of Memory" crashes + """ + # + # don't try to add 'is_done' column, because it will fail + # when executing the generation of records, in the openupgrade_records + # module. + # cr.execute("""ALTER TABLE "stock_move" ADD COLUMN "is_done" bool;""") + # cr.execute("""UPDATE stock_move + # SET is_done=COALESCE(state in ('done', 'cancel'), FALSE);""") + pass + # + + +mrp._pre_init_mrp = _pre_init_mrp diff --git a/upgrade_analysis/odoo_patch/addons/point_of_sale/__init__.py b/upgrade_analysis/odoo_patch/addons/point_of_sale/__init__.py new file mode 100644 index 000000000..0650744f6 --- /dev/null +++ b/upgrade_analysis/odoo_patch/addons/point_of_sale/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/upgrade_analysis/odoo_patch/addons/point_of_sale/models/__init__.py b/upgrade_analysis/odoo_patch/addons/point_of_sale/models/__init__.py new file mode 100644 index 000000000..db8634ade --- /dev/null +++ b/upgrade_analysis/odoo_patch/addons/point_of_sale/models/__init__.py @@ -0,0 +1 @@ +from . import pos_config diff --git a/upgrade_analysis/odoo_patch/addons/point_of_sale/models/pos_config.py b/upgrade_analysis/odoo_patch/addons/point_of_sale/models/pos_config.py new file mode 100644 index 000000000..ac0f5dc5a --- /dev/null +++ b/upgrade_analysis/odoo_patch/addons/point_of_sale/models/pos_config.py @@ -0,0 +1,21 @@ +from odoo import api +from odoo.addons.point_of_sale.models.pos_config import PosConfig + +if True: + + @api.model + def post_install_pos_localisation(self, companies=False): + # + # don't try to setup_defaults, because it will fail + # when executing the generation of records, in the openupgrade_records + # module. + # self = self.sudo() + # if not companies: + # companies = self.env['res.company'].search([]) + # for company in companies.filtered('chart_template_id'): + # pos_configs = self.search([('company_id', '=', company.id)]) + # pos_configs.setup_defaults(company) + pass + # + +PosConfig.post_install_pos_localisation = post_install_pos_localisation diff --git a/upgrade_analysis/odoo_patch/addons/stock/__init__.py b/upgrade_analysis/odoo_patch/addons/stock/__init__.py new file mode 100644 index 000000000..b66d7f484 --- /dev/null +++ b/upgrade_analysis/odoo_patch/addons/stock/__init__.py @@ -0,0 +1,17 @@ +from odoo.addons import stock + + +def pre_init_hook(cr): + # + # don't uninstall data as this breaks the analysis + # Origin of this code is https://github.com/odoo/odoo/issues/22243 + # env = api.Environment(cr, SUPERUSER_ID, {}) + # env['ir.model.data'].search([ + # ('model', 'like', '%stock%'), + # ('module', '=', 'stock') + # ]).unlink() + pass + # + + +stock.pre_init_hook = pre_init_hook diff --git a/upgrade_analysis/odoo_patch/odoo/__init__.py b/upgrade_analysis/odoo_patch/odoo/__init__.py new file mode 100644 index 000000000..5629ec98c --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo/__init__.py @@ -0,0 +1,5 @@ +from . import modules +from . import service +from . import tools +from . import http +from . import models diff --git a/upgrade_analysis/odoo_patch/odoo/http.py b/upgrade_analysis/odoo_patch/odoo/http.py new file mode 100644 index 000000000..e11c558fb --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo/http.py @@ -0,0 +1,32 @@ +# flake8: noqa +# pylint: skip-file + +import odoo +from odoo.service import security +from odoo.http import SessionExpiredException, request, OpenERPSession + +if True: + def _check_security(self): + """ + Check the current authentication parameters to know if those are still + valid. This method should be called at each request. If the + authentication fails, a :exc:`SessionExpiredException` is raised. + """ + if not self.db or not self.uid: + raise SessionExpiredException("Session expired") + # We create our own environment instead of the request's one. + # to avoid creating it without the uid since request.uid isn't set yet + env = odoo.api.Environment(request.cr, self.uid, self.context) + # here we check if the session is still valid + if not security.check_session(self, env): + # + # When asking openupgrade_records to generate records + # over jsonrpc, a query on res_users in the call above locks this + # table for the sql operations that are triggered by the + # reinstallation of the base module + env.cr.rollback() + # + raise SessionExpiredException("Session expired") + + +OpenERPSession.check_security = _check_security diff --git a/upgrade_analysis/odoo_patch/odoo/models.py b/upgrade_analysis/odoo_patch/odoo/models.py new file mode 100644 index 000000000..ee09595fb --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo/models.py @@ -0,0 +1,179 @@ +# flake8: noqa +# pylint: skip-file + +import odoo +import psycopg2 +from odoo import _ +from odoo.models import fix_import_export_id_paths, BaseModel, _logger +from odoo.addons.openupgrade_framework.openupgrade import openupgrade_log + + +if True: + def _load(self, fields, data): + """ + Attempts to load the data matrix, and returns a list of ids (or + ``False`` if there was an error and no id could be generated) and a + list of messages. + + The ids are those of the records created and saved (in database), in + the same order they were extracted from the file. They can be passed + directly to :meth:`~read` + + :param fields: list of fields to import, at the same index as the corresponding data + :type fields: list(str) + :param data: row-major matrix of data to import + :type data: list(list(str)) + :returns: {ids: list(int)|False, messages: [Message][, lastrow: int]} + """ + self.flush() + + # determine values of mode, current_module and noupdate + mode = self._context.get('mode', 'init') + current_module = self._context.get('module', '__import__') + noupdate = self._context.get('noupdate', False) + # add current module in context for the conversion of xml ids + self = self.with_context(_import_current_module=current_module) + + cr = self._cr + cr.execute('SAVEPOINT model_load') + + fields = [fix_import_export_id_paths(f) for f in fields] + fg = self.fields_get() + + ids = [] + messages = [] + ModelData = self.env['ir.model.data'] + + # list of (xid, vals, info) for records to be created in batch + batch = [] + batch_xml_ids = set() + # models in which we may have created / modified data, therefore might + # require flushing in order to name_search: the root model and any + # o2m + creatable_models = {self._name} + for field_path in fields: + if field_path[0] in (None, 'id', '.id'): + continue + model_fields = self._fields + if isinstance(model_fields[field_path[0]], odoo.fields.Many2one): + # this only applies for toplevel m2o (?) fields + if field_path[0] in (self.env.context.get('name_create_enabled_fieds') or {}): + creatable_models.add(model_fields[field_path[0]].comodel_name) + for field_name in field_path: + if field_name in (None, 'id', '.id'): + break + + if isinstance(model_fields[field_name], odoo.fields.One2many): + comodel = model_fields[field_name].comodel_name + creatable_models.add(comodel) + model_fields = self.env[comodel]._fields + + def flush(*, xml_id=None, model=None): + if not batch: + return + + assert not (xml_id and model), \ + "flush can specify *either* an external id or a model, not both" + + if xml_id and xml_id not in batch_xml_ids: + if xml_id not in self.env: + return + if model and model not in creatable_models: + return + + data_list = [ + dict(xml_id=xid, values=vals, info=info, noupdate=noupdate) + for xid, vals, info in batch + ] + batch.clear() + batch_xml_ids.clear() + + # try to create in batch + try: + with cr.savepoint(): + recs = self._load_records(data_list, mode == 'update') + ids.extend(recs.ids) + return + except psycopg2.InternalError as e: + # broken transaction, exit and hope the source error was already logged + if not any(message['type'] == 'error' for message in messages): + info = data_list[0]['info'] + messages.append(dict(info, type='error', message=_(u"Unknown database error: '%s'", e))) + return + except Exception: + pass + + errors = 0 + # try again, this time record by record + for i, rec_data in enumerate(data_list, 1): + try: + with cr.savepoint(): + rec = self._load_records([rec_data], mode == 'update') + ids.append(rec.id) + except psycopg2.Warning as e: + info = rec_data['info'] + messages.append(dict(info, type='warning', message=str(e))) + except psycopg2.Error as e: + info = rec_data['info'] + messages.append(dict(info, type='error', **PGERROR_TO_OE[e.pgcode](self, fg, info, e))) + # Failed to write, log to messages, rollback savepoint (to + # avoid broken transaction) and keep going + errors += 1 + except Exception as e: + _logger.debug("Error while loading record", exc_info=True) + info = rec_data['info'] + message = (_(u'Unknown error during import:') + u' %s: %s' % (type(e), e)) + moreinfo = _('Resolve other errors first') + messages.append(dict(info, type='error', message=message, moreinfo=moreinfo)) + # Failed for some reason, perhaps due to invalid data supplied, + # rollback savepoint and keep going + errors += 1 + if errors >= 10 and (errors >= i / 10): + messages.append({ + 'type': 'warning', + 'message': _(u"Found more than 10 errors and more than one error per 10 records, interrupted to avoid showing too many errors.") + }) + break + + # make 'flush' available to the methods below, in the case where XMLID + # resolution fails, for instance + flush_self = self.with_context(import_flush=flush) + + # TODO: break load's API instead of smuggling via context? + limit = self._context.get('_import_limit') + if limit is None: + limit = float('inf') + extracted = flush_self._extract_records(fields, data, log=messages.append, limit=limit) + + converted = flush_self._convert_records(extracted, log=messages.append) + + info = {'rows': {'to': -1}} + for id, xid, record, info in converted: + if xid: + xid = xid if '.' in xid else "%s.%s" % (current_module, xid) + batch_xml_ids.add(xid) + # + # log csv records + openupgrade_log.log_xml_id(self.env.cr, current_module, xid) + # + elif id: + record['id'] = id + batch.append((xid, record, info)) + + flush() + if any(message['type'] == 'error' for message in messages): + cr.execute('ROLLBACK TO SAVEPOINT model_load') + ids = False + # cancel all changes done to the registry/ormcache + self.pool.reset_changes() + + nextrow = info['rows']['to'] + 1 + if nextrow < limit: + nextrow = 0 + return { + 'ids': ids, + 'messages': messages, + 'nextrow': nextrow, + } + +BaseModel.load = _load diff --git a/upgrade_analysis/odoo_patch/odoo/modules/__init__.py b/upgrade_analysis/odoo_patch/odoo/modules/__init__.py new file mode 100644 index 000000000..90de5b4ff --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo/modules/__init__.py @@ -0,0 +1,12 @@ +# Minor changes. (call to safe_eval changed) +# otherwise : adapted to V14 +from . import graph + +# A lot of changes in the core functions. +from . import loading + +# Adapted to V14 +from . import migration + +# Adapted to V14 +from . import registry diff --git a/upgrade_analysis/odoo_patch/odoo/modules/graph.py b/upgrade_analysis/odoo_patch/odoo/modules/graph.py new file mode 100644 index 000000000..b0bedef3e --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo/modules/graph.py @@ -0,0 +1,108 @@ +# flake8: noqa +# pylint: skip-file + +import logging +import odoo +import odoo.tools as tools +from odoo.tools.safe_eval import safe_eval + +from odoo.modules.graph import Graph + +_logger = logging.getLogger(__name__) + + +if True: + + def _update_from_db(self, cr): + if not len(self): + return + # update the graph with values from the database (if exist) + ## First, we set the default values for each package in graph + additional_data = {key: {'id': 0, 'state': 'uninstalled', 'dbdemo': False, 'installed_version': None} for key in self.keys()} + ## Then we get the values from the database + cr.execute('SELECT name, id, state, demo AS dbdemo, latest_version AS installed_version' + ' FROM ir_module_module' + ' WHERE name IN %s',(tuple(additional_data),) + ) + + ## and we update the default values with values from the database + additional_data.update((x['name'], x) for x in cr.dictfetchall()) + + # + # Prevent reloading of demo data from the new version on major upgrade + if ('base' in self and additional_data['base']['dbdemo'] and + additional_data['base']['installed_version'] < + odoo.release.major_version): + cr.execute("UPDATE ir_module_module SET demo = false") + for data in additional_data.values(): + data['dbdemo'] = False + # + + for package in self.values(): + for k, v in additional_data[package.name].items(): + setattr(package, k, v) + + + def _add_modules(self, cr, module_list, force=None): + if force is None: + force = [] + packages = [] + len_graph = len(self) + + # + # force additional dependencies for the upgrade process if given + # in config file + forced_deps = tools.config.get_misc('openupgrade', 'force_deps', '{}') + forced_deps = tools.config.get_misc('openupgrade', + 'force_deps_' + odoo.release.version, + forced_deps) + forced_deps = safe_eval(forced_deps) + # + + for module in module_list: + # This will raise an exception if no/unreadable descriptor file. + # NOTE The call to load_information_from_description_file is already + # done by db.initialize, so it is possible to not do it again here. + info = odoo.modules.module.load_information_from_description_file(module) + if info and info['installable']: + # + info['depends'].extend(forced_deps.get(module, [])) + # + packages.append((module, info)) # TODO directly a dict, like in get_modules_with_version + elif module != 'studio_customization': + _logger.warning('module %s: not installable, skipped', module) + + dependencies = dict([(p, info['depends']) for p, info in packages]) + current, later = set([p for p, info in packages]), set() + + while packages and current > later: + package, info = packages[0] + deps = info['depends'] + + # if all dependencies of 'package' are already in the graph, add 'package' in the graph + if all(dep in self for dep in deps): + if not package in current: + packages.pop(0) + continue + later.clear() + current.remove(package) + node = self.add_node(package, info) + for kind in ('init', 'demo', 'update'): + if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force: + setattr(node, kind, True) + else: + later.add(package) + packages.append((package, info)) + packages.pop(0) + + self.update_from_db(cr) + + for package in later: + unmet_deps = [p for p in dependencies[package] if p not in self] + _logger.info('module %s: Unmet dependencies: %s', package, ', '.join(unmet_deps)) + + return len(self) - len_graph + + +Graph.update_from_db = _update_from_db +Graph.add_modules = _add_modules diff --git a/upgrade_analysis/odoo_patch/odoo/modules/loading.py b/upgrade_analysis/odoo_patch/odoo/modules/loading.py new file mode 100644 index 000000000..eb25c80ad --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo/modules/loading.py @@ -0,0 +1,556 @@ +# flake8: noqa +# pylint: skip-file + +import itertools +import logging +import sys +import time + +import odoo +import odoo.tools as tools +from odoo import api, SUPERUSER_ID +from odoo.modules import loading +from odoo.modules.module import adapt_version, load_openerp_module, initialize_sys_path + +from odoo.modules.loading import load_data, load_demo, _check_module_names +from odoo.addons.openupgrade_framework.openupgrade import openupgrade_loading + +import os + +_logger = logging.getLogger(__name__) +_test_logger = logging.getLogger('odoo.tests') + + +def _load_module_graph(cr, graph, status=None, perform_checks=True, + skip_modules=None, report=None, models_to_check=None, upg_registry=None): + # + """Migrates+Updates or Installs all module nodes from ``graph`` + :param graph: graph of module nodes to load + :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 + :param perform_checks: whether module descriptors should be checked for validity (prints warnings + for same cases) + :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped + :return: list of modules that were installed or updated + """ + if skip_modules is None: + skip_modules = [] + + if models_to_check is None: + models_to_check = set() + + processed_modules = [] + loaded_modules = [] + registry = odoo.registry(cr.dbname) + migrations = odoo.modules.migration.MigrationManager(cr, graph) + module_count = len(graph) + _logger.info('loading %d modules...', module_count) + + # + # suppress commits to have the upgrade of one module in just one transaction + cr.commit_org = cr.commit + cr.commit = lambda *args: None + cr.rollback_org = cr.rollback + cr.rollback = lambda *args: None + # + + # register, instantiate and initialize models for each modules + t0 = time.time() + loading_extra_query_count = odoo.sql_db.sql_counter + loading_cursor_query_count = cr.sql_log_count + + models_updated = set() + + for index, package in enumerate(graph, 1): + module_name = package.name + module_id = package.id + + # + if module_name in skip_modules or module_name in loaded_modules: + # + continue + + module_t0 = time.time() + module_cursor_query_count = cr.sql_log_count + module_extra_query_count = odoo.sql_db.sql_counter + + needs_update = ( + hasattr(package, "init") + or hasattr(package, "update") + or package.state in ("to install", "to upgrade") + ) + module_log_level = logging.DEBUG + if needs_update: + module_log_level = logging.INFO + _logger.log(module_log_level, 'Loading module %s (%d/%d)', module_name, index, module_count) + + if needs_update: + if package.name != 'base': + registry.setup_models(cr) + migrations.migrate_module(package, 'pre') + if package.name != 'base': + env = api.Environment(cr, SUPERUSER_ID, {}) + env['base'].flush() + + load_openerp_module(package.name) + + new_install = package.state == 'to install' + if new_install: + py_module = sys.modules['odoo.addons.%s' % (module_name,)] + pre_init = package.info.get('pre_init_hook') + if pre_init: + getattr(py_module, pre_init)(cr) + + model_names = registry.load(cr, package) + + mode = 'update' + if hasattr(package, 'init') or package.state == 'to install': + mode = 'init' + + loaded_modules.append(package.name) + if needs_update: + models_updated |= set(model_names) + models_to_check -= set(model_names) + registry.setup_models(cr) + # + # rebuild the local registry based on the loaded models + local_registry = {} + env = api.Environment(cr, SUPERUSER_ID, {}) + for model in env.values(): + if not model._auto: + continue + openupgrade_loading.log_model(model, local_registry) + openupgrade_loading.compare_registries( + cr, package.name, upg_registry, local_registry) + # + + registry.init_models(cr, model_names, {'module': package.name}, new_install) + elif package.state != 'to remove': + # The current module has simply been loaded. The models extended by this module + # and for which we updated the schema, must have their schema checked again. + # This is because the extension may have changed the model, + # e.g. adding required=True to an existing field, but the schema has not been + # updated by this module because it's not marked as 'to upgrade/to install'. + models_to_check |= set(model_names) & models_updated + + idref = {} + + if needs_update: + env = api.Environment(cr, SUPERUSER_ID, {}) + # Can't put this line out of the loop: ir.module.module will be + # registered by init_models() above. + module = env['ir.module.module'].browse(module_id) + + if perform_checks: + module._check() + + if package.state == 'to upgrade': + # upgrading the module information + module.write(module.get_values_from_terp(package.data)) + load_data(cr, idref, mode, kind='data', package=package) + demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode) + cr.execute('update ir_module_module set demo=%s where id=%s', (demo_loaded, module_id)) + module.invalidate_cache(['demo']) + + # + # add 'try' block for logging exceptions + # as errors in post scripts seem to be dropped + try: + migrations.migrate_module(package, 'post') + except Exception as exc: + _logger.error('Error executing post migration script for module %s: %s', + package, exc) + raise + # + + # Update translations for all installed languages + overwrite = odoo.tools.config["overwrite_existing_translations"] + module.with_context(overwrite=overwrite)._update_translations() + + if package.name is not None: + registry._init_modules.add(package.name) + + if needs_update: + if new_install: + post_init = package.info.get('post_init_hook') + if post_init: + getattr(py_module, post_init)(cr, registry) + + if mode == 'update': + # validate the views that have not been checked yet + env['ir.ui.view']._validate_module_views(module_name) + + # need to commit any modification the module's installation or + # update made to the schema or data so the tests can run + # (separately in their own transaction) + # + # commit after processing every module as well, for + # easier debugging and continuing an interrupted migration + cr.commit_org() + # + # run tests + if os.environ.get('OPENUPGRADE_TESTS') and package.name is not None: + prefix = '.migrations' + registry.openupgrade_test_prefixes[package.name] = prefix + report.record_result(odoo.modules.module.run_unit_tests(module_name, openupgrade_prefix=prefix)) + # + # commit module_n state and version immediatly + # to avoid invalid database state if module_n+1 raises an + # exception + cr.commit_org() + # + + package.load_state = package.state + package.load_version = package.installed_version + package.state = 'installed' + for kind in ('init', 'demo', 'update'): + if hasattr(package, kind): + delattr(package, kind) + module.flush() + + extra_queries = odoo.sql_db.sql_counter - module_extra_query_count - test_queries + extras = [] + if test_queries: + extras.append(f'+{test_queries} test') + if extra_queries: + extras.append(f'+{extra_queries} other') + _logger.log( + module_log_level, "Module %s loaded in %.2fs%s, %s queries%s", + module_name, time.time() - module_t0, + f' (incl. {test_time:.2f}s test)' if test_time else '', + cr.sql_log_count - module_cursor_query_count, + f' ({", ".join(extras)})' if extras else '' + ) + if test_results and not test_results.wasSuccessful(): + _logger.error( + "Module %s: %d failures, %d errors of %d tests", + module_name, len(test_results.failures), len(test_results.errors), + test_results.testsRun + ) + + _logger.runbot("%s modules loaded in %.2fs, %s queries (+%s extra)", + len(graph), + time.time() - t0, + cr.sql_log_count - loading_cursor_query_count, + odoo.sql_db.sql_counter - loading_extra_query_count) # extra queries: testes, notify, any other closed cursor + + # + # restore commit method + cr.commit = cr.commit_org + cr.commit() + # + + return loaded_modules, processed_modules + + +def _load_marked_modules(cr, graph, states, force, progressdict, report, + loaded_modules, perform_checks, models_to_check=None, upg_registry=None): + # + """Loads modules marked with ``states``, adding them to ``graph`` and + ``loaded_modules`` and returns a list of installed/upgraded modules.""" + + if models_to_check is None: + models_to_check = set() + + processed_modules = [] + while True: + cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),)) + module_list = [name for (name,) in cr.fetchall() if name not in graph] + # + module_list = openupgrade_loading.add_module_dependencies(cr, module_list) + # + if not module_list: + break + graph.add_modules(cr, module_list, force) + _logger.debug('Updating graph with %d more modules', len(module_list)) + # + # add upg_registry + loaded, processed = _load_module_graph( + cr, graph, progressdict, report=report, skip_modules=loaded_modules, + perform_checks=perform_checks, models_to_check=models_to_check, + upg_registry=upg_registry, + ) + # + processed_modules.extend(processed) + loaded_modules.extend(loaded) + if not processed: + break + return processed_modules + + +def _load_modules(db, force_demo=False, status=None, update_module=False): + initialize_sys_path() + + force = [] + if force_demo: + force.append('demo') + + # + upg_registry = {} + # + + models_to_check = set() + + with db.cursor() as cr: + if not odoo.modules.db.is_initialized(cr): + if not update_module: + _logger.error("Database %s not initialized, you can force it with `-i base`", cr.dbname) + return + _logger.info("init db") + odoo.modules.db.initialize(cr) + update_module = True # process auto-installed modules + tools.config["init"]["all"] = 1 + if not tools.config['without_demo']: + tools.config["demo"]['all'] = 1 + + # This is a brand new registry, just created in + # odoo.modules.registry.Registry.new(). + registry = odoo.registry(cr.dbname) + + if 'base' in tools.config['update'] or 'all' in tools.config['update']: + cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed')) + + # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps) + graph = odoo.modules.graph.Graph() + graph.add_module(cr, 'base', force) + if not graph: + _logger.critical('module base cannot be loaded! (hint: verify addons-path)') + raise ImportError('Module `base` cannot be loaded! (hint: verify addons-path)') + + # processed_modules: for cleanup step after install + # loaded_modules: to avoid double loading + report = registry._assertion_report + # + # add upg_registry + loaded_modules, processed_modules = _load_module_graph( + cr, graph, status, perform_checks=update_module, + report=report, models_to_check=models_to_check, upg_registry=upg_registry) + + # + load_lang = tools.config.pop('load_language') + if load_lang or update_module: + # some base models are used below, so make sure they are set up + registry.setup_models(cr) + + if load_lang: + for lang in load_lang.split(','): + tools.load_language(cr, lang) + + # STEP 2: Mark other modules to be loaded/updated + if update_module: + env = api.Environment(cr, SUPERUSER_ID, {}) + Module = env['ir.module.module'] + _logger.info('updating modules list') + Module.update_list() + + _check_module_names(cr, itertools.chain(tools.config['init'], tools.config['update'])) + + module_names = [k for k, v in tools.config['init'].items() if v] + if module_names: + modules = Module.search([('state', '=', 'uninstalled'), ('name', 'in', module_names)]) + if modules: + modules.button_install() + + module_names = [k for k, v in tools.config['update'].items() if v] + if module_names: + # + # in standard Odoo, '--update all' just means: + # '--update base + upward (installed) dependencies. This breaks + # the chain when new glue modules are encountered. + # E.g. purchase in 8.0 depends on stock_account and report, + # both of which are new. They may be installed, but purchase as + # an upward dependency is not selected for upgrade. + # Therefore, explicitely select all installed modules for + # upgrading in OpenUpgrade in that case. + domain = [('state', '=', 'installed')] + if 'all' not in module_names: + domain.append(('name', 'in', module_names)) + modules = Module.search(domain) + # + if modules: + modules.button_upgrade() + + cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base')) + Module.invalidate_cache(['state']) + Module.flush() + + # STEP 3: Load marked modules (skipping base which was done in STEP 1) + # IMPORTANT: this is done in two parts, first loading all installed or + # partially installed modules (i.e. installed/to upgrade), to + # offer a consistent system to the second part: installing + # newly selected modules. + # We include the modules 'to remove' in the first step, because + # they are part of the "currently installed" modules. They will + # be dropped in STEP 6 later, before restarting the loading + # process. + # IMPORTANT 2: We have to loop here until all relevant modules have been + # processed, because in some rare cases the dependencies have + # changed, and modules that depend on an uninstalled module + # will not be processed on the first pass. + # It's especially useful for migrations. + previously_processed = -1 + while previously_processed < len(processed_modules): + previously_processed = len(processed_modules) + # + # add upg_registry + processed_modules += _load_marked_modules(cr, graph, + ['installed', 'to upgrade', 'to remove'], + force, status, report, loaded_modules, update_module, models_to_check, upg_registry) + # + if update_module: + # + # add upg_registry + processed_modules += _load_marked_modules(cr, graph, + ['to install'], force, status, report, + loaded_modules, update_module, models_to_check, upg_registry) + # + # check that new module dependencies have been properly installed after a migration/upgrade + cr.execute("SELECT name from ir_module_module WHERE state IN ('to install', 'to upgrade')") + module_list = [name for (name,) in cr.fetchall()] + if module_list: + _logger.error("Some modules have inconsistent states, some dependencies may be missing: %s", sorted(module_list)) + + # check that all installed modules have been loaded by the registry after a migration/upgrade + cr.execute("SELECT name from ir_module_module WHERE state = 'installed' and name != 'studio_customization'") + module_list = [name for (name,) in cr.fetchall() if name not in graph] + if module_list: + _logger.error("Some modules are not loaded, some dependencies or manifest may be missing: %s", sorted(module_list)) + + registry.loaded = True + registry.setup_models(cr) + + # STEP 3.5: execute migration end-scripts + migrations = odoo.modules.migration.MigrationManager(cr, graph) + for package in graph: + migrations.migrate_module(package, 'end') + + # STEP 3.6: apply remaining constraints in case of an upgrade + registry.finalize_constraints() + + # STEP 4: Finish and cleanup installations + if processed_modules: + env = api.Environment(cr, SUPERUSER_ID, {}) + cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""") + for (model, name) in cr.fetchall(): + if model in registry and not registry[model]._abstract: + _logger.warning('The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,base.group_user,1,0,0,0', + model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_')) + + cr.execute("SELECT model from ir_model") + for (model,) in cr.fetchall(): + if model in registry: + env[model]._check_removed_columns(log=True) + elif _logger.isEnabledFor(logging.INFO): # more an info that a warning... + _logger.runbot("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model) + + # Cleanup orphan records + env['ir.model.data']._process_end(processed_modules) + env['base'].flush() + + for kind in ('init', 'demo', 'update'): + tools.config[kind] = {} + + # STEP 5: Uninstall modules to remove + if update_module: + # Remove records referenced from ir_model_data for modules to be + # removed (and removed the references from ir_model_data). + cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s", ('to remove',)) + modules_to_remove = dict(cr.fetchall()) + if modules_to_remove: + env = api.Environment(cr, SUPERUSER_ID, {}) + pkgs = reversed([p for p in graph if p.name in modules_to_remove]) + for pkg in pkgs: + uninstall_hook = pkg.info.get('uninstall_hook') + if uninstall_hook: + py_module = sys.modules['odoo.addons.%s' % (pkg.name,)] + getattr(py_module, uninstall_hook)(cr, registry) + + Module = env['ir.module.module'] + Module.browse(modules_to_remove.values()).module_uninstall() + # Recursive reload, should only happen once, because there should be no + # modules to remove next time + cr.commit() + _logger.info('Reloading registry once more after uninstalling modules') + api.Environment.reset() + registry = odoo.modules.registry.Registry.new( + cr.dbname, force_demo, status, update_module + ) + registry.check_tables_exist(cr) + cr.commit() + return registry + + # STEP 5.5: Verify extended fields on every model + # This will fix the schema of all models in a situation such as: + # - module A is loaded and defines model M; + # - module B is installed/upgraded and extends model M; + # - module C is loaded and extends model M; + # - module B and C depend on A but not on each other; + # The changes introduced by module C are not taken into account by the upgrade of B. + if models_to_check: + registry.init_models(cr, list(models_to_check), {'models_to_check': True}) + + # STEP 6: verify custom views on every model + if update_module: + env = api.Environment(cr, SUPERUSER_ID, {}) + env['res.groups']._update_user_groups_view() + View = env['ir.ui.view'] + for model in registry: + try: + View._validate_custom_views(model) + except Exception as e: + _logger.warning('invalid custom view(s) for model %s: %s', model, tools.ustr(e)) + + if report.wasSuccessful(): + _logger.info('Modules loaded.') + else: + _logger.error('At least one test failed when loading the modules.') + + # STEP 8: call _register_hook on every model + # This is done *exactly once* when the registry is being loaded. See the + # management of those hooks in `Registry.setup_models`: all the calls to + # setup_models() done here do not mess up with hooks, as registry.ready + # is False. + env = api.Environment(cr, SUPERUSER_ID, {}) + for model in env.values(): + model._register_hook() + env['base'].flush() + + # STEP 9: save installed/updated modules for post-install tests + registry.updated_modules += processed_modules + +loading.load_module_graph = _load_module_graph +loading.load_marked_modules = _load_marked_modules +loading.load_modules = _load_modules +odoo.modules.load_modules = _load_modules diff --git a/upgrade_analysis/odoo_patch/odoo/modules/migration.py b/upgrade_analysis/odoo_patch/odoo/modules/migration.py new file mode 100644 index 000000000..0346c2b8c --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo/modules/migration.py @@ -0,0 +1,118 @@ +# flake8: noqa +# pylint: skip-file + +import logging +import os +from os.path import join as opj +import odoo.release as release +from odoo.tools.parse_version import parse_version + +import odoo +from odoo.modules.migration import load_script +from odoo.modules import migration + +_logger = logging.getLogger(__name__) + + +if True: + def _migrate_module(self, pkg, stage): + assert stage in ('pre', 'post', 'end') + stageformat = { + 'pre': '[>%s]', + 'post': '[%s>]', + 'end': '[$%s]', + } + state = pkg.state if stage in ('pre', 'post') else getattr(pkg, 'load_state', None) + + # + # In openupgrade, also run migration scripts upon installation. + # We want to always pass in pre and post migration files and use a new + # argument in the migrate decorator (explained in the docstring) + # to decide if we want to do something if a new module is installed + # during the migration. + if not (hasattr(pkg, 'update') or state in ('to upgrade', 'to install')): + # + return + + def convert_version(version): + if version.count('.') >= 2: + return version # the version number already containt the server version + return "%s.%s" % (release.major_version, version) + + def _get_migration_versions(pkg, stage): + versions = sorted({ + ver + for lv in self.migrations[pkg.name].values() + for ver, lf in lv.items() + if lf + }, key=lambda k: parse_version(convert_version(k))) + if "0.0.0" in versions: + # reorder versions + versions.remove("0.0.0") + if stage == "pre": + versions.insert(0, "0.0.0") + else: + versions.append("0.0.0") + return versions + + def _get_migration_files(pkg, version, stage): + """ return a list of migration script files + """ + m = self.migrations[pkg.name] + lst = [] + + mapping = { + 'module': opj(pkg.name, 'migrations'), + 'module_upgrades': opj(pkg.name, 'upgrades'), + } + + for path in odoo.upgrade.__path__: + if os.path.exists(opj(path, pkg.name)): + mapping['upgrade'] = opj(path, pkg.name) + break + + for x in mapping: + if version in m.get(x): + for f in m[x][version]: + if not f.startswith(stage + '-'): + continue + lst.append(opj(mapping[x], version, f)) + lst.sort() + return lst + + installed_version = getattr(pkg, 'load_version', pkg.installed_version) or '' + parsed_installed_version = parse_version(installed_version) + current_version = parse_version(convert_version(pkg.data['version'])) + + versions = _get_migration_versions(pkg, stage) + + for version in versions: + if ((version == "0.0.0" and parsed_installed_version < current_version) + or parsed_installed_version < parse_version(convert_version(version)) <= current_version): + + strfmt = {'addon': pkg.name, + 'stage': stage, + 'version': stageformat[stage] % version, + } + + for pyfile in _get_migration_files(pkg, version, stage): + name, ext = os.path.splitext(os.path.basename(pyfile)) + if ext.lower() != '.py': + continue + mod = None + try: + mod = load_script(pyfile, name) + _logger.info('module %(addon)s: Running migration %(version)s %(name)s' % dict(strfmt, name=mod.__name__)) + migrate = mod.migrate + except ImportError: + _logger.exception('module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % dict(strfmt, file=pyfile)) + raise + except AttributeError: + _logger.error('module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt) + else: + migrate(self.cr, installed_version) + finally: + if mod: + del mod + +migration.migrate_module = _migrate_module diff --git a/upgrade_analysis/odoo_patch/odoo/modules/registry.py b/upgrade_analysis/odoo_patch/odoo/modules/registry.py new file mode 100644 index 000000000..4c5f50d4e --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo/modules/registry.py @@ -0,0 +1,58 @@ +# flake8: noqa +# pylint: skip-file + +from collections import deque +from contextlib import closing +import odoo +from odoo.tools.lru import LRU + +from odoo.modules import registry + + +if True: + + def _init(self, db_name): + self.models = {} # model name/model instance mapping + self._sql_constraints = set() + self._init = True + self._assertion_report = odoo.tests.runner.OdooTestResult() + self._fields_by_model = None + self._ordinary_tables = None + self._constraint_queue = deque() + self.__cache = LRU(8192) + + # modules fully loaded (maintained during init phase by `loading` module) + self._init_modules = set() + self.updated_modules = [] # installed/updated modules + # + self.openupgrade_test_prefixes = {} + # + self.loaded_xmlids = set() + + self.db_name = db_name + self._db = odoo.sql_db.db_connect(db_name) + + # cursor for test mode; None means "normal" mode + self.test_cr = None + self.test_lock = None + + # Indicates that the registry is + self.loaded = False # whether all modules are loaded + self.ready = False # whether everything is set up + + # Inter-process signaling: + # The `base_registry_signaling` sequence indicates the whole registry + # must be reloaded. + # The `base_cache_signaling sequence` indicates all caches must be + # invalidated (i.e. cleared). + self.registry_sequence = None + self.cache_sequence = None + + # Flags indicating invalidation of the registry or the cache. + self.registry_invalidated = False + self.cache_invalidated = False + + with closing(self.cursor()) as cr: + self.has_unaccent = odoo.modules.db.has_unaccent(cr) + +registry.init = _init diff --git a/upgrade_analysis/odoo_patch/odoo/service/__init__.py b/upgrade_analysis/odoo_patch/odoo/service/__init__.py new file mode 100644 index 000000000..a96314d0f --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo/service/__init__.py @@ -0,0 +1,4 @@ +# Import disabled, because the function run_unit_tests() +# disappeared in V14. +# TODO: OpenUpgrade Core maintainers : FIXME. +# from . import server diff --git a/upgrade_analysis/odoo_patch/odoo/service/server.py b/upgrade_analysis/odoo_patch/odoo/service/server.py new file mode 100644 index 000000000..a2a998e69 --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo/service/server.py @@ -0,0 +1,71 @@ +# flake8: noqa +# pylint: skip-file + +import logging +import os +import time + +import odoo +from odoo.tools import config +from odoo.modules.registry import Registry + +from odoo.service import server +from odoo.service.server import load_test_file_py + +_logger = logging.getLogger(__name__) + + +def preload_registries(dbnames): + """ Preload a registries, possibly run a test file.""" + # TODO: move all config checks to args dont check tools.config here + dbnames = dbnames or [] + rc = 0 + for dbname in dbnames: + try: + update_module = config['init'] or config['update'] + registry = Registry.new(dbname, update_module=update_module) + + # run test_file if provided + if config['test_file']: + test_file = config['test_file'] + if not os.path.isfile(test_file): + _logger.warning('test file %s cannot be found', test_file) + elif not test_file.endswith('py'): + _logger.warning('test file %s is not a python file', test_file) + else: + _logger.info('loading test file %s', test_file) + with odoo.api.Environment.manage(): + load_test_file_py(registry, test_file) + + # run post-install tests + if config['test_enable']: + t0 = time.time() + t0_sql = odoo.sql_db.sql_counter + module_names = (registry.updated_modules if update_module else + sorted(registry._init_modules)) + _logger.info("Starting post tests") + tests_before = registry._assertion_report.testsRun + with odoo.api.Environment.manage(): + for module_name in module_names: + result = loader.run_suite(loader.make_suite(module_name, 'post_install'), module_name) + registry._assertion_report.update(result) + # + # run deferred unit tests + for module_name, prefix in registry.openupgrade_test_prefixes: + result = run_unit_tests(module_name, position='post_install', openupgrade_prefix=prefix) + registry._assertion_report.record_result(result) + # + _logger.info("%d post-tests in %.2fs, %s queries", + registry._assertion_report.testsRun - tests_before, + time.time() - t0, + odoo.sql_db.sql_counter - t0_sql) + + if not registry._assertion_report.wasSuccessful(): + rc += 1 + except Exception: + _logger.critical('Failed to initialize database `%s`.', dbname, exc_info=True) + return -1 + return rc + + +server.preload_registries = preload_registries diff --git a/upgrade_analysis/odoo_patch/odoo/tools/__init__.py b/upgrade_analysis/odoo_patch/odoo/tools/__init__.py new file mode 100644 index 000000000..6ad156515 --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo/tools/__init__.py @@ -0,0 +1,2 @@ +from . import convert +from . import view_validation diff --git a/upgrade_analysis/odoo_patch/odoo/tools/convert.py b/upgrade_analysis/odoo_patch/odoo/tools/convert.py new file mode 100644 index 000000000..2af712a22 --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo/tools/convert.py @@ -0,0 +1,23 @@ +# flake8: noqa +# pylint: skip-file + +from .... import upgrade_log + +from odoo.tools.convert import xml_import + +if True: + + def __test_xml_id(self, xml_id): + if '.' in xml_id: + module, id = xml_id.split('.', 1) + assert '.' not in id, """The ID reference "%s" must contain +maximum one dot. They are used to refer to other modules ID, in the +form: module.record_id""" % (xml_id,) + if module != self.module: + modcnt = self.env['ir.module.module'].search_count([('name', '=', module), ('state', '=', 'installed')]) + assert modcnt == 1, """The ID "%s" refers to an uninstalled module""" % (xml_id,) + + # OpenUpgrade: log entry of XML imports + openupgrade_log.log_xml_id(self.env.cr, self.module, xml_id) + +xml_import._test_xml_id = __test_xml_id diff --git a/upgrade_analysis/odoo_patch/odoo/tools/view_validation.py b/upgrade_analysis/odoo_patch/odoo/tools/view_validation.py new file mode 100644 index 000000000..e6c824324 --- /dev/null +++ b/upgrade_analysis/odoo_patch/odoo/tools/view_validation.py @@ -0,0 +1,29 @@ +# flake8: noqa +# pylint: skip-file + +# from odoo.addons.openupgrade_framework.openupgrade import openupgrade_log + +from odoo.tools import view_validation +from odoo.tools.view_validation import _validators, _logger + + +def _valid_view(arch, **kwargs): + for pred in _validators[arch.tag]: + # + # Do not raise blocking error, because it's normal to + # have inconsistent views in an openupgrade process + check = pred(arch, **kwargs) or 'Warning' + # + if not check: + _logger.error("Invalid XML: %s", pred.__doc__) + return False + if check == "Warning": + # + # Don't show this warning as useless and too much verbose + # _logger.warning("Invalid XML: %s", pred.__doc__) + # + return "Warning" + return True + + +view_validation.valid_view = _valid_view diff --git a/upgrade_analysis/openupgrade_loading.py b/upgrade_analysis/openupgrade_loading.py new file mode 100644 index 000000000..ca3e1d430 --- /dev/null +++ b/upgrade_analysis/openupgrade_loading.py @@ -0,0 +1,318 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016-2019 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +# flake8: noqa: C901 + +import logging + +from openupgradelib.openupgrade_tools import table_exists + +from odoo import release +from odoo.modules.module import get_module_path +from odoo.tools.safe_eval import safe_eval +from odoo.tools.config import config + +# A collection of functions used in +# odoo/modules/loading.py + +logger = logging.getLogger("OpenUpgrade") + + +def add_module_dependencies(cr, module_list): + """ + Select (new) dependencies from the modules in the list + so that we can inject them into the graph at upgrade + time. Used in the modified OpenUpgrade Server, + not to be called from migration scripts + + Also take the OpenUpgrade configuration directives 'forced_deps' + and 'autoinstall' into account. From any additional modules + that these directives can add, the dependencies are added as + well (but these directives are not checked for the occurrence + of any of the dependencies). + """ + if not module_list: + return module_list + + modules_in = list(module_list) + forced_deps = safe_eval( + config.get_misc( + "openupgrade", + "forced_deps_" + release.version, + config.get_misc("openupgrade", "forced_deps", "{}"), + ) + ) + + autoinstall = safe_eval( + config.get_misc( + "openupgrade", + "autoinstall_" + release.version, + config.get_misc("openupgrade", "autoinstall", "{}"), + ) + ) + + for module in list(module_list): + module_list += forced_deps.get(module, []) + module_list += autoinstall.get(module, []) + + module_list = list(set(module_list)) + + dependencies = module_list + while dependencies: + cr.execute( + """ + SELECT DISTINCT dep.name + FROM + ir_module_module, + ir_module_module_dependency dep + WHERE + module_id = ir_module_module.id + AND ir_module_module.name in %s + AND dep.name not in %s + """, + ( + tuple(dependencies), + tuple(module_list), + ), + ) + + dependencies = [x[0] for x in cr.fetchall()] + module_list += dependencies + + # Select auto_install modules of which all dependencies + # are fulfilled based on the modules we know are to be + # installed + cr.execute( + """ + SELECT name from ir_module_module WHERE state IN %s + """, + (("installed", "to install", "to upgrade"),), + ) + modules = list(set(module_list + [row[0] for row in cr.fetchall()])) + cr.execute( + """ + SELECT name from ir_module_module m + WHERE auto_install IS TRUE + AND state = 'uninstalled' + AND NOT EXISTS( + SELECT id FROM ir_module_module_dependency d + WHERE d.module_id = m.id + AND name NOT IN %s) + """, + (tuple(modules),), + ) + auto_modules = [row[0] for row in cr.fetchall() if get_module_path(row[0])] + if auto_modules: + logger.info("Selecting autoinstallable modules %s", ",".join(auto_modules)) + module_list += auto_modules + + # Set proper state for new dependencies so that any init scripts are run + cr.execute( + """ + UPDATE ir_module_module SET state = 'to install' + WHERE name IN %s AND name NOT IN %s AND state = 'uninstalled' + """, + (tuple(module_list), tuple(modules_in)), + ) + return module_list + + +def log_model(model, local_registry): + """ + OpenUpgrade: Store the characteristics of the BaseModel and its fields + in the local registry, so that we can compare changes with the + main registry + """ + + if not model._name: + return + + typemap = {"monetary": "float"} + + # Deferred import to prevent import loop + from odoo import models + + # persistent models only + if isinstance(model, models.TransientModel): + return + + def isfunction(model, k): + if ( + model._fields[k].compute + and not model._fields[k].related + and not model._fields[k].company_dependent + ): + return "function" + return "" + + def isproperty(model, k): + if model._fields[k].company_dependent: + return "property" + return "" + + def isrelated(model, k): + if model._fields[k].related: + return "related" + return "" + + def _get_relation(v): + if v.type in ("many2many", "many2one", "one2many"): + return v.comodel_name + elif v.type == "many2one_reference": + return v.model_field + else: + return "" + + model_registry = local_registry.setdefault(model._name, {}) + if model._inherits: + model_registry["_inherits"] = {"_inherits": str(model._inherits)} + for k, v in model._fields.items(): + properties = { + "type": typemap.get(v.type, v.type), + "isfunction": isfunction(model, k), + "isproperty": isproperty(model, k), + "isrelated": isrelated(model, k), + "relation": _get_relation(v), + "table": v.relation if v.type == "many2many" else "", + "required": v.required and "required" or "", + "stored": v.store and "stored" or "", + "selection_keys": "", + "req_default": "", + "hasdefault": model._fields[k].default and "hasdefault" or "", + "inherits": "", + } + if v.type == "selection": + if isinstance(v.selection, (tuple, list)): + properties["selection_keys"] = str(sorted([x[0] for x in v.selection])) + else: + properties["selection_keys"] = "function" + elif v.type == "binary": + properties["attachment"] = str(getattr(v, "attachment", False)) + default = model._fields[k].default + if v.required and default: + if ( + callable(default) + or isinstance(default, str) + and getattr(model._fields[k], default, False) + and callable(getattr(model._fields[k], default)) + ): + # todo: in OpenERP 5 (and in 6 as well), + # literals are wrapped in a lambda function + properties["req_default"] = "function" + else: + properties["req_default"] = str(default) + for key, value in properties.items(): + if value: + model_registry.setdefault(k, {})[key] = value + + +def get_record_id(cr, module, model, field, mode): + """ + OpenUpgrade: get or create the id from the record table matching + the key parameter values + """ + cr.execute( + "SELECT id FROM openupgrade_record " + "WHERE module = %s AND model = %s AND " + "field = %s AND mode = %s AND type = %s", + (module, model, field, mode, "field"), + ) + record = cr.fetchone() + if record: + return record[0] + cr.execute( + "INSERT INTO openupgrade_record " + "(module, model, field, mode, type) " + "VALUES (%s, %s, %s, %s, %s)", + (module, model, field, mode, "field"), + ) + cr.execute( + "SELECT id FROM openupgrade_record " + "WHERE module = %s AND model = %s AND " + "field = %s AND mode = %s AND type = %s", + (module, model, field, mode, "field"), + ) + return cr.fetchone()[0] + + +def compare_registries(cr, module, registry, local_registry): + """ + OpenUpgrade: Compare the local registry with the global registry, + log any differences and merge the local registry with + the global one. + """ + if not table_exists(cr, "openupgrade_record"): + return + for model, flds in local_registry.items(): + registry.setdefault(model, {}) + for field, attributes in flds.items(): + old_field = registry[model].setdefault(field, {}) + mode = old_field and "modify" or "create" + record_id = False + for key, value in attributes.items(): + if key not in old_field or old_field[key] != value: + if not record_id: + record_id = get_record_id(cr, module, model, field, mode) + cr.execute( + "SELECT id FROM openupgrade_attribute " + "WHERE name = %s AND value = %s AND " + "record_id = %s", + (key, value, record_id), + ) + if not cr.fetchone(): + cr.execute( + "INSERT INTO openupgrade_attribute " + "(name, value, record_id) VALUES (%s, %s, %s)", + (key, value, record_id), + ) + old_field[key] = value + + +def update_field_xmlid(model, field): + """OpenUpgrade edit start: In rare cases, an old module defined a field + on a model that is not defined in another module earlier in the + chain of inheritance. Then we need to assign the ir.model.fields' + xmlid to this other module, otherwise the column would be dropped + when uninstalling the first module. + An example is res.partner#display_name defined in 7.0 by + account_report_company, but now the field belongs to the base + module + Given that we arrive here in order of inheritance, we simply check + if the field's xmlid belongs to a module already loaded, and if not, + update the record with the correct module name.""" + model.env.cr.execute( + "SELECT f.*, d.module, d.id as xmlid_id, d.name as xmlid " + "FROM ir_model_fields f LEFT JOIN ir_model_data d " + "ON f.id=d.res_id and d.model='ir.model.fields' WHERE f.model=%s", + (model._name,), + ) + for rec in model.env.cr.dictfetchall(): + if ( + "module" in model.env.context + and rec["module"] + and rec["name"] in model._fields.keys() + and rec["module"] != model.env.context["module"] + and rec["module"] not in model.env.registry._init_modules + ): + logging.getLogger(__name__).info( + "Moving XMLID for ir.model.fields record of %s#%s " "from %s to %s", + model._name, + rec["name"], + rec["module"], + model.env.context["module"], + ) + model.env.cr.execute( + "SELECT id FROM ir_model_data WHERE module=%(module)s " + "AND name=%(xmlid)s", + dict(rec, module=model.env.context["module"]), + ) + if model.env.cr.fetchone(): + logging.getLogger(__name__).info( + "Aborting, an XMLID for this module already exists." + ) + continue + model.env.cr.execute( + "UPDATE ir_model_data SET module=%(module)s " "WHERE id=%(xmlid_id)s", + dict(rec, module=model.env.context["module"]), + ) diff --git a/upgrade_analysis/openupgrade_log.py b/upgrade_analysis/openupgrade_log.py new file mode 100644 index 000000000..81c891673 --- /dev/null +++ b/upgrade_analysis/openupgrade_log.py @@ -0,0 +1,60 @@ +# coding: utf-8 +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from openupgradelib.openupgrade_tools import table_exists + + +def log_xml_id(cr, module, xml_id): + """ + Log xml_ids at load time in the records table. + Called from tools/convert.py:xml_import._test_xml_id() + + # Catcha's + - The module needs to be loaded with 'init', or the calling method + won't be called. This can be brought about by installing the + module or updating the 'state' field of the module to 'to install' + or call the server with '--init ' and the database argument. + + - Do you get the right results immediately when installing the module? + No, sorry. This method retrieves the model from the ir_model_table, but + when the xml id is encountered for the first time, this method is called + before the item is present in this table. Therefore, you will not + get any meaningful results until the *second* time that you 'init' + the module. + + - The good news is that the openupgrade_records module that comes + with this distribution allows you to deal with all of this with + one click on the menu item Settings -> Customizations -> + Database Structure -> OpenUpgrade -> Generate Records + + - You cannot reinitialize the modules in your production database + and expect to keep working on it happily ever after. Do not perform + this routine on your production database. + + :param module: The module that contains the xml_id + :param xml_id: the xml_id, with or without 'module.' prefix + """ + if not table_exists(cr, 'openupgrade_record'): + return + if '.' not in xml_id: + xml_id = '%s.%s' % (module, xml_id) + cr.execute( + "SELECT model FROM ir_model_data " + "WHERE module = %s AND name = %s", + xml_id.split('.')) + record = cr.fetchone() + if not record: + print("Cannot find xml_id %s" % xml_id) + return + else: + cr.execute( + "SELECT id FROM openupgrade_record " + "WHERE module=%s AND model=%s AND name=%s AND type=%s", + (module, record[0], xml_id, 'xmlid')) + if not cr.fetchone(): + cr.execute( + "INSERT INTO openupgrade_record " + "(module, model, name, type) values(%s, %s, %s, %s)", + (module, record[0], xml_id, 'xmlid'))