[MOVE] Port patches and framework from openupgrade project

Co-authored-by: Stefan Rijnhart <stefan@opener.am>

upgrade_analysis/odoo_patch/__init__.py

@@ -0,0 +1,3 @@
+from . import odoo
+from . import addons
+

upgrade_analysis/odoo_patch/addons/__init__.py

@@ -0,0 +1,3 @@
+from . import mrp
+from . import stock
+from . import point_of_sale

upgrade_analysis/odoo_patch/addons/mrp/__init__.py

@@ -0,0 +1,20 @@
+from odoo.addons import mrp
+
+
+def _pre_init_mrp(cr):
+    """ Allow installing MRP in databases with large stock.move table (>1M records)
+        - Creating the computed+stored field stock_move.is_done is terribly slow with the ORM and
+          leads to "Out of Memory" crashes
+    """
+    # <OpenUpgrade:REMOVE>
+    # don't try to add 'is_done' column, because it will fail
+    # when executing the generation of records, in the openupgrade_records
+    # module.
+    # cr.execute("""ALTER TABLE "stock_move" ADD COLUMN "is_done" bool;""")
+    # cr.execute("""UPDATE stock_move
+    #                  SET is_done=COALESCE(state in ('done', 'cancel'), FALSE);""")
+    pass
+    # </OpenUpgrade>
+
+
+mrp._pre_init_mrp = _pre_init_mrp

upgrade_analysis/odoo_patch/addons/point_of_sale/__init__.py

@@ -0,0 +1 @@
+from . import models

upgrade_analysis/odoo_patch/addons/point_of_sale/models/__init__.py

@@ -0,0 +1 @@
+from . import pos_config

upgrade_analysis/odoo_patch/addons/point_of_sale/models/pos_config.py

@@ -0,0 +1,21 @@
+from odoo import api
+from odoo.addons.point_of_sale.models.pos_config import PosConfig
+
+if True:
+
+    @api.model
+    def post_install_pos_localisation(self, companies=False):
+        # <OpenUpgrade:REMOVE>
+        # don't try to setup_defaults, because it will fail
+        # when executing the generation of records, in the openupgrade_records
+        # module.
+        # self = self.sudo()
+        # if not companies:
+        #     companies = self.env['res.company'].search([])
+        # for company in companies.filtered('chart_template_id'):
+        #     pos_configs = self.search([('company_id', '=', company.id)])
+        #     pos_configs.setup_defaults(company)
+        pass
+        # </OpenUpgrade>
+
+PosConfig.post_install_pos_localisation = post_install_pos_localisation

upgrade_analysis/odoo_patch/addons/stock/__init__.py

@@ -0,0 +1,17 @@
+from odoo.addons import stock
+
+
+def pre_init_hook(cr):
+    # <OpenUpgrade:REMOVE>
+    # don't uninstall data as this breaks the analysis
+    # Origin of this code is https://github.com/odoo/odoo/issues/22243
+    # env = api.Environment(cr, SUPERUSER_ID, {})
+    # env['ir.model.data'].search([
+    #     ('model', 'like', '%stock%'),
+    #     ('module', '=', 'stock')
+    # ]).unlink()
+    pass
+    # </OpenUpgrade>
+
+
+stock.pre_init_hook = pre_init_hook

upgrade_analysis/odoo_patch/odoo/__init__.py

@@ -0,0 +1,5 @@
+from . import modules
+from . import service
+from . import tools
+from . import http
+from . import models

upgrade_analysis/odoo_patch/odoo/http.py

@@ -0,0 +1,32 @@
+# flake8: noqa
+# pylint: skip-file
+
+import odoo
+from odoo.service import security
+from odoo.http import SessionExpiredException, request, OpenERPSession
+
+if True:
+    def _check_security(self):
+        """
+        Check the current authentication parameters to know if those are still
+        valid. This method should be called at each request. If the
+        authentication fails, a :exc:`SessionExpiredException` is raised.
+        """
+        if not self.db or not self.uid:
+            raise SessionExpiredException("Session expired")
+        # We create our own environment instead of the request's one.
+        # to avoid creating it without the uid since request.uid isn't set yet
+        env = odoo.api.Environment(request.cr, self.uid, self.context)
+        # here we check if the session is still valid
+        if not security.check_session(self, env):
+            # <OpenUpgrade:ADD>
+            # When asking openupgrade_records to generate records
+            # over jsonrpc, a query on res_users in the call above locks this
+            # table for the sql operations that are triggered by the
+            # reinstallation of the base module
+            env.cr.rollback()
+            # </OpenUpgrade>
+            raise SessionExpiredException("Session expired")
+
+
+OpenERPSession.check_security = _check_security

upgrade_analysis/odoo_patch/odoo/models.py

@@ -0,0 +1,179 @@
+# flake8: noqa
+# pylint: skip-file
+
+import odoo
+import psycopg2
+from odoo import _
+from odoo.models import fix_import_export_id_paths, BaseModel, _logger
+from odoo.addons.openupgrade_framework.openupgrade import openupgrade_log
+
+
+if True:
+    def _load(self, fields, data):
+        """
+        Attempts to load the data matrix, and returns a list of ids (or
+        ``False`` if there was an error and no id could be generated) and a
+        list of messages.
+
+        The ids are those of the records created and saved (in database), in
+        the same order they were extracted from the file. They can be passed
+        directly to :meth:`~read`
+
+        :param fields: list of fields to import, at the same index as the corresponding data
+        :type fields: list(str)
+        :param data: row-major matrix of data to import
+        :type data: list(list(str))
+        :returns: {ids: list(int)|False, messages: [Message][, lastrow: int]}
+        """
+        self.flush()
+
+        # determine values of mode, current_module and noupdate
+        mode = self._context.get('mode', 'init')
+        current_module = self._context.get('module', '__import__')
+        noupdate = self._context.get('noupdate', False)
+        # add current module in context for the conversion of xml ids
+        self = self.with_context(_import_current_module=current_module)
+
+        cr = self._cr
+        cr.execute('SAVEPOINT model_load')
+
+        fields = [fix_import_export_id_paths(f) for f in fields]
+        fg = self.fields_get()
+
+        ids = []
+        messages = []
+        ModelData = self.env['ir.model.data']
+
+        # list of (xid, vals, info) for records to be created in batch
+        batch = []
+        batch_xml_ids = set()
+        # models in which we may have created / modified data, therefore might
+        # require flushing in order to name_search: the root model and any
+        # o2m
+        creatable_models = {self._name}
+        for field_path in fields:
+            if field_path[0] in (None, 'id', '.id'):
+                continue
+            model_fields = self._fields
+            if isinstance(model_fields[field_path[0]], odoo.fields.Many2one):
+                # this only applies for toplevel m2o (?) fields
+                if field_path[0] in (self.env.context.get('name_create_enabled_fieds') or {}):
+                    creatable_models.add(model_fields[field_path[0]].comodel_name)
+            for field_name in field_path:
+                if field_name in (None, 'id', '.id'):
+                    break
+
+                if isinstance(model_fields[field_name], odoo.fields.One2many):
+                    comodel = model_fields[field_name].comodel_name
+                    creatable_models.add(comodel)
+                    model_fields = self.env[comodel]._fields
+
+        def flush(*, xml_id=None, model=None):
+            if not batch:
+                return
+
+            assert not (xml_id and model), \
+                "flush can specify *either* an external id or a model, not both"
+
+            if xml_id and xml_id not in batch_xml_ids:
+                if xml_id not in self.env:
+                    return
+            if model and model not in creatable_models:
+                return
+
+            data_list = [
+                dict(xml_id=xid, values=vals, info=info, noupdate=noupdate)
+                for xid, vals, info in batch
+            ]
+            batch.clear()
+            batch_xml_ids.clear()
+
+            # try to create in batch
+            try:
+                with cr.savepoint():
+                    recs = self._load_records(data_list, mode == 'update')
+                    ids.extend(recs.ids)
+                return
+            except psycopg2.InternalError as e:
+                # broken transaction, exit and hope the source error was already logged
+                if not any(message['type'] == 'error' for message in messages):
+                    info = data_list[0]['info']
+                    messages.append(dict(info, type='error', message=_(u"Unknown database error: '%s'", e)))
+                return
+            except Exception:
+                pass
+
+            errors = 0
+            # try again, this time record by record
+            for i, rec_data in enumerate(data_list, 1):
+                try:
+                    with cr.savepoint():
+                        rec = self._load_records([rec_data], mode == 'update')
+                        ids.append(rec.id)
+                except psycopg2.Warning as e:
+                    info = rec_data['info']
+                    messages.append(dict(info, type='warning', message=str(e)))
+                except psycopg2.Error as e:
+                    info = rec_data['info']
+                    messages.append(dict(info, type='error', **PGERROR_TO_OE[e.pgcode](self, fg, info, e)))
+                    # Failed to write, log to messages, rollback savepoint (to
+                    # avoid broken transaction) and keep going
+                    errors += 1
+                except Exception as e:
+                    _logger.debug("Error while loading record", exc_info=True)
+                    info = rec_data['info']
+                    message = (_(u'Unknown error during import:') + u' %s: %s' % (type(e), e))
+                    moreinfo = _('Resolve other errors first')
+                    messages.append(dict(info, type='error', message=message, moreinfo=moreinfo))
+                    # Failed for some reason, perhaps due to invalid data supplied,
+                    # rollback savepoint and keep going
+                    errors += 1
+                if errors >= 10 and (errors >= i / 10):
+                    messages.append({
+                        'type': 'warning',
+                        'message': _(u"Found more than 10 errors and more than one error per 10 records, interrupted to avoid showing too many errors.")
+                    })
+                    break
+
+        # make 'flush' available to the methods below, in the case where XMLID
+        # resolution fails, for instance
+        flush_self = self.with_context(import_flush=flush)
+
+        # TODO: break load's API instead of smuggling via context?
+        limit = self._context.get('_import_limit')
+        if limit is None:
+            limit = float('inf')
+        extracted = flush_self._extract_records(fields, data, log=messages.append, limit=limit)
+
+        converted = flush_self._convert_records(extracted, log=messages.append)
+
+        info = {'rows': {'to': -1}}
+        for id, xid, record, info in converted:
+            if xid:
+                xid = xid if '.' in xid else "%s.%s" % (current_module, xid)
+                batch_xml_ids.add(xid)
+                # <OpenUpgrade:ADD>
+                # log csv records
+                openupgrade_log.log_xml_id(self.env.cr, current_module, xid)
+                # </OpenUpgrade>
+            elif id:
+                record['id'] = id
+            batch.append((xid, record, info))
+
+        flush()
+        if any(message['type'] == 'error' for message in messages):
+            cr.execute('ROLLBACK TO SAVEPOINT model_load')
+            ids = False
+            # cancel all changes done to the registry/ormcache
+            self.pool.reset_changes()
+
+        nextrow = info['rows']['to'] + 1
+        if nextrow < limit:
+            nextrow = 0
+        return {
+            'ids': ids,
+            'messages': messages,
+            'nextrow': nextrow,
+        }
+
+BaseModel.load = _load

upgrade_analysis/odoo_patch/odoo/modules/__init__.py

@@ -0,0 +1,12 @@
+# Minor changes. (call to safe_eval changed)
+# otherwise : adapted to V14
+from . import graph
+
+# A lot of changes in the core functions.
+from . import loading
+
+# Adapted to V14
+from . import migration
+
+# Adapted to V14
+from . import registry

upgrade_analysis/odoo_patch/odoo/modules/graph.py

@@ -0,0 +1,108 @@
+# flake8: noqa
+# pylint: skip-file
+
+import logging
+import odoo
+import odoo.tools as tools
+from odoo.tools.safe_eval import safe_eval
+
+from odoo.modules.graph import Graph
+
+_logger = logging.getLogger(__name__)
+
+
+if True:
+
+    def _update_from_db(self, cr):
+        if not len(self):
+            return
+        # update the graph with values from the database (if exist)
+        ## First, we set the default values for each package in graph
+        additional_data = {key: {'id': 0, 'state': 'uninstalled', 'dbdemo': False, 'installed_version': None} for key in self.keys()}
+        ## Then we get the values from the database
+        cr.execute('SELECT name, id, state, demo AS dbdemo, latest_version AS installed_version'
+                   '  FROM ir_module_module'
+                   ' WHERE name IN %s',(tuple(additional_data),)
+                   )
+
+        ## and we update the default values with values from the database
+        additional_data.update((x['name'], x) for x in cr.dictfetchall())
+
+        # <OpenUpgrade:ADD>
+        # Prevent reloading of demo data from the new version on major upgrade
+        if ('base' in self and additional_data['base']['dbdemo'] and
+                additional_data['base']['installed_version'] <
+                odoo.release.major_version):
+            cr.execute("UPDATE ir_module_module SET demo = false")
+            for data in additional_data.values():
+                data['dbdemo'] = False
+        # </OpenUpgrade>
+
+        for package in self.values():
+            for k, v in additional_data[package.name].items():
+                setattr(package, k, v)
+
+
+    def _add_modules(self, cr, module_list, force=None):
+        if force is None:
+            force = []
+        packages = []
+        len_graph = len(self)
+
+        # <OpenUpgrade:ADD>
+        # force additional dependencies for the upgrade process if given
+        # in config file
+        forced_deps = tools.config.get_misc('openupgrade', 'force_deps', '{}')
+        forced_deps = tools.config.get_misc('openupgrade',
+                                            'force_deps_' + odoo.release.version,
+                                            forced_deps)
+        forced_deps = safe_eval(forced_deps)
+        # </OpenUpgrade>
+
+        for module in module_list:
+            # This will raise an exception if no/unreadable descriptor file.
+            # NOTE The call to load_information_from_description_file is already
+            # done by db.initialize, so it is possible to not do it again here.
+            info = odoo.modules.module.load_information_from_description_file(module)
+            if info and info['installable']:
+                # <OpenUpgrade:ADD>
+                info['depends'].extend(forced_deps.get(module, []))
+                # </OpenUpgrade>
+                packages.append((module, info)) # TODO directly a dict, like in get_modules_with_version
+            elif module != 'studio_customization':
+                _logger.warning('module %s: not installable, skipped', module)
+
+        dependencies = dict([(p, info['depends']) for p, info in packages])
+        current, later = set([p for p, info in packages]), set()
+
+        while packages and current > later:
+            package, info = packages[0]
+            deps = info['depends']
+
+            # if all dependencies of 'package' are already in the graph, add 'package' in the graph
+            if all(dep in self for dep in deps):
+                if not package in current:
+                    packages.pop(0)
+                    continue
+                later.clear()
+                current.remove(package)
+                node = self.add_node(package, info)
+                for kind in ('init', 'demo', 'update'):
+                    if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force:
+                        setattr(node, kind, True)
+            else:
+                later.add(package)
+                packages.append((package, info))
+            packages.pop(0)
+
+        self.update_from_db(cr)
+
+        for package in later:
+            unmet_deps = [p for p in dependencies[package] if p not in self]
+            _logger.info('module %s: Unmet dependencies: %s', package, ', '.join(unmet_deps))
+
+        return len(self) - len_graph
+
+
+Graph.update_from_db = _update_from_db
+Graph.add_modules = _add_modules

upgrade_analysis/odoo_patch/odoo/modules/loading.py

@@ -0,0 +1,556 @@
+# flake8: noqa
+# pylint: skip-file
+
+import itertools
+import logging
+import sys
+import time
+
+import odoo
+import odoo.tools as tools
+from odoo import api, SUPERUSER_ID
+from odoo.modules import loading
+from odoo.modules.module import adapt_version, load_openerp_module, initialize_sys_path
+
+from odoo.modules.loading import load_data, load_demo, _check_module_names
+from odoo.addons.openupgrade_framework.openupgrade import openupgrade_loading
+
+import os
+
+_logger = logging.getLogger(__name__)
+_test_logger = logging.getLogger('odoo.tests')
+
+
+def _load_module_graph(cr, graph, status=None, perform_checks=True,
+                      skip_modules=None, report=None, models_to_check=None, upg_registry=None):
+    # <OpenUpgrade:CHANGED-SIGNATURE/>
+    """Migrates+Updates or Installs all module nodes from ``graph``
+       :param graph: graph of module nodes to load
+       :param status: deprecated parameter, unused, left to avoid changing signature in 8.0
+       :param perform_checks: whether module descriptors should be checked for validity (prints warnings
+                              for same cases)
+       :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
+       :return: list of modules that were installed or updated
+    """
+    if skip_modules is None:
+        skip_modules = []
+
+    if models_to_check is None:
+        models_to_check = set()
+
+    processed_modules = []
+    loaded_modules = []
+    registry = odoo.registry(cr.dbname)
+    migrations = odoo.modules.migration.MigrationManager(cr, graph)
+    module_count = len(graph)
+    _logger.info('loading %d modules...', module_count)
+
+    # <OpenUpgrade:ADD>
+    # suppress commits to have the upgrade of one module in just one transaction
+    cr.commit_org = cr.commit
+    cr.commit = lambda *args: None
+    cr.rollback_org = cr.rollback
+    cr.rollback = lambda *args: None
+    # </OpenUpgrade>
+
+    # register, instantiate and initialize models for each modules
+    t0 = time.time()
+    loading_extra_query_count = odoo.sql_db.sql_counter
+    loading_cursor_query_count = cr.sql_log_count
+
+    models_updated = set()
+
+    for index, package in enumerate(graph, 1):
+        module_name = package.name
+        module_id = package.id
+
+        # <OpenUpgrade:CHANGE>
+        if module_name in skip_modules or module_name in loaded_modules:
+            # </OpenUpgrade>
+            continue
+
+        module_t0 = time.time()
+        module_cursor_query_count = cr.sql_log_count
+        module_extra_query_count = odoo.sql_db.sql_counter
+
+        needs_update = (
+            hasattr(package, "init")
+            or hasattr(package, "update")
+            or package.state in ("to install", "to upgrade")
+        )
+        module_log_level = logging.DEBUG
+        if needs_update:
+            module_log_level = logging.INFO
+        _logger.log(module_log_level, 'Loading module %s (%d/%d)', module_name, index, module_count)
+
+        if needs_update:
+            if package.name != 'base':
+                registry.setup_models(cr)
+            migrations.migrate_module(package, 'pre')
+            if package.name != 'base':
+                env = api.Environment(cr, SUPERUSER_ID, {})
+                env['base'].flush()
+
+        load_openerp_module(package.name)
+
+        new_install = package.state == 'to install'
+        if new_install:
+            py_module = sys.modules['odoo.addons.%s' % (module_name,)]
+            pre_init = package.info.get('pre_init_hook')
+            if pre_init:
+                getattr(py_module, pre_init)(cr)
+
+        model_names = registry.load(cr, package)
+
+        mode = 'update'
+        if hasattr(package, 'init') or package.state == 'to install':
+            mode = 'init'
+
+        loaded_modules.append(package.name)
+        if needs_update:
+            models_updated |= set(model_names)
+            models_to_check -= set(model_names)
+            registry.setup_models(cr)
+            # <OpenUpgrade:ADD>
+            # rebuild the local registry based on the loaded models
+            local_registry = {}
+            env = api.Environment(cr, SUPERUSER_ID, {})
+            for model in env.values():
+                if not model._auto:
+                    continue
+                openupgrade_loading.log_model(model, local_registry)
+            openupgrade_loading.compare_registries(
+                cr, package.name, upg_registry, local_registry)
+            # </OpenUpgrade>
+
+            registry.init_models(cr, model_names, {'module': package.name}, new_install)
+        elif package.state != 'to remove':
+            # The current module has simply been loaded. The models extended by this module
+            # and for which we updated the schema, must have their schema checked again.
+            # This is because the extension may have changed the model,
+            # e.g. adding required=True to an existing field, but the schema has not been
+            # updated by this module because it's not marked as 'to upgrade/to install'.
+            models_to_check |= set(model_names) & models_updated
+
+        idref = {}
+
+        if needs_update:
+            env = api.Environment(cr, SUPERUSER_ID, {})
+            # Can't put this line out of the loop: ir.module.module will be
+            # registered by init_models() above.
+            module = env['ir.module.module'].browse(module_id)
+
+            if perform_checks:
+                module._check()
+
+            if package.state == 'to upgrade':
+                # upgrading the module information
+                module.write(module.get_values_from_terp(package.data))
+            load_data(cr, idref, mode, kind='data', package=package)
+            demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode)
+            cr.execute('update ir_module_module set demo=%s where id=%s', (demo_loaded, module_id))
+            module.invalidate_cache(['demo'])
+
+            # <OpenUpgrade:CHANGE>
+            # add 'try' block for logging exceptions
+            # as errors in post scripts seem to be dropped
+            try:
+                migrations.migrate_module(package, 'post')
+            except Exception as exc:
+                _logger.error('Error executing post migration script for module %s: %s',
+                              package, exc)
+                raise
+            # </OpenUpgrade>
+
+            # Update translations for all installed languages
+            overwrite = odoo.tools.config["overwrite_existing_translations"]
+            module.with_context(overwrite=overwrite)._update_translations()
+
+        if package.name is not None:
+            registry._init_modules.add(package.name)
+
+        if needs_update:
+            if new_install:
+                post_init = package.info.get('post_init_hook')
+                if post_init:
+                    getattr(py_module, post_init)(cr, registry)
+
+            if mode == 'update':
+                # validate the views that have not been checked yet
+                env['ir.ui.view']._validate_module_views(module_name)
+
+            # need to commit any modification the module's installation or
+            # update made to the schema or data so the tests can run
+            # (separately in their own transaction)
+            # <OpenUpgrade:CHANGE>
+            # commit after processing every module as well, for
+            # easier debugging and continuing an interrupted migration
+            cr.commit_org()
+            # </OpenUpgrade
+
+        updating = tools.config.options['init'] or tools.config.options['update']
+        test_time = test_queries = 0
+        test_results = None
+        if tools.config.options['test_enable'] and (needs_update or not updating):
+            env = api.Environment(cr, SUPERUSER_ID, {})
+            loader = odoo.tests.loader
+            suite = loader.make_suite(module_name, 'at_install')
+            if suite.countTestCases():
+                if not needs_update:
+                    registry.setup_models(cr)
+                # Python tests
+                env['ir.http']._clear_routing_map()     # force routing map to be rebuilt
+
+                tests_t0, tests_q0 = time.time(), odoo.sql_db.sql_counter
+                test_results = loader.run_suite(suite, module_name)
+                report.update(test_results)
+                test_time = time.time() - tests_t0
+                test_queries = odoo.sql_db.sql_counter - tests_q0
+
+                # tests may have reset the environment
+                env = api.Environment(cr, SUPERUSER_ID, {})
+                module = env['ir.module.module'].browse(module_id)
+
+        if needs_update:
+            # <OpenUpgrade:CHANGE>
+            # run tests
+            if os.environ.get('OPENUPGRADE_TESTS') and package.name is not None:
+                prefix = '.migrations'
+                registry.openupgrade_test_prefixes[package.name] = prefix
+                report.record_result(odoo.modules.module.run_unit_tests(module_name, openupgrade_prefix=prefix))
+            # </OpenUpgrade
+
+            processed_modules.append(package.name)
+
+            ver = adapt_version(package.data['version'])
+            # Set new modules and dependencies
+            module.write({'state': 'installed', 'latest_version': ver})
+            # <OpenUpgrade:ADD>
+            # commit module_n state and version immediatly
+            # to avoid invalid database state if module_n+1 raises an
+            # exception
+            cr.commit_org()
+            # </OpenUpgrade>
+
+            package.load_state = package.state
+            package.load_version = package.installed_version
+            package.state = 'installed'
+            for kind in ('init', 'demo', 'update'):
+                if hasattr(package, kind):
+                    delattr(package, kind)
+            module.flush()
+
+        extra_queries = odoo.sql_db.sql_counter - module_extra_query_count - test_queries
+        extras = []
+        if test_queries:
+            extras.append(f'+{test_queries} test')
+        if extra_queries:
+            extras.append(f'+{extra_queries} other')
+        _logger.log(
+            module_log_level, "Module %s loaded in %.2fs%s, %s queries%s",
+            module_name, time.time() - module_t0,
+            f' (incl. {test_time:.2f}s test)' if test_time else '',
+            cr.sql_log_count - module_cursor_query_count,
+            f' ({", ".join(extras)})' if extras else ''
+        )
+        if test_results and not test_results.wasSuccessful():
+            _logger.error(
+                "Module %s: %d failures, %d errors of %d tests",
+                module_name, len(test_results.failures), len(test_results.errors),
+                test_results.testsRun
+            )
+
+    _logger.runbot("%s modules loaded in %.2fs, %s queries (+%s extra)",
+                   len(graph),
+                   time.time() - t0,
+                   cr.sql_log_count - loading_cursor_query_count,
+                   odoo.sql_db.sql_counter - loading_extra_query_count)  # extra queries: testes, notify, any other closed cursor
+
+    # <OpenUpgrade:ADD>
+    # restore commit method
+    cr.commit = cr.commit_org
+    cr.commit()
+    # </OpenUpgrade>
+
+    return loaded_modules, processed_modules
+
+
+def _load_marked_modules(cr, graph, states, force, progressdict, report,
+                        loaded_modules, perform_checks, models_to_check=None, upg_registry=None):
+    # <OpenUpgrade:CHANGED-SIGNATURE/>
+    """Loads modules marked with ``states``, adding them to ``graph`` and
+       ``loaded_modules`` and returns a list of installed/upgraded modules."""
+
+    if models_to_check is None:
+        models_to_check = set()
+
+    processed_modules = []
+    while True:
+        cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),))
+        module_list = [name for (name,) in cr.fetchall() if name not in graph]
+        # <OpenUpgrade:ADD>
+        module_list = openupgrade_loading.add_module_dependencies(cr, module_list)
+        # </OpenUpgrade>
+        if not module_list:
+            break
+        graph.add_modules(cr, module_list, force)
+        _logger.debug('Updating graph with %d more modules', len(module_list))
+        # <OpenUpgrade:CHANGE>
+        # add upg_registry
+        loaded, processed = _load_module_graph(
+            cr, graph, progressdict, report=report, skip_modules=loaded_modules,
+            perform_checks=perform_checks, models_to_check=models_to_check,
+            upg_registry=upg_registry,
+        )
+        # </OpenUpgrade>
+        processed_modules.extend(processed)
+        loaded_modules.extend(loaded)
+        if not processed:
+            break
+    return processed_modules
+
+
+def _load_modules(db, force_demo=False, status=None, update_module=False):
+    initialize_sys_path()
+
+    force = []
+    if force_demo:
+        force.append('demo')
+
+    # <OpenUpgrade:ADD>
+    upg_registry = {}
+    # </OpenUpgrade>
+
+    models_to_check = set()
+
+    with db.cursor() as cr:
+        if not odoo.modules.db.is_initialized(cr):
+            if not update_module:
+                _logger.error("Database %s not initialized, you can force it with `-i base`", cr.dbname)
+                return
+            _logger.info("init db")
+            odoo.modules.db.initialize(cr)
+            update_module = True # process auto-installed modules
+            tools.config["init"]["all"] = 1
+            if not tools.config['without_demo']:
+                tools.config["demo"]['all'] = 1
+
+        # This is a brand new registry, just created in
+        # odoo.modules.registry.Registry.new().
+        registry = odoo.registry(cr.dbname)
+
+        if 'base' in tools.config['update'] or 'all' in tools.config['update']:
+            cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
+
+        # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
+        graph = odoo.modules.graph.Graph()
+        graph.add_module(cr, 'base', force)
+        if not graph:
+            _logger.critical('module base cannot be loaded! (hint: verify addons-path)')
+            raise ImportError('Module `base` cannot be loaded! (hint: verify addons-path)')
+
+        # processed_modules: for cleanup step after install
+        # loaded_modules: to avoid double loading
+        report = registry._assertion_report
+        # <OpenUpgrade:CHANGE>
+        # add upg_registry
+        loaded_modules, processed_modules = _load_module_graph(
+            cr, graph, status, perform_checks=update_module,
+            report=report, models_to_check=models_to_check, upg_registry=upg_registry)
+
+        # </OpenUpgrade>
+        load_lang = tools.config.pop('load_language')
+        if load_lang or update_module:
+            # some base models are used below, so make sure they are set up
+            registry.setup_models(cr)
+
+        if load_lang:
+            for lang in load_lang.split(','):
+                tools.load_language(cr, lang)
+
+        # STEP 2: Mark other modules to be loaded/updated
+        if update_module:
+            env = api.Environment(cr, SUPERUSER_ID, {})
+            Module = env['ir.module.module']
+            _logger.info('updating modules list')
+            Module.update_list()
+
+            _check_module_names(cr, itertools.chain(tools.config['init'], tools.config['update']))
+
+            module_names = [k for k, v in tools.config['init'].items() if v]
+            if module_names:
+                modules = Module.search([('state', '=', 'uninstalled'), ('name', 'in', module_names)])
+                if modules:
+                    modules.button_install()
+
+            module_names = [k for k, v in tools.config['update'].items() if v]
+            if module_names:
+                # <OpenUpgrade:CHANGE>
+                # in standard Odoo, '--update all' just means:
+                # '--update base + upward (installed) dependencies. This breaks
+                # the chain when new glue modules are encountered.
+                # E.g. purchase in 8.0 depends on stock_account and report,
+                # both of which are new. They may be installed, but purchase as
+                # an upward dependency is not selected for upgrade.
+                # Therefore, explicitely select all installed modules for
+                # upgrading in OpenUpgrade in that case.
+                domain = [('state', '=', 'installed')]
+                if 'all' not in module_names:
+                    domain.append(('name', 'in', module_names))
+                modules = Module.search(domain)
+                # </OpenUpgrade>
+                if modules:
+                    modules.button_upgrade()
+
+            cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
+            Module.invalidate_cache(['state'])
+            Module.flush()
+
+        # STEP 3: Load marked modules (skipping base which was done in STEP 1)
+        # IMPORTANT: this is done in two parts, first loading all installed or
+        #            partially installed modules (i.e. installed/to upgrade), to
+        #            offer a consistent system to the second part: installing
+        #            newly selected modules.
+        #            We include the modules 'to remove' in the first step, because
+        #            they are part of the "currently installed" modules. They will
+        #            be dropped in STEP 6 later, before restarting the loading
+        #            process.
+        # IMPORTANT 2: We have to loop here until all relevant modules have been
+        #              processed, because in some rare cases the dependencies have
+        #              changed, and modules that depend on an uninstalled module
+        #              will not be processed on the first pass.
+        #              It's especially useful for migrations.
+        previously_processed = -1
+        while previously_processed < len(processed_modules):
+            previously_processed = len(processed_modules)
+            # <OpenUpgrade:CHANGE>
+            # add upg_registry
+            processed_modules += _load_marked_modules(cr, graph,
+                ['installed', 'to upgrade', 'to remove'],
+                force, status, report, loaded_modules, update_module, models_to_check, upg_registry)
+            # </OpenUpgrade>
+            if update_module:
+                # <OpenUpgrade:CHANGE>
+                # add upg_registry
+                processed_modules += _load_marked_modules(cr, graph,
+                    ['to install'], force, status, report,
+                    loaded_modules, update_module, models_to_check, upg_registry)
+                # </OpenUpgrade>
+        # check that new module dependencies have been properly installed after a migration/upgrade
+        cr.execute("SELECT name from ir_module_module WHERE state IN ('to install', 'to upgrade')")
+        module_list = [name for (name,) in cr.fetchall()]
+        if module_list:
+            _logger.error("Some modules have inconsistent states, some dependencies may be missing: %s", sorted(module_list))
+
+        # check that all installed modules have been loaded by the registry after a migration/upgrade
+        cr.execute("SELECT name from ir_module_module WHERE state = 'installed' and name != 'studio_customization'")
+        module_list = [name for (name,) in cr.fetchall() if name not in graph]
+        if module_list:
+            _logger.error("Some modules are not loaded, some dependencies or manifest may be missing: %s", sorted(module_list))
+
+        registry.loaded = True
+        registry.setup_models(cr)
+
+        # STEP 3.5: execute migration end-scripts
+        migrations = odoo.modules.migration.MigrationManager(cr, graph)
+        for package in graph:
+            migrations.migrate_module(package, 'end')
+
+        # STEP 3.6: apply remaining constraints in case of an upgrade
+        registry.finalize_constraints()
+
+        # STEP 4: Finish and cleanup installations
+        if processed_modules:
+            env = api.Environment(cr, SUPERUSER_ID, {})
+            cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
+            for (model, name) in cr.fetchall():
+                if model in registry and not registry[model]._abstract:
+                    _logger.warning('The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,base.group_user,1,0,0,0',
+                        model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_'))
+
+            cr.execute("SELECT model from ir_model")
+            for (model,) in cr.fetchall():
+                if model in registry:
+                    env[model]._check_removed_columns(log=True)
+                elif _logger.isEnabledFor(logging.INFO):    # more an info that a warning...
+                    _logger.runbot("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model)
+
+            # Cleanup orphan records
+            env['ir.model.data']._process_end(processed_modules)
+            env['base'].flush()
+
+        for kind in ('init', 'demo', 'update'):
+            tools.config[kind] = {}
+
+        # STEP 5: Uninstall modules to remove
+        if update_module:
+            # Remove records referenced from ir_model_data for modules to be
+            # removed (and removed the references from ir_model_data).
+            cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s", ('to remove',))
+            modules_to_remove = dict(cr.fetchall())
+            if modules_to_remove:
+                env = api.Environment(cr, SUPERUSER_ID, {})
+                pkgs = reversed([p for p in graph if p.name in modules_to_remove])
+                for pkg in pkgs:
+                    uninstall_hook = pkg.info.get('uninstall_hook')
+                    if uninstall_hook:
+                        py_module = sys.modules['odoo.addons.%s' % (pkg.name,)]
+                        getattr(py_module, uninstall_hook)(cr, registry)
+
+                Module = env['ir.module.module']
+                Module.browse(modules_to_remove.values()).module_uninstall()
+                # Recursive reload, should only happen once, because there should be no
+                # modules to remove next time
+                cr.commit()
+                _logger.info('Reloading registry once more after uninstalling modules')
+                api.Environment.reset()
+                registry = odoo.modules.registry.Registry.new(
+                    cr.dbname, force_demo, status, update_module
+                )
+                registry.check_tables_exist(cr)
+                cr.commit()
+                return registry
+
+        # STEP 5.5: Verify extended fields on every model
+        # This will fix the schema of all models in a situation such as:
+        #   - module A is loaded and defines model M;
+        #   - module B is installed/upgraded and extends model M;
+        #   - module C is loaded and extends model M;
+        #   - module B and C depend on A but not on each other;
+        # The changes introduced by module C are not taken into account by the upgrade of B.
+        if models_to_check:
+            registry.init_models(cr, list(models_to_check), {'models_to_check': True})
+
+        # STEP 6: verify custom views on every model
+        if update_module:
+            env = api.Environment(cr, SUPERUSER_ID, {})
+            env['res.groups']._update_user_groups_view()
+            View = env['ir.ui.view']
+            for model in registry:
+                try:
+                    View._validate_custom_views(model)
+                except Exception as e:
+                    _logger.warning('invalid custom view(s) for model %s: %s', model, tools.ustr(e))
+
+        if report.wasSuccessful():
+            _logger.info('Modules loaded.')
+        else:
+            _logger.error('At least one test failed when loading the modules.')
+
+        # STEP 8: call _register_hook on every model
+        # This is done *exactly once* when the registry is being loaded. See the
+        # management of those hooks in `Registry.setup_models`: all the calls to
+        # setup_models() done here do not mess up with hooks, as registry.ready
+        # is False.
+        env = api.Environment(cr, SUPERUSER_ID, {})
+        for model in env.values():
+            model._register_hook()
+        env['base'].flush()
+
+        # STEP 9: save installed/updated modules for post-install tests
+        registry.updated_modules += processed_modules
+
+loading.load_module_graph = _load_module_graph
+loading.load_marked_modules = _load_marked_modules
+loading.load_modules = _load_modules
+odoo.modules.load_modules = _load_modules

upgrade_analysis/odoo_patch/odoo/modules/migration.py

@@ -0,0 +1,118 @@
+# flake8: noqa
+# pylint: skip-file
+
+import logging
+import os
+from os.path import join as opj
+import odoo.release as release
+from odoo.tools.parse_version import parse_version
+
+import odoo
+from odoo.modules.migration import load_script
+from odoo.modules import migration
+
+_logger = logging.getLogger(__name__)
+
+
+if True:
+    def _migrate_module(self, pkg, stage):
+        assert stage in ('pre', 'post', 'end')
+        stageformat = {
+            'pre': '[>%s]',
+            'post': '[%s>]',
+            'end': '[$%s]',
+        }
+        state = pkg.state if stage in ('pre', 'post') else getattr(pkg, 'load_state', None)
+
+        # <OpenUpgrade:CHANGE>
+        # In openupgrade, also run migration scripts upon installation.
+        # We want to always pass in pre and post migration files and use a new
+        # argument in the migrate decorator (explained in the docstring)
+        # to decide if we want to do something if a new module is installed
+        # during the migration.
+        if not (hasattr(pkg, 'update') or state in ('to upgrade', 'to install')):
+        # </OpenUpgrade>
+            return
+
+        def convert_version(version):
+            if version.count('.') >= 2:
+                return version  # the version number already containt the server version
+            return "%s.%s" % (release.major_version, version)
+
+        def _get_migration_versions(pkg, stage):
+            versions = sorted({
+                ver
+                for lv in self.migrations[pkg.name].values()
+                for ver, lf in lv.items()
+                if lf
+            }, key=lambda k: parse_version(convert_version(k)))
+            if "0.0.0" in versions:
+                # reorder versions
+                versions.remove("0.0.0")
+                if stage == "pre":
+                    versions.insert(0, "0.0.0")
+                else:
+                    versions.append("0.0.0")
+            return versions
+
+        def _get_migration_files(pkg, version, stage):
+            """ return a list of migration script files
+            """
+            m = self.migrations[pkg.name]
+            lst = []
+
+            mapping = {
+                'module': opj(pkg.name, 'migrations'),
+                'module_upgrades': opj(pkg.name, 'upgrades'),
+            }
+
+            for path in odoo.upgrade.__path__:
+                if os.path.exists(opj(path, pkg.name)):
+                    mapping['upgrade'] = opj(path, pkg.name)
+                    break
+
+            for x in mapping:
+                if version in m.get(x):
+                    for f in m[x][version]:
+                        if not f.startswith(stage + '-'):
+                            continue
+                        lst.append(opj(mapping[x], version, f))
+            lst.sort()
+            return lst
+
+        installed_version = getattr(pkg, 'load_version', pkg.installed_version) or ''
+        parsed_installed_version = parse_version(installed_version)
+        current_version = parse_version(convert_version(pkg.data['version']))
+
+        versions = _get_migration_versions(pkg, stage)
+
+        for version in versions:
+            if ((version == "0.0.0" and parsed_installed_version < current_version)
+               or parsed_installed_version < parse_version(convert_version(version)) <= current_version):
+
+                strfmt = {'addon': pkg.name,
+                          'stage': stage,
+                          'version': stageformat[stage] % version,
+                          }
+
+                for pyfile in _get_migration_files(pkg, version, stage):
+                    name, ext = os.path.splitext(os.path.basename(pyfile))
+                    if ext.lower() != '.py':
+                        continue
+                    mod = None
+                    try:
+                        mod = load_script(pyfile, name)
+                        _logger.info('module %(addon)s: Running migration %(version)s %(name)s' % dict(strfmt, name=mod.__name__))
+                        migrate = mod.migrate
+                    except ImportError:
+                        _logger.exception('module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % dict(strfmt, file=pyfile))
+                        raise
+                    except AttributeError:
+                        _logger.error('module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt)
+                    else:
+                        migrate(self.cr, installed_version)
+                    finally:
+                        if mod:
+                            del mod
+
+migration.migrate_module = _migrate_module

upgrade_analysis/odoo_patch/odoo/modules/registry.py

@@ -0,0 +1,58 @@
+# flake8: noqa
+# pylint: skip-file
+
+from collections import deque
+from contextlib import closing
+import odoo
+from odoo.tools.lru import LRU
+
+from odoo.modules import registry
+
+
+if True:
+
+    def _init(self, db_name):
+        self.models = {}    # model name/model instance mapping
+        self._sql_constraints = set()
+        self._init = True
+        self._assertion_report = odoo.tests.runner.OdooTestResult()
+        self._fields_by_model = None
+        self._ordinary_tables = None
+        self._constraint_queue = deque()
+        self.__cache = LRU(8192)
+
+        # modules fully loaded (maintained during init phase by `loading` module)
+        self._init_modules = set()
+        self.updated_modules = []       # installed/updated modules
+        # <OpenUpgrade:ADD>
+        self.openupgrade_test_prefixes = {}
+        # </OpenUpgrade>
+        self.loaded_xmlids = set()
+
+        self.db_name = db_name
+        self._db = odoo.sql_db.db_connect(db_name)
+
+        # cursor for test mode; None means "normal" mode
+        self.test_cr = None
+        self.test_lock = None
+
+        # Indicates that the registry is
+        self.loaded = False             # whether all modules are loaded
+        self.ready = False              # whether everything is set up
+
+        # Inter-process signaling:
+        # The `base_registry_signaling` sequence indicates the whole registry
+        # must be reloaded.
+        # The `base_cache_signaling sequence` indicates all caches must be
+        # invalidated (i.e. cleared).
+        self.registry_sequence = None
+        self.cache_sequence = None
+
+        # Flags indicating invalidation of the registry or the cache.
+        self.registry_invalidated = False
+        self.cache_invalidated = False
+
+        with closing(self.cursor()) as cr:
+            self.has_unaccent = odoo.modules.db.has_unaccent(cr)
+
+registry.init = _init

upgrade_analysis/odoo_patch/odoo/service/__init__.py

@@ -0,0 +1,4 @@
+# Import disabled, because the function run_unit_tests()
+# disappeared in V14.
+# TODO: OpenUpgrade Core maintainers : FIXME.
+# from . import server

upgrade_analysis/odoo_patch/odoo/service/server.py

@@ -0,0 +1,71 @@
+# flake8: noqa
+# pylint: skip-file
+
+import logging
+import os
+import time
+
+import odoo
+from odoo.tools import config
+from odoo.modules.registry import Registry
+
+from odoo.service import server
+from odoo.service.server import load_test_file_py
+
+_logger = logging.getLogger(__name__)
+
+
+def preload_registries(dbnames):
+    """ Preload a registries, possibly run a test file."""
+    # TODO: move all config checks to args dont check tools.config here
+    dbnames = dbnames or []
+    rc = 0
+    for dbname in dbnames:
+        try:
+            update_module = config['init'] or config['update']
+            registry = Registry.new(dbname, update_module=update_module)
+
+            # run test_file if provided
+            if config['test_file']:
+                test_file = config['test_file']
+                if not os.path.isfile(test_file):
+                    _logger.warning('test file %s cannot be found', test_file)
+                elif not test_file.endswith('py'):
+                    _logger.warning('test file %s is not a python file', test_file)
+                else:
+                    _logger.info('loading test file %s', test_file)
+                    with odoo.api.Environment.manage():
+                        load_test_file_py(registry, test_file)
+
+            # run post-install tests
+            if config['test_enable']:
+                t0 = time.time()
+                t0_sql = odoo.sql_db.sql_counter
+                module_names = (registry.updated_modules if update_module else
+                                sorted(registry._init_modules))
+                _logger.info("Starting post tests")
+                tests_before = registry._assertion_report.testsRun
+                with odoo.api.Environment.manage():
+                    for module_name in module_names:
+                        result = loader.run_suite(loader.make_suite(module_name, 'post_install'), module_name)
+                        registry._assertion_report.update(result)
+                    # <OpenUpgrade:ADD>
+                    # run deferred unit tests
+                    for module_name, prefix in registry.openupgrade_test_prefixes:
+                        result = run_unit_tests(module_name, position='post_install', openupgrade_prefix=prefix)
+                        registry._assertion_report.record_result(result)
+                    # </OpenUpgrade>
+                _logger.info("%d post-tests in %.2fs, %s queries",
+                             registry._assertion_report.testsRun - tests_before,
+                             time.time() - t0,
+                             odoo.sql_db.sql_counter - t0_sql)
+
+            if not registry._assertion_report.wasSuccessful():
+                rc += 1
+        except Exception:
+            _logger.critical('Failed to initialize database `%s`.', dbname, exc_info=True)
+            return -1
+    return rc
+
+
+server.preload_registries = preload_registries

upgrade_analysis/odoo_patch/odoo/tools/__init__.py

@@ -0,0 +1,2 @@
+from . import convert
+from . import view_validation

upgrade_analysis/odoo_patch/odoo/tools/convert.py

@@ -0,0 +1,23 @@
+# flake8: noqa
+# pylint: skip-file
+
+from .... import upgrade_log
+
+from odoo.tools.convert import xml_import
+
+if True:
+
+    def __test_xml_id(self, xml_id):
+        if '.' in xml_id:
+            module, id = xml_id.split('.', 1)
+            assert '.' not in id, """The ID reference "%s" must contain
+maximum one dot. They are used to refer to other modules ID, in the
+form: module.record_id""" % (xml_id,)
+            if module != self.module:
+                modcnt = self.env['ir.module.module'].search_count([('name', '=', module), ('state', '=', 'installed')])
+                assert modcnt == 1, """The ID "%s" refers to an uninstalled module""" % (xml_id,)
+
+        # OpenUpgrade: log entry of XML imports
+        openupgrade_log.log_xml_id(self.env.cr, self.module, xml_id)
+
+xml_import._test_xml_id = __test_xml_id

upgrade_analysis/odoo_patch/odoo/tools/view_validation.py

@@ -0,0 +1,29 @@
+# flake8: noqa
+# pylint: skip-file
+
+# from odoo.addons.openupgrade_framework.openupgrade import openupgrade_log
+
+from odoo.tools import view_validation
+from odoo.tools.view_validation import _validators, _logger
+
+
+def _valid_view(arch, **kwargs):
+    for pred in _validators[arch.tag]:
+        # <OpenUpgrade:CHANGE>
+        # Do not raise blocking error, because it's normal to
+        # have inconsistent views in an openupgrade process
+        check = pred(arch, **kwargs) or 'Warning'
+        # </OpenUpgrade>
+        if not check:
+            _logger.error("Invalid XML: %s", pred.__doc__)
+            return False
+        if check == "Warning":
+            # <OpenUpgrade:REM>
+            # Don't show this warning as useless and too much verbose
+            # _logger.warning("Invalid XML: %s", pred.__doc__)
+            # </OpenUpgrade>
+            return "Warning"
+    return True
+
+
+view_validation.valid_view = _valid_view

upgrade_analysis/openupgrade_loading.py

@@ -0,0 +1,318 @@
+# Copyright 2011-2015 Therp BV <https://therp.nl>
+# Copyright 2016-2019 Opener B.V. <https://opener.am>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+# flake8: noqa: C901
+
+import logging
+
+from openupgradelib.openupgrade_tools import table_exists
+
+from odoo import release
+from odoo.modules.module import get_module_path
+from odoo.tools.safe_eval import safe_eval
+from odoo.tools.config import config
+
+# A collection of functions used in
+# odoo/modules/loading.py
+
+logger = logging.getLogger("OpenUpgrade")
+
+
+def add_module_dependencies(cr, module_list):
+    """
+    Select (new) dependencies from the modules in the list
+    so that we can inject them into the graph at upgrade
+    time. Used in the modified OpenUpgrade Server,
+    not to be called from migration scripts
+
+    Also take the OpenUpgrade configuration directives 'forced_deps'
+    and 'autoinstall' into account. From any additional modules
+    that these directives can add, the dependencies are added as
+    well (but these directives are not checked for the occurrence
+    of any of the dependencies).
+    """
+    if not module_list:
+        return module_list
+
+    modules_in = list(module_list)
+    forced_deps = safe_eval(
+        config.get_misc(
+            "openupgrade",
+            "forced_deps_" + release.version,
+            config.get_misc("openupgrade", "forced_deps", "{}"),
+        )
+    )
+
+    autoinstall = safe_eval(
+        config.get_misc(
+            "openupgrade",
+            "autoinstall_" + release.version,
+            config.get_misc("openupgrade", "autoinstall", "{}"),
+        )
+    )
+
+    for module in list(module_list):
+        module_list += forced_deps.get(module, [])
+        module_list += autoinstall.get(module, [])
+
+    module_list = list(set(module_list))
+
+    dependencies = module_list
+    while dependencies:
+        cr.execute(
+            """
+            SELECT DISTINCT dep.name
+            FROM
+                ir_module_module,
+                ir_module_module_dependency dep
+            WHERE
+                module_id = ir_module_module.id
+                AND ir_module_module.name in %s
+                AND dep.name not in %s
+            """,
+            (
+                tuple(dependencies),
+                tuple(module_list),
+            ),
+        )
+
+        dependencies = [x[0] for x in cr.fetchall()]
+        module_list += dependencies
+
+    # Select auto_install modules of which all dependencies
+    # are fulfilled based on the modules we know are to be
+    # installed
+    cr.execute(
+        """
+        SELECT name from ir_module_module WHERE state IN %s
+        """,
+        (("installed", "to install", "to upgrade"),),
+    )
+    modules = list(set(module_list + [row[0] for row in cr.fetchall()]))
+    cr.execute(
+        """
+        SELECT name from ir_module_module m
+        WHERE auto_install IS TRUE
+            AND state = 'uninstalled'
+            AND NOT EXISTS(
+                SELECT id FROM ir_module_module_dependency d
+                WHERE d.module_id = m.id
+                AND name NOT IN %s)
+         """,
+        (tuple(modules),),
+    )
+    auto_modules = [row[0] for row in cr.fetchall() if get_module_path(row[0])]
+    if auto_modules:
+        logger.info("Selecting autoinstallable modules %s", ",".join(auto_modules))
+        module_list += auto_modules
+
+    # Set proper state for new dependencies so that any init scripts are run
+    cr.execute(
+        """
+        UPDATE ir_module_module SET state = 'to install'
+        WHERE name IN %s AND name NOT IN %s AND state = 'uninstalled'
+        """,
+        (tuple(module_list), tuple(modules_in)),
+    )
+    return module_list
+
+
+def log_model(model, local_registry):
+    """
+    OpenUpgrade: Store the characteristics of the BaseModel and its fields
+    in the local registry, so that we can compare changes with the
+    main registry
+    """
+
+    if not model._name:
+        return
+
+    typemap = {"monetary": "float"}
+
+    # Deferred import to prevent import loop
+    from odoo import models
+
+    # persistent models only
+    if isinstance(model, models.TransientModel):
+        return
+
+    def isfunction(model, k):
+        if (
+            model._fields[k].compute
+            and not model._fields[k].related
+            and not model._fields[k].company_dependent
+        ):
+            return "function"
+        return ""
+
+    def isproperty(model, k):
+        if model._fields[k].company_dependent:
+            return "property"
+        return ""
+
+    def isrelated(model, k):
+        if model._fields[k].related:
+            return "related"
+        return ""
+
+    def _get_relation(v):
+        if v.type in ("many2many", "many2one", "one2many"):
+            return v.comodel_name
+        elif v.type == "many2one_reference":
+            return v.model_field
+        else:
+            return ""
+
+    model_registry = local_registry.setdefault(model._name, {})
+    if model._inherits:
+        model_registry["_inherits"] = {"_inherits": str(model._inherits)}
+    for k, v in model._fields.items():
+        properties = {
+            "type": typemap.get(v.type, v.type),
+            "isfunction": isfunction(model, k),
+            "isproperty": isproperty(model, k),
+            "isrelated": isrelated(model, k),
+            "relation": _get_relation(v),
+            "table": v.relation if v.type == "many2many" else "",
+            "required": v.required and "required" or "",
+            "stored": v.store and "stored" or "",
+            "selection_keys": "",
+            "req_default": "",
+            "hasdefault": model._fields[k].default and "hasdefault" or "",
+            "inherits": "",
+        }
+        if v.type == "selection":
+            if isinstance(v.selection, (tuple, list)):
+                properties["selection_keys"] = str(sorted([x[0] for x in v.selection]))
+            else:
+                properties["selection_keys"] = "function"
+        elif v.type == "binary":
+            properties["attachment"] = str(getattr(v, "attachment", False))
+        default = model._fields[k].default
+        if v.required and default:
+            if (
+                callable(default)
+                or isinstance(default, str)
+                and getattr(model._fields[k], default, False)
+                and callable(getattr(model._fields[k], default))
+            ):
+                # todo: in OpenERP 5 (and in 6 as well),
+                # literals are wrapped in a lambda function
+                properties["req_default"] = "function"
+            else:
+                properties["req_default"] = str(default)
+        for key, value in properties.items():
+            if value:
+                model_registry.setdefault(k, {})[key] = value
+
+
+def get_record_id(cr, module, model, field, mode):
+    """
+    OpenUpgrade: get or create the id from the record table matching
+    the key parameter values
+    """
+    cr.execute(
+        "SELECT id FROM openupgrade_record "
+        "WHERE module = %s AND model = %s AND "
+        "field = %s AND mode = %s AND type = %s",
+        (module, model, field, mode, "field"),
+    )
+    record = cr.fetchone()
+    if record:
+        return record[0]
+    cr.execute(
+        "INSERT INTO openupgrade_record "
+        "(module, model, field, mode, type) "
+        "VALUES (%s, %s, %s, %s, %s)",
+        (module, model, field, mode, "field"),
+    )
+    cr.execute(
+        "SELECT id FROM openupgrade_record "
+        "WHERE module = %s AND model = %s AND "
+        "field = %s AND mode = %s AND type = %s",
+        (module, model, field, mode, "field"),
+    )
+    return cr.fetchone()[0]
+
+
+def compare_registries(cr, module, registry, local_registry):
+    """
+    OpenUpgrade: Compare the local registry with the global registry,
+    log any differences and merge the local registry with
+    the global one.
+    """
+    if not table_exists(cr, "openupgrade_record"):
+        return
+    for model, flds in local_registry.items():
+        registry.setdefault(model, {})
+        for field, attributes in flds.items():
+            old_field = registry[model].setdefault(field, {})
+            mode = old_field and "modify" or "create"
+            record_id = False
+            for key, value in attributes.items():
+                if key not in old_field or old_field[key] != value:
+                    if not record_id:
+                        record_id = get_record_id(cr, module, model, field, mode)
+                    cr.execute(
+                        "SELECT id FROM openupgrade_attribute "
+                        "WHERE name = %s AND value = %s AND "
+                        "record_id = %s",
+                        (key, value, record_id),
+                    )
+                    if not cr.fetchone():
+                        cr.execute(
+                            "INSERT INTO openupgrade_attribute "
+                            "(name, value, record_id) VALUES (%s, %s, %s)",
+                            (key, value, record_id),
+                        )
+                    old_field[key] = value
+
+
+def update_field_xmlid(model, field):
+    """OpenUpgrade edit start: In rare cases, an old module defined a field
+    on a model that is not defined in another module earlier in the
+    chain of inheritance. Then we need to assign the ir.model.fields'
+    xmlid to this other module, otherwise the column would be dropped
+    when uninstalling the first module.
+    An example is res.partner#display_name defined in 7.0 by
+    account_report_company, but now the field belongs to the base
+    module
+    Given that we arrive here in order of inheritance, we simply check
+    if the field's xmlid belongs to a module already loaded, and if not,
+    update the record with the correct module name."""
+    model.env.cr.execute(
+        "SELECT f.*, d.module, d.id as xmlid_id, d.name as xmlid "
+        "FROM ir_model_fields f LEFT JOIN ir_model_data d "
+        "ON f.id=d.res_id and d.model='ir.model.fields' WHERE f.model=%s",
+        (model._name,),
+    )
+    for rec in model.env.cr.dictfetchall():
+        if (
+            "module" in model.env.context
+            and rec["module"]
+            and rec["name"] in model._fields.keys()
+            and rec["module"] != model.env.context["module"]
+            and rec["module"] not in model.env.registry._init_modules
+        ):
+            logging.getLogger(__name__).info(
+                "Moving XMLID for ir.model.fields record of %s#%s " "from %s to %s",
+                model._name,
+                rec["name"],
+                rec["module"],
+                model.env.context["module"],
+            )
+            model.env.cr.execute(
+                "SELECT id FROM ir_model_data WHERE module=%(module)s "
+                "AND name=%(xmlid)s",
+                dict(rec, module=model.env.context["module"]),
+            )
+            if model.env.cr.fetchone():
+                logging.getLogger(__name__).info(
+                    "Aborting, an XMLID for this module already exists."
+                )
+                continue
+            model.env.cr.execute(
+                "UPDATE ir_model_data SET module=%(module)s " "WHERE id=%(xmlid_id)s",
+                dict(rec, module=model.env.context["module"]),
+            )

upgrade_analysis/openupgrade_log.py

@@ -0,0 +1,60 @@
+# coding: utf-8
+# Copyright 2011-2015 Therp BV <https://therp.nl>
+# Copyright 2016 Opener B.V. <https://opener.am>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+from openupgradelib.openupgrade_tools import table_exists
+
+
+def log_xml_id(cr, module, xml_id):
+    """
+    Log xml_ids at load time in the records table.
+    Called from tools/convert.py:xml_import._test_xml_id()
+
+    # Catcha's
+    - The module needs to be loaded with 'init', or the calling method
+    won't be called. This can be brought about by installing the
+    module or updating the 'state' field of the module to 'to install'
+    or call the server with '--init <module>' and the database argument.
+
+    - Do you get the right results immediately when installing the module?
+    No, sorry. This method retrieves the model from the ir_model_table, but
+    when the xml id is encountered for the first time, this method is called
+    before the item is present in this table. Therefore, you will not
+    get any meaningful results until the *second* time that you 'init'
+    the module.
+
+    - The good news is that the openupgrade_records module that comes
+    with this distribution allows you to deal with all of this with
+    one click on the menu item Settings -> Customizations ->
+    Database Structure -> OpenUpgrade -> Generate Records
+
+    - You cannot reinitialize the modules in your production database
+    and expect to keep working on it happily ever after. Do not perform
+    this routine on your production database.
+
+    :param module: The module that contains the xml_id
+    :param xml_id: the xml_id, with or without 'module.' prefix
+    """
+    if not table_exists(cr, 'openupgrade_record'):
+        return
+    if '.' not in xml_id:
+        xml_id = '%s.%s' % (module, xml_id)
+    cr.execute(
+        "SELECT model FROM ir_model_data "
+        "WHERE module = %s AND name = %s",
+        xml_id.split('.'))
+    record = cr.fetchone()
+    if not record:
+        print("Cannot find xml_id %s" % xml_id)
+        return
+    else:
+        cr.execute(
+            "SELECT id FROM openupgrade_record "
+            "WHERE module=%s AND model=%s AND name=%s AND type=%s",
+            (module, record[0], xml_id, 'xmlid'))
+        if not cr.fetchone():
+            cr.execute(
+                "INSERT INTO openupgrade_record "
+                "(module, model, name, type) values(%s, %s, %s, %s)",
+                (module, record[0], xml_id, 'xmlid'))