Techsystech
/
server-tools
mirror of https://github.com/OCA/server-tools.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[RFR] Improve patchwork

pull/2417/head
Stefan Rijnhart 2020-12-01 10:35:52 +01:00
parent b7a3528af8
commit 3a2b838f41
28 changed files with 324 additions and 1579 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
upgrade_analysis/__init__.py
View File

@ -4,5 +4,4 @@ from . import wizards
from . import blacklist
from . import apriori
from . import compare
from . import upgrade_loading
from . import upgrade_log

17
upgrade_analysis/models/upgrade_analysis.py
View File

@ -3,6 +3,7 @@
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
# flake8: noqa: C901
import logging
import os
from odoo import fields, models
@ -11,6 +12,7 @@ from odoo.tools import config
from .. import compare
_logger = logging.getLogger(__name__)
_IGNORE_MODULES = ["openupgrade_records", "upgrade_analysis"]
@ -31,6 +33,10 @@ class UpgradeAnalysis(models.Model):
)
log = fields.Text(readonly=True)
upgrade_path = fields.Char(
default=config.get("upgrade_path", False),
help="The base file path to save the analyse files of Odoo modules",
)
write_files = fields.Boolean(
help="Write analysis files to the module directories", default=True
@ -50,10 +56,12 @@ class UpgradeAnalysis(models.Model):
):
module = self.env["ir.module.module"].search([("name", "=", module_name)])[0]
if module.is_odoo_module:
upgrade_path = config.get("upgrade_path", False)
if not upgrade_path:
return "ERROR: could not find 'upgrade_path' config:\n"
module_path = os.path.join(upgrade_path, module_name)
if not self.upgrade_path:
return (
"ERROR: no upgrade_path set when writing analysis of %s\n"
% module_name
)
module_path = os.path.join(self.upgrade_path, module_name)
else:
module_path = get_module_path(module_name)
if not module_path:
@ -71,6 +79,7 @@ class UpgradeAnalysis(models.Model):
f = open(logfile, "w")
except Exception:
return "ERROR: could not open file %s for writing:\n" % logfile
_logger.debug("Writing analysis to %s", logfile)
f.write(content)
f.close()
return None

3
upgrade_analysis/odoo_patch/__init__.py
View File

@ -1,2 +1,3 @@
from . import odoo
from . import addons
from . import odoo
from . import odoo_patch

2
upgrade_analysis/odoo_patch/addons/__init__.py
View File

@ -1,3 +1,3 @@
from . import mrp
from . import stock
from . import point_of_sale
from . import stock

21
upgrade_analysis/odoo_patch/addons/mrp/__init__.py
View File

@ -1,19 +1,10 @@
from odoo.addons import mrp
from ...odoo_patch import OdooPatch
def _pre_init_mrp(cr):
""" Allow installing MRP in databases with large stock.move table (>1M records)
- Creating the computed+stored field stock_move.is_done is terribly slow with the ORM and
leads to "Out of Memory" crashes
"""
# <OpenUpgrade:REMOVE>
# don't try to add 'is_done' column, because it will fail
# when executing the generation of records, in the openupgrade_records
# module.
# cr.execute("""ALTER TABLE "stock_move" ADD COLUMN "is_done" bool;""")
# cr.execute("""UPDATE stock_move
# SET is_done=COALESCE(state in ('done', 'cancel'), FALSE);""")
# </OpenUpgrade>
class PreInitHookPatch(OdooPatch):
target = mrp
method_names = ['_pre_init_mrp']
mrp._pre_init_mrp = _pre_init_mrp
def _pre_init_mrp(cr):
""" Don't try to create an existing column on reinstall """

1
upgrade_analysis/odoo_patch/addons/point_of_sale/__init__.py
View File

@ -1 +0,0 @@
from . import models

1
upgrade_analysis/odoo_patch/addons/point_of_sale/models/__init__.py
View File

@ -1 +0,0 @@
from . import pos_config

21
upgrade_analysis/odoo_patch/addons/point_of_sale/models/pos_config.py
View File

@ -1,21 +0,0 @@
from odoo import api
from odoo.addons.point_of_sale.models.pos_config import PosConfig
if True:
@api.model
def post_install_pos_localisation(self, companies=False):
# <OpenUpgrade:REMOVE>
# don't try to setup_defaults, because it will fail
# when executing the generation of records, in the openupgrade_records
# module.
# self = self.sudo()
# if not companies:
# companies = self.env['res.company'].search([])
# for company in companies.filtered('chart_template_id'):
# pos_configs = self.search([('company_id', '=', company.id)])
# pos_configs.setup_defaults(company)
pass
# </OpenUpgrade>
PosConfig.post_install_pos_localisation = post_install_pos_localisation

19
upgrade_analysis/odoo_patch/addons/stock/__init__.py
View File

@ -1,17 +1,10 @@
from odoo.addons import stock
from ...odoo_patch import OdooPatch
def pre_init_hook(cr):
# <OpenUpgrade:REMOVE>
# don't uninstall data as this breaks the analysis
# Origin of this code is https://github.com/odoo/odoo/issues/22243
# env = api.Environment(cr, SUPERUSER_ID, {})
# env['ir.model.data'].search([
# ('model', 'like', '%stock%'),
# ('module', '=', 'stock')
# ]).unlink()
pass
# </OpenUpgrade>
class PreInitHookPatch(OdooPatch):
target = stock
method_names = ['pre_init_hook']
stock.pre_init_hook = pre_init_hook
def pre_init_hook(cr):
""" Don't unlink stock data on reinstall """

6
upgrade_analysis/odoo_patch/odoo/__init__.py
View File

@ -1,5 +1,3 @@
from . import modules
from . import service
from . import tools
from . import http
from . import models
from . import modules
from . import tools

32
upgrade_analysis/odoo_patch/odoo/http.py
View File

@ -1,32 +0,0 @@
# flake8: noqa
# pylint: skip-file
import odoo
from odoo.service import security
from odoo.http import SessionExpiredException, request, OpenERPSession
if True:
def _check_security(self):
"""
Check the current authentication parameters to know if those are still
valid. This method should be called at each request. If the
authentication fails, a :exc:`SessionExpiredException` is raised.
"""
if not self.db or not self.uid:
raise SessionExpiredException("Session expired")
# We create our own environment instead of the request's one.
# to avoid creating it without the uid since request.uid isn't set yet
env = odoo.api.Environment(request.cr, self.uid, self.context)
# here we check if the session is still valid
if not security.check_session(self, env):
# <OpenUpgrade:ADD>
# When asking openupgrade_records to generate records
# over jsonrpc, a query on res_users in the call above locks this
# table for the sql operations that are triggered by the
# reinstallation of the base module
env.cr.rollback()
# </OpenUpgrade>
raise SessionExpiredException("Session expired")
OpenERPSession.check_security = _check_security

184
upgrade_analysis/odoo_patch/odoo/models.py
View File

@ -1,179 +1,21 @@
# flake8: noqa
# pylint: skip-file
import odoo
import psycopg2
from odoo import _
from odoo.models import fix_import_export_id_paths, BaseModel, _logger
from odoo import api, models
from ..odoo_patch import OdooPatch
from ... import upgrade_log
if True:
def _load(self, fields, data):
"""
Attempts to load the data matrix, and returns a list of ids (or
``False`` if there was an error and no id could be generated) and a
list of messages.
class BaseModelPatch(OdooPatch):
target = models.BaseModel
method_names = ['_convert_records']
The ids are those of the records created and saved (in database), in
the same order they were extracted from the file. They can be passed
directly to :meth:`~read`
:param fields: list of fields to import, at the same index as the corresponding data
:type fields: list(str)
:param data: row-major matrix of data to import
:type data: list(list(str))
:returns: {ids: list(int)|False, messages: [Message][, lastrow: int]}
"""
self.flush()
# determine values of mode, current_module and noupdate
mode = self._context.get('mode', 'init')
current_module = self._context.get('module', '__import__')
noupdate = self._context.get('noupdate', False)
# add current module in context for the conversion of xml ids
self = self.with_context(_import_current_module=current_module)
cr = self._cr
cr.execute('SAVEPOINT model_load')
fields = [fix_import_export_id_paths(f) for f in fields]
fg = self.fields_get()
ids = []
messages = []
ModelData = self.env['ir.model.data']
# list of (xid, vals, info) for records to be created in batch
batch = []
batch_xml_ids = set()
# models in which we may have created / modified data, therefore might
# require flushing in order to name_search: the root model and any
# o2m
creatable_models = {self._name}
for field_path in fields:
if field_path[0] in (None, 'id', '.id'):
continue
model_fields = self._fields
if isinstance(model_fields[field_path[0]], odoo.fields.Many2one):
# this only applies for toplevel m2o (?) fields
if field_path[0] in (self.env.context.get('name_create_enabled_fieds') or {}):
creatable_models.add(model_fields[field_path[0]].comodel_name)
for field_name in field_path:
if field_name in (None, 'id', '.id'):
break
if isinstance(model_fields[field_name], odoo.fields.One2many):
comodel = model_fields[field_name].comodel_name
creatable_models.add(comodel)
model_fields = self.env[comodel]._fields
def flush(*, xml_id=None, model=None):
if not batch:
return
assert not (xml_id and model), \
"flush can specify *either* an external id or a model, not both"
if xml_id and xml_id not in batch_xml_ids:
if xml_id not in self.env:
return
if model and model not in creatable_models:
return
data_list = [
dict(xml_id=xid, values=vals, info=info, noupdate=noupdate)
for xid, vals, info in batch
]
batch.clear()
batch_xml_ids.clear()
# try to create in batch
try:
with cr.savepoint():
recs = self._load_records(data_list, mode == 'update')
ids.extend(recs.ids)
return
except psycopg2.InternalError as e:
# broken transaction, exit and hope the source error was already logged
if not any(message['type'] == 'error' for message in messages):
info = data_list[0]['info']
messages.append(dict(info, type='error', message=_(u"Unknown database error: '%s'", e)))
return
except Exception:
pass
errors = 0
# try again, this time record by record
for i, rec_data in enumerate(data_list, 1):
try:
with cr.savepoint():
rec = self._load_records([rec_data], mode == 'update')
ids.append(rec.id)
except psycopg2.Warning as e:
info = rec_data['info']
messages.append(dict(info, type='warning', message=str(e)))
except psycopg2.Error as e:
info = rec_data['info']
messages.append(dict(info, type='error', **PGERROR_TO_OE[e.pgcode](self, fg, info, e)))
# Failed to write, log to messages, rollback savepoint (to
# avoid broken transaction) and keep going
errors += 1
except Exception as e:
_logger.debug("Error while loading record", exc_info=True)
info = rec_data['info']
message = (_(u'Unknown error during import:') + u' %s: %s' % (type(e), e))
moreinfo = _('Resolve other errors first')
messages.append(dict(info, type='error', message=message, moreinfo=moreinfo))
# Failed for some reason, perhaps due to invalid data supplied,
# rollback savepoint and keep going
errors += 1
if errors >= 10 and (errors >= i / 10):
messages.append({
'type': 'warning',
'message': _(u"Found more than 10 errors and more than one error per 10 records, interrupted to avoid showing too many errors.")
})
break
# make 'flush' available to the methods below, in the case where XMLID
# resolution fails, for instance
flush_self = self.with_context(import_flush=flush)
# TODO: break load's API instead of smuggling via context?
limit = self._context.get('_import_limit')
if limit is None:
limit = float('inf')
extracted = flush_self._extract_records(fields, data, log=messages.append, limit=limit)
converted = flush_self._convert_records(extracted, log=messages.append)
info = {'rows': {'to': -1}}
for id, xid, record, info in converted:
@api.model
def _convert_records(self, records, log=lambda a: None):
""" Log data ids that are imported with `load` """
current_module = self.env.context['module']
for res in BaseModelPatch._convert_records._original_method(
self, records, log=log):
_id, xid, _record, _info = res
if xid:
xid = xid if '.' in xid else "%s.%s" % (current_module, xid)
batch_xml_ids.add(xid)
# <OpenUpgrade:ADD>
# log csv records
upgrade_log.log_xml_id(self.env.cr, current_module, xid)
# </OpenUpgrade>
elif id:
record['id'] = id
batch.append((xid, record, info))
flush()
if any(message['type'] == 'error' for message in messages):
cr.execute('ROLLBACK TO SAVEPOINT model_load')
ids = False
# cancel all changes done to the registry/ormcache
self.pool.reset_changes()
nextrow = info['rows']['to'] + 1
if nextrow < limit:
nextrow = 0
return {
'ids': ids,
'messages': messages,
'nextrow': nextrow,
}
BaseModel.load = _load
yield res

11
upgrade_analysis/odoo_patch/odoo/modules/__init__.py
View File

@ -1,12 +1 @@
# Minor changes. (call to safe_eval changed)
# otherwise : adapted to V14
from . import graph
# A lot of changes in the core functions.
from . import loading
# Adapted to V14
from . import migration
# Adapted to V14
from . import registry

108
upgrade_analysis/odoo_patch/odoo/modules/graph.py
View File

@ -1,108 +0,0 @@
# flake8: noqa
# pylint: skip-file
import logging
import odoo
import odoo.tools as tools
from odoo.tools.safe_eval import safe_eval
from odoo.modules.graph import Graph
_logger = logging.getLogger(__name__)
if True:
def _update_from_db(self, cr):
if not len(self):
return
# update the graph with values from the database (if exist)
## First, we set the default values for each package in graph
additional_data = {key: {'id': 0, 'state': 'uninstalled', 'dbdemo': False, 'installed_version': None} for key in self.keys()}
## Then we get the values from the database
cr.execute('SELECT name, id, state, demo AS dbdemo, latest_version AS installed_version'
' FROM ir_module_module'
' WHERE name IN %s',(tuple(additional_data),)
)
## and we update the default values with values from the database
additional_data.update((x['name'], x) for x in cr.dictfetchall())
# <OpenUpgrade:ADD>
# Prevent reloading of demo data from the new version on major upgrade
if ('base' in self and additional_data['base']['dbdemo'] and
additional_data['base']['installed_version'] <
odoo.release.major_version):
cr.execute("UPDATE ir_module_module SET demo = false")
for data in additional_data.values():
data['dbdemo'] = False
# </OpenUpgrade>
for package in self.values():
for k, v in additional_data[package.name].items():
setattr(package, k, v)
def _add_modules(self, cr, module_list, force=None):
if force is None:
force = []
packages = []
len_graph = len(self)
# <OpenUpgrade:ADD>
# force additional dependencies for the upgrade process if given
# in config file
forced_deps = tools.config.get_misc('openupgrade', 'force_deps', '{}')
forced_deps = tools.config.get_misc('openupgrade',
'force_deps_' + odoo.release.version,
forced_deps)
forced_deps = safe_eval(forced_deps)
# </OpenUpgrade>
for module in module_list:
# This will raise an exception if no/unreadable descriptor file.
# NOTE The call to load_information_from_description_file is already
# done by db.initialize, so it is possible to not do it again here.
info = odoo.modules.module.load_information_from_description_file(module)
if info and info['installable']:
# <OpenUpgrade:ADD>
info['depends'].extend(forced_deps.get(module, []))
# </OpenUpgrade>
packages.append((module, info)) # TODO directly a dict, like in get_modules_with_version
elif module != 'studio_customization':
_logger.warning('module %s: not installable, skipped', module)
dependencies = dict([(p, info['depends']) for p, info in packages])
current, later = set([p for p, info in packages]), set()
while packages and current > later:
package, info = packages[0]
deps = info['depends']
# if all dependencies of 'package' are already in the graph, add 'package' in the graph
if all(dep in self for dep in deps):
if not package in current:
packages.pop(0)
continue
later.clear()
current.remove(package)
node = self.add_node(package, info)
for kind in ('init', 'demo', 'update'):
if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force:
setattr(node, kind, True)
else:
later.add(package)
packages.append((package, info))
packages.pop(0)
self.update_from_db(cr)
for package in later:
unmet_deps = [p for p in dependencies[package] if p not in self]
_logger.info('module %s: Unmet dependencies: %s', package, ', '.join(unmet_deps))
return len(self) - len_graph
Graph.update_from_db = _update_from_db
Graph.add_modules = _add_modules

556
upgrade_analysis/odoo_patch/odoo/modules/loading.py
View File

@ -1,556 +0,0 @@
# flake8: noqa
# pylint: skip-file
import itertools
import logging
import sys
import time
import odoo
import odoo.tools as tools
from odoo import api, SUPERUSER_ID
from odoo.modules import loading
from odoo.modules.module import adapt_version, load_openerp_module, initialize_sys_path
from odoo.modules.loading import load_data, load_demo, _check_module_names
from .... import upgrade_loading
import os
_logger = logging.getLogger(__name__)
_test_logger = logging.getLogger('odoo.tests')
def _load_module_graph(cr, graph, status=None, perform_checks=True,
skip_modules=None, report=None, models_to_check=None, upg_registry=None):
# <OpenUpgrade:CHANGED-SIGNATURE/>
"""Migrates+Updates or Installs all module nodes from ``graph``
:param graph: graph of module nodes to load
:param status: deprecated parameter, unused, left to avoid changing signature in 8.0
:param perform_checks: whether module descriptors should be checked for validity (prints warnings
for same cases)
:param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped
:return: list of modules that were installed or updated
"""
if skip_modules is None:
skip_modules = []
if models_to_check is None:
models_to_check = set()
processed_modules = []
loaded_modules = []
registry = odoo.registry(cr.dbname)
migrations = odoo.modules.migration.MigrationManager(cr, graph)
module_count = len(graph)
_logger.info('loading %d modules...', module_count)
# <OpenUpgrade:ADD>
# suppress commits to have the upgrade of one module in just one transaction
cr.commit_org = cr.commit
cr.commit = lambda *args: None
cr.rollback_org = cr.rollback
cr.rollback = lambda *args: None
# </OpenUpgrade>
# register, instantiate and initialize models for each modules
t0 = time.time()
loading_extra_query_count = odoo.sql_db.sql_counter
loading_cursor_query_count = cr.sql_log_count
models_updated = set()
for index, package in enumerate(graph, 1):
module_name = package.name
module_id = package.id
# <OpenUpgrade:CHANGE>
if module_name in skip_modules or module_name in loaded_modules:
# </OpenUpgrade>
continue
module_t0 = time.time()
module_cursor_query_count = cr.sql_log_count
module_extra_query_count = odoo.sql_db.sql_counter
needs_update = (
hasattr(package, "init")
or hasattr(package, "update")
or package.state in ("to install", "to upgrade")
)
module_log_level = logging.DEBUG
if needs_update:
module_log_level = logging.INFO
_logger.log(module_log_level, 'Loading module %s (%d/%d)', module_name, index, module_count)
if needs_update:
if package.name != 'base':
registry.setup_models(cr)
migrations.migrate_module(package, 'pre')
if package.name != 'base':
env = api.Environment(cr, SUPERUSER_ID, {})
env['base'].flush()
load_openerp_module(package.name)
new_install = package.state == 'to install'
if new_install:
py_module = sys.modules['odoo.addons.%s' % (module_name,)]
pre_init = package.info.get('pre_init_hook')
if pre_init:
getattr(py_module, pre_init)(cr)
model_names = registry.load(cr, package)
mode = 'update'
if hasattr(package, 'init') or package.state == 'to install':
mode = 'init'
loaded_modules.append(package.name)
if needs_update:
models_updated |= set(model_names)
models_to_check -= set(model_names)
registry.setup_models(cr)
# <OpenUpgrade:ADD>
# rebuild the local registry based on the loaded models
local_registry = {}
env = api.Environment(cr, SUPERUSER_ID, {})
for model in env.values():
if not model._auto:
continue
upgrade_loading.log_model(model, local_registry)
upgrade_loading.compare_registries(
cr, package.name, upg_registry, local_registry)
# </OpenUpgrade>
registry.init_models(cr, model_names, {'module': package.name}, new_install)
elif package.state != 'to remove':
# The current module has simply been loaded. The models extended by this module
# and for which we updated the schema, must have their schema checked again.
# This is because the extension may have changed the model,
# e.g. adding required=True to an existing field, but the schema has not been
# updated by this module because it's not marked as 'to upgrade/to install'.
models_to_check |= set(model_names) & models_updated
idref = {}
if needs_update:
env = api.Environment(cr, SUPERUSER_ID, {})
# Can't put this line out of the loop: ir.module.module will be
# registered by init_models() above.
module = env['ir.module.module'].browse(module_id)
if perform_checks:
module._check()
if package.state == 'to upgrade':
# upgrading the module information
module.write(module.get_values_from_terp(package.data))
load_data(cr, idref, mode, kind='data', package=package)
demo_loaded = package.dbdemo = load_demo(cr, package, idref, mode)
cr.execute('update ir_module_module set demo=%s where id=%s', (demo_loaded, module_id))
module.invalidate_cache(['demo'])
# <OpenUpgrade:CHANGE>
# add 'try' block for logging exceptions
# as errors in post scripts seem to be dropped
try:
migrations.migrate_module(package, 'post')
except Exception as exc:
_logger.error('Error executing post migration script for module %s: %s',
package, exc)
raise
# </OpenUpgrade>
# Update translations for all installed languages
overwrite = odoo.tools.config["overwrite_existing_translations"]
module.with_context(overwrite=overwrite)._update_translations()
if package.name is not None:
registry._init_modules.add(package.name)
if needs_update:
if new_install:
post_init = package.info.get('post_init_hook')
if post_init:
getattr(py_module, post_init)(cr, registry)
if mode == 'update':
# validate the views that have not been checked yet
env['ir.ui.view']._validate_module_views(module_name)
# need to commit any modification the module's installation or
# update made to the schema or data so the tests can run
# (separately in their own transaction)
# <OpenUpgrade:CHANGE>
# commit after processing every module as well, for
# easier debugging and continuing an interrupted migration
cr.commit_org()
# </OpenUpgrade
updating = tools.config.options['init'] or tools.config.options['update']
test_time = test_queries = 0
test_results = None
if tools.config.options['test_enable'] and (needs_update or not updating):
env = api.Environment(cr, SUPERUSER_ID, {})
loader = odoo.tests.loader
suite = loader.make_suite(module_name, 'at_install')
if suite.countTestCases():
if not needs_update:
registry.setup_models(cr)
# Python tests
env['ir.http']._clear_routing_map() # force routing map to be rebuilt
tests_t0, tests_q0 = time.time(), odoo.sql_db.sql_counter
test_results = loader.run_suite(suite, module_name)
report.update(test_results)
test_time = time.time() - tests_t0
test_queries = odoo.sql_db.sql_counter - tests_q0
# tests may have reset the environment
env = api.Environment(cr, SUPERUSER_ID, {})
module = env['ir.module.module'].browse(module_id)
if needs_update:
# <OpenUpgrade:CHANGE>
# run tests
if os.environ.get('OPENUPGRADE_TESTS') and package.name is not None:
prefix = '.migrations'
registry.openupgrade_test_prefixes[package.name] = prefix
report.record_result(odoo.modules.module.run_unit_tests(module_name, openupgrade_prefix=prefix))
# </OpenUpgrade
processed_modules.append(package.name)
ver = adapt_version(package.data['version'])
# Set new modules and dependencies
module.write({'state': 'installed', 'latest_version': ver})
# <OpenUpgrade:ADD>
# commit module_n state and version immediatly
# to avoid invalid database state if module_n+1 raises an
# exception
cr.commit_org()
# </OpenUpgrade>
package.load_state = package.state
package.load_version = package.installed_version
package.state = 'installed'
for kind in ('init', 'demo', 'update'):
if hasattr(package, kind):
delattr(package, kind)
module.flush()
extra_queries = odoo.sql_db.sql_counter - module_extra_query_count - test_queries
extras = []
if test_queries:
extras.append(f'+{test_queries} test')
if extra_queries:
extras.append(f'+{extra_queries} other')
_logger.log(
module_log_level, "Module %s loaded in %.2fs%s, %s queries%s",
module_name, time.time() - module_t0,
f' (incl. {test_time:.2f}s test)' if test_time else '',
cr.sql_log_count - module_cursor_query_count,
f' ({", ".join(extras)})' if extras else ''
)
if test_results and not test_results.wasSuccessful():
_logger.error(
"Module %s: %d failures, %d errors of %d tests",
module_name, len(test_results.failures), len(test_results.errors),
test_results.testsRun
)
_logger.runbot("%s modules loaded in %.2fs, %s queries (+%s extra)",
len(graph),
time.time() - t0,
cr.sql_log_count - loading_cursor_query_count,
odoo.sql_db.sql_counter - loading_extra_query_count) # extra queries: testes, notify, any other closed cursor
# <OpenUpgrade:ADD>
# restore commit method
cr.commit = cr.commit_org
cr.commit()
# </OpenUpgrade>
return loaded_modules, processed_modules
def _load_marked_modules(cr, graph, states, force, progressdict, report,
loaded_modules, perform_checks, models_to_check=None, upg_registry=None):
# <OpenUpgrade:CHANGED-SIGNATURE/>
"""Loads modules marked with ``states``, adding them to ``graph`` and
``loaded_modules`` and returns a list of installed/upgraded modules."""
if models_to_check is None:
models_to_check = set()
processed_modules = []
while True:
cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),))
module_list = [name for (name,) in cr.fetchall() if name not in graph]
# <OpenUpgrade:ADD>
module_list = upgrade_loading.add_module_dependencies(cr, module_list)
# </OpenUpgrade>
if not module_list:
break
graph.add_modules(cr, module_list, force)
_logger.debug('Updating graph with %d more modules', len(module_list))
# <OpenUpgrade:CHANGE>
# add upg_registry
loaded, processed = _load_module_graph(
cr, graph, progressdict, report=report, skip_modules=loaded_modules,
perform_checks=perform_checks, models_to_check=models_to_check,
upg_registry=upg_registry,
)
# </OpenUpgrade>
processed_modules.extend(processed)
loaded_modules.extend(loaded)
if not processed:
break
return processed_modules
def _load_modules(db, force_demo=False, status=None, update_module=False):
initialize_sys_path()
force = []
if force_demo:
force.append('demo')
# <OpenUpgrade:ADD>
upg_registry = {}
# </OpenUpgrade>
models_to_check = set()
with db.cursor() as cr:
if not odoo.modules.db.is_initialized(cr):
if not update_module:
_logger.error("Database %s not initialized, you can force it with `-i base`", cr.dbname)
return
_logger.info("init db")
odoo.modules.db.initialize(cr)
update_module = True # process auto-installed modules
tools.config["init"]["all"] = 1
if not tools.config['without_demo']:
tools.config["demo"]['all'] = 1
# This is a brand new registry, just created in
# odoo.modules.registry.Registry.new().
registry = odoo.registry(cr.dbname)
if 'base' in tools.config['update'] or 'all' in tools.config['update']:
cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed'))
# STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps)
graph = odoo.modules.graph.Graph()
graph.add_module(cr, 'base', force)
if not graph:
_logger.critical('module base cannot be loaded! (hint: verify addons-path)')
raise ImportError('Module `base` cannot be loaded! (hint: verify addons-path)')
# processed_modules: for cleanup step after install
# loaded_modules: to avoid double loading
report = registry._assertion_report
# <OpenUpgrade:CHANGE>
# add upg_registry
loaded_modules, processed_modules = _load_module_graph(
cr, graph, status, perform_checks=update_module,
report=report, models_to_check=models_to_check, upg_registry=upg_registry)
# </OpenUpgrade>
load_lang = tools.config.pop('load_language')
if load_lang or update_module:
# some base models are used below, so make sure they are set up
registry.setup_models(cr)
if load_lang:
for lang in load_lang.split(','):
tools.load_language(cr, lang)
# STEP 2: Mark other modules to be loaded/updated
if update_module:
env = api.Environment(cr, SUPERUSER_ID, {})
Module = env['ir.module.module']
_logger.info('updating modules list')
Module.update_list()
_check_module_names(cr, itertools.chain(tools.config['init'], tools.config['update']))
module_names = [k for k, v in tools.config['init'].items() if v]
if module_names:
modules = Module.search([('state', '=', 'uninstalled'), ('name', 'in', module_names)])
if modules:
modules.button_install()
module_names = [k for k, v in tools.config['update'].items() if v]
if module_names:
# <OpenUpgrade:CHANGE>
# in standard Odoo, '--update all' just means:
# '--update base + upward (installed) dependencies. This breaks
# the chain when new glue modules are encountered.
# E.g. purchase in 8.0 depends on stock_account and report,
# both of which are new. They may be installed, but purchase as
# an upward dependency is not selected for upgrade.
# Therefore, explicitely select all installed modules for
# upgrading in OpenUpgrade in that case.
domain = [('state', '=', 'installed')]
if 'all' not in module_names:
domain.append(('name', 'in', module_names))
modules = Module.search(domain)
# </OpenUpgrade>
if modules:
modules.button_upgrade()
cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base'))
Module.invalidate_cache(['state'])
Module.flush()
# STEP 3: Load marked modules (skipping base which was done in STEP 1)
# IMPORTANT: this is done in two parts, first loading all installed or
# partially installed modules (i.e. installed/to upgrade), to
# offer a consistent system to the second part: installing
# newly selected modules.
# We include the modules 'to remove' in the first step, because
# they are part of the "currently installed" modules. They will
# be dropped in STEP 6 later, before restarting the loading
# process.
# IMPORTANT 2: We have to loop here until all relevant modules have been
# processed, because in some rare cases the dependencies have
# changed, and modules that depend on an uninstalled module
# will not be processed on the first pass.
# It's especially useful for migrations.
previously_processed = -1
while previously_processed < len(processed_modules):
previously_processed = len(processed_modules)
# <OpenUpgrade:CHANGE>
# add upg_registry
processed_modules += _load_marked_modules(cr, graph,
['installed', 'to upgrade', 'to remove'],
force, status, report, loaded_modules, update_module, models_to_check, upg_registry)
# </OpenUpgrade>
if update_module:
# <OpenUpgrade:CHANGE>
# add upg_registry
processed_modules += _load_marked_modules(cr, graph,
['to install'], force, status, report,
loaded_modules, update_module, models_to_check, upg_registry)
# </OpenUpgrade>
# check that new module dependencies have been properly installed after a migration/upgrade
cr.execute("SELECT name from ir_module_module WHERE state IN ('to install', 'to upgrade')")
module_list = [name for (name,) in cr.fetchall()]
if module_list:
_logger.error("Some modules have inconsistent states, some dependencies may be missing: %s", sorted(module_list))
# check that all installed modules have been loaded by the registry after a migration/upgrade
cr.execute("SELECT name from ir_module_module WHERE state = 'installed' and name != 'studio_customization'")
module_list = [name for (name,) in cr.fetchall() if name not in graph]
if module_list:
_logger.error("Some modules are not loaded, some dependencies or manifest may be missing: %s", sorted(module_list))
registry.loaded = True
registry.setup_models(cr)
# STEP 3.5: execute migration end-scripts
migrations = odoo.modules.migration.MigrationManager(cr, graph)
for package in graph:
migrations.migrate_module(package, 'end')
# STEP 3.6: apply remaining constraints in case of an upgrade
registry.finalize_constraints()
# STEP 4: Finish and cleanup installations
if processed_modules:
env = api.Environment(cr, SUPERUSER_ID, {})
cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")
for (model, name) in cr.fetchall():
if model in registry and not registry[model]._abstract:
_logger.warning('The model %s has no access rules, consider adding one. E.g. access_%s,access_%s,model_%s,base.group_user,1,0,0,0',
model, model.replace('.', '_'), model.replace('.', '_'), model.replace('.', '_'))
cr.execute("SELECT model from ir_model")
for (model,) in cr.fetchall():
if model in registry:
env[model]._check_removed_columns(log=True)
elif _logger.isEnabledFor(logging.INFO): # more an info that a warning...
_logger.runbot("Model %s is declared but cannot be loaded! (Perhaps a module was partially removed or renamed)", model)
# Cleanup orphan records
env['ir.model.data']._process_end(processed_modules)
env['base'].flush()
for kind in ('init', 'demo', 'update'):
tools.config[kind] = {}
# STEP 5: Uninstall modules to remove
if update_module:
# Remove records referenced from ir_model_data for modules to be
# removed (and removed the references from ir_model_data).
cr.execute("SELECT name, id FROM ir_module_module WHERE state=%s", ('to remove',))
modules_to_remove = dict(cr.fetchall())
if modules_to_remove:
env = api.Environment(cr, SUPERUSER_ID, {})
pkgs = reversed([p for p in graph if p.name in modules_to_remove])
for pkg in pkgs:
uninstall_hook = pkg.info.get('uninstall_hook')
if uninstall_hook:
py_module = sys.modules['odoo.addons.%s' % (pkg.name,)]
getattr(py_module, uninstall_hook)(cr, registry)
Module = env['ir.module.module']
Module.browse(modules_to_remove.values()).module_uninstall()
# Recursive reload, should only happen once, because there should be no
# modules to remove next time
cr.commit()
_logger.info('Reloading registry once more after uninstalling modules')
api.Environment.reset()
registry = odoo.modules.registry.Registry.new(
cr.dbname, force_demo, status, update_module
)
registry.check_tables_exist(cr)
cr.commit()
return registry
# STEP 5.5: Verify extended fields on every model
# This will fix the schema of all models in a situation such as:
# - module A is loaded and defines model M;
# - module B is installed/upgraded and extends model M;
# - module C is loaded and extends model M;
# - module B and C depend on A but not on each other;
# The changes introduced by module C are not taken into account by the upgrade of B.
if models_to_check:
registry.init_models(cr, list(models_to_check), {'models_to_check': True})
# STEP 6: verify custom views on every model
if update_module:
env = api.Environment(cr, SUPERUSER_ID, {})
env['res.groups']._update_user_groups_view()
View = env['ir.ui.view']
for model in registry:
try:
View._validate_custom_views(model)
except Exception as e:
_logger.warning('invalid custom view(s) for model %s: %s', model, tools.ustr(e))
if report.wasSuccessful():
_logger.info('Modules loaded.')
else:
_logger.error('At least one test failed when loading the modules.')
# STEP 8: call _register_hook on every model
# This is done *exactly once* when the registry is being loaded. See the
# management of those hooks in `Registry.setup_models`: all the calls to
# setup_models() done here do not mess up with hooks, as registry.ready
# is False.
env = api.Environment(cr, SUPERUSER_ID, {})
for model in env.values():
model._register_hook()
env['base'].flush()
# STEP 9: save installed/updated modules for post-install tests
registry.updated_modules += processed_modules
loading.load_module_graph = _load_module_graph
loading.load_marked_modules = _load_marked_modules
loading.load_modules = _load_modules
odoo.modules.load_modules = _load_modules

118
upgrade_analysis/odoo_patch/odoo/modules/migration.py
View File

@ -1,118 +0,0 @@
# flake8: noqa
# pylint: skip-file
import logging
import os
from os.path import join as opj
import odoo.release as release
from odoo.tools.parse_version import parse_version
import odoo
from odoo.modules.migration import load_script
from odoo.modules import migration
_logger = logging.getLogger(__name__)
if True:
def _migrate_module(self, pkg, stage):
assert stage in ('pre', 'post', 'end')
stageformat = {
'pre': '[>%s]',
'post': '[%s>]',
'end': '[$%s]',
}
state = pkg.state if stage in ('pre', 'post') else getattr(pkg, 'load_state', None)
# <OpenUpgrade:CHANGE>
# In openupgrade, also run migration scripts upon installation.
# We want to always pass in pre and post migration files and use a new
# argument in the migrate decorator (explained in the docstring)
# to decide if we want to do something if a new module is installed
# during the migration.
if not (hasattr(pkg, 'update') or state in ('to upgrade', 'to install')):
# </OpenUpgrade>
return
def convert_version(version):
if version.count('.') >= 2:
return version # the version number already containt the server version
return "%s.%s" % (release.major_version, version)
def _get_migration_versions(pkg, stage):
versions = sorted({
ver
for lv in self.migrations[pkg.name].values()
for ver, lf in lv.items()
if lf
}, key=lambda k: parse_version(convert_version(k)))
if "0.0.0" in versions:
# reorder versions
versions.remove("0.0.0")
if stage == "pre":
versions.insert(0, "0.0.0")
else:
versions.append("0.0.0")
return versions
def _get_migration_files(pkg, version, stage):
""" return a list of migration script files
"""
m = self.migrations[pkg.name]
lst = []
mapping = {
'module': opj(pkg.name, 'migrations'),
'module_upgrades': opj(pkg.name, 'upgrades'),
}
for path in odoo.upgrade.__path__:
if os.path.exists(opj(path, pkg.name)):
mapping['upgrade'] = opj(path, pkg.name)
break
for x in mapping:
if version in m.get(x):
for f in m[x][version]:
if not f.startswith(stage + '-'):
continue
lst.append(opj(mapping[x], version, f))
lst.sort()
return lst
installed_version = getattr(pkg, 'load_version', pkg.installed_version) or ''
parsed_installed_version = parse_version(installed_version)
current_version = parse_version(convert_version(pkg.data['version']))
versions = _get_migration_versions(pkg, stage)
for version in versions:
if ((version == "0.0.0" and parsed_installed_version < current_version)
or parsed_installed_version < parse_version(convert_version(version)) <= current_version):
strfmt = {'addon': pkg.name,
'stage': stage,
'version': stageformat[stage] % version,
}
for pyfile in _get_migration_files(pkg, version, stage):
name, ext = os.path.splitext(os.path.basename(pyfile))
if ext.lower() != '.py':
continue
mod = None
try:
mod = load_script(pyfile, name)
_logger.info('module %(addon)s: Running migration %(version)s %(name)s' % dict(strfmt, name=mod.__name__))
migrate = mod.migrate
except ImportError:
_logger.exception('module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % dict(strfmt, file=pyfile))
raise
except AttributeError:
_logger.error('module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt)
else:
migrate(self.cr, installed_version)
finally:
if mod:
del mod
migration.migrate_module = _migrate_module

77
upgrade_analysis/odoo_patch/odoo/modules/registry.py
View File

@ -1,58 +1,29 @@
# flake8: noqa
# pylint: skip-file
import logging
from threading import current_thread
from odoo import api, SUPERUSER_ID
from ...odoo_patch import OdooPatch
from .... import upgrade_log
from odoo.modules.registry import Registry
from collections import deque
from contextlib import closing
import odoo
from odoo.tools.lru import LRU
from odoo.modules import registry
_logger = logging.getLogger(__name__)
if True:
class RegistryPatch(OdooPatch):
target = Registry
method_names = ['init_models']
def _init(self, db_name):
self.models = {} # model name/model instance mapping
self._sql_constraints = set()
self._init = True
self._assertion_report = odoo.tests.runner.OdooTestResult()
self._fields_by_model = None
self._ordinary_tables = None
self._constraint_queue = deque()
self.__cache = LRU(8192)
def init_models(self, cr, model_names, context, install=True):
module_name = context['module']
_logger.debug('Logging models of module %s', module_name)
upg_registry = current_thread()._upgrade_registry
local_registry = {}
env = api.Environment(cr, SUPERUSER_ID, {})
for model in env.values():
if not model._auto:
continue
upgrade_log.log_model(model, local_registry)
upgrade_log.compare_registries(
cr, context['module'], upg_registry, local_registry)
# modules fully loaded (maintained during init phase by `loading` module)
self._init_modules = set()
self.updated_modules = [] # installed/updated modules
# <OpenUpgrade:ADD>
self.openupgrade_test_prefixes = {}
# </OpenUpgrade>
self.loaded_xmlids = set()
self.db_name = db_name
self._db = odoo.sql_db.db_connect(db_name)
# cursor for test mode; None means "normal" mode
self.test_cr = None
self.test_lock = None
# Indicates that the registry is
self.loaded = False # whether all modules are loaded
self.ready = False # whether everything is set up
# Inter-process signaling:
# The `base_registry_signaling` sequence indicates the whole registry
# must be reloaded.
# The `base_cache_signaling sequence` indicates all caches must be
# invalidated (i.e. cleared).
self.registry_sequence = None
self.cache_sequence = None
# Flags indicating invalidation of the registry or the cache.
self.registry_invalidated = False
self.cache_invalidated = False
with closing(self.cursor()) as cr:
self.has_unaccent = odoo.modules.db.has_unaccent(cr)
registry.init = _init
return RegistryPatch.init_models._original_method(
self, cr, model_names, context, install=install)

4
upgrade_analysis/odoo_patch/odoo/service/__init__.py
View File

@ -1,4 +0,0 @@
# Import disabled, because the function run_unit_tests()
# disappeared in V14.
# TODO: OpenUpgrade Core maintainers : FIXME.
# from . import server

71
upgrade_analysis/odoo_patch/odoo/service/server.py
View File

@ -1,71 +0,0 @@
# flake8: noqa
# pylint: skip-file
import logging
import os
import time
import odoo
from odoo.tools import config
from odoo.modules.registry import Registry
from odoo.service import server
from odoo.service.server import load_test_file_py
_logger = logging.getLogger(__name__)
def preload_registries(dbnames):
""" Preload a registries, possibly run a test file."""
# TODO: move all config checks to args dont check tools.config here
dbnames = dbnames or []
rc = 0
for dbname in dbnames:
try:
update_module = config['init'] or config['update']
registry = Registry.new(dbname, update_module=update_module)
# run test_file if provided
if config['test_file']:
test_file = config['test_file']
if not os.path.isfile(test_file):
_logger.warning('test file %s cannot be found', test_file)
elif not test_file.endswith('py'):
_logger.warning('test file %s is not a python file', test_file)
else:
_logger.info('loading test file %s', test_file)
with odoo.api.Environment.manage():
load_test_file_py(registry, test_file)
# run post-install tests
if config['test_enable']:
t0 = time.time()
t0_sql = odoo.sql_db.sql_counter
module_names = (registry.updated_modules if update_module else
sorted(registry._init_modules))
_logger.info("Starting post tests")
tests_before = registry._assertion_report.testsRun
with odoo.api.Environment.manage():
for module_name in module_names:
result = loader.run_suite(loader.make_suite(module_name, 'post_install'), module_name)
registry._assertion_report.update(result)
# <OpenUpgrade:ADD>
# run deferred unit tests
for module_name, prefix in registry.openupgrade_test_prefixes:
result = run_unit_tests(module_name, position='post_install', openupgrade_prefix=prefix)
registry._assertion_report.record_result(result)
# </OpenUpgrade>
_logger.info("%d post-tests in %.2fs, %s queries",
registry._assertion_report.testsRun - tests_before,
time.time() - t0,
odoo.sql_db.sql_counter - t0_sql)
if not registry._assertion_report.wasSuccessful():
rc += 1
except Exception:
_logger.critical('Failed to initialize database `%s`.', dbname, exc_info=True)
return -1
return rc
server.preload_registries = preload_registries

1
upgrade_analysis/odoo_patch/odoo/tools/__init__.py
View File

@ -1,2 +1 @@
from . import convert
from . import view_validation

24
upgrade_analysis/odoo_patch/odoo/tools/convert.py
View File

@ -1,23 +1,13 @@
# flake8: noqa
# pylint: skip-file
from ...odoo_patch import OdooPatch
from .... import upgrade_log
from odoo.tools.convert import xml_import
if True:
def __test_xml_id(self, xml_id):
if '.' in xml_id:
module, id = xml_id.split('.', 1)
assert '.' not in id, """The ID reference "%s" must contain
maximum one dot. They are used to refer to other modules ID, in the
form: module.record_id""" % (xml_id,)
if module != self.module:
modcnt = self.env['ir.module.module'].search_count([('name', '=', module), ('state', '=', 'installed')])
assert modcnt == 1, """The ID "%s" refers to an uninstalled module""" % (xml_id,)
class XMLImportPatch(OdooPatch):
target = xml_import
method_names = ['_test_xml_id']
# OpenUpgrade: log entry of XML imports
def _test_xml_id(self, xml_id):
res = XMLImportPatch._test_xml_id._original_method(self, xml_id)
upgrade_log.log_xml_id(self.env.cr, self.module, xml_id)
xml_import._test_xml_id = __test_xml_id
return res

29
upgrade_analysis/odoo_patch/odoo/tools/view_validation.py
View File

@ -1,29 +0,0 @@
# flake8: noqa
# pylint: skip-file
# from odoo.addons.openupgrade_framework.openupgrade import openupgrade_log
from odoo.tools import view_validation
from odoo.tools.view_validation import _validators, _logger
def _valid_view(arch, **kwargs):
for pred in _validators[arch.tag]:
# <OpenUpgrade:CHANGE>
# Do not raise blocking error, because it's normal to
# have inconsistent views in an openupgrade process
check = pred(arch, **kwargs) or 'Warning'
# </OpenUpgrade>
if not check:
_logger.error("Invalid XML: %s", pred.__doc__)
return False
if check == "Warning":
# <OpenUpgrade:REM>
# Don't show this warning as useless and too much verbose
# _logger.warning("Invalid XML: %s", pred.__doc__)
# </OpenUpgrade>
return "Warning"
return True
view_validation.valid_view = _valid_view

59
upgrade_analysis/odoo_patch/odoo_patch.py 100644
View File

@ -0,0 +1,59 @@
import logging
_logger = logging.getLogger(__name__)
class OdooPatch(object):
""" Simple mechanism to apply a collection of monkeypatches using a
context manager.
Classes can register their monkeypatches by inheriting from this class.
They need to define a `target` member, referring to the object or module
that needs to be patched, and a list `method_names`. They also need to
redefine those methods under the same name.
The original method is made available on the new method as
`_original_method`.
Example:
```
from odoo import api
from odoo.addons.some_module.models.my_model import MyModel
class MyModelPatch(OdooPatch):
target = MyModel
method_names = ['do_something']
@api.model
def do_something(self):
res = MyModelPatch.do_something._original_method()
...
return res
```
Usage:
```
with OdooPatch():
do_something()
```
"""
def __enter__(self):
for cls in OdooPatch.__subclasses__():
for method_name in cls.method_names:
method = getattr(cls, method_name)
setattr(method, '_original_method',
getattr(cls.target, method_name))
setattr(cls.target, method_name, method)
def __exit__(self, exc_type, exc_value, tb):
for cls in OdooPatch.__subclasses__():
for method_name in cls.method_names:
method = getattr(cls.target, method_name)
if hasattr(method, '_original_method'):
setattr(cls.target, method_name, method._original_method)
else:
_logger.warn(
'_original_method not found on method %s of class %s',
method_name, cls.target)

316
upgrade_analysis/upgrade_loading.py
View File

@ -1,316 +0,0 @@
# Copyright 2011-2015 Therp BV <https://therp.nl>
# Copyright 2016-2019 Opener B.V. <https://opener.am>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
# flake8: noqa: C901
import logging
from openupgradelib.openupgrade_tools import table_exists
from odoo import release
from odoo.modules.module import get_module_path
from odoo.tools.config import config
from odoo.tools.safe_eval import safe_eval
# A collection of functions used in
# odoo/modules/loading.py
_logger = logging.getLogger(__name__)
def add_module_dependencies(cr, module_list):
"""
Select (new) dependencies from the modules in the list
so that we can inject them into the graph at upgrade
time. Used in the modified OpenUpgrade Server,
not to be called from migration scripts
Also take the OpenUpgrade configuration directives 'forced_deps'
and 'autoinstall' into account. From any additional modules
that these directives can add, the dependencies are added as
well (but these directives are not checked for the occurrence
of any of the dependencies).
"""
if not module_list:
return module_list
modules_in = list(module_list)
forced_deps = safe_eval(
config.get_misc(
"openupgrade",
"forced_deps_" + release.version,
config.get_misc("openupgrade", "forced_deps", "{}"),
)
)
autoinstall = safe_eval(
config.get_misc(
"openupgrade",
"autoinstall_" + release.version,
config.get_misc("openupgrade", "autoinstall", "{}"),
)
)
for module in list(module_list):
module_list += forced_deps.get(module, [])
module_list += autoinstall.get(module, [])
module_list = list(set(module_list))
dependencies = module_list
while dependencies:
cr.execute(
"""
SELECT DISTINCT dep.name
FROM
ir_module_module,
ir_module_module_dependency dep
WHERE
module_id = ir_module_module.id
AND ir_module_module.name in %s
AND dep.name not in %s
""",
(
tuple(dependencies),
tuple(module_list),
),
)
dependencies = [x[0] for x in cr.fetchall()]
module_list += dependencies
# Select auto_install modules of which all dependencies
# are fulfilled based on the modules we know are to be
# installed
cr.execute(
"""
SELECT name from ir_module_module WHERE state IN %s
""",
(("installed", "to install", "to upgrade"),),
)
modules = list(set(module_list + [row[0] for row in cr.fetchall()]))
cr.execute(
"""
SELECT name from ir_module_module m
WHERE auto_install IS TRUE
AND state = 'uninstalled'
AND NOT EXISTS(
SELECT id FROM ir_module_module_dependency d
WHERE d.module_id = m.id
AND name NOT IN %s)
""",
(tuple(modules),),
)
auto_modules = [row[0] for row in cr.fetchall() if get_module_path(row[0])]
if auto_modules:
_logger.info("Selecting autoinstallable modules %s", ",".join(auto_modules))
module_list += auto_modules
# Set proper state for new dependencies so that any init scripts are run
cr.execute(
"""
UPDATE ir_module_module SET state = 'to install'
WHERE name IN %s AND name NOT IN %s AND state = 'uninstalled'
""",
(tuple(module_list), tuple(modules_in)),
)
return module_list
def log_model(model, local_registry):
"""
OpenUpgrade: Store the characteristics of the BaseModel and its fields
in the local registry, so that we can compare changes with the
main registry
"""
if not model._name:
return
typemap = {"monetary": "float"}
# Deferred import to prevent import loop
from odoo import models
# persistent models only
if isinstance(model, models.TransientModel):
return
def isfunction(model, k):
if (
model._fields[k].compute
and not model._fields[k].related
and not model._fields[k].company_dependent
):
return "function"
return ""
def isproperty(model, k):
if model._fields[k].company_dependent:
return "property"
return ""
def isrelated(model, k):
if model._fields[k].related:
return "related"
return ""
def _get_relation(v):
if v.type in ("many2many", "many2one", "one2many"):
return v.comodel_name
elif v.type == "many2one_reference":
return v.model_field
else:
return ""
model_registry = local_registry.setdefault(model._name, {})
if model._inherits:
model_registry["_inherits"] = {"_inherits": str(model._inherits)}
for k, v in model._fields.items():
properties = {
"type": typemap.get(v.type, v.type),
"isfunction": isfunction(model, k),
"isproperty": isproperty(model, k),
"isrelated": isrelated(model, k),
"relation": _get_relation(v),
"table": v.relation if v.type == "many2many" else "",
"required": v.required and "required" or "",
"stored": v.store and "stored" or "",
"selection_keys": "",
"req_default": "",
"hasdefault": model._fields[k].default and "hasdefault" or "",
"inherits": "",
}
if v.type == "selection":
if isinstance(v.selection, (tuple, list)):
properties["selection_keys"] = str(sorted([x[0] for x in v.selection]))
else:
properties["selection_keys"] = "function"
elif v.type == "binary":
properties["attachment"] = str(getattr(v, "attachment", False))
default = model._fields[k].default
if v.required and default:
if (
callable(default)
or isinstance(default, str)
and getattr(model._fields[k], default, False)
and callable(getattr(model._fields[k], default))
):
# todo: in OpenERP 5 (and in 6 as well),
# literals are wrapped in a lambda function
properties["req_default"] = "function"
else:
properties["req_default"] = str(default)
for key, value in properties.items():
if value:
model_registry.setdefault(k, {})[key] = value
def get_record_id(cr, module, model, field, mode):
"""
OpenUpgrade: get or create the id from the record table matching
the key parameter values
"""
cr.execute(
"SELECT id FROM upgrade_record "
"WHERE module = %s AND model = %s AND "
"field = %s AND mode = %s AND type = %s",
(module, model, field, mode, "field"),
)
record = cr.fetchone()
if record:
return record[0]
cr.execute(
"INSERT INTO upgrade_record "
"(module, model, field, mode, type) "
"VALUES (%s, %s, %s, %s, %s)",
(module, model, field, mode, "field"),
)
cr.execute(
"SELECT id FROM upgrade_record "
"WHERE module = %s AND model = %s AND "
"field = %s AND mode = %s AND type = %s",
(module, model, field, mode, "field"),
)
return cr.fetchone()[0]
def compare_registries(cr, module, registry, local_registry):
"""
OpenUpgrade: Compare the local registry with the global registry,
log any differences and merge the local registry with
the global one.
"""
if not table_exists(cr, "upgrade_record"):
return
for model, flds in local_registry.items():
registry.setdefault(model, {})
for field, attributes in flds.items():
old_field = registry[model].setdefault(field, {})
mode = old_field and "modify" or "create"
record_id = False
for key, value in attributes.items():
if key not in old_field or old_field[key] != value:
if not record_id:
record_id = get_record_id(cr, module, model, field, mode)
cr.execute(
"SELECT id FROM upgrade_attribute "
"WHERE name = %s AND value = %s AND "
"record_id = %s",
(key, value, record_id),
)
if not cr.fetchone():
cr.execute(
"INSERT INTO upgrade_attribute "
"(name, value, record_id) VALUES (%s, %s, %s)",
(key, value, record_id),
)
old_field[key] = value
def update_field_xmlid(model, field):
"""OpenUpgrade edit start: In rare cases, an old module defined a field
on a model that is not defined in another module earlier in the
chain of inheritance. Then we need to assign the ir.model.fields'
xmlid to this other module, otherwise the column would be dropped
when uninstalling the first module.
An example is res.partner#display_name defined in 7.0 by
account_report_company, but now the field belongs to the base
module
Given that we arrive here in order of inheritance, we simply check
if the field's xmlid belongs to a module already loaded, and if not,
update the record with the correct module name."""
model.env.cr.execute(
"SELECT f.*, d.module, d.id as xmlid_id, d.name as xmlid "
"FROM ir_model_fields f LEFT JOIN ir_model_data d "
"ON f.id=d.res_id and d.model='ir.model.fields' WHERE f.model=%s",
(model._name,),
)
for rec in model.env.cr.dictfetchall():
if (
"module" in model.env.context
and rec["module"]
and rec["name"] in model._fields.keys()
and rec["module"] != model.env.context["module"]
and rec["module"] not in model.env.registry._init_modules
):
_logger.info(
"Moving XMLID for ir.model.fields record of %s#%s " "from %s to %s",
model._name,
rec["name"],
rec["module"],
model.env.context["module"],
)
model.env.cr.execute(
"SELECT id FROM ir_model_data WHERE module=%(module)s "
"AND name=%(xmlid)s",
dict(rec, module=model.env.context["module"]),
)
if model.env.cr.fetchone():
_logger.info("Aborting, an XMLID for this module already exists.")
continue
model.env.cr.execute(
"UPDATE ir_model_data SET module=%(module)s " "WHERE id=%(xmlid_id)s",
dict(rec, module=model.env.context["module"]),
)

154
upgrade_analysis/upgrade_log.py
View File

@ -6,9 +6,161 @@ import logging
from openupgradelib.openupgrade_tools import table_exists
from odoo import models
_logger = logging.getLogger(__name__)
def get_record_id(cr, module, model, field, mode):
"""
OpenUpgrade: get or create the id from the record table matching
the key parameter values
"""
cr.execute(
"SELECT id FROM upgrade_record "
"WHERE module = %s AND model = %s AND "
"field = %s AND mode = %s AND type = %s",
(module, model, field, mode, "field"),
)
record = cr.fetchone()
if record:
return record[0]
cr.execute(
"INSERT INTO upgrade_record "
"(module, model, field, mode, type) "
"VALUES (%s, %s, %s, %s, %s)",
(module, model, field, mode, "field"),
)
cr.execute(
"SELECT id FROM upgrade_record "
"WHERE module = %s AND model = %s AND "
"field = %s AND mode = %s AND type = %s",
(module, model, field, mode, "field"),
)
return cr.fetchone()[0]
def compare_registries(cr, module, registry, local_registry):
"""
OpenUpgrade: Compare the local registry with the global registry,
log any differences and merge the local registry with
the global one.
"""
if not table_exists(cr, "upgrade_record"):
return
for model, flds in local_registry.items():
registry.setdefault(model, {})
for field, attributes in flds.items():
old_field = registry[model].setdefault(field, {})
mode = old_field and "modify" or "create"
record_id = False
for key, value in attributes.items():
if key not in old_field or old_field[key] != value:
if not record_id:
record_id = get_record_id(cr, module, model, field, mode)
cr.execute(
"SELECT id FROM upgrade_attribute "
"WHERE name = %s AND value = %s AND "
"record_id = %s",
(key, value, record_id),
)
if not cr.fetchone():
cr.execute(
"INSERT INTO upgrade_attribute "
"(name, value, record_id) VALUES (%s, %s, %s)",
(key, value, record_id),
)
old_field[key] = value
def isfunction(model, k):
if (
model._fields[k].compute
and not model._fields[k].related
and not model._fields[k].company_dependent
):
return "function"
return ""
def isproperty(model, k):
if model._fields[k].company_dependent:
return "property"
return ""
def isrelated(model, k):
if model._fields[k].related:
return "related"
return ""
def _get_relation(v):
if v.type in ("many2many", "many2one", "one2many"):
return v.comodel_name
elif v.type == "many2one_reference":
return v.model_field
else:
return ""
def log_model(model, local_registry):
"""
OpenUpgrade: Store the characteristics of the BaseModel and its fields
in the local registry, so that we can compare changes with the
main registry
"""
if not model._name:
return
typemap = {"monetary": "float"}
# persistent models only
if isinstance(model, models.TransientModel):
return
model_registry = local_registry.setdefault(model._name, {})
if model._inherits:
model_registry["_inherits"] = {"_inherits": str(model._inherits)}
for k, v in model._fields.items():
properties = {
"type": typemap.get(v.type, v.type),
"isfunction": isfunction(model, k),
"isproperty": isproperty(model, k),
"isrelated": isrelated(model, k),
"relation": _get_relation(v),
"table": v.relation if v.type == "many2many" else "",
"required": v.required and "required" or "",
"stored": v.store and "stored" or "",
"selection_keys": "",
"req_default": "",
"hasdefault": model._fields[k].default and "hasdefault" or "",
"inherits": "",
}
if v.type == "selection":
if isinstance(v.selection, (tuple, list)):
properties["selection_keys"] = str(sorted([x[0] for x in v.selection]))
else:
properties["selection_keys"] = "function"
elif v.type == "binary":
properties["attachment"] = str(getattr(v, "attachment", False))
default = model._fields[k].default
if v.required and default:
if (
callable(default)
or isinstance(default, str)
and getattr(model._fields[k], default, False)
and callable(getattr(model._fields[k], default))
):
properties["req_default"] = "function"
else:
properties["req_default"] = str(default)
for key, value in properties.items():
if value:
model_registry.setdefault(k, {})[key] = value
def log_xml_id(cr, module, xml_id):
"""
Log xml_ids at load time in the records table.
@ -49,7 +201,7 @@ def log_xml_id(cr, module, xml_id):
)
record = cr.fetchone()
if not record:
_logger.warning("Cannot find xml_id %s" % xml_id)
_logger.warning("Cannot find xml_id %s", xml_id)
return
else:
cr.execute(

4
upgrade_analysis/views/view_upgrade_analysis.xml
View File

@ -36,6 +36,10 @@
name="write_files"
attrs="{'readonly': [('state', '=', 'done')]}"
/>
<field
name="upgrade_path"
attrs="{'readonly': [('state', '=', 'done')], 'invisible': [('write_files', '=', False)]}"
/>
<field
name="analysis_date"
attrs="{'invisible': [('analysis_date', '=', False)]}"

35
upgrade_analysis/wizards/upgrade_generate_record_wizard.py
View File

@ -2,11 +2,14 @@
# Copyright 2016 Opener B.V. <https://opener.am>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
from threading import current_thread
from odoo import _, fields, models
from odoo.exceptions import UserError
from odoo.modules.registry import Registry
from ..odoo_patch.odoo_patch import OdooPatch
class GenerateWizard(models.TransientModel):
_name = "upgrade.generate.record.wizard"
@ -14,26 +17,6 @@ class GenerateWizard(models.TransientModel):
state = fields.Selection([("draft", "Draft"), ("done", "Done")], default="draft")
# from openupgradelib import openupgrade_tools
# TODO, SLG, make better a patch in odoo_patch
# def quirk_standard_calendar_attendances(self):
# """Introduced in Odoo 13. The reinstallation causes a one2many value
# in [(0, 0, {})] format to be loaded on top of the first load, causing a
# violation of database constraint."""
# for cal in ("resource_calendar_std_35h", "resource_calendar_std_38h"):
# record = self.env.ref("resource.%s" % cal, False)
# if record:
# record.attendance_ids.unlink()
# # Truncate the records table
# if openupgrade_tools.table_exists(
# self.env.cr, "upgrade_attribute"
# ) and openupgrade_tools.table_exists(self.env.cr, "upgrade_record"):
# self.env.cr.execute("TRUNCATE upgrade_attribute, upgrade_record;")
# # Run any quirks
# self.quirk_standard_calendar_attendances()
def generate(self):
"""Reinitialize all installed modules.
Equivalent of running the server with '-d <database> --init all'
@ -56,7 +39,17 @@ class GenerateWizard(models.TransientModel):
{"state": "to install"}
)
self.env.cr.commit() # pylint: disable=invalid-commit
Registry.new(self.env.cr.dbname, update_module=True)
# Patch the registry on the thread
thread = current_thread()
thread._upgrade_registry = {}
# Regenerate the registry with monkeypatches that log the records
with OdooPatch():
Registry.new(self.env.cr.dbname, update_module=True)
# Free the registry
delattr(thread, "_upgrade_registry")
# Set domain property
self.env.cr.execute(

28
upgrade_analysis/wizards/upgrade_install_wizard.py
View File

@ -17,6 +17,10 @@ class UpgradeInstallWizard(models.TransientModel):
_name = "upgrade.install.wizard"
_description = "Upgrade Install Wizard"
name = fields.Char(
default=_description,
help="Workaround for https://github.com/OCA/odoorpc/issues/57",
)
state = fields.Selection(
[("draft", "Draft"), ("done", "Done")], readonly=True, default="draft"
)
@ -52,30 +56,38 @@ class UpgradeInstallWizard(models.TransientModel):
for wizard in self:
wizard.module_qty = len(wizard.module_ids)
def select_odoo_modules(self):
def select_odoo_modules(self, extra_domain=None):
self.ensure_one()
modules = self.env["ir.module.module"].search(self._module_ids_domain())
modules = self.env["ir.module.module"].search(
self._module_ids_domain(extra_domain=extra_domain)
)
modules = modules.filtered(lambda x: x.is_odoo_module)
self.module_ids = modules
return self.return_same_form_view()
def select_oca_modules(self):
def select_oca_modules(self, extra_domain=None):
self.ensure_one()
modules = self.env["ir.module.module"].search(self._module_ids_domain())
modules = self.env["ir.module.module"].search(
self._module_ids_domain(extra_domain=extra_domain)
)
modules = modules.filtered(lambda x: x.is_oca_module)
self.module_ids = modules
return self.return_same_form_view()
def select_other_modules(self):
def select_other_modules(self, extra_domain=None):
self.ensure_one()
modules = self.env["ir.module.module"].search(self._module_ids_domain())
modules = self.env["ir.module.module"].search(
self._module_ids_domain(extra_domain=extra_domain)
)
modules = modules.filtered(lambda x: not (x.is_oca_module or x.is_odoo_module))
self.module_ids = modules
return self.return_same_form_view()
def select_installable_modules(self):
def select_installable_modules(self, extra_domain=None):
self.ensure_one()
self.module_ids = self.env["ir.module.module"].search(self._module_ids_domain())
self.module_ids = self.env["ir.module.module"].search(
self._module_ids_domain(extra_domain=extra_domain)
)
return self.return_same_form_view()
def unselect_modules(self):