diff --git a/upgrade_analysis/README.rst b/upgrade_analysis/README.rst new file mode 100644 index 000000000..f3d1c48de --- /dev/null +++ b/upgrade_analysis/README.rst @@ -0,0 +1,59 @@ +.. image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: https://www.gnu.org/licenses/agpl + :alt: License: AGPL-3 + +=============================== +OpenUpgrade Database Comparison +=============================== + +This module provides the tool to generate the database analysis files that indicate how the Odoo data model and module data have changed between two versions of Odoo. Database analysis files for the core modules are included in the OpenUpgrade distribution so as a migration script developer you will not usually need to use this tool yourself. If you do need to run your analysis of a custom set of modules, please refer to the documentation here: https://doc.therp.nl/openupgrade/analysis.html + +Installation +============ + +This module has a python dependency on openerp-client-lib. You need to make this module available in your Python environment, for instance by installing it with the pip tool. + +Known issues / Roadmap +====================== + +* scripts/compare_noupdate_xml_records.py should be integrated in the analysis process (#590) +* Log removed modules in the module that owned them (#468) +* Detect renamed many2many tables (#213) + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues +`_. In case of trouble, please +check there if your issue has already been reported. If you spotted it first, +help us smash it by providing detailed and welcomed feedback. + +Images +------ + +* Odoo Community Association: `Icon `_. + +Contributors +------------ + +* Stefan Rijnhart +* Holger Brunn +* Pedro M. Baeza +* Ferdinand Gassauer +* Florent Xicluna +* Miquel Raïch + +Maintainer +---------- + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit https://odoo-community.org. diff --git a/upgrade_analysis/__init__.py b/upgrade_analysis/__init__.py new file mode 100644 index 000000000..c102a8ca6 --- /dev/null +++ b/upgrade_analysis/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import blacklist diff --git a/upgrade_analysis/__manifest__.py b/upgrade_analysis/__manifest__.py new file mode 100644 index 000000000..79ebfc939 --- /dev/null +++ b/upgrade_analysis/__manifest__.py @@ -0,0 +1,23 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +{ + "name": "OpenUpgrade Records", + "version": "14.0.1.0.0", + "category": "Migration", + "author": "Therp BV, Opener B.V., Odoo Community Association (OCA)", + "website": "https://github.com/OCA/server-tools", + "data": [ + "views/openupgrade_record.xml", + "views/comparison_config.xml", + "views/analysis_wizard.xml", + "views/generate_records_wizard.xml", + "views/install_all_wizard.xml", + "security/ir.model.access.csv", + ], + "installable": True, + "external_dependencies": { + "python": ["odoorpc", "openupgradelib"], + }, + "license": "AGPL-3", +} diff --git a/upgrade_analysis/blacklist.py b/upgrade_analysis/blacklist.py new file mode 100644 index 000000000..814396ad6 --- /dev/null +++ b/upgrade_analysis/blacklist.py @@ -0,0 +1,7 @@ +BLACKLIST_MODULES = [ + # the hw_* modules are not affected by a migration as they don't + # contain any ORM functionality, but they do start up threads that + # delay the process and spit out annoying log messages continously. + "hw_escpos", + "hw_proxy", +] diff --git a/upgrade_analysis/models/__init__.py b/upgrade_analysis/models/__init__.py new file mode 100644 index 000000000..6e4458424 --- /dev/null +++ b/upgrade_analysis/models/__init__.py @@ -0,0 +1,5 @@ +from . import openupgrade_record +from . import comparison_config +from . import analysis_wizard +from . import generate_records_wizard +from . import install_all_wizard diff --git a/upgrade_analysis/models/analysis_wizard.py b/upgrade_analysis/models/analysis_wizard.py new file mode 100644 index 000000000..61a5e2082 --- /dev/null +++ b/upgrade_analysis/models/analysis_wizard.py @@ -0,0 +1,182 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# flake8: noqa: C901 + +import os + +from odoo import fields, models +from odoo.modules import get_module_path + +from ..lib import compare + + +class AnalysisWizard(models.TransientModel): + _name = "openupgrade.analysis.wizard" + _description = "OpenUpgrade Analysis Wizard" + + server_config = fields.Many2one( + "openupgrade.comparison.config", "Configuration", required=True + ) + state = fields.Selection( + [("init", "Init"), ("ready", "Ready")], readonly=True, default="init" + ) + log = fields.Text() + write_files = fields.Boolean( + help="Write analysis files to the module directories", default=True + ) + + def get_communication(self): + """ + Retrieve both sets of database representations, + perform the comparison and register the resulting + change set + """ + + def write_file(module, version, content, filename="openupgrade_analysis.txt"): + module_path = get_module_path(module) + if not module_path: + return "ERROR: could not find module path:\n" + full_path = os.path.join(module_path, "migrations", version) + if not os.path.exists(full_path): + try: + os.makedirs(full_path) + except os.error: + return "ERROR: could not create migrations directory:\n" + logfile = os.path.join(full_path, filename) + try: + f = open(logfile, "w") + except Exception: + return "ERROR: could not open file %s for writing:\n" % logfile + f.write(content) + f.close() + return None + + self.ensure_one() + connection = self.server_config.get_connection() + remote_record_obj = connection.env["openupgrade.record"] + local_record_obj = self.env["openupgrade.record"] + + # Retrieve field representations and compare + remote_records = remote_record_obj.field_dump() + local_records = local_record_obj.field_dump() + res = compare.compare_sets(remote_records, local_records) + + # Retrieve xml id representations and compare + flds = [ + "module", + "model", + "name", + "noupdate", + "prefix", + "suffix", + "domain", + ] + local_xml_records = [ + {field: record[field] for field in flds} + for record in local_record_obj.search([("type", "=", "xmlid")]) + ] + remote_xml_record_ids = remote_record_obj.search([("type", "=", "xmlid")]) + remote_xml_records = [ + {field: record[field] for field in flds} + for record in remote_record_obj.read(remote_xml_record_ids, flds) + ] + res_xml = compare.compare_xml_sets(remote_xml_records, local_xml_records) + + # Retrieve model representations and compare + flds = [ + "module", + "model", + "name", + "model_original_module", + "model_type", + ] + local_model_records = [ + {field: record[field] for field in flds} + for record in local_record_obj.search([("type", "=", "model")]) + ] + remote_model_record_ids = remote_record_obj.search([("type", "=", "model")]) + remote_model_records = [ + {field: record[field] for field in flds} + for record in remote_record_obj.read(remote_model_record_ids, flds) + ] + res_model = compare.compare_model_sets( + remote_model_records, local_model_records + ) + + affected_modules = sorted( + { + record["module"] + for record in remote_records + + local_records + + remote_xml_records + + local_xml_records + + remote_model_records + + local_model_records + } + ) + + # reorder and output the result + keys = ["general"] + affected_modules + modules = { + module["name"]: module + for module in self.env["ir.module.module"].search( + [("state", "=", "installed")] + ) + } + general = "" + for key in keys: + contents = "---Models in module '%s'---\n" % key + if key in res_model: + contents += "\n".join([str(line) for line in res_model[key]]) + if res_model[key]: + contents += "\n" + contents += "---Fields in module '%s'---\n" % key + if key in res: + contents += "\n".join([str(line) for line in sorted(res[key])]) + if res[key]: + contents += "\n" + contents += "---XML records in module '%s'---\n" % key + if key in res_xml: + contents += "\n".join([str(line) for line in res_xml[key]]) + if res_xml[key]: + contents += "\n" + if key not in res and key not in res_xml and key not in res_model: + contents += "---nothing has changed in this module--\n" + if key == "general": + general += contents + continue + if compare.module_map(key) not in modules: + general += ( + "ERROR: module not in list of installed modules:\n" + contents + ) + continue + if key not in modules: + # no need to log in general the merged/renamed modules + continue + if self.write_files: + error = write_file(key, modules[key].installed_version, contents) + if error: + general += error + general += contents + else: + general += contents + + # Store the general log in as many places as possible ;-) + if self.write_files and "base" in modules: + write_file( + "base", + modules["base"].installed_version, + general, + "openupgrade_general_log.txt", + ) + self.server_config.write({"last_log": general}) + self.write({"state": "ready", "log": general}) + + return { + "name": self._description, + "view_mode": "form", + "res_model": self._name, + "type": "ir.actions.act_window", + "res_id": self.id, + } diff --git a/upgrade_analysis/models/comparison_config.py b/upgrade_analysis/models/comparison_config.py new file mode 100644 index 000000000..0e5274bdf --- /dev/null +++ b/upgrade_analysis/models/comparison_config.py @@ -0,0 +1,88 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import logging + +from odoo import fields, models +from odoo.exceptions import UserError +from odoo.tools.translate import _ + +from ..lib import apriori + + +class OpenupgradeComparisonConfig(models.Model): + _name = "openupgrade.comparison.config" + _description = "OpenUpgrade Comparison Configuration" + + name = fields.Char() + server = fields.Char(required=True) + port = fields.Integer(required=True, default=8069) + protocol = fields.Selection( + [("http://", "XML-RPC")], + # ('https://', 'XML-RPC Secure')], not supported by libopenerp + required=True, + default="http://", + ) + database = fields.Char(required=True) + username = fields.Char(required=True) + password = fields.Char(required=True) + last_log = fields.Text() + + def get_connection(self): + self.ensure_one() + import odoorpc + + remote = odoorpc.ODOO(self.server, port=self.port) + remote.login(self.database, self.username, self.password) + return remote + + def test_connection(self): + self.ensure_one() + try: + connection = self.get_connection() + user_model = connection.env["res.users"] + ids = user_model.search([("login", "=", "admin")]) + user_info = user_model.read([ids[0]], ["name"])[0] + except Exception as e: + raise UserError(_("Connection failed.\n\nDETAIL: %s") % e) + raise UserError(_("%s is connected.") % user_info["name"]) + + def analyze(self): + """ Run the analysis wizard """ + self.ensure_one() + wizard = self.env["openupgrade.analysis.wizard"].create( + {"server_config": self.id} + ) + return { + "name": wizard._description, + "view_mode": "form", + "res_model": wizard._name, + "type": "ir.actions.act_window", + "target": "new", + "res_id": wizard.id, + "nodestroy": True, + } + + def install_modules(self): + """ Install same modules as in source DB """ + self.ensure_one() + connection = self.get_connection() + remote_module_obj = connection.env["ir.module.module"] + remote_module_ids = remote_module_obj.search([("state", "=", "installed")]) + + modules = [] + for module_id in remote_module_ids: + mod = remote_module_obj.read([module_id], ["name"])[0] + mod_name = mod["name"] + mod_name = apriori.renamed_modules.get(mod_name, mod_name) + modules.append(mod_name) + _logger = logging.getLogger(__name__) + _logger.debug("remote modules %s", modules) + local_modules = self.env["ir.module.module"].search( + [("name", "in", modules), ("state", "=", "uninstalled")] + ) + _logger.debug("local modules %s", ",".join(local_modules.mapped("name"))) + if local_modules: + local_modules.write({"state": "to install"}) + return {} diff --git a/upgrade_analysis/models/generate_records_wizard.py b/upgrade_analysis/models/generate_records_wizard.py new file mode 100644 index 000000000..b09f6839e --- /dev/null +++ b/upgrade_analysis/models/generate_records_wizard.py @@ -0,0 +1,119 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from openupgradelib import openupgrade_tools + +from odoo import _, fields, models +from odoo.exceptions import UserError +from odoo.modules.registry import Registry + + +class GenerateWizard(models.TransientModel): + _name = "openupgrade.generate.records.wizard" + _description = "OpenUpgrade Generate Records Wizard" + _rec_name = "state" + + state = fields.Selection([("init", "init"), ("ready", "ready")], default="init") + + def quirk_standard_calendar_attendances(self): + """Introduced in Odoo 13. The reinstallation causes a one2many value + in [(0, 0, {})] format to be loaded on top of the first load, causing a + violation of database constraint.""" + for cal in ("resource_calendar_std_35h", "resource_calendar_std_38h"): + record = self.env.ref("resource.%s" % cal, False) + if record: + record.attendance_ids.unlink() + + def generate(self): + """Main wizard step. Make sure that all modules are up-to-date, + then reinitialize all installed modules. + Equivalent of running the server with '-d --init all' + + The goal of this is to fill the records table. + + TODO: update module list and versions, then update all modules?""" + # Truncate the records table + if openupgrade_tools.table_exists( + self.env.cr, "openupgrade_attribute" + ) and openupgrade_tools.table_exists(self.env.cr, "openupgrade_record"): + self.env.cr.execute("TRUNCATE openupgrade_attribute, openupgrade_record;") + + # Run any quirks + self.quirk_standard_calendar_attendances() + + # Need to get all modules in state 'installed' + modules = self.env["ir.module.module"].search( + [("state", "in", ["to install", "to upgrade"])] + ) + if modules: + self.env.cr.commit() # pylint: disable=invalid-commit + Registry.new(self.env.cr.dbname, update_module=True) + # Did we succeed above? + modules = self.env["ir.module.module"].search( + [("state", "in", ["to install", "to upgrade"])] + ) + if modules: + raise UserError( + _("Cannot seem to install or upgrade modules %s") + % (", ".join([module.name for module in modules])) + ) + # Now reinitialize all installed modules + self.env["ir.module.module"].search([("state", "=", "installed")]).write( + {"state": "to install"} + ) + self.env.cr.commit() # pylint: disable=invalid-commit + Registry.new(self.env.cr.dbname, update_module=True) + + # Set domain property + self.env.cr.execute( + """ UPDATE openupgrade_record our + SET domain = iaw.domain + FROM ir_model_data imd + JOIN ir_act_window iaw ON imd.res_id = iaw.id + WHERE our.type = 'xmlid' + AND imd.model = 'ir.actions.act_window' + AND our.model = imd.model + AND our.name = imd.module || '.' || imd.name + """ + ) + self.env.cache.invalidate( + [ + (self.env["openupgrade.record"]._fields["domain"], None), + ] + ) + + # Set noupdate property from ir_model_data + self.env.cr.execute( + """ UPDATE openupgrade_record our + SET noupdate = imd.noupdate + FROM ir_model_data imd + WHERE our.type = 'xmlid' + AND our.model = imd.model + AND our.name = imd.module || '.' || imd.name + """ + ) + self.env.cache.invalidate( + [ + (self.env["openupgrade.record"]._fields["noupdate"], None), + ] + ) + + # Log model records + self.env.cr.execute( + """INSERT INTO openupgrade_record + (module, name, model, type) + SELECT imd2.module, imd2.module || '.' || imd.name AS name, + im.model, 'model' AS type + FROM ( + SELECT min(id) as id, name, res_id + FROM ir_model_data + WHERE name LIKE 'model_%' AND model = 'ir.model' + GROUP BY name, res_id + ) imd + JOIN ir_model_data imd2 ON imd2.id = imd.id + JOIN ir_model im ON imd.res_id = im.id + ORDER BY imd.name, imd.id""", + ) + + return self.write({"state": "ready"}) diff --git a/upgrade_analysis/models/install_all_wizard.py b/upgrade_analysis/models/install_all_wizard.py new file mode 100644 index 000000000..c1085418e --- /dev/null +++ b/upgrade_analysis/models/install_all_wizard.py @@ -0,0 +1,51 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import api, fields, models +from odoo.modules.registry import Registry +from odoo.osv.expression import AND + +from ..blacklist import BLACKLIST_MODULES + + +class InstallAll(models.TransientModel): + _name = "openupgrade.install.all.wizard" + _description = "OpenUpgrade Install All Wizard" + + state = fields.Selection( + [("init", "init"), ("ready", "ready")], readonly=True, default="init" + ) + to_install = fields.Integer("Number of modules to install", readonly=True) + + @api.model + def default_get(self, fields): + """Update module list and retrieve the number + of installable modules""" + res = super(InstallAll, self).default_get(fields) + update, add = self.env["ir.module.module"].update_list() + modules = self.env["ir.module.module"].search( + [("state", "not in", ["uninstallable", "unknown"])] + ) + res["to_install"] = len(modules) + return res + + def install_all(self, extra_domain=None): + """Main wizard step. Set all installable modules to install + and actually install them. Exclude testing modules.""" + domain = [ + "&", + "&", + ("state", "not in", ["uninstallable", "unknown"]), + ("category_id.name", "!=", "Tests"), + ("name", "not in", BLACKLIST_MODULES), + ] + if extra_domain: + domain = AND([domain, extra_domain]) + modules = self.env["ir.module.module"].search(domain) + if modules: + modules.write({"state": "to install"}) + self.env.cr.commit() # pylint: disable=invalid-commit + Registry.new(self.env.cr.dbname, update_module=True) + self.write({"state": "ready"}) + return True diff --git a/upgrade_analysis/models/openupgrade_record.py b/upgrade_analysis/models/openupgrade_record.py new file mode 100644 index 000000000..80b5a8a3a --- /dev/null +++ b/upgrade_analysis/models/openupgrade_record.py @@ -0,0 +1,113 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import api, fields, models + + +class Attribute(models.Model): + _name = "openupgrade.attribute" + _description = "OpenUpgrade Attribute" + + name = fields.Char(readonly=True) + value = fields.Char(readonly=True) + record_id = fields.Many2one( + "openupgrade.record", + ondelete="CASCADE", + readonly=True, + ) + + +class Record(models.Model): + _name = "openupgrade.record" + _description = "OpenUpgrade Record" + + name = fields.Char(readonly=True) + module = fields.Char(readonly=True) + model = fields.Char(readonly=True) + field = fields.Char(readonly=True) + mode = fields.Selection( + [("create", "Create"), ("modify", "Modify")], + help="Set to Create if a field is newly created " + "in this module. If this module modifies an attribute of an " + "existing field, set to Modify.", + readonly=True, + ) + type = fields.Selection( # Uh oh, reserved keyword + [("field", "Field"), ("xmlid", "XML ID"), ("model", "Model")], + readonly=True, + ) + attribute_ids = fields.One2many("openupgrade.attribute", "record_id", readonly=True) + noupdate = fields.Boolean(readonly=True) + domain = fields.Char(readonly=True) + prefix = fields.Char(compute="_compute_prefix_and_suffix") + suffix = fields.Char(compute="_compute_prefix_and_suffix") + model_original_module = fields.Char(compute="_compute_model_original_module") + model_type = fields.Char(compute="_compute_model_type") + + @api.depends("name") + def _compute_prefix_and_suffix(self): + for rec in self: + rec.prefix, rec.suffix = rec.name.split(".", 1) + + @api.depends("model", "type") + def _compute_model_original_module(self): + for rec in self: + if rec.type == "model": + rec.model_original_module = self.env[rec.model]._original_module + else: + rec.model_original_module = "" + + @api.depends("model", "type") + def _compute_model_type(self): + for rec in self: + if rec.type == "model": + model = self.env[rec.model] + if model._auto and model._transient: + rec.model_type = "transient" + elif model._auto: + rec.model_type = "" + elif not model._auto and model._abstract: + rec.model_type = "abstract" + else: + rec.model_type = "sql_view" + else: + rec.model_type = "" + + @api.model + def field_dump(self): + keys = [ + "attachment", + "module", + "mode", + "model", + "field", + "type", + "isfunction", + "isproperty", + "isrelated", + "relation", + "required", + "stored", + "selection_keys", + "req_default", + "hasdefault", + "table", + "inherits", + ] + + template = {x: False for x in keys} + data = [] + for record in self.search([("type", "=", "field")]): + repre = template.copy() + repre.update( + { + "module": record.module, + "model": record.model, + "field": record.field, + "mode": record.mode, + } + ) + repre.update({x.name: x.value for x in record.attribute_ids}) + data.append(repre) + return data diff --git a/upgrade_analysis/security/ir.model.access.csv b/upgrade_analysis/security/ir.model.access.csv new file mode 100644 index 000000000..2ab5e67c1 --- /dev/null +++ b/upgrade_analysis/security/ir.model.access.csv @@ -0,0 +1,4 @@ +"id","name","model_id:id","group_id:id","perm_read","perm_write","perm_create","perm_unlink" +"access_openupgrade_record","openupgrade.record all","model_openupgrade_record",,1,0,0,0 +"access_openupgrade_attribute","openupgrade.attribute all","model_openupgrade_attribute",,1,0,0,0 +"access_openupgrade_comparison_config","openupgrade.comparison.config","model_openupgrade_comparison_config",base.group_system,1,1,1,1 diff --git a/upgrade_analysis/views/analysis_wizard.xml b/upgrade_analysis/views/analysis_wizard.xml new file mode 100644 index 000000000..efaa5294f --- /dev/null +++ b/upgrade_analysis/views/analysis_wizard.xml @@ -0,0 +1,36 @@ + + + + + view.openupgrade.analysis_wizard.form + openupgrade.analysis.wizard + +
+ + + + + + +
+
+
+ + + + diff --git a/upgrade_analysis/views/comparison_config.xml b/upgrade_analysis/views/comparison_config.xml new file mode 100644 index 000000000..db0ae7770 --- /dev/null +++ b/upgrade_analysis/views/comparison_config.xml @@ -0,0 +1,77 @@ + + + + + view.openupgrade.comparison_config.tree + openupgrade.comparison.config + + + + + + + + + + + + + view.openupgrade.comparison_config.form + openupgrade.comparison.config + +
+ + + + + + + + + +