From 2c0e612cb99edefb53599328b26940221736dde3 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Fri, 6 Nov 2020 21:39:26 +0100 Subject: [PATCH 01/67] [MOVE] openupgrade_records : move module from OCA/OpenUpgrade (branch 13.0) to OCA/server-tools Based on commit 746b7acbd90d62f9ffe6ee17472a1a3533e36597 (Fri Nov 6 17:18:47 2020 +0100) Co-authored-by: Stefan Rijnhart --- upgrade_analysis/apriori.py | 98 ++++++++ upgrade_analysis/compare.py | 438 ++++++++++++++++++++++++++++++++++++ 2 files changed, 536 insertions(+) create mode 100644 upgrade_analysis/apriori.py create mode 100644 upgrade_analysis/compare.py diff --git a/upgrade_analysis/apriori.py b/upgrade_analysis/apriori.py new file mode 100644 index 000000000..973300fa0 --- /dev/null +++ b/upgrade_analysis/apriori.py @@ -0,0 +1,98 @@ +""" Encode any known changes to the database here +to help the matching process +""" + +renamed_modules = { + # Odoo + 'crm_reveal': 'crm_iap_lead', + 'document': 'attachment_indexation', + 'payment_ogone': 'payment_ingenico', + # OCA/hr + # TODO: Transform possible data + 'hr_skill': 'hr_skills' +} + +merged_modules = { + # Odoo + 'account_cancel': 'account', + 'account_voucher': 'account', + 'crm_phone_validation': 'crm', + 'decimal_precision': 'base', + 'delivery_hs_code': 'delivery', + 'hw_scale': 'hw_drivers', + 'hw_scanner': 'hw_drivers', + 'hw_screen': 'hw_drivers', + 'l10n_fr_certification': 'account', + 'l10n_fr_sale_closing': 'l10n_fr', + 'mrp_bom_cost': 'mrp_account', + 'mrp_byproduct': 'mrp', + 'payment_stripe_sca': 'payment_stripe', + 'stock_zebra': 'stock', + 'survey_crm': 'survey', + 'test_pylint': 'test_lint', + 'web_settings_dashboard': 'base_setup', + 'website_crm_phone_validation': 'website_crm', + 'website_sale_link_tracker': 'website_sale', + 'website_survey': 'survey', + # OCA/account-financial-tools + 'account_move_chatter': 'account', + # OCA/account-reconcile + 'account_set_reconcilable': 'account', + # OCA/l10n-spain + 'l10n_es_aeat_sii': 'l10n_es_aeat_sii_oca', + # OCA/server-backend + 'base_suspend_security': 'base', + # OCA/social + 'mass_mailing_unique': 'mass_mailing', + # OCA/timesheet + 'sale_timesheet_existing_project': 'sale_timesheet', + # OCA/web + 'web_favicon': 'base', + 'web_widget_color': 'web', + 'web_widget_many2many_tags_multi_selection': 'web', + # OCA/website + 'website_canonical_url': 'website', + 'website_logo': 'website', +} + +# only used here for openupgrade_records analysis: +renamed_models = { + # Odoo + 'account.register.payments': 'account.payment.register', + 'crm.reveal.industry': 'crm.iap.lead.industry', + 'crm.reveal.role': 'crm.iap.lead.role', + 'crm.reveal.seniority': 'crm.iap.lead.seniority', + 'mail.blacklist.mixin': 'mail.thread.blacklist', + 'mail.mail.statistics': 'mailing.trace', + 'mail.statistics.report': 'mailing.trace.report', + 'mail.mass_mailing': 'mailing.mailing', + 'mail.mass_mailing.contact': 'mailing.contact', + 'mail.mass_mailing.list': 'mailing.list', + 'mail.mass_mailing.list_contact_rel': 'mailing.contact.subscription', + 'mail.mass_mailing.stage': 'utm.stage', + 'mail.mass_mailing.tag': 'utm.tag', + 'mail.mass_mailing.test': 'mailing.mailing.test', + 'mass.mailing.list.merge': 'mailing.list.merge', + 'mass.mailing.schedule.date': 'mailing.mailing.schedule.date', + 'mrp.subproduct': 'mrp.bom.byproduct', + 'sms.send_sms': 'sms.composer', + 'stock.fixed.putaway.strat': 'stock.putaway.rule', + 'survey.mail.compose.message': 'survey.invite', + 'website.redirect': 'website.rewrite', + # OCA/... +} + +# only used here for openupgrade_records analysis: +merged_models = { + # Odoo + 'account.invoice': 'account.move', + 'account.invoice.line': 'account.move.line', + 'account.invoice.tax': 'account.move.line', + 'account.voucher': 'account.move', + 'account.voucher.line': 'account.move.line', + 'lunch.order.line': 'lunch.order', + 'mail.mass_mailing.campaign': 'utm.campaign', + 'slide.category': 'slide.slide', + 'survey.page': 'survey.question', + # OCA/... +} diff --git a/upgrade_analysis/compare.py b/upgrade_analysis/compare.py new file mode 100644 index 000000000..52ac7bbe8 --- /dev/null +++ b/upgrade_analysis/compare.py @@ -0,0 +1,438 @@ +# coding: utf-8 +# Copyright 2011-2015 Therp BV +# Copyright 2015-2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +##################################################################### +# library providing a function to analyse two progressive database +# layouts from the OpenUpgrade server. +##################################################################### + +import collections +import copy + +from odoo.addons.openupgrade_records.lib import apriori + + +def module_map(module): + return apriori.renamed_modules.get( + module, apriori.merged_modules.get(module, module)) + + +def model_rename_map(model): + return apriori.renamed_models.get(model, model) + + +def model_map(model): + return apriori.renamed_models.get( + model, apriori.merged_models.get(model, model)) + + +def inv_model_map(model): + inv_model_map_dict = {v: k for k, v in apriori.renamed_models.items()} + return inv_model_map_dict.get(model, model) + + +IGNORE_FIELDS = [ + 'create_date', + 'create_uid', + 'id', + 'write_date', + 'write_uid', + ] + + +def compare_records(dict_old, dict_new, fields): + """ + Check equivalence of two OpenUpgrade field representations + with respect to the keys in the 'fields' arguments. + Take apriori knowledge into account for mapped modules or + model names. + Return True of False. + """ + for field in fields: + if field == 'module': + if module_map(dict_old['module']) != dict_new['module']: + return False + elif field == 'model': + if model_rename_map(dict_old['model']) != dict_new['model']: + return False + elif field == 'other_prefix': + if dict_old['module'] != dict_old['prefix'] or \ + dict_new['module'] != dict_new['prefix']: + return False + if dict_old['model'] == 'ir.ui.view': + # basically, to avoid the assets_backend case + return False + elif dict_old[field] != dict_new[field]: + return False + return True + + +def search(item, item_list, fields): + """ + Find a match of a dictionary in a list of similar dictionaries + with respect to the keys in the 'fields' arguments. + Return the item if found or None. + """ + for other in item_list: + if not compare_records(item, other, fields): + continue + return other + # search for renamed fields + if 'field' in fields: + for other in item_list: + if not item['field'] or item['field'] is not None or \ + item['isproperty']: + continue + if compare_records( + dict(item, field=other['field']), other, fields): + return other + return None + + +def fieldprint(old, new, field, text, reprs): + fieldrepr = "%s (%s)" % (old['field'], old['type']) + fullrepr = '%-12s / %-24s / %-30s' % ( + old['module'], old['model'], fieldrepr) + if not text: + text = "%s is now '%s' ('%s')" % (field, new[field], old[field]) + if field == 'relation': + text += ' [nothing to do]' + reprs[module_map(old['module'])].append("%s: %s" % (fullrepr, text)) + if field == 'module': + text = "previously in module %s" % old[field] + fullrepr = '%-12s / %-24s / %-30s' % ( + new['module'], old['model'], fieldrepr) + reprs[module_map(new['module'])].append("%s: %s" % (fullrepr, text)) + + +def report_generic(new, old, attrs, reprs): + for attr in attrs: + if attr == 'required': + if old[attr] != new['required'] and new['required']: + text = "now required" + if new['req_default']: + text += ', req_default: %s' % new['req_default'] + fieldprint(old, new, '', text, reprs) + elif attr == 'stored': + if old[attr] != new[attr]: + if new['stored']: + text = "is now stored" + else: + text = "not stored anymore" + fieldprint(old, new, '', text, reprs) + elif attr == 'isfunction': + if old[attr] != new[attr]: + if new['isfunction']: + text = "now a function" + else: + text = "not a function anymore" + fieldprint(old, new, '', text, reprs) + elif attr == 'isproperty': + if old[attr] != new[attr]: + if new[attr]: + text = "now a property" + else: + text = "not a property anymore" + fieldprint(old, new, '', text, reprs) + elif attr == 'isrelated': + if old[attr] != new[attr]: + if new[attr]: + text = "now related" + else: + text = "not related anymore" + fieldprint(old, new, '', text, reprs) + elif old[attr] != new[attr]: + fieldprint(old, new, attr, '', reprs) + + +def compare_sets(old_records, new_records): + """ + Compare a set of OpenUpgrade field representations. + Try to match the equivalent fields in both sets. + Return a textual representation of changes in a dictionary with + module names as keys. Special case is the 'general' key + which contains overall remarks and matching statistics. + """ + reprs = collections.defaultdict(list) + + def clean_records(records): + result = [] + for record in records: + if record['field'] not in IGNORE_FIELDS: + result.append(record) + return result + + old_records = clean_records(old_records) + new_records = clean_records(new_records) + + origlen = len(old_records) + new_models = set([column['model'] for column in new_records]) + old_models = set([column['model'] for column in old_records]) + + matched_direct = 0 + matched_other_module = 0 + matched_other_type = 0 + in_obsolete_models = 0 + + obsolete_models = [] + for model in old_models: + if model not in new_models: + if model_map(model) not in new_models: + obsolete_models.append(model) + + non_obsolete_old_records = [] + for column in copy.copy(old_records): + if column['model'] in obsolete_models: + in_obsolete_models += 1 + else: + non_obsolete_old_records.append(column) + + def match(match_fields, report_fields, warn=False): + count = 0 + for column in copy.copy(non_obsolete_old_records): + found = search(column, new_records, match_fields) + if found: + if warn: + pass + # print "Tentatively" + report_generic(found, column, report_fields, reprs) + old_records.remove(column) + non_obsolete_old_records.remove(column) + new_records.remove(found) + count += 1 + return count + + matched_direct = match( + ['module', 'mode', 'model', 'field'], + ['relation', 'type', 'selection_keys', 'inherits', 'stored', + 'isfunction', 'isrelated', 'required', 'table']) + + # other module, same type and operation + matched_other_module = match( + ['mode', 'model', 'field', 'type'], + ['module', 'relation', 'selection_keys', 'inherits', 'stored', + 'isfunction', 'isrelated', 'required', 'table']) + + # other module, same operation, other type + matched_other_type = match( + ['mode', 'model', 'field'], + ['relation', 'type', 'selection_keys', 'inherits', 'stored', + 'isfunction', 'isrelated', 'required', 'table']) + + printkeys = [ + 'relation', 'required', 'selection_keys', + 'req_default', 'inherits', 'mode', 'attachment', + ] + for column in old_records: + # we do not care about removed non stored function fields + if not column['stored'] and ( + column['isfunction'] or column['isrelated']): + continue + if column['mode'] == 'create': + column['mode'] = '' + extra_message = ", ".join( + [k + ': ' + str(column[k]) if k != str(column[k]) else k + for k in printkeys if column[k]] + ) + if extra_message: + extra_message = " " + extra_message + fieldprint( + column, '', '', "DEL" + extra_message, reprs) + + printkeys.extend([ + 'hasdefault', + ]) + for column in new_records: + # we do not care about newly added non stored function fields + if not column['stored'] and ( + column['isfunction'] or column['isrelated']): + continue + if column['mode'] == 'create': + column['mode'] = '' + printkeys_plus = printkeys.copy() + if column['isfunction'] or column['isrelated']: + printkeys_plus.extend(['isfunction', 'isrelated', 'stored']) + extra_message = ", ".join( + [k + ': ' + str(column[k]) if k != str(column[k]) else k + for k in printkeys_plus if column[k]] + ) + if extra_message: + extra_message = " " + extra_message + fieldprint( + column, '', '', "NEW" + extra_message, reprs) + + for line in [ + "# %d fields matched," % (origlen - len(old_records)), + "# Direct match: %d" % matched_direct, + "# Found in other module: %d" % matched_other_module, + "# Found with different type: %d" % matched_other_type, + "# In obsolete models: %d" % in_obsolete_models, + "# Not matched: %d" % len(old_records), + "# New columns: %d" % len(new_records) + ]: + reprs['general'].append(line) + return reprs + + +def compare_xml_sets(old_records, new_records): + reprs = collections.defaultdict(list) + + def match(match_fields, match_type='direct'): + matched_records = [] + for column in copy.copy(old_records): + found = search(column, new_records, match_fields) + if found: + old_records.remove(column) + new_records.remove(found) + if match_type != 'direct': + column['old'] = True + found['new'] = True + column[match_type] = found['module'] + found[match_type] = column['module'] + found['domain'] = column['domain'] != found['domain'] and \ + column['domain'] != '[]' and found['domain'] is False + column['domain'] = False + column['noupdate_switched'] = False + found['noupdate_switched'] = \ + column['noupdate'] != found['noupdate'] + if match_type != 'direct': + matched_records.append(column) + matched_records.append(found) + elif (match_type == 'direct' and found['domain']) or \ + found['noupdate_switched']: + matched_records.append(found) + return matched_records + + # direct match + modified_records = match(['module', 'model', 'name']) + + # other module, same full xmlid + moved_records = match(['model', 'name'], 'moved') + + # other module, same suffix, other prefix + renamed_records = match(['model', 'suffix', 'other_prefix'], 'renamed') + + for record in old_records: + record['old'] = True + record['domain'] = False + record['noupdate_switched'] = False + for record in new_records: + record['new'] = True + record['domain'] = False + record['noupdate_switched'] = False + + sorted_records = sorted( + old_records + new_records + moved_records + renamed_records + + modified_records, + key=lambda k: (k['model'], 'old' in k, k['name']) + ) + for entry in sorted_records: + content = '' + if 'old' in entry: + content = 'DEL %(model)s: %(name)s' % entry + if 'moved' in entry: + content += ' [potentially moved to %(moved)s module]' % entry + elif 'renamed' in entry: + content += ' [renamed to %(renamed)s module]' % entry + elif 'new' in entry: + content = 'NEW %(model)s: %(name)s' % entry + if 'moved' in entry: + content += ' [potentially moved from %(moved)s module]' % entry + elif 'renamed' in entry: + content += ' [renamed from %(renamed)s module]' % entry + if 'old' not in entry and 'new' not in entry: + content = '%(model)s: %(name)s' % entry + if entry['domain']: + content += ' (deleted domain)' + if entry['noupdate']: + content += ' (noupdate)' + if entry['noupdate_switched']: + content += ' (noupdate switched)' + reprs[module_map(entry['module'])].append(content) + return reprs + + +def compare_model_sets(old_records, new_records): + """ + Compare a set of OpenUpgrade model representations. + """ + reprs = collections.defaultdict(list) + + new_models = {column['model']: column['module'] for column in new_records} + old_models = {column['model']: column['module'] for column in old_records} + + obsolete_models = [] + for column in copy.copy(old_records): + model = column['model'] + if model in old_models: + if model not in new_models: + if model_map(model) not in new_models: + obsolete_models.append(model) + text = 'obsolete model %s' % model + if column['model_type']: + text += " [%s]" % column['model_type'] + reprs[module_map(column['module'])].append(text) + reprs['general'].append('obsolete model %s [module %s]' % ( + model, module_map(column['module']))) + else: + moved_module = '' + if module_map(column['module']) != new_models[model_map( + model)]: + moved_module = ' in module %s' % new_models[model_map( + model)] + text = 'obsolete model %s (renamed to %s%s)' % ( + model, model_map(model), moved_module) + if column['model_type']: + text += " [%s]" % column['model_type'] + reprs[module_map(column['module'])].append(text) + reprs['general'].append( + 'obsolete model %s (renamed to %s) [module %s]' % ( + model, model_map(model), + module_map(column['module']))) + else: + if module_map(column['module']) != new_models[model]: + text = 'model %s (moved to %s)' % ( + model, new_models[model]) + if column['model_type']: + text += " [%s]" % column['model_type'] + reprs[module_map(column['module'])].append(text) + text = 'model %s (moved from %s)' % ( + model, old_models[model]) + if column['model_type']: + text += " [%s]" % column['model_type'] + + for column in copy.copy(new_records): + model = column['model'] + if model in new_models: + if model not in old_models: + if inv_model_map(model) not in old_models: + text = 'new model %s' % model + if column['model_type']: + text += " [%s]" % column['model_type'] + reprs[column['module']].append(text) + reprs['general'].append('new model %s [module %s]' % ( + model, column['module'])) + else: + moved_module = '' + if column['module'] != module_map(old_models[inv_model_map( + model)]): + moved_module = ' in module %s' % old_models[ + inv_model_map(model)] + text = 'new model %s (renamed from %s%s)' % ( + model, inv_model_map(model), moved_module) + if column['model_type']: + text += " [%s]" % column['model_type'] + reprs[column['module']].append(text) + reprs['general'].append( + 'new model %s (renamed from %s) [module %s]' % ( + model, inv_model_map(model), column['module'])) + else: + if column['module'] != module_map(old_models[model]): + text = 'model %s (moved from %s)' % ( + model, old_models[model]) + if column['model_type']: + text += " [%s]" % column['model_type'] + reprs[column['module']].append(text) + return reprs From 3ada67d9e7bec3d86961e02746eb0c8adbe124f2 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Fri, 6 Nov 2020 22:24:04 +0100 Subject: [PATCH 02/67] [REF] rename module openupgrade_records into upgrade_analysis --- upgrade_analysis/README.rst | 59 ++++++ upgrade_analysis/__init__.py | 2 + upgrade_analysis/__manifest__.py | 23 +++ upgrade_analysis/blacklist.py | 7 + upgrade_analysis/models/__init__.py | 5 + upgrade_analysis/models/analysis_wizard.py | 182 ++++++++++++++++++ upgrade_analysis/models/comparison_config.py | 88 +++++++++ .../models/generate_records_wizard.py | 119 ++++++++++++ upgrade_analysis/models/install_all_wizard.py | 51 +++++ upgrade_analysis/models/openupgrade_record.py | 113 +++++++++++ upgrade_analysis/security/ir.model.access.csv | 4 + upgrade_analysis/views/analysis_wizard.xml | 36 ++++ upgrade_analysis/views/comparison_config.xml | 77 ++++++++ .../views/generate_records_wizard.xml | 47 +++++ upgrade_analysis/views/install_all_wizard.xml | 48 +++++ upgrade_analysis/views/openupgrade_record.xml | 96 +++++++++ 16 files changed, 957 insertions(+) create mode 100644 upgrade_analysis/README.rst create mode 100644 upgrade_analysis/__init__.py create mode 100644 upgrade_analysis/__manifest__.py create mode 100644 upgrade_analysis/blacklist.py create mode 100644 upgrade_analysis/models/__init__.py create mode 100644 upgrade_analysis/models/analysis_wizard.py create mode 100644 upgrade_analysis/models/comparison_config.py create mode 100644 upgrade_analysis/models/generate_records_wizard.py create mode 100644 upgrade_analysis/models/install_all_wizard.py create mode 100644 upgrade_analysis/models/openupgrade_record.py create mode 100644 upgrade_analysis/security/ir.model.access.csv create mode 100644 upgrade_analysis/views/analysis_wizard.xml create mode 100644 upgrade_analysis/views/comparison_config.xml create mode 100644 upgrade_analysis/views/generate_records_wizard.xml create mode 100644 upgrade_analysis/views/install_all_wizard.xml create mode 100644 upgrade_analysis/views/openupgrade_record.xml diff --git a/upgrade_analysis/README.rst b/upgrade_analysis/README.rst new file mode 100644 index 000000000..f3d1c48de --- /dev/null +++ b/upgrade_analysis/README.rst @@ -0,0 +1,59 @@ +.. image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: https://www.gnu.org/licenses/agpl + :alt: License: AGPL-3 + +=============================== +OpenUpgrade Database Comparison +=============================== + +This module provides the tool to generate the database analysis files that indicate how the Odoo data model and module data have changed between two versions of Odoo. Database analysis files for the core modules are included in the OpenUpgrade distribution so as a migration script developer you will not usually need to use this tool yourself. If you do need to run your analysis of a custom set of modules, please refer to the documentation here: https://doc.therp.nl/openupgrade/analysis.html + +Installation +============ + +This module has a python dependency on openerp-client-lib. You need to make this module available in your Python environment, for instance by installing it with the pip tool. + +Known issues / Roadmap +====================== + +* scripts/compare_noupdate_xml_records.py should be integrated in the analysis process (#590) +* Log removed modules in the module that owned them (#468) +* Detect renamed many2many tables (#213) + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues +`_. In case of trouble, please +check there if your issue has already been reported. If you spotted it first, +help us smash it by providing detailed and welcomed feedback. + +Images +------ + +* Odoo Community Association: `Icon `_. + +Contributors +------------ + +* Stefan Rijnhart +* Holger Brunn +* Pedro M. Baeza +* Ferdinand Gassauer +* Florent Xicluna +* Miquel Raïch + +Maintainer +---------- + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit https://odoo-community.org. diff --git a/upgrade_analysis/__init__.py b/upgrade_analysis/__init__.py new file mode 100644 index 000000000..c102a8ca6 --- /dev/null +++ b/upgrade_analysis/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import blacklist diff --git a/upgrade_analysis/__manifest__.py b/upgrade_analysis/__manifest__.py new file mode 100644 index 000000000..79ebfc939 --- /dev/null +++ b/upgrade_analysis/__manifest__.py @@ -0,0 +1,23 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +{ + "name": "OpenUpgrade Records", + "version": "14.0.1.0.0", + "category": "Migration", + "author": "Therp BV, Opener B.V., Odoo Community Association (OCA)", + "website": "https://github.com/OCA/server-tools", + "data": [ + "views/openupgrade_record.xml", + "views/comparison_config.xml", + "views/analysis_wizard.xml", + "views/generate_records_wizard.xml", + "views/install_all_wizard.xml", + "security/ir.model.access.csv", + ], + "installable": True, + "external_dependencies": { + "python": ["odoorpc", "openupgradelib"], + }, + "license": "AGPL-3", +} diff --git a/upgrade_analysis/blacklist.py b/upgrade_analysis/blacklist.py new file mode 100644 index 000000000..814396ad6 --- /dev/null +++ b/upgrade_analysis/blacklist.py @@ -0,0 +1,7 @@ +BLACKLIST_MODULES = [ + # the hw_* modules are not affected by a migration as they don't + # contain any ORM functionality, but they do start up threads that + # delay the process and spit out annoying log messages continously. + "hw_escpos", + "hw_proxy", +] diff --git a/upgrade_analysis/models/__init__.py b/upgrade_analysis/models/__init__.py new file mode 100644 index 000000000..6e4458424 --- /dev/null +++ b/upgrade_analysis/models/__init__.py @@ -0,0 +1,5 @@ +from . import openupgrade_record +from . import comparison_config +from . import analysis_wizard +from . import generate_records_wizard +from . import install_all_wizard diff --git a/upgrade_analysis/models/analysis_wizard.py b/upgrade_analysis/models/analysis_wizard.py new file mode 100644 index 000000000..61a5e2082 --- /dev/null +++ b/upgrade_analysis/models/analysis_wizard.py @@ -0,0 +1,182 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# flake8: noqa: C901 + +import os + +from odoo import fields, models +from odoo.modules import get_module_path + +from ..lib import compare + + +class AnalysisWizard(models.TransientModel): + _name = "openupgrade.analysis.wizard" + _description = "OpenUpgrade Analysis Wizard" + + server_config = fields.Many2one( + "openupgrade.comparison.config", "Configuration", required=True + ) + state = fields.Selection( + [("init", "Init"), ("ready", "Ready")], readonly=True, default="init" + ) + log = fields.Text() + write_files = fields.Boolean( + help="Write analysis files to the module directories", default=True + ) + + def get_communication(self): + """ + Retrieve both sets of database representations, + perform the comparison and register the resulting + change set + """ + + def write_file(module, version, content, filename="openupgrade_analysis.txt"): + module_path = get_module_path(module) + if not module_path: + return "ERROR: could not find module path:\n" + full_path = os.path.join(module_path, "migrations", version) + if not os.path.exists(full_path): + try: + os.makedirs(full_path) + except os.error: + return "ERROR: could not create migrations directory:\n" + logfile = os.path.join(full_path, filename) + try: + f = open(logfile, "w") + except Exception: + return "ERROR: could not open file %s for writing:\n" % logfile + f.write(content) + f.close() + return None + + self.ensure_one() + connection = self.server_config.get_connection() + remote_record_obj = connection.env["openupgrade.record"] + local_record_obj = self.env["openupgrade.record"] + + # Retrieve field representations and compare + remote_records = remote_record_obj.field_dump() + local_records = local_record_obj.field_dump() + res = compare.compare_sets(remote_records, local_records) + + # Retrieve xml id representations and compare + flds = [ + "module", + "model", + "name", + "noupdate", + "prefix", + "suffix", + "domain", + ] + local_xml_records = [ + {field: record[field] for field in flds} + for record in local_record_obj.search([("type", "=", "xmlid")]) + ] + remote_xml_record_ids = remote_record_obj.search([("type", "=", "xmlid")]) + remote_xml_records = [ + {field: record[field] for field in flds} + for record in remote_record_obj.read(remote_xml_record_ids, flds) + ] + res_xml = compare.compare_xml_sets(remote_xml_records, local_xml_records) + + # Retrieve model representations and compare + flds = [ + "module", + "model", + "name", + "model_original_module", + "model_type", + ] + local_model_records = [ + {field: record[field] for field in flds} + for record in local_record_obj.search([("type", "=", "model")]) + ] + remote_model_record_ids = remote_record_obj.search([("type", "=", "model")]) + remote_model_records = [ + {field: record[field] for field in flds} + for record in remote_record_obj.read(remote_model_record_ids, flds) + ] + res_model = compare.compare_model_sets( + remote_model_records, local_model_records + ) + + affected_modules = sorted( + { + record["module"] + for record in remote_records + + local_records + + remote_xml_records + + local_xml_records + + remote_model_records + + local_model_records + } + ) + + # reorder and output the result + keys = ["general"] + affected_modules + modules = { + module["name"]: module + for module in self.env["ir.module.module"].search( + [("state", "=", "installed")] + ) + } + general = "" + for key in keys: + contents = "---Models in module '%s'---\n" % key + if key in res_model: + contents += "\n".join([str(line) for line in res_model[key]]) + if res_model[key]: + contents += "\n" + contents += "---Fields in module '%s'---\n" % key + if key in res: + contents += "\n".join([str(line) for line in sorted(res[key])]) + if res[key]: + contents += "\n" + contents += "---XML records in module '%s'---\n" % key + if key in res_xml: + contents += "\n".join([str(line) for line in res_xml[key]]) + if res_xml[key]: + contents += "\n" + if key not in res and key not in res_xml and key not in res_model: + contents += "---nothing has changed in this module--\n" + if key == "general": + general += contents + continue + if compare.module_map(key) not in modules: + general += ( + "ERROR: module not in list of installed modules:\n" + contents + ) + continue + if key not in modules: + # no need to log in general the merged/renamed modules + continue + if self.write_files: + error = write_file(key, modules[key].installed_version, contents) + if error: + general += error + general += contents + else: + general += contents + + # Store the general log in as many places as possible ;-) + if self.write_files and "base" in modules: + write_file( + "base", + modules["base"].installed_version, + general, + "openupgrade_general_log.txt", + ) + self.server_config.write({"last_log": general}) + self.write({"state": "ready", "log": general}) + + return { + "name": self._description, + "view_mode": "form", + "res_model": self._name, + "type": "ir.actions.act_window", + "res_id": self.id, + } diff --git a/upgrade_analysis/models/comparison_config.py b/upgrade_analysis/models/comparison_config.py new file mode 100644 index 000000000..0e5274bdf --- /dev/null +++ b/upgrade_analysis/models/comparison_config.py @@ -0,0 +1,88 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import logging + +from odoo import fields, models +from odoo.exceptions import UserError +from odoo.tools.translate import _ + +from ..lib import apriori + + +class OpenupgradeComparisonConfig(models.Model): + _name = "openupgrade.comparison.config" + _description = "OpenUpgrade Comparison Configuration" + + name = fields.Char() + server = fields.Char(required=True) + port = fields.Integer(required=True, default=8069) + protocol = fields.Selection( + [("http://", "XML-RPC")], + # ('https://', 'XML-RPC Secure')], not supported by libopenerp + required=True, + default="http://", + ) + database = fields.Char(required=True) + username = fields.Char(required=True) + password = fields.Char(required=True) + last_log = fields.Text() + + def get_connection(self): + self.ensure_one() + import odoorpc + + remote = odoorpc.ODOO(self.server, port=self.port) + remote.login(self.database, self.username, self.password) + return remote + + def test_connection(self): + self.ensure_one() + try: + connection = self.get_connection() + user_model = connection.env["res.users"] + ids = user_model.search([("login", "=", "admin")]) + user_info = user_model.read([ids[0]], ["name"])[0] + except Exception as e: + raise UserError(_("Connection failed.\n\nDETAIL: %s") % e) + raise UserError(_("%s is connected.") % user_info["name"]) + + def analyze(self): + """ Run the analysis wizard """ + self.ensure_one() + wizard = self.env["openupgrade.analysis.wizard"].create( + {"server_config": self.id} + ) + return { + "name": wizard._description, + "view_mode": "form", + "res_model": wizard._name, + "type": "ir.actions.act_window", + "target": "new", + "res_id": wizard.id, + "nodestroy": True, + } + + def install_modules(self): + """ Install same modules as in source DB """ + self.ensure_one() + connection = self.get_connection() + remote_module_obj = connection.env["ir.module.module"] + remote_module_ids = remote_module_obj.search([("state", "=", "installed")]) + + modules = [] + for module_id in remote_module_ids: + mod = remote_module_obj.read([module_id], ["name"])[0] + mod_name = mod["name"] + mod_name = apriori.renamed_modules.get(mod_name, mod_name) + modules.append(mod_name) + _logger = logging.getLogger(__name__) + _logger.debug("remote modules %s", modules) + local_modules = self.env["ir.module.module"].search( + [("name", "in", modules), ("state", "=", "uninstalled")] + ) + _logger.debug("local modules %s", ",".join(local_modules.mapped("name"))) + if local_modules: + local_modules.write({"state": "to install"}) + return {} diff --git a/upgrade_analysis/models/generate_records_wizard.py b/upgrade_analysis/models/generate_records_wizard.py new file mode 100644 index 000000000..b09f6839e --- /dev/null +++ b/upgrade_analysis/models/generate_records_wizard.py @@ -0,0 +1,119 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from openupgradelib import openupgrade_tools + +from odoo import _, fields, models +from odoo.exceptions import UserError +from odoo.modules.registry import Registry + + +class GenerateWizard(models.TransientModel): + _name = "openupgrade.generate.records.wizard" + _description = "OpenUpgrade Generate Records Wizard" + _rec_name = "state" + + state = fields.Selection([("init", "init"), ("ready", "ready")], default="init") + + def quirk_standard_calendar_attendances(self): + """Introduced in Odoo 13. The reinstallation causes a one2many value + in [(0, 0, {})] format to be loaded on top of the first load, causing a + violation of database constraint.""" + for cal in ("resource_calendar_std_35h", "resource_calendar_std_38h"): + record = self.env.ref("resource.%s" % cal, False) + if record: + record.attendance_ids.unlink() + + def generate(self): + """Main wizard step. Make sure that all modules are up-to-date, + then reinitialize all installed modules. + Equivalent of running the server with '-d --init all' + + The goal of this is to fill the records table. + + TODO: update module list and versions, then update all modules?""" + # Truncate the records table + if openupgrade_tools.table_exists( + self.env.cr, "openupgrade_attribute" + ) and openupgrade_tools.table_exists(self.env.cr, "openupgrade_record"): + self.env.cr.execute("TRUNCATE openupgrade_attribute, openupgrade_record;") + + # Run any quirks + self.quirk_standard_calendar_attendances() + + # Need to get all modules in state 'installed' + modules = self.env["ir.module.module"].search( + [("state", "in", ["to install", "to upgrade"])] + ) + if modules: + self.env.cr.commit() # pylint: disable=invalid-commit + Registry.new(self.env.cr.dbname, update_module=True) + # Did we succeed above? + modules = self.env["ir.module.module"].search( + [("state", "in", ["to install", "to upgrade"])] + ) + if modules: + raise UserError( + _("Cannot seem to install or upgrade modules %s") + % (", ".join([module.name for module in modules])) + ) + # Now reinitialize all installed modules + self.env["ir.module.module"].search([("state", "=", "installed")]).write( + {"state": "to install"} + ) + self.env.cr.commit() # pylint: disable=invalid-commit + Registry.new(self.env.cr.dbname, update_module=True) + + # Set domain property + self.env.cr.execute( + """ UPDATE openupgrade_record our + SET domain = iaw.domain + FROM ir_model_data imd + JOIN ir_act_window iaw ON imd.res_id = iaw.id + WHERE our.type = 'xmlid' + AND imd.model = 'ir.actions.act_window' + AND our.model = imd.model + AND our.name = imd.module || '.' || imd.name + """ + ) + self.env.cache.invalidate( + [ + (self.env["openupgrade.record"]._fields["domain"], None), + ] + ) + + # Set noupdate property from ir_model_data + self.env.cr.execute( + """ UPDATE openupgrade_record our + SET noupdate = imd.noupdate + FROM ir_model_data imd + WHERE our.type = 'xmlid' + AND our.model = imd.model + AND our.name = imd.module || '.' || imd.name + """ + ) + self.env.cache.invalidate( + [ + (self.env["openupgrade.record"]._fields["noupdate"], None), + ] + ) + + # Log model records + self.env.cr.execute( + """INSERT INTO openupgrade_record + (module, name, model, type) + SELECT imd2.module, imd2.module || '.' || imd.name AS name, + im.model, 'model' AS type + FROM ( + SELECT min(id) as id, name, res_id + FROM ir_model_data + WHERE name LIKE 'model_%' AND model = 'ir.model' + GROUP BY name, res_id + ) imd + JOIN ir_model_data imd2 ON imd2.id = imd.id + JOIN ir_model im ON imd.res_id = im.id + ORDER BY imd.name, imd.id""", + ) + + return self.write({"state": "ready"}) diff --git a/upgrade_analysis/models/install_all_wizard.py b/upgrade_analysis/models/install_all_wizard.py new file mode 100644 index 000000000..c1085418e --- /dev/null +++ b/upgrade_analysis/models/install_all_wizard.py @@ -0,0 +1,51 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import api, fields, models +from odoo.modules.registry import Registry +from odoo.osv.expression import AND + +from ..blacklist import BLACKLIST_MODULES + + +class InstallAll(models.TransientModel): + _name = "openupgrade.install.all.wizard" + _description = "OpenUpgrade Install All Wizard" + + state = fields.Selection( + [("init", "init"), ("ready", "ready")], readonly=True, default="init" + ) + to_install = fields.Integer("Number of modules to install", readonly=True) + + @api.model + def default_get(self, fields): + """Update module list and retrieve the number + of installable modules""" + res = super(InstallAll, self).default_get(fields) + update, add = self.env["ir.module.module"].update_list() + modules = self.env["ir.module.module"].search( + [("state", "not in", ["uninstallable", "unknown"])] + ) + res["to_install"] = len(modules) + return res + + def install_all(self, extra_domain=None): + """Main wizard step. Set all installable modules to install + and actually install them. Exclude testing modules.""" + domain = [ + "&", + "&", + ("state", "not in", ["uninstallable", "unknown"]), + ("category_id.name", "!=", "Tests"), + ("name", "not in", BLACKLIST_MODULES), + ] + if extra_domain: + domain = AND([domain, extra_domain]) + modules = self.env["ir.module.module"].search(domain) + if modules: + modules.write({"state": "to install"}) + self.env.cr.commit() # pylint: disable=invalid-commit + Registry.new(self.env.cr.dbname, update_module=True) + self.write({"state": "ready"}) + return True diff --git a/upgrade_analysis/models/openupgrade_record.py b/upgrade_analysis/models/openupgrade_record.py new file mode 100644 index 000000000..80b5a8a3a --- /dev/null +++ b/upgrade_analysis/models/openupgrade_record.py @@ -0,0 +1,113 @@ +# Copyright 2011-2015 Therp BV +# Copyright 2016 Opener B.V. +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import api, fields, models + + +class Attribute(models.Model): + _name = "openupgrade.attribute" + _description = "OpenUpgrade Attribute" + + name = fields.Char(readonly=True) + value = fields.Char(readonly=True) + record_id = fields.Many2one( + "openupgrade.record", + ondelete="CASCADE", + readonly=True, + ) + + +class Record(models.Model): + _name = "openupgrade.record" + _description = "OpenUpgrade Record" + + name = fields.Char(readonly=True) + module = fields.Char(readonly=True) + model = fields.Char(readonly=True) + field = fields.Char(readonly=True) + mode = fields.Selection( + [("create", "Create"), ("modify", "Modify")], + help="Set to Create if a field is newly created " + "in this module. If this module modifies an attribute of an " + "existing field, set to Modify.", + readonly=True, + ) + type = fields.Selection( # Uh oh, reserved keyword + [("field", "Field"), ("xmlid", "XML ID"), ("model", "Model")], + readonly=True, + ) + attribute_ids = fields.One2many("openupgrade.attribute", "record_id", readonly=True) + noupdate = fields.Boolean(readonly=True) + domain = fields.Char(readonly=True) + prefix = fields.Char(compute="_compute_prefix_and_suffix") + suffix = fields.Char(compute="_compute_prefix_and_suffix") + model_original_module = fields.Char(compute="_compute_model_original_module") + model_type = fields.Char(compute="_compute_model_type") + + @api.depends("name") + def _compute_prefix_and_suffix(self): + for rec in self: + rec.prefix, rec.suffix = rec.name.split(".", 1) + + @api.depends("model", "type") + def _compute_model_original_module(self): + for rec in self: + if rec.type == "model": + rec.model_original_module = self.env[rec.model]._original_module + else: + rec.model_original_module = "" + + @api.depends("model", "type") + def _compute_model_type(self): + for rec in self: + if rec.type == "model": + model = self.env[rec.model] + if model._auto and model._transient: + rec.model_type = "transient" + elif model._auto: + rec.model_type = "" + elif not model._auto and model._abstract: + rec.model_type = "abstract" + else: + rec.model_type = "sql_view" + else: + rec.model_type = "" + + @api.model + def field_dump(self): + keys = [ + "attachment", + "module", + "mode", + "model", + "field", + "type", + "isfunction", + "isproperty", + "isrelated", + "relation", + "required", + "stored", + "selection_keys", + "req_default", + "hasdefault", + "table", + "inherits", + ] + + template = {x: False for x in keys} + data = [] + for record in self.search([("type", "=", "field")]): + repre = template.copy() + repre.update( + { + "module": record.module, + "model": record.model, + "field": record.field, + "mode": record.mode, + } + ) + repre.update({x.name: x.value for x in record.attribute_ids}) + data.append(repre) + return data diff --git a/upgrade_analysis/security/ir.model.access.csv b/upgrade_analysis/security/ir.model.access.csv new file mode 100644 index 000000000..2ab5e67c1 --- /dev/null +++ b/upgrade_analysis/security/ir.model.access.csv @@ -0,0 +1,4 @@ +"id","name","model_id:id","group_id:id","perm_read","perm_write","perm_create","perm_unlink" +"access_openupgrade_record","openupgrade.record all","model_openupgrade_record",,1,0,0,0 +"access_openupgrade_attribute","openupgrade.attribute all","model_openupgrade_attribute",,1,0,0,0 +"access_openupgrade_comparison_config","openupgrade.comparison.config","model_openupgrade_comparison_config",base.group_system,1,1,1,1 diff --git a/upgrade_analysis/views/analysis_wizard.xml b/upgrade_analysis/views/analysis_wizard.xml new file mode 100644 index 000000000..efaa5294f --- /dev/null +++ b/upgrade_analysis/views/analysis_wizard.xml @@ -0,0 +1,36 @@ + + + + + view.openupgrade.analysis_wizard.form + openupgrade.analysis.wizard + +
+ + + + + + +
+
+
+ + + + diff --git a/upgrade_analysis/views/comparison_config.xml b/upgrade_analysis/views/comparison_config.xml new file mode 100644 index 000000000..db0ae7770 --- /dev/null +++ b/upgrade_analysis/views/comparison_config.xml @@ -0,0 +1,77 @@ + + + + + view.openupgrade.comparison_config.tree + openupgrade.comparison.config + + + + + + + + + + + + + view.openupgrade.comparison_config.form + openupgrade.comparison.config + +
+ + + + + + + + + +
+ + + + + + + + + + +