Techsystech
/
server-tools
mirror of https://github.com/OCA/server-tools.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

base_search_fuzzy: black, isort

pull/2530/head
Ernesto Tejeda 2020-03-25 17:09:14 -04:00 committed by Daniel Reis
parent d36af2cc80
commit 37fb022bee
5 changed files with 153 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
base_search_fuzzy/__manifest__.py
View File

@ -2,20 +2,17 @@
# Copyright 2016 Serpent Consulting Services Pvt. Ltd.
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
{
'name': "Fuzzy Search",
'summary': "Fuzzy search with the PostgreSQL trigram extension",
'category': 'Uncategorized',
'version': '12.0.1.0.0',
'website': 'https://github.com/OCA/server-tools',
'author': 'bloopark systems GmbH & Co. KG, '
'Eficent, '
'Serpent CS, '
'Odoo Community Association (OCA)',
'license': 'AGPL-3',
'depends': ['base'],
'data': [
'views/trgm_index.xml',
'security/ir.model.access.csv',
],
'installable': True,
"name": "Fuzzy Search",
"summary": "Fuzzy search with the PostgreSQL trigram extension",
"category": "Uncategorized",
"version": "12.0.1.0.0",
"website": "https://github.com/OCA/server-tools",
"author": "bloopark systems GmbH & Co. KG, "
"Eficent, "
"Serpent CS, "
"Odoo Community Association (OCA)",
"license": "AGPL-3",
"depends": ["base"],
"data": ["views/trgm_index.xml", "security/ir.model.access.csv",],
"installable": True,
}

43
base_search_fuzzy/models/ir_model.py
View File

@ -7,7 +7,6 @@ import logging
from odoo import _, api, models
from odoo.osv import expression
_logger = logging.getLogger(__name__)
@ -18,26 +17,23 @@ def patch_leaf_trgm(method):
left, operator, right = leaf
table_alias = '"%s"' % (eleaf.generate_alias())
if operator == '%':
if operator == "%":
sql_operator = '%%'
sql_operator = "%%"
params = []
if left in model._fields:
column = '%s.%s' % (table_alias, expression._quote(left))
query = '(%s %s %s)' % (
column = "{}.{}".format(table_alias, expression._quote(left))
query = "({} {} {})".format(
column,
sql_operator,
model._fields[left].column_format,
)
elif left in models.MAGIC_COLUMNS:
query = "(%s.\"%s\" %s %%s)" % (
table_alias, left, sql_operator)
query = '({}."{}" {} %s)'.format(table_alias, left, sql_operator)
params = right
else: # Must not happen
raise ValueError(_(
"Invalid field %r in domain term %r" % (left, leaf)
))
raise ValueError(_("Invalid field {!r} in domain term {!r}".format(left, leaf)))
if left in model._fields:
params = str(right)
@ -45,8 +41,8 @@ def patch_leaf_trgm(method):
if isinstance(params, str):
params = [params]
return query, params
elif operator == 'inselect':
right = (right[0].replace(' % ', ' %% '), right[1])
elif operator == "inselect":
right = (right[0].replace(" % ", " %% "), right[1])
eleaf.leaf = (left, operator, right)
return method(self, eleaf)
@ -56,11 +52,10 @@ def patch_leaf_trgm(method):
def patch_generate_order_by(method):
@api.model
def decorate_generate_order_by(self, order_spec, query):
if order_spec and order_spec.startswith('similarity('):
return ' ORDER BY ' + order_spec
if order_spec and order_spec.startswith("similarity("):
return " ORDER BY " + order_spec
return method(self, order_spec, query)
decorate_generate_order_by.__decorated__ = True
@ -70,22 +65,22 @@ def patch_generate_order_by(method):
class IrModel(models.Model):
_inherit = 'ir.model'
_inherit = "ir.model"
@api.model_cr
def _register_hook(self):
# We have to prevent wrapping the function twice to avoid recursion
# errors
if not hasattr(expression.expression._expression__leaf_to_sql,
'__decorated__'):
if not hasattr(expression.expression._expression__leaf_to_sql, "__decorated__"):
expression.expression._expression__leaf_to_sql = patch_leaf_trgm(
expression.expression._expression__leaf_to_sql)
expression.expression._expression__leaf_to_sql
)
if '%' not in expression.TERM_OPERATORS:
expression.TERM_OPERATORS += ('%',)
if "%" not in expression.TERM_OPERATORS:
expression.TERM_OPERATORS += ("%",)
if not hasattr(models.BaseModel._generate_order_by,
'__decorated__'):
if not hasattr(models.BaseModel._generate_order_by, "__decorated__"):
models.BaseModel._generate_order_by = patch_generate_order_by(
models.BaseModel._generate_order_by)
models.BaseModel._generate_order_by
)
return super(IrModel, self)._register_hook()

131
base_search_fuzzy/models/trgm_index.py
View File

@ -4,10 +4,10 @@
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
import logging
from odoo import _, api, exceptions, fields, models
from psycopg2.extensions import AsIs
from odoo import _, api, exceptions, fields, models
_logger = logging.getLogger(__name__)
@ -15,76 +15,81 @@ class TrgmIndex(models.Model):
"""Model for Trigram Index."""
_name = 'trgm.index'
_rec_name = 'field_id'
_description = 'Trigram Index'
_name = "trgm.index"
_rec_name = "field_id"
_description = "Trigram Index"
field_id = fields.Many2one(
comodel_name='ir.model.fields',
string='Field',
comodel_name="ir.model.fields",
string="Field",
required=True,
help='You can either select a field of type "text" or "char".'
help='You can either select a field of type "text" or "char".',
)
index_name = fields.Char(
string='Index Name',
string="Index Name",
readonly=True,
help='The index name is automatically generated like '
'fieldname_indextype_idx. If the index already exists and the '
'index is located in the same table then this index is reused. '
'If the index is located in another table then a number is added '
'at the end of the index name.'
help="The index name is automatically generated like "
"fieldname_indextype_idx. If the index already exists and the "
"index is located in the same table then this index is reused. "
"If the index is located in another table then a number is added "
"at the end of the index name.",
)
index_type = fields.Selection(
selection=[('gin', 'GIN'), ('gist', 'GiST')],
string='Index Type',
default='gin',
selection=[("gin", "GIN"), ("gist", "GiST")],
string="Index Type",
default="gin",
required=True,
help='Cite from PostgreSQL documentation: "As a rule of thumb, a '
'GIN index is faster to search than a GiST index, but slower to '
'build or update; so GIN is better suited for static data and '
'GiST for often-updated data."'
"GIN index is faster to search than a GiST index, but slower to "
"build or update; so GIN is better suited for static data and "
'GiST for often-updated data."',
)
@api.model_cr
def _trgm_extension_exists(self):
self.env.cr.execute("""
self.env.cr.execute(
"""
SELECT name, installed_version
FROM pg_available_extensions
WHERE name = 'pg_trgm'
LIMIT 1;
""")
"""
)
extension = self.env.cr.fetchone()
if extension is None:
return 'missing'
return "missing"
if extension[1] is None:
return 'uninstalled'
return "uninstalled"
return 'installed'
return "installed"
@api.model_cr
def _is_postgres_superuser(self):
self.env.cr.execute("SHOW is_superuser;")
superuser = self.env.cr.fetchone()
return superuser is not None and superuser[0] == 'on' or False
return superuser is not None and superuser[0] == "on" or False
@api.model_cr
def _install_trgm_extension(self):
extension = self._trgm_extension_exists()
if extension == 'missing':
_logger.warning('To use pg_trgm you have to install the '
'postgres-contrib module.')
elif extension == 'uninstalled':
if extension == "missing":
_logger.warning(
"To use pg_trgm you have to install the " "postgres-contrib module."
)
elif extension == "uninstalled":
if self._is_postgres_superuser():
self.env.cr.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
return True
else:
_logger.warning('To use pg_trgm you have to create the '
'extension pg_trgm in your database or you '
'have to be the superuser.')
_logger.warning(
"To use pg_trgm you have to create the "
"extension pg_trgm in your database or you "
"have to be the superuser."
)
else:
return True
return False
@ -93,8 +98,10 @@ class TrgmIndex(models.Model):
def _auto_init(self):
res = super(TrgmIndex, self)._auto_init()
if self._install_trgm_extension():
_logger.info('The pg_trgm is loaded in the database and the '
'fuzzy search can be used.')
_logger.info(
"The pg_trgm is loaded in the database and the "
"fuzzy search can be used."
)
return res
@api.model_cr
@ -103,18 +110,20 @@ class TrgmIndex(models.Model):
new_index_name = index_name + str(inc)
else:
new_index_name = index_name
self.env.cr.execute("""
self.env.cr.execute(
"""
SELECT tablename, indexname
FROM pg_indexes
WHERE indexname = %(index)s;
""", {'index': new_index_name})
""",
{"index": new_index_name},
)
indexes = self.env.cr.fetchone()
if indexes is not None and indexes[0] == table_name:
return True, index_name
elif indexes is not None:
return self.get_not_used_index(index_name, table_name,
inc + 1)
return self.get_not_used_index(index_name, table_name, inc + 1)
return False, new_index_name
@ -123,39 +132,42 @@ class TrgmIndex(models.Model):
self.ensure_one()
if not self._install_trgm_extension():
raise exceptions.UserError(_(
'The pg_trgm extension does not exists or cannot be '
'installed.'))
raise exceptions.UserError(
_("The pg_trgm extension does not exists or cannot be " "installed.")
)
table_name = self.env[self.field_id.model_id.model]._table
column_name = self.field_id.name
index_type = self.index_type
index_name = '%s_%s_idx' % (column_name, index_type)
index_exists, index_name = self.get_not_used_index(
index_name, table_name)
index_name = "{}_{}_idx".format(column_name, index_type)
index_exists, index_name = self.get_not_used_index(index_name, table_name)
if not index_exists:
self.env.cr.execute("""
self.env.cr.execute(
"""
CREATE INDEX %(index)s
ON %(table)s
USING %(indextype)s (%(column)s %(indextype)s_trgm_ops);
""", {
'table': AsIs(table_name),
'index': AsIs(index_name),
'column': AsIs(column_name),
'indextype': AsIs(index_type)
})
""",
{
"table": AsIs(table_name),
"index": AsIs(index_name),
"column": AsIs(column_name),
"indextype": AsIs(index_type),
},
)
return index_name
@api.model
def index_exists(self, model_name, field_name):
field = self.env['ir.model.fields'].search([
('model', '=', model_name), ('name', '=', field_name)], limit=1)
field = self.env["ir.model.fields"].search(
[("model", "=", model_name), ("name", "=", field_name)], limit=1
)
if not field:
return False
trgm_index = self.search([('field_id', '=', field.id)], limit=1)
trgm_index = self.search([("field_id", "=", field.id)], limit=1)
return bool(trgm_index)
@api.model
@ -167,9 +179,10 @@ class TrgmIndex(models.Model):
@api.multi
def unlink(self):
for rec in self:
self.env.cr.execute("""
self.env.cr.execute(
"""
DROP INDEX IF EXISTS %(index)s;
""", {
'index': AsIs(rec.index_name),
})
""",
{"index": AsIs(rec.index_name),},
)
return super(TrgmIndex, self).unlink()

73
base_search_fuzzy/tests/test_query_generation.py
View File

@ -8,22 +8,20 @@ from odoo.tests.common import TransactionCase, at_install, post_install
@at_install(False)
@post_install(True)
class QueryGenerationCase(TransactionCase):
def setUp(self):
super(QueryGenerationCase, self).setUp()
self.ResPartner = self.env['res.partner']
self.TrgmIndex = self.env['trgm.index']
self.ResPartnerCategory = self.env['res.partner.category']
self.ResPartner = self.env["res.partner"]
self.TrgmIndex = self.env["trgm.index"]
self.ResPartnerCategory = self.env["res.partner.category"]
def test_fuzzy_where_generation(self):
"""Check the generation of the where clause."""
# the added fuzzy search operator should be available in the allowed
# operators
self.assertIn('%', expression.TERM_OPERATORS)
self.assertIn("%", expression.TERM_OPERATORS)
# create new query with fuzzy search operator
query = self.ResPartner._where_calc(
[('name', '%', 'test')], active_test=False)
query = self.ResPartner._where_calc([("name", "%", "test")], active_test=False)
from_clause, where_clause, where_clause_params = query.get_sql()
# the % parameter has to be escaped (%%) for the string replation
@ -32,70 +30,65 @@ class QueryGenerationCase(TransactionCase):
# test the right sql query statement creation
# now there should be only one '%'
complete_where = self.env.cr.mogrify(
"SELECT FROM %s WHERE %s" % (from_clause, where_clause),
where_clause_params)
"SELECT FROM {} WHERE {}".format(from_clause, where_clause), where_clause_params
)
self.assertEqual(
complete_where,
b'SELECT FROM "res_partner" WHERE '
b'("res_partner"."name" % \'test\')')
b'SELECT FROM "res_partner" WHERE ' b'("res_partner"."name" % \'test\')',
)
def test_fuzzy_where_generation_translatable(self):
"""Check the generation of the where clause for translatable fields."""
ctx = {'lang': 'de_DE'}
ctx = {"lang": "de_DE"}
# create new query with fuzzy search operator
query = self.ResPartnerCategory.with_context(ctx)\
._where_calc([('name', '%', 'Goschaeftlic')], active_test=False)
query = self.ResPartnerCategory.with_context(ctx)._where_calc(
[("name", "%", "Goschaeftlic")], active_test=False
)
from_clause, where_clause, where_clause_params = query.get_sql()
# the % parameter has to be escaped (%%) for the string replation
self.assertIn("""SELECT id FROM temp_irt_current WHERE name %% %s""",
where_clause)
self.assertIn(
"""SELECT id FROM temp_irt_current WHERE name %% %s""", where_clause
)
complete_where = self.env.cr.mogrify(
"SELECT FROM %s WHERE %s" % (from_clause, where_clause),
where_clause_params)
"SELECT FROM {} WHERE {}".format(from_clause, where_clause), where_clause_params
)
self.assertIn(
b"""SELECT id FROM temp_irt_current WHERE name % 'Goschaeftlic'""",
complete_where)
complete_where,
)
def test_fuzzy_order_generation(self):
"""Check the generation of the where clause."""
order = "similarity(%s.name, 'test') DESC" % self.ResPartner._table
query = self.ResPartner._where_calc(
[('name', '%', 'test')], active_test=False)
query = self.ResPartner._where_calc([("name", "%", "test")], active_test=False)
order_by = self.ResPartner._generate_order_by(order, query)
self.assertEqual(' ORDER BY %s' % order, order_by)
self.assertEqual(" ORDER BY %s" % order, order_by)
def test_fuzzy_search(self):
"""Test the fuzzy search itself."""
if self.TrgmIndex._trgm_extension_exists() != 'installed':
if self.TrgmIndex._trgm_extension_exists() != "installed":
return
if not self.TrgmIndex.index_exists('res.partner', 'name'):
field_partner_name = self.env.ref('base.field_res_partner__name')
self.TrgmIndex.create({
'field_id': field_partner_name.id,
'index_type': 'gin',
})
if not self.TrgmIndex.index_exists("res.partner", "name"):
field_partner_name = self.env.ref("base.field_res_partner__name")
self.TrgmIndex.create(
{"field_id": field_partner_name.id, "index_type": "gin",}
)
partner1 = self.ResPartner.create({
'name': 'John Smith'
})
partner2 = self.ResPartner.create(
{'name': 'John Smizz'}
)
partner3 = self.ResPartner.create({
'name': 'Linus Torvalds'
})
partner1 = self.ResPartner.create({"name": "John Smith"})
partner2 = self.ResPartner.create({"name": "John Smizz"})
partner3 = self.ResPartner.create({"name": "Linus Torvalds"})
res = self.ResPartner.search([('name', '%', 'Jon Smith')])
res = self.ResPartner.search([("name", "%", "Jon Smith")])
self.assertIn(partner1.id, res.ids)
self.assertIn(partner2.id, res.ids)
self.assertNotIn(partner3.id, res.ids)
res = self.ResPartner.search([('name', '%', 'Smith John')])
res = self.ResPartner.search([("name", "%", "Smith John")])
self.assertIn(partner1.id, res.ids)
self.assertIn(partner2.id, res.ids)
self.assertNotIn(partner3.id, res.ids)

32
base_search_fuzzy/views/trgm_index.xml
View File

@ -1,6 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8" ?>
<odoo>
<record model="ir.ui.view" id="trgm_index_view_form">
<field name="name">trgm.index.view.form</field>
<field name="model">trgm.index</field>
@ -8,27 +7,28 @@
<form string="Trigram Index">
<sheet>
<group col="4">
<field name="field_id" domain="[('ttype', 'in', ['char', 'text'])]"/>
<field name="index_name"/>
<field name="index_type"/>
<field
name="field_id"
domain="[('ttype', 'in', ['char', 'text'])]"
/>
<field name="index_name" />
<field name="index_type" />
</group>
</sheet>
</form>
</field>
</record>
<record model="ir.ui.view" id="trgm_index_view_tree">
<field name="name">trgm.index.view.tree</field>
<field name="model">trgm.index</field>
<field name="arch" type="xml">
<tree string="Trigram Index">
<field name="field_id"/>
<field name="index_name"/>
<field name="index_type"/>
<field name="field_id" />
<field name="index_name" />
<field name="index_type" />
</tree>
</field>
</record>
<record model="ir.actions.act_window" id="trgm_index_action">
<field name="name">Trigram Index</field>
<field name="res_model">trgm.index</field>
@ -36,10 +36,10 @@
<field name="view_mode">tree,form</field>
<field name="type">ir.actions.act_window</field>
</record>
<menuitem id="trgm_index_menu"
parent="base.next_id_9"
action="trgm_index_action"
groups="base.group_no_one"/>
<menuitem
id="trgm_index_menu"
parent="base.next_id_9"
action="trgm_index_action"
groups="base.group_no_one"
/>
</odoo>