[FIX] account_reconcile_model_oca: reconciliation models

Some behavior were incorrect with reconciliation
rules:

1) Whit invoice_matching rule, no match_text_location
  and no match partner set. Nothing was matching, even
  by filling the right fields.
2) We were taking into account only the digits of the
   payment reference of the invoice, by removing all
   non digits characters. It has been decided to also
   taking non digit strings from 'payment_ref' and 'ref'
   into account, as long as they don't exceed one word.

Steps for point 1:

- Reco model with invoice_atching rule, payment_tolerance 0%,
  no match_text_location and no match_partner
- An invoice for 100$ with name of the invoice as payment reference
  (eg 'INV/2023/00001')
- A statement line of 100$ with either 'payment_ref, 'ref' or 'narration'
  set as 'INV/2023/00001', and no partner
-> No match

Steps for point 2:

- Same reco model but with match_text_location_label set to True
- An invoice for 100$ with eg 'abcdef' as payment reference
- A  statement line of 100$ with payment_reference (label) set
  as 'abcdef' and no partner
-> No match

Related to cddea14954

account_reconcile_model_oca/models/account_bank_statement_line.py

@@ -118,23 +118,15 @@ class AccountBankStatementLine(models.Model):
         """
         self.ensure_one()
 
-        def _get_text_value(field_name):
-            if self._fields[field_name].type == "html":
-                return self[field_name] and html2plaintext(self[field_name])
-            else:
-                return self[field_name]
-
         st_line_text_values = []
-        if allowed_fields is None or "payment_ref" in allowed_fields:
+        if not allowed_fields or "payment_ref" in allowed_fields:
-            value = _get_text_value("payment_ref")
+            if self.payment_ref:
-            if value:
+                st_line_text_values.append(self.payment_ref)
-                st_line_text_values.append(value)
+        if not allowed_fields or "narration" in allowed_fields:
-        if allowed_fields is None or "narration" in allowed_fields:
+            value = html2plaintext(self.narration or "")
-            value = _get_text_value("narration")
-            if value:
-                st_line_text_values.append(value)
-        if allowed_fields is None or "ref" in allowed_fields:
-            value = _get_text_value("ref")
             if value:
                 st_line_text_values.append(value)
+        if not allowed_fields or "ref" in allowed_fields:
+            if self.ref:
+                st_line_text_values.append(self.ref)
         return st_line_text_values

account_reconcile_model_oca/models/account_reconcile_model.py

@@ -314,24 +314,42 @@ class AccountReconcileModel(models.Model):
 
         return aml_domain
 
+    def _get_st_line_text_values_for_matching(self, st_line):
+        """Collect the strings that could be used on the statement line to perform
+        some matching.
+
+        :param st_line: The current statement line.
+        :return: A list of strings.
+        """
+        self.ensure_one()
+        allowed_fields = []
+        if self.match_text_location_label:
+            allowed_fields.append("payment_ref")
+        if self.match_text_location_note:
+            allowed_fields.append("narration")
+        if self.match_text_location_reference:
+            allowed_fields.append("ref")
+        return st_line._get_st_line_strings_for_matching(allowed_fields=allowed_fields)
+
     def _get_invoice_matching_st_line_tokens(self, st_line):
         """Parse the textual information from the statement line passed as parameter
         in order to extract from it the meaningful information in order to perform the
         matching.
         :param st_line: A statement line.
-        :return: A list of tokens, each one being a string.
+        :return:    A tuple of list of tokens, each one being a string.
+                    The first element is a list of tokens you may match on
+                    numerical information.
+                    The second element is a list of tokens you may match exactly.
         """
-        st_line_text_values = st_line._get_st_line_strings_for_matching(
+        st_line_text_values = self._get_st_line_text_values_for_matching(st_line)
-            allowed_fields=(
-                "payment_ref" if self.match_text_location_label else None,
-                "narration" if self.match_text_location_note else None,
-                "ref" if self.match_text_location_reference else None,
-            )
-        )
         significant_token_size = 4
-        tokens = []
+        numerical_tokens = []
+        exact_tokens = []
         for text_value in st_line_text_values:
-            for token in (text_value or "").split():
+            tokens = (text_value or "").split()
+
+            # Numerical tokens
+            for token in tokens:
                 # The token is too short to be significant.
                 if len(token) < significant_token_size:
                     continue
@@ -342,8 +360,12 @@ class AccountReconcileModel(models.Model):
                 if len(formatted_token) < significant_token_size:
                     continue
 
-                tokens.append(formatted_token)
+                numerical_tokens.append(formatted_token)
-        return tokens
+
+            # Exact tokens.
+            if len(tokens) == 1:
+                exact_tokens.append(tokens[0])
+        return numerical_tokens, exact_tokens
 
     def _get_invoice_matching_amls_candidates(self, st_line, partner):
         """Returns the match candidates for the 'invoice_matching' rule, with respect to
@@ -366,17 +388,18 @@ class AccountReconcileModel(models.Model):
         where_string, where_params = query.where_clause
         from_clause = from_string
         where_clause = where_string
-        query_params = from_params + where_params
 
-        tokens = self._get_invoice_matching_st_line_tokens(st_line)
+        sub_queries = []
-        if tokens:
+        all_params = []
-            search_fields = [
+        numerical_tokens, exact_tokens = self._get_invoice_matching_st_line_tokens(
+            st_line
+        )
+        if numerical_tokens:
+            for table_alias, field in (
                 ("account_move_line", "name"),
                 ("account_move_line__move_id", "name"),
                 ("account_move_line__move_id", "ref"),
-            ]
+            ):
-            sub_queries = []
-            for table_alias, field in search_fields:
                 sub_queries.append(
                     rf"""
                     SELECT
@@ -400,7 +423,30 @@ class AccountReconcileModel(models.Model):
                     WHERE {where_clause} AND {table_alias}.{field} IS NOT NULL
                 """
                 )
+                all_params += where_params
 
+        if exact_tokens:
+            for table_alias, field in (
+                ("account_move_line", "name"),
+                ("account_move_line__move_id", "name"),
+                ("account_move_line__move_id", "ref"),
+            ):
+                sub_queries.append(
+                    rf"""
+                    SELECT
+                        account_move_line.id,
+                        account_move_line.date,
+                        account_move_line.date_maturity,
+                        {table_alias}.{field} AS token
+                    FROM {from_clause}
+                    JOIN account_move account_move_line__move_id
+                        ON account_move_line__move_id.id = account_move_line.move_id
+                    WHERE {where_clause} AND {table_alias}.{field} IS NOT NULL
+                """
+                )
+                all_params += where_params
+
+        if sub_queries:
             self._cr.execute(
                 """
                     SELECT
@@ -416,7 +462,7 @@ class AccountReconcileModel(models.Model):
                 + order_by
                 + """
                 """,
-                (query_params * 3) + [tuple(tokens)],
+                all_params + [tuple(numerical_tokens + exact_tokens)],
             )
             candidate_ids = [r[0] for r in self._cr.fetchall()]
             if candidate_ids:

account_reconcile_model_oca/tests/test_reconciliation_match.py

@@ -1,3 +1,5 @@
+from contextlib import contextmanager
+
 from freezegun import freeze_time
 
 from odoo import Command
@@ -296,122 +298,6 @@ class TestReconciliationMatchingRules(AccountTestInvoicingCommon):
             },
         )
 
-    @freeze_time("2020-01-01")
-    def test_matching_fields_match_text_location(self):
-        st_line = self._create_st_line(
-            payment_ref="1111", ref="2222 3333", narration="4444 5555 6666"
-        )
-
-        inv1 = self._create_invoice_line(
-            1000, self.partner_a, "out_invoice", pay_reference="bernard 1111 gagnant"
-        )
-        inv2 = self._create_invoice_line(
-            1000, self.partner_a, "out_invoice", pay_reference="2222 turlututu 3333"
-        )
-        inv3 = self._create_invoice_line(
-            1000,
-            self.partner_a,
-            "out_invoice",
-            pay_reference="4444 tsoin 5555 tsoin 6666",
-        )
-
-        rule = self._create_reconcile_model(
-            allow_payment_tolerance=False,
-            match_text_location_label=True,
-            match_text_location_reference=False,
-            match_text_location_note=False,
-        )
-        self.assertDictEqual(
-            rule._apply_rules(st_line, st_line._retrieve_partner()),
-            {"amls": inv1, "model": rule},
-        )
-
-        rule.match_text_location_reference = True
-        self.assertDictEqual(
-            rule._apply_rules(st_line, st_line._retrieve_partner()),
-            {"amls": inv2, "model": rule},
-        )
-
-        rule.match_text_location_note = True
-        self.assertDictEqual(
-            rule._apply_rules(st_line, st_line._retrieve_partner()),
-            {"amls": inv3, "model": rule},
-        )
-
-    def test_matching_fields_match_text_location_no_partner(self):
-        self.bank_line_2.unlink()  # One line is enough for this test
-        self.bank_line_1.partner_id = None
-
-        self.partner_1.name = "Bernard Gagnant"
-
-        self.rule_1.write(
-            {
-                "match_partner": False,
-                "match_partner_ids": [(5, 0, 0)],
-                "line_ids": [(5, 0, 0)],
-            }
-        )
-
-        st_line_initial_vals = {
-            "ref": None,
-            "payment_ref": "nothing",
-            "narration": None,
-        }
-        recmod_initial_vals = {
-            "match_text_location_label": False,
-            "match_text_location_note": False,
-            "match_text_location_reference": False,
-        }
-
-        rec_mod_options_to_fields = {
-            "match_text_location_label": "payment_ref",
-            "match_text_location_note": "narration",
-            "match_text_location_reference": "ref",
-        }
-
-        for rec_mod_field, st_line_field in rec_mod_options_to_fields.items():
-            self.rule_1.write({**recmod_initial_vals, rec_mod_field: True})
-            # Fully reinitialize the statement line
-            self.bank_line_1.write(st_line_initial_vals)
-
-            # Nothing should match
-            self._check_statement_matching(
-                self.rule_1,
-                {
-                    self.bank_line_1: {},
-                },
-            )
-
-            # Test matching with the invoice ref
-            self.bank_line_1.write(
-                {st_line_field: self.invoice_line_1.move_id.payment_reference}
-            )
-
-            self._check_statement_matching(
-                self.rule_1,
-                {
-                    self.bank_line_1: {
-                        "amls": self.invoice_line_1,
-                        "model": self.rule_1,
-                    },
-                },
-            )
-
-            # Test matching with the partner name (resetting the statement line first)
-            self.bank_line_1.write(
-                {**st_line_initial_vals, st_line_field: self.partner_1.name}
-            )
-
-            self._check_statement_matching(
-                self.rule_1,
-                {
-                    self.bank_line_1: {
-                        "amls": self.invoice_line_1,
-                        "model": self.rule_1,
-                    },
-                },
-            )
-
     def test_matching_fields_match_journal_ids(self):
         self.rule_1.match_journal_ids |= self.cash_line_1.journal_id
         self._check_statement_matching(
@@ -1487,3 +1373,146 @@ class TestReconciliationMatchingRules(AccountTestInvoicingCommon):
                 },
             },
         )
+
+    @freeze_time("2019-01-01")
+    def test_invoice_matching_using_match_text_location(self):
+        @contextmanager
+        def rollback():
+            savepoint = self.cr.savepoint()
+            yield
+            savepoint.rollback()
+
+        rule = self._create_reconcile_model(
+            match_partner=False,
+            allow_payment_tolerance=False,
+            match_text_location_label=False,
+            match_text_location_reference=False,
+            match_text_location_note=False,
+        )
+        st_line = self._create_st_line(amount=1000, partner_id=False)
+        invoice = self.env["account.move"].create(
+            {
+                "move_type": "out_invoice",
+                "partner_id": self.partner_a.id,
+                "invoice_date": "2019-01-01",
+                "invoice_line_ids": [
+                    Command.create(
+                        {
+                            "product_id": self.product_a.id,
+                            "price_unit": 100,
+                        }
+                    )
+                ],
+            }
+        )
+        invoice.action_post()
+        term_line = invoice.line_ids.filtered(
+            lambda x: x.display_type == "payment_term"
+        )
+
+        # No match at all.
+        self.assertDictEqual(
+            rule._apply_rules(st_line, None),
+            {},
+        )
+
+        with rollback():
+            term_line.name = "1234"
+            st_line.payment_ref = "1234"
+
+            # Matching if no checkbox checked.
+            self.assertDictEqual(
+                rule._apply_rules(st_line, None),
+                {"amls": term_line, "model": rule},
+            )
+
+            # No matching if other checkbox is checked.
+            rule.match_text_location_note = True
+            self.assertDictEqual(
+                rule._apply_rules(st_line, None),
+                {},
+            )
+
+        for rule_field, st_line_field in (
+            ("match_text_location_label", "payment_ref"),
+            ("match_text_location_reference", "ref"),
+            ("match_text_location_note", "narration"),
+        ):
+            with self.subTest(rule_field=rule_field, st_line_field=st_line_field):
+                with rollback():
+                    rule[rule_field] = True
+                    st_line[st_line_field] = "123456"
+                    term_line.name = "123456"
+
+                    # Matching if the corresponding flag is enabled.
+                    self.assertDictEqual(
+                        rule._apply_rules(st_line, None),
+                        {"amls": term_line, "model": rule},
+                    )
+
+                    # It works also if the statement line contains the word.
+                    st_line[st_line_field] = "payment for 123456 urgent!"
+                    self.assertDictEqual(
+                        rule._apply_rules(st_line, None),
+                        {"amls": term_line, "model": rule},
+                    )
+
+                    # Not if the invoice has nothing in common even if numerical.
+                    term_line.name = "78910"
+                    self.assertDictEqual(
+                        rule._apply_rules(st_line, None),
+                        {},
+                    )
+
+                    # Exact matching on a single word.
+                    st_line[st_line_field] = "TURLUTUTU21"
+                    term_line.name = "TURLUTUTU21"
+                    self.assertDictEqual(
+                        rule._apply_rules(st_line, None),
+                        {"amls": term_line, "model": rule},
+                    )
+
+                    # No matching if not enough numerical values.
+                    st_line[st_line_field] = "12"
+                    term_line.name = "selling 3 apples, 2 tomatoes and 12kg of potatoes"
+                    self.assertDictEqual(
+                        rule._apply_rules(st_line, None),
+                        {},
+                    )
+
+        invoice2 = self.env["account.move"].create(
+            {
+                "move_type": "out_invoice",
+                "partner_id": self.partner_a.id,
+                "invoice_date": "2019-01-01",
+                "invoice_line_ids": [
+                    Command.create(
+                        {
+                            "product_id": self.product_a.id,
+                            "price_unit": 100,
+                        }
+                    )
+                ],
+            }
+        )
+        invoice2.action_post()
+        term_lines = (invoice + invoice2).line_ids.filtered(
+            lambda x: x.display_type == "payment_term"
+        )
+
+        # Matching multiple invoices.
+        rule.match_text_location_label = True
+        st_line.payment_ref = "paying invoices 1234 & 5678"
+        term_lines[0].name = "INV/1234"
+        term_lines[1].name = "INV/5678"
+        self.assertDictEqual(
+            rule._apply_rules(st_line, None),
+            {"amls": term_lines, "model": rule},
+        )
+
+        # Matching multiple invoices sharing the same reference.
+        term_lines[1].name = "INV/1234"
+        self.assertDictEqual(
+            rule._apply_rules(st_line, None),
+            {"amls": term_lines, "model": rule},
+        )