313 lines
12 KiB
Python
313 lines
12 KiB
Python
# Copyright 2012-2016 Camptocamp SA
|
|
# Copyright 2010 Sébastien Beau
|
|
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
|
|
|
|
import logging
|
|
from itertools import product
|
|
|
|
from odoo import api, models, registry
|
|
from odoo.tools.translate import _
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
|
|
class MassReconcileAdvanced(models.AbstractModel):
|
|
_name = "mass.reconcile.advanced"
|
|
_inherit = "mass.reconcile.base"
|
|
_description = "Mass Reconcile Advanced"
|
|
|
|
def _query_debit(self):
|
|
"""Select all move (debit>0) as candidate."""
|
|
select = self._select_query()
|
|
sql_from = self._from_query()
|
|
where, params = self._where_query()
|
|
where += " AND account_move_line.debit > 0 "
|
|
where2, params2 = self._get_filter()
|
|
query = " ".join((select, sql_from, where, where2))
|
|
self.env.cr.execute(query, params + params2)
|
|
return self.env.cr.dictfetchall()
|
|
|
|
def _query_credit(self):
|
|
"""Select all move (credit>0) as candidate."""
|
|
select = self._select_query()
|
|
sql_from = self._from_query()
|
|
where, params = self._where_query()
|
|
where += " AND account_move_line.credit > 0 "
|
|
where2, params2 = self._get_filter()
|
|
query = " ".join((select, sql_from, where, where2))
|
|
self.env.cr.execute(query, params + params2)
|
|
return self.env.cr.dictfetchall()
|
|
|
|
@staticmethod
|
|
def _matchers(move_line):
|
|
"""
|
|
Return the values used as matchers to find the opposite lines
|
|
|
|
All the matcher keys in the dict must have their equivalent in
|
|
the `_opposite_matchers`.
|
|
|
|
The values of each matcher key will be searched in the
|
|
one returned by the `_opposite_matchers`
|
|
|
|
Must be inherited to implement the matchers for one method
|
|
|
|
As instance, it can return:
|
|
return ('ref', move_line['rec'])
|
|
|
|
or
|
|
return (('partner_id', move_line['partner_id']),
|
|
('ref', "prefix_%s" % move_line['rec']))
|
|
|
|
All the matchers have to be found in the opposite lines
|
|
to consider them as "opposite"
|
|
|
|
The matchers will be evaluated in the same order as declared
|
|
vs the the opposite matchers, so you can gain performance by
|
|
declaring first the partners with the less computation.
|
|
|
|
All matchers should match with their opposite to be considered
|
|
as "matching".
|
|
So with the previous example, partner_id and ref have to be
|
|
equals on the opposite line matchers.
|
|
|
|
:return: tuple of tuples (key, value) where the keys are
|
|
the matchers keys
|
|
(They must be the same that `_opposite_matchers` returns,
|
|
and their values to match in the opposite lines.
|
|
A matching key can have multiples values.)
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
@staticmethod
|
|
def _opposite_matchers(move_line):
|
|
"""
|
|
Return the values of the opposite line used as matchers
|
|
so the line is matched
|
|
|
|
Must be inherited to implement the matchers for one method
|
|
It can be inherited to apply some formatting of fields
|
|
(strip(), lower() and so on)
|
|
|
|
This method is the counterpart of the `_matchers()` method.
|
|
|
|
Each matcher has to yield its value respecting the order
|
|
of the `_matchers()`.
|
|
|
|
When a matcher does not correspond, the next matchers won't
|
|
be evaluated so the ones which need the less computation
|
|
have to be executed first.
|
|
|
|
If the `_matchers()` returns:
|
|
(('partner_id', move_line['partner_id']),
|
|
('ref', move_line['ref']))
|
|
|
|
Here, you should yield :
|
|
yield ('partner_id', move_line['partner_id'])
|
|
yield ('ref', move_line['ref'])
|
|
|
|
Note that a matcher can contain multiple values, as instance,
|
|
if for a move line, you want to search from its `ref` in the
|
|
`ref` or `name` fields of the opposite move lines, you have to
|
|
yield ('partner_id', move_line['partner_id'])
|
|
yield ('ref', (move_line['ref'], move_line['name'])
|
|
|
|
An OR is used between the values for the same key.
|
|
An AND is used between the different keys.
|
|
|
|
:param dict move_line: values of the move_line
|
|
:yield: matchers as tuple ('matcher key', value(s))
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
@staticmethod
|
|
def _compare_values(key, value, opposite_value):
|
|
"""Can be inherited to modify the equality condition
|
|
specifically according to the matcher key (maybe using
|
|
a like operator instead of equality on 'ref' as instance)
|
|
"""
|
|
# consider that empty vals are not valid matchers
|
|
# it can still be inherited for some special cases
|
|
# where it would be allowed
|
|
if not (value and opposite_value):
|
|
return False
|
|
|
|
if value == opposite_value:
|
|
return True
|
|
return False
|
|
|
|
@classmethod
|
|
def _compare_matcher_values(cls, key, values, opposite_values):
|
|
"""Compare every `values` from a matcher vs an opposite matcher
|
|
and return True if it matches
|
|
"""
|
|
for value, ovalue in product(values, opposite_values):
|
|
# we do not need to compare all values, if one matches
|
|
# we are done
|
|
if cls._compare_values(key, value, ovalue):
|
|
return True
|
|
return False
|
|
|
|
@staticmethod
|
|
def _compare_matchers(matcher, opposite_matcher):
|
|
"""
|
|
Prepare and check the matchers to compare
|
|
"""
|
|
mkey, mvalue = matcher
|
|
omkey, omvalue = opposite_matcher
|
|
assert mkey == omkey, _(
|
|
"A matcher %(mkey)s is compared with a matcher %(omkey)s, the _matchers "
|
|
"and _opposite_matchers are probably wrong"
|
|
) % {"mkey": mkey, "omkey": omkey}
|
|
if not isinstance(mvalue, list | tuple):
|
|
mvalue = (mvalue,)
|
|
if not isinstance(omvalue, list | tuple):
|
|
omvalue = (omvalue,)
|
|
return MassReconcileAdvanced._compare_matcher_values(mkey, mvalue, omvalue)
|
|
|
|
def _compare_opposite(self, move_line, opposite_move_line, matchers):
|
|
"""Iterate over the matchers of the move lines vs opposite move lines
|
|
and if they all match, return True.
|
|
|
|
If all the matchers match for a move line and an opposite move line,
|
|
they are candidate for a reconciliation.
|
|
"""
|
|
opp_matchers = self._opposite_matchers(opposite_move_line)
|
|
for matcher in matchers:
|
|
try:
|
|
opp_matcher = next(opp_matchers)
|
|
except StopIteration as e:
|
|
# if you fall here, you probably missed to put a `yield`
|
|
# in `_opposite_matchers()`
|
|
raise ValueError("Missing _opposite_matcher: %s" % matcher[0]) from e
|
|
|
|
if not self._compare_matchers(matcher, opp_matcher):
|
|
# if any of the matcher fails, the opposite line
|
|
# is not a valid counterpart
|
|
# directly returns so the next yield of _opposite_matchers
|
|
# are not evaluated
|
|
return False
|
|
|
|
return True
|
|
|
|
def _search_opposites(self, move_line, opposite_move_lines):
|
|
"""Search the opposite move lines for a move line
|
|
|
|
:param dict move_line: the move line for which we search opposites
|
|
:param list opposite_move_lines: list of dict of move lines values,
|
|
the move lines we want to search for
|
|
:return: list of matching lines
|
|
"""
|
|
matchers = self._matchers(move_line)
|
|
return [
|
|
op
|
|
for op in opposite_move_lines
|
|
if self._compare_opposite(move_line, op, matchers)
|
|
]
|
|
|
|
def _action_rec(self):
|
|
self.env.flush_all()
|
|
credit_lines = self._query_credit()
|
|
debit_lines = self._query_debit()
|
|
result = self._rec_auto_lines_advanced(credit_lines, debit_lines)
|
|
return result
|
|
|
|
def _skip_line(self, move_line):
|
|
"""
|
|
When True is returned on some conditions, the credit move line
|
|
will be skipped for reconciliation. Can be inherited to
|
|
skip on some conditions. ie: ref or partner_id is empty.
|
|
"""
|
|
return False
|
|
|
|
def _rec_group(self, reconcile_groups, lines_by_id):
|
|
reconciled_ids = []
|
|
for group_count, reconcile_group_ids in enumerate(reconcile_groups, start=1):
|
|
_logger.debug(
|
|
"Reconciling group %d/%d with ids %s",
|
|
group_count,
|
|
len(reconcile_groups),
|
|
reconcile_group_ids,
|
|
)
|
|
group_lines = [lines_by_id[lid] for lid in reconcile_group_ids]
|
|
reconciled, full = self._reconcile_lines(group_lines, allow_partial=True)
|
|
if reconciled and full:
|
|
reconciled_ids += reconcile_group_ids
|
|
return reconciled_ids
|
|
|
|
def _rec_group_by_chunk(self, reconcile_groups, lines_by_id, chunk_size):
|
|
"""Commit after each chunk
|
|
|
|
:param list reconcile_groups: all groups to reconcile, will be split
|
|
by chunk
|
|
:param dict lines_by_id: dict of move lines values,
|
|
the move lines we want to search for
|
|
:return: list of reconciled lines
|
|
"""
|
|
reconciled_ids = []
|
|
|
|
_logger.info("Reconciling by chunk of %d", chunk_size)
|
|
|
|
# Copy and commit current transient model before creating a new cursor
|
|
# This is required to avoid CacheMiss when using data from `self`
|
|
# which is created during current transaction.
|
|
with registry(self.env.cr.dbname).cursor() as new_cr:
|
|
new_env = api.Environment(new_cr, self.env.uid, self.env.context)
|
|
self_env = self.with_env(new_env)
|
|
rec = self_env.create(self.copy_data())
|
|
for i in range(0, len(reconcile_groups), chunk_size):
|
|
chunk = reconcile_groups[i : i + chunk_size]
|
|
_logger.debug("Reconcile group chunk %s", chunk)
|
|
try:
|
|
with registry(self.env.cr.dbname).cursor() as new_cr:
|
|
new_env = api.Environment(new_cr, self.env.uid, self.env.context)
|
|
# Re-use the committed transient we just committed
|
|
self_env = self.with_env(new_env).browse(rec.id)
|
|
reconciled_ids += self_env._rec_group(chunk, lines_by_id)
|
|
except Exception as e:
|
|
msg = "Reconciliation failed for group chunk %s with error:\n%s"
|
|
_logger.exception(msg, chunk, e)
|
|
return reconciled_ids
|
|
|
|
def _rec_auto_lines_advanced(self, credit_lines, debit_lines):
|
|
"""Advanced reconciliation main loop"""
|
|
# pylint: disable=invalid-commit
|
|
reconciled_ids = []
|
|
for rec in self:
|
|
commit_every = rec.account_id.company_id.reconciliation_commit_every
|
|
reconcile_groups = []
|
|
_logger.info("%d credit lines to reconcile", len(credit_lines))
|
|
for idx, credit_line in enumerate(credit_lines, start=1):
|
|
if idx % 50 == 0:
|
|
_logger.info(
|
|
"... %d/%d credit lines inspected ...", idx, len(credit_lines)
|
|
)
|
|
if self._skip_line(credit_line):
|
|
continue
|
|
opposite_lines = self._search_opposites(credit_line, debit_lines)
|
|
if not opposite_lines:
|
|
continue
|
|
opposite_ids = [opp["id"] for opp in opposite_lines]
|
|
line_ids = opposite_ids + [credit_line["id"]]
|
|
for group in reconcile_groups:
|
|
if any([lid in group for lid in opposite_ids]):
|
|
_logger.debug(
|
|
"New lines %s matched with an existing " "group %s",
|
|
line_ids,
|
|
group,
|
|
)
|
|
group.update(line_ids)
|
|
break
|
|
else:
|
|
_logger.debug("New group of lines matched %s", line_ids)
|
|
reconcile_groups.append(set(line_ids))
|
|
lines_by_id = {line["id"]: line for line in credit_lines + debit_lines}
|
|
_logger.info("Found %d groups to reconcile", len(reconcile_groups))
|
|
if commit_every:
|
|
reconciled_ids = self._rec_group_by_chunk(
|
|
reconcile_groups, lines_by_id, commit_every
|
|
)
|
|
else:
|
|
reconciled_ids = self._rec_group(reconcile_groups, lines_by_id)
|
|
_logger.info("Reconciliation is over")
|
|
return reconciled_ids
|