Merge PR #804 into 13.0

Signed-off-by pedrobaeza

mail_tracking/controllers/main.py

@@ -41,6 +41,7 @@ class MailTrackingController(MailController):
             "ua_family": request.user_agent.browser or False,
         }
 
+    # TODO Remove useless controller
     @http.route(
         [
             "/mail/tracking/all/<string:db>",

mail_tracking/models/mail_tracking_email.py

@@ -379,6 +379,7 @@ class MailTrackingEmail(models.Model):
                 _logger.debug("Concurrent event '%s' discarded", event_type)
         return event_ids
 
+    # TODO Remove useless method
     @api.model
     def event_process(self, request, post, metadata, event_type=None):
         # Generic event process hook, inherit it and

mail_tracking/models/mail_tracking_event.py

@@ -122,6 +122,10 @@ class MailTrackingEvent(models.Model):
         )
         return self._process_data(tracking_email, metadata, event_type, state)
 
+    @api.model
+    def process_sent(self, tracking_email, metadata):
+        return self._process_status(tracking_email, metadata, "sent", "sent")
+
     @api.model
     def process_delivered(self, tracking_email, metadata):
         return self._process_status(tracking_email, metadata, "delivered", "delivered")

mail_tracking_mailgun/README.rst

@@ -38,33 +38,27 @@ function used here.
 .. contents::
    :local:
 
+Installation
+============
+
+If you're using a multi-database installation (with or without dbfilter option)
+where /web/databse/selector returns a list of more than one database, then
+you need to add ``mail_tracking_mailgun`` addon to wide load addons list
+(by default, only ``web`` addon), setting ``--load`` option.
+
+Example: ``--load=web,mail_tracking,mail_tracking_mailgun``
+
 Configuration
 =============
 
-You must configure Mailgun webhooks in order to receive mail events:
+To configure this module, you need to:
 
-1. Got a Mailgun account and validate your sending domain.
-2. Go to Webhook tab and configure the below URL for each event:
-
-.. code:: html
-
-   https://<your_domain>/mail/tracking/all/<your_database>
-
-Replace '<your_domain>' with your Odoo install domain name
-and '<your_database>' with your database name.
-
-In order to validate Mailgun webhooks you have to configure the following system
-parameters:
-
-- `mailgun.apikey`: You can find Mailgun api_key in your validated sending
-  domain.
-- `mailgun.api_url`: It should be fine as it is, but it could change in the
-  future.
-- `mailgun.domain`: In case your sending domain is different from the one
-  configured in `mail.catchall.domain`.
-- `mailgun.validation_key`: If you want to be able to check mail address
-  validity you must config this parameter with your account Public Validation
-  Key.
+#. Go to Mailgun, create an account and validate your sending domain.
+#. Go back to Odoo.
+#. Go to *Settings > General Settings > Discuss > Enable mail tracking with Mailgun*.
+#. Fill all the values. The only one required is the API key.
+#. Optionally click *Unregister Mailgun webhooks* and accept.
+#. Click *Register Mailgun webhooks*.
 
 You can also config partner email autocheck with this system parameter:
 
@@ -94,6 +88,11 @@ Known issues / Roadmap
 
 * There's no support for more than one Mailgun mail server.
 
+* Automate more webhook registration. It would be nice to not have to click the
+  "Unregister Mailgun webhooks" and "Register Mailgun webhooks" when setting up
+  Mailgun in Odoo. However, it doesn't come without its `conceptual complexities
+  <https://github.com/OCA/social/pull/787#discussion_r734275262>`__.
+
 Bug Tracker
 ===========
 
@@ -123,6 +122,7 @@ Contributors
   * David Vidal
   * Rafael Blasco
   * Ernesto Tejeda
+  * Jairo Llopis
 
 Other credits
 ~~~~~~~~~~~~~

mail_tracking_mailgun/__init__.py

@@ -1,3 +1,5 @@
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
 
+from . import controllers
 from . import models
+from . import wizards

mail_tracking_mailgun/__manifest__.py

@@ -6,7 +6,7 @@
 {
     "name": "Mail tracking for Mailgun",
     "summary": "Mail tracking and Mailgun webhooks integration",
-    "version": "13.0.1.0.0",
+    "version": "13.0.2.0.0",
     "category": "Social Network",
     "website": "https://github.com/OCA/social",
     "author": "Tecnativa, Odoo Community Association (OCA)",
@@ -14,5 +14,9 @@
     "application": False,
     "installable": True,
     "depends": ["mail_tracking"],
-    "data": ["views/res_partner.xml", "views/mail_tracking_email.xml"],
+    "data": [
+        "views/res_partner.xml",
+        "views/mail_tracking_email.xml",
+        "wizards/res_config_settings_views.xml",
+    ],
 }

mail_tracking_mailgun/controllers/__init__.py

@@ -0,0 +1 @@
+from . import main

mail_tracking_mailgun/controllers/main.py

@@ -0,0 +1,74 @@
+# Copyright 2021 Tecnativa - Jairo Llopis
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+
+import hashlib
+import hmac
+import logging
+from datetime import datetime, timedelta
+
+from werkzeug.exceptions import NotAcceptable
+
+from odoo import _
+from odoo.exceptions import ValidationError
+from odoo.http import request, route
+
+from ...mail_tracking.controllers import main
+from ...web.controllers.main import ensure_db
+
+_logger = logging.getLogger(__name__)
+
+
+class MailTrackingController(main.MailTrackingController):
+    def _mail_tracking_mailgun_webhook_verify(self, timestamp, token, signature):
+        """Avoid mailgun webhook attacks.
+
+        See https://documentation.mailgun.com/en/latest/user_manual.html#securing-webhooks
+        """  # noqa: E501
+        # Request cannot be old
+        processing_time = datetime.utcnow() - datetime.utcfromtimestamp(int(timestamp))
+        if not timedelta() < processing_time < timedelta(minutes=10):
+            raise ValidationError(_("Request is too old"))
+        # Avoid replay attacks
+        try:
+            processed_tokens = (
+                request.env.registry._mail_tracking_mailgun_processed_tokens
+            )
+        except AttributeError:
+            processed_tokens = (
+                request.env.registry._mail_tracking_mailgun_processed_tokens
+            ) = set()
+        if token in processed_tokens:
+            raise ValidationError(_("Request was already processed"))
+        processed_tokens.add(token)
+        params = request.env["mail.tracking.email"]._mailgun_values()
+        # Assert signature
+        if not params.webhook_signing_key:
+            _logger.warning(
+                "Skipping webhook payload verification. "
+                "Set `mailgun.webhook_signing_key` config parameter to enable"
+            )
+            return
+        hmac_digest = hmac.new(
+            key=params.webhook_signing_key.encode(),
+            msg=("{}{}".format(timestamp, token)).encode(),
+            digestmod=hashlib.sha256,
+        ).hexdigest()
+        if not hmac.compare_digest(str(signature), str(hmac_digest)):
+            raise ValidationError(_("Wrong signature"))
+
+    @route(["/mail/tracking/mailgun/all"], auth="none", type="json", csrf=False)
+    def mail_tracking_mailgun_webhook(self):
+        """Process webhooks from Mailgun."""
+        ensure_db()
+        # Verify and return 406 in case of failure, to avoid retries
+        # See https://documentation.mailgun.com/en/latest/user_manual.html#routes
+        try:
+            self._mail_tracking_mailgun_webhook_verify(
+                **request.jsonrequest["signature"]
+            )
+        except ValidationError as error:
+            raise NotAcceptable from error
+        # Process event
+        request.env["mail.tracking.email"].sudo()._mailgun_event_process(
+            request.jsonrequest["event-data"], self._request_metadata(),
+        )

mail_tracking_mailgun/migrations/13.0.2.0.0/post-migration.py

@@ -0,0 +1,32 @@
+# Copyright 2021 Tecnativa - Jairo Llopis
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+
+import logging
+
+from openupgradelib import openupgrade
+
+_logger = logging.getLogger(__name__)
+
+
+@openupgrade.migrate()
+def migrate(env, version):
+    """Update webhooks.
+
+    This version dropped support for legacy webhooks and added support for
+    webhook autoregistering. Do that process now.
+    """
+    settings = env["res.config.settings"].create()
+    if not settings.mail_tracking_mailgun_enabled:
+        _logger.warning("Not updating webhooks because mailgun is not configured")
+        return
+    _logger.info("Updating mailgun webhooks")
+    try:
+        settings.mail_tracking_mailgun_unregister_webhooks()
+        settings.mail_tracking_mailgun_register_webhooks()
+    except Exception:
+        # Don't fail the update if you can't register webhooks; it can be a
+        # failing network condition or air-gapped upgrade, and that's OK, you
+        # can just update them later
+        _logger.warning(
+            "Failed to update mailgun webhooks; do that manually", exc_info=True
+        )

mail_tracking_mailgun/models/mail_tracking_email.py

@@ -1,11 +1,12 @@
 # Copyright 2016 Tecnativa - Antonio Espinosa
 # Copyright 2017 Tecnativa - David Vidal
+# Copyright 2021 Tecnativa - Jairo Llopis
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
 
-import hashlib
-import hmac
 import logging
+from collections import namedtuple
 from datetime import datetime
+from urllib.parse import urljoin
 
 import requests
 
@@ -15,6 +16,22 @@ from odoo.tools import email_split
 
 _logger = logging.getLogger(__name__)
 
+MailgunParameters = namedtuple(
+    "MailgunParameters",
+    (
+        "api_key",
+        "api_url",
+        "domain",
+        "validation_key",
+        "webhooks_domain",
+        "webhook_signing_key",
+    ),
+)
+
+
+class EventNotFoundWarning(Warning):
+    pass
+
 
 class MailTrackingEmail(models.Model):
     _inherit = "mail.tracking.email"
@@ -29,49 +46,30 @@ class MailTrackingEmail(models.Model):
             return country.id
         return False
 
-    @property
-    def _mailgun_mandatory_fields(self):
-        return (
-            "event",
-            "timestamp",
-            "token",
-            "signature",
-            "tracking_email_id",
-            "odoo_db",
-        )
+    @api.model
+    def _mailgun_event2type(self, event, default="UNKNOWN"):
+        """Return the ``mail.tracking.event`` equivalent event
 
-    @property
-    def _mailgun_event_type_mapping(self):
-        return {
-            # Mailgun event type: tracking event type
+        Args:
+            event: Mailgun event response from API.
+            default: Value to return when not found.
+        """
+        # Mailgun event type: tracking event type
+        equivalents = {
             "delivered": "delivered",
             "opened": "open",
             "clicked": "click",
             "unsubscribed": "unsub",
             "complained": "spam",
-            "bounced": "hard_bounce",
-            "dropped": "reject",
             "accepted": "sent",
-            "failed": "error",
-            "rejected": "error",
+            "failed": (
+                "hard_bounce" if event.get("severity") == "permanent" else "soft_bounce"
+            ),
+            "rejected": "reject",
         }
+        return equivalents.get(event.get("event"), default)
 
-    def _mailgun_event_type_verify(self, event):
-        event = event or {}
-        mailgun_event_type = event.get("event")
-        if mailgun_event_type not in self._mailgun_event_type_mapping:
-            _logger.error("Mailgun: event type '%s' not supported", mailgun_event_type)
-            return False
-        # OK, event type is valid
-        return True
-
-    def _mailgun_signature(self, api_key, timestamp, token):
-        return hmac.new(
-            key=bytes(api_key, "utf-8"),
-            msg=bytes("{}{}".format(str(timestamp), str(token)), "utf-8"),
-            digestmod=hashlib.sha256,
-        ).hexdigest()
-
+    @api.model
     def _mailgun_values(self):
         icp = self.env["ir.config_parameter"].sudo()
         api_key = icp.get_param("mailgun.apikey")
@@ -83,43 +81,17 @@ class MailTrackingEmail(models.Model):
         if not domain:
             raise ValidationError(_("A Mailgun domain value is needed!"))
         validation_key = icp.get_param("mailgun.validation_key")
-        return api_key, api_url, domain, validation_key
-
-    def _mailgun_signature_verify(self, event):
-        event = event or {}
-        icp = self.env["ir.config_parameter"].sudo()
-        api_key = icp.get_param("mailgun.apikey")
-        if not api_key:
-            _logger.warning(
-                "No Mailgun api key configured. "
-                "Please add 'mailgun.apikey' to System parameters "
-                "to enable Mailgun authentication webhoook "
-                "requests. More info at: "
-                "https://documentation.mailgun.com/"
-                "user_manual.html#webhooks"
-            )
-        else:
-            timestamp = event.get("timestamp")
-            token = event.get("token")
-            signature = event.get("signature")
-            event_digest = self._mailgun_signature(api_key, timestamp, token)
-            if signature != event_digest:
-                _logger.error(
-                    "Mailgun: Invalid signature '%s' != '%s'", signature, event_digest
-                )
-                return False
-        # OK, signature is valid
-        return True
-
-    def _db_verify(self, event):
-        event = event or {}
-        odoo_db = event.get("odoo_db")
-        current_db = self.env.cr.dbname
-        if odoo_db != current_db:
-            _logger.error("Mailgun: Database '%s' is not the current database", odoo_db)
-            return False
-        # OK, DB is current
-        return True
+        web_base_url = icp.get_param("web.base.url")
+        webhooks_domain = icp.get_param("mailgun.webhooks_domain", web_base_url)
+        webhook_signing_key = icp.get_param("mailgun.webhook_signing_key")
+        return MailgunParameters(
+            api_key,
+            api_url,
+            domain,
+            validation_key,
+            webhooks_domain,
+            webhook_signing_key,
+        )
 
     def _mailgun_metadata(self, mailgun_event_type, event, metadata):
         # Get Mailgun timestamp when found
@@ -159,20 +131,22 @@ class MailTrackingEmail(models.Model):
             }
         )
         # Mapping for special events
-        if mailgun_event_type == "bounced":
+        if mailgun_event_type == "failed":
+            delivery_status = event.get("delivery-status", {})
             metadata.update(
                 {
-                    "error_type": event.get("code", False),
-                    "error_description": event.get("error", False),
-                    "error_details": event.get("notification", False),
+                    "error_type": delivery_status.get("code", False),
+                    "error_description": delivery_status.get("message", False),
+                    "error_details": delivery_status.get("description", False),
                 }
             )
-        elif mailgun_event_type == "dropped":
+        elif mailgun_event_type == "rejected":
+            reject = event.get("reject", {})
             metadata.update(
                 {
-                    "error_type": event.get("reason", False),
-                    "error_description": event.get("code", False),
-                    "error_details": event.get("description", False),
+                    "error_type": "rejected",
+                    "error_description": reject.get("reason", False),
+                    "error_details": reject.get("description", False),
                 }
             )
         elif mailgun_event_type == "complained":
@@ -185,89 +159,75 @@ class MailTrackingEmail(models.Model):
             )
         return metadata
 
-    def _mailgun_tracking_get(self, event):
-        tracking = False
-        tracking_email_id = event.get("tracking_email_id", False)
-        if tracking_email_id and tracking_email_id.isdigit():
-            tracking = self.search([("id", "=", tracking_email_id)], limit=1)
-        return tracking
-
-    def _event_is_from_mailgun(self, event):
-        event = event or {}
-        return all([k in event for k in self._mailgun_mandatory_fields])
-
     @api.model
-    def event_process(self, request, post, metadata, event_type=None):
-        res = super(MailTrackingEmail, self).event_process(
-            request, post, metadata, event_type=event_type
+    def _mailgun_event_process(self, event_data, metadata):
+        """Retrieve (and maybe create) mailgun event from API data payload.
+
+        In https://documentation.mailgun.com/en/latest/api-events.html#event-structure
+        you can read the event payload format as obtained from webhooks or calls to API.
+        """
+        if event_data["user-variables"]["odoo_db"] != self.env.cr.dbname:
+            raise ValidationError(_("Wrong database for event!"))
+        # Do nothing if event was already processed
+        mailgun_id = event_data["id"]
+        db_event = self.env["mail.tracking.event"].search(
+            [("mailgun_id", "=", mailgun_id)], limit=1
         )
-        if res == "NONE" and self._event_is_from_mailgun(post):
-            if not self._mailgun_signature_verify(post):
-                res = "ERROR: Signature"
-            elif not self._mailgun_event_type_verify(post):
-                res = "ERROR: Event type not supported"
-            elif not self._db_verify(post):
-                res = "ERROR: Invalid DB"
-            else:
-                res = "OK"
-        if res == "OK":
-            mailgun_event_type = post.get("event")
-            mapped_event_type = (
-                self._mailgun_event_type_mapping.get(mailgun_event_type) or event_type
-            )
-            if not mapped_event_type:  # pragma: no cover
-                res = "ERROR: Bad event"
-            tracking = self._mailgun_tracking_get(post)
-            if not tracking:
-                res = "ERROR: Tracking not found"
-        if res == "OK":
-            # Complete metadata with mailgun event info
-            metadata = self._mailgun_metadata(mailgun_event_type, post, metadata)
-            # Create event
-            tracking.event_create(mapped_event_type, metadata)
-        if res != "NONE":
-            if event_type:
-                _logger.info("Mailgun: event '%s' process '%s'", event_type, res)
-            else:
-                _logger.info("Mailgun: event process '%s'", res)
-        return res
+        if db_event:
+            _logger.debug("Mailgun event already found in DB: %s", mailgun_id)
+            return db_event
+        # Do nothing if tracking email for event is not found
+        message_id = event_data["message"]["headers"]["message-id"]
+        recipient = event_data["recipient"]
+        tracking_email = self.browse(
+            int(event_data["user-variables"]["tracking_email_id"])
+        )
+        mailgun_event_type = event_data["event"]
+        # Process event
+        state = self._mailgun_event2type(event_data, mailgun_event_type)
+        metadata = self._mailgun_metadata(mailgun_event_type, event_data, metadata)
+        _logger.info(
+            "Importing mailgun event %s (%s message %s for %s)",
+            mailgun_id,
+            mailgun_event_type,
+            message_id,
+            recipient,
+        )
+        tracking_email.event_create(state, metadata)
 
     def action_manual_check_mailgun(self):
-        """
-        Manual check against Mailgun API
+        """Manual check against Mailgun API
+
         API Documentation:
         https://documentation.mailgun.com/en/latest/api-events.html
         """
-        api_key, api_url, domain, validation_key = self._mailgun_values()
+        api_key, api_url, domain, *__ = self._mailgun_values()
         for tracking in self:
             if not tracking.mail_message_id:
                 raise UserError(_("There is no tracked message!"))
             message_id = tracking.mail_message_id.message_id.replace("<", "").replace(
                 ">", ""
             )
-            res = requests.get(
-                "{}/{}/events".format(api_url, domain),
-                auth=("api", api_key),
-                params={
-                    "begin": tracking.timestamp,
-                    "ascending": "yes",
-                    "message-id": message_id,
-                },
-            )
-            if not res or res.status_code != 200:
-                raise ValidationError(_("Couldn't retrieve Mailgun information"))
-            content = res.json()
-            if "items" not in content:
-                raise ValidationError(_("Event information not longer stored"))
-            for item in content["items"]:
-                # mailgun event hasn't been synced and recipient is the same as
-                # in the evaluated tracking. We use email_split since tracking
-                # recipient could come in format: "example" <to@dest.com>
-                if not self.env["mail.tracking.event"].search(
-                    [("mailgun_id", "=", item["id"])]
-                ) and (item.get("recipient", "") == email_split(tracking.recipient)[0]):
-                    mapped_event_type = self._mailgun_event_type_mapping.get(
-                        item["event"], item["event"]
-                    )
-                    metadata = self._mailgun_metadata(mapped_event_type, item, {})
-                    tracking.event_create(mapped_event_type, metadata)
+            events = []
+            url = urljoin(api_url, "/v3/%s/events" % domain)
+            params = {
+                "begin": tracking.timestamp,
+                "ascending": "yes",
+                "message-id": message_id,
+                "recipient": email_split(tracking.recipient)[0],
+            }
+            while url:
+                res = requests.get(url, auth=("api", api_key), params=params,)
+                if not res or res.status_code != 200:
+                    raise UserError(_("Couldn't retrieve Mailgun information"))
+                iter_events = res.json().get("items", [])
+                if not iter_events:
+                    # Loop no more
+                    break
+                events.extend(iter_events)
+                # Loop over pagination
+                url = res.json().get("paging", {}).get("next")
+            if not events:
+                raise UserError(_("Event information not longer stored"))
+            for event in events:
+                self.sudo()._mailgun_event_process(event, {})

mail_tracking_mailgun/models/mail_tracking_event.py

@@ -7,7 +7,13 @@ from odoo import fields, models
 class MailTrackingEvent(models.Model):
     _inherit = "mail.tracking.event"
 
-    mailgun_id = fields.Char(string="Mailgun Event ID", copy="False", readonly=True)
+    _sql_constraints = [
+        ("mailgun_id_unique", "UNIQUE(mailgun_id)", "Mailgun event IDs must be unique!")
+    ]
+
+    mailgun_id = fields.Char(
+        string="Mailgun Event ID", copy="False", readonly=True, index=True,
+    )
 
     def _process_data(self, tracking_email, metadata, event_type, state):
         res = super(MailTrackingEvent, self)._process_data(

mail_tracking_mailgun/models/res_partner.py

@@ -4,6 +4,8 @@
 # Copyright 2017 Tecnativa - David Vidal
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
 
+from urllib.parse import urljoin
+
 import requests
 
 from odoo import _, api, models
@@ -45,10 +47,8 @@ class ResPartner(models.Model):
         API documentation:
         https://documentation.mailgun.com/en/latest/api-email-validation.html
         """
-        api_key, api_url, domain, validation_key = self.env[
-            "mail.tracking.email"
-        ]._mailgun_values()
-        if not validation_key:
+        params = self.env["mail.tracking.email"]._mailgun_values()
+        if not params.validation_key:
             raise UserError(
                 _(
                     "You need to configure mailgun.validation_key"
@@ -57,9 +57,8 @@ class ResPartner(models.Model):
             )
         for partner in self.filtered("email"):
             res = requests.get(
-                # Validation API url is always the same
-                "https://api.mailgun.net/v3/address/validate",
-                auth=("api", validation_key),
+                urljoin(params.api_url, "/v3/address/validate"),
+                auth=("api", params.validation_key),
                 params={"address": partner.email, "mailbox_verification": True},
             )
             if (
@@ -69,7 +68,7 @@ class ResPartner(models.Model):
             ):
                 raise UserError(
                     _(
-                        "Error %s trying to check mail" % res.status_code
+                        "Error %s trying to " "check mail" % res.status_code
                         or "of connection"
                     )
                 )
@@ -129,12 +128,12 @@ class ResPartner(models.Model):
         API documentation:
         https://documentation.mailgun.com/en/latest/api-suppressions.html
         """
-        api_key, api_url, domain, validation_key = self.env[
+        api_key, api_url, domain, *__ = self.env[
             "mail.tracking.email"
         ]._mailgun_values()
         for partner in self:
             res = requests.get(
-                "{}/{}/bounces/{}".format(api_url, domain, partner.email),
+                urljoin(api_url, "/v3/%s/bounces/%s" % (domain, partner.email)),
                 auth=("api", api_key),
             )
             if res.status_code == 200 and not partner.email_bounced:
@@ -148,12 +147,12 @@ class ResPartner(models.Model):
         API documentation:
         https://documentation.mailgun.com/en/latest/api-suppressions.html
         """
-        api_key, api_url, domain, validation_key = self.env[
+        api_key, api_url, domain, *__ = self.env[
             "mail.tracking.email"
         ]._mailgun_values()
         for partner in self:
             res = requests.post(
-                "{}/{}/bounces".format(api_url, domain),
+                urljoin(api_url, "/v3/%s/bounces" % domain),
                 auth=("api", api_key),
                 data={"address": partner.email},
             )
@@ -165,12 +164,12 @@ class ResPartner(models.Model):
         API documentation:
         https://documentation.mailgun.com/en/latest/api-suppressions.html
         """
-        api_key, api_url, domain, validation_key = self.env[
+        api_key, api_url, domain, *__ = self.env[
             "mail.tracking.email"
         ]._mailgun_values()
         for partner in self:
             res = requests.delete(
-                "{}/{}/bounces/{}".format(api_url, domain, partner.email),
+                urljoin(api_url, "/v3/%s/bounces/%s" % (domain, partner.email)),
                 auth=("api", api_key),
             )
             if res.status_code in (200, 404) and partner.email_bounced:

mail_tracking_mailgun/readme/CONFIGURE.rst

@@ -1,27 +1,11 @@
-You must configure Mailgun webhooks in order to receive mail events:
+To configure this module, you need to:
 
-1. Got a Mailgun account and validate your sending domain.
-2. Go to Webhook tab and configure the below URL for each event:
-
-.. code:: html
-
-   https://<your_domain>/mail/tracking/all/<your_database>
-
-Replace '<your_domain>' with your Odoo install domain name
-and '<your_database>' with your database name.
-
-In order to validate Mailgun webhooks you have to configure the following system
-parameters:
-
-- `mailgun.apikey`: You can find Mailgun api_key in your validated sending
-  domain.
-- `mailgun.api_url`: It should be fine as it is, but it could change in the
-  future.
-- `mailgun.domain`: In case your sending domain is different from the one
-  configured in `mail.catchall.domain`.
-- `mailgun.validation_key`: If you want to be able to check mail address
-  validity you must config this parameter with your account Public Validation
-  Key.
+#. Go to Mailgun, create an account and validate your sending domain.
+#. Go back to Odoo.
+#. Go to *Settings > General Settings > Discuss > Enable mail tracking with Mailgun*.
+#. Fill all the values. The only one required is the API key.
+#. Optionally click *Unregister Mailgun webhooks* and accept.
+#. Click *Register Mailgun webhooks*.
 
 You can also config partner email autocheck with this system parameter:
 

mail_tracking_mailgun/readme/CONTRIBUTORS.rst

@@ -6,3 +6,4 @@
   * David Vidal
   * Rafael Blasco
   * Ernesto Tejeda
+  * Jairo Llopis

mail_tracking_mailgun/readme/INSTALL.rst

@@ -0,0 +1,6 @@
+If you're using a multi-database installation (with or without dbfilter option)
+where /web/databse/selector returns a list of more than one database, then
+you need to add ``mail_tracking_mailgun`` addon to wide load addons list
+(by default, only ``web`` addon), setting ``--load`` option.
+
+Example: ``--load=web,mail_tracking,mail_tracking_mailgun``

mail_tracking_mailgun/readme/ROADMAP.rst

@@ -1 +1,6 @@
 * There's no support for more than one Mailgun mail server.
+
+* Automate more webhook registration. It would be nice to not have to click the
+  "Unregister Mailgun webhooks" and "Register Mailgun webhooks" when setting up
+  Mailgun in Odoo. However, it doesn't come without its `conceptual complexities
+  <https://github.com/OCA/social/pull/787#discussion_r734275262>`__.

mail_tracking_mailgun/static/description/index.html

@@ -3,7 +3,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-<meta name="generator" content="Docutils 0.15.1: http://docutils.sourceforge.net/" />
+<meta name="generator" content="Docutils: http://docutils.sourceforge.net/" />
 <title>Mail tracking for Mailgun</title>
 <style type="text/css">
 
@@ -377,54 +377,49 @@ function used here.</p>
 <p><strong>Table of contents</strong></p>
 <div class="contents local topic" id="contents">
 <ul class="simple">
-<li><a class="reference internal" href="#configuration" id="id1">Configuration</a></li>
-<li><a class="reference internal" href="#usage" id="id2">Usage</a></li>
-<li><a class="reference internal" href="#known-issues-roadmap" id="id3">Known issues / Roadmap</a></li>
-<li><a class="reference internal" href="#bug-tracker" id="id4">Bug Tracker</a></li>
-<li><a class="reference internal" href="#credits" id="id5">Credits</a><ul>
-<li><a class="reference internal" href="#authors" id="id6">Authors</a></li>
-<li><a class="reference internal" href="#contributors" id="id7">Contributors</a></li>
-<li><a class="reference internal" href="#other-credits" id="id8">Other credits</a><ul>
-<li><a class="reference internal" href="#images" id="id9">Images</a></li>
+<li><a class="reference internal" href="#installation" id="id1">Installation</a></li>
+<li><a class="reference internal" href="#configuration" id="id2">Configuration</a></li>
+<li><a class="reference internal" href="#usage" id="id3">Usage</a></li>
+<li><a class="reference internal" href="#known-issues-roadmap" id="id4">Known issues / Roadmap</a></li>
+<li><a class="reference internal" href="#bug-tracker" id="id5">Bug Tracker</a></li>
+<li><a class="reference internal" href="#credits" id="id6">Credits</a><ul>
+<li><a class="reference internal" href="#authors" id="id7">Authors</a></li>
+<li><a class="reference internal" href="#contributors" id="id8">Contributors</a></li>
+<li><a class="reference internal" href="#other-credits" id="id9">Other credits</a><ul>
+<li><a class="reference internal" href="#images" id="id10">Images</a></li>
 </ul>
 </li>
-<li><a class="reference internal" href="#maintainers" id="id10">Maintainers</a></li>
+<li><a class="reference internal" href="#maintainers" id="id11">Maintainers</a></li>
 </ul>
 </li>
 </ul>
 </div>
+<div class="section" id="installation">
+<h1><a class="toc-backref" href="#id1">Installation</a></h1>
+<p>If you’re using a multi-database installation (with or without dbfilter option)
+where /web/databse/selector returns a list of more than one database, then
+you need to add <tt class="docutils literal">mail_tracking_mailgun</tt> addon to wide load addons list
+(by default, only <tt class="docutils literal">web</tt> addon), setting <tt class="docutils literal"><span class="pre">--load</span></tt> option.</p>
+<p>Example: <tt class="docutils literal"><span class="pre">--load=web,mail_tracking,mail_tracking_mailgun</span></tt></p>
+</div>
 <div class="section" id="configuration">
-<h1><a class="toc-backref" href="#id1">Configuration</a></h1>
-<p>You must configure Mailgun webhooks in order to receive mail events:</p>
+<h1><a class="toc-backref" href="#id2">Configuration</a></h1>
+<p>To configure this module, you need to:</p>
 <ol class="arabic simple">
-<li>Got a Mailgun account and validate your sending domain.</li>
-<li>Go to Webhook tab and configure the below URL for each event:</li>
+<li>Go to Mailgun, create an account and validate your sending domain.</li>
+<li>Go back to Odoo.</li>
+<li>Go to <em>Settings &gt; General Settings &gt; Discuss &gt; Enable mail tracking with Mailgun</em>.</li>
+<li>Fill all the values. The only one required is the API key.</li>
+<li>Optionally click <em>Unregister Mailgun webhooks</em> and accept.</li>
+<li>Click <em>Register Mailgun webhooks</em>.</li>
 </ol>
-<pre class="code html literal-block">
-https://<span class="p">&lt;</span><span class="nt">your_domain</span><span class="p">&gt;</span>/mail/tracking/all/<span class="p">&lt;</span><span class="nt">your_database</span><span class="p">&gt;</span>
-</pre>
-<p>Replace ‘&lt;your_domain&gt;’ with your Odoo install domain name
-and ‘&lt;your_database&gt;’ with your database name.</p>
-<p>In order to validate Mailgun webhooks you have to configure the following system
-parameters:</p>
-<ul class="simple">
-<li><cite>mailgun.apikey</cite>: You can find Mailgun api_key in your validated sending
-domain.</li>
-<li><cite>mailgun.api_url</cite>: It should be fine as it is, but it could change in the
-future.</li>
-<li><cite>mailgun.domain</cite>: In case your sending domain is different from the one
-configured in <cite>mail.catchall.domain</cite>.</li>
-<li><cite>mailgun.validation_key</cite>: If you want to be able to check mail address
-validity you must config this parameter with your account Public Validation
-Key.</li>
-</ul>
 <p>You can also config partner email autocheck with this system parameter:</p>
 <ul class="simple">
 <li><cite>mailgun.auto_check_partner_email</cite>: Set it to True.</li>
 </ul>
 </div>
 <div class="section" id="usage">
-<h1><a class="toc-backref" href="#id2">Usage</a></h1>
+<h1><a class="toc-backref" href="#id3">Usage</a></h1>
 <p>In your mail tracking status screens (explained on module <em>mail_tracking</em>), you
 will see a more accurate information, like the ‘Received’ or ‘Bounced’ status,
 which are not usually detected by normal SMTP servers.</p>
@@ -441,13 +436,16 @@ button <em>Check Mailgun</em>. It’s important to note that tracking events hav
 short lifespan, so after 24h they won’t be recoverable.</p>
 </div>
 <div class="section" id="known-issues-roadmap">
-<h1><a class="toc-backref" href="#id3">Known issues / Roadmap</a></h1>
+<h1><a class="toc-backref" href="#id4">Known issues / Roadmap</a></h1>
 <ul class="simple">
 <li>There’s no support for more than one Mailgun mail server.</li>
+<li>Automate more webhook registration. It would be nice to not have to click the
+“Unregister Mailgun webhooks” and “Register Mailgun webhooks” when setting up
+Mailgun in Odoo. However, it doesn’t come without its <a class="reference external" href="https://github.com/OCA/social/pull/787#discussion_r734275262">conceptual complexities</a>.</li>
 </ul>
 </div>
 <div class="section" id="bug-tracker">
-<h1><a class="toc-backref" href="#id4">Bug Tracker</a></h1>
+<h1><a class="toc-backref" href="#id5">Bug Tracker</a></h1>
 <p>Bugs are tracked on <a class="reference external" href="https://github.com/OCA/social/issues">GitHub Issues</a>.
 In case of trouble, please check there if your issue has already been reported.
 If you spotted it first, help us smashing it by providing a detailed and welcomed
@@ -455,15 +453,15 @@ If you spotted it first, help us smashing it by providing a detailed and welcome
 <p>Do not contact contributors directly about support or help with technical issues.</p>
 </div>
 <div class="section" id="credits">
-<h1><a class="toc-backref" href="#id5">Credits</a></h1>
+<h1><a class="toc-backref" href="#id6">Credits</a></h1>
 <div class="section" id="authors">
-<h2><a class="toc-backref" href="#id6">Authors</a></h2>
+<h2><a class="toc-backref" href="#id7">Authors</a></h2>
 <ul class="simple">
 <li>Tecnativa</li>
 </ul>
 </div>
 <div class="section" id="contributors">
-<h2><a class="toc-backref" href="#id7">Contributors</a></h2>
+<h2><a class="toc-backref" href="#id8">Contributors</a></h2>
 <ul class="simple">
 <li><a class="reference external" href="https://www.tecnativa.com">Tecnativa</a>:<ul>
 <li>Antonio Espinosa</li>
@@ -472,21 +470,22 @@ If you spotted it first, help us smashing it by providing a detailed and welcome
 <li>David Vidal</li>
 <li>Rafael Blasco</li>
 <li>Ernesto Tejeda</li>
+<li>Jairo Llopis</li>
 </ul>
 </li>
 </ul>
 </div>
 <div class="section" id="other-credits">
-<h2><a class="toc-backref" href="#id8">Other credits</a></h2>
+<h2><a class="toc-backref" href="#id9">Other credits</a></h2>
 <div class="section" id="images">
-<h3><a class="toc-backref" href="#id9">Images</a></h3>
+<h3><a class="toc-backref" href="#id10">Images</a></h3>
 <ul class="simple">
 <li>Mailgun logo: <a class="reference external" href="http://seeklogo.com/mailgun-logo-273630.html">SVG Icon</a>.</li>
 </ul>
 </div>
 </div>
 <div class="section" id="maintainers">
-<h2><a class="toc-backref" href="#id10">Maintainers</a></h2>
+<h2><a class="toc-backref" href="#id11">Maintainers</a></h2>
 <p>This module is maintained by the OCA.</p>
 <a class="reference external image-reference" href="https://odoo-community.org"><img alt="Odoo Community Association" src="https://odoo-community.org/logo.png" /></a>
 <p>OCA, or the Odoo Community Association, is a nonprofit organization whose

mail_tracking_mailgun/tests/test_mailgun.py

@@ -1,16 +1,31 @@
 # Copyright 2016 Tecnativa - Antonio Espinosa
 # Copyright 2017 Tecnativa - David Vidal
+# Copyright 2021 Tecnativa - Jairo Llopis
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
 
-from unittest import mock
+from contextlib import contextmanager, suppress
 
-from odoo.exceptions import UserError, ValidationError
-from odoo.tests.common import TransactionCase
+import mock
+from freezegun import freeze_time
+from werkzeug.exceptions import NotAcceptable
+
+from odoo.exceptions import MissingError, UserError, ValidationError
+from odoo.tests.common import Form, TransactionCase
 from odoo.tools import mute_logger
 
+from ..controllers.main import MailTrackingController
+
+# HACK https://github.com/odoo/odoo/pull/78424 because website is not dependency
+try:
+    from odoo.addons.website.tools import MockRequest
+except ImportError:
+    MockRequest = None
+
+
 _packagepath = "odoo.addons.mail_tracking_mailgun"
 
 
+@freeze_time("2016-08-12 17:00:00", tick=True)
 class TestMailgun(TransactionCase):
     def mail_send(self):
         mail = self.env["mail.mail"].create(
@@ -19,6 +34,7 @@ class TestMailgun(TransactionCase):
                 "email_from": "from@example.com",
                 "email_to": self.recipient,
                 "body_html": "<p>This is a test message</p>",
+                "message_id": "<test-id@f187c54734e8>",
             }
         )
         mail.send()
@@ -32,32 +48,45 @@ class TestMailgun(TransactionCase):
         super(TestMailgun, self).setUp()
         self.recipient = "to@example.com"
         self.mail, self.tracking_email = self.mail_send()
-        self.api_key = "key-12345678901234567890123456789012"
         self.domain = "example.com"
+        # Configure Mailgun through GUI
+        cf = Form(self.env["res.config.settings"])
+        cf.mail_tracking_mailgun_enabled = True
+        cf.mail_tracking_mailgun_api_key = (
+            cf.mail_tracking_mailgun_webhook_signing_key
+        ) = (
+            cf.mail_tracking_mailgun_validation_key
+        ) = "key-12345678901234567890123456789012"
+        cf.mail_tracking_mailgun_domain = False
+        cf.mail_tracking_mailgun_auto_check_partner_emails = False
+        config = cf.save()
+        # Done this way as `hr_expense` adds this field again as readonly, and thus Form
+        # doesn't process it correctly
+        config.alias_domain = self.domain
+        config.execute()
         self.token = "f1349299097a51b9a7d886fcb5c2735b426ba200ada6e9e149"
         self.timestamp = "1471021089"
         self.signature = (
-            "4fb6d4dbbe10ce5d620265dcd7a3c0b8ca0dede1433103891bc1ae4086e9d5b2"
-        )
-        self.env["ir.config_parameter"].set_param("mailgun.apikey", self.api_key)
-        self.env["ir.config_parameter"].set_param("mail.catchall.domain", self.domain)
-        self.env["ir.config_parameter"].set_param(
-            "mailgun.validation_key", self.api_key
-        )
-        self.env["ir.config_parameter"].set_param(
-            "mailgun.auto_check_partner_email", ""
+            "4fb6d4dbbe10ce5d620265dcd7a3c0b8" "ca0dede1433103891bc1ae4086e9d5b2"
         )
         self.event = {
-            "Message-Id": "<xxx.xxx.xxx-openerp-xxx-res.partner@test_db>",
-            "X-Mailgun-Sid": "WyIwNjgxZSIsICJ0b0BleGFtcGxlLmNvbSIsICI3MGI0MWYiXQ==",
-            "token": self.token,
-            "timestamp": self.timestamp,
-            "signature": self.signature,
-            "domain": "example.com",
-            "message-headers": "[]",
-            "recipient": self.recipient,
-            "odoo_db": self.env.cr.dbname,
-            "tracking_email_id": "%s" % self.tracking_email.id,
+            "log-level": "info",
+            "id": "oXAVv5URCF-dKv8c6Sa7T",
+            "timestamp": 1471021089.0,
+            "message": {
+                "headers": {
+                    "to": "test@test.com",
+                    "message-id": "test-id@f187c54734e8",
+                    "from": "Mr. Odoo <mrodoo@odoo.com>",
+                    "subject": "This is a test",
+                },
+            },
+            "event": "delivered",
+            "recipient": "to@example.com",
+            "user-variables": {
+                "odoo_db": self.env.cr.dbname,
+                "tracking_email_id": self.tracking_email.id,
+            },
         }
         self.metadata = {
             "ip": "127.0.0.1",
@@ -68,25 +97,31 @@ class TestMailgun(TransactionCase):
         self.partner = self.env["res.partner"].create(
             {"name": "Mr. Odoo", "email": "mrodoo@example.com"}
         )
-        self.response = {
-            "items": [
-                {
-                    "log-level": "info",
-                    "id": "oXAVv5URCF-dKv8c6Sa7T",
-                    "timestamp": 1509119329.0,
-                    "message": {
-                        "headers": {
-                            "to": "test@test.com",
-                            "message-id": "test-id@f187c54734e8",
-                            "from": "Mr. Odoo <mrodoo@odoo.com>",
-                            "subject": "This is a test",
-                        }
-                    },
-                    "event": "delivered",
-                    "recipient": "to@example.com",
-                }
-            ]
-        }
+        self.response = {"items": [self.event]}
+        self.MailTrackingController = MailTrackingController()
+
+    @contextmanager
+    def _request_mock(self, reset_replay_cache=True):
+        # HACK https://github.com/odoo/odoo/pull/78424
+        if MockRequest is None:
+            self.skipTest("MockRequest not found, sorry")
+        if reset_replay_cache:
+            with suppress(AttributeError):
+                del self.env.registry._mail_tracking_mailgun_processed_tokens
+        # Imitate Mailgun JSON request
+        mock = MockRequest(self.env)
+        with mock as request:
+            request.jsonrequest = {
+                "signature": {
+                    "timestamp": self.timestamp,
+                    "token": self.token,
+                    "signature": self.signature,
+                },
+                "event-data": self.event,
+            }
+            request.params = {"db": self.env.cr.dbname}
+            request.session.db = self.env.cr.dbname
+            yield request
 
     def event_search(self, event_type):
         event = self.env["mail.tracking.event"].search(
@@ -100,13 +135,11 @@ class TestMailgun(TransactionCase):
 
     def test_no_api_key(self):
         self.env["ir.config_parameter"].set_param("mailgun.apikey", "")
-        self.test_event_delivered()
         with self.assertRaises(ValidationError):
             self.env["mail.tracking.email"]._mailgun_values()
 
     def test_no_domain(self):
         self.env["ir.config_parameter"].set_param("mail.catchall.domain", "")
-        self.test_event_delivered()
         with self.assertRaises(ValidationError):
             self.env["mail.tracking.email"]._mailgun_values()
         # now we set an specific domain for Mailgun:
@@ -116,60 +149,65 @@ class TestMailgun(TransactionCase):
 
     @mute_logger("odoo.addons.mail_tracking_mailgun.models.mail_tracking_email")
     def test_bad_signature(self):
-        self.event.update({"event": "delivered", "signature": "bad_signature"})
-        response = self.env["mail.tracking.email"].event_process(
-            None, self.event, self.metadata
-        )
-        self.assertEqual("ERROR: Signature", response)
+        self.signature = "bad_signature"
+        with self._request_mock(), self.assertRaises(NotAcceptable):
+            self.MailTrackingController.mail_tracking_mailgun_webhook()
 
     @mute_logger("odoo.addons.mail_tracking_mailgun.models.mail_tracking_email")
     def test_bad_event_type(self):
+        old_events = self.tracking_email.tracking_event_ids
         self.event.update({"event": "bad_event"})
-        response = self.env["mail.tracking.email"].event_process(
-            None, self.event, self.metadata
-        )
-        self.assertEqual("ERROR: Event type not supported", response)
-
-    @mute_logger("odoo.addons.mail_tracking_mailgun.models.mail_tracking_email")
-    def test_bad_db(self):
-        self.event.update({"event": "delivered", "odoo_db": "bad_db"})
-        response = self.env["mail.tracking.email"].event_process(
-            None, self.event, self.metadata
-        )
-        self.assertEqual("ERROR: Invalid DB", response)
+        with self._request_mock():
+            self.MailTrackingController.mail_tracking_mailgun_webhook()
+        self.assertFalse(self.tracking_email.tracking_event_ids - old_events)
 
     def test_bad_ts(self):
-        timestamp = "7a"  # Now time will be used instead
-        signature = "06cc05680f6e8110e59b41152b2d1c0f1045d755ef2880ff922344325c89a6d4"
-        self.event.update(
-            {"event": "delivered", "timestamp": timestamp, "signature": signature}
+        self.timestamp = "7a"  # Now time will be used instead
+        self.signature = (
+            "06cc05680f6e8110e59b41152b2d1c0f1045d755ef2880ff922344325c89a6d4"
         )
-        response = self.env["mail.tracking.email"].event_process(
-            None, self.event, self.metadata
-        )
-        self.assertEqual("OK", response)
+        with self._request_mock(), self.assertRaises(ValueError):
+            self.MailTrackingController.mail_tracking_mailgun_webhook()
 
     @mute_logger("odoo.addons.mail_tracking_mailgun.models.mail_tracking_email")
     def test_tracking_not_found(self):
-        self.event.update({"event": "delivered", "tracking_email_id": "bad_id"})
-        response = self.env["mail.tracking.email"].event_process(
-            None, self.event, self.metadata
+        self.event.update(
+            {
+                "event": "delivered",
+                "message": {
+                    "headers": {
+                        "to": "else@test.com",
+                        "message-id": "test-id-else@f187c54734e8",
+                        "from": "Mr. Odoo <mrodoo@odoo.com>",
+                        "subject": "This is a bad test",
+                    },
+                },
+                "user-variables": {
+                    "odoo_db": self.env.cr.dbname,
+                    "tracking_email_id": -1,
+                },
+            }
         )
-        self.assertEqual("ERROR: Tracking not found", response)
+        with self._request_mock(), self.assertRaises(MissingError):
+            self.MailTrackingController.mail_tracking_mailgun_webhook()
 
-    # https://documentation.mailgun.com/user_manual.html#tracking-deliveries
+    @mute_logger("odoo.addons.mail_tracking_mailgun.models.mail_tracking_email")
+    def test_tracking_wrong_db(self):
+        self.event["user-variables"]["odoo_db"] = "%s_nope" % self.env.cr.dbname
+        with self._request_mock(), self.assertRaises(ValidationError):
+            self.MailTrackingController.mail_tracking_mailgun_webhook()
+
+    # https://documentation.mailgun.com/en/latest/user_manual.html#tracking-deliveries
     def test_event_delivered(self):
         self.event.update({"event": "delivered"})
-        response = self.env["mail.tracking.email"].event_process(
-            None, self.event, self.metadata
-        )
-        self.assertEqual("OK", response)
+        with self._request_mock():
+            self.MailTrackingController.mail_tracking_mailgun_webhook()
         events = self.event_search("delivered")
         for event in events:
             self.assertEqual(event.timestamp, float(self.timestamp))
             self.assertEqual(event.recipient, self.recipient)
 
-    # https://documentation.mailgun.com/user_manual.html#tracking-opens
+    # https://documentation.mailgun.com/en/latest/user_manual.html#tracking-opens
     def test_event_opened(self):
         ip = "127.0.0.1"
         user_agent = "Odoo Test/8.0 Gecko Firefox/11.0"
@@ -190,10 +228,8 @@ class TestMailgun(TransactionCase):
                 "user-agent": user_agent,
             }
         )
-        response = self.env["mail.tracking.email"].event_process(
-            None, self.event, self.metadata
-        )
-        self.assertEqual("OK", response)
+        with self._request_mock():
+            self.MailTrackingController.mail_tracking_mailgun_webhook()
         event = self.event_search("open")
         self.assertEqual(event.timestamp, float(self.timestamp))
         self.assertEqual(event.recipient, self.recipient)
@@ -205,7 +241,7 @@ class TestMailgun(TransactionCase):
         self.assertEqual(event.mobile, False)
         self.assertEqual(event.user_country_id.code, "US")
 
-    # https://documentation.mailgun.com/user_manual.html#tracking-clicks
+    # https://documentation.mailgun.com/en/latest/user_manual.html#tracking-clicks
     def test_event_clicked(self):
         ip = "127.0.0.1"
         user_agent = "Odoo Test/8.0 Gecko Firefox/11.0"
@@ -228,10 +264,8 @@ class TestMailgun(TransactionCase):
                 "url": url,
             }
         )
-        response = self.env["mail.tracking.email"].event_process(
-            None, self.event, self.metadata, event_type="click"
-        )
-        self.assertEqual("OK", response)
+        with self._request_mock():
+            self.MailTrackingController.mail_tracking_mailgun_webhook()
         event = self.event_search("click")
         self.assertEqual(event.timestamp, float(self.timestamp))
         self.assertEqual(event.recipient, self.recipient)
@@ -243,7 +277,7 @@ class TestMailgun(TransactionCase):
         self.assertEqual(event.mobile, True)
         self.assertEqual(event.url, url)
 
-    # https://documentation.mailgun.com/user_manual.html#tracking-unsubscribes
+    # https://documentation.mailgun.com/en/latest/user_manual.html#tracking-unsubscribes
     def test_event_unsubscribed(self):
         ip = "127.0.0.1"
         user_agent = "Odoo Test/8.0 Gecko Firefox/11.0"
@@ -264,10 +298,8 @@ class TestMailgun(TransactionCase):
                 "user-agent": user_agent,
             }
         )
-        response = self.env["mail.tracking.email"].event_process(
-            None, self.event, self.metadata
-        )
-        self.assertEqual("OK", response)
+        with self._request_mock():
+            self.MailTrackingController.mail_tracking_mailgun_webhook()
         event = self.event_search("unsub")
         self.assertEqual(event.timestamp, float(self.timestamp))
         self.assertEqual(event.recipient, self.recipient)
@@ -278,22 +310,19 @@ class TestMailgun(TransactionCase):
         self.assertEqual(event.ua_type, ua_type)
         self.assertEqual(event.mobile, True)
 
-    # https://documentation.mailgun.com/
-    #   user_manual.html#tracking-spam-complaints
+    # https://documentation.mailgun.com/en/latest/user_manual.html#tracking-spam-complaints
     def test_event_complained(self):
         self.event.update({"event": "complained"})
-        response = self.env["mail.tracking.email"].event_process(
-            None, self.event, self.metadata
-        )
-        self.assertEqual("OK", response)
+        with self._request_mock():
+            self.MailTrackingController.mail_tracking_mailgun_webhook()
         event = self.event_search("spam")
         self.assertEqual(event.timestamp, float(self.timestamp))
         self.assertEqual(event.recipient, self.recipient)
         self.assertEqual(event.error_type, "spam")
 
-    # https://documentation.mailgun.com/user_manual.html#tracking-bounces
-    def test_event_bounced(self):
-        code = "550"
+    # https://documentation.mailgun.com/en/latest/user_manual.html#tracking-bounces
+    def test_event_failed(self):
+        code = 550
         error = (
             "5.1.1 The email account does not exist.\n"
             "5.1.1 double-checking the recipient's email address"
@@ -301,45 +330,42 @@ class TestMailgun(TransactionCase):
         notification = "Please, check recipient's email address"
         self.event.update(
             {
-                "event": "bounced",
-                "code": code,
-                "error": error,
-                "notification": notification,
+                "event": "failed",
+                "delivery-status": {
+                    "attempt-no": 1,
+                    "code": code,
+                    "description": notification,
+                    "message": error,
+                    "session-seconds": 0.0,
+                },
+                "severity": "permanent",
             }
         )
-        response = self.env["mail.tracking.email"].event_process(
-            None, self.event, self.metadata
-        )
-        self.assertEqual("OK", response)
+        with self._request_mock():
+            self.MailTrackingController.mail_tracking_mailgun_webhook()
         event = self.event_search("hard_bounce")
         self.assertEqual(event.timestamp, float(self.timestamp))
         self.assertEqual(event.recipient, self.recipient)
-        self.assertEqual(event.error_type, code)
+        self.assertEqual(event.error_type, str(code))
         self.assertEqual(event.error_description, error)
         self.assertEqual(event.error_details, notification)
 
-    # https://documentation.mailgun.com/user_manual.html#tracking-failures
-    def test_event_dropped(self):
+    def test_event_rejected(self):
         reason = "hardfail"
-        code = "605"
         description = "Not delivering to previously bounced address"
         self.event.update(
             {
-                "event": "dropped",
-                "reason": reason,
-                "code": code,
-                "description": description,
+                "event": "rejected",
+                "reject": {"reason": reason, "description": description},
             }
         )
-        response = self.env["mail.tracking.email"].event_process(
-            None, self.event, self.metadata
-        )
-        self.assertEqual("OK", response)
+        with self._request_mock():
+            self.MailTrackingController.mail_tracking_mailgun_webhook()
         event = self.event_search("reject")
         self.assertEqual(event.timestamp, float(self.timestamp))
         self.assertEqual(event.recipient, self.recipient)
-        self.assertEqual(event.error_type, reason)
-        self.assertEqual(event.error_description, code)
+        self.assertEqual(event.error_type, "rejected")
+        self.assertEqual(event.error_description, reason)
         self.assertEqual(event.error_details, description)
 
     @mock.patch(_packagepath + ".models.res_partner.requests")
@@ -420,14 +446,15 @@ class TestMailgun(TransactionCase):
         event = self.env["mail.tracking.event"].search(
             [("mailgun_id", "=", self.response["items"][0]["id"])]
         )
+        self.assertTrue(event)
         self.assertEqual(event.event_type, self.response["items"][0]["event"])
 
     @mock.patch(_packagepath + ".models.mail_tracking_email.requests")
     def test_manual_check_exceptions(self, mock_request):
         mock_request.get.return_value.status_code = 404
-        with self.assertRaises(ValidationError):
+        with self.assertRaises(UserError):
             self.tracking_email.action_manual_check_mailgun()
         mock_request.get.return_value.status_code = 200
         mock_request.get.return_value.json.return_value = {}
-        with self.assertRaises(ValidationError):
+        with self.assertRaises(UserError):
             self.tracking_email.action_manual_check_mailgun()

mail_tracking_mailgun/wizards/__init__.py

@@ -0,0 +1 @@
+from . import res_config_settings

mail_tracking_mailgun/wizards/res_config_settings.py

@@ -0,0 +1,121 @@
+# Copyright 2021 Tecnativa - Jairo Llopis
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+
+import logging
+from urllib.parse import urljoin
+
+import requests
+
+from odoo import fields, models
+
+_logger = logging.getLogger(__name__)
+
+WEBHOOK_EVENTS = (
+    "clicked",
+    "complained",
+    "delivered",
+    "opened",
+    "permanent_fail",
+    "temporary_fail",
+    "unsubscribed",
+)
+
+
+class ResConfigSettings(models.TransientModel):
+    _inherit = "res.config.settings"
+
+    mail_tracking_mailgun_enabled = fields.Boolean(
+        string="Enable mail tracking with Mailgun",
+        help="Enable to enhance mail tracking with Mailgun",
+    )
+    mail_tracking_mailgun_api_key = fields.Char(
+        string="Mailgun API key",
+        config_parameter="mailgun.apikey",
+        help="Secret API key used to authenticate with Mailgun.",
+    )
+    mail_tracking_mailgun_webhook_signing_key = fields.Char(
+        string="Mailgun webhook signing key",
+        config_parameter="mailgun.webhook_signing_key",
+        help="Secret key used to validate incoming webhooks payload.",
+    )
+    mail_tracking_mailgun_validation_key = fields.Char(
+        string="Mailgun validation key",
+        config_parameter="mailgun.validation_key",
+        help="Key used to validate emails.",
+    )
+    mail_tracking_mailgun_api_url = fields.Char(
+        string="Mailgun API endpoint",
+        config_parameter="mailgun.api_url",
+        help=(
+            "Leave this empty if your API endpoint is the default "
+            "(https://api.mailgun.net/)."
+        ),
+    )
+    mail_tracking_mailgun_domain = fields.Char(
+        string="Mailgun domain",
+        config_parameter="mailgun.domain",
+        help="Leave empty to use the catch-all domain.",
+    )
+    mail_tracking_mailgun_webhooks_domain = fields.Char(
+        string="Mailgun webhooks domain",
+        config_parameter="mailgun.webhooks_domain",
+        help="Leave empty to use the base Odoo URL.",
+    )
+    mail_tracking_mailgun_auto_check_partner_emails = fields.Boolean(
+        string="Check partner emails automatically",
+        config_parameter="mailgun.auto_check_partner_email",
+        help="Attempt to check partner emails always. This may cost money.",
+    )
+
+    def get_values(self):
+        """Is Mailgun enabled?"""
+        result = super().get_values()
+        result["mail_tracking_mailgun_enabled"] = bool(
+            self.env["ir.config_parameter"].get_param("mailgun.apikey")
+        )
+        return result
+
+    def mail_tracking_mailgun_unregister_webhooks(self):
+        """Remove existing Mailgun webhooks."""
+        params = self.env["mail.tracking.email"]._mailgun_values()
+        _logger.info("Getting current webhooks")
+        webhooks = requests.get(
+            urljoin(params.api_url, "/v3/domains/%s/webhooks" % params.domain),
+            auth=("api", params.api_key),
+        )
+        webhooks.raise_for_status()
+        for event, data in webhooks.json()["webhooks"].items():
+            # Modern webhooks return a list of URLs; old ones just one
+            urls = []
+            if "urls" in data:
+                urls.extend(data["urls"])
+            elif "url" in data:
+                urls.append(data["url"])
+            _logger.info(
+                "Deleting webhooks. Event: %s. URLs: %s", event, ", ".join(urls)
+            )
+            response = requests.delete(
+                urljoin(
+                    params.api_url,
+                    "/v3/domains/%s/webhooks/%s" % (params.domain, event),
+                ),
+                auth=("api", params.api_key),
+            )
+            response.raise_for_status()
+
+    def mail_tracking_mailgun_register_webhooks(self):
+        """Register Mailgun webhooks to get mail statuses automatically."""
+        params = self.env["mail.tracking.email"]._mailgun_values()
+        for event in WEBHOOK_EVENTS:
+            odoo_webhook = urljoin(
+                params.webhooks_domain,
+                "/mail/tracking/mailgun/all?db=%s" % self.env.cr.dbname,
+            )
+            _logger.info("Registering webhook. Event: %s. URL: %s", event, odoo_webhook)
+            response = requests.post(
+                urljoin(params.api_url, "/v3/domains/%s/webhooks" % params.domain),
+                auth=("api", params.api_key),
+                data={"id": event, "url": [odoo_webhook]},
+            )
+            # Assert correct registration
+            response.raise_for_status()

mail_tracking_mailgun/wizards/res_config_settings_views.xml

@@ -0,0 +1,145 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright 2021 Tecnativa - Jairo Llopis
+     License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). -->
+<data>
+
+    <record id="res_config_settings_view_form" model="ir.ui.view">
+        <field name="name">res.config.settings.view.form.inherit</field>
+        <field name="model">res.config.settings</field>
+        <field name="inherit_id" ref="mail.res_config_settings_view_form" />
+        <field name="arch" type="xml">
+            <div id="emails" position="inside">
+                <div id="mail_tracking_mailgun" class="col-12 o_setting_box">
+                    <div class="o_setting_left_pane">
+                        <field name="mail_tracking_mailgun_enabled" />
+                    </div>
+                    <div class="o_setting_right_pane">
+                        <label for="mail_tracking_mailgun_enabled" />
+                        <div class="text-muted">
+                            Connecting Odoo with <a
+                                href="https://www.mailgun.com/"
+                                target="_blank"
+                            >Mailgun</a> enhances Odoo's mail tracking features.
+                        </div>
+                        <div
+                            class="content-group"
+                            attrs="{'invisible': [('mail_tracking_mailgun_enabled', '=', False)]}"
+                        >
+                            <div class="row">
+                                <div class="col-12 col-lg-6">
+                                    <div class="text-muted mt16 mb4">
+                                        Obtain keys in <a
+                                            href="https://app.mailgun.com/app/account/security/api_keys"
+                                            target="_blank"
+                                        >Mailgun &gt; Settings &gt; API keys</a>:
+                                    </div>
+                                    <div class="row mt16">
+                                        <label
+                                            for="mail_tracking_mailgun_api_key"
+                                            class="col-lg-3 o_light_label"
+                                        />
+                                        <field
+                                            name="mail_tracking_mailgun_api_key"
+                                            password="True"
+                                            placeholder="key-abcde0123456789abcde0123456789ab"
+                                            attrs="{'required': [('mail_tracking_mailgun_enabled', '=', True)]}"
+                                        />
+                                    </div>
+                                    <div class="row mt16">
+                                        <label
+                                            for="mail_tracking_mailgun_webhook_signing_key"
+                                            class="col-lg-3 o_light_label"
+                                        />
+                                        <field
+                                            name="mail_tracking_mailgun_webhook_signing_key"
+                                            password="True"
+                                            placeholder="abcde0123456789abcde0123456789ab"
+                                        />
+                                    </div>
+                                    <div class="row mt16">
+                                        <label
+                                            for="mail_tracking_mailgun_validation_key"
+                                            class="col-lg-3 o_light_label"
+                                        />
+                                        <field
+                                            name="mail_tracking_mailgun_validation_key"
+                                            password="True"
+                                            placeholder="pubkey-abcde0123456789abcde0123456789ab"
+                                            attrs="{'required': [('mail_tracking_mailgun_auto_check_partner_emails', '=', True)]}"
+                                        />
+                                    </div>
+                                </div>
+                                <div class="col-12 col-lg-6">
+                                    <div class="text-muted mt16 mb4">
+                                        Other settings:
+                                    </div>
+                                    <div class="mt16">
+                                        <field
+                                            name="mail_tracking_mailgun_auto_check_partner_emails"
+                                            class="oe_inline"
+                                        />
+                                        <label
+                                            for="mail_tracking_mailgun_auto_check_partner_emails"
+                                            class="o_light_label"
+                                        />
+                                    </div>
+                                    <div class="row mt16">
+                                        <label
+                                            for="mail_tracking_mailgun_domain"
+                                            class="col-lg-3 o_light_label"
+                                        />
+                                        <field
+                                            name="mail_tracking_mailgun_domain"
+                                            placeholder="odoo.example.com"
+                                        />
+                                    </div>
+                                    <div class="row mt16">
+                                        <label
+                                            for="mail_tracking_mailgun_api_url"
+                                            class="col-lg-3 o_light_label"
+                                        />
+                                        <field
+                                            name="mail_tracking_mailgun_api_url"
+                                            placeholder="https://api.mailgun.net"
+                                        />
+                                    </div>
+                                    <div class="row mt16">
+                                        <label
+                                            for="mail_tracking_mailgun_webhooks_domain"
+                                            class="col-lg-3 o_light_label"
+                                        />
+                                        <field
+                                            name="mail_tracking_mailgun_webhooks_domain"
+                                            placeholder="https://odoo.example.com"
+                                        />
+                                    </div>
+                                </div>
+                                <div class="col-12">
+                                    <div class="text-muted mt16 mb4">
+                                        If you change Mailgun settings, your Odoo URL or your sending domain, unregister webhooks and register them again to get automatic updates about sent emails:
+                                    </div>
+                                    <button
+                                        type="object"
+                                        name="mail_tracking_mailgun_unregister_webhooks"
+                                        string="Unregister Mailgun webhooks"
+                                        icon="fa-arrow-right"
+                                        class="btn-link"
+                                        confirm="This will unregister ALL webhooks from Mailgun, which can include webhooks for other apps."
+                                    />
+                                    <button
+                                        type="object"
+                                        name="mail_tracking_mailgun_register_webhooks"
+                                        string="Register Mailgun webhooks"
+                                        icon="fa-arrow-right"
+                                        class="btn-link"
+                                    />
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </field>
+    </record>
+
+</data>