From 17b4196e2eae6a632ccc2f501aa9950f88beef0b Mon Sep 17 00:00:00 2001 From: SiteRelEnby <125829806+SiteRelEnby@users.noreply.github.com> Date: Fri, 12 Jun 2026 18:16:32 -0400 Subject: [PATCH] Add SMTP2GO delivery webhook Feeds SMTP2GO's delivery/bounce/spam events into the same deliverability lifecycle as the SES and SendGrid handlers, so bounce suppression and soft-bounce self-healing work when sending via SMTP2GO over the smtp backend. Event mapping (pure, unit-tested): delivered clears transient soft-bounce state; bounce maps SMTP2GO's hard/soft classification, with an unclassified bounce defaulting to soft (conservative - soft doesn't block until the threshold, so a misclassification can't lock anyone out); spam is a complaint. reject is ignored - SMTP2GO emits it when refusing an already-flagged address, so it carries no new state. SMTP2GO doesn't sign payloads, so the endpoint is guarded by a shared secret in the URL (SMTP2GO_WEBHOOK_SECRET; 404 when unset, constant-time compared). Accepts either webhook output type - JSON (single object or array) or form-encoded (single event) - so a webhook left on the wrong output type still works instead of silently rejecting every event. Recipient is read from SMTP2GO's `rcpt` field. --- CHANGELOG.md | 6 ++ docs/SELFHOSTING.md | 19 ++++- sheaf/api/v1/webhooks.py | 133 +++++++++++++++++++++++++++++++++ sheaf/config.py | 10 +++ tests/test_webhooks_smtp2go.py | 98 ++++++++++++++++++++++++ 5 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 tests/test_webhooks_smtp2go.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 68bce91..a6e6c5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to Sheaf are documented here. The format is based on [Keep a `v1.0.0` is the first stable release. The `v0.x.y` releases were betas; from 1.0 on, the v1 API and database schema carry semver compatibility guarantees. +## [Unreleased] + +### Added + +- **SMTP2GO delivery webhook.** A new `POST /v1/webhooks/smtp2go/events` endpoint feeds SMTP2GO's delivery/bounce/spam events into the same deliverability lifecycle as the SES and SendGrid handlers, so bounce suppression and soft-bounce self-healing work when sending via SMTP2GO (over the `smtp` backend). `delivered` clears transient soft-bounce state, `bounce` maps hard/soft from SMTP2GO's classification (an unclassified bounce defaults to soft, the conservative choice), and `spam` is treated as a complaint. SMTP2GO does not sign payloads, so the endpoint is guarded by a shared secret in the URL (`SMTP2GO_WEBHOOK_SECRET`; returns 404 when unset) - point the SMTP2GO webhook at `/v1/webhooks/smtp2go/events?token=` (JSON or form-encoded output both accepted) and enable at least the Delivered, Bounce, and Spam events. See SELFHOSTING for setup. + ## [1.0.0] - 2026-06-12 ### Added diff --git a/docs/SELFHOSTING.md b/docs/SELFHOSTING.md index 9abab06..5521621 100644 --- a/docs/SELFHOSTING.md +++ b/docs/SELFHOSTING.md @@ -178,6 +178,23 @@ Port 465 uses implicit TLS; all other ports use STARTTLS (when `SMTP_TLS=true`). **Requires the `smtp` extra** — see [Optional dependencies](#optional-dependencies). +Plain SMTP has no feedback channel, so by default nothing flags a bounced or complained address - the [deliverability gate](#deliverability-state) simply never trips. If your SMTP provider offers a webhook, wire it up to keep bounce suppression working (see SMTP2GO below). + +#### SMTP2GO bounce/complaint feedback + +If you send via SMTP2GO (`EMAIL_BACKEND=smtp` pointed at SMTP2GO's relay), add a webhook so delivery/bounce/spam events drive the deliverability state. SMTP2GO does not sign payloads, so the endpoint is guarded by a shared secret in the URL. Set it in `.env`: + +```env +SMTP2GO_WEBHOOK_SECRET=your-random-secret-here +``` + +Then in the SMTP2GO app (Settings -> Webhooks), add a webhook with: +- **URL:** `https://your-instance/v1/webhooks/smtp2go/events?token=your-random-secret-here` +- **Output type:** JSON or Form-encoded (both accepted) +- **Events:** at minimum Delivered, Bounce, and Spam. Delivered is what lets a greylisted first attempt self-heal once the retry lands - see [Deliverability state](#deliverability-state). + +When `SMTP2GO_WEBHOOK_SECRET` is empty the endpoint returns 404. For defence in depth, also restrict the endpoint to SMTP2GO's published source IPs (`webhooks.smtp2go.com`) at your reverse proxy, since the secret travels in the URL. + ### AWS SES ```env @@ -237,7 +254,7 @@ Bounce and complaint feedback (SES queue or SendGrid webhook above) drives a per - **Users are never silently locked out.** When an address is flagged, the user sees a banner on sign-in prompting them to re-verify or change their email. Re-verifying (the verification email is sent even to a currently-blocked address) clears the block. No admin intervention or manual database edit is required. -If you run plain SMTP (`EMAIL_BACKEND=smtp`) with no bounce-feedback channel wired up, no addresses are ever auto-flagged - the deliverability gate simply never trips. Wire your provider's webhook/feedback into the SES or SendGrid handlers if you want bounce suppression on SMTP. +Bounce/complaint feedback comes from a provider webhook or queue: the SES SQS handler, the SendGrid Event Webhook, or the SMTP2GO webhook (see [SMTP2GO bounce/complaint feedback](#smtp2go-bouncecomplaint-feedback)). If you run plain SMTP with no such channel wired up, no addresses are ever auto-flagged - the deliverability gate simply never trips, which is a safe (if unfiltered) default. ### Disabling email diff --git a/sheaf/api/v1/webhooks.py b/sheaf/api/v1/webhooks.py index 9bdfa59..2f58e90 100644 --- a/sheaf/api/v1/webhooks.py +++ b/sheaf/api/v1/webhooks.py @@ -1,9 +1,11 @@ """Webhook endpoints for third-party service callbacks.""" import base64 +import json import logging import secrets import time +from urllib.parse import parse_qs from cryptography.exceptions import InvalidSignature from cryptography.hazmat.primitives import hashes @@ -176,3 +178,134 @@ async def sendgrid_events( await db.commit() logger.info("Processed %d SendGrid events from batch of %d", processed, len(events)) return {"processed": processed} + + +# SMTP2GO event -> deliverability action. Pure mapping, unit-tested +# separately from the endpoint. Returns one of "hard_bounce", +# "soft_bounce", "complaint", "delivered", or None (event we don't act +# on: processed / open / click / unsubscribe / resubscribe / reject). +# +# `reject` is deliberately ignored: SMTP2GO emits it when it refuses to +# send to an address that ALREADY hard-bounced/complained/unsubscribed, +# so it carries no new state - the address is already flagged. An +# unclassified bounce defaults to soft, the conservative choice: soft +# bounces don't block until the threshold, so a misclassification can't +# wrongly lock anyone out. +def parse_smtp2go_payload(raw: bytes, content_type: str) -> list[dict]: + """Parse an SMTP2GO webhook body into a list of event dicts. + + SMTP2GO's output type is operator-configurable: JSON or form-encoded + (`application/x-www-form-urlencoded`). We accept either, so a webhook + left on the wrong output type still works instead of silently 400ing + every event. JSON may be a single object or an array; form-encoded is + always a single event. Raises ValueError on anything we can't read as + event dict(s). + """ + text = raw.decode("utf-8", "replace").strip() + if not text: + raise ValueError("empty body") + # JSON if declared as such, or if the body opens like JSON. + if "json" in content_type.lower() or text[:1] in ("{", "["): + data = json.loads(text) # raises ValueError (JSONDecodeError) on junk + if isinstance(data, list): + return [e for e in data if isinstance(e, dict)] + if isinstance(data, dict): + return [data] + raise ValueError("unexpected JSON shape") + # Form-encoded fallback: one event as key=value pairs. parse_qs keeps + # blank values and never raises; an empty result means it wasn't form + # data either. + parsed = parse_qs(text, keep_blank_values=True) + if not parsed: + raise ValueError("not JSON or form-encoded") + return [{k: v[0] for k, v in parsed.items()}] + + +def smtp2go_event_action(event_type: str, bounce_kind: str | None) -> str | None: + if event_type == "delivered": + return "delivered" + if event_type == "bounce": + return "hard_bounce" if bounce_kind == "hard" else "soft_bounce" + if event_type == "spam": + return "complaint" + return None + + +@router.post("/smtp2go/events", dependencies=[rate_limit(300, 60)]) +async def smtp2go_events( + request: Request, + db: AsyncSession = Depends(get_db), +): + """Receive SMTP2GO webhook callbacks for delivery/bounce/spam events. + + SMTP2GO does not sign payloads (no HMAC), so the endpoint is guarded + by a shared secret in the URL: configure the SMTP2GO webhook to POST + to /v1/webhooks/smtp2go/events?token= with JSON output. + When no secret is configured the endpoint returns 404. Feeds the + same deliverability lifecycle (apply_bounce / apply_complaint / + apply_delivered) as the SES and SendGrid handlers. + + Accepts either webhook output type: JSON (single object or array) or + form-encoded (a single event). SMTP2GO usually posts one event per + request. + """ + if not settings.smtp2go_webhook_secret: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + + token = request.query_params.get("token", "") + if not secrets.compare_digest(token, settings.smtp2go_webhook_secret): + webhook_signature_failures_total.labels(endpoint="smtp2go").inc() + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN) + + from sheaf.services.email_events import ( + apply_bounce, + apply_complaint, + apply_delivered, + ) + + raw_body = await request.body() + try: + events = parse_smtp2go_payload( + raw_body, request.headers.get("content-type", "") + ) + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Request body is not valid JSON or form-encoded data", + ) from None + + processed = 0 + for event in events: + if not isinstance(event, dict): + continue + event_type = event.get("event") + # SMTP2GO names the recipient `rcpt` (not `email` like SendGrid). + email = event.get("rcpt") or event.get("email") + if not email or not event_type: + continue + + action = smtp2go_event_action(event_type, event.get("bounce")) + if action is None: + continue + email_provider_events_total.labels( + provider="smtp2go", event=event_type, + ).inc() + + try: + if action == "hard_bounce": + if await apply_bounce(db, email, permanent=True): + processed += 1 + elif action == "soft_bounce": + if await apply_bounce(db, email, permanent=False): + processed += 1 + elif action == "complaint": + if await apply_complaint(db, email): + processed += 1 + elif action == "delivered" and await apply_delivered(db, email): + processed += 1 + except Exception: + logger.exception("Failed to process SMTP2GO event: %s", event_type) + + await db.commit() + logger.info("Processed %d SMTP2GO events from batch of %d", processed, len(events)) + return {"processed": processed} diff --git a/sheaf/config.py b/sheaf/config.py index 8ae9968..d31b73a 100644 --- a/sheaf/config.py +++ b/sheaf/config.py @@ -220,6 +220,16 @@ class Settings(BaseSettings): # as a possible replay. sendgrid_webhook_max_skew_seconds: int = 600 + # SMTP2GO webhook. SMTP2GO does not sign payloads (no HMAC), so the + # endpoint is guarded by a shared secret in the URL: configure the + # SMTP2GO webhook to POST to + # /v1/webhooks/smtp2go/events?token= with JSON output. + # When empty, the webhook endpoint returns 404. Pair with SMTP2GO's + # IP allowlist (webhooks.smtp2go.com) at the proxy for defence in + # depth. Feeds the same deliverability lifecycle as the SES/SendGrid + # handlers. + smtp2go_webhook_secret: str = "" + # Registration registration_mode: str = "open" # "open", "approval", "invite", "closed" invite_codes_enabled: bool = False # Accept invite codes in open/approval modes too diff --git a/tests/test_webhooks_smtp2go.py b/tests/test_webhooks_smtp2go.py new file mode 100644 index 0000000..c825a44 --- /dev/null +++ b/tests/test_webhooks_smtp2go.py @@ -0,0 +1,98 @@ +"""Unit tests for the SMTP2GO webhook event mapping. + +SMTP2GO doesn't sign payloads, so there's no crypto to verify (the +endpoint guards on a shared URL secret). The bug-prone part is the +event -> deliverability-action mapping, which is a pure function tested +here. The apply_* state transitions it dispatches to are covered in +test_email_events.py. +""" + +import pytest + +from sheaf.api.v1.webhooks import parse_smtp2go_payload, smtp2go_event_action + + +def test_delivered_maps_to_delivered(): + assert smtp2go_event_action("delivered", None) == "delivered" + + +def test_hard_bounce_maps_to_hard(): + assert smtp2go_event_action("bounce", "hard") == "hard_bounce" + + +def test_soft_bounce_maps_to_soft(): + assert smtp2go_event_action("bounce", "soft") == "soft_bounce" + + +def test_unclassified_bounce_defaults_to_soft(): + # Conservative: an unknown/missing bounce classification must not + # immediately hard-block. Soft only blocks past the threshold. + assert smtp2go_event_action("bounce", None) == "soft_bounce" + assert smtp2go_event_action("bounce", "weird") == "soft_bounce" + + +def test_spam_maps_to_complaint(): + assert smtp2go_event_action("spam", None) == "complaint" + + +def test_reject_is_ignored(): + # SMTP2GO emits `reject` when refusing to send to an already-flagged + # address - no new state, so no action. + assert smtp2go_event_action("reject", None) is None + + +def test_non_actionable_events_ignored(): + for ev in ("processed", "open", "click", "unsubscribe", "resubscribe"): + assert smtp2go_event_action(ev, None) is None + + +def test_unknown_event_ignored(): + assert smtp2go_event_action("totally_made_up", None) is None + assert smtp2go_event_action("", None) is None + + +# --- payload parsing (JSON or form-encoded, operator-configurable) --------- + + +def test_parse_json_single_object(): + body = b'{"event":"bounce","rcpt":"x@example.com","bounce":"hard"}' + events = parse_smtp2go_payload(body, "application/json") + assert events == [ + {"event": "bounce", "rcpt": "x@example.com", "bounce": "hard"} + ] + + +def test_parse_json_array(): + body = b'[{"event":"delivered","rcpt":"a@x.com"},{"event":"spam","rcpt":"b@x.com"}]' + events = parse_smtp2go_payload(body, "application/json") + assert len(events) == 2 + assert events[0]["event"] == "delivered" + assert events[1]["event"] == "spam" + + +def test_parse_json_without_content_type_sniffed_by_shape(): + # SMTP2GO doesn't always set a JSON content-type; a body that opens + # like JSON is still parsed as JSON. + body = b'{"event":"delivered","rcpt":"a@x.com"}' + events = parse_smtp2go_payload(body, "") + assert events[0]["event"] == "delivered" + + +def test_parse_form_encoded_single_event(): + body = b"event=bounce&rcpt=x%40example.com&bounce=soft" + events = parse_smtp2go_payload( + body, "application/x-www-form-urlencoded" + ) + assert events == [ + {"event": "bounce", "rcpt": "x@example.com", "bounce": "soft"} + ] + + +def test_parse_empty_body_raises(): + with pytest.raises(ValueError): + parse_smtp2go_payload(b" ", "application/json") + + +def test_parse_garbage_json_raises(): + with pytest.raises(ValueError): + parse_smtp2go_payload(b"{not valid json", "application/json")