From cc17c20f2e5d3176651643e2eac0eb84093c39ac Mon Sep 17 00:00:00 2001 From: Lio Date: Mon, 9 Mar 2026 00:11:45 +0100 Subject: [PATCH 01/15] Add global sender blacklist (regex) with admin UI and SMTP blocking --- app/admin/__init__.py | 2 + app/admin/global_sender_blacklist.py | 15 +++++ app/admin/index.py | 3 + app/models.py | 18 ++++++ app/regex_utils.py | 20 ++++++- app/sender_blacklist.py | 55 +++++++++++++++++++ email_handler.py | 13 ++++- ...08_b7c1d6a4f2e1_global_sender_blacklist.py | 40 ++++++++++++++ tests/test_email_handler.py | 22 ++++++++ 9 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 app/admin/global_sender_blacklist.py create mode 100644 app/sender_blacklist.py create mode 100644 migrations/versions/2026_0308_b7c1d6a4f2e1_global_sender_blacklist.py diff --git a/app/admin/__init__.py b/app/admin/__init__.py index 7d434aa5a..3a3cd2445 100644 --- a/app/admin/__init__.py +++ b/app/admin/__init__.py @@ -16,6 +16,7 @@ from app.admin.metrics import DailyMetricAdmin, MetricAdmin from app.admin.invalid_mailbox_domain import InvalidMailboxDomainAdmin from app.admin.forbidden_mx_ip import ForbiddenMxIpAdmin +from app.admin.global_sender_blacklist import GlobalSenderBlacklistAdmin from app.admin.email_search import ( EmailSearchResult, EmailSearchHelpers, @@ -52,6 +53,7 @@ "MetricAdmin", "InvalidMailboxDomainAdmin", "ForbiddenMxIpAdmin", + "GlobalSenderBlacklistAdmin", # Search views "EmailSearchResult", "EmailSearchHelpers", diff --git a/app/admin/global_sender_blacklist.py b/app/admin/global_sender_blacklist.py new file mode 100644 index 000000000..cc1ec1c36 --- /dev/null +++ b/app/admin/global_sender_blacklist.py @@ -0,0 +1,15 @@ +from flask_admin.form import SecureForm + +from app.admin.base import SLModelView + + +class GlobalSenderBlacklistAdmin(SLModelView): + form_base_class = SecureForm + + can_create = True + can_edit = True + can_delete = True + + column_searchable_list = ("pattern", "comment") + column_filters = ("enabled",) + column_editable_list = ("enabled", "comment") diff --git a/app/admin/index.py b/app/admin/index.py index 46cb6780f..ec1e04962 100644 --- a/app/admin/index.py +++ b/app/admin/index.py @@ -17,6 +17,7 @@ Metric2, InvalidMailboxDomain, ForbiddenMxIp, + GlobalSenderBlacklist, ) from app.admin.base import SLAdminIndexView from app.admin.user import UserAdmin @@ -31,6 +32,7 @@ from app.admin.metrics import DailyMetricAdmin, MetricAdmin from app.admin.invalid_mailbox_domain import InvalidMailboxDomainAdmin from app.admin.forbidden_mx_ip import ForbiddenMxIpAdmin +from app.admin.global_sender_blacklist import GlobalSenderBlacklistAdmin from app.admin.email_search import EmailSearchAdmin from app.admin.custom_domain_search import CustomDomainSearchAdmin from app.admin.abuser_lookup import AbuserLookupAdmin @@ -63,3 +65,4 @@ def init_admin(app: Flask): admin.add_view(MetricAdmin(Metric2, Session)) admin.add_view(InvalidMailboxDomainAdmin(InvalidMailboxDomain, Session)) admin.add_view(ForbiddenMxIpAdmin(ForbiddenMxIp, Session)) + admin.add_view(GlobalSenderBlacklistAdmin(GlobalSenderBlacklist, Session)) diff --git a/app/models.py b/app/models.py index 581b52f6b..13c091b2f 100644 --- a/app/models.py +++ b/app/models.py @@ -3672,6 +3672,24 @@ class ForbiddenMxIp(Base, ModelMixin): comment = sa.Column(sa.Text, unique=False, nullable=True) +class GlobalSenderBlacklist(Base, ModelMixin): + """Global blacklist for inbound senders (envelope MAIL FROM). + + Pattern is a (re2-compatible) regex that is applied via search() against the + full envelope sender address. + + Examples: + - "@spamdomain\\.com$" + - "^no-?reply@.*" + """ + + __tablename__ = "global_sender_blacklist" + + pattern = sa.Column(sa.String(512), unique=True, nullable=False) + enabled = sa.Column(sa.Boolean, nullable=False, default=True, server_default="1") + comment = sa.Column(sa.Text, nullable=True) + + # region Phone class PhoneCountry(Base, ModelMixin): __tablename__ = "phone_country" diff --git a/app/regex_utils.py b/app/regex_utils.py index e32413338..788103126 100644 --- a/app/regex_utils.py +++ b/app/regex_utils.py @@ -5,7 +5,8 @@ from app.log import LOG -def regex_match(rule_regex: str, local): +def regex_match(rule_regex: str, local) -> bool: + """Return True if *full string* matches rule_regex.""" regex = re2.compile(rule_regex) try: if re2.fullmatch(regex, local): @@ -16,3 +17,20 @@ def regex_match(rule_regex: str, local): if re.fullmatch(regex, local): return True return False + + +def regex_search(rule_regex: str, text: str) -> bool: + """Return True if any substring of text matches rule_regex. + + Uses re2 when possible to avoid catastrophic backtracking. + """ + regex = re2.compile(rule_regex) + try: + if re2.search(regex, text): + return True + except TypeError: # re2 bug "Argument 'pattern' has incorrect type (expected bytes, got PythonRePattern)" + LOG.w("use re instead of re2 for %s %s", rule_regex, text) + regex = re.compile(rule_regex) + if re.search(regex, text): + return True + return False diff --git a/app/sender_blacklist.py b/app/sender_blacklist.py new file mode 100644 index 000000000..40b52805c --- /dev/null +++ b/app/sender_blacklist.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from cachetools import TTLCache, cached + +from app.db import Session +from app.log import LOG +from app.models import GlobalSenderBlacklist +from app.regex_utils import regex_search + + +# Cache enabled patterns briefly to avoid a DB query per inbound email. +# Admin changes should take effect quickly but don't need to be instant. +@cached(cache=TTLCache(maxsize=1, ttl=30)) +def _get_enabled_patterns() -> list[str]: + return [ + r.pattern + for r in Session.query(GlobalSenderBlacklist) + .filter(GlobalSenderBlacklist.enabled.is_(True)) + .order_by(GlobalSenderBlacklist.id.asc()) + .all() + ] + + +def is_sender_globally_blocked(*candidates: str) -> bool: + """Return True if any candidate sender string matches the global blacklist. + + Typical candidates: + - SMTP envelope MAIL FROM + - parsed header From address + """ + + patterns = _get_enabled_patterns() + if not patterns: + return False + + for candidate in candidates: + if not candidate: + continue + # Ignore bounce/null reverse-path + if candidate == "<>": + continue + + for pattern in patterns: + try: + if regex_search(pattern, candidate): + return True + except Exception: + # Never crash the SMTP handler because of a bad regex. + LOG.exception( + "Global sender blacklist regex failed: pattern=%s candidate=%s", + pattern, + candidate, + ) + + return False diff --git a/email_handler.py b/email_handler.py index 214887c7a..aa40af7e9 100644 --- a/email_handler.py +++ b/email_handler.py @@ -153,6 +153,7 @@ from app.handler.unsubscribe_generator import UnsubscribeGenerator from app.handler.unsubscribe_handler import UnsubscribeHandler from app.log import LOG, set_message_id +from app.sender_blacklist import is_sender_globally_blocked from app.mail_sender import sl_sendmail from app.mailbox_utils import ( get_mailbox_for_reply_phase, @@ -658,7 +659,9 @@ def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str ) return [(True, status.E502)] - if not alias.enabled or alias.is_trashed() or contact.block_forward: + sender_blocked = is_sender_globally_blocked(envelope.mail_from, contact.website_email) + + if not alias.enabled or alias.is_trashed() or contact.block_forward or sender_blocked: if not alias.enabled: LOG.d("%s is disabled, do not forward", alias) @@ -668,6 +671,14 @@ def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str if contact.block_forward: LOG.d("Contact %s of alias %s is blocked, do not forward", contact, alias) + if sender_blocked: + LOG.i( + "Sender is blocked by global sender blacklist: mail_from=%s header_from=%s alias=%s", + envelope.mail_from, + contact.website_email, + alias, + ) + EmailLog.create( contact_id=contact.id, user_id=contact.user_id, diff --git a/migrations/versions/2026_0308_b7c1d6a4f2e1_global_sender_blacklist.py b/migrations/versions/2026_0308_b7c1d6a4f2e1_global_sender_blacklist.py new file mode 100644 index 000000000..c161a2a3e --- /dev/null +++ b/migrations/versions/2026_0308_b7c1d6a4f2e1_global_sender_blacklist.py @@ -0,0 +1,40 @@ +"""Add global sender blacklist + +Revision ID: b7c1d6a4f2e1 +Revises: 3ee37864eb67 +Create Date: 2026-03-08 + +""" + +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "b7c1d6a4f2e1" +down_revision = "3ee37864eb67" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "global_sender_blacklist", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column( + "created_at", sqlalchemy_utils.types.arrow.ArrowType(), nullable=False + ), + sa.Column("updated_at", sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column("pattern", sa.String(length=512), nullable=False), + sa.Column( + "enabled", sa.Boolean(), server_default=sa.text("true"), nullable=False + ), + sa.Column("comment", sa.Text(), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("pattern"), + ) + + +def downgrade(): + op.drop_table("global_sender_blacklist") diff --git a/tests/test_email_handler.py b/tests/test_email_handler.py index e12efb2f1..e41fe065b 100644 --- a/tests/test_email_handler.py +++ b/tests/test_email_handler.py @@ -21,6 +21,7 @@ VerpType, Contact, SentAlert, + GlobalSenderBlacklist, ) from app.utils import random_string, canonicalize_email from email_handler import ( @@ -40,6 +41,27 @@ def test_should_ignore(flask_client): assert should_ignore("mail_from", ["rcpt_to"]) +def test_global_sender_blacklist_blocks(flask_client): + user = create_new_user() + alias = Alias.create_new_random(user) + + # Block all senders from spam.test + GlobalSenderBlacklist.create(pattern=r"@spam\\.test$", enabled=True, commit=True) + + msg = EmailMessage() + msg[headers.FROM] = "Bad Guy " + msg[headers.TO] = alias.email + msg[headers.SUBJECT] = "hello" + msg.set_content("test") + + envelope = Envelope() + envelope.mail_from = "bad@spam.test" + envelope.rcpt_tos = [alias.email] + + result = email_handler.handle(envelope, msg) + assert result == status.E200 + + def test_is_automatic_out_of_office(): msg = EmailMessage() assert not is_automatic_out_of_office(msg) From 90bd0cb9a410c96ab75b4b5e32429d4147852835 Mon Sep 17 00:00:00 2001 From: Lio Date: Tue, 17 Mar 2026 21:37:01 +0100 Subject: [PATCH 02/15] Blacklist: only apply when no Contact exists; create disabled contact --- app/contact_utils.py | 2 ++ email_handler.py | 56 +++++++++++++++++++++++++++++-------- tests/test_email_handler.py | 49 ++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 11 deletions(-) diff --git a/app/contact_utils.py b/app/contact_utils.py index 9d452e41f..529e59e9d 100644 --- a/app/contact_utils.py +++ b/app/contact_utils.py @@ -47,6 +47,7 @@ def create_contact( mail_from: Optional[str] = None, allow_empty_email: bool = False, automatic_created: bool = False, + block_forward: bool = False, from_partner: bool = False, ) -> ContactCreateResult: LOG.i( @@ -105,6 +106,7 @@ def create_contact( automatic_created=automatic_created, flags=flags, invalid_email=is_invalid_email, + block_forward=block_forward, commit=True, ) contact_id = contact.id diff --git a/email_handler.py b/email_handler.py index aa40af7e9..f2842ff1a 100644 --- a/email_handler.py +++ b/email_handler.py @@ -214,6 +214,49 @@ def get_or_create_contact( mail_from, ) contact_email = mail_from + + # Decide whether we already have a matching contact BEFORE applying the global sender blacklist. + # This allows users to whitelist a specific sender by manually creating/enabling a Contact. + sanitized_contact_email = sanitize_email(contact_email, not_lower=True) + existing_contact = Contact.get_by(alias_id=alias.id, website_email=sanitized_contact_email) + if existing_contact is not None: + # Still update name/mail_from if needed (create_contact handles that). + contact_result = contact_utils.create_contact( + email=contact_email, + alias=alias, + name=contact_name, + mail_from=mail_from, + allow_empty_email=True, + automatic_created=True, + from_partner=False, + ) + if contact_result.error: + LOG.w(f"Error creating contact: {contact_result.error.value}") + return contact_result.contact + + # No existing contact: only now consult the global blacklist. + # If matched, create a disabled Contact; the existing block_forward logic will refuse the email. + if is_sender_globally_blocked(mail_from, sanitized_contact_email): + LOG.i( + "Sender matched global sender blacklist; creating disabled contact: mail_from=%s header_from=%s alias=%s", + mail_from, + sanitized_contact_email, + alias, + ) + contact_result = contact_utils.create_contact( + email=contact_email, + alias=alias, + name=contact_name, + mail_from=mail_from, + allow_empty_email=True, + automatic_created=True, + block_forward=True, + from_partner=False, + ) + if contact_result.error: + LOG.w(f"Error creating contact: {contact_result.error.value}") + return contact_result.contact + contact_result = contact_utils.create_contact( email=contact_email, alias=alias, @@ -221,6 +264,7 @@ def get_or_create_contact( mail_from=mail_from, allow_empty_email=True, automatic_created=True, + block_forward=False, from_partner=False, ) if contact_result.error: @@ -659,9 +703,7 @@ def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str ) return [(True, status.E502)] - sender_blocked = is_sender_globally_blocked(envelope.mail_from, contact.website_email) - - if not alias.enabled or alias.is_trashed() or contact.block_forward or sender_blocked: + if not alias.enabled or alias.is_trashed() or contact.block_forward: if not alias.enabled: LOG.d("%s is disabled, do not forward", alias) @@ -671,14 +713,6 @@ def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str if contact.block_forward: LOG.d("Contact %s of alias %s is blocked, do not forward", contact, alias) - if sender_blocked: - LOG.i( - "Sender is blocked by global sender blacklist: mail_from=%s header_from=%s alias=%s", - envelope.mail_from, - contact.website_email, - alias, - ) - EmailLog.create( contact_id=contact.id, user_id=contact.user_id, diff --git a/tests/test_email_handler.py b/tests/test_email_handler.py index e41fe065b..34c435e64 100644 --- a/tests/test_email_handler.py +++ b/tests/test_email_handler.py @@ -61,6 +61,55 @@ def test_global_sender_blacklist_blocks(flask_client): result = email_handler.handle(envelope, msg) assert result == status.E200 + email_logs = ( + EmailLog.filter_by(user_id=user.id, alias_id=alias.id) + .order_by(EmailLog.id.desc()) + .all() + ) + assert len(email_logs) == 1 + assert email_logs[0].blocked + + +def test_global_sender_blacklist_not_applied_when_contact_exists(flask_client): + user = create_new_user() + alias = Alias.create_new_random(user) + + # Create a manual/previous contact: this should act as a whitelist entry. + contact = Contact.create( + user_id=user.id, + alias_id=alias.id, + website_email="bad@spam.test", + name="Bad Guy", + reply_email=f"{random_string(8)}@{EMAIL_DOMAIN}", + block_forward=False, + commit=True, + ) + assert contact is not None + + # Now enable a global blacklist that would match this sender. + GlobalSenderBlacklist.create(pattern=r"@spam\\.test$", enabled=True, commit=True) + + msg = EmailMessage() + msg[headers.FROM] = "Bad Guy " + msg[headers.TO] = alias.email + msg[headers.SUBJECT] = "hello" + msg.set_content("test") + + envelope = Envelope() + envelope.mail_from = "bad@spam.test" + envelope.rcpt_tos = [alias.email] + + result = email_handler.handle(envelope, msg) + assert result == status.E200 + + email_logs = ( + EmailLog.filter_by(user_id=user.id, alias_id=alias.id) + .order_by(EmailLog.id.desc()) + .all() + ) + assert len(email_logs) == 1 + assert email_logs[0].blocked is False + def test_is_automatic_out_of_office(): msg = EmailMessage() From 2b8f535e799ab853978d0891f59dc45dbfe41f85 Mon Sep 17 00:00:00 2001 From: Lio Date: Tue, 17 Mar 2026 22:17:16 +0100 Subject: [PATCH 03/15] Refactor contact creation + add pattern help text --- app/admin/global_sender_blacklist.py | 7 ++++ email_handler.py | 52 ++++++++-------------------- 2 files changed, 22 insertions(+), 37 deletions(-) diff --git a/app/admin/global_sender_blacklist.py b/app/admin/global_sender_blacklist.py index cc1ec1c36..e3cbc6770 100644 --- a/app/admin/global_sender_blacklist.py +++ b/app/admin/global_sender_blacklist.py @@ -13,3 +13,10 @@ class GlobalSenderBlacklistAdmin(SLModelView): column_searchable_list = ("pattern", "comment") column_filters = ("enabled",) column_editable_list = ("enabled", "comment") + + # Help text for admins when adding patterns + form_args = { + "pattern": { + "description": r"Regex, i.e. `@domain\.com`", + } + } diff --git a/email_handler.py b/email_handler.py index f2842ff1a..5776c2b88 100644 --- a/email_handler.py +++ b/email_handler.py @@ -218,44 +218,22 @@ def get_or_create_contact( # Decide whether we already have a matching contact BEFORE applying the global sender blacklist. # This allows users to whitelist a specific sender by manually creating/enabling a Contact. sanitized_contact_email = sanitize_email(contact_email, not_lower=True) - existing_contact = Contact.get_by(alias_id=alias.id, website_email=sanitized_contact_email) - if existing_contact is not None: - # Still update name/mail_from if needed (create_contact handles that). - contact_result = contact_utils.create_contact( - email=contact_email, - alias=alias, - name=contact_name, - mail_from=mail_from, - allow_empty_email=True, - automatic_created=True, - from_partner=False, - ) - if contact_result.error: - LOG.w(f"Error creating contact: {contact_result.error.value}") - return contact_result.contact + existing_contact = Contact.get_by( + alias_id=alias.id, website_email=sanitized_contact_email + ) - # No existing contact: only now consult the global blacklist. + # Only consult the global blacklist if NO matching contact exists yet. # If matched, create a disabled Contact; the existing block_forward logic will refuse the email. - if is_sender_globally_blocked(mail_from, sanitized_contact_email): - LOG.i( - "Sender matched global sender blacklist; creating disabled contact: mail_from=%s header_from=%s alias=%s", - mail_from, - sanitized_contact_email, - alias, - ) - contact_result = contact_utils.create_contact( - email=contact_email, - alias=alias, - name=contact_name, - mail_from=mail_from, - allow_empty_email=True, - automatic_created=True, - block_forward=True, - from_partner=False, - ) - if contact_result.error: - LOG.w(f"Error creating contact: {contact_result.error.value}") - return contact_result.contact + block_forward = False + if existing_contact is None: + block_forward = is_sender_globally_blocked(mail_from, sanitized_contact_email) + if block_forward: + LOG.i( + "Sender matched global sender blacklist; creating disabled contact: mail_from=%s header_from=%s alias=%s", + mail_from, + sanitized_contact_email, + alias, + ) contact_result = contact_utils.create_contact( email=contact_email, @@ -264,7 +242,7 @@ def get_or_create_contact( mail_from=mail_from, allow_empty_email=True, automatic_created=True, - block_forward=False, + block_forward=block_forward, from_partner=False, ) if contact_result.error: From e2980075743c64d919dd8ed3cfaec21e0d11c56b Mon Sep 17 00:00:00 2001 From: Lio Date: Wed, 18 Mar 2026 23:07:08 +0100 Subject: [PATCH 04/15] Add per-user sender blacklist --- app/admin/global_sender_blacklist.py | 18 +++++++++++ app/dashboard/views/setting.py | 42 +++++++++++++++++++++++++ app/models.py | 10 +++++- app/sender_blacklist.py | 46 ++++++++++++++++++++++++---- email_handler.py | 8 +++-- templates/dashboard/setting.html | 42 +++++++++++++++++++++++++ 6 files changed, 156 insertions(+), 10 deletions(-) diff --git a/app/admin/global_sender_blacklist.py b/app/admin/global_sender_blacklist.py index e3cbc6770..c6b34509d 100644 --- a/app/admin/global_sender_blacklist.py +++ b/app/admin/global_sender_blacklist.py @@ -14,6 +14,24 @@ class GlobalSenderBlacklistAdmin(SLModelView): column_filters = ("enabled",) column_editable_list = ("enabled", "comment") + # Keep the admin UI strictly on GLOBAL entries (user_id is NULL) + column_exclude_list = ("user_id", "user") + form_excluded_columns = ("user_id", "user") + + def get_query(self): + return ( + super() + .get_query() + .filter(self.model.user_id.is_(None)) # type: ignore[attr-defined] + ) + + def get_count_query(self): + return ( + super() + .get_count_query() + .filter(self.model.user_id.is_(None)) # type: ignore[attr-defined] + ) + # Help text for admins when adding patterns form_args = { "pattern": { diff --git a/app/dashboard/views/setting.py b/app/dashboard/views/setting.py index 7fa1d3101..088da26ae 100644 --- a/app/dashboard/views/setting.py +++ b/app/dashboard/views/setting.py @@ -39,6 +39,7 @@ PartnerSubscription, UnsubscribeBehaviourEnum, UserAliasDeleteAction, + GlobalSenderBlacklist, ) from app.proton.proton_unlink import can_unlink_proton_account from app.utils import ( @@ -285,6 +286,39 @@ def setting(): Session.commit() flash("Your preference has been updated", "success") + elif request.form.get("form-name") == "user-sender-blacklist-add": + pattern = (request.form.get("pattern") or "").strip() + if not pattern: + flash("Pattern cannot be empty", "warning") + return redirect(url_for("dashboard.setting") + "#sender-blacklist") + + GlobalSenderBlacklist.create( + user_id=current_user.id, + pattern=pattern, + enabled=True, + comment=None, + commit=True, + ) + flash("Sender blacklist entry added", "success") + return redirect(url_for("dashboard.setting") + "#sender-blacklist") + + elif request.form.get("form-name") == "user-sender-blacklist-delete": + try: + entry_id = int(request.form.get("entry-id")) + except Exception: + flash("Invalid request", "warning") + return redirect(url_for("dashboard.setting") + "#sender-blacklist") + + entry = GlobalSenderBlacklist.get_by(id=entry_id) + if entry is None or entry.user_id != current_user.id: + flash("Not found", "warning") + return redirect(url_for("dashboard.setting") + "#sender-blacklist") + + Session.delete(entry) + Session.commit() + flash("Sender blacklist entry deleted", "success") + return redirect(url_for("dashboard.setting") + "#sender-blacklist") + manual_sub = ManualSubscription.get_by(user_id=current_user.id) apple_sub = AppleSubscription.get_by(user_id=current_user.id) coinbase_sub = CoinbaseSubscription.get_by(user_id=current_user.id) @@ -296,6 +330,13 @@ def setting(): if partner_sub_name: partner_sub, partner_name = partner_sub_name + user_sender_blacklist_entries = ( + Session.query(GlobalSenderBlacklist) + .filter(GlobalSenderBlacklist.user_id == current_user.id) + .order_by(GlobalSenderBlacklist.id.asc()) + .all() + ) + return render_template( "dashboard/setting.html", csrf_form=csrf_form, @@ -318,4 +359,5 @@ def setting(): ALIAS_RAND_SUFFIX_LENGTH=ALIAS_RANDOM_SUFFIX_LENGTH, connect_with_proton=CONNECT_WITH_PROTON, can_unlink_proton_account=can_unlink_proton_account(current_user), + user_sender_blacklist_entries=user_sender_blacklist_entries, ) diff --git a/app/models.py b/app/models.py index 13c091b2f..9399e3857 100644 --- a/app/models.py +++ b/app/models.py @@ -3685,10 +3685,18 @@ class GlobalSenderBlacklist(Base, ModelMixin): __tablename__ = "global_sender_blacklist" - pattern = sa.Column(sa.String(512), unique=True, nullable=False) + # NULL user_id => global blacklist entry (admin-managed) + # non-NULL user_id => per-user blacklist entry (user-managed) + user_id = sa.Column(sa.ForeignKey(User.id, ondelete="cascade"), nullable=True) + + pattern = sa.Column(sa.String(512), nullable=False) enabled = sa.Column(sa.Boolean, nullable=False, default=True, server_default="1") comment = sa.Column(sa.Text, nullable=True) + user = orm.relationship(User) + + __table_args__ = (sa.Index("ix_global_sender_blacklist_user_id", "user_id"),) + # region Phone class PhoneCountry(Base, ModelMixin): diff --git a/app/sender_blacklist.py b/app/sender_blacklist.py index 40b52805c..d4db6c3d2 100644 --- a/app/sender_blacklist.py +++ b/app/sender_blacklist.py @@ -11,25 +11,50 @@ # Cache enabled patterns briefly to avoid a DB query per inbound email. # Admin changes should take effect quickly but don't need to be instant. @cached(cache=TTLCache(maxsize=1, ttl=30)) -def _get_enabled_patterns() -> list[str]: +def _get_enabled_global_patterns() -> list[str]: return [ r.pattern for r in Session.query(GlobalSenderBlacklist) - .filter(GlobalSenderBlacklist.enabled.is_(True)) + .filter( + GlobalSenderBlacklist.enabled.is_(True), + GlobalSenderBlacklist.user_id.is_(None), + ) .order_by(GlobalSenderBlacklist.id.asc()) .all() ] -def is_sender_globally_blocked(*candidates: str) -> bool: - """Return True if any candidate sender string matches the global blacklist. +# Per-user cache: keep it small-ish but avoid a DB query per email per user. +@cached(cache=TTLCache(maxsize=1024, ttl=30)) +def _get_enabled_user_patterns(user_id: int) -> list[str]: + return [ + r.pattern + for r in Session.query(GlobalSenderBlacklist) + .filter( + GlobalSenderBlacklist.enabled.is_(True), + GlobalSenderBlacklist.user_id == user_id, + ) + .order_by(GlobalSenderBlacklist.id.asc()) + .all() + ] + + +def is_sender_blocked_for_user(user_id: int | None, *candidates: str) -> bool: + """Return True if any candidate sender string matches: + + - the global sender blacklist (user_id is NULL), OR + - the given user's sender blacklist (user_id matches) Typical candidates: - SMTP envelope MAIL FROM - parsed header From address """ - patterns = _get_enabled_patterns() + patterns: list[str] = [] + patterns.extend(_get_enabled_global_patterns()) + if user_id is not None: + patterns.extend(_get_enabled_user_patterns(int(user_id))) + if not patterns: return False @@ -46,10 +71,19 @@ def is_sender_globally_blocked(*candidates: str) -> bool: return True except Exception: # Never crash the SMTP handler because of a bad regex. + # Never crash the SMTP handler because of a bad regex. + # (Global or user entry — both are user-provided.) LOG.exception( - "Global sender blacklist regex failed: pattern=%s candidate=%s", + "Sender blacklist regex failed: user_id=%s pattern=%s candidate=%s", + user_id, pattern, candidate, ) return False + + +def is_sender_globally_blocked(*candidates: str) -> bool: + """Backwards-compatible helper (global-only).""" + + return is_sender_blocked_for_user(None, *candidates) diff --git a/email_handler.py b/email_handler.py index 5776c2b88..d16ef0d49 100644 --- a/email_handler.py +++ b/email_handler.py @@ -153,7 +153,7 @@ from app.handler.unsubscribe_generator import UnsubscribeGenerator from app.handler.unsubscribe_handler import UnsubscribeHandler from app.log import LOG, set_message_id -from app.sender_blacklist import is_sender_globally_blocked +from app.sender_blacklist import is_sender_blocked_for_user from app.mail_sender import sl_sendmail from app.mailbox_utils import ( get_mailbox_for_reply_phase, @@ -226,10 +226,12 @@ def get_or_create_contact( # If matched, create a disabled Contact; the existing block_forward logic will refuse the email. block_forward = False if existing_contact is None: - block_forward = is_sender_globally_blocked(mail_from, sanitized_contact_email) + block_forward = is_sender_blocked_for_user( + alias.user_id, mail_from, sanitized_contact_email + ) if block_forward: LOG.i( - "Sender matched global sender blacklist; creating disabled contact: mail_from=%s header_from=%s alias=%s", + "Sender matched sender blacklist (global or user); creating disabled contact: mail_from=%s header_from=%s alias=%s", mail_from, sanitized_contact_email, alias, diff --git a/templates/dashboard/setting.html b/templates/dashboard/setting.html index 60a68796f..2617262ae 100644 --- a/templates/dashboard/setting.html +++ b/templates/dashboard/setting.html @@ -259,6 +259,48 @@ + + +
+
+
Sender blacklist
+
+ Automatically block new Contacts (and therefore forwarding) when the sender matches a blacklist pattern. +
+ Patterns are regular expressions (regex) and are checked with a search()-style match against the full sender address. +
+ Examples: @spamdomain\.com$, ^no-?reply@.* +
+ Note: if you already have a Contact for that sender, it will not be blocked (existing "exception" logic remains). +
+ +
+ {{ csrf_form.csrf_token }} + + + +
+ + {% if user_sender_blacklist_entries and user_sender_blacklist_entries|length > 0 %} +
    + {% for entry in user_sender_blacklist_entries %} +
  • + {{ entry.pattern }} +
    + {{ csrf_form.csrf_token }} + + + +
    +
    • + {% endfor %} +
+ {% else %} +
No entries yet.
+ {% endif %} +
+
+
From c39dca25fb16ac65c3c735d6992c8d431f622b8a Mon Sep 17 00:00:00 2001 From: Lio Date: Wed, 18 Mar 2026 23:07:11 +0100 Subject: [PATCH 05/15] DB: add user_id to global_sender_blacklist --- ...0318_9c2a7f3c1b21_user_sender_blacklist.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 migrations/versions/2026_0318_9c2a7f3c1b21_user_sender_blacklist.py diff --git a/migrations/versions/2026_0318_9c2a7f3c1b21_user_sender_blacklist.py b/migrations/versions/2026_0318_9c2a7f3c1b21_user_sender_blacklist.py new file mode 100644 index 000000000..6f5ecaea8 --- /dev/null +++ b/migrations/versions/2026_0318_9c2a7f3c1b21_user_sender_blacklist.py @@ -0,0 +1,62 @@ +"""User sender blacklist (extend global sender blacklist) + +Revision ID: 9c2a7f3c1b21 +Revises: b7c1d6a4f2e1 +Create Date: 2026-03-18 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "9c2a7f3c1b21" +down_revision = "b7c1d6a4f2e1" +branch_labels = None +depends_on = None + + +def upgrade(): + # 1) Add user_id nullable so existing global entries stay valid. + with op.batch_alter_table("global_sender_blacklist") as batch: + batch.add_column( + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="cascade"), + nullable=True, + ) + ) + batch.create_index("ix_global_sender_blacklist_user_id", ["user_id"]) + + # 2) Drop unique constraint on pattern so users can use the same pattern independently. + # Constraint name is backend-dependent; try a few common names. + for name in ( + "global_sender_blacklist_pattern_key", # PostgreSQL default + "uq_global_sender_blacklist_pattern", # potential naming convention + "uq_global_sender_blacklist_pattern", # (duplicate on purpose; harmless) + ): + try: + op.drop_constraint(name, "global_sender_blacklist", type_="unique") + except Exception: + pass + + +def downgrade(): + # Re-create unique constraint on pattern (best-effort). + try: + op.create_unique_constraint( + "global_sender_blacklist_pattern_key", + "global_sender_blacklist", + ["pattern"], + ) + except Exception: + pass + + with op.batch_alter_table("global_sender_blacklist") as batch: + try: + batch.drop_index("ix_global_sender_blacklist_user_id") + except Exception: + pass + batch.drop_column("user_id") From 96c1fbc4b2f43be948023ea4e404527dc2386113 Mon Sep 17 00:00:00 2001 From: Lio Date: Wed, 18 Mar 2026 23:33:02 +0100 Subject: [PATCH 06/15] Fix sender blacklist UI text, show global entries, and make migration safe --- app/dashboard/views/setting.py | 8 ++++ app/sender_blacklist.py | 1 - ...0318_9c2a7f3c1b21_user_sender_blacklist.py | 38 ++++++++++++++----- templates/dashboard/setting.html | 21 +++++++++- 4 files changed, 56 insertions(+), 12 deletions(-) diff --git a/app/dashboard/views/setting.py b/app/dashboard/views/setting.py index 088da26ae..b82cb9b04 100644 --- a/app/dashboard/views/setting.py +++ b/app/dashboard/views/setting.py @@ -337,6 +337,13 @@ def setting(): .all() ) + global_sender_blacklist_entries = ( + Session.query(GlobalSenderBlacklist) + .filter(GlobalSenderBlacklist.user_id.is_(None)) + .order_by(GlobalSenderBlacklist.id.asc()) + .all() + ) + return render_template( "dashboard/setting.html", csrf_form=csrf_form, @@ -360,4 +367,5 @@ def setting(): connect_with_proton=CONNECT_WITH_PROTON, can_unlink_proton_account=can_unlink_proton_account(current_user), user_sender_blacklist_entries=user_sender_blacklist_entries, + global_sender_blacklist_entries=global_sender_blacklist_entries, ) diff --git a/app/sender_blacklist.py b/app/sender_blacklist.py index d4db6c3d2..ded84e727 100644 --- a/app/sender_blacklist.py +++ b/app/sender_blacklist.py @@ -70,7 +70,6 @@ def is_sender_blocked_for_user(user_id: int | None, *candidates: str) -> bool: if regex_search(pattern, candidate): return True except Exception: - # Never crash the SMTP handler because of a bad regex. # Never crash the SMTP handler because of a bad regex. # (Global or user entry — both are user-provided.) LOG.exception( diff --git a/migrations/versions/2026_0318_9c2a7f3c1b21_user_sender_blacklist.py b/migrations/versions/2026_0318_9c2a7f3c1b21_user_sender_blacklist.py index 6f5ecaea8..8a6794ab6 100644 --- a/migrations/versions/2026_0318_9c2a7f3c1b21_user_sender_blacklist.py +++ b/migrations/versions/2026_0318_9c2a7f3c1b21_user_sender_blacklist.py @@ -10,6 +10,33 @@ import sqlalchemy as sa +def _drop_unique_constraint_on_pattern_if_present(): + """Drop the UNIQUE(pattern) constraint safely. + + Important: On PostgreSQL, attempting to drop a non-existent constraint + aborts the transaction. Catching the exception in Python is not enough + because the transaction remains in a failed state. + + Therefore we *reflect* existing unique constraints first and only drop + when we have an actual name. + """ + + bind = op.get_bind() + insp = sa.inspect(bind) + try: + uniques = insp.get_unique_constraints("global_sender_blacklist") + except Exception: + uniques = [] + + for uc in uniques: + cols = uc.get("column_names") or [] + if cols == ["pattern"]: + name = uc.get("name") + if name: + op.drop_constraint(name, "global_sender_blacklist", type_="unique") + break + + # revision identifiers, used by Alembic. revision = "9c2a7f3c1b21" down_revision = "b7c1d6a4f2e1" @@ -31,16 +58,7 @@ def upgrade(): batch.create_index("ix_global_sender_blacklist_user_id", ["user_id"]) # 2) Drop unique constraint on pattern so users can use the same pattern independently. - # Constraint name is backend-dependent; try a few common names. - for name in ( - "global_sender_blacklist_pattern_key", # PostgreSQL default - "uq_global_sender_blacklist_pattern", # potential naming convention - "uq_global_sender_blacklist_pattern", # (duplicate on purpose; harmless) - ): - try: - op.drop_constraint(name, "global_sender_blacklist", type_="unique") - except Exception: - pass + _drop_unique_constraint_on_pattern_if_present() def downgrade(): diff --git a/templates/dashboard/setting.html b/templates/dashboard/setting.html index 2617262ae..3c0d4ed15 100644 --- a/templates/dashboard/setting.html +++ b/templates/dashboard/setting.html @@ -271,7 +271,7 @@
Examples: @spamdomain\.com$, ^no-?reply@.*
- Note: if you already have a Contact for that sender, it will not be blocked (existing "exception" logic remains). + Note: if a Contact already exists for that specific sender, it keeps its current status. This can be used to create exceptions (allow/deny) per sender.
@@ -298,6 +298,25 @@ {% else %}
No entries yet.
{% endif %} + +
+
Global blacklist (read-only)
+ {% if global_sender_blacklist_entries and global_sender_blacklist_entries|length > 0 %} +
    + {% for entry in global_sender_blacklist_entries %} +
  • + {{ entry.pattern }} + {% if entry.enabled %} + enabled + {% else %} + disabled + {% endif %} +
    • + {% endfor %} +
+ {% else %} +
No global blacklist entries.
+ {% endif %}
From cfdbbfb180d2201bfea68d3d2ac3854afa7e6db0 Mon Sep 17 00:00:00 2001 From: chrisblech Date: Wed, 18 Mar 2026 23:37:22 +0000 Subject: [PATCH 07/15] cleanup, only show active global entries --- app/admin/global_sender_blacklist.py | 10 +++------- app/dashboard/views/setting.py | 5 ++++- app/sender_blacklist.py | 10 ++-------- templates/dashboard/setting.html | 29 ++++++++++------------------ 4 files changed, 19 insertions(+), 35 deletions(-) diff --git a/app/admin/global_sender_blacklist.py b/app/admin/global_sender_blacklist.py index c6b34509d..fd114e7c4 100644 --- a/app/admin/global_sender_blacklist.py +++ b/app/admin/global_sender_blacklist.py @@ -20,21 +20,17 @@ class GlobalSenderBlacklistAdmin(SLModelView): def get_query(self): return ( - super() - .get_query() - .filter(self.model.user_id.is_(None)) # type: ignore[attr-defined] + super().get_query().filter(self.model.user_id.is_(None)) # type: ignore[attr-defined] ) def get_count_query(self): return ( - super() - .get_count_query() - .filter(self.model.user_id.is_(None)) # type: ignore[attr-defined] + super().get_count_query().filter(self.model.user_id.is_(None)) # type: ignore[attr-defined] ) # Help text for admins when adding patterns form_args = { "pattern": { - "description": r"Regex, i.e. `@domain\.com`", + "description": r"Regex, i.e. `@domain\.com$`", } } diff --git a/app/dashboard/views/setting.py b/app/dashboard/views/setting.py index b82cb9b04..c66e81756 100644 --- a/app/dashboard/views/setting.py +++ b/app/dashboard/views/setting.py @@ -339,7 +339,10 @@ def setting(): global_sender_blacklist_entries = ( Session.query(GlobalSenderBlacklist) - .filter(GlobalSenderBlacklist.user_id.is_(None)) + .filter( + GlobalSenderBlacklist.enabled.is_(True), + GlobalSenderBlacklist.user_id.is_(None), + ) .order_by(GlobalSenderBlacklist.id.asc()) .all() ) diff --git a/app/sender_blacklist.py b/app/sender_blacklist.py index ded84e727..37110bb5d 100644 --- a/app/sender_blacklist.py +++ b/app/sender_blacklist.py @@ -10,7 +10,7 @@ # Cache enabled patterns briefly to avoid a DB query per inbound email. # Admin changes should take effect quickly but don't need to be instant. -@cached(cache=TTLCache(maxsize=1, ttl=30)) +@cached(cache=TTLCache(maxsize=128, ttl=30)) def _get_enabled_global_patterns() -> list[str]: return [ r.pattern @@ -25,7 +25,7 @@ def _get_enabled_global_patterns() -> list[str]: # Per-user cache: keep it small-ish but avoid a DB query per email per user. -@cached(cache=TTLCache(maxsize=1024, ttl=30)) +@cached(cache=TTLCache(maxsize=128, ttl=30)) def _get_enabled_user_patterns(user_id: int) -> list[str]: return [ r.pattern @@ -80,9 +80,3 @@ def is_sender_blocked_for_user(user_id: int | None, *candidates: str) -> bool: ) return False - - -def is_sender_globally_blocked(*candidates: str) -> bool: - """Backwards-compatible helper (global-only).""" - - return is_sender_blocked_for_user(None, *candidates) diff --git a/templates/dashboard/setting.html b/templates/dashboard/setting.html index 3c0d4ed15..8984223b5 100644 --- a/templates/dashboard/setting.html +++ b/templates/dashboard/setting.html @@ -269,9 +269,9 @@
Patterns are regular expressions (regex) and are checked with a search()-style match against the full sender address.
- Examples: @spamdomain\.com$, ^no-?reply@.* + Examples: @(spam|junk)-?domain\.com$, ^no-?reply@.*
- Note: if a Contact already exists for that specific sender, it keeps its current status. This can be used to create exceptions (allow/deny) per sender. + Note: if a Contact already exists for that specific sender, it keeps its current status. This can be used to create allow-exceptions. @@ -299,25 +299,16 @@
No entries yet.
{% endif %} -
-
Global blacklist (read-only)
{% if global_sender_blacklist_entries and global_sender_blacklist_entries|length > 0 %} -
    - {% for entry in global_sender_blacklist_entries %} -
  • - {{ entry.pattern }} - {% if entry.enabled %} - enabled - {% else %} - disabled - {% endif %} -
    • - {% endfor %} -
- {% else %} -
No global blacklist entries.
+
+
Global blacklist (admin-managed)
+
    + {% for entry in global_sender_blacklist_entries %} +
  • {{ entry.pattern }}
    • + {% endfor %} +
+ {% endif %} - From 1aeab12fe82a2ff1b067134b369b1895e2ebd760 Mon Sep 17 00:00:00 2001 From: Lio Date: Wed, 25 Mar 2026 23:45:50 +0100 Subject: [PATCH 08/15] Rename GlobalSenderBlacklist model to ForbiddenEnvelopeSender --- app/admin/index.py | 4 ++-- app/dashboard/views/setting.py | 20 ++++++++++---------- app/models.py | 7 ++++--- app/sender_blacklist.py | 18 +++++++++--------- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/app/admin/index.py b/app/admin/index.py index ec1e04962..42ed50e0d 100644 --- a/app/admin/index.py +++ b/app/admin/index.py @@ -17,7 +17,7 @@ Metric2, InvalidMailboxDomain, ForbiddenMxIp, - GlobalSenderBlacklist, + ForbiddenEnvelopeSender, ) from app.admin.base import SLAdminIndexView from app.admin.user import UserAdmin @@ -65,4 +65,4 @@ def init_admin(app: Flask): admin.add_view(MetricAdmin(Metric2, Session)) admin.add_view(InvalidMailboxDomainAdmin(InvalidMailboxDomain, Session)) admin.add_view(ForbiddenMxIpAdmin(ForbiddenMxIp, Session)) - admin.add_view(GlobalSenderBlacklistAdmin(GlobalSenderBlacklist, Session)) + admin.add_view(GlobalSenderBlacklistAdmin(ForbiddenEnvelopeSender, Session)) diff --git a/app/dashboard/views/setting.py b/app/dashboard/views/setting.py index c66e81756..f204d4dd7 100644 --- a/app/dashboard/views/setting.py +++ b/app/dashboard/views/setting.py @@ -39,7 +39,7 @@ PartnerSubscription, UnsubscribeBehaviourEnum, UserAliasDeleteAction, - GlobalSenderBlacklist, + ForbiddenEnvelopeSender, ) from app.proton.proton_unlink import can_unlink_proton_account from app.utils import ( @@ -292,7 +292,7 @@ def setting(): flash("Pattern cannot be empty", "warning") return redirect(url_for("dashboard.setting") + "#sender-blacklist") - GlobalSenderBlacklist.create( + ForbiddenEnvelopeSender.create( user_id=current_user.id, pattern=pattern, enabled=True, @@ -309,7 +309,7 @@ def setting(): flash("Invalid request", "warning") return redirect(url_for("dashboard.setting") + "#sender-blacklist") - entry = GlobalSenderBlacklist.get_by(id=entry_id) + entry = ForbiddenEnvelopeSender.get_by(id=entry_id) if entry is None or entry.user_id != current_user.id: flash("Not found", "warning") return redirect(url_for("dashboard.setting") + "#sender-blacklist") @@ -331,19 +331,19 @@ def setting(): partner_sub, partner_name = partner_sub_name user_sender_blacklist_entries = ( - Session.query(GlobalSenderBlacklist) - .filter(GlobalSenderBlacklist.user_id == current_user.id) - .order_by(GlobalSenderBlacklist.id.asc()) + Session.query(ForbiddenEnvelopeSender) + .filter(ForbiddenEnvelopeSender.user_id == current_user.id) + .order_by(ForbiddenEnvelopeSender.id.asc()) .all() ) global_sender_blacklist_entries = ( - Session.query(GlobalSenderBlacklist) + Session.query(ForbiddenEnvelopeSender) .filter( - GlobalSenderBlacklist.enabled.is_(True), - GlobalSenderBlacklist.user_id.is_(None), + ForbiddenEnvelopeSender.enabled.is_(True), + ForbiddenEnvelopeSender.user_id.is_(None), ) - .order_by(GlobalSenderBlacklist.id.asc()) + .order_by(ForbiddenEnvelopeSender.id.asc()) .all() ) diff --git a/app/models.py b/app/models.py index 9399e3857..6c34f1687 100644 --- a/app/models.py +++ b/app/models.py @@ -3672,8 +3672,8 @@ class ForbiddenMxIp(Base, ModelMixin): comment = sa.Column(sa.Text, unique=False, nullable=True) -class GlobalSenderBlacklist(Base, ModelMixin): - """Global blacklist for inbound senders (envelope MAIL FROM). +class ForbiddenEnvelopeSender(Base, ModelMixin): + """Forbidden inbound senders (SMTP envelope MAIL FROM). Pattern is a (re2-compatible) regex that is applied via search() against the full envelope sender address. @@ -3689,7 +3689,8 @@ class GlobalSenderBlacklist(Base, ModelMixin): # non-NULL user_id => per-user blacklist entry (user-managed) user_id = sa.Column(sa.ForeignKey(User.id, ondelete="cascade"), nullable=True) - pattern = sa.Column(sa.String(512), nullable=False) + # RFC5321 states that an email address cannot be longer than 254 characters. + pattern = sa.Column(sa.String(255), nullable=False) enabled = sa.Column(sa.Boolean, nullable=False, default=True, server_default="1") comment = sa.Column(sa.Text, nullable=True) diff --git a/app/sender_blacklist.py b/app/sender_blacklist.py index 37110bb5d..cd2e7070c 100644 --- a/app/sender_blacklist.py +++ b/app/sender_blacklist.py @@ -4,7 +4,7 @@ from app.db import Session from app.log import LOG -from app.models import GlobalSenderBlacklist +from app.models import ForbiddenEnvelopeSender from app.regex_utils import regex_search @@ -14,12 +14,12 @@ def _get_enabled_global_patterns() -> list[str]: return [ r.pattern - for r in Session.query(GlobalSenderBlacklist) + for r in Session.query(ForbiddenEnvelopeSender) .filter( - GlobalSenderBlacklist.enabled.is_(True), - GlobalSenderBlacklist.user_id.is_(None), + ForbiddenEnvelopeSender.enabled.is_(True), + ForbiddenEnvelopeSender.user_id.is_(None), ) - .order_by(GlobalSenderBlacklist.id.asc()) + .order_by(ForbiddenEnvelopeSender.id.asc()) .all() ] @@ -29,12 +29,12 @@ def _get_enabled_global_patterns() -> list[str]: def _get_enabled_user_patterns(user_id: int) -> list[str]: return [ r.pattern - for r in Session.query(GlobalSenderBlacklist) + for r in Session.query(ForbiddenEnvelopeSender) .filter( - GlobalSenderBlacklist.enabled.is_(True), - GlobalSenderBlacklist.user_id == user_id, + ForbiddenEnvelopeSender.enabled.is_(True), + ForbiddenEnvelopeSender.user_id == user_id, ) - .order_by(GlobalSenderBlacklist.id.asc()) + .order_by(ForbiddenEnvelopeSender.id.asc()) .all() ] From 00113609a6d3b2989657f9c5fabd2efb35e89109 Mon Sep 17 00:00:00 2001 From: Lio Date: Wed, 25 Mar 2026 23:46:01 +0100 Subject: [PATCH 09/15] Admin: show all forbidden envelope sender entries --- app/admin/global_sender_blacklist.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/app/admin/global_sender_blacklist.py b/app/admin/global_sender_blacklist.py index fd114e7c4..19e64e834 100644 --- a/app/admin/global_sender_blacklist.py +++ b/app/admin/global_sender_blacklist.py @@ -11,23 +11,9 @@ class GlobalSenderBlacklistAdmin(SLModelView): can_delete = True column_searchable_list = ("pattern", "comment") - column_filters = ("enabled",) + column_filters = ("enabled", "user_id") column_editable_list = ("enabled", "comment") - # Keep the admin UI strictly on GLOBAL entries (user_id is NULL) - column_exclude_list = ("user_id", "user") - form_excluded_columns = ("user_id", "user") - - def get_query(self): - return ( - super().get_query().filter(self.model.user_id.is_(None)) # type: ignore[attr-defined] - ) - - def get_count_query(self): - return ( - super().get_count_query().filter(self.model.user_id.is_(None)) # type: ignore[attr-defined] - ) - # Help text for admins when adding patterns form_args = { "pattern": { From 203f8dda1078668cb8b7068de85144b363d1c7ce Mon Sep 17 00:00:00 2001 From: Lio Date: Wed, 25 Mar 2026 23:47:55 +0100 Subject: [PATCH 10/15] Sender blacklist: cache patterns 5min, use fullmatch and explicit types --- app/sender_blacklist.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/app/sender_blacklist.py b/app/sender_blacklist.py index cd2e7070c..1850d48e5 100644 --- a/app/sender_blacklist.py +++ b/app/sender_blacklist.py @@ -5,12 +5,18 @@ from app.db import Session from app.log import LOG from app.models import ForbiddenEnvelopeSender -from app.regex_utils import regex_search +from app.regex_utils import regex_match -# Cache enabled patterns briefly to avoid a DB query per inbound email. -# Admin changes should take effect quickly but don't need to be instant. -@cached(cache=TTLCache(maxsize=128, ttl=30)) +# Cache enabled patterns to avoid a DB query per inbound email. +# +# TTL: keep changes reasonably fresh while avoiding hammering the DB. +# Memory: cachetools.TTLCache is an in-process dict with an upper bound (maxsize). +_GLOBAL_PATTERNS_CACHE = TTLCache(maxsize=1, ttl=300) +_USER_PATTERNS_CACHE = TTLCache(maxsize=128, ttl=300) + + +@cached(cache=_GLOBAL_PATTERNS_CACHE) def _get_enabled_global_patterns() -> list[str]: return [ r.pattern @@ -24,8 +30,8 @@ def _get_enabled_global_patterns() -> list[str]: ] -# Per-user cache: keep it small-ish but avoid a DB query per email per user. -@cached(cache=TTLCache(maxsize=128, ttl=30)) +# Per-user cache: avoid a DB query per email per user, but cap memory via maxsize. +@cached(cache=_USER_PATTERNS_CACHE) def _get_enabled_user_patterns(user_id: int) -> list[str]: return [ r.pattern @@ -39,7 +45,7 @@ def _get_enabled_user_patterns(user_id: int) -> list[str]: ] -def is_sender_blocked_for_user(user_id: int | None, *candidates: str) -> bool: +def is_sender_blocked_for_user(user_id: int | None, candidates: list[str]) -> bool: """Return True if any candidate sender string matches: - the global sender blacklist (user_id is NULL), OR @@ -67,7 +73,8 @@ def is_sender_blocked_for_user(user_id: int | None, *candidates: str) -> bool: for pattern in patterns: try: - if regex_search(pattern, candidate): + # Full-string match to avoid false positives (partial hits). + if regex_match(pattern, candidate): return True except Exception: # Never crash the SMTP handler because of a bad regex. From 2ca7a7743868eb5326e6406d63041ad21539bdb7 Mon Sep 17 00:00:00 2001 From: Lio Date: Wed, 25 Mar 2026 23:47:58 +0100 Subject: [PATCH 11/15] Email handler: apply sender blacklist before creating contacts --- email_handler.py | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/email_handler.py b/email_handler.py index d16ef0d49..ff7b01380 100644 --- a/email_handler.py +++ b/email_handler.py @@ -215,30 +215,25 @@ def get_or_create_contact( ) contact_email = mail_from - # Decide whether we already have a matching contact BEFORE applying the global sender blacklist. - # This allows users to whitelist a specific sender by manually creating/enabling a Contact. - sanitized_contact_email = sanitize_email(contact_email, not_lower=True) - existing_contact = Contact.get_by( - alias_id=alias.id, website_email=sanitized_contact_email + # Normalize sender address to lowercase so blacklist patterns are easy to write. + sanitized_contact_email = sanitize_email(contact_email) + + # Check the blacklist BEFORE creating/updating contacts. + # Otherwise an auto-created contact could allow subsequent emails to bypass the blacklist. + block_forward = is_sender_blocked_for_user( + alias.user_id, + candidates=[mail_from, sanitized_contact_email], ) - - # Only consult the global blacklist if NO matching contact exists yet. - # If matched, create a disabled Contact; the existing block_forward logic will refuse the email. - block_forward = False - if existing_contact is None: - block_forward = is_sender_blocked_for_user( - alias.user_id, mail_from, sanitized_contact_email + if block_forward: + LOG.i( + "Sender matched sender blacklist (global or user); creating disabled contact: mail_from=%s header_from=%s alias=%s", + mail_from, + sanitized_contact_email, + alias, ) - if block_forward: - LOG.i( - "Sender matched sender blacklist (global or user); creating disabled contact: mail_from=%s header_from=%s alias=%s", - mail_from, - sanitized_contact_email, - alias, - ) contact_result = contact_utils.create_contact( - email=contact_email, + email=sanitized_contact_email, alias=alias, name=contact_name, mail_from=mail_from, From 851f58e4b0c0eabb08ef650bf0586b574cd6e08f Mon Sep 17 00:00:00 2001 From: Lio Date: Wed, 25 Mar 2026 23:48:01 +0100 Subject: [PATCH 12/15] Dashboard: comment, validation and audit log for sender blacklist --- app/dashboard/views/setting.py | 29 ++++++++++++++++++++++++++--- app/regex_utils.py | 29 +++++++++++++++++++++++++++++ app/user_audit_log_utils.py | 3 +++ templates/dashboard/setting.html | 6 +++--- 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/app/dashboard/views/setting.py b/app/dashboard/views/setting.py index f204d4dd7..b859b2e73 100644 --- a/app/dashboard/views/setting.py +++ b/app/dashboard/views/setting.py @@ -42,6 +42,8 @@ ForbiddenEnvelopeSender, ) from app.proton.proton_unlink import can_unlink_proton_account +from app.regex_utils import validate_sender_blacklist_pattern +from app.user_audit_log_utils import emit_user_audit_log, UserAuditLogAction from app.utils import ( random_string, CSRFValidationForm, @@ -288,15 +290,29 @@ def setting(): elif request.form.get("form-name") == "user-sender-blacklist-add": pattern = (request.form.get("pattern") or "").strip() - if not pattern: - flash("Pattern cannot be empty", "warning") + comment = (request.form.get("comment") or "").strip() or None + + if len(pattern) > 255: + flash("Pattern too long (max 255 characters)", "warning") + return redirect(url_for("dashboard.setting") + "#sender-blacklist") + + err = validate_sender_blacklist_pattern(pattern) + if err: + flash(err, "warning") return redirect(url_for("dashboard.setting") + "#sender-blacklist") ForbiddenEnvelopeSender.create( user_id=current_user.id, pattern=pattern, enabled=True, - comment=None, + comment=comment, + commit=True, + ) + + emit_user_audit_log( + user=current_user, + action=UserAuditLogAction.AddSenderBlacklist, + message=f"Added sender blacklist pattern: {pattern}", commit=True, ) flash("Sender blacklist entry added", "success") @@ -316,6 +332,13 @@ def setting(): Session.delete(entry) Session.commit() + + emit_user_audit_log( + user=current_user, + action=UserAuditLogAction.DeleteSenderBlacklist, + message=f"Deleted sender blacklist pattern: {entry.pattern}", + commit=True, + ) flash("Sender blacklist entry deleted", "success") return redirect(url_for("dashboard.setting") + "#sender-blacklist") diff --git a/app/regex_utils.py b/app/regex_utils.py index 788103126..3134e5b32 100644 --- a/app/regex_utils.py +++ b/app/regex_utils.py @@ -5,6 +5,35 @@ from app.log import LOG +_SENDER_BLACKLIST_ALLOWED_RE = re.compile(r"^[A-Za-z0-9@._\-\+\*\^\$\(\)\|\?\[\]\\]+$") + + +def validate_sender_blacklist_pattern(pattern: str) -> str | None: + """Validate a user-provided sender-blacklist regex pattern. + + The goal is to keep patterns simple and prevent expensive/unsafe constructs. + We also validate the regex compiles (re2 preferred). + + Returns: + None if valid; otherwise an error message string. + """ + if not pattern: + return "Pattern cannot be empty" + + # Keep the allowed character set intentionally small. + if not _SENDER_BLACKLIST_ALLOWED_RE.fullmatch(pattern): + return ( + "Invalid characters in pattern. Allowed: letters, digits, and @ . _ - + * ^ $ ( ) | ? [ ] \\" + ) + + try: + re2.compile(pattern) + except Exception: + return "Invalid regex pattern" + + return None + + def regex_match(rule_regex: str, local) -> bool: """Return True if *full string* matches rule_regex.""" regex = re2.compile(rule_regex) diff --git a/app/user_audit_log_utils.py b/app/user_audit_log_utils.py index f9521e714..9a659b501 100644 --- a/app/user_audit_log_utils.py +++ b/app/user_audit_log_utils.py @@ -29,6 +29,9 @@ class UserAuditLogAction(Enum): UpdateDirectory = "update_directory" DeleteDirectory = "delete_directory" + AddSenderBlacklist = "add_sender_blacklist" + DeleteSenderBlacklist = "delete_sender_blacklist" + UserMarkedForDeletion = "user_marked_for_deletion" DeleteUser = "delete_user" diff --git a/templates/dashboard/setting.html b/templates/dashboard/setting.html index 8984223b5..6402419f6 100644 --- a/templates/dashboard/setting.html +++ b/templates/dashboard/setting.html @@ -267,17 +267,16 @@
Automatically block new Contacts (and therefore forwarding) when the sender matches a blacklist pattern.
- Patterns are regular expressions (regex) and are checked with a search()-style match against the full sender address. + Patterns are regular expressions (regex) and are checked with a fullmatch()-style match against the full sender address.
Examples: @(spam|junk)-?domain\.com$, ^no-?reply@.* -
- Note: if a Contact already exists for that specific sender, it keeps its current status. This can be used to create allow-exceptions.
{{ csrf_form.csrf_token }} + @@ -286,6 +285,7 @@ {% for entry in user_sender_blacklist_entries %}
  • {{ entry.pattern }} + {% if entry.comment %}— {{ entry.comment }}{% endif %}
    {{ csrf_form.csrf_token }} From 28554a83cb2fb48282f68339a19655815678d07c Mon Sep 17 00:00:00 2001 From: Lio Date: Wed, 25 Mar 2026 23:48:04 +0100 Subject: [PATCH 13/15] Deps: add cachetools --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 67a6022a2..d03131772 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ dependencies = [ "newrelic ~= 8.8.0", "flanker ~= 0.9.11", "pyre2 ~= 0.3.6", + "cachetools ~= 5.3.3", "tldextract ~= 3.1.2", "flask-debugtoolbar-sqlalchemy ~= 0.2.0", "twilio ~= 7.3.2", From 1f6496ec5893c3452869a55e429f813b10df1ba6 Mon Sep 17 00:00:00 2001 From: Lio Date: Wed, 25 Mar 2026 23:48:27 +0100 Subject: [PATCH 14/15] Tests: update model name for sender blacklist --- tests/test_email_handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_email_handler.py b/tests/test_email_handler.py index 34c435e64..7215f0580 100644 --- a/tests/test_email_handler.py +++ b/tests/test_email_handler.py @@ -21,7 +21,7 @@ VerpType, Contact, SentAlert, - GlobalSenderBlacklist, + ForbiddenEnvelopeSender, ) from app.utils import random_string, canonicalize_email from email_handler import ( @@ -46,7 +46,7 @@ def test_global_sender_blacklist_blocks(flask_client): alias = Alias.create_new_random(user) # Block all senders from spam.test - GlobalSenderBlacklist.create(pattern=r"@spam\\.test$", enabled=True, commit=True) + ForbiddenEnvelopeSender.create(pattern=r"@spam\\.test$", enabled=True, commit=True) msg = EmailMessage() msg[headers.FROM] = "Bad Guy " @@ -87,7 +87,7 @@ def test_global_sender_blacklist_not_applied_when_contact_exists(flask_client): assert contact is not None # Now enable a global blacklist that would match this sender. - GlobalSenderBlacklist.create(pattern=r"@spam\\.test$", enabled=True, commit=True) + ForbiddenEnvelopeSender.create(pattern=r"@spam\\.test$", enabled=True, commit=True) msg = EmailMessage() msg[headers.FROM] = "Bad Guy " From 43173d15932b77ddcbb42b3218d31574608fe7b5 Mon Sep 17 00:00:00 2001 From: Lio Date: Thu, 26 Mar 2026 00:08:14 +0100 Subject: [PATCH 15/15] Adjust sender blacklist validation, contact precedence, and cache sizing --- app/regex_utils.py | 8 ++++++-- app/sender_blacklist.py | 2 +- email_handler.py | 20 +++++++++++++++----- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/app/regex_utils.py b/app/regex_utils.py index 3134e5b32..818e252d4 100644 --- a/app/regex_utils.py +++ b/app/regex_utils.py @@ -5,7 +5,11 @@ from app.log import LOG -_SENDER_BLACKLIST_ALLOWED_RE = re.compile(r"^[A-Za-z0-9@._\-\+\*\^\$\(\)\|\?\[\]\\]+$") +# Keep this permissive enough for practical regexes, but still a strict whitelist. +# Note: We intentionally do NOT allow whitespace. +_SENDER_BLACKLIST_ALLOWED_RE = re.compile( + r"^[A-Za-z0-9\[\]\{\}\(\)\|\?\^\$@,._\-\+\*\\\.]+$" +) def validate_sender_blacklist_pattern(pattern: str) -> str | None: @@ -23,7 +27,7 @@ def validate_sender_blacklist_pattern(pattern: str) -> str | None: # Keep the allowed character set intentionally small. if not _SENDER_BLACKLIST_ALLOWED_RE.fullmatch(pattern): return ( - "Invalid characters in pattern. Allowed: letters, digits, and @ . _ - + * ^ $ ( ) | ? [ ] \\" + "Invalid characters in pattern. Allowed: letters, digits, and []{}(),._-+*\\.^$@|?" ) try: diff --git a/app/sender_blacklist.py b/app/sender_blacklist.py index 1850d48e5..474513d21 100644 --- a/app/sender_blacklist.py +++ b/app/sender_blacklist.py @@ -12,7 +12,7 @@ # # TTL: keep changes reasonably fresh while avoiding hammering the DB. # Memory: cachetools.TTLCache is an in-process dict with an upper bound (maxsize). -_GLOBAL_PATTERNS_CACHE = TTLCache(maxsize=1, ttl=300) +_GLOBAL_PATTERNS_CACHE = TTLCache(maxsize=128, ttl=300) _USER_PATTERNS_CACHE = TTLCache(maxsize=128, ttl=300) diff --git a/email_handler.py b/email_handler.py index ff7b01380..c430496b3 100644 --- a/email_handler.py +++ b/email_handler.py @@ -218,12 +218,22 @@ def get_or_create_contact( # Normalize sender address to lowercase so blacklist patterns are easy to write. sanitized_contact_email = sanitize_email(contact_email) - # Check the blacklist BEFORE creating/updating contacts. - # Otherwise an auto-created contact could allow subsequent emails to bypass the blacklist. - block_forward = is_sender_blocked_for_user( - alias.user_id, - candidates=[mail_from, sanitized_contact_email], + # If a Contact already exists and is NOT blocked, it takes precedence over the blacklist. + # This allows users to "whitelist" a specific sender by manually creating/enabling a Contact. + existing_contact = Contact.get_by( + alias_id=alias.id, + website_email=sanitized_contact_email, ) + + block_forward = False + if existing_contact is None: + # Check the blacklist BEFORE creating the contact. + # Otherwise an auto-created contact could allow subsequent emails to bypass the blacklist. + block_forward = is_sender_blocked_for_user( + alias.user_id, + candidates=[mail_from, sanitized_contact_email], + ) + if block_forward: LOG.i( "Sender matched sender blacklist (global or user); creating disabled contact: mail_from=%s header_from=%s alias=%s",