Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions docs/environment-variables.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1092,6 +1092,24 @@
- "" (ריק)
- ``secret123``
- WebApp
* - ``ALERTMANAGER_API_URL``
- כתובת בסיס של Alertmanager API (כולל ``https://``). משמשת את הבוט כדי לשלוח התראה ל־``POST /api/v2/alerts`` (למשל עבור הפקודה ``/admin``).
- לא
- "" (ריק)
- ``https://code-keeper-alertmanager.onrender.com``
- Bot
* - ``ALERTMANAGER_API_TOKEN``
- טוקן Bearer אופציונלי ל־Alertmanager API (יישלח כ־``Authorization: Bearer ...``) עבור ``/api/v2/alerts``.
- לא
- "" (ריק)
- ``secret123``
- Bot
* - ``ALERTMANAGER_TARGET``
- יעד Alertmanager בפורמט ``host:port`` (נפוץ בפריסת Prometheus). הבוט יכול להשתמש בו כ-fallback לבניית URL ל־``/api/v2/alerts`` אם ``ALERTMANAGER_API_URL`` לא מוגדר.
- לא
- "" (ריק)
- ``code-keeper-alertmanager.onrender.com:443``
- Prometheus/Bot
* - ``SENTRY_WEBHOOK_SECRET``
- סוד לאימות קריאות ל־``/webhooks/sentry`` (אם ריק, השירות מאפשר קריאות ללא אימות – מומלץ להגדיר). נבדק כ-``Authorization: Bearer`` או ``?token=`` וגם תומך בחתימת HMAC כאשר קיימת.
- לא
Expand Down
202 changes: 198 additions & 4 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -756,6 +756,190 @@ async def notify_admins(context: ContextTypes.DEFAULT_TYPE, text: str) -> bool:
return False


def _get_alertmanager_api_alerts_url() -> str | None:
"""בונה URL ל-Alertmanager API עבור POST /api/v2/alerts.

תומך ב-2 מצבים:
- ALERTMANAGER_API_URL: URL בסיס מלא (כולל scheme), לדוגמה: https://code-keeper-alertmanager.onrender.com
- ALERTMANAGER_TARGET: host:port (כמו בפריסת Prometheus), לדוגמה: code-keeper-alertmanager.onrender.com:443
"""
raw = (os.getenv("ALERTMANAGER_API_URL") or os.getenv("ALERTMANAGER_TARGET") or "").strip()
if not raw:
return None

base = raw
if "://" not in base:
# ניחוש scheme: 443 => https, אחרת נוטים ל-http (למשל docker-compose: alertmanager:9093)
scheme = "https"
try:
low = base.lower()
if low.endswith(":443"):
scheme = "https"
elif low.endswith(":9093") or low.endswith(":80") or low.startswith("localhost") or low.startswith("127.") or low.startswith("alertmanager"):
scheme = "http"

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Scheme guessing fails for non-standard HTTPS ports

Low Severity

The scheme guessing logic checks if the hostname starts with alertmanager and defaults to http, which incorrectly handles URLs like alertmanager.prod.example.com:8443 or alertmanager-api.example.com:9443 where HTTPS is running on non-standard ports. The check for startswith("alertmanager") on line 778 takes precedence over port-based detection for any port other than 443, causing HTTPS requests to fail. Users can work around this by using ALERTMANAGER_API_URL with the full scheme.

Fix in Cursor Fix in Web

else:
# ברירת מחדל סבירה לסביבות ענן
scheme = "https"
except Exception:
scheme = "https"
base = f"{scheme}://{base}"

base = base.rstrip("/")
return f"{base}/api/v2/alerts"


async def _send_admin_report_via_alertmanager(
report_text: str,
*,
user_id: Any = None,
username: Any = None,
display: Any = None,
) -> bool:
"""שולח דיווח ל-Alertmanager ומחזיר True רק אם התקבלה תשובת 2xx.

חשוב: זה "אישור קבלה" מצד Alertmanager (accepted), לא הבטחה שההודעה הגיעה ליעד הסופי.
"""
url = _get_alertmanager_api_alerts_url()
if not url:
return False

if not report_text:
return False

def _truncate(s: Any, limit: int = 400) -> str:
try:
text = str(s or "")
except Exception:
text = ""
text = text.strip()
if limit and len(text) > limit:
return text[: max(0, limit - 1)] + "…"
return text

# Alertmanager expects a list of alerts
now = datetime.now(timezone.utc).isoformat()
alert = {
"labels": {
"alertname": "AdminUserReport",
"severity": "info",
"component": "bot",
},
"annotations": {
"summary": _truncate(report_text, 1800),
"user_id": _truncate(user_id, 80),
"username": _truncate(username, 120),
"user_display": _truncate(display, 200),
},
"startsAt": now,
}

headers: dict[str, str] = {"Content-Type": "application/json"}
token = (os.getenv("ALERTMANAGER_API_TOKEN") or "").strip()
if token:
headers["Authorization"] = f"Bearer {token}"

last_err: str | None = None

# Prefer the project's resilient async HTTP helper (aiohttp + retries/circuit breaker).
try:
try:
from http_async import request as http_request # type: ignore
except Exception:
http_request = None # type: ignore

# aiohttp דורש ClientTimeout, לא int. אם aiohttp לא זמין, לא נעביר timeout כלל.
timeout_obj = None
try:
import aiohttp # type: ignore

timeout_obj = aiohttp.ClientTimeout(total=6) # type: ignore[attr-defined]
except Exception:
timeout_obj = None

if http_request is not None:
req_kwargs = {
"json": [alert],
"headers": headers,
"service": "alertmanager",
"endpoint": "/api/v2/alerts",
"max_attempts": 2,
}
if timeout_obj is not None:
req_kwargs["timeout"] = timeout_obj

async with http_request("POST", url, **req_kwargs) as resp:
try:
status = int(getattr(resp, "status", 0) or 0)
except Exception:
status = 0

if 200 <= status < 300:
return True

# לא מציגים למשתמש; רק לוג דיבוג קצר
body_preview = ""
try:
body_preview = _truncate(await resp.text(), 300) # type: ignore[func-returns-value]
except Exception:
body_preview = ""
try:
logger.warning(
"alertmanager_admin_report_failed status=%s body=%s",
status,
body_preview or "—",
)
except Exception:
pass
Comment thread
cursor[bot] marked this conversation as resolved.
# אם קיבלנו תשובה מהשרת (גם אם דחייה), לא ננסה לשלוח שוב ב-sync fallback
# כדי לא ליצור כפילות/latency מיותרת.
return False
except Exception as e:
last_err = str(e)
try:
logger.warning("alertmanager_admin_report_async_exception error=%s", last_err)
except Exception:
pass

# Fallback: synchronous request in a thread (do not skip if async path failed)
try:
try:
from http_sync import request as http_sync_request # type: ignore
except Exception:
http_sync_request = None # type: ignore

if http_sync_request is None:
return False

def _do_sync() -> bool:
try:
resp = http_sync_request(
"POST",
url,
json=[alert],
headers=headers,
timeout=6,
service="alertmanager",
endpoint="/api/v2/alerts",
max_attempts=2,
)
code = int(getattr(resp, "status_code", 0) or 0)
return 200 <= code < 300
except Exception:
return False

return bool(await asyncio.to_thread(_do_sync))
except Exception as e:
try:
logger.warning(
"alertmanager_admin_report_sync_exception error=%s prev=%s",
str(e),
last_err or "",
)
except Exception:
pass
return False


async def _send_direct_admins(context: ContextTypes.DEFAULT_TYPE, text: str) -> bool:
"""Fallback לשליחת הודעה ישירה לאדמינים בטלגרם."""
try:
Expand Down Expand Up @@ -809,11 +993,21 @@ async def admin_report_command(update: Update, context: ContextTypes.DEFAULT_TYP
f"• user_id: {user_id}\n"
f"• הודעה: {args_text}"
)
sent = await notify_admins(context, report)
if not sent:
sent = await _send_direct_admins(context, report)
if sent:
# חשוב: נציג "נשלח" רק אם Alertmanager קיבל את ההודעה בהצלחה (2xx).
# אם Alertmanager לא זמין, ננסה fallback ישיר לאדמינים – אבל ננסח זאת במפורש כדי לא להטעות.
sent_via_am = await _send_admin_report_via_alertmanager(
report,
user_id=user_id,
username=username,
display=display,
)
if sent_via_am:
await message.reply_text("תודה! הדיווח נשלח לאדמין.")
return

sent_direct = await _send_direct_admins(context, report)
if sent_direct:
await message.reply_text("כרגע Alertmanager לא זמין, אבל שלחתי את הדיווח ישירות לאדמין.")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fallback bypasses Rule Engine and internal alerts

High Severity

The new /admin command flow bypasses notify_admins() entirely. When Alertmanager fails and _send_direct_admins() is used as fallback, messages are sent directly via bot.send_message, which bypasses both emit_internal_alert() and the Rule Engine's suppress/routing logic. The existing notify_admins() function (line 734) explicitly documents that admin messages should not be sent directly to avoid bypassing the Rule Engine. This breaks alert consolidation, suppression rules, and observability for fallback cases.

Fix in Cursor Fix in Web

else:
await message.reply_text("לא הצלחתי לשלוח את הדיווח כרגע.")
except Exception:
Expand Down
19 changes: 19 additions & 0 deletions services/config_inspector_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -860,6 +860,25 @@ class ConfigService:
description="רשימת IP מותרים ל-Alertmanager",
category="alerts",
),
"ALERTMANAGER_API_URL": ConfigDefinition(
key="ALERTMANAGER_API_URL",
default="",
description="כתובת בסיס של Alertmanager API (למשל https://...); משמשת לשליחה ל-/api/v2/alerts (לדוגמה עבור /admin)",
category="alerts",
),
"ALERTMANAGER_API_TOKEN": ConfigDefinition(
key="ALERTMANAGER_API_TOKEN",
default="",
description="טוקן Bearer אופציונלי ל-Alertmanager API (Authorization: Bearer ...), עבור /api/v2/alerts",
category="alerts",
sensitive=True,
),
"ALERTMANAGER_TARGET": ConfigDefinition(
key="ALERTMANAGER_TARGET",
default="",
description="יעד Alertmanager בפורמט host:port (משמש גם כ-fallback לבניית URL ל-/api/v2/alerts אם ALERTMANAGER_API_URL לא מוגדר)",
category="alerts",
),
"ALLOWED_WEBHOOK_HOSTS": ConfigDefinition(
key="ALLOWED_WEBHOOK_HOSTS",
default="",
Expand Down
100 changes: 100 additions & 0 deletions tests/test_admin_report_command_alertmanager_ack.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
import types
import pytest


@pytest.mark.asyncio
async def test_admin_report_ack_only_on_alertmanager_success(monkeypatch):
import main as main_mod

class Msg:
def __init__(self):
self.replies = []

async def reply_text(self, text):
self.replies.append(text)

msg = Msg()
upd = types.SimpleNamespace(
message=msg,
effective_message=msg,
effective_user=types.SimpleNamespace(id=123, username="alice", full_name="Alice"),
)
ctx = types.SimpleNamespace(args=["יש", "בעיה"])

async def _am_ok(*_a, **_k):
return True

async def _direct_should_not_run(*_a, **_k):
raise AssertionError("_send_direct_admins should not be called when Alertmanager succeeds")

monkeypatch.setattr(main_mod, "_send_admin_report_via_alertmanager", _am_ok)
monkeypatch.setattr(main_mod, "_send_direct_admins", _direct_should_not_run)

await main_mod.admin_report_command(upd, ctx)
assert msg.replies and msg.replies[-1] == "תודה! הדיווח נשלח לאדמין."


@pytest.mark.asyncio
async def test_admin_report_fallback_message_when_alertmanager_fails(monkeypatch):
import main as main_mod

class Msg:
def __init__(self):
self.replies = []

async def reply_text(self, text):
self.replies.append(text)

msg = Msg()
upd = types.SimpleNamespace(
message=msg,
effective_message=msg,
effective_user=types.SimpleNamespace(id=7, username=None, full_name="Bob"),
)
ctx = types.SimpleNamespace(args=["הודעה"])

async def _am_fail(*_a, **_k):
return False

async def _direct_ok(*_a, **_k):
return True

monkeypatch.setattr(main_mod, "_send_admin_report_via_alertmanager", _am_fail)
monkeypatch.setattr(main_mod, "_send_direct_admins", _direct_ok)

await main_mod.admin_report_command(upd, ctx)
assert msg.replies
assert "Alertmanager לא זמין" in msg.replies[-1]


@pytest.mark.asyncio
async def test_admin_report_failure_when_both_paths_fail(monkeypatch):
import main as main_mod

class Msg:
def __init__(self):
self.replies = []

async def reply_text(self, text):
self.replies.append(text)

msg = Msg()
upd = types.SimpleNamespace(
message=msg,
effective_message=msg,
effective_user=types.SimpleNamespace(id=7, username="u", full_name="U"),
)
ctx = types.SimpleNamespace(args=["הודעה"])

async def _am_fail(*_a, **_k):
return False

async def _direct_fail(*_a, **_k):
return False

monkeypatch.setattr(main_mod, "_send_admin_report_via_alertmanager", _am_fail)
monkeypatch.setattr(main_mod, "_send_direct_admins", _direct_fail)

await main_mod.admin_report_command(upd, ctx)
assert msg.replies and msg.replies[-1] == "לא הצלחתי לשלוח את הדיווח כרגע."

Loading
Loading