pmb/yttrx-welcomebot
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add read-only blocklist API (off by default)
docker-build-push / build-push (push) Successful in 5s
Details

Browse Source 
Publishes the IP ranges and email domains the bot has auto-blocked as
JSON: GET /api/blocks, /api/blocks/ip-ranges, /api/blocks/email-domains,
each taking an optional ?limit=N.

The endpoints are unauthenticated, so BLOCKS_API_ENABLED gates them and
defaults to false — with it unset the paths 404 as if they didn't exist.
Publishing which ranges are already blocked also tells a spam signup
which ones to avoid, so opting in is a per-deployment call; yttrx.com
sets it true in the box's .env.

Responses assert only that a block exists: the blocked range/domain, a
signups count, and first/last seen. No account identifiers, and none of
the ipapi.is or check-mail.org classification that drove the decision —
republishing a paid reputation feed's verdicts on an open endpoint would
be redistributing their data rather than ours.

Supporting changes:

- signup_ip gains a block_cidr column (migrated in place on open) so the
  API reports the network actually blocked rather than the signup IP.
  Rows predating it fall back to that IP's /32 or /128, which is exactly
  what was registered before range-blocking started 2026-07-15.
- register_ip_block / register_email_domain_block now return whether the
  block landed, and the caller only marks it registered on success, so a
  failed POST is no longer published as a block that exists.
This commit is contained in:
pmb committed 2026-09-13 00:41:48 -07:00
1 parent 1a3e112946
commit eb6d7cad46
6 files changed
+422 -22
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/main.py
+222 -15
View File
@@ -29,6 +29,11 @@ Mastodon signs every delivery with:
One Mastodon webhook subscribing to both events points at /webhook; both
share the single WEBHOOK_SECRET. See README.md for the deploy + Mastodon-side
configuration runbook.
Alongside the webhook, an optional read-only blocklist API publishes what the
bot has auto-blocked — ``GET /api/blocks``, ``/api/blocks/ip-ranges``,
``/api/blocks/email-domains``. It is unauthenticated and therefore off unless
``BLOCKS_API_ENABLED`` is set; see that config entry.
"""
from __future__ import annotations
@@ -45,6 +50,7 @@ from datetime import datetime, timedelta, timezone
import httpx
from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Request
from fastapi.responses import JSONResponse
logging.basicConfig(
level=logging.INFO,
@@ -220,6 +226,23 @@ SUSPICIOUS_COMBINED_ACTION = os.environ.get("SUSPICIOUS_COMBINED_ACTION", "suspe
# by-design "false" default above) since this is a brand-new action path.
SUSPICIOUS_COMBINED_DRY_RUN = os.environ.get("SUSPICIOUS_COMBINED_DRY_RUN", "true").lower() in ("1", "true", "yes")
# --- Public blocklist API ---------------------------------------------------
# Read-only JSON of the blocks this bot has registered (GET /api/blocks and
# friends). Served WITHOUT authentication when enabled, so it defaults to
# "off" and must be opted into per-deployment: publishing which ranges and
# domains are already blocked also tells a would-be spam signup which ones to
# avoid, and that's a per-instance call, not a safe default. yttrx.com sets
# BLOCKS_API_ENABLED=true in its .env; every other deployment 404s.
BLOCKS_API_ENABLED = os.environ.get("BLOCKS_API_ENABLED", "false").lower() in ("1", "true", "yes")
# Hard ceiling on how many entries one response may return, whatever ?limit
# asks for — this endpoint is unauthenticated, so the response size can't be
# left to the caller.
BLOCKS_API_MAX_ITEMS = int(os.environ.get("BLOCKS_API_MAX_ITEMS", "5000"))
# Cache-Control max-age (seconds) on blocklist responses. The underlying data
# changes only when a signup is flagged, so letting clients and any
# intermediate cache hold it for a few minutes costs nothing.
BLOCKS_API_CACHE_SECONDS = int(os.environ.get("BLOCKS_API_CACHE_SECONDS", "300"))
if not WEBHOOK_SECRET:
log.warning("WEBHOOK_SECRET is empty — signature verification will reject all requests.")
if not BOT_ACCESS_TOKEN:
@@ -238,6 +261,10 @@ app = FastAPI(title="yttrx welcome-bot + abuse-bot")
_db_lock = threading.Lock()
def _column_exists(conn: sqlite3.Connection, table: str, column: str) -> bool:
return any(row[1] == column for row in conn.execute(f"PRAGMA table_info({table})"))
def _init_db() -> None:
os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
with sqlite3.connect(DB_PATH) as conn:
@@ -268,9 +295,18 @@ def _init_db() -> None:
" org TEXT,"
" flagged INTEGER,"
" ipblock_registered INTEGER DEFAULT 0,"
" block_cidr TEXT,"
" classified_at TEXT DEFAULT CURRENT_TIMESTAMP"
")"
)
# block_cidr was added after the table shipped (it records the network
# register_ip_block actually blocks, which is what /api/blocks
# publishes). Rows written before this column existed keep it NULL and
# fall back to the single-address /32 or /128 of their signup IP —
# which is exactly what was registered back then, since blocking the
# ASN route CIDR only started 2026-07-15.
if not _column_exists(conn, "signup_ip", "block_cidr"):
conn.execute("ALTER TABLE signup_ip ADD COLUMN block_cidr TEXT")
conn.execute(
"CREATE TABLE IF NOT EXISTS ipapi_range_cache ("
" family INTEGER NOT NULL," # 4 or 6
@@ -368,15 +404,16 @@ def record_action(target_id: str, acct: str, action: str, tier: str,
def record_signup_ip(account_id: str, acct: str, ip: str, classification: str,
org: str, flagged: bool) -> None:
org: str, flagged: bool, block_cidr: str = "") -> None:
with _db() as conn:
conn.execute(
"INSERT OR REPLACE INTO signup_ip "
"(account_id, acct, ip, classification, org, flagged, "
" ipblock_registered) "
" ipblock_registered, block_cidr) "
"VALUES (?, ?, ?, ?, ?, ?, "
" COALESCE((SELECT ipblock_registered FROM signup_ip WHERE account_id = ?), 0))",
(account_id, acct, ip, classification, org, int(flagged), account_id),
" COALESCE((SELECT ipblock_registered FROM signup_ip WHERE account_id = ?), 0), ?)",
(account_id, acct, ip, classification, org, int(flagged), account_id,
block_cidr or None),
)
@@ -526,6 +563,104 @@ def start_suspicious_watch(account_id: str, acct: str, reasons: str,
)
def _fallback_cidr(ip: str) -> str:
"""The single-address CIDR for ip, used for signup_ip rows written before
block_cidr existed — that's the block that was registered for them."""
try:
addr = ipaddress.ip_address(ip)
except ValueError:
return ""
return f"{addr}/{32 if addr.version == 4 else 128}"
def blocked_ip_ranges(limit: int = 0) -> list[dict]:
"""Every network this bot has registered an Admin::IpBlock for, newest
first, collapsed to one entry per distinct CIDR.
Repeat signups out of the same range are common (that's the point of
blocking the range rather than the address), so entries carry a `signups`
count and the first/last time the range was hit rather than appearing once
per account.
Entries state only *that* a range is blocked. The ipapi.is classification
and org that drove the decision stay out of the response on purpose: this
is an unauthenticated endpoint, and republishing a paid IP-reputation
feed's verdicts would be redistributing their data, not ours. The same
goes for identifying anything about the accounts involved.
"""
with _db() as conn:
rows = conn.execute(
"SELECT block_cidr, ip, classified_at "
"FROM signup_ip WHERE ipblock_registered = 1 "
"ORDER BY classified_at DESC"
).fetchall()
grouped: dict[str, dict] = {}
for block_cidr, ip, classified_at in rows:
cidr = block_cidr or _fallback_cidr(ip or "")
if not cidr:
continue
entry = grouped.get(cidr)
if entry is None:
grouped[cidr] = {
"cidr": cidr,
"signups": 1,
"first_blocked_at": classified_at,
"last_blocked_at": classified_at,
}
else:
# rows are newest-first, so every later row is an older sighting
entry["signups"] += 1
entry["first_blocked_at"] = classified_at
return _capped(list(grouped.values()), limit)
def blocked_email_domains(limit: int = 0) -> list[dict]:
"""Every domain this bot has registered an Admin::EmailDomainBlock for,
newest first, one entry per distinct domain.
Domains only — the bot never stores or forwards a full signup address (see
classify_email_domain), so there is none to expose here.
As with blocked_ip_ranges, entries state only *that* a domain is blocked:
check-mail.org's disposable flag and risk score stay out of the response
so this doesn't republish a third-party reputation feed's verdicts.
"""
with _db() as conn:
rows = conn.execute(
"SELECT domain, classified_at "
"FROM signup_email WHERE domain_blocked = 1 "
"ORDER BY classified_at DESC"
).fetchall()
grouped: dict[str, dict] = {}
for domain, classified_at in rows:
if not domain:
continue
entry = grouped.get(domain)
if entry is None:
grouped[domain] = {
"domain": domain,
"signups": 1,
"first_blocked_at": classified_at,
"last_blocked_at": classified_at,
}
else:
entry["signups"] += 1
entry["first_blocked_at"] = classified_at
return _capped(list(grouped.values()), limit)
def _capped(items: list[dict], limit: int) -> list[dict]:
"""Apply a caller's ?limit on top of BLOCKS_API_MAX_ITEMS. limit <= 0
means "as many as the ceiling allows"."""
ceiling = max(BLOCKS_API_MAX_ITEMS, 0)
n = ceiling if limit <= 0 else min(limit, ceiling)
return items[:n]
_init_db()
# --- Signature verification ------------------------------------------------
@@ -604,12 +739,16 @@ def classify_email_domain(domain: str) -> tuple[bool, int]:
return is_disposable, risk
def register_email_domain_block(domain: str, acct: str) -> None:
def register_email_domain_block(domain: str, acct: str) -> bool:
"""Register a flagged signup's email domain in Mastodon's native
Admin::EmailDomainBlock, blocking future signups from that domain.
Requires the admin:write:email_domain_blocks OAuth scope on
ABUSE_BOT_TOKEN — see CLAUDE.md's moderator-token-gotcha section.
Returns True if the domain is blocked on the Mastodon side afterwards
(including the 422 "already blocked" case), so the caller only records it
as blocked when it really is — /api/blocks publishes those rows.
"""
try:
resp = httpx.post(
@@ -624,6 +763,8 @@ def register_email_domain_block(domain: str, acct: str) -> None:
resp.raise_for_status()
except httpx.HTTPError as exc:
log.error("failed to register email_domain_block for %s (acct=%s): %s", domain, acct, exc)
return False
return True
def process_signup(account_id: str, acct: str, ip: str, email: str = "",
@@ -662,15 +803,15 @@ def process_signup(account_id: str, acct: str, ip: str, email: str = "",
if IP_SCRUTINY_ENABLED and ip:
classification, org, ip_flagged, block_cidr = classify_signup_ip(ip)
record_signup_ip(account_id, acct, ip, classification, org, ip_flagged)
record_signup_ip(account_id, acct, ip, classification, org, ip_flagged, block_cidr)
if ip_flagged:
ip_strong_flagged = bool(set(classification.split("+")) & IP_SCRUTINY_STRONG_SIGNALS)
log.warning("flagged signup acct=%s ip=%s classification=%s org=%s",
acct, ip, classification, org)
if IP_SCRUTINY_AUTO_IPBLOCK and not IP_SCRUTINY_DRY_RUN:
register_ip_block(ip, acct, org, block_cidr)
mark_ipblock_registered(account_id)
if register_ip_block(ip, acct, org, block_cidr):
mark_ipblock_registered(account_id)
prefix = "[DRY-RUN] " if IP_SCRUTINY_DRY_RUN else ""
reasons.append(f"{prefix}IP {ip} ({org or 'unknown org'}, {classification})")
hold = hold or (IP_SCRUTINY_HOLD_WELCOME and not IP_SCRUTINY_DRY_RUN)
@@ -685,8 +826,8 @@ def process_signup(account_id: str, acct: str, ip: str, email: str = "",
log.warning("flagged signup acct=%s email_domain=%s is_disposable=%s risk=%d",
acct, domain, is_disposable, risk)
if CHECK_MAIL_AUTO_DOMAIN_BLOCK and not CHECK_MAIL_DRY_RUN:
register_email_domain_block(domain, acct)
mark_email_domain_blocked(account_id)
if register_email_domain_block(domain, acct):
mark_email_domain_blocked(account_id)
prefix = "[DRY-RUN] " if CHECK_MAIL_DRY_RUN else ""
reasons.append(f"{prefix}email domain {domain} (disposable={is_disposable}, risk={risk})")
hold = hold or (CHECK_MAIL_HOLD_WELCOME and not CHECK_MAIL_DRY_RUN)
@@ -975,7 +1116,7 @@ def classify_signup_ip(ip: str) -> tuple[str, str, bool, str]:
return classification, intel["org"], bool(reasons), intel.get("cidr") or f"{ip}/32"
def register_ip_block(ip: str, acct: str, org: str, cidr: str = "") -> None:
def register_ip_block(ip: str, acct: str, org: str, cidr: str = "") -> bool:
"""Register a flagged signup's network in Mastodon's native Admin::IpBlock.
cidr should be classify_signup_ip's block_cidr — the ASN/route-level
@@ -983,13 +1124,17 @@ def register_ip_block(ip: str, acct: str, org: str, cidr: str = "") -> None:
datacenter/VPN/proxy block is blocked too, not just this one address.
Falls back to a single-address block if no cidr is given or ip itself
doesn't parse (defensive only; classify_signup_ip always supplies one).
Returns True if the range is blocked on the Mastodon side afterwards
(including the 422 "already blocked" case), so the caller only records it
as blocked when it really is — /api/blocks publishes those rows.
"""
if not cidr:
try:
prefix_len = 32 if ipaddress.ip_address(ip).version == 4 else 128
except ValueError:
log.warning("skipping ip_block registration for unparseable ip=%r (acct=%s)", ip, acct)
return
return False
cidr = f"{ip}/{prefix_len}"
comment = f"welcomebot: flagged signup @{acct} ({org or 'unknown org'})"[:200]
try:
@@ -1005,7 +1150,8 @@ def register_ip_block(ip: str, acct: str, org: str, cidr: str = "") -> None:
resp.raise_for_status()
except httpx.HTTPError as exc:
log.error("failed to register ip_block for %s (acct=%s): %s", cidr, acct, exc)
return
return False
return True
def count_distinct_reporters(target_id: str) -> int:
@@ -1147,8 +1293,8 @@ def handle_report(report_id: str, target_id: str, acct: str, is_local: bool,
if CHECK_MAIL_AUTO_DOMAIN_BLOCK:
record_signup_email(target_id, acct, domain, is_disposable, risk, True)
register_email_domain_block(domain, acct)
mark_email_domain_blocked(target_id)
if register_email_domain_block(domain, acct):
mark_email_domain_blocked(target_id)
return
try:
@@ -1215,6 +1361,67 @@ def healthz():
return {"ok": True}
# --- Blocklist API ---
# Read-only JSON view of what this bot has auto-blocked, served unauthenticated
# when BLOCKS_API_ENABLED is on (off by default — see the config block). It
# reports only the bot's own record: blocks a human added by hand in Mastodon's
# admin UI are not in here. Responses assert only that a range or domain is
# blocked — no account handles, account ids or email addresses, and none of
# the ipapi.is / check-mail.org classification data behind the decision (see
# blocked_ip_ranges).
def _blocks_guard(limit: int) -> int:
"""404 unless the API is switched on for this deployment; a disabled
endpoint should be indistinguishable from one that doesn't exist. Returns
the sanitised limit."""
if not BLOCKS_API_ENABLED:
raise HTTPException(status_code=404, detail="Not Found")
return max(limit, 0)
def _blocks_response(payload: dict) -> JSONResponse:
return JSONResponse(
payload,
headers={"Cache-Control": f"public, max-age={BLOCKS_API_CACHE_SECONDS}"},
)
def _now_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
@app.get("/api/blocks")
def api_blocks(limit: int = 0):
"""Both blocklists in one document."""
limit = _blocks_guard(limit)
ip_ranges = blocked_ip_ranges(limit)
email_domains = blocked_email_domains(limit)
return _blocks_response({
"generated_at": _now_iso(),
"ip_ranges": {"count": len(ip_ranges), "items": ip_ranges},
"email_domains": {"count": len(email_domains), "items": email_domains},
})
@app.get("/api/blocks/ip-ranges")
def api_blocks_ip_ranges(limit: int = 0):
limit = _blocks_guard(limit)
items = blocked_ip_ranges(limit)
return _blocks_response({
"generated_at": _now_iso(), "count": len(items), "items": items,
})
@app.get("/api/blocks/email-domains")
def api_blocks_email_domains(limit: int = 0):
limit = _blocks_guard(limit)
items = blocked_email_domains(limit)
return _blocks_response({
"generated_at": _now_iso(), "count": len(items), "items": items,
})
@app.post("/webhook")
async def webhook(
request: Request,