From af49aab79ca5ec8b3e65c9da79113ae548b564fb Mon Sep 17 00:00:00 2001 From: waffle2k Date: Sun, 13 Sep 2026 00:48:20 -0700 Subject: [PATCH] Backfill block_cidr so /api/blocks reports real ranges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every row blocked before the block_cidr column existed had nothing to report, so the endpoint fell back to the signup IP's /32 — the address that signed up rather than the range that was blocked, which is the opposite of what the list is for. The range is recoverable from the bot's own ipapi range cache, which stores the CIDR each classification was cached under — the same value that was handed to register_ip_block. Run once on admin.yttrx.com: of 154 blocked rows, 65 gained their real range, 79 genuinely were single-address blocks (no asn.route from ipapi.is), and 10 with no cache entry stay as a /32, since under-reporting beats claiming a range is blocked when it isn't. Rows classified before 2026-07-15 are skipped — range-blocking didn't exist yet, so only the /32 was ever registered. Live output went from 150 entries (all /32) to 145, of which 59 are real networks. --- README.md | 11 ++++++ bin/backfill-block-cidr.py | 68 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100755 bin/backfill-block-cidr.py diff --git a/README.md b/README.md index 37fa5b0..13e2523 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,16 @@ What the response deliberately does **not** contain: paid reputation feed's verdicts on an open endpoint would be redistributing their data rather than ours. An entry asserts only that the block exists. +Entries report the **network** that was blocked (ipapi.is's `asn.route` for +the signup IP, which is what `register_ip_block` registers), not the address +that signed up. A `/32` or `/128` entry means a single address really is what +was blocked — either the classification returned no usable route CIDR, or the +signup predates range-blocking (2026-07-15). The `block_cidr` column recording +this arrived with the API itself, so existing rows were backfilled once from +the ipapi range cache by `bin/backfill-block-cidr.py`; ten rows with no cache +entry left are shown as their `/32`, since a narrower entry is a safer error +than claiming a range is blocked when it isn't. + Scope: this is the **bot's own record**, not Mastodon's full blocklist — blocks a moderator added by hand in the admin UI are not in it. Entries appear only once the block actually landed on the Mastodon side (a failed @@ -258,6 +268,7 @@ only once the block actually landed on the Mastodon side (a failed | `.env.example` | Config template (copy to `.env`, never commit) | | `nginx/hooks.yttrx.com.conf` | nginx site for admin.yttrx.com (webhook, healthz, blocklist API) | | `bin/welcomebot-logs`, `bin/welcomebot-signups`, `bin/welcomebot-suspicious` | Ops CLIs installed to `/root/bin` on admin.yttrx.com | +| `bin/backfill-block-cidr.py` | One-shot: reconstruct `block_cidr` for rows predating it (run 2026-09-13) | | `test_local.py` | Offline smoke test (no network) | ## Configuration diff --git a/bin/backfill-block-cidr.py b/bin/backfill-block-cidr.py new file mode 100755 index 0000000..0772502 --- /dev/null +++ b/bin/backfill-block-cidr.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""One-shot backfill of signup_ip.block_cidr, run once on admin.yttrx.com +2026-09-13 when the blocklist API shipped. + +block_cidr records the network register_ip_block actually blocked. It didn't +exist before the API did, so every pre-existing blocked row had nothing to +report and /api/blocks fell back to its signup IP's /32 — narrower than what +is really blocked, and not what the endpoint is for. The range is recoverable +from the bot's own ipapi range cache, which stores the CIDR each +classification was cached under (the same value that was handed to +register_ip_block). + +Run inside the container, dry-run first: + + docker cp bin/backfill-block-cidr.py yttrx-welcomebot:/tmp/ + docker exec -e PYTHONPATH=/app yttrx-welcomebot python /tmp/backfill-block-cidr.py + docker exec -e PYTHONPATH=/app -e APPLY=1 yttrx-welcomebot python /tmp/backfill-block-cidr.py + +Result of the 2026-09-13 run: 154 blocked rows, 65 backfilled to a real range, +79 that genuinely were single-address blocks (ipapi.is returned no asn.route +for them), 10 with no cache entry at all whose range can't be reconstructed +and stay as a /32. +""" +import os + +import app.main as m + +# Route-CIDR blocking landed 2026-07-15 (commit b39528a); the ipapi range +# cache landed 2026-07-13. Only signups blocked after the former had a network +# CIDR handed to register_ip_block — for anything earlier only the /32 was +# ever registered, so backfilling a wider range from the cache would publish a +# block that doesn't exist. Under-reporting is safe; over-reporting isn't. +CUTOFF = "2026-07-16" +APPLY = os.environ.get("APPLY") == "1" + +# NB: _db_lock is not reentrant — never call cached_ip_intel() inside a _db() +# block, it opens its own. +with m._db() as conn: + rows = conn.execute( + "SELECT account_id, ip, classified_at FROM signup_ip " + "WHERE ipblock_registered = 1 AND block_cidr IS NULL" + ).fetchall() + +updates, skipped_old, no_cache = [], 0, 0 +for account_id, ip, classified_at in rows: + intel = m.cached_ip_intel(ip or "") + cidr = (intel or {}).get("cidr") or "" + if not cidr or cidr.endswith(("/32", "/128")): + no_cache += 1 + continue + if (classified_at or "") < CUTOFF: + skipped_old += 1 + continue + updates.append((cidr, account_id)) + +print(f"rows missing block_cidr : {len(rows)}") +print(f"no wider cached CIDR : {no_cache} (stay as their /32 or /128)") +print(f"pre-{CUTOFF} rows : {skipped_old} (only a /32 was ever blocked)") +print(f"to backfill : {len(updates)}") +for cidr, account_id in updates[:5]: + print(f" e.g. account {account_id} -> {cidr}") + +if APPLY: + with m._db() as conn: + conn.executemany("UPDATE signup_ip SET block_cidr = ? WHERE account_id = ?", updates) + print(f"applied {len(updates)} updates") +else: + print("DRY RUN — set APPLY=1 to write")