Add a reputation query API for pre-signup blocking

Everything the bot does today is reactive: Mastodon creates the account,
the account.created webhook arrives, we classify the signup IP and email
domain, and then we block the range and suspend what already exists.

POST /api/reputation is the pre-emptive half. A caller asks about an IP
and/or email domain; we classify it, record the answer, register the
matching Admin::IpBlock / EmailDomainBlock, and only then return. A
caller that runs its own block lookup immediately afterwards therefore
finds a block that did not exist when its request started. The intended
caller is Mastodon's signup path, which does exactly that.

It writes, so unlike /api/blocks it is authenticated and refuses to serve
without a key (REPUTATION_API_KEY): an unauthenticated endpoint that
installs IP blocks is a denial-of-service lever, so there is no open mode.

Notes on the pieces that aren't obvious:

- REPUTATION_API_MAX_LOOKUPS_PER_HOUR caps *fresh* provider lookups. The
  caller hits this on every view of the signup form, not just on
  submission, so without a ceiling a crawler walking /auth/sign_up from a
  new range each time would set our ipapi.is bill. Over budget we answer
  "unknown" and install nothing -- never "flagged". Cache hits are free
  and uncounted, and the webhook path never spends from this budget.

- Blocks installed here are recorded in their own table, keyed by the
  CIDR/domain rather than by account (there is no account yet), and are
  folded into /api/blocks and the reconcile pass so neither one
  under-reports as blocking shifts to the pre-signup path. The published
  entry shape is unchanged; a range blocked before anyone signed up from
  it simply reads "signups": 0.

- A repeat query for an already-installed block skips the Mastodon round
  trip. This runs on a live signup request, so that is saved latency.

test_local.py

@@ -979,6 +979,209 @@ def reconcile_blocks_tests():
     assert "198.18.5.0/24" in active_ips(), "re-registering must reactivate the row"
 
 
+def reputation_api_tests():
+    """Drive POST /api/reputation — the pre-signup query path Mastodon calls
+    before its own IpBlock/EmailDomainBlock lookup. The load-bearing property
+    is ordering: the block must be installed by the time the call returns,
+    because the caller looks it up microseconds later."""
+    dms = []
+    ipblocks = []
+    domain_blocks = []
+    main.dm_moderator = lambda message: dms.append(message)
+    main.register_ip_block = (
+        lambda ip, acct, org, cidr="", comment="":
+        ipblocks.append((ip, cidr, comment)) or True)
+    main.register_email_domain_block = (
+        lambda domain, acct: domain_blocks.append((domain, acct)) or True)
+
+    classifications = {
+        "203.0.113.50": ("clean", "Example Residential ISP", False, "203.0.113.50/32"),
+        "198.51.100.50": ("datacenter", "Example Cloud Inc", True, "198.51.100.0/24"),
+        "198.51.100.51": ("datacenter", "Example Cloud Inc", True, "198.51.100.0/24"),
+        "198.51.100.60": ("proxy", "Example Proxy Co", True, "198.51.100.64/26"),
+    }
+    main.classify_signup_ip = lambda ip: classifications[ip]
+    main.classify_email_domain = lambda domain: {
+        "rep-clean.test": (False, 0),
+        "rep-spam.test": (True, 99),
+    }[domain]
+
+    main.IP_SCRUTINY_ENABLED = True
+    main.IP_SCRUTINY_DRY_RUN = False
+    main.IP_SCRUTINY_AUTO_IPBLOCK = True
+    main.CHECK_MAIL_ENABLED = True
+    main.CHECK_MAIL_API_KEY = "dummy"
+    main.CHECK_MAIL_DRY_RUN = False
+    main.CHECK_MAIL_AUTO_DOMAIN_BLOCK = True
+    main.CHECK_MAIL_RISK_THRESHOLD = 80
+    main.REPUTATION_API_INSTALL_BLOCKS = True
+    main.REPUTATION_API_DM_MODERATOR = True
+    main.REPUTATION_API_MAX_LOOKUPS_PER_HOUR = 0  # no ceiling except where tested
+
+    def ask(payload, key="rep-secret"):
+        headers = {"X-Reputation-Key": key} if key is not None else {}
+        return client.post("/api/reputation", json=payload, headers=headers)
+
+    # A. The gate. Disabled, or enabled with no key configured, is a 404 —
+    #    a route that installs blocks must never be reachable unauthenticated,
+    #    and "off" should look like "not a route".
+    main.REPUTATION_API_ENABLED = False
+    main.REPUTATION_API_KEY = "rep-secret"
+    assert ask({"ip": "203.0.113.50"}).status_code == 404
+
+    main.REPUTATION_API_ENABLED = True
+    main.REPUTATION_API_KEY = ""
+    assert ask({"ip": "203.0.113.50"}).status_code == 404, \
+        "enabled without a key must stay closed, not open"
+
+    main.REPUTATION_API_KEY = "rep-secret"
+    assert ask({"ip": "203.0.113.50"}, key="wrong").status_code == 401
+    assert ask({"ip": "203.0.113.50"}, key=None).status_code == 401
+
+    # B. Something to look up is required.
+    assert ask({}).status_code == 400
+    assert ask({"source": "registration"}).status_code == 400
+
+    # C. A clean IP: answered, recorded, nothing installed.
+    ipblocks.clear(); dms.clear()
+    r = ask({"ip": "203.0.113.50", "source": "registration"})
+    assert r.status_code == 200, r.status_code
+    body = r.json()
+    assert body["flagged"] is False and body["blocked"] is False, body
+    assert body["ip"]["classification"] == "clean"
+    assert body["email"] is None, "no email asked about, no email answer"
+    assert ipblocks == [], "a clean IP must not be blocked"
+    assert dms == [], "a clean IP must not wake a moderator"
+
+    # D. THE POINT OF THE FEATURE: a flagged IP is blocked *during* the call,
+    #    so the caller's own IpBlock lookup right afterwards finds it.
+    r = ask({"ip": "198.51.100.50", "source": "registration"})
+    body = r.json()
+    assert body["flagged"] is True and body["blocked"] is True, body
+    assert body["ip"]["cidr"] == "198.51.100.0/24"
+    assert len(ipblocks) == 1, ipblocks
+    assert ipblocks[0][1] == "198.51.100.0/24", "the range is blocked, not the address"
+    assert "pre-signup" in ipblocks[0][2], "the block says why it exists"
+    assert any("198.51.100.0/24" in d for d in dms), dms
+    assert main.reputation_block_installed("ip", "198.51.100.0/24")
+
+    # E. A second address in the same range: still blocked, but we don't
+    #    re-POST a block we already installed — this runs on the signup
+    #    request path, so the saved round trip is saved latency.
+    dms.clear()
+    r = ask({"ip": "198.51.100.51"})
+    assert r.json()["blocked"] is True, "an already-installed block still counts as blocked"
+    assert len(ipblocks) == 1, "an already-installed block must not be re-POSTed"
+    assert dms == [], "only a newly-installed block wakes a moderator"
+
+    # F. REPUTATION_API_INSTALL_BLOCKS=false classifies and records but
+    #    installs nothing — the dry-run equivalent for this path.
+    main.REPUTATION_API_INSTALL_BLOCKS = False
+    r = ask({"ip": "198.51.100.60"})
+    body = r.json()
+    assert body["flagged"] is True, body
+    assert body["blocked"] is False, "install-blocks off must not install"
+    assert len(ipblocks) == 1, ipblocks
+    main.REPUTATION_API_INSTALL_BLOCKS = True
+
+    # G. The email side, and the standing rule that a full address is never
+    #    stored, logged or echoed — only its domain.
+    domain_blocks.clear(); dms.clear()
+    r = ask({"ip": "203.0.113.50", "email": "[email protected]", "source": "sign_up"})
+    body = r.json()
+    assert body["email"]["domain"] == "rep-spam.test", "domain is normalised to lower case"
+    assert body["email"]["flagged"] is True and body["email"]["blocked"] is True
+    assert body["blocked"] is True, "either side flagging makes the whole answer blocked"
+    assert body["ip"]["flagged"] is False, "the two sides are independent"
+    assert domain_blocks and domain_blocks[0][0] == "rep-spam.test", domain_blocks
+    assert "SpamGuy" not in r.text and "spamguy" not in r.text.lower(), \
+        "the local part must never come back in the response"
+    with main._db() as conn:
+        stored = conn.execute(
+            "SELECT value FROM reputation_query WHERE kind = 'email'").fetchall()
+    assert all("@" not in v[0] for v in stored), \
+        "reputation_query must store domains, never addresses"
+
+    # H. A clean domain is answered but not blocked.
+    domain_blocks.clear()
+    r = ask({"email": "[email protected]"})
+    assert r.json()["email"]["flagged"] is False
+    assert r.json()["blocked"] is False
+    assert domain_blocks == [], domain_blocks
+
+    # I. The hourly ceiling on fresh provider lookups. Over budget, an
+    #    uncached value answers "unknown" and blocks nothing — the guard
+    #    exists to stop paying, so it must never guess "flagged" instead.
+    #    Addresses here must fall outside every range cached by earlier tests,
+    #    or they'd be cache hits and never reach the budget at all.
+    main._lookup_budget.clear()
+    main.REPUTATION_API_MAX_LOOKUPS_PER_HOUR = 1
+    classifications["198.51.102.10"] = ("datacenter", "Example Cloud Inc", True,
+                                        "198.51.102.0/25")
+    classifications["198.51.102.200"] = ("datacenter", "Other Cloud Inc", True,
+                                         "198.51.102.128/25")
+    before = len(ipblocks)
+    first = ask({"ip": "198.51.102.10"}).json()["ip"]
+    assert first["cached"] is False and first["deferred"] is False, first
+    assert len(ipblocks) == before + 1, "the in-budget lookup still works normally"
+
+    over = ask({"ip": "198.51.102.200"}).json()["ip"]
+    assert over["deferred"] is True, over
+    assert over["classification"] == "unknown" and over["flagged"] is False, over
+    assert over["blocked"] is False
+    assert len(ipblocks) == before + 1, "an over-budget lookup must install nothing"
+    assert not main.reputation_block_installed("ip", "198.51.102.128/25")
+
+    #    A cache hit costs nothing, so it is answered normally even with the
+    #    budget spent — the ceiling limits provider spend, not service.
+    cached_hit = ask({"ip": "198.51.100.51"}).json()["ip"]
+    assert cached_hit["cached"] is True and cached_hit["deferred"] is False, cached_hit
+    assert cached_hit["blocked"] is True, "a cached flagged range still answers blocked"
+    main.REPUTATION_API_MAX_LOOKUPS_PER_HOUR = 0
+    main._lookup_budget.clear()
+
+    # J. Blocks installed here show up in the public blocklist alongside the
+    #    signup-derived ones — otherwise the published list under-reports as
+    #    blocking shifts to the pre-signup path. `signups` still counts
+    #    signups, so a range blocked before anyone signed up reads 0.
+    main.BLOCKS_API_ENABLED = True
+    ranges = {e["cidr"]: e for e in client.get("/api/blocks/ip-ranges").json()["items"]}
+    domains = {e["domain"]: e for e in client.get("/api/blocks/email-domains").json()["items"]}
+    assert "198.51.100.0/24" in ranges, ranges
+    assert ranges["198.51.100.0/24"]["signups"] == 0, \
+        "nothing signed up from this range — it was blocked first"
+    assert "rep-spam.test" in domains, domains
+    assert set(ranges["198.51.100.0/24"]) == {
+        "cidr", "signups", "first_blocked_at", "last_blocked_at"}, \
+        "merging must not change the published entry shape"
+
+    # K. ...and the classification behind them stays out of the public list,
+    #    same rule as for the signup-derived entries.
+    raw = client.get("/api/blocks").text
+    for leak in ("Example Cloud Inc", "Example Proxy Co", "datacenter", "proxy"):
+        assert leak not in raw, f"{leak!r} must not reach the public blocklist"
+
+    # L. The reconcile pass covers these rows too: a block a moderator removes
+    #    stops being published, and reappears if it's registered again.
+    import app.reconcile_blocks as rec
+    rec.live_ip_cidrs = lambda: set()
+    rec.live_email_domains = lambda: set()
+    rec.reconcile()
+    active = {e["cidr"] for e in client.get("/api/blocks/ip-ranges").json()["items"]}
+    assert "198.51.100.0/24" not in active, \
+        "a query-API block removed on the Mastodon side must stop being published"
+    assert not main.reputation_block_installed("ip", "198.51.100.0/24")
+
+    ipblocks.clear()
+    r = ask({"ip": "198.51.100.50"})
+    assert r.json()["blocked"] is True
+    assert len(ipblocks) == 1, "a reconciled-away block is re-installed, not assumed present"
+    active = {e["cidr"] for e in client.get("/api/blocks/ip-ranges").json()["items"]}
+    assert "198.51.100.0/24" in active, "re-installing republishes it"
+
+    main.REPUTATION_API_ENABLED = False
+
+
 if __name__ == "__main__":
     routing_tests()
     policy_tests()
@@ -991,4 +1194,5 @@ if __name__ == "__main__":
     suspicious_sweep_tests()
     blocks_api_tests()
     reconcile_blocks_tests()
+    reputation_api_tests()
     print("ALL TESTS PASSED")