pmb
95ca996a72
docker-build-push / build-push (push) Successful in 13s
Suspend (without the warning email) and delete any signup whose username
fully matches JUNK_USERNAME_PATTERNS, default bp[0-9a-f]{16}: a campaign of
~100/day never-confirmed signups whose confirmation emails were bouncing
into the yttrx inbox and bombing plus-tagged real inboxes. Runs before any
IP/email lookup or welcome; ships dry-run by default.
1414 lines
67 KiB
Python
1414 lines
67 KiB
Python
"""Local smoke test: signature verification, payload routing, abuse policy.
|
|
|
|
Run after `pip install -r requirements.txt`:
|
|
WEBHOOK_SECRET=testsecret BOT_ACCESS_TOKEN=x python test_local.py
|
|
|
|
It uses FastAPI's TestClient and monkeypatches every outbound Mastodon call,
|
|
so it makes no network requests.
|
|
"""
|
|
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import os
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
os.environ.setdefault("WEBHOOK_SECRET", "testsecret")
|
|
os.environ.setdefault("BOT_ACCESS_TOKEN", "dummy")
|
|
os.environ.setdefault("ABUSE_BOT_TOKEN", "dummy-admin")
|
|
os.environ.setdefault("ABUSE_DRY_RUN", "false")
|
|
os.environ.setdefault("MOD_ALERT_ACCT", "pete")
|
|
os.environ.setdefault("DB_PATH", "/tmp/welcomebot-test.db")
|
|
|
|
if os.path.exists(os.environ["DB_PATH"]):
|
|
os.remove(os.environ["DB_PATH"])
|
|
|
|
from fastapi.testclient import TestClient # noqa: E402
|
|
|
|
import app.main as main # noqa: E402
|
|
|
|
# Capture welcome calls instead of hitting the network.
|
|
sent = []
|
|
main.send_welcome = lambda account_id, acct: sent.append((account_id, acct))
|
|
|
|
# Capture report handling at the route boundary for the routing test.
|
|
reports = []
|
|
_real_handle_report = main.handle_report
|
|
main.handle_report = lambda *a: reports.append(a)
|
|
|
|
client = TestClient(main.app)
|
|
|
|
|
|
def sign(body: bytes) -> str:
|
|
return "sha256=" + hmac.new(b"testsecret", body, hashlib.sha256).hexdigest()
|
|
|
|
|
|
def post(payload, secret_ok=True):
|
|
body = json.dumps(payload).encode()
|
|
sig = sign(body) if secret_ok else "sha256=deadbeef"
|
|
return client.post(
|
|
"/webhook", content=body,
|
|
headers={"X-Hub-Signature": sig, "Content-Type": "application/json"},
|
|
)
|
|
|
|
|
|
def days_ago(n):
|
|
return (datetime.now(timezone.utc) - timedelta(days=n)).isoformat()
|
|
|
|
|
|
def routing_tests():
|
|
# 1. health
|
|
assert client.get("/healthz").json() == {"ok": True}
|
|
|
|
# 2. bad signature rejected
|
|
r = post({"event": "account.created", "object": {}}, secret_ok=False)
|
|
assert r.status_code == 401, r.status_code
|
|
|
|
# 3. missing signature rejected
|
|
body = json.dumps({"event": "account.created"}).encode()
|
|
assert client.post("/webhook", content=body).status_code == 401
|
|
|
|
# 4. valid account.created -> queued
|
|
payload = {
|
|
"event": "account.created",
|
|
"object": {
|
|
"id": "1001", "username": "alice", "domain": None,
|
|
"account": {"id": "1001", "username": "alice", "acct": "alice",
|
|
"url": "https://yttrx.com/@alice"},
|
|
},
|
|
}
|
|
r = post(payload)
|
|
assert r.status_code == 200, r.text
|
|
assert r.json().get("queued") == "alice", r.json()
|
|
assert sent == [("1001", "alice")], sent
|
|
|
|
# 5. report.created -> routed to handle_report with extracted fields
|
|
r = post({
|
|
"event": "report.created",
|
|
"object": {
|
|
"id": "55", "category": "spam",
|
|
"target_account": {
|
|
"id": "777", "username": "spammer", "domain": None,
|
|
"suspended": False, "silenced": False,
|
|
"account": {"id": "777", "acct": "spammer"},
|
|
},
|
|
},
|
|
})
|
|
assert r.status_code == 200, r.text
|
|
assert r.json().get("report") == "55", r.json()
|
|
# (report_id, target_id, acct, is_local, suspended, silenced, privileged, email)
|
|
assert reports == [("55", "777", "spammer", True, False, False, False, "")], reports
|
|
|
|
# 5b. report against a staff account -> privileged=True extracted from role
|
|
r = post({
|
|
"event": "report.created",
|
|
"object": {
|
|
"id": "56",
|
|
"target_account": {
|
|
"id": "778", "username": "waffles", "domain": None,
|
|
"suspended": False, "silenced": False,
|
|
"role": {"id": 3, "name": "Owner", "permissions": "1"},
|
|
"account": {"id": "778", "acct": "waffles"},
|
|
},
|
|
},
|
|
})
|
|
assert reports[-1] == ("56", "778", "waffles", True, False, False, True, ""), reports[-1]
|
|
|
|
# 6. account.approved also welcomes — but dedup means alice (id 1001,
|
|
# already welcomed via account.created above) is not welcomed twice.
|
|
r = post({
|
|
"event": "account.approved",
|
|
"object": {"id": "1001", "username": "alice", "domain": None,
|
|
"account": {"id": "1001", "acct": "alice"}},
|
|
})
|
|
assert r.status_code == 200, r.text
|
|
assert r.json().get("queued") == "alice", r.json()
|
|
# send_welcome is stubbed to append; real dedup is exercised in policy run.
|
|
# A brand-new account via account.approved should also route to welcome.
|
|
r = post({
|
|
"event": "account.approved",
|
|
"object": {"id": "2002", "username": "carol", "domain": None,
|
|
"account": {"id": "2002", "acct": "carol"}},
|
|
})
|
|
assert r.json().get("queued") == "carol", r.json()
|
|
assert ("2002", "carol") in sent, sent
|
|
|
|
# 7. truly unknown event ignored
|
|
r = post({"event": "status.updated", "object": {"id": "5"}})
|
|
assert r.json().get("ignored") == "status.updated", r.json()
|
|
|
|
# 8. remote account skipped when LOCAL_ONLY (welcome)
|
|
r = post({
|
|
"event": "account.created",
|
|
"object": {"id": "9", "username": "bob", "domain": "other.social",
|
|
"account": {"id": "9", "acct": "[email protected]"}},
|
|
})
|
|
assert r.json().get("skipped"), r.json()
|
|
|
|
|
|
def policy_tests():
|
|
"""Drive handle_report directly with stubbed Mastodon calls."""
|
|
main.handle_report = _real_handle_report
|
|
|
|
actions, dms, user_dms = [], [], []
|
|
main.apply_action = lambda target_id, action, text: actions.append((target_id, action))
|
|
main.dm_moderator = lambda message: dms.append(message)
|
|
main.dm_silenced_user = lambda acct: user_dms.append(acct)
|
|
|
|
def run(target_id, acct, *, profile, sources, **kw):
|
|
main.classify_account = lambda tid: profile
|
|
main.count_distinct_reporters = lambda tid: sources
|
|
defaults = dict(is_local=True, suspended=False, silenced=False, privileged=False)
|
|
defaults.update(kw)
|
|
main.handle_report("R", target_id, acct, defaults["is_local"],
|
|
defaults["suspended"], defaults["silenced"],
|
|
defaults["privileged"])
|
|
|
|
young = {"has_posts": True, "young": True, "dormant": False,
|
|
"newest_at": None, "oldest_seen": None, "truncated": False}
|
|
active = {"has_posts": True, "young": False, "dormant": False,
|
|
"newest_at": None, "oldest_seen": None, "truncated": False}
|
|
dormant = {"has_posts": True, "young": False, "dormant": True,
|
|
"newest_at": None, "oldest_seen": None, "truncated": False}
|
|
noposts = {"has_posts": False, "young": False, "dormant": False,
|
|
"newest_at": None, "oldest_seen": None, "truncated": False}
|
|
|
|
# young, 1 source -> below threshold (needs 2), no action
|
|
run("1", "y1", profile=young, sources=1)
|
|
assert actions == [], actions
|
|
|
|
# young, 2 sources -> silenced; both the mod and the user are DMed
|
|
run("2", "y2", profile=young, sources=2)
|
|
assert ("2", "silence") in actions, actions
|
|
assert any("y2" in d for d in dms), dms
|
|
assert "y2" in user_dms, user_dms
|
|
|
|
# dormant, 2 sources -> silenced
|
|
run("3", "d1", profile=dormant, sources=2)
|
|
assert ("3", "silence") in actions, actions
|
|
|
|
# active, 2 sources -> below threshold (needs 3), no action
|
|
run("4", "a1", profile=active, sources=2)
|
|
assert ("4", "silence") not in actions, actions
|
|
|
|
# active, 3 sources -> silenced
|
|
run("5", "a2", profile=active, sources=3)
|
|
assert ("5", "silence") in actions, actions
|
|
|
|
# no-posts, 2 sources -> silenced (grouped with young/dormant)
|
|
run("6", "n1", profile=noposts, sources=2)
|
|
assert ("6", "silence") in actions, actions
|
|
|
|
# already silenced -> skipped even with enough sources
|
|
run("7", "s1", profile=young, sources=5, silenced=True)
|
|
assert ("7", "silence") not in actions, actions
|
|
|
|
# remote -> skipped (ABUSE_LOCAL_ONLY)
|
|
run("8", "r1@elsewhere", profile=young, sources=5, is_local=False)
|
|
assert ("8", "silence") not in actions, actions
|
|
|
|
# privileged (staff role) -> skipped even with plenty of sources
|
|
run("9", "waffles", profile=young, sources=9, privileged=True)
|
|
assert ("9", "silence") not in actions, actions
|
|
|
|
# dedup: re-running an already-actioned target takes no new action
|
|
before = len(actions)
|
|
run("2", "y2", profile=young, sources=9)
|
|
assert len(actions) == before, "expected dedup to suppress repeat action"
|
|
|
|
|
|
def ip_scrutiny_tests():
|
|
"""Drive process_signup + the abuse threshold override, network stubbed out."""
|
|
sent = []
|
|
dms = []
|
|
ipblocks = []
|
|
main.send_welcome = lambda account_id, acct: sent.append((account_id, acct))
|
|
main.dm_moderator = lambda message: dms.append(message)
|
|
main.register_ip_block = lambda ip, acct, org, cidr: ipblocks.append((ip, acct, org, cidr)) or True
|
|
# A flagged-but-not-held signup starts the suspicious watch inline, which
|
|
# would otherwise hit the network for a baseline snapshot — stub it.
|
|
main.fetch_account_counts = lambda account_id: (0, 0)
|
|
|
|
def classify(ip):
|
|
return classifications[ip]
|
|
|
|
classifications = {
|
|
"203.0.113.10": ("clean", "Example Residential ISP", False, "203.0.113.10/32"),
|
|
"198.51.100.20": ("datacenter", "Example Cloud Hosting Inc", True, "198.51.100.0/24"),
|
|
"198.51.100.21": ("datacenter", "Example Cloud Hosting Inc", True, "198.51.100.0/24"),
|
|
}
|
|
main.classify_signup_ip = classify
|
|
|
|
main.IP_SCRUTINY_ENABLED = True
|
|
|
|
# A. residential IP -> welcomed immediately, no ip-block, no flag recorded.
|
|
main.IP_SCRUTINY_DRY_RUN = False
|
|
main.IP_SCRUTINY_HOLD_WELCOME = True
|
|
main.IP_SCRUTINY_AUTO_IPBLOCK = True
|
|
main.process_signup("101", "res1", "203.0.113.10")
|
|
assert ("101", "res1") in sent, sent
|
|
assert ipblocks == [], ipblocks
|
|
assert main.get_signup_flag("101") is False, "residential signup should not be flagged"
|
|
|
|
# B. datacenter IP, HOLD_WELCOME + AUTO_IPBLOCK, live (not dry-run) ->
|
|
# welcome held, ip-block registered, moderator DM'd; account.approved
|
|
# later releases the welcome.
|
|
sent.clear(); dms.clear(); ipblocks.clear()
|
|
main.process_signup("102", "dc1", "198.51.100.20")
|
|
assert ("102", "dc1") not in sent, "welcome should be held for a flagged signup"
|
|
assert ipblocks == [("198.51.100.20", "dc1", "Example Cloud Hosting Inc", "198.51.100.0/24")], ipblocks
|
|
assert any("dc1" in d and "held" in d for d in dms), dms
|
|
assert main.get_signup_flag("102") is True
|
|
|
|
main.send_welcome("102", "dc1") # simulate account.approved's unconditional welcome
|
|
assert ("102", "dc1") in sent, sent
|
|
|
|
# B2. datacenter IP, held, but Mastodon already auto-approved the signup
|
|
# (open registration, never queued) -> account.approved will never
|
|
# fire, so process_signup must welcome + start the watch immediately
|
|
# instead of waiting for an event that isn't coming.
|
|
sent.clear(); dms.clear(); ipblocks.clear()
|
|
classifications["198.51.100.22"] = ("datacenter", "Example Cloud Hosting Inc", True, "198.51.100.0/24")
|
|
main.process_signup("104", "dc3", "198.51.100.22", "", True)
|
|
assert ("104", "dc3") in sent, "already-approved flagged signup must be welcomed immediately"
|
|
assert main.get_signup_flag("104") is True
|
|
assert main.already_watched_suspicious("104"), "already-approved flagged signup must start its watch immediately"
|
|
|
|
# C. same datacenter IP, but IP_SCRUTINY_DRY_RUN -> welcomed immediately
|
|
# (no hold), no ip-block write, DM carries the dry-run prefix.
|
|
sent.clear(); dms.clear(); ipblocks.clear()
|
|
main.IP_SCRUTINY_DRY_RUN = True
|
|
main.process_signup("103", "dc2", "198.51.100.21")
|
|
assert ("103", "dc2") in sent, "dry-run must not hold the welcome"
|
|
assert ipblocks == [], "dry-run must not write an ip_block"
|
|
assert any("[DRY-RUN]" in d for d in dms), dms
|
|
|
|
# D. abuse threshold override: a flagged account needs only
|
|
# IP_SCRUTINY_ABUSE_THRESHOLD (1) distinct reporter to silence, vs. the
|
|
# normal young-tier threshold of 2 for an unflagged account.
|
|
main.IP_SCRUTINY_DRY_RUN = False
|
|
main.dm_moderator = lambda message: dms.append(message)
|
|
main.handle_report = _real_handle_report
|
|
actions = []
|
|
main.apply_action = lambda target_id, action, text: actions.append((target_id, action))
|
|
main.dm_silenced_user = lambda acct: None
|
|
young = {"has_posts": True, "young": True, "dormant": False,
|
|
"newest_at": None, "oldest_seen": None, "truncated": False}
|
|
main.classify_account = lambda tid: young
|
|
main.count_distinct_reporters = lambda tid: 1
|
|
|
|
# "102" (dc1) was flagged above -> 1 reporter is enough.
|
|
main.handle_report("R2", "102", "dc1", True, False, False, False)
|
|
assert ("102", "silence") in actions, actions
|
|
|
|
# "101" (res1) was not flagged -> 1 reporter stays below the tier's usual
|
|
# threshold of 2.
|
|
main.handle_report("R3", "101", "res1", True, False, False, False)
|
|
assert ("101", "silence") not in actions, actions
|
|
|
|
|
|
def email_scrutiny_tests():
|
|
"""Drive process_signup's email-domain path + the report-triggered
|
|
suspend+domain-block override, network stubbed out."""
|
|
sent = []
|
|
dms = []
|
|
domain_blocks = []
|
|
main.send_welcome = lambda account_id, acct: sent.append((account_id, acct))
|
|
main.dm_moderator = lambda message: dms.append(message)
|
|
main.register_email_domain_block = lambda domain, acct: domain_blocks.append((domain, acct)) or True
|
|
# A flagged-but-not-held signup starts the suspicious watch inline, which
|
|
# would otherwise hit the network for a baseline snapshot — stub it.
|
|
main.fetch_account_counts = lambda account_id: (0, 0)
|
|
|
|
classifications = {
|
|
"gmail.com": (False, 5),
|
|
"temp-mail.org": (True, 99),
|
|
"sketchy-disposable.example": (True, 95),
|
|
}
|
|
main.classify_email_domain = lambda domain: classifications[domain]
|
|
|
|
main.CHECK_MAIL_ENABLED = True
|
|
main.CHECK_MAIL_API_KEY = "test-key" # required for the signup-path gate
|
|
main.IP_SCRUTINY_ENABLED = False # isolate the email-only signal
|
|
|
|
# A. clean domain -> welcomed immediately, no domain-block, not flagged.
|
|
main.CHECK_MAIL_DRY_RUN = False
|
|
main.CHECK_MAIL_HOLD_WELCOME = True
|
|
main.CHECK_MAIL_AUTO_DOMAIN_BLOCK = True
|
|
main.process_signup("201", "clean1", "", "[email protected]")
|
|
assert ("201", "clean1") in sent, sent
|
|
assert domain_blocks == [], domain_blocks
|
|
assert main.get_signup_email("201") == ("gmail.com", False)
|
|
|
|
# B. disposable domain, HOLD_WELCOME + AUTO_DOMAIN_BLOCK, live -> welcome
|
|
# held, domain blocked, moderator DM'd; account.approved later releases
|
|
# the welcome.
|
|
sent.clear(); dms.clear(); domain_blocks.clear()
|
|
main.process_signup("202", "spam1", "", "[email protected]")
|
|
assert ("202", "spam1") not in sent, "welcome should be held for a flagged signup"
|
|
assert domain_blocks == [("temp-mail.org", "spam1")], domain_blocks
|
|
assert any("spam1" in d and "held" in d for d in dms), dms
|
|
assert main.get_signup_email("202") == ("temp-mail.org", True)
|
|
|
|
main.send_welcome("202", "spam1") # simulate account.approved's unconditional welcome
|
|
assert ("202", "spam1") in sent, sent
|
|
|
|
# C. same idea, but CHECK_MAIL_DRY_RUN -> welcomed immediately (no hold),
|
|
# no domain-block write, DM carries the dry-run marker.
|
|
sent.clear(); dms.clear(); domain_blocks.clear()
|
|
main.CHECK_MAIL_DRY_RUN = True
|
|
main.process_signup("203", "spam2", "", "[email protected]")
|
|
assert ("203", "spam2") in sent, "dry-run must not hold the welcome"
|
|
assert domain_blocks == [], "dry-run must not write a domain block"
|
|
assert any("[DRY-RUN]" in d for d in dms), dms
|
|
|
|
# D0. CHECK_MAIL_DRY_RUN alone holds the report-triggered suspend even
|
|
# though ABUSE_DRY_RUN=False in this test env (set in the module
|
|
# preamble) — the two dry-run switches are independent; either one
|
|
# holding is enough to keep this brand-new action path inert.
|
|
actions = []
|
|
main.apply_action = lambda target_id, action, text: actions.append((target_id, action))
|
|
main.dm_silenced_user = lambda acct: None
|
|
domain_blocks.clear()
|
|
|
|
main.handle_report("R9", "997", "heldbydryrun", True, False, False, False,
|
|
"[email protected]")
|
|
assert ("997", "suspend") not in actions, actions
|
|
assert domain_blocks == [], "CHECK_MAIL_DRY_RUN alone must prevent the domain block too"
|
|
|
|
# D. report-triggered override: ANY report against an account with a
|
|
# high-risk email domain suspends immediately (no distinct-reporter
|
|
# threshold) and blocks the domain — classified LIVE from the report
|
|
# payload's target_account.email, independent of whatever (if
|
|
# anything) process_signup recorded at signup time. This account was
|
|
# never seen by process_signup at all.
|
|
main.CHECK_MAIL_DRY_RUN = False
|
|
actions.clear()
|
|
domain_blocks.clear()
|
|
|
|
main.handle_report("R10", "999", "neversignedup", True, False, False, False,
|
|
"[email protected]")
|
|
assert ("999", "suspend") in actions, actions
|
|
assert domain_blocks == [("temp-mail.org", "neversignedup")], domain_blocks
|
|
|
|
# E. report against an account with a clean email domain -> untouched by
|
|
# this path, falls through to the normal reporter-count/tier policy.
|
|
actions.clear(); domain_blocks.clear()
|
|
noposts = {"has_posts": False, "young": False, "dormant": False,
|
|
"newest_at": None, "oldest_seen": None, "truncated": False}
|
|
main.classify_account = lambda tid: noposts
|
|
main.count_distinct_reporters = lambda tid: 1
|
|
main.handle_report("R11", "998", "clean2", True, False, False, False,
|
|
"[email protected]")
|
|
assert ("998", "suspend") not in actions, actions
|
|
assert domain_blocks == [], domain_blocks
|
|
|
|
|
|
def range_cache_tests():
|
|
"""Drive the ipapi range-cache helpers (_ip_range_bounds/cache_ip_intel/
|
|
cached_ip_intel) directly — pure sqlite + ipaddress logic, no network.
|
|
This is what classify_signup_ip's block_cidr (the network register_
|
|
ip_block blocks) is built on."""
|
|
intel = {
|
|
"is_datacenter": True, "is_vpn": False, "is_proxy": False,
|
|
"is_tor": False, "is_abuser": False, "org": "Example Cloud Hosting Inc",
|
|
}
|
|
|
|
# A. a valid route containing the ip caches (and returns) the wider CIDR,
|
|
# not just the single address.
|
|
main.cache_ip_intel("198.51.100.77", intel, route="198.51.100.0/24")
|
|
cached = main.cached_ip_intel("198.51.100.77")
|
|
assert cached["cidr"] == "198.51.100.0/24", cached
|
|
assert cached["is_datacenter"] is True
|
|
|
|
# B. any other address in that same cached range hits the cache with the
|
|
# same wider CIDR — this is the point of range-based caching, and
|
|
# exactly what lets a repeat signup from elsewhere in the block also
|
|
# resolve to blocking the whole range.
|
|
cached2 = main.cached_ip_intel("198.51.100.200")
|
|
assert cached2["cidr"] == "198.51.100.0/24", cached2
|
|
|
|
# C. an address outside the cached range is a cache miss.
|
|
assert main.cached_ip_intel("198.51.101.1") is None
|
|
|
|
# D. a route that doesn't actually contain the ip falls back to a /32 —
|
|
# never trust a route wide enough to not even cover the IP it came from.
|
|
main.cache_ip_intel("203.0.113.9", intel, route="10.0.0.0/8")
|
|
cached3 = main.cached_ip_intel("203.0.113.9")
|
|
assert cached3["cidr"] == "203.0.113.9/32", cached3
|
|
|
|
# E. no route at all -> /32.
|
|
main.cache_ip_intel("203.0.113.10", intel, route="")
|
|
cached4 = main.cached_ip_intel("203.0.113.10")
|
|
assert cached4["cidr"] == "203.0.113.10/32", cached4
|
|
|
|
|
|
def block_width_cap_tests():
|
|
"""capped_block_cidr: every registered block lands in the width window
|
|
between the floor and the ceiling. The ceiling is there because ipapi.is
|
|
reports routes as wide as /8, so one `abuser` flag on a residential or
|
|
mobile address would otherwise wall off millions of consumer addresses.
|
|
The floor is there because the opposite extreme is just as useless: a /32
|
|
blocks the one address we saw while the next signup comes from its
|
|
neighbour in the same datacenter range."""
|
|
main.IP_SCRUTINY_MAX_BLOCK_PREFIX_V4 = 24
|
|
main.IP_SCRUTINY_MAX_BLOCK_PREFIX_V6 = 48
|
|
main.IP_SCRUTINY_MIN_BLOCK_PREFIX_V4 = 24
|
|
main.IP_SCRUTINY_MIN_BLOCK_PREFIX_V6 = 48
|
|
|
|
# A. A route already inside the window is blocked as-is.
|
|
assert main.capped_block_cidr("5.255.96.30", "5.255.96.0/24") == "5.255.96.0/24", \
|
|
"the ceiling itself is allowed, not excluded"
|
|
|
|
# B. Narrower than the floor: widened to the floor network containing the
|
|
# address. A /28 hosting allocation is a fine cache answer but a poor
|
|
# block — and a bare /32 is the case this floor exists for.
|
|
assert main.capped_block_cidr("5.255.96.30", "5.255.96.0/28") == "5.255.96.0/24"
|
|
assert main.capped_block_cidr("5.255.96.30", "5.255.96.30/32") == "5.255.96.0/24", \
|
|
"a single-address route must widen to the /24, never block just the /32"
|
|
|
|
# C. Wider than the ceiling: back down to the floor network — NOT to the
|
|
# single address, which is what this used to do. A /11 is a consumer
|
|
# ISP aggregate; blocking it would take out 2m addresses.
|
|
assert main.capped_block_cidr("98.45.12.7", "98.32.0.0/11") == "98.45.12.0/24"
|
|
assert main.capped_block_cidr("197.211.45.9", "197.211.32.0/19") == "197.211.45.0/24"
|
|
# ...including the VPS-sized ranges the bot used to block wholesale.
|
|
assert main.capped_block_cidr("51.222.9.9", "51.222.0.0/16") == "51.222.9.0/24"
|
|
assert main.capped_block_cidr("194.163.140.1", "194.163.128.0/18") == "194.163.140.0/24"
|
|
|
|
# D. Same rules for v6, on its own window (/48 = the standard end-site
|
|
# allocation, the rough v6 equivalent of a v4 /24).
|
|
assert main.capped_block_cidr("2001:db8:1::5", "2001:db8:1::/48") == "2001:db8:1::/48"
|
|
assert main.capped_block_cidr("2001:db8:1::5", "2001:db8::/40") == "2001:db8:1::/48"
|
|
assert main.capped_block_cidr("2001:db8:1::5", "2001:db8::/29") == "2001:db8:1::/48"
|
|
assert main.capped_block_cidr("2001:db8:1::5", "2001:db8:1::5/128") == "2001:db8:1::/48"
|
|
|
|
# E. Missing or unparseable input degrades to the floor network, not to
|
|
# something wide and not to the bare address.
|
|
assert main.capped_block_cidr("203.0.113.5", "") == "203.0.113.0/24"
|
|
assert main.capped_block_cidr("203.0.113.5", "not-a-cidr") == "203.0.113.0/24"
|
|
# ...and an unparseable IP yields nothing at all, so callers can refuse.
|
|
assert main.capped_block_cidr("not-an-ip", "") == ""
|
|
|
|
# F. The window is applied to what gets BLOCKED, not to what gets CACHED:
|
|
# the wide route is still the right cache key, since the whole route
|
|
# genuinely belongs to one operator and one lookup answers for all of
|
|
# it. Narrowing the cache key instead would multiply provider lookups.
|
|
intel = {"is_datacenter": False, "is_vpn": False, "is_proxy": False,
|
|
"is_tor": False, "is_abuser": True, "org": "Big ISP"}
|
|
main.cache_ip_intel("100.64.5.5", intel, route="100.64.0.0/10")
|
|
assert main.cached_ip_intel("100.64.200.200") is not None, \
|
|
"a wide route must still answer from cache for its whole range"
|
|
assert main.cached_ip_intel("100.64.200.200")["cidr"] == "100.64.0.0/10"
|
|
assert main.capped_block_cidr("100.64.200.200", "100.64.0.0/10") == "100.64.200.0/24"
|
|
|
|
# G. The floor is a knob: setting it to /32 restores the old behaviour of
|
|
# blocking exactly what the route says, down to the single address.
|
|
main.IP_SCRUTINY_MIN_BLOCK_PREFIX_V4 = 32
|
|
try:
|
|
assert main.capped_block_cidr("98.45.12.7", "98.32.0.0/11") == "98.45.12.7/32"
|
|
assert main.capped_block_cidr("5.255.96.30", "5.255.96.0/28") == "5.255.96.0/28"
|
|
assert main.capped_block_cidr("203.0.113.5", "") == "203.0.113.5/32"
|
|
finally:
|
|
main.IP_SCRUTINY_MIN_BLOCK_PREFIX_V4 = 24
|
|
|
|
# H. A floor configured WIDER than the ceiling is nonsense; the ceiling
|
|
# wins, since it is the one bounding collateral damage.
|
|
main.IP_SCRUTINY_MIN_BLOCK_PREFIX_V4 = 16
|
|
try:
|
|
assert main.capped_block_cidr("98.45.12.7", "98.32.0.0/11") == "98.45.12.0/24"
|
|
assert main.capped_block_cidr("98.45.12.7", "") == "98.45.12.0/24"
|
|
finally:
|
|
main.IP_SCRUTINY_MIN_BLOCK_PREFIX_V4 = 24
|
|
|
|
|
|
def combined_signal_tests():
|
|
"""Drive process_signup's combined IP+email immediate-suspend path
|
|
(SUSPICIOUS_COMBINED_*), independent of the OR-based hourly sweep."""
|
|
sent = []
|
|
dms = []
|
|
actions = []
|
|
ipblocks = []
|
|
domain_blocks = []
|
|
main.send_welcome = lambda account_id, acct: sent.append((account_id, acct))
|
|
main.dm_moderator = lambda message: dms.append(message)
|
|
main.apply_action = lambda target_id, action, text: actions.append((target_id, action))
|
|
main.register_ip_block = lambda ip, acct, org, cidr: ipblocks.append((ip, acct, org, cidr)) or True
|
|
main.register_email_domain_block = lambda domain, acct: domain_blocks.append((domain, acct)) or True
|
|
main.fetch_account_counts = lambda account_id: (0, 0)
|
|
|
|
main.IP_SCRUTINY_ENABLED = True
|
|
main.IP_SCRUTINY_DRY_RUN = False
|
|
main.IP_SCRUTINY_HOLD_WELCOME = True
|
|
main.IP_SCRUTINY_AUTO_IPBLOCK = True
|
|
main.CHECK_MAIL_ENABLED = True
|
|
main.CHECK_MAIL_API_KEY = "test-key"
|
|
main.CHECK_MAIL_DRY_RUN = False
|
|
main.CHECK_MAIL_HOLD_WELCOME = True
|
|
main.CHECK_MAIL_AUTO_DOMAIN_BLOCK = True
|
|
main.SUSPICIOUS_COMBINED_ENABLED = True
|
|
main.SUSPICIOUS_COMBINED_ACTION = "suspend"
|
|
main.SUSPICIOUS_COMBINED_DRY_RUN = False
|
|
main.ABUSE_ALLOWLIST = {"trustedstaff"}
|
|
|
|
ip_classifications = {
|
|
"198.51.100.40": ("datacenter", "Example Cloud Hosting Inc", True, "198.51.100.32/28"),
|
|
"198.51.100.41": ("datacenter", "Example Cloud Hosting Inc", True, "198.51.100.32/28"),
|
|
}
|
|
main.classify_signup_ip = lambda ip: ip_classifications[ip]
|
|
email_classifications = {
|
|
"temp-mail.org": (True, 99),
|
|
"gmail.com": (False, 5),
|
|
}
|
|
main.classify_email_domain = lambda domain: email_classifications[domain]
|
|
|
|
# A. both signals flagged, live -> suspended immediately, no welcome, no
|
|
# suspicious-watch entry (already actioned, nothing left to sweep).
|
|
# The individual ip/email auto-blocks still fire independently.
|
|
main.process_signup("501", "bothbad", "198.51.100.40", "[email protected]")
|
|
assert ("501", "suspend") in actions, actions
|
|
assert ("501", "bothbad") not in sent, "must not welcome an immediately-suspended signup"
|
|
assert any("bothbad" in d and "BOTH" in d for d in dms), dms
|
|
with main._db() as conn:
|
|
row = conn.execute(
|
|
"SELECT 1 FROM suspicious_watch WHERE account_id = ?", ("501",),
|
|
).fetchone()
|
|
assert row is None, "an immediately-suspended signup must not also enter the hourly watch"
|
|
assert ("198.51.100.40", "bothbad", "Example Cloud Hosting Inc", "198.51.100.32/28") in ipblocks, ipblocks
|
|
assert ("temp-mail.org", "bothbad") in domain_blocks, domain_blocks
|
|
|
|
# B. only IP flagged -> untouched by the combined path, falls through to
|
|
# the normal single-signal hold/welcome behavior.
|
|
actions.clear(); sent.clear(); dms.clear()
|
|
main.process_signup("502", "iponly", "198.51.100.41", "[email protected]")
|
|
assert ("502", "suspend") not in actions, actions
|
|
assert ("502", "iponly") not in sent, "single-signal flag should still be held"
|
|
|
|
# C. both flagged, but allowlisted acct -> combined path skipped; falls
|
|
# through to the normal (still-held) path like any other flagged signup.
|
|
actions.clear(); sent.clear(); dms.clear()
|
|
main.process_signup("503", "trustedstaff", "198.51.100.40", "[email protected]")
|
|
assert ("503", "suspend") not in actions, "allowlisted acct must not be auto-suspended"
|
|
|
|
# D. both flagged, SUSPICIOUS_COMBINED_DRY_RUN -> DM only, no real
|
|
# suspend; falls through to the normal held-welcome path.
|
|
actions.clear(); sent.clear(); dms.clear()
|
|
main.SUSPICIOUS_COMBINED_DRY_RUN = True
|
|
main.process_signup("504", "drybothbad", "198.51.100.40", "[email protected]")
|
|
assert ("504", "suspend") not in actions, "dry-run must not call apply_action"
|
|
assert any("[DRY-RUN]" in d and "drybothbad" in d for d in dms), dms
|
|
assert ("504", "drybothbad") not in sent, "still held by the individual signal holds"
|
|
|
|
# E. both flagged, live, but apply_action fails -> falls back to the
|
|
# normal held-welcome path instead of silently dropping the signup.
|
|
actions.clear(); sent.clear(); dms.clear()
|
|
main.SUSPICIOUS_COMBINED_DRY_RUN = False
|
|
|
|
def boom(target_id, action, text):
|
|
raise main.httpx.HTTPError("boom")
|
|
|
|
main.apply_action = boom
|
|
main.process_signup("505", "failsuspend", "198.51.100.40", "[email protected]")
|
|
assert ("505", "failsuspend") not in sent, "still held, falls back to the normal flagged path"
|
|
|
|
|
|
def strong_ip_suspend_tests():
|
|
"""Drive process_signup's strong-IP-signal immediate-suspend path
|
|
(IP_SCRUTINY_STRONG_SUSPEND_*): datacenter/proxy/tor/abuser suspend
|
|
immediately regardless of the email signal, but vpn alone does not — it
|
|
still falls through to the normal hold/welcome path (or the existing
|
|
combined-signal suspend, if also paired with a flagged email domain)."""
|
|
sent = []
|
|
dms = []
|
|
actions = []
|
|
ipblocks = []
|
|
main.send_welcome = lambda account_id, acct: sent.append((account_id, acct))
|
|
main.dm_moderator = lambda message: dms.append(message)
|
|
main.apply_action = lambda target_id, action, text: actions.append((target_id, action))
|
|
main.register_ip_block = lambda ip, acct, org, cidr: ipblocks.append((ip, acct, org, cidr)) or True
|
|
main.register_email_domain_block = lambda domain, acct: True
|
|
main.fetch_account_counts = lambda account_id: (0, 0)
|
|
|
|
main.IP_SCRUTINY_ENABLED = True
|
|
main.IP_SCRUTINY_DRY_RUN = False
|
|
main.IP_SCRUTINY_HOLD_WELCOME = True
|
|
main.IP_SCRUTINY_AUTO_IPBLOCK = True
|
|
main.CHECK_MAIL_ENABLED = True
|
|
main.CHECK_MAIL_API_KEY = "test-key"
|
|
main.CHECK_MAIL_DRY_RUN = False
|
|
main.CHECK_MAIL_HOLD_WELCOME = True
|
|
main.CHECK_MAIL_AUTO_DOMAIN_BLOCK = True
|
|
main.SUSPICIOUS_COMBINED_ENABLED = True
|
|
main.SUSPICIOUS_COMBINED_ACTION = "suspend"
|
|
main.SUSPICIOUS_COMBINED_DRY_RUN = False
|
|
main.ABUSE_ALLOWLIST = {"trustedstaff"}
|
|
main.IP_SCRUTINY_STRONG_SUSPEND_ENABLED = True
|
|
main.IP_SCRUTINY_STRONG_SUSPEND_ACTION = "suspend"
|
|
main.IP_SCRUTINY_STRONG_SUSPEND_DRY_RUN = False
|
|
|
|
ip_classifications = {
|
|
"198.51.100.60": ("datacenter", "Example Cloud Hosting Inc", True, "198.51.100.48/28"),
|
|
"198.51.100.61": ("proxy", "Example Proxy Networks", True, "198.51.100.48/28"),
|
|
"198.51.100.62": ("vpn", "Example VPN Provider", True, "198.51.100.48/28"),
|
|
"198.51.100.63": ("vpn", "Example VPN Provider", True, "198.51.100.48/28"),
|
|
}
|
|
main.classify_signup_ip = lambda ip: ip_classifications[ip]
|
|
email_classifications = {
|
|
"temp-mail.org": (True, 99),
|
|
"gmail.com": (False, 5),
|
|
}
|
|
main.classify_email_domain = lambda domain: email_classifications[domain]
|
|
|
|
# A. datacenter (strong), clean email -> suspended immediately regardless
|
|
# of the (non-flagged) email signal; ip_block still registered.
|
|
main.process_signup("601", "strongdc", "198.51.100.60", "[email protected]")
|
|
assert ("601", "suspend") in actions, actions
|
|
assert ("601", "strongdc") not in sent, "must not welcome an immediately-suspended signup"
|
|
assert any("strongdc" in d and "strong IP signal" in d for d in dms), dms
|
|
assert ("198.51.100.60", "strongdc", "Example Cloud Hosting Inc", "198.51.100.48/28") in ipblocks, ipblocks
|
|
|
|
# B. proxy (strong), no email at all -> still suspended immediately.
|
|
actions.clear(); sent.clear(); dms.clear()
|
|
main.process_signup("602", "strongproxy", "198.51.100.61")
|
|
assert ("602", "suspend") in actions, actions
|
|
assert ("602", "strongproxy") not in sent
|
|
|
|
# C. vpn only (not strong), clean email -> NOT suspended, falls through to
|
|
# the normal held-welcome path.
|
|
actions.clear(); sent.clear(); dms.clear()
|
|
main.process_signup("603", "vpnonly", "198.51.100.62", "[email protected]")
|
|
assert ("603", "suspend") not in actions, actions
|
|
assert ("603", "vpnonly") not in sent, "still held by the individual IP-scrutiny hold"
|
|
|
|
# D. vpn (not strong) + flagged email -> not caught by the strong path,
|
|
# but still caught by the existing combined-signal suspend.
|
|
actions.clear(); sent.clear(); dms.clear()
|
|
main.process_signup("604", "vpnplusemail", "198.51.100.63", "[email protected]")
|
|
assert ("604", "suspend") in actions, actions
|
|
assert ("604", "vpnplusemail") not in sent
|
|
assert any("vpnplusemail" in d and "BOTH" in d for d in dms), dms
|
|
|
|
# E. allowlisted acct with a strong signal -> not suspended.
|
|
actions.clear(); sent.clear(); dms.clear()
|
|
main.process_signup("605", "trustedstaff", "198.51.100.60", "[email protected]")
|
|
assert ("605", "suspend") not in actions, "allowlisted acct must not be auto-suspended"
|
|
|
|
# F. strong signal, but IP_SCRUTINY_STRONG_SUSPEND_DRY_RUN -> DM only, no
|
|
# real suspend; falls through to the normal held-welcome path.
|
|
actions.clear(); sent.clear(); dms.clear()
|
|
main.IP_SCRUTINY_STRONG_SUSPEND_DRY_RUN = True
|
|
main.process_signup("606", "drystrong", "198.51.100.60", "[email protected]")
|
|
assert ("606", "suspend") not in actions, "dry-run must not call apply_action"
|
|
assert any("[DRY-RUN]" in d and "drystrong" in d for d in dms), dms
|
|
assert ("606", "drystrong") not in sent, "still held by the individual IP-scrutiny hold"
|
|
|
|
# G. strong signal, live, but apply_action fails -> falls back to the
|
|
# normal held-welcome path instead of silently dropping the signup.
|
|
actions.clear(); sent.clear(); dms.clear()
|
|
main.IP_SCRUTINY_STRONG_SUSPEND_DRY_RUN = False
|
|
|
|
def boom(target_id, action, text):
|
|
raise main.httpx.HTTPError("boom")
|
|
|
|
main.apply_action = boom
|
|
main.process_signup("607", "failstrong", "198.51.100.60", "[email protected]")
|
|
assert ("607", "failstrong") not in sent, "still held, falls back to the normal flagged path"
|
|
|
|
|
|
def junk_username_tests():
|
|
"""Drive process_signup's junk-username purge (JUNK_USERNAME_*): a
|
|
matching username is purged before any lookup or welcome; near-misses,
|
|
allowlisted accts, dry-run, and a failed purge all fall through."""
|
|
sent = []
|
|
purged = []
|
|
lookups = []
|
|
main.send_welcome = lambda account_id, acct: sent.append((account_id, acct))
|
|
main.dm_moderator = lambda message: None
|
|
main.purge_account = lambda target_id, text: purged.append(target_id)
|
|
main.classify_signup_ip = lambda ip: lookups.append(ip) or ("residential", "Example ISP", False, "")
|
|
main.IP_SCRUTINY_ENABLED = True
|
|
main.CHECK_MAIL_ENABLED = False
|
|
main.ABUSE_ALLOWLIST = {"bp0000000000000000"}
|
|
main.JUNK_USERNAME_PATTERNS = [main.re.compile(r"bp[0-9a-f]{16}", main.re.IGNORECASE)]
|
|
main.JUNK_USERNAME_DRY_RUN = False
|
|
|
|
# A. matching username -> purged, no IP lookup, no welcome.
|
|
main.process_signup("701", "bp56ac8d816ff8d793", "203.0.113.70", "[email protected]", True)
|
|
assert purged == ["701"], purged
|
|
assert not lookups, "junk signups must not spend IP/email lookups"
|
|
assert not sent
|
|
|
|
# B. near-misses (too short, extra suffix, non-hex) -> normal path.
|
|
for i, name in enumerate(["bp56ac8d816ff8d79", "bp56ac8d816ff8d7930", "bp56ac8d816ff8d79z", "bpm"]):
|
|
main.process_signup(f"71{i}", name, "203.0.113.71", "", True)
|
|
assert purged == ["701"], purged
|
|
assert len(sent) == 4, sent
|
|
|
|
# C. allowlisted -> not purged.
|
|
main.process_signup("720", "bp0000000000000000", "203.0.113.72", "", True)
|
|
assert "720" not in purged
|
|
|
|
# D. dry-run -> not purged, normal path.
|
|
purged.clear(); sent.clear()
|
|
main.JUNK_USERNAME_DRY_RUN = True
|
|
main.process_signup("730", "bpaaaaaaaaaaaaaaaa", "203.0.113.73", "", True)
|
|
assert not purged and ("730", "bpaaaaaaaaaaaaaaaa") in sent
|
|
|
|
# E. purge fails -> falls back to the normal signup path.
|
|
main.JUNK_USERNAME_DRY_RUN = False
|
|
sent.clear()
|
|
|
|
def boom(target_id, text):
|
|
raise main.httpx.HTTPError("boom")
|
|
|
|
main.purge_account = boom
|
|
main.process_signup("740", "bpbbbbbbbbbbbbbbbb", "203.0.113.74", "", True)
|
|
assert ("740", "bpbbbbbbbbbbbbbbbb") in sent
|
|
|
|
# F. account.approved for a junk username (live) -> no welcome queued.
|
|
sent.clear()
|
|
r = post({"event": "account.approved",
|
|
"object": {"id": "750", "username": "bpcccccccccccccccc",
|
|
"account": {"id": "750", "acct": "bpcccccccccccccccc"}}})
|
|
assert r.status_code == 200 and r.json().get("skipped") == "junk username", r.json()
|
|
assert not sent
|
|
|
|
main.ABUSE_ALLOWLIST = set()
|
|
main.JUNK_USERNAME_DRY_RUN = True
|
|
|
|
|
|
def suspicious_watch_tests():
|
|
"""Drive maybe_start_suspicious_watch: baseline capture on flagged
|
|
signups only, never on clean ones, and never twice for the same account."""
|
|
main.SUSPICIOUS_SWEEP_ENABLED = True
|
|
counts = {"301": (2, 5), "302": (0, 0)}
|
|
main.fetch_account_counts = lambda account_id: counts[account_id]
|
|
|
|
# A. flagged by IP only -> watch started with that baseline.
|
|
main.record_signup_ip("301", "watched1", "198.51.100.30", "datacenter", "Cloud Inc", True)
|
|
main.maybe_start_suspicious_watch("301", "watched1")
|
|
with main._db() as conn:
|
|
row = conn.execute(
|
|
"SELECT reasons, baseline_statuses_count, baseline_following_count, status "
|
|
"FROM suspicious_watch WHERE account_id = ?", ("301",),
|
|
).fetchone()
|
|
assert row == ("ip", 2, 5, "pending"), row
|
|
|
|
# B. calling again (e.g. account.created then account.approved both
|
|
# firing) does not reset/duplicate the baseline.
|
|
counts["301"] = (99, 99)
|
|
main.maybe_start_suspicious_watch("301", "watched1")
|
|
with main._db() as conn:
|
|
row = conn.execute(
|
|
"SELECT baseline_statuses_count FROM suspicious_watch WHERE account_id = ?", ("301",),
|
|
).fetchone()
|
|
assert row == (2,), "re-calling must not overwrite the original baseline"
|
|
|
|
# C. never flagged -> no watch row at all.
|
|
main.maybe_start_suspicious_watch("302", "clean3")
|
|
with main._db() as conn:
|
|
row = conn.execute(
|
|
"SELECT 1 FROM suspicious_watch WHERE account_id = ?", ("302",),
|
|
).fetchone()
|
|
assert row is None, "an unflagged signup must never be watched"
|
|
|
|
|
|
def suspicious_sweep_tests():
|
|
"""Drive app.suspicious_sweep.sweep() directly against rows seeded into
|
|
the shared sqlite store, with the Mastodon admin-account fetch and
|
|
apply_action/dm_moderator stubbed out (no network)."""
|
|
import app.suspicious_sweep as sweep_mod
|
|
|
|
sweep_mod.SUSPICIOUS_SWEEP_ENABLED = True
|
|
sweep_mod.SUSPICIOUS_GRACE_HOURS = 24
|
|
sweep_mod.SUSPICIOUS_ACTION = "suspend"
|
|
sweep_mod.SUSPICIOUS_DRY_RUN = False
|
|
sweep_mod.ABUSE_SKIP_PRIVILEGED = True
|
|
sweep_mod.ABUSE_ALLOWLIST = {"trustedstaff"}
|
|
|
|
def seed(account_id, acct, reasons, baseline_statuses, baseline_following, hours_ago):
|
|
started_at = (datetime.now(timezone.utc) - timedelta(hours=hours_ago)).isoformat()
|
|
with main._db() as conn:
|
|
conn.execute(
|
|
"INSERT INTO suspicious_watch "
|
|
"(account_id, acct, reasons, started_at, baseline_statuses_count, "
|
|
" baseline_following_count) VALUES (?, ?, ?, ?, ?, ?)",
|
|
(account_id, acct, reasons, started_at, baseline_statuses, baseline_following),
|
|
)
|
|
|
|
def status_of(account_id):
|
|
with main._db() as conn:
|
|
row = conn.execute(
|
|
"SELECT status FROM suspicious_watch WHERE account_id = ?", (account_id,),
|
|
).fetchone()
|
|
return row[0] if row else None
|
|
|
|
admin_accounts = {}
|
|
sweep_mod._fetch_admin_account = lambda account_id: admin_accounts[account_id]
|
|
|
|
actions = []
|
|
dms = []
|
|
sweep_mod.apply_action = lambda target_id, action, text: actions.append((target_id, action))
|
|
sweep_mod.dm_moderator = lambda message: dms.append(message)
|
|
|
|
def public(statuses_count=0, following_count=0, **extra):
|
|
acct = {"statuses_count": statuses_count, "following_count": following_count}
|
|
return {"account": acct, **extra}
|
|
|
|
# A. not yet due (started 1h ago, 24h grace) -> untouched, still pending.
|
|
seed("401", "toosoon", "ip", 0, 0, hours_ago=1)
|
|
admin_accounts["401"] = public(0, 0)
|
|
|
|
# B. due, zero new posts/follows since baseline -> suspended.
|
|
seed("402", "dormantbot", "ip", 0, 0, hours_ago=25)
|
|
admin_accounts["402"] = public(0, 0)
|
|
|
|
# C. due, but posted since baseline -> cleared, not suspended.
|
|
seed("403", "postedsince", "email", 3, 1, hours_ago=25)
|
|
admin_accounts["403"] = public(4, 1)
|
|
|
|
# D. due, but followed someone since baseline -> cleared, not suspended.
|
|
seed("404", "followedsince", "ip,email", 0, 0, hours_ago=25)
|
|
admin_accounts["404"] = public(0, 1)
|
|
|
|
# E. due, but already suspended/silenced by another mechanism -> skipped.
|
|
seed("405", "alreadylimited", "ip", 0, 0, hours_ago=25)
|
|
admin_accounts["405"] = public(0, 0, suspended=True)
|
|
|
|
# F. due, but holds a staff role -> skipped, never suspended.
|
|
seed("406", "staffer", "ip", 0, 0, hours_ago=25)
|
|
admin_accounts["406"] = public(0, 0, role={"id": 3, "name": "Owner"})
|
|
|
|
# G. due, allowlisted by handle -> skipped.
|
|
seed("407", "trustedstaff", "ip", 0, 0, hours_ago=25)
|
|
admin_accounts["407"] = public(0, 0)
|
|
|
|
sweep_mod.sweep()
|
|
|
|
assert status_of("401") == "pending", "not-yet-due watch must be left alone"
|
|
assert ("402", "suspend") in actions, actions
|
|
assert status_of("402") == "suspend"
|
|
assert any("dormantbot" in d for d in dms), dms
|
|
|
|
assert ("403", "suspend") not in actions, actions
|
|
assert status_of("403") == "cleared"
|
|
|
|
assert ("404", "suspend") not in actions, actions
|
|
assert status_of("404") == "cleared"
|
|
|
|
assert ("405", "suspend") not in actions, actions
|
|
assert status_of("405") == "skipped-already-limited"
|
|
|
|
assert ("406", "suspend") not in actions, actions
|
|
assert status_of("406") == "skipped-privileged"
|
|
|
|
assert ("407", "suspend") not in actions, actions
|
|
assert status_of("407") == "skipped-privileged"
|
|
|
|
# H. re-running the sweep must not re-suspend or re-DM an already-resolved row.
|
|
before_actions, before_dms = len(actions), len(dms)
|
|
sweep_mod.sweep()
|
|
assert len(actions) == before_actions, "resolved rows must not be re-actioned"
|
|
assert len(dms) == before_dms, "resolved rows must not be re-DMed"
|
|
|
|
# I. dry-run mode logs/DMs but takes no real action.
|
|
seed("408", "dryrunbot", "ip", 0, 0, hours_ago=25)
|
|
admin_accounts["408"] = public(0, 0)
|
|
sweep_mod.SUSPICIOUS_DRY_RUN = True
|
|
sweep_mod.sweep()
|
|
assert ("408", "suspend") not in actions, "dry-run must not call apply_action"
|
|
assert status_of("408") == "dry-run:suspend"
|
|
assert any("[DRY-RUN]" in d and "dryrunbot" in d for d in dms), dms
|
|
|
|
|
|
def blocks_api_tests():
|
|
"""Drive the read-only blocklist API: the BLOCKS_API_ENABLED gate, the
|
|
response shape, the deliberate omission of provider classification data
|
|
and account identifiers, and the pre-block_cidr fallback."""
|
|
# Seed deterministic rows rather than relying on what earlier tests left
|
|
# in the shared db; assertions below check for presence, not exact lists.
|
|
main.record_signup_ip("900", "blockedone", "198.18.0.90", "datacenter",
|
|
"Cloud Inc", True, "198.18.0.0/24")
|
|
main.mark_ipblock_registered("900")
|
|
# A second signup out of the same range collapses into one entry.
|
|
main.record_signup_ip("901", "blockedtwo", "198.18.0.91", "datacenter",
|
|
"Cloud Inc", True, "198.18.0.0/24")
|
|
main.mark_ipblock_registered("901")
|
|
# Classified but never blocked -> must not appear at all.
|
|
main.record_signup_ip("902", "notblockedacct", "203.0.113.199", "clean", "", False)
|
|
# A row predating the block_cidr column (NULL) -> falls back to the /32
|
|
# that was in fact registered for it back then.
|
|
main.record_signup_ip("903", "legacyacct", "192.0.2.77", "datacenter", "Old Corp", True)
|
|
main.mark_ipblock_registered("903")
|
|
|
|
main.record_signup_email("910", "spamone", "blocked-example.test", True, 95, True)
|
|
main.mark_email_domain_blocked("910")
|
|
main.record_signup_email("911", "spamtwo", "blocked-example.test", True, 95, True)
|
|
main.mark_email_domain_blocked("911")
|
|
main.record_signup_email("912", "cleanacct", "notblocked-example.test", False, 0, False)
|
|
|
|
paths = ("/api/blocks", "/api/blocks/ip-ranges", "/api/blocks/email-domains")
|
|
|
|
# A. Disabled (the shipped default) -> 404, indistinguishable from a path
|
|
# that doesn't exist.
|
|
main.BLOCKS_API_ENABLED = False
|
|
for path in paths:
|
|
assert client.get(path).status_code == 404, path
|
|
|
|
# B. Enabled -> 200, with the ranges and domains actually blocked.
|
|
main.BLOCKS_API_ENABLED = True
|
|
r = client.get("/api/blocks")
|
|
assert r.status_code == 200, r.status_code
|
|
assert r.headers["cache-control"] == f"public, max-age={main.BLOCKS_API_CACHE_SECONDS}"
|
|
body = r.json()
|
|
ranges = {e["cidr"]: e for e in body["ip_ranges"]["items"]}
|
|
domains = {e["domain"]: e for e in body["email_domains"]["items"]}
|
|
assert body["ip_ranges"]["count"] == len(body["ip_ranges"]["items"])
|
|
assert body["email_domains"]["count"] == len(body["email_domains"]["items"])
|
|
|
|
assert "198.18.0.0/24" in ranges, ranges
|
|
assert ranges["198.18.0.0/24"]["signups"] == 2, "repeat signups collapse into one entry"
|
|
assert "192.0.2.77/32" in ranges, "a NULL block_cidr row falls back to its /32"
|
|
assert "blocked-example.test" in domains, domains
|
|
assert domains["blocked-example.test"]["signups"] == 2
|
|
assert "notblocked-example.test" not in domains, "an unblocked domain must not be listed"
|
|
|
|
# C. Nothing that identifies an account, and nothing sourced from the
|
|
# ipapi.is / check-mail.org feeds, may appear in the response.
|
|
raw = r.text
|
|
for leak in ("blockedone", "legacyacct", "notblockedacct", "spamone", "cleanacct",
|
|
"203.0.113.199", "datacenter", "Cloud Inc", "Old Corp"):
|
|
assert leak not in raw, f"{leak!r} must not be exposed by the blocklist API"
|
|
for entry in body["ip_ranges"]["items"]:
|
|
assert set(entry) == {"cidr", "signups", "first_blocked_at", "last_blocked_at"}, entry
|
|
for entry in body["email_domains"]["items"]:
|
|
assert set(entry) == {"domain", "signups", "first_blocked_at", "last_blocked_at"}, entry
|
|
|
|
# D. The per-list endpoints agree with the combined document.
|
|
ips_only = client.get("/api/blocks/ip-ranges").json()
|
|
domains_only = client.get("/api/blocks/email-domains").json()
|
|
assert ips_only["items"] == body["ip_ranges"]["items"]
|
|
assert domains_only["items"] == body["email_domains"]["items"]
|
|
|
|
# E. ?limit truncates; a nonsensical limit is treated as "no limit".
|
|
assert len(client.get("/api/blocks/ip-ranges?limit=1").json()["items"]) == 1
|
|
assert (client.get("/api/blocks/ip-ranges?limit=-5").json()["items"]
|
|
== ips_only["items"]), "a negative limit must not empty the list"
|
|
|
|
# F. BLOCKS_API_MAX_ITEMS caps the caller's limit, not the other way round.
|
|
main.BLOCKS_API_MAX_ITEMS = 1
|
|
assert len(client.get("/api/blocks/ip-ranges?limit=999").json()["items"]) == 1
|
|
main.BLOCKS_API_MAX_ITEMS = 5000
|
|
|
|
# G. CORS is open — other instances' tooling may fetch this from a browser.
|
|
assert r.headers["access-control-allow-origin"] == "*"
|
|
|
|
# H. ?format=txt renders one entry per line under a comment header, so it
|
|
# pipes into another admin's tooling without a JSON parser.
|
|
t = client.get("/api/blocks/ip-ranges?format=txt")
|
|
assert t.status_code == 200, t.status_code
|
|
assert t.headers["content-type"].startswith("text/plain"), t.headers["content-type"]
|
|
assert t.headers["access-control-allow-origin"] == "*"
|
|
lines = t.text.strip().split("\n")
|
|
header = [ln for ln in lines if ln.startswith("#")]
|
|
payload = [ln for ln in lines if not ln.startswith("#")]
|
|
assert any("source:" in ln for ln in header), header
|
|
assert f"entries: {len(ips_only['items'])}" in "\n".join(header)
|
|
assert payload == [e["cidr"] for e in ips_only["items"]], payload[:3]
|
|
|
|
td = client.get("/api/blocks/email-domains?format=txt")
|
|
assert [ln for ln in td.text.strip().split("\n") if not ln.startswith("#")] \
|
|
== [e["domain"] for e in domains_only["items"]]
|
|
|
|
# I. An unknown format is rejected rather than quietly serving JSON, and
|
|
# the combined endpoint refuses text (it would mix CIDRs with domains).
|
|
assert client.get("/api/blocks/ip-ranges?format=yaml").status_code == 400
|
|
assert client.get("/api/blocks?format=txt").status_code == 400
|
|
assert client.get("/api/blocks?format=json").status_code == 200
|
|
|
|
# J. The format gate never leaks existence when the API is switched off.
|
|
main.BLOCKS_API_ENABLED = False
|
|
assert client.get("/api/blocks/ip-ranges?format=txt").status_code == 404
|
|
assert client.get("/api/blocks?format=txt").status_code == 404
|
|
main.BLOCKS_API_ENABLED = True
|
|
main.BLOCKS_API_MAX_ITEMS = 5000
|
|
|
|
|
|
def reconcile_blocks_tests():
|
|
"""Drive app.reconcile_blocks.reconcile() against seeded rows with the
|
|
Mastodon admin list fetches stubbed out (no network). The important case
|
|
is the failure path: a 403 or an outage must never be read as "every
|
|
block was removed"."""
|
|
import app.reconcile_blocks as rec
|
|
|
|
# Seed: two blocked ranges and two blocked domains, all currently active.
|
|
main.record_signup_ip("950", "recone", "198.18.5.1", "datacenter", "X", True, "198.18.5.0/24")
|
|
main.mark_ipblock_registered("950")
|
|
main.record_signup_ip("951", "rectwo", "198.18.6.1", "datacenter", "X", True, "198.18.6.0/24")
|
|
main.mark_ipblock_registered("951")
|
|
main.record_signup_email("960", "recthree", "rec-blocked.test", True, 90, True)
|
|
main.mark_email_domain_blocked("960")
|
|
main.record_signup_email("961", "recfour", "rec-removed.test", True, 90, True)
|
|
main.mark_email_domain_blocked("961")
|
|
|
|
def active_ips():
|
|
return {e["cidr"] for e in main.blocked_ip_ranges()}
|
|
|
|
def active_domains():
|
|
return {e["domain"] for e in main.blocked_email_domains()}
|
|
|
|
assert "198.18.5.0/24" in active_ips() and "198.18.6.0/24" in active_ips()
|
|
assert "rec-removed.test" in active_domains()
|
|
|
|
# A. Mastodon no longer has 198.18.6.0/24 or rec-removed.test -> both drop
|
|
# out of what the API publishes; the survivors stay.
|
|
rec.live_ip_cidrs = lambda: {"198.18.5.0/24"}
|
|
rec.live_email_domains = lambda: {"rec-blocked.test"}
|
|
changed = rec.reconcile()
|
|
assert changed >= 2, changed
|
|
ips, domains = active_ips(), active_domains()
|
|
assert "198.18.5.0/24" in ips, "a block still present must stay published"
|
|
assert "198.18.6.0/24" not in ips, "a block removed in Mastodon must stop being published"
|
|
assert "rec-blocked.test" in domains
|
|
assert "rec-removed.test" not in domains
|
|
|
|
# B. It reappears (re-registered by hand, or the bot blocks it again) ->
|
|
# the row comes back rather than being stuck off.
|
|
rec.live_ip_cidrs = lambda: {"198.18.5.0/24", "198.18.6.0/24"}
|
|
rec.live_email_domains = lambda: {"rec-blocked.test", "rec-removed.test"}
|
|
rec.reconcile()
|
|
assert "198.18.6.0/24" in active_ips(), "a reappearing block must be republished"
|
|
assert "rec-removed.test" in active_domains()
|
|
|
|
# C. THE IMPORTANT ONE: the fetch fails (403 on a missing scope, or
|
|
# Mastodon down). Nothing may be marked inactive — an outage must not
|
|
# empty the published list.
|
|
rec.live_ip_cidrs = lambda: None
|
|
rec.live_email_domains = lambda: None
|
|
assert rec.reconcile() == 0, "a failed fetch must change nothing"
|
|
assert "198.18.5.0/24" in active_ips() and "198.18.6.0/24" in active_ips()
|
|
assert len(active_domains()) >= 2
|
|
|
|
# D. Half-failure: IP fetch works, domain fetch doesn't. Only the side
|
|
# that answered is reconciled.
|
|
rec.live_ip_cidrs = lambda: {"198.18.5.0/24"}
|
|
rec.live_email_domains = lambda: None
|
|
rec.reconcile()
|
|
assert "198.18.6.0/24" not in active_ips(), "the side that answered is reconciled"
|
|
assert "rec-removed.test" in active_domains(), "the side that failed is left alone"
|
|
|
|
# E. --dry-run reports without writing.
|
|
rec.live_ip_cidrs = lambda: set()
|
|
rec.live_email_domains = lambda: set()
|
|
assert rec.reconcile(dry_run=True) > 0
|
|
assert "198.18.5.0/24" in active_ips(), "dry-run must not change stored state"
|
|
|
|
# F. An expired block is treated as gone even though Mastodon still lists it.
|
|
future = (datetime.now(timezone.utc) + timedelta(days=1)).isoformat()
|
|
past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat()
|
|
assert rec._expired({"expires_at": past}) is True
|
|
assert rec._expired({"expires_at": future}) is False
|
|
assert rec._expired({"expires_at": None}) is False
|
|
assert rec._expired({}) is False
|
|
|
|
# G. A re-registration reactivates a row that had been reconciled away.
|
|
rec.live_ip_cidrs = lambda: set()
|
|
rec.live_email_domains = lambda: set()
|
|
rec.reconcile()
|
|
assert "198.18.5.0/24" not in active_ips()
|
|
main.mark_ipblock_registered("950")
|
|
assert "198.18.5.0/24" in active_ips(), "re-registering must reactivate the row"
|
|
|
|
|
|
def reputation_api_tests():
|
|
"""Drive POST /api/reputation — the pre-signup query path Mastodon calls
|
|
before its own IpBlock/EmailDomainBlock lookup. The load-bearing property
|
|
is ordering: the block must be installed by the time the call returns,
|
|
because the caller looks it up microseconds later."""
|
|
dms = []
|
|
ipblocks = []
|
|
domain_blocks = []
|
|
main.dm_moderator = lambda message: dms.append(message)
|
|
main.register_ip_block = (
|
|
lambda ip, acct, org, cidr="", comment="":
|
|
ipblocks.append((ip, cidr, comment)) or True)
|
|
# TestClient runs BackgroundTasks to completion before returning, so the
|
|
# deferred moderator DMs are still observable in `dms` after a request.
|
|
main.register_email_domain_block = (
|
|
lambda domain, acct: domain_blocks.append((domain, acct)) or True)
|
|
|
|
classifications = {
|
|
"203.0.113.50": ("clean", "Example Residential ISP", False, "203.0.113.50/32"),
|
|
"198.51.100.50": ("datacenter", "Example Cloud Inc", True, "198.51.100.0/24"),
|
|
"198.51.100.51": ("datacenter", "Example Cloud Inc", True, "198.51.100.0/24"),
|
|
"198.51.100.60": ("proxy", "Example Proxy Co", True, "198.51.100.64/26"),
|
|
}
|
|
main.classify_signup_ip = lambda ip: classifications[ip]
|
|
main.classify_email_domain = lambda domain: {
|
|
"rep-clean.test": (False, 0),
|
|
"rep-spam.test": (True, 99),
|
|
}[domain]
|
|
|
|
main.IP_SCRUTINY_ENABLED = True
|
|
main.IP_SCRUTINY_DRY_RUN = False
|
|
main.IP_SCRUTINY_AUTO_IPBLOCK = True
|
|
main.CHECK_MAIL_ENABLED = True
|
|
main.CHECK_MAIL_API_KEY = "dummy"
|
|
main.CHECK_MAIL_DRY_RUN = False
|
|
main.CHECK_MAIL_AUTO_DOMAIN_BLOCK = True
|
|
main.CHECK_MAIL_RISK_THRESHOLD = 80
|
|
main.REPUTATION_API_INSTALL_BLOCKS = True
|
|
main.REPUTATION_API_DM_MODERATOR = True
|
|
main.REPUTATION_API_MAX_LOOKUPS_PER_HOUR = 0 # no ceiling except where tested
|
|
|
|
def ask(payload, key="rep-secret"):
|
|
headers = {"X-Reputation-Key": key} if key is not None else {}
|
|
return client.post("/api/reputation", json=payload, headers=headers)
|
|
|
|
# A. The gate. Disabled, or enabled with no key configured, is a 404 —
|
|
# a route that installs blocks must never be reachable unauthenticated,
|
|
# and "off" should look like "not a route".
|
|
main.REPUTATION_API_ENABLED = False
|
|
main.REPUTATION_API_KEY = "rep-secret"
|
|
assert ask({"ip": "203.0.113.50"}).status_code == 404
|
|
|
|
main.REPUTATION_API_ENABLED = True
|
|
main.REPUTATION_API_KEY = ""
|
|
assert ask({"ip": "203.0.113.50"}).status_code == 404, \
|
|
"enabled without a key must stay closed, not open"
|
|
|
|
main.REPUTATION_API_KEY = "rep-secret"
|
|
assert ask({"ip": "203.0.113.50"}, key="wrong").status_code == 401
|
|
assert ask({"ip": "203.0.113.50"}, key=None).status_code == 401
|
|
|
|
# B. Something to look up is required.
|
|
assert ask({}).status_code == 400
|
|
assert ask({"source": "registration"}).status_code == 400
|
|
|
|
# C. A clean IP: answered, recorded, nothing installed.
|
|
ipblocks.clear(); dms.clear()
|
|
r = ask({"ip": "203.0.113.50", "source": "registration"})
|
|
assert r.status_code == 200, r.status_code
|
|
body = r.json()
|
|
assert body["flagged"] is False and body["blocked"] is False, body
|
|
assert body["ip"]["classification"] == "clean"
|
|
assert body["email"] is None, "no email asked about, no email answer"
|
|
assert ipblocks == [], "a clean IP must not be blocked"
|
|
assert dms == [], "a clean IP must not wake a moderator"
|
|
|
|
# D. THE POINT OF THE FEATURE: a flagged IP is blocked *during* the call,
|
|
# so the caller's own IpBlock lookup right afterwards finds it.
|
|
r = ask({"ip": "198.51.100.50", "source": "registration"})
|
|
body = r.json()
|
|
assert body["flagged"] is True and body["blocked"] is True, body
|
|
assert body["ip"]["cidr"] == "198.51.100.0/24"
|
|
assert len(ipblocks) == 1, ipblocks
|
|
assert ipblocks[0][1] == "198.51.100.0/24", "the range is blocked, not the address"
|
|
assert "pre-signup" in ipblocks[0][2], "the block says why it exists"
|
|
assert any("198.51.100.0/24" in d for d in dms), dms
|
|
assert main.reputation_block_installed("ip", "198.51.100.0/24")
|
|
|
|
# E. A second address in the same range: still blocked, but we don't
|
|
# re-POST a block we already installed — this runs on the signup
|
|
# request path, so the saved round trip is saved latency.
|
|
dms.clear()
|
|
r = ask({"ip": "198.51.100.51"})
|
|
assert r.json()["blocked"] is True, "an already-installed block still counts as blocked"
|
|
assert len(ipblocks) == 1, "an already-installed block must not be re-POSTed"
|
|
assert dms == [], "only a newly-installed block wakes a moderator"
|
|
|
|
# F. REPUTATION_API_INSTALL_BLOCKS=false classifies and records but
|
|
# installs nothing — the dry-run equivalent for this path.
|
|
main.REPUTATION_API_INSTALL_BLOCKS = False
|
|
r = ask({"ip": "198.51.100.60"})
|
|
body = r.json()
|
|
assert body["flagged"] is True, body
|
|
assert body["blocked"] is False, "install-blocks off must not install"
|
|
assert len(ipblocks) == 1, ipblocks
|
|
main.REPUTATION_API_INSTALL_BLOCKS = True
|
|
|
|
# G. The email side, and the standing rule that a full address is never
|
|
# stored, logged or echoed — only its domain.
|
|
domain_blocks.clear(); dms.clear()
|
|
r = ask({"ip": "203.0.113.50", "email": "[email protected]", "source": "sign_up"})
|
|
body = r.json()
|
|
assert body["email"]["domain"] == "rep-spam.test", "domain is normalised to lower case"
|
|
assert body["email"]["flagged"] is True and body["email"]["blocked"] is True
|
|
assert body["blocked"] is True, "either side flagging makes the whole answer blocked"
|
|
assert body["ip"]["flagged"] is False, "the two sides are independent"
|
|
assert domain_blocks and domain_blocks[0][0] == "rep-spam.test", domain_blocks
|
|
assert "SpamGuy" not in r.text and "spamguy" not in r.text.lower(), \
|
|
"the local part must never come back in the response"
|
|
with main._db() as conn:
|
|
stored = conn.execute(
|
|
"SELECT value FROM reputation_query WHERE kind = 'email'").fetchall()
|
|
assert all("@" not in v[0] for v in stored), \
|
|
"reputation_query must store domains, never addresses"
|
|
|
|
# H. A clean domain is answered but not blocked.
|
|
domain_blocks.clear()
|
|
r = ask({"email": "[email protected]"})
|
|
assert r.json()["email"]["flagged"] is False
|
|
assert r.json()["blocked"] is False
|
|
assert domain_blocks == [], domain_blocks
|
|
|
|
# I. The hourly ceiling on fresh provider lookups. Over budget, an
|
|
# uncached value answers "unknown" and blocks nothing — the guard
|
|
# exists to stop paying, so it must never guess "flagged" instead.
|
|
# Addresses here must fall outside every range cached by earlier tests,
|
|
# or they'd be cache hits and never reach the budget at all.
|
|
main._lookup_budget.clear()
|
|
main.REPUTATION_API_MAX_LOOKUPS_PER_HOUR = 1
|
|
classifications["198.51.102.10"] = ("datacenter", "Example Cloud Inc", True,
|
|
"198.51.102.0/25")
|
|
classifications["198.51.102.200"] = ("datacenter", "Other Cloud Inc", True,
|
|
"198.51.102.128/25")
|
|
before = len(ipblocks)
|
|
first = ask({"ip": "198.51.102.10"}).json()["ip"]
|
|
assert first["cached"] is False and first["deferred"] is False, first
|
|
assert len(ipblocks) == before + 1, "the in-budget lookup still works normally"
|
|
|
|
over = ask({"ip": "198.51.102.200"}).json()["ip"]
|
|
assert over["deferred"] is True, over
|
|
assert over["classification"] == "unknown" and over["flagged"] is False, over
|
|
assert over["blocked"] is False
|
|
assert len(ipblocks) == before + 1, "an over-budget lookup must install nothing"
|
|
assert not main.reputation_block_installed("ip", "198.51.102.128/25")
|
|
|
|
# A cache hit costs nothing, so it is answered normally even with the
|
|
# budget spent — the ceiling limits provider spend, not service.
|
|
cached_hit = ask({"ip": "198.51.100.51"}).json()["ip"]
|
|
assert cached_hit["cached"] is True and cached_hit["deferred"] is False, cached_hit
|
|
assert cached_hit["blocked"] is True, "a cached flagged range still answers blocked"
|
|
main.REPUTATION_API_MAX_LOOKUPS_PER_HOUR = 0
|
|
main._lookup_budget.clear()
|
|
|
|
# J. Blocks installed here show up in the public blocklist alongside the
|
|
# signup-derived ones — otherwise the published list under-reports as
|
|
# blocking shifts to the pre-signup path. `signups` still counts
|
|
# signups, so a range blocked before anyone signed up reads 0.
|
|
main.BLOCKS_API_ENABLED = True
|
|
ranges = {e["cidr"]: e for e in client.get("/api/blocks/ip-ranges").json()["items"]}
|
|
domains = {e["domain"]: e for e in client.get("/api/blocks/email-domains").json()["items"]}
|
|
assert "198.51.100.0/24" in ranges, ranges
|
|
assert ranges["198.51.100.0/24"]["signups"] == 0, \
|
|
"nothing signed up from this range — it was blocked first"
|
|
assert "rep-spam.test" in domains, domains
|
|
assert set(ranges["198.51.100.0/24"]) == {
|
|
"cidr", "signups", "first_blocked_at", "last_blocked_at"}, \
|
|
"merging must not change the published entry shape"
|
|
|
|
# K. ...and the classification behind them stays out of the public list,
|
|
# same rule as for the signup-derived entries.
|
|
raw = client.get("/api/blocks").text
|
|
for leak in ("Example Cloud Inc", "Example Proxy Co", "datacenter", "proxy"):
|
|
assert leak not in raw, f"{leak!r} must not reach the public blocklist"
|
|
|
|
# L. The reconcile pass covers these rows too: a block a moderator removes
|
|
# stops being published, and reappears if it's registered again.
|
|
import app.reconcile_blocks as rec
|
|
rec.live_ip_cidrs = lambda: set()
|
|
rec.live_email_domains = lambda: set()
|
|
rec.reconcile()
|
|
active = {e["cidr"] for e in client.get("/api/blocks/ip-ranges").json()["items"]}
|
|
assert "198.51.100.0/24" not in active, \
|
|
"a query-API block removed on the Mastodon side must stop being published"
|
|
assert not main.reputation_block_installed("ip", "198.51.100.0/24")
|
|
|
|
ipblocks.clear()
|
|
r = ask({"ip": "198.51.100.50"})
|
|
assert r.json()["blocked"] is True
|
|
assert len(ipblocks) == 1, "a reconciled-away block is re-installed, not assumed present"
|
|
active = {e["cidr"] for e in client.get("/api/blocks/ip-ranges").json()["items"]}
|
|
assert "198.51.100.0/24" in active, "re-installing republishes it"
|
|
|
|
# M. IP blocks are registered with an expiry, and our own dedup record
|
|
# lapses with them. Without that, a query between a block expiring and
|
|
# the next hourly reconcile would claim a block Mastodon has already
|
|
# dropped -- and skip re-installing it.
|
|
main.REPUTATION_API_ENABLED = True
|
|
params = main._ip_block_params("198.51.100.0/24", "test")
|
|
assert params["expires_in"] == main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN, params
|
|
assert main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN == 86400, "default is 24 hours"
|
|
|
|
saved = main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN
|
|
main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN = 0
|
|
assert "expires_in" not in main._ip_block_params("198.51.100.0/24", "test"), \
|
|
"a 0 expiry omits the field rather than sending 0"
|
|
main.IP_SCRUTINY_IPBLOCK_EXPIRES_IN = saved
|
|
|
|
# An expired record reads as not-installed, so the next query
|
|
# re-installs rather than assuming it's still there.
|
|
main.record_reputation_block("ip", "203.0.113.0/24", "datacenter", "Expiring Inc", 86400)
|
|
assert main.reputation_block_installed("ip", "203.0.113.0/24")
|
|
with main._db() as conn:
|
|
conn.execute("UPDATE reputation_block SET expires_at = ? WHERE target = ?",
|
|
("2020-01-01T00:00:00+00:00", "203.0.113.0/24"))
|
|
assert not main.reputation_block_installed("ip", "203.0.113.0/24"), \
|
|
"an expired block must not count as installed"
|
|
active = {e["cidr"] for e in client.get("/api/blocks/ip-ranges").json()["items"]}
|
|
assert "203.0.113.0/24" not in active, "an expired block must not be published"
|
|
|
|
# N. The moderator DM is deferred until after the response. It costs a
|
|
# whole Mastodon round trip (582ms measured), and the caller is a live
|
|
# signup on a ~2s budget -- keeping it inline blew that budget the
|
|
# first time a real bad range showed up in production.
|
|
sent_order = []
|
|
main.register_ip_block = (
|
|
lambda ip, acct, org, cidr="", comment="":
|
|
sent_order.append("block") or ipblocks.append((ip, cidr, comment)) or True)
|
|
main.dm_moderator = lambda message: sent_order.append("dm") or dms.append(message)
|
|
classifications["198.51.103.7"] = ("datacenter", "Late DM Inc", True, "198.51.103.0/24")
|
|
|
|
dms.clear()
|
|
r = ask({"ip": "198.51.103.7", "source": "defer-check"})
|
|
assert r.json()["blocked"] is True, r.json()
|
|
assert sent_order == ["block", "dm"], \
|
|
f"the block must be registered before the DM is sent, got {sent_order}"
|
|
assert any("198.51.103.0/24" in d for d in dms), "the DM still goes out, just later"
|
|
|
|
# ...and the response body never depends on the DM having been sent.
|
|
main.dm_moderator = lambda message: (_ for _ in ()).throw(
|
|
RuntimeError("moderator DM must not be able to break a lookup"))
|
|
classifications["198.51.104.7"] = ("datacenter", "Broken DM Inc", True, "198.51.104.0/24")
|
|
try:
|
|
r = ask({"ip": "198.51.104.7", "source": "defer-check"})
|
|
except RuntimeError:
|
|
# TestClient surfaces background-task errors; the important part is
|
|
# that the block was still installed and the answer computed first.
|
|
pass
|
|
assert main.reputation_block_installed("ip", "198.51.104.0/24"), \
|
|
"a failing DM must not prevent the block being registered"
|
|
main.dm_moderator = lambda message: dms.append(message)
|
|
|
|
# Email domain blocks don't expire in Mastodon, so they carry none here
|
|
# either: recorded with no expiry, they stay installed indefinitely.
|
|
main.record_reputation_block("email", "rep-forever.test", "", "")
|
|
assert main.reputation_block_installed("email", "rep-forever.test"), \
|
|
"email domain blocks have no expiry and stay installed"
|
|
with main._db() as conn:
|
|
row = conn.execute(
|
|
"SELECT expires_at FROM reputation_block WHERE target = ?",
|
|
("rep-forever.test",)).fetchone()
|
|
assert row[0] is None, row
|
|
|
|
main.REPUTATION_API_ENABLED = False
|
|
|
|
|
|
if __name__ == "__main__":
|
|
routing_tests()
|
|
policy_tests()
|
|
ip_scrutiny_tests()
|
|
email_scrutiny_tests()
|
|
range_cache_tests()
|
|
block_width_cap_tests()
|
|
combined_signal_tests()
|
|
strong_ip_suspend_tests()
|
|
junk_username_tests()
|
|
suspicious_watch_tests()
|
|
suspicious_sweep_tests()
|
|
blocks_api_tests()
|
|
reconcile_blocks_tests()
|
|
reputation_api_tests()
|
|
print("ALL TESTS PASSED")
|