Buckets:

download
raw
9.9 kB
#!/usr/bin/env python3
"""Report on publisher attribution and the platform detection gap.
Two questions the snapshot could not answer until flagged skills carried an
owner and ClawHub's own moderation state:
1. **Who publishes the bad skills?** Every campaign hypothesis this project has
tested was built from slug prefixes, and every one collapsed when checked:
a shared prefix is a naming convention, not an operator. An account is an
operator. Clustering on ``publisher`` is the first version of this question
that can actually be answered.
2. **What does the platform say about the skills we call malicious?** ClawHub
screens uploads and exposes the result. Holding our verdict and theirs on the
same skill is the only way to measure, from outside, how much gets through.
Both reports refuse to count what was never checked. ``moderation_blocked``
defaults to False on an unenriched record, so treating "not blocked" as the
platform's opinion would silently convert unenriched skills into evidence of a
gap. Every denominator here is the *checked* population, and the uncovered
remainder is printed rather than dropped.
Usage:
analyze-attribution.py <snapshot.json> [--min-cluster N] [--json out.json]
"""
from __future__ import annotations
import argparse
import json
import sys
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any
FLAGGED = {"MALICIOUS", "SUSPICIOUS", "CAUTION"}
def load_skills(path: Path) -> dict[str, dict[str, Any]]:
with path.open(encoding="utf-8") as fh:
snapshot = json.load(fh)
skills = snapshot.get("skills", {})
if not isinstance(skills, dict):
raise SystemExit(f"{path}: 'skills' is not an object")
return skills
def detection_gap(skills: dict[str, dict[str, Any]]) -> dict[str, Any]:
"""Compare our verdicts against ClawHub's moderation state.
Only skills we actually asked about are counted. A skill we verified
malicious that the platform has scanned and left unblocked is the finding;
one still pending its scan is not, and one we never enriched is neither.
"""
malicious = [s for s in skills.values() if s.get("verdict") == "MALICIOUS"]
# detail_fetched is required as well: snapshots written before the flag was
# tightened set moderation_checked on any successful fetch, including ones
# carrying no moderation at all, and those records survive until backfill
# reaches them. Without this the denominator counts skills we never
# actually got a screening result for.
checked = [
s for s in malicious if s.get("moderation_checked") and s.get("detail_fetched")
]
# Fetched, but the registry has no moderation record for this skill at all.
# That is not the platform clearing it, and not a broken pipeline either:
# roughly 40% of a sampled cohort came back this way. Counted separately
# because "screened and called clean" and "never screened" are different
# claims, and lumping them together overstates whichever one you name.
no_data = [
s for s in malicious if s.get("detail_fetched") and not s.get("moderation_checked")
]
# The registry's verdict is authoritative; the booleans are derived from it
# and all default False, so a cleared skill must actually say so.
cleared = [
s
for s in checked
if str(s.get("moderation_verdict", "")).lower() == "clean"
and not s.get("moderation_blocked")
and not s.get("moderation_pending")
and not s.get("moderation_suspicious")
]
blocked = [s for s in checked if s.get("moderation_blocked")]
pending = [s for s in checked if s.get("moderation_pending") and not s.get("moderation_blocked")]
suspicious = [
s
for s in checked
if s.get("moderation_suspicious")
and not s.get("moderation_blocked")
and not s.get("moderation_pending")
]
unflagged = cleared
return {
"malicious_total": len(malicious),
"malicious_checked": len(checked),
"malicious_unchecked": len(malicious) - len(checked) - len(no_data),
"malicious_fetched_no_moderation_data": len(no_data),
"platform_blocked": len(blocked),
"platform_suspicious": len(suspicious),
"platform_pending_scan": len(pending),
"platform_no_flag": len(unflagged),
"gap_rate": (len(unflagged) / len(checked)) if checked else None,
"cleared_by_engine": dict(
Counter(s.get("moderation_engine") or "?" for s in cleared).most_common()
),
"examples": sorted(s["slug"] for s in unflagged)[:25],
}
def publisher_clusters(
skills: dict[str, dict[str, Any]], min_cluster: int
) -> list[dict[str, Any]]:
"""Group flagged skills by the account that published them.
A cluster is reported with its full published footprint, not just its
flagged members: an account with 3 malicious skills out of 4 is a very
different claim from 3 out of 300, and only the first is worth calling a
campaign.
"""
# Keyed on the immutable account id where we have it, because a handle can
# be renamed and would then split one operator across two clusters.
by_publisher: dict[str, list[dict[str, Any]]] = defaultdict(list)
for skill in skills.values():
key = skill.get("publisher_id") or skill.get("publisher") or ""
if key:
by_publisher[key].append(skill)
clusters = []
for key, members in by_publisher.items():
publisher = next(
(m.get("publisher") for m in members if m.get("publisher")), key
)
flagged = [s for s in members if s.get("verdict") in FLAGGED]
malicious = [s for s in flagged if s.get("verdict") == "MALICIOUS"]
if len(malicious) < min_cluster:
continue
rules: Counter[str] = Counter()
for s in malicious:
rules.update(s.get("finding_rule_ids") or [])
clusters.append(
{
"publisher": publisher,
# Attributed skills only. We enrich the flagged tail, so this is
# not the account's total output and must not be read as such.
"attributed_skills": len(members),
"flagged": len(flagged),
"malicious": len(malicious),
"shared_rules": [r for r, n in rules.most_common(5) if n >= 2],
"slugs": sorted(s["slug"] for s in malicious)[:20],
}
)
clusters.sort(key=lambda c: (-c["malicious"], c["publisher"]))
return clusters
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("snapshot", type=Path)
parser.add_argument(
"--min-cluster",
type=int,
default=2,
help="Minimum malicious skills for a publisher to be reported (default 2)",
)
parser.add_argument("--json", type=Path, help="Write the full report here")
args = parser.parse_args()
skills = load_skills(args.snapshot)
gap = detection_gap(skills)
clusters = publisher_clusters(skills, args.min_cluster)
attributed = sum(1 for s in skills.values() if s.get("publisher"))
flagged_total = sum(1 for s in skills.values() if s.get("verdict") in FLAGGED)
print(f"snapshot: {args.snapshot}")
print(f"skills: {len(skills):,}")
print(f"flagged: {flagged_total:,}")
print(f"attributed: {attributed:,}")
print()
print("== platform detection gap ==")
print(f"we verified malicious: {gap['malicious_total']:,}")
print(f" of which we checked: {gap['malicious_checked']:,}")
print(f" not yet asked: {gap['malicious_unchecked']:,}")
if gap["malicious_fetched_no_moderation_data"]:
print(
f" no screening record: "
f"{gap['malicious_fetched_no_moderation_data']:,} "
f"<-- no screening record exists; not the same as cleared"
)
if gap["malicious_checked"]:
print(f" platform blocked: {gap['platform_blocked']:,}")
print(f" platform marked suspect: {gap['platform_suspicious']:,}")
print(f" platform scan pending: {gap['platform_pending_scan']:,}")
print(f" platform no flag at all: {gap['platform_no_flag']:,}")
print(f" gap rate: {gap['gap_rate']:.1%} of checked")
if gap["cleared_by_engine"]:
spread = ", ".join(f"{v} ({n})" for v, n in gap["cleared_by_engine"].items())
print(f" cleared by engine: {spread}")
if gap["examples"]:
print(" examples: " + ", ".join(gap["examples"][:10]))
else:
print(" nothing enriched yet; run a sweep with enrichment enabled")
print()
print(f"== publisher clusters (>= {args.min_cluster} malicious) ==")
if not clusters:
print(" none")
for c in clusters[:25]:
rules = ", ".join(c["shared_rules"]) or "no shared rule"
print(
f" {c['publisher']:<28} {c['malicious']:>4} malicious "
f"/ {c['flagged']:>4} flagged / {c['attributed_skills']:>4} attributed [{rules}]"
)
if len(clusters) > 25:
print(f" ... and {len(clusters) - 25} more")
if args.json:
args.json.write_text(
json.dumps(
{
"snapshot": str(args.snapshot),
"skills": len(skills),
"flagged": flagged_total,
"attributed": attributed,
"detection_gap": gap,
"publisher_clusters": clusters,
},
indent=2,
),
encoding="utf-8",
)
print(f"\nwrote {args.json}")
return 0
if __name__ == "__main__":
sys.exit(main())

Xet Storage Details

Size:
9.9 kB
·
Xet hash:
bd2f0a4c2838ca8ddba288a3ad1c4376ffd009a9e254e06c66fa53cd0c47e9f0

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.