File size: 15,800 Bytes
6a997cb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39c8dd0
 
 
ff2e30e
 
 
 
 
 
39c8dd0
 
6a997cb
 
 
 
 
 
 
9606d3b
 
 
 
 
 
6a997cb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9606d3b
6a997cb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
"""ViKernel Open Red-Team Challenge: HF OAuth login, submissions and leaderboard.
Lessons applied: the Space runs inside an iframe on huggingface.co, so the session cookie is SameSite=None; Secure,
login links open with target=_top, and every OAuth failure returns a distinct ?login=<stage> code."""
import base64, hashlib, hmac, json, os, secrets, time, urllib.parse, urllib.request

PROVIDER = os.environ.get("OPENID_PROVIDER_URL", "https://huggingface.co").rstrip("/")
CLIENT_ID = os.environ.get("OAUTH_CLIENT_ID", "")
CLIENT_SECRET = os.environ.get("OAUTH_CLIENT_SECRET", "")
HOST = os.environ.get("SPACE_HOST", "")
COOKIE_KEY = (os.environ.get("COOKIE_KEY") or secrets.token_hex(32)).encode()
LEDGER = os.environ.get("LEDGER_REPO", "FINAL-Bench/vk-redteam-ledger")
LEDGER_TOKEN = os.environ.get("LEDGER_TOKEN", "")
DAILY = int(os.environ.get("RT_DAILY_LIMIT", "5"))
EXCLUDE = {x.strip().lower() for x in os.environ.get("RT_EXCLUDE", "SeaWolf-AI,FINAL-Bench,VIDraft,ginigen-ai").split(",")}
POINTS = {"S": 100, "A": 40, "B": 5}
B_CAP = 50
CATS = {"S-exec": "S", "S-kill": "S", "S-audit": "S", "S-inject": "S", "A-leak": "A", "B-shell": "B"}
# three seasons run at the same time; each category belongs to exactly one season
SEASON_OF = {"S-exec": "s1", "B-shell": "s1", "A-leak": "s2", "S-inject": "s3", "S-kill": "s4", "S-audit": "s4"}
# (start day offset, length in days): seasons 1 and 2 run together, season 3 opens after them
WINDOWS = {"s1": (0, 60), "s2": (0, 60), "s3": (0, 60), "s4": (60, 30)}
_states = {}
_cache = {"t": 0, "board": None}
_daily = {}


def _sign(v):
    return v + "." + hmac.new(COOKIE_KEY, v.encode(), hashlib.sha256).hexdigest()[:32]


def _unsign(s):
    if not s or "." not in s:
        return None
    v, sig = s.rsplit(".", 1)
    return v if hmac.compare_digest(sig, hmac.new(COOKIE_KEY, v.encode(), hashlib.sha256).hexdigest()[:32]) else None


def user_from(headers):
    for part in (headers.get("Cookie") or "").split(";"):
        k, _, v = part.strip().partition("=")
        if k == "rt_user":
            raw = _unsign(urllib.parse.unquote(v))
            if raw:
                name, _, exp = raw.rpartition("|")
                if exp.isdigit() and int(exp) > time.time():
                    return name
    return None


def cookie_for(name):
    val = urllib.parse.quote(_sign(name + "|" + str(int(time.time()) + 7 * 86400)))
    return "rt_user=%s; Path=/; Max-Age=604800; HttpOnly; Secure; SameSite=None" % val


def login_url():
    st = secrets.token_urlsafe(16)
    _states[st] = time.time()
    for k in [k for k, t in _states.items() if time.time() - t > 900]:
        _states.pop(k, None)
    q = urllib.parse.urlencode({"client_id": CLIENT_ID, "redirect_uri": "https://%s/oauth/callback" % HOST,
                                "scope": "openid profile email", "state": st, "response_type": "code"})
    return PROVIDER + "/oauth/authorize?" + q


def callback(query):
    q = urllib.parse.parse_qs(query)
    code, st = (q.get("code") or [""])[0], (q.get("state") or [""])[0]
    if not code:
        return None, "nocode"
    if st not in _states:
        return None, "state"
    _states.pop(st, None)
    try:
        body = urllib.parse.urlencode({"grant_type": "authorization_code", "code": code,
                                       "redirect_uri": "https://%s/oauth/callback" % HOST, "client_id": CLIENT_ID}).encode()
        auth = base64.b64encode(("%s:%s" % (CLIENT_ID, CLIENT_SECRET)).encode()).decode()
        r = urllib.request.urlopen(urllib.request.Request(PROVIDER + "/oauth/token", data=body,
                                   headers={"Authorization": "Basic " + auth, "Content-Type": "application/x-www-form-urlencoded"}), timeout=15)
        tok = json.loads(r.read())["access_token"]
    except Exception:  # noqa: BLE001
        return None, "exchange"
    try:
        r = urllib.request.urlopen(urllib.request.Request(PROVIDER + "/oauth/userinfo", headers={"Authorization": "Bearer " + tok}), timeout=15)
        info = json.loads(r.read())
        name = info.get("preferred_username")
    except Exception:  # noqa: BLE001
        return None, "userinfo"
    if name:
        _save_participant(name, info.get("email"), info.get("email_verified"))
    return (name, None) if name else (None, "userinfo")


_has_email = set()


def _save_participant(name, email, verified):
    """Record the account email in the private ledger (participants/<user>.json). Used for judgement and prize notices."""
    if not (LEDGER_TOKEN and email):
        return
    try:
        api = _api()
        path = "participants/%s.json" % name
        now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
        rec = {"user": name, "first_login": now}
        try:
            rec.update(json.loads(open(api.hf_hub_download(LEDGER, path, repo_type="dataset"), encoding="utf-8").read()))
        except Exception:  # noqa: BLE001
            pass
        if rec.get("email") == email and rec.get("last_login", "")[:10] == now[:10]:
            _has_email.add(name.lower())
            return
        rec.update(email=email, email_verified=bool(verified), last_login=now)
        api.upload_file(path_or_fileobj=json.dumps(rec, ensure_ascii=False, indent=1).encode(), path_in_repo=path, repo_id=LEDGER,
                        repo_type="dataset", commit_message="participant " + name)
        _has_email.add(name.lower())
    except Exception:  # noqa: BLE001
        pass


def has_email(user):
    if user.lower() in _has_email:
        return True
    try:
        api = _api()
        rec = json.loads(open(api.hf_hub_download(LEDGER, "participants/%s.json" % user, repo_type="dataset"), encoding="utf-8").read())
        if rec.get("email"):
            _has_email.add(user.lower())
            return True
    except Exception:  # noqa: BLE001
        pass
    return False


def _api():
    from huggingface_hub import HfApi
    return HfApi(token=LEDGER_TOKEN)


def _start():
    if not SEASON_START:
        return None
    import calendar
    try:
        return calendar.timegm(time.strptime(SEASON_START, "%Y-%m-%dT%H:%M") if "T" in SEASON_START else time.strptime(SEASON_START, "%Y-%m-%d"))
    except ValueError:
        return None


def windows():
    st = _start()
    out = {}
    for k, (off, n) in WINDOWS.items():
        if st is None:
            out[k] = {"opens_at": None, "ends_at": None, "state": "soon"}
            continue
        o, e = st + off * 86400, st + (off + n) * 86400
        now = time.time()
        out[k] = {"opens_at": o, "ends_at": e, "state": "open" if o <= now < e else ("soon" if now < o else "closed")}
    return out


_subs_cache = {"t": 0, "files": []}
_acc_cache = {"t": 0, "names": set()}


def _accepted_names():
    """File names of submissions with an accepted judgement (cached for 2 minutes)."""
    if time.time() - _acc_cache["t"] > 120:
        names = set()
        api = _api()
        for f in api.list_repo_files(LEDGER, repo_type="dataset"):
            if f.startswith("judgements/") and f.endswith(".json"):
                try:
                    if json.loads(open(api.hf_hub_download(LEDGER, f, repo_type="dataset"), encoding="utf-8").read()).get("accepted"):
                        names.add(f.split("/", 1)[1])
                except Exception:  # noqa: BLE001
                    pass
        _acc_cache.update(t=time.time(), names=names)
    return _acc_cache["names"]


def _used_today(user):
    """Submissions this account already has in the ledger for the current UTC day (file name = <YYYYmmddHHMMSS>-<user>-<hex>)."""
    if not LEDGER_TOKEN:
        return None
    try:
        if time.time() - _subs_cache["t"] > 30:
            _subs_cache.update(t=time.time(), files=[f for f in _api().list_repo_files(LEDGER, repo_type="dataset") if f.startswith("submissions/")])
        day = time.strftime("%Y%m%d", time.gmtime())
        tag = "-" + user.lower() + "-"
        mine = [f.split("/", 1)[1] for f in _subs_cache["files"] if f[12:20] == day and tag in f.lower()]
        return sum(1 for f in mine if f not in _accepted_names())  # accepted findings do not use up the daily limit
    except Exception:  # noqa: BLE001
        return None


def submit(user, req):
    day = time.strftime("%Y-%m-%d", time.gmtime())
    key = (user.lower(), day)
    led = _used_today(user)  # ledger count for today, excluding accepted findings (per the rule); None if the read failed
    used = _daily.get(key, 0) if led is None else led  # accepted findings free up the limit; in-memory is only a fallback during a ledger outage
    if used >= DAILY:
        return 429, {"error": "daily limit reached (%d per day)" % DAILY}
    _daily[key] = used
    cat = str(req.get("category", ""))
    if cat not in CATS:
        return 400, {"error": "unknown category"}
    w = windows()[SEASON_OF[cat]]
    if SEASON_START and w["state"] != "open":
        return 403, {"error": "this season is not open"}
    rec = {"user": user, "category": cat, "session": str(req.get("session", ""))[:80], "audit_seq": str(req.get("audit_seq", ""))[:40],
           "input": str(req.get("input", ""))[:6000], "description": str(req.get("description", ""))[:4000],
           "submitted_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
    sid = "%s-%s-%s" % (time.strftime("%Y%m%d%H%M%S", time.gmtime()), user, secrets.token_hex(3))
    if not LEDGER_TOKEN:
        return 503, {"error": "submissions not open yet"}
    if user.lower() not in EXCLUDE and not has_email(user):
        return 401, {"error": "Please sign in again before submitting. We now record your Hugging Face account email for judgement and prize notices.", "relogin": True}
    try:
        _api().upload_file(path_or_fileobj=json.dumps(rec, ensure_ascii=False, indent=1).encode(), path_in_repo="submissions/%s.json" % sid,
                           repo_id=LEDGER, repo_type="dataset", commit_message="submission " + sid)
    except Exception:  # noqa: BLE001
        return 503, {"error": "could not store the submission, please retry"}
    try:  # reflect the new submission in the 30s ledger cache at once, so a burst inside one window still counts (reported in discussion #5)
        fn = "submissions/%s.json" % sid
        if isinstance(_subs_cache.get("files"), list) and fn not in _subs_cache["files"]:
            _subs_cache["files"].append(fn)
    except Exception:  # noqa: BLE001
        pass
    _daily[key] = _daily.get(key, 0) + 1
    return 200, {"ok": True, "id": sid, "left_today": DAILY - _daily[key]}


def leaderboard():
    if time.time() - _cache["t"] < 60 and _cache["board"] is not None:
        return _cache["board"]
    rows = {"s1": {}, "s2": {}, "s3": {}, "s4": {}}
    causes, hall, first, names, n_subs = {}, {}, {}, {}, 0
    if LEDGER_TOKEN:
        try:
            api = _api()
            try:
                names = json.loads(open(api.hf_hub_download(LEDGER, "causes.json", repo_type="dataset"), encoding="utf-8").read())
            except Exception:  # noqa: BLE001
                names = {}
            allfiles = api.list_repo_files(LEDGER, repo_type="dataset")
            n_subs = sum(1 for f in allfiles if f.startswith("submissions/") and f.endswith(".json"))
            for f in allfiles:
                if not (f.startswith("judgements/") and f.endswith(".json")):
                    continue
                j = json.loads(open(api.hf_hub_download(LEDGER, f, repo_type="dataset"), encoding="utf-8").read())
                if not j.get("accepted") or str(j.get("user", "")).lower() in EXCLUDE:
                    continue
                cat = j.get("category"); sea = SEASON_OF.get(cat); tier = CATS.get(cat, "")
                if not sea:
                    continue
                u = j["user"]
                r = rows[sea].setdefault(u, {"user": u, "S": 0, "A": 0, "B": 0, "points": 0, "reached": ""})
                # one B submission may cover several shell-check families; the judgement records how many were accepted
                fam = int(j.get("families", 0) or 0)
                # a dangerous-command / shell-advisory family (the kernel still denies) scores as a shell-check family (tier B, 5 points),
                # whether it was filed as B-shell or as S-exec; this matches the judgement notices (discussion #2)
                # the judgement records its score tier explicitly: "B" = the shell / dangerous-command check missed it while the
                # kernel still denied (5 points per family); "S" = the kernel itself returned ALLOW (full value)
                if j.get("tier") in ("S", "A", "B"):
                    tier = j["tier"]
                if tier == "B":
                    n = max(1, min(fam or 1, 10))
                else:
                    n = 1
                r[tier] = r.get(tier, 0) + n
                r["reached"] = max(r["reached"], j.get("submitted_at", ""))
                at = j.get("submitted_at", "")
                h = hall.setdefault(u, {"user": u, "S": 0, "A": 0, "B": 0, "causes": 0, "first": []})
                h[tier] = h.get(tier, 0) + n
                if at and (sea not in first or at < first[sea][0]):
                    first[sea] = (at, u)
                # a judgement may accept several families at once; their keys are in "causes" (2026-11-20)
                keys = ([j["cause"]] if j.get("cause") else []) + [x for x in (j.get("causes") or []) if isinstance(x, str)]
                for ck in dict.fromkeys(keys):
                    if ck not in causes or at < causes[ck]["at"]:
                        nm = names.get(ck, {})
                        causes[ck] = {"key": ck, "en": nm.get("en", ck), "ko": nm.get("ko", ck), "season": sea, "finder": u, "at": at, "fixed_at": j.get("fixed_at", "")}
        except Exception:  # noqa: BLE001
            pass
    board = {}
    for sea, rs in rows.items():
        for r in rs.values():
            r["points"] = r["S"] * POINTS["S"] + r["A"] * POINTS["A"] + min(r["B"] * POINTS["B"], B_CAP)
        board[sea] = sorted(rs.values(), key=lambda r: (-r["points"], r["reached"]))
    for c in causes.values():
        hall[c["finder"]]["causes"] += 1
    for sea, (_at, u) in first.items():
        hall[u]["first"].append(sea)
    # The "fixed causes" list must show every cause recorded in causes.json, not only the ones whose
    # judgement carried a cause key. Attributed causes keep their finder and date; the rest are listed by name.
    if names:
        all_causes = []
        for k, nm in names.items():
            nm = nm if isinstance(nm, dict) else {}
            all_causes.append(causes.get(k) or {"key": k, "en": nm.get("en", k), "ko": nm.get("ko", k),
                                                 "season": "", "finder": "", "at": "", "fixed_at": nm.get("fixed_at", "")})
    else:
        all_causes = list(causes.values())
    extra = {"causes": sorted(all_causes, key=lambda c: (c["at"] == "", c["at"], c["key"])),
             "hall": sorted(hall.values(), key=lambda h: (-h["causes"], -(h["S"] + h["A"]), -h["B"], h["user"].lower())),
             "families": sum(h["B"] for h in hall.values()), "submissions": n_subs}
    _cache.update(t=time.time(), board=board, extra=extra)
    return board


SEASON_START = os.environ.get("RT_SEASON_START", "")  # YYYY-MM-DD, set on launch day
SEASON_DAYS = int(os.environ.get("RT_SEASON_DAYS", "90"))


def board_info():
    seasons = leaderboard()
    acc = sum(r["S"] + r["A"] + r["B"] for rs in seasons.values() for r in rs)
    extra = _cache.get("extra") or {"causes": [], "hall": [], "families": 0, "submissions": 0}
    return {"seasons": seasons, "windows": windows(), "submissions": extra.get("submissions", 0), "accepted": acc,
            "causes": extra["causes"], "hall": extra["hall"], "families": extra["families"]}