File size: 2,850 Bytes
028a5cd
 
d12e2ce
028a5cd
 
d12e2ce
 
 
 
 
 
 
 
 
 
028a5cd
 
 
 
d12e2ce
028a5cd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d12e2ce
 
 
028a5cd
 
 
 
 
 
 
 
d12e2ce
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
// Where the explorer reads from: in development the local export (public/data), on the Space the private dataset,
// with the signed-in viewer's own token.
import type { RecEpisode, RecIndex, RecRow } from "./records";
import type { Episode, Index, Version } from "./types";

// One dataset per task line. kind "cb" = full CooperBench trajectories, "records" = curves, scores, per-episode stats and a few conversations.
export interface Project { id: string; name: string; dataset: string; kind: "cb" | "records" }
export const PROJECTS: Project[] = [
  { id: "cooperbench", name: "CooperBench", dataset: "dsh-marl/cooperbench-grow-trajectories", kind: "cb" },
  { id: "math", name: "数学", dataset: "dsh-marl/math-team-runs", kind: "records" },
  { id: "agentgym", name: "AgentGym", dataset: "dsh-marl/agentgym-team-runs", kind: "records" },
];
let project: Project = PROJECTS[0];
export const setProject = (p: Project) => { project = p; };

let token: string | null = null;
export const setToken = (t: string | null) => { token = t; };

const url = (path: string) =>
  import.meta.env.DEV ? `${import.meta.env.BASE_URL}data/${project.id}/${path}` : `https://huggingface.co/datasets/${project.dataset}/resolve/main/${path}`;

async function bytes(path: string): Promise<Uint8Array> {
  const r = await fetch(url(path), { headers: token ? { Authorization: `Bearer ${token}` } : {} });
  if ((r.status === 401 || r.status === 403) && !import.meta.env.DEV) dispatchEvent(new Event("auth-expired"));
  if (!r.ok) throw new Error(`${r.status} ${path}`);
  return new Uint8Array(await r.arrayBuffer());
}

async function json<T>(path: string): Promise<T> {
  let b = await bytes(path);
  if (b[0] === 0x1f && b[1] === 0x8b) {   // gzip, unless the server already undid it
    const s = new Blob([b]).stream().pipeThrough(new DecompressionStream("gzip"));
    b = new Uint8Array(await new Response(s).arrayBuffer());
  }
  return JSON.parse(new TextDecoder().decode(b)) as T;
}

const cache = new Map<string, Promise<unknown>>();
function cached<T>(path: string): Promise<T> {
  const k = project.id + ":" + path;
  if (!cache.has(k)) cache.set(k, json<T>(path).catch((e) => { cache.delete(k); throw e; }));
  return cache.get(k) as Promise<T>;
}

export const loadVersions = () => cached<Version[]>("versions.json");
export const loadIndex = (v: string) => cached<Index>(`${v}/site/index.json`);
export const loadEpisode = (v: string, id: string) => {
  const [run, ...rest] = id.split("/");
  return cached<Episode>(`${v}/site/episodes/${run}/${rest.join("/")}.json.gz`);
};

export const loadRecIndex = (v: string) => cached<RecIndex>(`${v}/site/index.json`);
export const loadRecRows = (v: string) => cached<RecRow[]>(`${v}/site/episodes.json.gz`);
export const loadRecEpisode = (v: string, file: string) => cached<RecEpisode>(`${v}/site/${file}`);