2Btainment commited on
Commit
e73dcb3
·
verified ·
1 Parent(s): f8253b7

P3.1: jobs persisted in bucket

Browse files
Files changed (6) hide show
  1. app.js +5 -5
  2. avatar.js +1 -1
  3. index.html +2 -2
  4. master.js +1 -1
  5. server/agents.py +58 -16
  6. workspace.js +5 -5
app.js CHANGED
@@ -1,10 +1,10 @@
1
  import * as THREE from './vendor/three.module.min.js';
2
  import { STUDIOS, stopLoops } from './widgets.js?v=50';
3
- import './real.js?v=70';
4
- import './avatar.js?v=70';
5
- import './master.js?v=70';
6
- import './workspace.js?v=70';
7
- import { icon } from './icons.js?v=70';
8
  import { RoomEnvironment } from './vendor/addons/environments/RoomEnvironment.js';
9
  import { EffectComposer } from './vendor/addons/postprocessing/EffectComposer.js';
10
  import { RenderPass } from './vendor/addons/postprocessing/RenderPass.js';
 
1
  import * as THREE from './vendor/three.module.min.js';
2
  import { STUDIOS, stopLoops } from './widgets.js?v=50';
3
+ import './real.js?v=71';
4
+ import './avatar.js?v=71';
5
+ import './master.js?v=71';
6
+ import './workspace.js?v=71';
7
+ import { icon } from './icons.js?v=71';
8
  import { RoomEnvironment } from './vendor/addons/environments/RoomEnvironment.js';
9
  import { EffectComposer } from './vendor/addons/postprocessing/EffectComposer.js';
10
  import { RenderPass } from './vendor/addons/postprocessing/RenderPass.js';
avatar.js CHANGED
@@ -1,7 +1,7 @@
1
  // Avatar / Persona studio (DoD): persistent avatars as workspace asset type `avatar`
2
  // avatars/<slug>/avatar.json + ref.webp + voice.wav · reused by Chat, Image (Kontext), agents.
3
  import { STUDIOS, onStop } from './widgets.js?v=50';
4
- import { run, result, MODULES, esc, hf, stTxt, url, badge, setS, keep, LS, FLUX, KONTEXT, TTS, CHAT, signedIn } from './real.js?v=70';
5
 
6
  const LANGS = ['German', 'English', 'French', 'Spanish', 'Auto'];
7
  const slug = (n) => (n || 'avatar').toLowerCase().normalize('NFKD').replace(/[^\w]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'avatar';
 
1
  // Avatar / Persona studio (DoD): persistent avatars as workspace asset type `avatar`
2
  // avatars/<slug>/avatar.json + ref.webp + voice.wav · reused by Chat, Image (Kontext), agents.
3
  import { STUDIOS, onStop } from './widgets.js?v=50';
4
+ import { run, result, MODULES, esc, hf, stTxt, url, badge, setS, keep, LS, FLUX, KONTEXT, TTS, CHAT, signedIn } from './real.js?v=71';
5
 
6
  const LANGS = ['German', 'English', 'French', 'Spanish', 'Auto'];
7
  const slug = (n) => (n || 'avatar').toLowerCase().normalize('NFKD').replace(/[^\w]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'avatar';
index.html CHANGED
@@ -8,7 +8,7 @@
8
  <title>STUDIUS · DAS Studio — Einswarm</title>
9
  <link rel="icon" href="assets/einswarm-mascot.webp">
10
  <link rel="preload" href="fonts/outfit-latin-wght.woff2" as="font" type="font/woff2" crossorigin>
11
- <link rel="stylesheet" href="style.css?v=70">
12
  </head>
13
  <body>
14
  <canvas id="gl"></canvas>
@@ -49,6 +49,6 @@
49
 
50
  <a id="bmc" href="https://www.buymeacoffee.com/2btainment" target="_blank" rel="noopener noreferrer">Buy me a coffee</a>
51
  <noscript><p style="color:#f2c94c;padding:2em">STUDIUS braucht JavaScript.</p></noscript>
52
- <script type="module" src="app.js?v=70"></script>
53
  </body>
54
  </html>
 
8
  <title>STUDIUS · DAS Studio — Einswarm</title>
9
  <link rel="icon" href="assets/einswarm-mascot.webp">
10
  <link rel="preload" href="fonts/outfit-latin-wght.woff2" as="font" type="font/woff2" crossorigin>
11
+ <link rel="stylesheet" href="style.css?v=71">
12
  </head>
13
  <body>
14
  <canvas id="gl"></canvas>
 
49
 
50
  <a id="bmc" href="https://www.buymeacoffee.com/2btainment" target="_blank" rel="noopener noreferrer">Buy me a coffee</a>
51
  <noscript><p style="color:#f2c94c;padding:2em">STUDIUS braucht JavaScript.</p></noscript>
52
+ <script type="module" src="app.js?v=71"></script>
53
  </body>
54
  </html>
master.js CHANGED
@@ -1,7 +1,7 @@
1
  // Mastering studio (DoD): client-side WebAudio chain EQ → compressor → limiter, loudness normalisation to target LUFS
2
  // (ITU-R BS.1770-4 K-weighting + gating, measured offline), live meter, A/B, WAV/MP3 export, save to workspace.
3
  import { STUDIOS, onStop, loop, cv } from './widgets.js?v=50';
4
- import { result, MODULES, esc, setS, keep } from './real.js?v=70';
5
 
6
  /* ---------- BS.1770 loudness ---------- */
7
  function biquad(x, b0, b1, b2, a0, a1, a2) { const y = new Float32Array(x.length); b0 /= a0; b1 /= a0; b2 /= a0; a1 /= a0; a2 /= a0; let x1 = 0, x2 = 0, y1 = 0, y2 = 0;
 
1
  // Mastering studio (DoD): client-side WebAudio chain EQ → compressor → limiter, loudness normalisation to target LUFS
2
  // (ITU-R BS.1770-4 K-weighting + gating, measured offline), live meter, A/B, WAV/MP3 export, save to workspace.
3
  import { STUDIOS, onStop, loop, cv } from './widgets.js?v=50';
4
+ import { result, MODULES, esc, setS, keep } from './real.js?v=71';
5
 
6
  /* ---------- BS.1770 loudness ---------- */
7
  function biquad(x, b0, b1, b2, a0, a1, a2) { const y = new Float32Array(x.length); b0 /= a0; b1 /= a0; b2 /= a0; a1 /= a0; a2 /= a0; let x1 = 0, x2 = 0, y1 = 0, y2 = 0;
server/agents.py CHANGED
@@ -31,6 +31,36 @@ def parse_json(txt):
31
  except Exception: pass
32
  return None
33
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
34
  def make_agents_router(session, need, bid, client_for):
35
  r = APIRouter()
36
 
@@ -129,37 +159,49 @@ def make_agents_router(session, need, bid, client_for):
129
  return {"final": None, "raw": txt[:300], "via": "json:llama-3.2-3B", "provider_errors": errs}
130
 
131
  # ---------- server-side job list with approval ----------
132
- def job_of(s, jid):
 
 
 
 
 
 
 
133
  j = JOBS.get(jid)
 
134
  if not j or j["user"] != uid(s): raise HTTPException(404, "job")
135
  return j
136
 
137
- def pub(j): return {k: v for k, v in j.items() if k != "user"}
 
138
 
139
  @r.get("/api/jobs")
140
- def jobs(req: Request):
141
- s = need(req); return {"jobs": [pub(j) for j in sorted(JOBS.values(), key=lambda x: -x["created"]) if j["user"] == uid(s)][:50]}
 
 
142
 
143
  @r.post("/api/jobs")
144
  async def job_new(req: Request):
145
  s = need(req); b = await req.json(); jid = "j" + secrets.token_hex(5)
146
- JOBS[jid] = {"id": jid, "user": uid(s), "task": str(b.get("task", ""))[:500], "agent": str(b.get("agent", ""))[:60], "team": str(b.get("team", ""))[:60],
 
147
  "state": "running", "steps": [], "created": int(time.time())}
148
- return pub(JOBS[jid])
149
 
150
  @r.post("/api/jobs/{jid}/step")
151
  async def job_step(jid: str, req: Request):
152
  """Register a tool call. External actions are held as 'awaiting_approval' until approved."""
153
- s = need(req); j = job_of(s, jid); b = await req.json(); tool = str(b.get("tool", ""))[:60]
154
  st = {"n": len(j["steps"]) + 1, "tool": tool, "agent": str(b.get("agent", ""))[:60], "args": b.get("args") or {}, "external": tool in EXTERNAL,
155
  "state": "awaiting_approval" if tool in EXTERNAL else "approved", "t": int(time.time())}
156
  j["steps"].append(st)
157
  if st["external"]: j["state"] = "awaiting_approval"
158
- return {"step": st, "job": pub(j)}
159
 
160
  @r.post("/api/jobs/{jid}/steps/{n}/{decision}")
161
- def job_decide(jid: str, n: int, decision: str, req: Request):
162
- s = need(req); j = job_of(s, jid)
163
  if decision not in ("approve", "reject", "done", "error"): raise HTTPException(400, "decision")
164
  if not 0 < n <= len(j["steps"]): raise HTTPException(404, "step")
165
  st = j["steps"][n - 1]
@@ -169,18 +211,18 @@ def make_agents_router(session, need, bid, client_for):
169
  j["state"] = "running" if decision == "approve" else "rejected"
170
  else:
171
  if st["state"] != "approved": raise HTTPException(409, "step not approved")
172
- st["state"] = decision
173
- return {"step": st, "job": pub(j)}
174
 
175
  @r.get("/api/jobs/{jid}/steps/{n}")
176
- def job_poll(jid: str, n: int, req: Request):
177
- s = need(req); j = job_of(s, jid)
178
  if not 0 < n <= len(j["steps"]): raise HTTPException(404, "step")
179
  return {"step": j["steps"][n - 1], "job": pub(j)}
180
 
181
  @r.post("/api/jobs/{jid}/finish")
182
  async def job_finish(jid: str, req: Request):
183
- s = need(req); j = job_of(s, jid); b = await req.json()
184
  j["state"] = b.get("state") if b.get("state") in ("done", "error", "rejected") else "done"; j["summary"] = str(b.get("summary", ""))[:1000]; j["finished"] = int(time.time())
185
- return pub(j)
186
  return r
 
31
  except Exception: pass
32
  return None
33
 
34
+ import threading
35
+ JD = ".studius/jobs/"
36
+ LOCKS: dict[str, threading.Lock] = {}
37
+ def persist(j, token):
38
+ """Write the job's latest state to the bucket in a background thread (serialized per job, newest wins)."""
39
+ if not j.get("bucket"): return
40
+ j["_rev"] = j.get("_rev", 0) + 1
41
+ lock = LOCKS.setdefault(j["id"], threading.Lock())
42
+ def w():
43
+ with lock:
44
+ rev = j["_rev"]
45
+ if j.get("persisted", 0) >= rev: return
46
+ data = json.dumps({k: v for k, v in j.items() if k not in ("user", "_rev", "persisted", "persist_error")} | {"owner": j["user"]}, indent=1, ensure_ascii=False).encode()
47
+ try: HfApi(token=token).batch_bucket_files(j["bucket"], add=[(data, f"{JD}{j['id']}.json")]); j["persisted"] = rev; j.pop("persist_error", None)
48
+ except Exception as e: j["persist_error"] = str(e)[:120]
49
+ threading.Thread(target=w, daemon=True).start()
50
+
51
+ def load_jobs(api, bucket, user):
52
+ out = []
53
+ try:
54
+ files = sorted([f for f in api.list_bucket_tree(bucket, prefix=JD.rstrip("/"), recursive=True) if getattr(f, "type", "") == "file" and f.path.endswith(".json")], key=lambda f: str(f.mtime), reverse=True)[:40]
55
+ with tempfile.TemporaryDirectory() as t:
56
+ api.download_bucket_files(bucket, [(f.path, f"{t}/{i}.json") for i, f in enumerate(files)])
57
+ for i, _ in enumerate(files):
58
+ try: j = json.loads(Path(f"{t}/{i}.json").read_text())
59
+ except Exception: continue
60
+ if j.get("owner") == user: out.append(j)
61
+ except Exception: pass
62
+ return out
63
+
64
  def make_agents_router(session, need, bid, client_for):
65
  r = APIRouter()
66
 
 
159
  return {"final": None, "raw": txt[:300], "via": "json:llama-3.2-3B", "provider_errors": errs}
160
 
161
  # ---------- server-side job list with approval ----------
162
+ def restore(s, bucket):
163
+ for j in load_jobs(HfApi(token=s["token"]), bucket, uid(s)):
164
+ if j["id"] not in JOBS:
165
+ j["user"] = j.pop("owner", uid(s)); j["restored"] = True
166
+ if j.get("state") in ("running", "awaiting_approval"): j["state"] = "interrupted" # server restarted mid-run; runner (browser) is gone
167
+ JOBS[j["id"]] = j
168
+
169
+ def job_of(s, jid, bucket=None):
170
  j = JOBS.get(jid)
171
+ if not j and bucket: restore(s, bid(bucket, s)); j = JOBS.get(jid)
172
  if not j or j["user"] != uid(s): raise HTTPException(404, "job")
173
  return j
174
 
175
+ def pub(j): return {k: v for k, v in j.items() if k not in ("user", "_rev")} | {"saved": j.get("persisted", 0) >= j.get("_rev", 0)}
176
+ def save(s, j): persist(j, s["token"]); return pub(j)
177
 
178
  @r.get("/api/jobs")
179
+ def jobs(req: Request, bucket: str = ""):
180
+ s = need(req)
181
+ if bucket: restore(s, bid(bucket, s))
182
+ return {"jobs": [pub(j) for j in sorted(JOBS.values(), key=lambda x: -x["created"]) if j["user"] == uid(s) and (not bucket or j.get("bucket") == bucket)][:50]}
183
 
184
  @r.post("/api/jobs")
185
  async def job_new(req: Request):
186
  s = need(req); b = await req.json(); jid = "j" + secrets.token_hex(5)
187
+ bk = bid(b["bucket"], s) if b.get("bucket") else ""
188
+ JOBS[jid] = {"id": jid, "user": uid(s), "bucket": bk, "task": str(b.get("task", ""))[:500], "agent": str(b.get("agent", ""))[:60], "team": str(b.get("team", ""))[:60],
189
  "state": "running", "steps": [], "created": int(time.time())}
190
+ return save(s, JOBS[jid])
191
 
192
  @r.post("/api/jobs/{jid}/step")
193
  async def job_step(jid: str, req: Request):
194
  """Register a tool call. External actions are held as 'awaiting_approval' until approved."""
195
+ b = await req.json(); s = need(req); j = job_of(s, jid, b.get("bucket")); tool = str(b.get("tool", ""))[:60]
196
  st = {"n": len(j["steps"]) + 1, "tool": tool, "agent": str(b.get("agent", ""))[:60], "args": b.get("args") or {}, "external": tool in EXTERNAL,
197
  "state": "awaiting_approval" if tool in EXTERNAL else "approved", "t": int(time.time())}
198
  j["steps"].append(st)
199
  if st["external"]: j["state"] = "awaiting_approval"
200
+ return {"step": st, "job": save(s, j)}
201
 
202
  @r.post("/api/jobs/{jid}/steps/{n}/{decision}")
203
+ def job_decide(jid: str, n: int, decision: str, req: Request, bucket: str = ""):
204
+ s = need(req); j = job_of(s, jid, bucket)
205
  if decision not in ("approve", "reject", "done", "error"): raise HTTPException(400, "decision")
206
  if not 0 < n <= len(j["steps"]): raise HTTPException(404, "step")
207
  st = j["steps"][n - 1]
 
211
  j["state"] = "running" if decision == "approve" else "rejected"
212
  else:
213
  if st["state"] != "approved": raise HTTPException(409, "step not approved")
214
+ st["state"] = decision; st["finished"] = int(time.time())
215
+ return {"step": st, "job": save(s, j)}
216
 
217
  @r.get("/api/jobs/{jid}/steps/{n}")
218
+ def job_poll(jid: str, n: int, req: Request, bucket: str = ""):
219
+ s = need(req); j = job_of(s, jid, bucket)
220
  if not 0 < n <= len(j["steps"]): raise HTTPException(404, "step")
221
  return {"step": j["steps"][n - 1], "job": pub(j)}
222
 
223
  @r.post("/api/jobs/{jid}/finish")
224
  async def job_finish(jid: str, req: Request):
225
+ b = await req.json(); s = need(req); j = job_of(s, jid, b.get("bucket"))
226
  j["state"] = b.get("state") if b.get("state") in ("done", "error", "rejected") else "done"; j["summary"] = str(b.get("summary", ""))[:1000]; j["finished"] = int(time.time())
227
+ return save(s, j)
228
  return r
workspace.js CHANGED
@@ -1,6 +1,6 @@
1
  // STUDIUS empire backbone (frontend): HF sign-in, Workspace (user's HF Bucket), asset bus hand-off, bee agent dock.
2
- import { MODULES } from './real.js?v=70';
3
- import { Avatars } from './avatar.js?v=70';
4
  const $ = (s, r = document) => r.querySelector(s);
5
  const esc = (t) => String(t ?? '').replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
6
  const ST = () => window.__studius.STUDIOS;
@@ -153,7 +153,7 @@ async function renderAgents(reload) { if (reload) { await loadCrew(); await load
153
  const blog = () => $('#bees .blog');
154
  function say(who, html) { const p = document.createElement('div'); p.className = 'bm ' + who; p.innerHTML = (who === 'bee' ? '<i class="bee">🐝</i>' : '') + `<div>${html}</div>`; blog().append(p); blog().scrollTop = 1e6; return p.lastChild; }
155
  const toolsOf = (a) => a.modules.flatMap(m => Object.entries(MODULES[m]?.actions || {}).map(([k, v]) => ({ name: m + '.' + k, desc: v.desc, params: v.params, run: v.run, external: !!v.external })));
156
- const jpost = (u, b) => api(u, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(b || {}) });
157
  const show = (res) => res.kind === 'image' ? `<img alt="" src="${esc(res.url)}">` : res.kind === 'audio' ? `<audio controls src="${esc(res.url)}"></audio>` : `<p>${esc((res.text || '').slice(0, 600))}</p>`;
158
  function approval(job, st, box) { return new Promise((ok) => {
159
  box.innerHTML = `<b>Freigabe nötig</b> · <code>${esc(st.tool)}</code><br><small>${esc(JSON.stringify(st.args).slice(0, 200))}</small><div class="row"><button class="btn ap">Freigeben</button><button class="btn alt rj">Ablehnen</button></div>`;
@@ -213,6 +213,6 @@ async function runAgent() { const inp = $('#bin'), task = inp.value.trim(); if (
213
  window.__agent = { ...window.__agent, state: 'done', steps: shared.history.map(h => h.tool), kinds: shared.history.map(h => h.result?.kind || (h.result?.rejected ? 'rejected' : h.result?.error ? 'error' : '')), url: shared.assets.at(-1)?.url, text: shared.history.at(-1)?.result?.text };
214
  } catch (e) { say('bee', '⚠ ' + esc(e.message)); window.__agent = { ...window.__agent, state: 'error', err: e.message }; jpost(`/api/jobs/${job.id}/finish`, { state: 'error', summary: e.message }).catch(() => {}); }
215
  renderJobs(); }
216
- async function renderJobs() { const el = $('#bees .jobs'); if (!el || !W.me?.signedIn) return; try { const { jobs } = await api('/api/jobs'); window.__jobs = jobs;
217
- el.innerHTML = jobs.slice(0, 8).map(j => `<div class="job"><b>${{ done: '✔', running: '⏳', awaiting_approval: '✋', rejected: '⛔', error: '⚠' }[j.state] || '·'}</b> ${esc(j.task.slice(0, 50))} <small>${j.steps.map(s => esc(s.tool) + (s.external ? '🔒' : '') + ':' + s.state).join(' → ')}</small></div>`).join('') || '<small class="mut">Noch keine Jobs.</small>'; } catch (e) {} }
218
  $('#bsend').onclick = runAgent; $('#bin').addEventListener('keydown', e => { if (e.key === 'Enter') runAgent(); });
 
1
  // STUDIUS empire backbone (frontend): HF sign-in, Workspace (user's HF Bucket), asset bus hand-off, bee agent dock.
2
+ import { MODULES } from './real.js?v=71';
3
+ import { Avatars } from './avatar.js?v=71';
4
  const $ = (s, r = document) => r.querySelector(s);
5
  const esc = (t) => String(t ?? '').replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
6
  const ST = () => window.__studius.STUDIOS;
 
153
  const blog = () => $('#bees .blog');
154
  function say(who, html) { const p = document.createElement('div'); p.className = 'bm ' + who; p.innerHTML = (who === 'bee' ? '<i class="bee">🐝</i>' : '') + `<div>${html}</div>`; blog().append(p); blog().scrollTop = 1e6; return p.lastChild; }
155
  const toolsOf = (a) => a.modules.flatMap(m => Object.entries(MODULES[m]?.actions || {}).map(([k, v]) => ({ name: m + '.' + k, desc: v.desc, params: v.params, run: v.run, external: !!v.external })));
156
+ const jpost = (u, b) => { const bk = W.bucket ? (u.includes('?') ? '&' : '?') + 'bucket=' + encodeURIComponent(W.bucket) : ''; return api(u + (u.startsWith('/api/jobs') ? bk : ''), { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ ...(b || {}), ...(u.startsWith('/api/jobs') && W.bucket ? { bucket: W.bucket } : {}) }) }); };
157
  const show = (res) => res.kind === 'image' ? `<img alt="" src="${esc(res.url)}">` : res.kind === 'audio' ? `<audio controls src="${esc(res.url)}"></audio>` : `<p>${esc((res.text || '').slice(0, 600))}</p>`;
158
  function approval(job, st, box) { return new Promise((ok) => {
159
  box.innerHTML = `<b>Freigabe nötig</b> · <code>${esc(st.tool)}</code><br><small>${esc(JSON.stringify(st.args).slice(0, 200))}</small><div class="row"><button class="btn ap">Freigeben</button><button class="btn alt rj">Ablehnen</button></div>`;
 
213
  window.__agent = { ...window.__agent, state: 'done', steps: shared.history.map(h => h.tool), kinds: shared.history.map(h => h.result?.kind || (h.result?.rejected ? 'rejected' : h.result?.error ? 'error' : '')), url: shared.assets.at(-1)?.url, text: shared.history.at(-1)?.result?.text };
214
  } catch (e) { say('bee', '⚠ ' + esc(e.message)); window.__agent = { ...window.__agent, state: 'error', err: e.message }; jpost(`/api/jobs/${job.id}/finish`, { state: 'error', summary: e.message }).catch(() => {}); }
215
  renderJobs(); }
216
+ async function renderJobs() { const el = $('#bees .jobs'); if (!el || !W.me?.signedIn) return; try { const { jobs } = await api('/api/jobs' + (W.bucket ? '?bucket=' + encodeURIComponent(W.bucket) : '')); window.__jobs = jobs;
217
+ el.innerHTML = jobs.slice(0, 8).map(j => `<div class="job"><b>${{ done: '✔', running: '⏳', awaiting_approval: '✋', rejected: '⛔', error: '⚠', interrupted: '⏸' }[j.state] || '·'}</b> ${esc(j.task.slice(0, 50))} <small class="mut">${new Date(j.created * 1000).toLocaleString('de-DE', { dateStyle: 'short', timeStyle: 'short' })}${j.saved ? ' · 🪣' : ''}${j.restored ? ' · aus Bucket' : ''}</small> <small>${j.steps.map(s => esc(s.tool) + (s.external ? '🔒' : '') + ':' + s.state).join(' → ')}</small></div>`).join('') || '<small class="mut">Noch keine Jobs.</small>'; } catch (e) {} }
218
  $('#bsend').onclick = runAgent; $('#bin').addEventListener('keydown', e => { if (e.key === 'Enter') runAgent(); });