2Btainment commited on
Commit
1c96c02
·
verified ·
1 Parent(s): e73dcb3

P3.2: stronger validated planner

Browse files
Files changed (6) hide show
  1. app.js +5 -5
  2. avatar.js +1 -1
  3. index.html +2 -2
  4. master.js +1 -1
  5. server/agents.py +82 -44
  6. workspace.js +4 -4
app.js CHANGED
@@ -1,10 +1,10 @@
1
  import * as THREE from './vendor/three.module.min.js';
2
  import { STUDIOS, stopLoops } from './widgets.js?v=50';
3
- import './real.js?v=71';
4
- import './avatar.js?v=71';
5
- import './master.js?v=71';
6
- import './workspace.js?v=71';
7
- import { icon } from './icons.js?v=71';
8
  import { RoomEnvironment } from './vendor/addons/environments/RoomEnvironment.js';
9
  import { EffectComposer } from './vendor/addons/postprocessing/EffectComposer.js';
10
  import { RenderPass } from './vendor/addons/postprocessing/RenderPass.js';
 
1
  import * as THREE from './vendor/three.module.min.js';
2
  import { STUDIOS, stopLoops } from './widgets.js?v=50';
3
+ import './real.js?v=72';
4
+ import './avatar.js?v=72';
5
+ import './master.js?v=72';
6
+ import './workspace.js?v=72';
7
+ import { icon } from './icons.js?v=72';
8
  import { RoomEnvironment } from './vendor/addons/environments/RoomEnvironment.js';
9
  import { EffectComposer } from './vendor/addons/postprocessing/EffectComposer.js';
10
  import { RenderPass } from './vendor/addons/postprocessing/RenderPass.js';
avatar.js CHANGED
@@ -1,7 +1,7 @@
1
  // Avatar / Persona studio (DoD): persistent avatars as workspace asset type `avatar`
2
  // avatars/<slug>/avatar.json + ref.webp + voice.wav · reused by Chat, Image (Kontext), agents.
3
  import { STUDIOS, onStop } from './widgets.js?v=50';
4
- import { run, result, MODULES, esc, hf, stTxt, url, badge, setS, keep, LS, FLUX, KONTEXT, TTS, CHAT, signedIn } from './real.js?v=71';
5
 
6
  const LANGS = ['German', 'English', 'French', 'Spanish', 'Auto'];
7
  const slug = (n) => (n || 'avatar').toLowerCase().normalize('NFKD').replace(/[^\w]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'avatar';
 
1
  // Avatar / Persona studio (DoD): persistent avatars as workspace asset type `avatar`
2
  // avatars/<slug>/avatar.json + ref.webp + voice.wav · reused by Chat, Image (Kontext), agents.
3
  import { STUDIOS, onStop } from './widgets.js?v=50';
4
+ import { run, result, MODULES, esc, hf, stTxt, url, badge, setS, keep, LS, FLUX, KONTEXT, TTS, CHAT, signedIn } from './real.js?v=72';
5
 
6
  const LANGS = ['German', 'English', 'French', 'Spanish', 'Auto'];
7
  const slug = (n) => (n || 'avatar').toLowerCase().normalize('NFKD').replace(/[^\w]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'avatar';
index.html CHANGED
@@ -8,7 +8,7 @@
8
  <title>STUDIUS · DAS Studio — Einswarm</title>
9
  <link rel="icon" href="assets/einswarm-mascot.webp">
10
  <link rel="preload" href="fonts/outfit-latin-wght.woff2" as="font" type="font/woff2" crossorigin>
11
- <link rel="stylesheet" href="style.css?v=71">
12
  </head>
13
  <body>
14
  <canvas id="gl"></canvas>
@@ -49,6 +49,6 @@
49
 
50
  <a id="bmc" href="https://www.buymeacoffee.com/2btainment" target="_blank" rel="noopener noreferrer">Buy me a coffee</a>
51
  <noscript><p style="color:#f2c94c;padding:2em">STUDIUS braucht JavaScript.</p></noscript>
52
- <script type="module" src="app.js?v=71"></script>
53
  </body>
54
  </html>
 
8
  <title>STUDIUS · DAS Studio — Einswarm</title>
9
  <link rel="icon" href="assets/einswarm-mascot.webp">
10
  <link rel="preload" href="fonts/outfit-latin-wght.woff2" as="font" type="font/woff2" crossorigin>
11
+ <link rel="stylesheet" href="style.css?v=72">
12
  </head>
13
  <body>
14
  <canvas id="gl"></canvas>
 
49
 
50
  <a id="bmc" href="https://www.buymeacoffee.com/2btainment" target="_blank" rel="noopener noreferrer">Buy me a coffee</a>
51
  <noscript><p style="color:#f2c94c;padding:2em">STUDIUS braucht JavaScript.</p></noscript>
52
+ <script type="module" src="app.js?v=72"></script>
53
  </body>
54
  </html>
master.js CHANGED
@@ -1,7 +1,7 @@
1
  // Mastering studio (DoD): client-side WebAudio chain EQ → compressor → limiter, loudness normalisation to target LUFS
2
  // (ITU-R BS.1770-4 K-weighting + gating, measured offline), live meter, A/B, WAV/MP3 export, save to workspace.
3
  import { STUDIOS, onStop, loop, cv } from './widgets.js?v=50';
4
- import { result, MODULES, esc, setS, keep } from './real.js?v=71';
5
 
6
  /* ---------- BS.1770 loudness ---------- */
7
  function biquad(x, b0, b1, b2, a0, a1, a2) { const y = new Float32Array(x.length); b0 /= a0; b1 /= a0; b2 /= a0; a1 /= a0; a2 /= a0; let x1 = 0, x2 = 0, y1 = 0, y2 = 0;
 
1
  // Mastering studio (DoD): client-side WebAudio chain EQ → compressor → limiter, loudness normalisation to target LUFS
2
  // (ITU-R BS.1770-4 K-weighting + gating, measured offline), live meter, A/B, WAV/MP3 export, save to workspace.
3
  import { STUDIOS, onStop, loop, cv } from './widgets.js?v=50';
4
+ import { result, MODULES, esc, setS, keep } from './real.js?v=72';
5
 
6
  /* ---------- BS.1770 loudness ---------- */
7
  function biquad(x, b0, b1, b2, a0, a1, a2) { const y = new Float32Array(x.length); b0 /= a0; b1 /= a0; b2 /= a0; a1 /= a0; a2 /= a0; let x1 = 0, x2 = 0, y1 = 0, y2 = 0;
server/agents.py CHANGED
@@ -9,6 +9,7 @@ from huggingface_hub import HfApi, InferenceClient
9
 
10
  PROVIDER_MODELS = ["meta-llama/Llama-3.3-70B-Instruct", "Qwen/Qwen2.5-72B-Instruct"]
11
  LLAMA = "huggingface-projects/llama-3.2-3B-Instruct"
 
12
  EXTERNAL = {"workspace.save", "avatar.save"} # actions that persist/publish → need approval
13
  JOBS: dict[str, dict] = {}
14
  NOPROV: dict[str, float] = {} # token-hash -> until (provider returned 402: skip for a while) # job id -> job (scoped by user)
@@ -101,62 +102,99 @@ def make_agents_router(session, need, bid, client_for):
101
  except Exception as e: raise HTTPException(502, str(e)[:200])
102
  return {"saved": obj}
103
 
104
- # ---------- LLM function-calling ----------
105
- def llama(token, prompt):
106
- from gradio_client import Client # fresh session per step: the Space keeps chat history per session
107
- c = Client(LLAMA, hf_token=token, verbose=False)
108
- out = c.predict(prompt, 400, 0.2, 0.9, 50, 1.1, api_name="/generate")
 
109
  return out if isinstance(out, str) else json.dumps(out)
110
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
111
  @r.post("/api/agent/step")
112
  async def step(req: Request):
113
- """One planning step. Body: {task, tools:[{name,desc,params}], history:[{tool,args,result}], system}.
114
- Returns {call:{tool,args}} or {final:text}, plus via."""
115
  s = session(req)
116
  if not s: raise HTTPException(401, "not signed in")
117
- b = await req.json(); tools = b.get("tools") or []; names = {t["name"] for t in tools}
118
  hist = b.get("history") or []; system = str(b.get("system") or "")[:2000]; task = str(b.get("task") or "")[:2000]
 
119
  if not tools: raise HTTPException(400, "no tools")
120
- sysmsg = (system + "\n\n" if system else "") + "Du bist ein STUDIUS-Mitarbeiter. Erledige den Auftrag mit den Werkzeugen, Schritt für Schritt. Nutze Ergebnisse vorheriger Schritte (z. B. deren url) als Argumente. Wenn alles erledigt ist, antworte kurz ohne Werkzeug."
 
 
 
 
 
 
 
 
 
 
121
  msgs = [{"role": "system", "content": sysmsg}, {"role": "user", "content": task}]
122
  for i, h in enumerate(hist):
123
- cid = f"c{i}"
124
- msgs.append({"role": "assistant", "content": "", "tool_calls": [{"id": cid, "type": "function", "function": {"name": h["tool"].replace(".", "__"), "arguments": json.dumps(h.get("args", {}))}}]})
125
- msgs.append({"role": "tool", "tool_call_id": cid, "content": json.dumps(h.get("result", {}))[:1500]})
126
- errs = []; th = str(hash(s["token"]))
127
- for m in ([] if NOPROV.get(th, 0) > time.time() else PROVIDER_MODELS): # native function-calling (needs Inference Providers credits/PRO)
128
- try:
129
- ic = InferenceClient(token=s["token"], timeout=60)
130
- out = ic.chat_completion(model=m, messages=msgs, tools=to_schema(tools), tool_choice="auto", max_tokens=400)
131
- msg = out.choices[0].message
132
- if msg.tool_calls:
133
- tc = msg.tool_calls[0].function; a = tc.arguments
134
- a = json.loads(a) if isinstance(a, str) else (a or {})
135
- name = tc.name.replace("__", ".")
136
- if name in names: return {"call": {"tool": name, "args": a}, "via": f"tools:{m}"}
137
- return {"final": msg.content or "", "via": f"tools:{m}"}
138
- except Exception as e:
139
- errs.append(f"{m}: {str(e)[:80]}")
140
- if "402" in str(e): NOPROV[th] = time.time() + 600; break
141
- # fallback: JSON function-calling via Llama Space (user's ZeroGPU quota)
 
142
  tl = "\n".join(f'- {t["name"]}({", ".join(f"{k}: {v}" for k, v in (t.get("params") or {}).items())}): {t.get("desc", "")}' for t in tools)
143
  done = "\n".join(f'{i+1}. {h["tool"]}({json.dumps(h.get("args", {}), ensure_ascii=False)}) -> {json.dumps(h.get("result", {}), ensure_ascii=False)[:300]}' for i, h in enumerate(hist)) or "(noch nichts)"
144
- prompt = (f"{sysmsg}\n\nWerkzeuge:\n{tl}\n\nAuftrag: {task}\n\nBisher ausgeführt:\n{done}\n\n"
145
- 'Antworte NUR mit einem JSON-Objekt in einer Zeile. Nächster Werkzeugaufruf: {"tool":"<name>","args":{...}} '
146
- 'oder, wenn der Auftrag vollständig erledigt ist: {"final":"<kurze Zusammenfassung>"}. Rufe kein Werkzeug doppelt mit gleichen Argumenten auf.')
147
- txt = None
148
- for k in range(2):
149
- try: txt = llama(s["token"], prompt); break
150
- except Exception as e: err = e
151
- if txt is None: raise HTTPException(502, "LLM nicht erreichbar: " + str(err)[:150])
152
- j = parse_json(txt) or {}
153
- for _ in range(2):
154
- if not (b.get("require_tool") and j.get("tool") not in names): break
155
- try: j = parse_json(llama(s["token"], prompt + "\nWICHTIG: Du hast noch KEIN Werkzeug benutzt. Antworte NUR mit {\"tool\":\"<eines der Werkzeuge>\",\"args\":{...}} – kein final.")) or j
156
- except Exception: pass
157
- if j.get("tool") in names: return {"call": {"tool": j["tool"], "args": j.get("args") or {}}, "via": "json:llama-3.2-3B", "provider_errors": errs}
158
- if "final" in j: return {"final": str(j["final"]), "via": "json:llama-3.2-3B", "provider_errors": errs}
159
- return {"final": None, "raw": txt[:300], "via": "json:llama-3.2-3B", "provider_errors": errs}
 
 
 
160
 
161
  # ---------- server-side job list with approval ----------
162
  def restore(s, bucket):
 
9
 
10
  PROVIDER_MODELS = ["meta-llama/Llama-3.3-70B-Instruct", "Qwen/Qwen2.5-72B-Instruct"]
11
  LLAMA = "huggingface-projects/llama-3.2-3B-Instruct"
12
+ GEMMA = "huggingface-projects/gemma-3-12b-it"
13
  EXTERNAL = {"workspace.save", "avatar.save"} # actions that persist/publish → need approval
14
  JOBS: dict[str, dict] = {}
15
  NOPROV: dict[str, float] = {} # token-hash -> until (provider returned 402: skip for a while) # job id -> job (scoped by user)
 
102
  except Exception as e: raise HTTPException(502, str(e)[:200])
103
  return {"saved": obj}
104
 
105
+ # ---------- LLM function-calling: best available model, validated plans, retries ----------
106
+ def gr_llm(space, token, prompt, system):
107
+ from gradio_client import Client # fresh session per call: chat Spaces keep history per session
108
+ c = Client(space, hf_token=token, verbose=False)
109
+ if space == GEMMA: out = c.predict({"text": prompt, "files": []}, system, 500, api_name="/run")
110
+ else: out = c.predict((system + "\n\n" if system else "") + prompt, 500, 0.2, 0.9, 50, 1.1, api_name="/generate")
111
  return out if isinstance(out, str) else json.dumps(out)
112
 
113
+ def validate(call, tools, hist, known):
114
+ """Return (call, problem). Coerces types, repairs unknown asset urls to the latest known one."""
115
+ t = next((x for x in tools if x["name"] == call.get("tool")), None)
116
+ if not t: return None, f"Werkzeug '{call.get('tool')}' existiert nicht. Erlaubt: {', '.join(x['name'] for x in tools)}"
117
+ args = call.get("args") if isinstance(call.get("args"), dict) else {}
118
+ params = t.get("params") or {}; fixed = []
119
+ args = {k: v for k, v in args.items() if k in params}
120
+ for k, typ in params.items():
121
+ v = args.get(k)
122
+ if typ == "number":
123
+ try: args[k] = float(v) if v not in (None, "") else None
124
+ except Exception: args[k] = None
125
+ if args[k] is None: args.pop(k)
126
+ elif k in ("url", "source"):
127
+ if v not in known and known: args[k] = known[-1]; fixed.append(k)
128
+ elif not known and not v: return None, f"'{k}' braucht die url eines vorherigen Ergebnisses – zuerst ein Werkzeug aufrufen, das eines erzeugt"
129
+ elif k == "prefix": args.setdefault(k, "")
130
+ elif not str(v or "").strip(): return None, f"Parameter '{k}' fehlt für {t['name']}"
131
+ if any(h.get("tool") == t["name"] and h.get("args") == args for h in hist): return None, f"{t['name']} wurde mit diesen Argumenten schon ausgeführt – nächster Schritt oder final"
132
+ return {"tool": t["name"], "args": args, **({"repaired": fixed} if fixed else {})}, None
133
+
134
  @r.post("/api/agent/step")
135
  async def step(req: Request):
136
+ """One planning step. Body: {task, tools:[{name,desc,params}], history:[{tool,args,result}], system, known_urls, require_tool}.
137
+ Returns {call:{tool,args}} | {final}, plus via + attempts (model/validation trail)."""
138
  s = session(req)
139
  if not s: raise HTTPException(401, "not signed in")
140
+ b = await req.json(); tools = b.get("tools") or []
141
  hist = b.get("history") or []; system = str(b.get("system") or "")[:2000]; task = str(b.get("task") or "")[:2000]
142
+ known = [u for u in (b.get("known_urls") or []) if isinstance(u, str)][-10:]; need_tool = bool(b.get("require_tool"))
143
  if not tools: raise HTTPException(400, "no tools")
144
+ sysmsg = (system + "\n\n" if system else "") + "Du bist ein STUDIUS-Mitarbeiter. Erledige den Auftrag mit den Werkzeugen, Schritt für Schritt. Nutze Ergebnisse vorheriger Schritte (deren url) als Argumente. Wenn alles erledigt ist, antworte kurz ohne Werkzeug."
145
+ attempts = []
146
+ def accept(call, via):
147
+ c, prob = validate(call, tools, hist, known)
148
+ attempts.append({"via": via, "tool": call.get("tool"), "ok": not prob, **({"problem": prob} if prob else {})})
149
+ return c, prob
150
+ def final_ok(via):
151
+ ok = not need_tool
152
+ attempts.append({"via": via, "final": True, "ok": ok, **({} if ok else {"problem": "Werkzeug nötig"})}); return ok
153
+ # 1) Inference Providers with native tools (needs credits/PRO on the user's account)
154
+ th = str(hash(s["token"]))
155
  msgs = [{"role": "system", "content": sysmsg}, {"role": "user", "content": task}]
156
  for i, h in enumerate(hist):
157
+ msgs.append({"role": "assistant", "content": "", "tool_calls": [{"id": f"c{i}", "type": "function", "function": {"name": h["tool"].replace(".", "__"), "arguments": json.dumps(h.get("args", {}))}}]})
158
+ msgs.append({"role": "tool", "tool_call_id": f"c{i}", "content": json.dumps(h.get("result", {}))[:1500]})
159
+ for m in ([] if NOPROV.get(th, 0) > time.time() else PROVIDER_MODELS):
160
+ for k in range(2):
161
+ try:
162
+ out = InferenceClient(token=s["token"], timeout=60).chat_completion(model=m, messages=msgs, tools=to_schema(tools), tool_choice="required" if need_tool else "auto", max_tokens=400)
163
+ msg = out.choices[0].message
164
+ if msg.tool_calls:
165
+ tc = msg.tool_calls[0].function; a = tc.arguments; a = json.loads(a) if isinstance(a, str) else (a or {})
166
+ c, prob = accept({"tool": tc.name.replace("__", "."), "args": a}, f"tools:{m}")
167
+ if c: return {"call": c, "via": f"tools:{m}", "attempts": attempts}
168
+ msgs.append({"role": "user", "content": "Ungültiger Aufruf: " + prob})
169
+ elif final_ok(f"tools:{m}"): return {"final": msg.content or "", "via": f"tools:{m}", "attempts": attempts}
170
+ else: msgs.append({"role": "user", "content": "Rufe jetzt genau ein Werkzeug auf."})
171
+ except Exception as e:
172
+ attempts.append({"via": f"tools:{m}", "ok": False, "problem": str(e)[:90]})
173
+ if "402" in str(e): NOPROV[th] = time.time() + 600
174
+ break
175
+ if NOPROV.get(th, 0) > time.time(): break
176
+ # 2) JSON function-calling on ZeroGPU LLM Spaces (user's quota): strongest first
177
  tl = "\n".join(f'- {t["name"]}({", ".join(f"{k}: {v}" for k, v in (t.get("params") or {}).items())}): {t.get("desc", "")}' for t in tools)
178
  done = "\n".join(f'{i+1}. {h["tool"]}({json.dumps(h.get("args", {}), ensure_ascii=False)}) -> {json.dumps(h.get("result", {}), ensure_ascii=False)[:300]}' for i, h in enumerate(hist)) or "(noch nichts)"
179
+ base = (f"Werkzeuge:\n{tl}\n\nAuftrag: {task}\n\nBisher ausgeführt:\n{done}\n\nVerfügbare Ergebnis-URLs: {', '.join(known) or '(keine)'}\n\n"
180
+ 'Antworte NUR mit einem JSON-Objekt in einer Zeile. Nächster Werkzeugaufruf: {"tool":"<name>","args":{...}} '
181
+ + ('(ein Werkzeugaufruf ist jetzt Pflicht).' if need_tool else 'oder, wenn der Auftrag vollständig erledigt ist: {"final":"<kurze Zusammenfassung>"}.'))
182
+ for space, label in ((GEMMA, "json:gemma-3-12b"), (LLAMA, "json:llama-3.2-3B")):
183
+ prompt = base
184
+ for k in range(2):
185
+ try: txt = gr_llm(space, s["token"], prompt, sysmsg)
186
+ except Exception as e:
187
+ attempts.append({"via": label, "ok": False, "problem": str(e)[:110]}); break
188
+ j = parse_json(txt) or {}
189
+ if j.get("tool"):
190
+ c, prob = accept(j, label)
191
+ if c: return {"call": c, "via": label, "attempts": attempts}
192
+ elif "final" in j and final_ok(label): return {"final": str(j["final"]), "via": label, "attempts": attempts}
193
+ else:
194
+ prob = "Werkzeug nötig" if "final" in j else "keine gültige JSON-Antwort"
195
+ if "final" not in j: attempts.append({"via": label, "ok": False, "problem": prob, "raw": txt[:120]})
196
+ prompt = base + f"\n\nDein letzter Vorschlag war ungültig: {prob}. Korrigiere ihn."
197
+ raise HTTPException(502, "Kein Planer-Modell lieferte einen gültigen Plan: " + "; ".join(f"{a['via']}: {a.get('problem','')}" for a in attempts)[-400:])
198
 
199
  # ---------- server-side job list with approval ----------
200
  def restore(s, bucket):
workspace.js CHANGED
@@ -1,6 +1,6 @@
1
  // STUDIUS empire backbone (frontend): HF sign-in, Workspace (user's HF Bucket), asset bus hand-off, bee agent dock.
2
- import { MODULES } from './real.js?v=71';
3
- import { Avatars } from './avatar.js?v=71';
4
  const $ = (s, r = document) => r.querySelector(s);
5
  const esc = (t) => String(t ?? '').replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
6
  const ST = () => window.__studius.STUDIOS;
@@ -166,8 +166,8 @@ async function work(job, a, task, shared, maxSteps) { // one agent's function-ca
166
  let final = null; const startLen = shared.history.length; const mustTool = !(a.modules.length === 1 && a.modules[0] === 'chat');
167
  for (let i = 0; i < maxSteps; i++) {
168
  const box = say('bee', `<b>${esc(a.name)}</b> <span class="buzz"><i>🐝</i><i>🐝</i><i>🐝</i></span> plant…`);
169
- let p = await jpost('/api/agent/step', { task, system, tools: tools.map(({ name, desc, params }) => ({ name, desc, params })), history: shared.history, require_tool: mustTool && shared.history.length === startLen });
170
- window.__agent = { ...window.__agent, via: p.via, state: 'plan' };
171
  if (!p.call) {
172
  // if this agent hasn't used a tool yet and has an external save available with a pending asset, force it
173
  if (shared.history.length === startLen && tools.some(t => t.name === 'workspace.save') && shared.assets.length) {
 
1
  // STUDIUS empire backbone (frontend): HF sign-in, Workspace (user's HF Bucket), asset bus hand-off, bee agent dock.
2
+ import { MODULES } from './real.js?v=72';
3
+ import { Avatars } from './avatar.js?v=72';
4
  const $ = (s, r = document) => r.querySelector(s);
5
  const esc = (t) => String(t ?? '').replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
6
  const ST = () => window.__studius.STUDIOS;
 
166
  let final = null; const startLen = shared.history.length; const mustTool = !(a.modules.length === 1 && a.modules[0] === 'chat');
167
  for (let i = 0; i < maxSteps; i++) {
168
  const box = say('bee', `<b>${esc(a.name)}</b> <span class="buzz"><i>🐝</i><i>🐝</i><i>🐝</i></span> plant…`);
169
+ let p = await jpost('/api/agent/step', { task, system, tools: tools.map(({ name, desc, params }) => ({ name, desc, params })), history: shared.history, known_urls: shared.assets.filter(x => x.url).map(x => x.url), require_tool: mustTool && shared.history.length === startLen });
170
+ window.__agent = { ...window.__agent, via: p.via, state: 'plan', attempts: [...(window.__agent?.attempts || []), ...(p.attempts || [])] };
171
  if (!p.call) {
172
  // if this agent hasn't used a tool yet and has an external save available with a pending asset, force it
173
  if (shared.history.length === startLen && tools.some(t => t.name === 'workspace.save') && shared.assets.length) {