2b-studio / studio /agent.py
2Btainment's picture
fix: ZeroGPU-Kontingent-Fallback (Besucher → Space), KI-Producer versteht „ohne Gesang“, Dropdown-Listen über der Raum-Leiste
518c021 verified
Raw History Blame Contribute Delete
21.6 kB
from __future__ import annotations
import json
import os
import re
import urllib.error
import urllib.request
from pathlib import Path
from typing import Any, Literal, Protocol
from huggingface_hub import InferenceClient
from pydantic import BaseModel, Field
from studio.adapters.router import route_id_for_model
from studio.registry import ModelRegistry, load_registry
PROMPT_PATH = Path(__file__).resolve().parents[1] / "prompts" / "producer.md"
MASTER_PROMPT_PATH = Path(__file__).resolve().parents[1] / "prompts" / "mastering.md"
_FALLBACK_PRODUCER = (
"Du bist der Musikproducer-Agent von 2B-Studio. Antworte ausschließlich als JSON "
"für fill_create_fields. Spiegele die Sprache des Users. Keine freien Generierungen starten."
)
_FALLBACK_MASTER = (
"Du bist der Mastering-Agent von 2B-Studio. Antworte ausschließlich als JSON "
"für fill_master_fields. Spiegele die Sprache des Users. Kein Mastering selbst starten."
)
def _read_prompt(env_key: str, path: Path, fallback: str) -> str:
env = (os.environ.get(env_key) or "").strip()
if env and not env.startswith("# STUB"):
return env
if path.is_file():
text = path.read_text(encoding="utf-8").strip()
if text and not text.startswith("# STUB"):
return text
return fallback
def gemini_api_key() -> str | None:
return os.environ.get("GEMINI_API_KEY", "").strip() or None
class FieldProposal(BaseModel):
model_id: str | None = None
genre_style_instruments: str
lyrics: str
negative: str
rationale: str
duration_s: int = Field(default=20, ge=5, le=180)
idea: str = ""
studio: dict[str, str] = Field(default_factory=dict)
renderings: dict[str, str] = Field(default_factory=dict)
preview: str = ""
ask_user: str | None = None
warnings: list[str] = Field(default_factory=list)
source: Literal["gemini", "hf-inference", "fallback"] = "fallback"
class ProducerAction(BaseModel):
type: Literal["fill_create_fields"] = "fill_create_fields"
payload: FieldProposal
preview: str
ask_user: str | None = None
class MasterProposal(BaseModel):
profile: Literal[
"streaming_safe",
"streaming_loud",
"club",
"dynamic_hifi",
"premium_clarity",
"audiophile_preserve",
] = "streaming_safe"
purity_mode: Literal["soft_keep_air", "strict", "off"] = "soft_keep_air"
keep_air_db: float = Field(default=-20.0, ge=-48, le=0)
margin: float = Field(default=2.0, ge=0.5, le=2.9)
intensity: int = Field(default=50, ge=0, le=100)
analyze_only: bool = False
rationale: str
preview: str
requires_strict_approval: bool = False
warnings: list[str] = Field(default_factory=list)
class MasterAction(BaseModel):
type: Literal["fill_master_params"] = "fill_master_params"
payload: MasterProposal
preview: str
class ProducerProvider(Protocol):
def propose(self, message: str, current_fields: dict[str, Any]) -> dict[str, Any]: ...
class HFInferenceProducerProvider:
def __init__(self) -> None:
token = os.environ.get("HF_TOKEN", "")
if not token:
raise ValueError("HF_TOKEN ist nicht gesetzt.")
self.model_id = os.environ.get(
"AGENT_MODEL_ID", "meta-llama/Llama-3.2-3B-Instruct"
)
self.client = InferenceClient(token=token)
self.system_prompt = _read_prompt("PRODUCER_SYSTEM_PROMPT", PROMPT_PATH, _FALLBACK_PRODUCER)
def propose(self, message: str, current_fields: dict[str, Any]) -> dict[str, Any]:
response = self.client.chat.completions.create(
model=self.model_id,
messages=[
{"role": "system", "content": self.system_prompt},
{
"role": "user",
"content": json.dumps(
{"idea": message, "current_fields": current_fields},
ensure_ascii=False,
),
},
],
temperature=0.2,
max_tokens=1400,
)
content = response.choices[0].message.content or "{}"
cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", content.strip())
parsed = json.loads(cleaned)
if not isinstance(parsed, dict):
raise ValueError("Der Agent lieferte kein JSON-Objekt.")
payload = parsed.get("payload", parsed)
if not isinstance(payload, dict):
raise ValueError("Der Agent lieferte kein gültiges Payload-Objekt.")
return payload
class GeminiProducerProvider:
"""Gemini via REST API (keine Zusatz-Dependency, Free-Tier-tauglich)."""
def __init__(self) -> None:
key = gemini_api_key()
if not key:
raise ValueError("GEMINI_API_KEY ist nicht gesetzt.")
self.api_key = key
self.model_id = os.environ.get("GEMINI_MODEL", "gemini-3.5-flash")
self.system_prompt = _read_prompt("PRODUCER_SYSTEM_PROMPT", PROMPT_PATH, _FALLBACK_PRODUCER)
self.base_url = "https://generativelanguage.googleapis.com/v1beta"
def propose(self, message: str, current_fields: dict[str, Any]) -> dict[str, Any]:
url = (
f"{self.base_url}/models/{self.model_id}:generateContent"
f"?key={self.api_key}"
)
body = {
"system_instruction": {"parts": [{"text": self.system_prompt}]},
"contents": [
{
"parts": [
{
"text": json.dumps(
{"idea": message, "current_fields": current_fields},
ensure_ascii=False,
)
}
]
}
],
"generationConfig": {
"temperature": 0.2,
"maxOutputTokens": 4096,
"responseMimeType": "application/json",
},
}
request = urllib.request.Request(
url,
data=json.dumps(body).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=60) as response:
payload = json.loads(response.read().decode("utf-8"))
except urllib.error.HTTPError as error:
detail = error.read().decode("utf-8", errors="replace")[:300]
raise ValueError(f"Gemini HTTP {error.code}: {detail}") from error
except urllib.error.URLError as error:
raise ValueError(f"Gemini nicht erreichbar: {error.reason}") from error
candidates = payload.get("candidates") or []
if not candidates:
block = payload.get("promptFeedback", {}).get("blockReason")
raise ValueError(f"Gemini lieferte keine Antwort (blockReason={block}).")
text = candidates[0].get("content", {}).get("parts", [{}])[0].get("text", "")
cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip())
parsed = json.loads(cleaned)
if not isinstance(parsed, dict):
raise ValueError("Gemini lieferte kein JSON-Objekt.")
payload_out = parsed.get("payload", parsed)
if not isinstance(payload_out, dict):
raise ValueError("Gemini lieferte kein gültiges Payload-Objekt.")
return payload_out
def _duration_from_message(message: str) -> int:
match = re.search(
r"\b(?P<value>\d{1,3})\s*(?P<unit>sekunden?|seconds?|secs?|s|minuten?|minutes?|min)\b",
message.casefold(),
)
if not match:
return 20
value = int(match.group("value"))
if match.group("unit").startswith(("min", "minute")):
value *= 60
return max(5, min(180, value))
def _fallback_proposal(message: str) -> FieldProposal:
normalized = message.casefold()
duration_s = _duration_from_message(message)
vocals_requested = any(
term in normalized
for term in (
"vocal",
"gesang",
"sänger",
"sängerin",
"männlich",
"weiblich",
"lyrics",
"stimme",
)
) and "instrumental" not in normalized and not any(
neg in normalized
for neg in (
"ohne gesang", "kein gesang", "keinen gesang", "ohne vocals", "keine vocals", "ohne stimme",
"ohne text", "keine lyrics", "ohne lyrics", "no vocals", "without vocals", "no lyrics",
)
)
if any(term in normalized for term in ("riddim", "reggae", "dub")):
style = (
"dark roots riddim, 140 BPM, analog sub bass, tight live drums, "
"warm organ skanks, evolving dub transitions"
)
elif any(term in normalized for term in ("techno", "club", "rave")):
style = (
"hypnotic techno, 132 BPM, rolling sub bass, crisp percussion, "
"warehouse depth, tension-building arrangement"
)
elif any(term in normalized for term in ("lofi", "chill", "entspannt")):
style = (
"lo-fi soul, 82 BPM, dusty drums, mellow Rhodes piano, warm tape texture, "
"human timing, intimate dynamics"
)
elif any(term in normalized for term in ("film", "cinematic", "episch")):
style = (
"cinematic electronica, 96 BPM, evolving strings, deep percussion, "
"wide synth layers, clear emotional arc"
)
elif any(term in normalized for term in ("rock", "gitarre", "guitar")):
style = (
"modern alternative rock, 118 BPM, expressive electric guitars, live drums, "
"driving bass, dynamic verse-to-chorus lift"
)
else:
style = (
"modern electronic soul, 108 BPM, punchy drums, warm bass, memorable hook, "
"intentional transitions, polished stereo depth"
)
if vocals_requested:
style += ", expressive lead vocal with clear diction"
lyrics = (
"[Verse]\nWe turn the quiet into light\nEvery step is drawing near\n\n"
"[Chorus]\nWe hold the line tonight\nMake the hidden signal clear"
)
model_id = "minimax-music3"
else:
lyrics = "[Instrumental]"
model_id = "2b-musicgen-medium" if duration_s > 20 else "2b-musicgen-small"
short_caption = style[:800]
return FieldProposal(
model_id=model_id,
genre_style_instruments=style,
lyrics=lyrics,
negative=(
"muddy low end, harsh sibilance, uncontrolled clipping, phase cancellation, "
"static arrangement, generic AI artifacts"
),
duration_s=duration_s,
idea=message.strip(),
studio={
"global_metadata": f"Basic Attributes: {style}",
"vocal_details": (
"Vocal Gender & Timbre: expressive lead, clear diction"
if vocals_requested
else "Vocal Direction: instrumental, no lead vocals"
),
"arrangement": "Instrument Lifecycle: focused opening, developed middle, resolved ending",
},
renderings={
"musicgen_prompt": style,
"short_caption": short_caption,
},
preview=f"{duration_s}s · {'Vocal' if vocals_requested else 'Instrumental'} · {model_id}",
rationale=(
"MiniMax passt zur strukturierten Vocal-Anforderung."
if vocals_requested
else "Der live angebundene 2B-Audio-Worker passt zum instrumentalen Entwurf."
),
source="fallback",
)
class ProducerAgent:
def __init__(
self,
registry: ModelRegistry,
provider: ProducerProvider | None = None,
) -> None:
self.registry = registry
self.provider = provider
if self.provider is None:
if gemini_api_key():
self.provider = GeminiProducerProvider()
elif os.environ.get("HF_TOKEN"):
self.provider = HFInferenceProducerProvider()
def propose(
self,
message: str,
current_fields: dict[str, Any] | None = None,
) -> ProducerAction:
warnings: list[str] = []
if self.provider is not None:
try:
raw = self.provider.propose(message, current_fields or {})
source = (
"gemini"
if isinstance(self.provider, GeminiProducerProvider)
else "hf-inference"
)
proposal = FieldProposal.model_validate({**raw, "source": source})
except Exception as error:
proposal = _fallback_proposal(message)
warnings.append(f"LLM-Fallback aktiv: {type(error).__name__}")
else:
proposal = _fallback_proposal(message)
# Deterministische Absicherung: vage Eingabe -> immer eine Rückfrage
if proposal.ask_user is None and _is_vague_input(message):
proposal.ask_user = _default_ask(message)
try:
model = self.registry.get(proposal.model_id or "")
except KeyError:
model = self.registry.get("2b-musicgen-small")
proposal.model_id = model.id
warnings.append("Unbekanntes Agentenmodell wurde auf 2B Audio Small zurückgesetzt.")
if proposal.duration_s > model.max_duration_s:
warnings.append(
f"Dauer wurde an {model.label} angepasst: maximal {model.max_duration_s} s."
)
proposal.duration_s = model.max_duration_s
if proposal.lyrics.strip().casefold() != "[instrumental]" and not model.supports_vocals:
warnings.append(f"{model.label} unterstützt Gesang nur eingeschränkt.")
if not route_id_for_model(model).endswith("-real"):
warnings.append(
f"{model.label} ist gerade nicht erreichbar; der Worker antwortet nicht."
)
proposal.warnings = [*proposal.warnings, *warnings]
if not proposal.preview:
proposal.preview = f"{proposal.duration_s}s · {model.label}"
return ProducerAction(
payload=proposal,
preview=proposal.preview,
ask_user=proposal.ask_user,
)
def _is_vague_input(message: str) -> bool:
"""True, wenn die Eingabe keine entscheidenden Produktions-Infos enthält."""
normalized = message.casefold()
genre_terms = (
"riddim", "reggae", "dub", "techno", "club", "rave", "lofi", "lo-fi", "chill",
"film", "cinematic", "episch", "rock", "gitarre", "guitar", "pop", "hiphop",
"hip-hop", "house", "drum and bass", "dnb", "ambient", "jazz", "soul", "trap",
"edm", "trance", "metal", "punk", "blues", "funk", "disco", "country", "folk",
)
mood_terms = (
"dunkel", "dark", "euphorisch", "euphoric", "entspannt", "relaxed", "traurig",
"sad", "wütend", "angry", "aggressiv", "aggressive", "melancholisch", "melancholic",
"fröhlich", "happy", "warm", "kalt", "cold", "mysteriös", "mysterious", "episch",
"cinematic", "energetisch", "energetic", "ruhig", "calm", "böse", "evil", "düster",
)
instrument_terms = (
"bass", "drums", "drum", "gitarre", "guitar", "piano", "keys", "synth", "synthesizer",
"strings", "streicher", "violine", "saxophon", "sax", "horn", "flöte", "flute",
"percussion", "schlagzeug", "kick", "snare", "hi-hat", "pad", "lead", "vocal", "gesang",
"stimme", "choir", "chor", "organ", "orgel", "brass",
)
structure_terms = (
"instrumental", "vocal", "gesang", "lyrics", "text", "sänger", "sängerin",
"sekunden", "seconds", "minuten", "minutes", "bpm", "tempo", "dauer", "länge",
"intro", "verse", "chorus", "bridge", "drop", "hook",
)
has_info = any(
term in normalized
for term in (*genre_terms, *mood_terms, *instrument_terms, *structure_terms)
)
return not has_info
def _default_ask(message: str) -> str:
"""Deterministische Rückfrage, wenn die Eingabe zu wenig Info enthält."""
normalized = message.casefold()
if "instrumental" not in normalized and not any(
term in normalized for term in ("vocal", "gesang", "stimme", "lyrics", "text")
):
return "Soll es ein Instrumental sein oder mit Gesang?"
if not any(
term in normalized
for term in (
"bpm", "tempo", "sekunden", "seconds", "minuten", "minutes", "dauer", "länge",
)
):
return "Wie lang soll der Track sein (5–180 Sekunden)?"
return "Welche Stimmung soll der Track vermitteln (z. B. dunkel, euphorisch, entspannt)?"
def propose_fields(message: str) -> FieldProposal:
"""Compatibility helper using the validated local fallback/provider chain."""
return ProducerAgent(load_registry()).propose(message).payload
def propose_master(message: str) -> MasterAction:
if gemini_api_key():
try:
return _propose_master_llm(message)
except Exception:
pass # Fallback auf deterministische Heuristik
return _propose_master_fallback(message)
def _propose_master_llm(message: str) -> MasterAction:
"""Master-Vorschlag via Gemini; nutzt prompts/mastering.md als System-Prompt."""
api_key = gemini_api_key()
assert api_key is not None
model_id = os.environ.get("GEMINI_MODEL", "gemini-3.5-flash")
system_prompt = _read_prompt("MASTERING_SYSTEM_PROMPT", MASTER_PROMPT_PATH, _FALLBACK_MASTER)
base_url = "https://generativelanguage.googleapis.com/v1beta"
url = f"{base_url}/models/{model_id}:generateContent?key={api_key}"
body = {
"system_instruction": {"parts": [{"text": system_prompt}]},
"contents": [{"parts": [{"text": json.dumps({"idea": message}, ensure_ascii=False)}]}],
"generationConfig": {
"temperature": 0.2,
"maxOutputTokens": 2048,
"responseMimeType": "application/json",
},
}
request = urllib.request.Request(
url,
data=json.dumps(body).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=60) as response:
payload = json.loads(response.read().decode("utf-8"))
except urllib.error.HTTPError as error:
detail = error.read().decode("utf-8", errors="replace")[:300]
raise ValueError(f"Gemini HTTP {error.code}: {detail}") from error
except urllib.error.URLError as error:
raise ValueError(f"Gemini nicht erreichbar: {error.reason}") from error
candidates = payload.get("candidates") or []
if not candidates:
block = payload.get("promptFeedback", {}).get("blockReason")
raise ValueError(f"Gemini lieferte keine Antwort (blockReason={block}).")
text = candidates[0].get("content", {}).get("parts", [{}])[0].get("text", "")
cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip())
parsed = json.loads(cleaned)
if not isinstance(parsed, dict):
raise ValueError("Gemini lieferte kein JSON-Objekt.")
payload_out = parsed.get("payload", parsed)
if not isinstance(payload_out, dict):
raise ValueError("Gemini lieferte kein gültiges Payload-Objekt.")
proposal = MasterProposal.model_validate(payload_out)
if proposal.purity_mode == "strict":
proposal.requires_strict_approval = True
return MasterAction(payload=proposal, preview=proposal.preview)
def _propose_master_fallback(message: str) -> MasterAction:
normalized = message.casefold()
analyze_only = any(
term in normalized for term in ("nur messen", "nur analyse", "only analyze", "analyse only")
)
if any(term in normalized for term in ("club", "dj", "dancefloor")):
profile = "club"
intensity = 72
elif any(term in normalized for term in ("lauter", "loud", "druck")):
profile = "streaming_loud"
intensity = 65
elif any(term in normalized for term in ("hifi", "dynamisch", "audiophil")):
profile = "dynamic_hifi"
intensity = 42
else:
profile = "streaming_safe"
intensity = 50
strict = "strict" in normalized
purity_off = any(
term in normalized for term in ("ohne purity", "kein purity", "nur master", "skip purity")
)
purity_mode = "strict" if strict else "off" if purity_off else "soft_keep_air"
preserve_air = any(
term in normalized for term in ("luft", "air", "wärme", "hall", "nicht tot")
)
keep_air_db = -24.0 if preserve_air else -20.0
margin = 1.5 if preserve_air else 2.0
rationale = (
"Strict wurde nur als Vorschlag markiert und benötigt eine separate Bestätigung."
if strict
else "soft_keep_air schützt Air, Hall und Wärme; das Original bleibt unverändert."
)
if analyze_only:
rationale = "Es wird ausschließlich gemessen; keine Audiodatei wird verändert oder erzeugt."
proposal = MasterProposal(
profile=profile,
purity_mode=purity_mode,
keep_air_db=keep_air_db,
margin=margin,
intensity=intensity,
analyze_only=analyze_only,
rationale=rationale,
preview=f"{profile} · {purity_mode}{' · nur Analyse' if analyze_only else ''}",
requires_strict_approval=strict,
)
return MasterAction(payload=proposal, preview=proposal.preview)