yue2-studio / src /components /render-panel.tsx
RalphThings's picture
Upload folder using huggingface_hub
88a2108 verified
Raw History Blame Contribute Delete
14.7 kB
"use client";
import { useState } from "react";
import { CaretRight, DiceFive, Waveform as WaveIcon } from "@phosphor-icons/react";
import { Button, Field, NumberInput, Panel, SegmentedControl, Slider, Switch } from "@/components/ui";
import {
INSTRUMENTAL_LORAS,
REAL_AUDIO_BASE_CHECKPOINT,
REAL_AUDIO_LORAS,
instrumentalRequirementsMet,
} from "@/lib/comfy/graph";
import { formatClock } from "@/lib/presets";
import type {
AdvancedParams,
ComfyCapabilities,
OutputFormat,
PlanMode,
ScoreSource,
} from "@/lib/types";
export interface RenderSettings {
scoreSource: ScoreSource;
mode: PlanMode;
maxDuration: number;
seed: number;
randomizeSeed: boolean;
tiledDecode: boolean;
realAudioMatch: boolean;
realAudioStrength: number;
instrumentalArrangement: boolean;
instrumentalStrength: number;
outputFormat: OutputFormat;
checkpoint: string;
audioEncoder: string;
advanced: AdvancedParams;
}
export function RenderPanel({
settings,
onChange,
capabilities,
hasReference,
hasScore,
onOpenScore,
busy,
error,
onRender,
}: {
settings: RenderSettings;
onChange: (next: RenderSettings) => void;
capabilities: ComfyCapabilities | null;
hasReference: boolean;
hasScore: boolean;
onOpenScore: () => void;
busy: boolean;
error: string | null;
onRender: () => void;
}) {
const [showAdvanced, setShowAdvanced] = useState(false);
const installed = capabilities?.loras ?? [];
const loraReady = REAL_AUDIO_LORAS.every((file) => installed.includes(file));
// The adapter was built against the bf16 layout. On the int8 checkpoint the
// deltas land on rotated, quantised weights and the result is unpredictable.
const wrongBase =
Boolean(settings.checkpoint) && settings.checkpoint !== REAL_AUDIO_BASE_CHECKPOINT;
const instReady = INSTRUMENTAL_LORAS.every((file) => installed.includes(file));
const instUsable = instrumentalRequirementsMet(settings);
const set = <K extends keyof RenderSettings>(key: K, value: RenderSettings[K]) =>
onChange({ ...settings, [key]: value });
const setAdvanced = <K extends keyof AdvancedParams>(key: K, value: AdvancedParams[K]) =>
onChange({ ...settings, advanced: { ...settings.advanced, [key]: value } });
return (
<Panel title="Render" hint={settings.mode === "melody" ? "Melody plan" : "Melody and chords"}>
<div className="flex flex-col gap-5">
<Field label="Score source" helper={SCORE_HELP[settings.scoreSource]}>
<SegmentedControl<ScoreSource>
name="Score source"
value={settings.scoreSource}
onChange={(value) => set("scoreSource", value)}
options={[
{ value: "model", label: "YuE2 writes it" },
{
value: "reference",
label: hasReference ? "From the reference" : "From a reference",
hint: hasReference ? undefined : "Upload a song first",
},
{ value: "manual", label: "Score I edited", hint: hasScore ? undefined : "Paste one in" },
{ value: "none", label: "Skip the score" },
]}
/>
{/*
The editor used to be reachable only from the dock, which needs a
finished take, so "Score I edited" could never be populated before
the first render. This is the way in.
*/}
<button
type="button"
onClick={onOpenScore}
className="mt-1 self-start text-[12px] text-ink-faint underline-offset-4 transition-colors hover:text-citron hover:underline"
>
{hasScore ? "Open the score editor" : "Write or paste a score"}
</button>
</Field>
<Field label="Plan" helper="Melody only is the recommended setting for covers.">
<SegmentedControl<PlanMode>
name="Plan mode"
value={settings.mode}
onChange={(value) => set("mode", value)}
options={[
{ value: "full", label: "Melody and chords" },
{ value: "melody", label: "Melody only" },
]}
/>
</Field>
{/*
The honest framing matters here. This adapter changes how the
renderer sounds, it does not hand the model a second channel into
the reference, because the tokenizer head that would do that has no
ComfyUI node. Overselling it would send people chasing a knob.
*/}
<Field
label="Renderer"
helper={
!loraReady
? "The Real-Audio adapter is not on this engine yet."
: !settings.realAudioMatch
? "Stock. Clean, and recognisably synthetic on sustained notes."
: wrongBase
? `Built for ${REAL_AUDIO_BASE_CHECKPOINT}. On this checkpoint the result is unpredictable.`
: "Pushes the render toward the texture of recorded audio. It does not copy your reference."
}
>
<Switch
checked={settings.realAudioMatch && loraReady}
disabled={!loraReady}
onChange={(value) => set("realAudioMatch", value)}
label="Real-audio adapter"
/>
{settings.realAudioMatch && loraReady ? (
<div className="mt-2">
<Slider
id="real-audio-strength"
label="Strength"
value={settings.realAudioStrength}
display={settings.realAudioStrength.toFixed(2)}
min={0.1}
max={1.5}
step={0.05}
onChange={(value) => set("realAudioStrength", value)}
/>
</div>
) : null}
</Field>
<Field
label="Arrangement"
helper={
!instReady
? "The instrumental adapter is not on this engine yet."
: !settings.instrumentalArrangement
? "Off. Renders follow the lyric sheet and sing it."
: !instUsable
? "Needs melody and chords, plus a score. It is sitting out this render."
: "Writes and plays an arrangement instead of a vocal. Leave the lyric sheet empty."
}
>
<Switch
checked={settings.instrumentalArrangement && instReady}
disabled={!instReady}
onChange={(value) => set("instrumentalArrangement", value)}
label="Instrumental arrangement"
/>
{settings.instrumentalArrangement && instReady ? (
<div className="mt-2 flex flex-col gap-2">
<Slider
id="instrumental-strength"
label="Strength"
value={settings.instrumentalStrength}
display={settings.instrumentalStrength.toFixed(2)}
min={0.1}
max={1.5}
step={0.05}
onChange={(value) => set("instrumentalStrength", value)}
/>
{!instUsable ? (
<button
type="button"
onClick={() =>
onChange({
...settings,
mode: "full",
scoreSource: settings.scoreSource === "none" ? "model" : settings.scoreSource,
})
}
className="self-start text-[12px] text-citron underline-offset-4 hover:underline"
>
Set the plan it needs
</button>
) : null}
</div>
) : null}
</Field>
<Field
label="Length ceiling"
helper={`Generation can stop earlier. Hard cap is ${formatClock(900)}.`}
htmlFor="max-duration"
>
<Slider
id="max-duration"
label="Up to"
display={formatClock(settings.maxDuration)}
value={settings.maxDuration}
min={30}
max={900}
step={10}
onChange={(value) => set("maxDuration", value)}
/>
</Field>
<Field label="Seed" htmlFor="seed-input">
<div className="flex items-center gap-2">
<NumberInput
id="seed-input"
value={settings.seed}
min={0}
max={4294967295}
onChange={(value) => set("seed", value)}
/>
<Button
size="sm"
variant="ghost"
onClick={() => set("seed", Math.floor(Math.random() * 4294967295))}
aria-label="Roll a new seed"
className="h-9 shrink-0 px-2.5"
>
<DiceFive size={15} weight="bold" />
</Button>
</div>
<div className="mt-1">
<Switch
checked={settings.randomizeSeed}
onChange={(value) => set("randomizeSeed", value)}
label="New seed on every render"
/>
</div>
</Field>
<div className="border-t border-line pt-4">
<button
type="button"
onClick={() => setShowAdvanced((open) => !open)}
className="flex w-full items-center gap-1.5 text-[12px] text-ink-dim transition-colors hover:text-ink"
>
<CaretRight
size={12}
weight="bold"
className={`transition-transform duration-150 ${showAdvanced ? "rotate-90" : ""}`}
/>
Sampling and weights
</button>
{showAdvanced ? (
<div className="mt-4 flex flex-col gap-4">
<Slider id="temp" label="Temperature" value={settings.advanced.temperature} min={0} max={2} step={0.05} onChange={(v) => setAdvanced("temperature", v)} />
<Slider id="topp" label="Top p" value={settings.advanced.topP} min={0.1} max={1} step={0.01} onChange={(v) => setAdvanced("topP", v)} />
<Slider id="topk" label="Top k" value={settings.advanced.topK} min={1} max={500} step={1} onChange={(v) => setAdvanced("topK", v)} />
<Slider id="rep" label="Repetition penalty" value={settings.advanced.repetitionPenalty} min={1} max={2} step={0.01} onChange={(v) => setAdvanced("repetitionPenalty", v)} />
<Slider id="cfg" label="Guidance" value={settings.advanced.cfgScale} min={1} max={1.5} step={0.01} onChange={(v) => setAdvanced("cfgScale", v)} />
<Slider id="steps" label="Flow steps" value={settings.advanced.steps} min={8} max={64} step={1} onChange={(v) => setAdvanced("steps", v)} />
<Slider id="abctemp" label="Score temperature" value={settings.advanced.abcTemperature} min={0.1} max={1.5} step={0.05} onChange={(v) => setAdvanced("abcTemperature", v)} />
<Field
label="Output"
helper={
settings.outputFormat === "flac"
? "Lossless. Some browsers will not play it back, the download always works."
: "Compressed, and playable everywhere."
}
>
<div className="flex gap-1.5">
{(["flac", "mp3", "opus"] as OutputFormat[]).map((format) => (
<button
key={format}
type="button"
onClick={() => set("outputFormat", format)}
className={`flex-1 rounded-console border py-1.5 font-mono text-[12px] transition-colors duration-150 ${
settings.outputFormat === format
? "border-citron bg-citron/10 text-citron"
: "border-line-strong text-ink-dim hover:border-ink-faint hover:text-ink"
}`}
>
{format}
</button>
))}
</div>
</Field>
<div className="border-t border-line pt-4">
<Switch
checked={settings.tiledDecode}
onChange={(value) => set("tiledDecode", value)}
label="Tiled decode"
/>
<p className="mt-1 text-[11.5px] leading-snug text-ink-faint">
Slower, but it keeps the VAE inside a 24 GB card on long songs.
</p>
</div>
{capabilities?.checkpoints.length ? (
<Field label="Checkpoint" htmlFor="ckpt">
<Picker
id="ckpt"
value={settings.checkpoint}
options={capabilities.checkpoints}
onChange={(value) => set("checkpoint", value)}
/>
</Field>
) : null}
{capabilities?.audioEncoders.length ? (
<Field label="Transcriber" htmlFor="encoder">
<Picker
id="encoder"
value={settings.audioEncoder}
options={capabilities.audioEncoders}
onChange={(value) => set("audioEncoder", value)}
/>
</Field>
) : null}
</div>
) : null}
</div>
<div className="border-t border-line pt-4">
<Button variant="primary" className="w-full" onClick={onRender} disabled={busy}>
<WaveIcon size={16} weight="bold" />
{busy ? "Sending to the GPU" : "Render take"}
</Button>
{error ? <p className="mt-2 text-[12px] leading-snug text-alarm">{error}</p> : null}
</div>
</div>
</Panel>
);
}
const SCORE_HELP: Record<ScoreSource, string> = {
model: "YuE2 drafts melody and chords first, then sings them.",
reference: "SheetSage2 transcribes the uploaded song, YuE2 performs your lyrics on that melody.",
manual: "Uses the score in the panel as written, edits included.",
none: "No symbolic plan. Fastest, least control over the tune.",
};
function Picker({
id,
value,
options,
onChange,
}: {
id: string;
value: string;
options: string[];
onChange: (value: string) => void;
}) {
return (
<select
id={id}
value={options.includes(value) ? value : options[0]}
onChange={(event) => onChange(event.target.value)}
className="h-9 w-full rounded-console border border-line-strong bg-void px-2.5 font-mono text-[12px] text-ink"
>
{options.map((option) => (
<option key={option} value={option}>
{option}
</option>
))}
</select>
);
}