// Tests music generation runtime dispatch and provider fallback behavior. import { beforeEach, describe, expect, it } from "vitest"; import type { OpenClawConfig } from "../config/types.js"; import type { GenerateMusicParams } from "./runtime-types.js"; import { generateMusic, listRuntimeMusicGenerationProviders } from "./runtime.js"; import type { MusicGenerationProvider } from "./types.js"; type MusicGenerationRuntimeDeps = NonNullable[1]>; let providers: MusicGenerationProvider[] = []; let listedConfigs: Array = []; const runtimeDeps: MusicGenerationRuntimeDeps = { getProvider: (providerId) => providers.find((provider) => provider.id === providerId), listProviders: (config) => { listedConfigs.push(config); return providers; }, log: { debug: () => {}, }, }; function runGenerateMusic(params: GenerateMusicParams) { const defaults = params.cfg.agents?.defaults as | (NonNullable["defaults"] & { musicGenerationModel?: unknown; }) | undefined; const cfg = defaults?.musicGenerationModel !== undefined && defaults.mediaModels?.music === undefined ? { ...params.cfg, agents: { ...params.cfg.agents, defaults: { ...defaults, mediaModels: { ...defaults.mediaModels, music: defaults.musicGenerationModel }, }, }, } : params.cfg; return generateMusic({ ...params, cfg }, runtimeDeps); } function createBufferedMusicProvider(id: string, buffers: Buffer[]): MusicGenerationProvider { return { id, capabilities: {}, generateMusic: async () => ({ tracks: buffers.map((buffer) => ({ buffer, mimeType: "audio/mpeg" })), }), }; } describe("music-generation runtime", () => { beforeEach(() => { providers = []; listedConfigs = []; }); it("generates tracks through the active music-generation provider", async () => { const authStore = { version: 1, profiles: {} } as const; let seenAuthStore: unknown; let seenTimeoutMs: number | undefined; const provider: MusicGenerationProvider = { id: "music-plugin", capabilities: {}, async generateMusic(req: { authStore?: unknown; timeoutMs?: number }) { seenAuthStore = req.authStore; seenTimeoutMs = req.timeoutMs; return { tracks: [ { buffer: Buffer.from("mp3-bytes"), mimeType: "audio/mpeg", fileName: "sample.mp3", }, ], model: "track-v1", }; }, }; providers = [provider]; const result = await runGenerateMusic({ cfg: { agents: { defaults: { musicGenerationModel: { primary: "music-plugin/track-v1" }, }, }, } as OpenClawConfig, prompt: "play a synth line", agentDir: "/tmp/agent", authStore, timeoutMs: 12_345, }); expect(result.provider).toBe("music-plugin"); expect(result.model).toBe("track-v1"); expect(result.attempts).toStrictEqual([]); expect(result.ignoredOverrides).toStrictEqual([]); expect(seenAuthStore).toEqual(authStore); expect(seenTimeoutMs).toBe(12_345); expect(result.tracks).toEqual([ { buffer: Buffer.from("mp3-bytes"), mimeType: "audio/mpeg", fileName: "sample.mp3", }, ]); }); it("uses configured music-generation timeout when call omits timeoutMs", async () => { let seenTimeoutMs: number | undefined; providers = [ { id: "music-plugin", capabilities: {}, async generateMusic(req: { timeoutMs?: number }) { seenTimeoutMs = req.timeoutMs; return { tracks: [{ buffer: Buffer.from("mp3-bytes"), mimeType: "audio/mpeg" }], model: "track-v1", }; }, }, ]; await runGenerateMusic({ cfg: { agents: { defaults: { musicGenerationModel: { primary: "music-plugin/track-v1", timeoutMs: 300_000 }, }, }, } as OpenClawConfig, prompt: "play a synth line", }); expect(seenTimeoutMs).toBe(300_000); }); it("does not list providers when explicit config disables auto provider fallback", async () => { const provider: MusicGenerationProvider = { id: "music-plugin", capabilities: {}, async generateMusic() { return { tracks: [ { buffer: Buffer.from("mp3-bytes"), mimeType: "audio/mpeg", fileName: "sample.mp3", }, ], model: "track-v1", }; }, }; providers = [provider]; const params: GenerateMusicParams = { cfg: { agents: { defaults: { musicGenerationModel: { primary: "music-plugin/track-v1" }, }, }, } as OpenClawConfig, prompt: "play a synth line", autoProviderFallback: false, }; const result = await runGenerateMusic(params); expect(result.provider).toBe("music-plugin"); expect(listedConfigs).toStrictEqual([]); }); it("auto-detects and falls through to another configured music-generation provider by default", async () => { providers = [ { id: "google", defaultModel: "lyria-3-clip-preview", capabilities: {}, isConfigured: () => true, async generateMusic() { throw new Error("Google music generation response missing audio data"); }, }, { id: "minimax", defaultModel: "music-2.6", capabilities: {}, isConfigured: () => true, async generateMusic() { return { tracks: [{ buffer: Buffer.from("mp3-bytes"), mimeType: "audio/mpeg" }], model: "music-2.6", }; }, }, ]; const result = await runGenerateMusic({ cfg: {} as OpenClawConfig, prompt: "play a synth line", }); expect(result.provider).toBe("minimax"); expect(result.model).toBe("music-2.6"); expect(result.attempts).toEqual([ { provider: "google", model: "lyria-3-clip-preview", error: "Google music generation response missing audio data", }, ]); }); it("falls through when a music provider returns an empty buffer", async () => { providers = [ createBufferedMusicProvider("empty", [Buffer.from("partial"), Buffer.alloc(0)]), createBufferedMusicProvider("valid", [Buffer.from("mp3-bytes")]), ]; const result = await runGenerateMusic({ cfg: { agents: { defaults: { mediaModels: { music: { primary: "empty/track-v1", fallbacks: ["valid/track-v2"] }, }, }, }, } as OpenClawConfig, prompt: "play a synth line", }); expect(result.provider).toBe("valid"); expect(result.tracks[0]?.buffer).toEqual(Buffer.from("mp3-bytes")); expect(result.attempts).toEqual([ { provider: "empty", model: "track-v1", error: "Music generation provider returned an empty track buffer at index 1.", }, ]); }); it("fails visibly when every music provider returns an empty buffer", async () => { providers = [ createBufferedMusicProvider("empty-primary", [Buffer.alloc(0)]), createBufferedMusicProvider("empty-fallback", [Buffer.alloc(0)]), ]; await expect( runGenerateMusic({ cfg: { agents: { defaults: { mediaModels: { music: { primary: "empty-primary/track-v1", fallbacks: ["empty-fallback/track-v2"], }, }, }, }, } as OpenClawConfig, prompt: "play a synth line", }), ).rejects.toThrow( "All music generation models failed (2): empty-primary/track-v1: Music generation provider returned an empty track buffer at index 0. | empty-fallback/track-v2: Music generation provider returned an empty track buffer at index 0.", ); }); it("lists runtime music-generation providers through the provider registry", () => { const registryProviders: MusicGenerationProvider[] = [ { id: "music-plugin", defaultModel: "track-v1", models: ["track-v1"], capabilities: { generate: { supportsDuration: true, }, }, generateMusic: async () => ({ tracks: [{ buffer: Buffer.from("mp3-bytes"), mimeType: "audio/mpeg" }], }), }, ]; providers = registryProviders; expect( listRuntimeMusicGenerationProviders({ config: {} as OpenClawConfig }, runtimeDeps), ).toEqual(registryProviders); expect(listedConfigs).toEqual([{} as OpenClawConfig]); }); it("ignores unsupported optional overrides per provider and model", async () => { let seenRequest: | { lyrics?: string; instrumental?: boolean; durationSeconds?: number; format?: string; } | undefined; providers = [ { id: "google", capabilities: { generate: { supportsLyrics: true, supportsInstrumental: true, supportsFormat: true, supportedFormatsByModel: { "lyria-3-clip-preview": ["mp3"], }, }, }, generateMusic: async (req) => { seenRequest = { lyrics: req.lyrics, instrumental: req.instrumental, durationSeconds: req.durationSeconds, format: req.format, }; return { tracks: [{ buffer: Buffer.from("mp3-bytes"), mimeType: "audio/mpeg" }], model: "lyria-3-clip-preview", }; }, }, ]; const result = await runGenerateMusic({ cfg: { agents: { defaults: { musicGenerationModel: { primary: "google/lyria-3-clip-preview" }, }, }, } as OpenClawConfig, prompt: "energetic arcade anthem", lyrics: "Hero crab in the neon tide", instrumental: true, durationSeconds: 30, format: "wav", }); expect(seenRequest).toEqual({ lyrics: "Hero crab in the neon tide", instrumental: true, durationSeconds: undefined, format: undefined, }); expect(result.ignoredOverrides).toEqual([ { key: "durationSeconds", value: 30 }, { key: "format", value: "wav" }, ]); }); it("ignores model-specific unsupported lyrics and instrumental overrides", async () => { let seenRequest: | { lyrics?: string; instrumental?: boolean; } | undefined; providers = [ { id: "fal", capabilities: { generate: { supportsLyrics: true, supportsLyricsByModel: { "fal-ai/stable-audio-25/text-to-audio": false, }, supportsInstrumental: true, supportsInstrumentalByModel: { "fal-ai/stable-audio-25/text-to-audio": false, }, }, }, generateMusic: async (req) => { seenRequest = { lyrics: req.lyrics, instrumental: req.instrumental, }; return { tracks: [{ buffer: Buffer.from("wav-bytes"), mimeType: "audio/wav" }], model: "fal-ai/stable-audio-25/text-to-audio", }; }, }, ]; const result = await runGenerateMusic({ cfg: { agents: { defaults: { musicGenerationModel: { primary: "fal/fal-ai/stable-audio-25/text-to-audio" }, }, }, } as OpenClawConfig, prompt: "orchestral hit", lyrics: "rise up", instrumental: true, }); expect(seenRequest).toEqual({ lyrics: undefined, instrumental: undefined, }); expect(result.ignoredOverrides).toEqual([ { key: "lyrics", value: "rise up" }, { key: "instrumental", value: true }, ]); }); it("uses mode-specific capabilities for edit requests", async () => { let seenRequest: | { lyrics?: string; instrumental?: boolean; durationSeconds?: number; format?: string; } | undefined; providers = [ { id: "google", capabilities: { generate: { supportsLyrics: false, supportsInstrumental: false, supportsFormat: true, supportedFormats: ["mp3"], }, edit: { enabled: true, maxInputImages: 1, supportsLyrics: true, supportsInstrumental: true, supportsDuration: false, supportsFormat: false, }, }, generateMusic: async (req) => { seenRequest = { lyrics: req.lyrics, instrumental: req.instrumental, durationSeconds: req.durationSeconds, format: req.format, }; return { tracks: [{ buffer: Buffer.from("mp3-bytes"), mimeType: "audio/mpeg" }], model: "lyria-3-pro-preview", }; }, }, ]; const result = await runGenerateMusic({ cfg: { agents: { defaults: { musicGenerationModel: { primary: "google/lyria-3-pro-preview" }, }, }, } as OpenClawConfig, prompt: "turn this cover image into a trailer cue", lyrics: "rise up", instrumental: true, durationSeconds: 30, format: "mp3", inputImages: [{ buffer: Buffer.from("png"), mimeType: "image/png" }], }); expect(seenRequest).toEqual({ lyrics: "rise up", instrumental: true, durationSeconds: undefined, format: undefined, }); expect(result.ignoredOverrides).toEqual([ { key: "durationSeconds", value: 30 }, { key: "format", value: "mp3" }, ]); }); it("normalizes requested durations to the closest supported max duration", async () => { let seenRequest: | { durationSeconds?: number; } | undefined; providers = [ { id: "minimax", capabilities: { generate: { supportsDuration: true, maxDurationSeconds: 30, }, }, generateMusic: async (req) => { seenRequest = { durationSeconds: req.durationSeconds, }; return { tracks: [{ buffer: Buffer.from("mp3-bytes"), mimeType: "audio/mpeg" }], model: "music-2.6", }; }, }, ]; const result = await runGenerateMusic({ cfg: { agents: { defaults: { musicGenerationModel: { primary: "minimax/music-2.6" }, }, }, } as OpenClawConfig, prompt: "energetic arcade anthem", durationSeconds: 45, }); expect(seenRequest).toEqual({ durationSeconds: 30, }); expect(result.ignoredOverrides).toStrictEqual([]); if (!result.normalization || !result.metadata) { throw new Error("Expected normalization and metadata"); } expect(result.normalization.durationSeconds?.requested).toBe(45); expect(result.normalization.durationSeconds?.applied).toBe(30); expect(result.metadata.requestedDurationSeconds).toBe(45); expect(result.metadata.normalizedDurationSeconds).toBe(30); }); it.each([ { edit: { enabled: false }, count: 1 }, { edit: { enabled: true, maxInputImages: 1 }, count: 2 }, { edit: { enabled: true }, count: 11 }, ])("skips incompatible reference-image fallback candidates", async ({ edit, count }) => { let incompatibleProviderCalled = false; const inputImages = Array.from({ length: count }, () => ({ buffer: Buffer.from("reference"), mimeType: "image/png", })); providers = [ { id: "fal", defaultModel: "prompt-only", capabilities: { edit }, async generateMusic() { incompatibleProviderCalled = true; return { tracks: [{ buffer: Buffer.from("incorrect"), mimeType: "audio/mpeg" }] }; }, }, { id: "google", defaultModel: "lyria", capabilities: { edit: { enabled: true, maxInputImages: 14 } }, async generateMusic() { return { tracks: [{ buffer: Buffer.from("correct"), mimeType: "audio/mpeg" }] }; }, }, ]; const result = await runGenerateMusic({ cfg: { agents: { defaults: { musicGenerationModel: { primary: "fal/prompt-only", fallbacks: ["google/lyria"] }, }, }, } as OpenClawConfig, prompt: "score the cover art", inputImages, }); expect(incompatibleProviderCalled).toBe(false); expect(result.provider).toBe("google"); expect(result.attempts).toHaveLength(1); }); });