| |
| import { resolveAgentModelTimeoutMsValue } from "../config/model-input.js"; |
| import type { OpenClawConfig } from "../config/types.openclaw.js"; |
| import { createSubsystemLogger } from "../logging/subsystem.js"; |
| import { parseMusicGenerationModelRef } from "../media-generation/model-ref.js"; |
| import { |
| withMusicGenerationProviders, |
| getMusicGenerationProvider, |
| listMusicGenerationProviders, |
| } from "../media-generation/registry.js"; |
| import { |
| buildMediaGenerationNormalizationMetadata, |
| buildNoCapabilityModelConfiguredMessage, |
| resolveCapabilityModelCandidates, |
| resolveReferenceImageCapabilityError, |
| runMediaGenerationCandidates, |
| } from "../media-generation/runtime-shared.js"; |
| import { |
| buildCapabilityProviderIndex, |
| normalizeCapabilityProviderId, |
| } from "../plugins/provider-registry-shared.js"; |
| import { getProviderEnvVars } from "../secrets/provider-env-vars.js"; |
| import { resolveMusicGenerationOverrides } from "./normalization.js"; |
| import type { GenerateMusicParams, GenerateMusicRuntimeResult } from "./runtime-types.js"; |
| import type { MusicGenerationResult } from "./types.js"; |
|
|
| |
| |
| |
| |
| |
| |
| |
| const log = createSubsystemLogger("music-generation"); |
|
|
| |
| type MusicGenerationRuntimeDeps = { |
| getProvider?: typeof getMusicGenerationProvider; |
| listProviders?: typeof listMusicGenerationProviders; |
| getProviderEnvVars?: typeof getProviderEnvVars; |
| log?: Pick<typeof log, "debug">; |
| }; |
|
|
| |
| export function listRuntimeMusicGenerationProviders( |
| params?: { config?: OpenClawConfig }, |
| deps: MusicGenerationRuntimeDeps = {}, |
| ) { |
| return (deps.listProviders ?? listMusicGenerationProviders)(params?.config); |
| } |
|
|
| |
| export async function generateMusic( |
| params: GenerateMusicParams, |
| deps: MusicGenerationRuntimeDeps = {}, |
| ): Promise<GenerateMusicRuntimeResult> { |
| if (deps.getProvider && deps.listProviders) { |
| return runMusicGeneration(params, deps); |
| } |
| return withMusicGenerationProviders(params.cfg, (providers) => { |
| const canonical = buildCapabilityProviderIndex(providers, "canonical"); |
| const aliases = buildCapabilityProviderIndex(providers, "aliases"); |
| return runMusicGeneration(params, { |
| ...deps, |
| getProvider: |
| deps.getProvider ?? |
| ((id) => { |
| const normalized = normalizeCapabilityProviderId(id); |
| return normalized ? aliases.get(normalized) : undefined; |
| }), |
| listProviders: deps.listProviders ?? (() => [...canonical.values()]), |
| }); |
| }); |
| } |
|
|
| async function runMusicGeneration( |
| params: GenerateMusicParams, |
| deps: MusicGenerationRuntimeDeps, |
| ): Promise<GenerateMusicRuntimeResult> { |
| const getProvider = deps.getProvider ?? getMusicGenerationProvider; |
| const listProviders = deps.listProviders ?? listMusicGenerationProviders; |
| const logger = deps.log ?? log; |
| const timeoutMs = |
| params.timeoutMs ?? |
| resolveAgentModelTimeoutMsValue(params.cfg.agents?.defaults?.mediaModels?.music); |
| const candidates = resolveCapabilityModelCandidates({ |
| cfg: params.cfg, |
| modelConfig: params.cfg.agents?.defaults?.mediaModels?.music, |
| modelOverride: params.modelOverride, |
| parseModelRef: parseMusicGenerationModelRef, |
| agentDir: params.agentDir, |
| listProviders, |
| autoProviderFallback: params.autoProviderFallback, |
| }); |
| if (candidates.length === 0) { |
| throw new Error( |
| buildNoCapabilityModelConfiguredMessage({ |
| capabilityLabel: "music-generation", |
| modelConfigKey: "mediaModels.music", |
| providers: listProviders(params.cfg), |
| fallbackSampleRef: "google/lyria-3-clip-preview", |
| getProviderEnvVars: deps.getProviderEnvVars, |
| }), |
| ); |
| } |
|
|
| return runMediaGenerationCandidates({ |
| candidates, |
| capability: "music", |
| getProvider: (providerId) => getProvider(providerId, params.cfg), |
| includeSkipFailureDetails: true, |
| onFailure: (attempt) => { |
| logger.debug(`music-generation candidate failed: ${attempt.provider}/${attempt.model}`); |
| }, |
| prepareCandidate(candidate, provider) { |
| const referenceImageError = resolveReferenceImageCapabilityError({ |
| candidateRef: `${candidate.provider}/${candidate.model}`, |
| inputImageCount: params.inputImages?.length ?? 0, |
| edit: provider.capabilities.edit, |
| }); |
| if (referenceImageError) { |
| logger.debug(`music-generation candidate skipped: ${referenceImageError}`); |
| return referenceImageError; |
| } |
|
|
| return async (attempts): Promise<GenerateMusicRuntimeResult> => { |
| const sanitized = resolveMusicGenerationOverrides({ |
| provider, |
| model: candidate.model, |
| lyrics: params.lyrics, |
| instrumental: params.instrumental, |
| durationSeconds: params.durationSeconds, |
| format: params.format, |
| inputImages: params.inputImages, |
| }); |
| const result: MusicGenerationResult = await provider.generateMusic({ |
| provider: candidate.provider, |
| model: candidate.model, |
| prompt: params.prompt, |
| cfg: params.cfg, |
| agentDir: params.agentDir, |
| authStore: params.authStore, |
| lyrics: sanitized.lyrics, |
| instrumental: sanitized.instrumental, |
| durationSeconds: sanitized.durationSeconds, |
| format: sanitized.format, |
| inputImages: params.inputImages, |
| ...(timeoutMs !== undefined ? { timeoutMs } : {}), |
| }); |
| if (!Array.isArray(result.tracks) || result.tracks.length === 0) { |
| throw new Error("Music generation provider returned no tracks."); |
| } |
| const emptyTrackIndex = result.tracks.findIndex((track) => track.buffer.byteLength === 0); |
| if (emptyTrackIndex >= 0) { |
| throw new Error( |
| `Music generation provider returned an empty track buffer at index ${emptyTrackIndex}.`, |
| ); |
| } |
| return { |
| tracks: result.tracks, |
| provider: candidate.provider, |
| model: result.model ?? candidate.model, |
| attempts, |
| lyrics: result.lyrics, |
| normalization: sanitized.normalization, |
| metadata: { |
| ...result.metadata, |
| ...buildMediaGenerationNormalizationMetadata({ |
| normalization: sanitized.normalization, |
| }), |
| }, |
| ignoredOverrides: sanitized.ignoredOverrides, |
| }; |
| }; |
| }, |
| }); |
| } |
|
|
