import type { VideoStatusResponse, SegmentsPayload, HealthStatus, ExtractedFrame, LoginRequest, LoginResponse, AuthUser, GenerationStep } from '@/types'; // ==================== RETRY CONSTANTS ==================== export const RETRY_DELAY_MS = 2000; export const MAX_VIDEO_ATTEMPTS = 2; /** Max attempts before skipping auto-fix and using flow retry (attemptCount 0, 1 = 2 tries). */ export const AUTO_FIX_MAX_ATTEMPTS = 3; const API_BASE = import.meta.env.VITE_API_BASE_URL || 'http://localhost:4000'; // Get auth token from localStorage function getAuthToken(): string | null { return localStorage.getItem('auth_token'); } // Set auth token in localStorage export function setAuthToken(token: string): void { localStorage.setItem('auth_token', token); } // Remove auth token from localStorage export function removeAuthToken(): void { localStorage.removeItem('auth_token'); } function getStatusBasedErrorMessage(status: number): string { if (status === 401) return 'Incorrect username or password.'; if (status === 403) return 'Access forbidden.'; if (status === 404) return 'Resource not found.'; if (status >= 500) return 'Server error. Please try again later.'; return `Request failed with status ${status}`; } /** Parse error message from a non-ok fetch response (JSON or text). */ async function parseFetchError(response: Response, defaultMessage: string): Promise { const contentType = response.headers.get('content-type'); const isJson = contentType?.includes('application/json'); try { if (isJson) { const errorData = await response.json(); return errorData.detail || errorData.message || errorData.error || (typeof errorData === 'string' ? errorData : defaultMessage); } const text = await response.text(); return text?.trim() ? text : getStatusBasedErrorMessage(response.status); } catch { return getStatusBasedErrorMessage(response.status); } } /** POST JSON to API with auth. */ async function apiPostJson(path: string, body: object): Promise { return apiRequest(path, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), }); } // Generic API request handler async function apiRequest( path: string, options: RequestInit = {} ): Promise { const url = path.startsWith('http') ? path : `${API_BASE}${path}`; const token = getAuthToken(); // Normalize headers to a plain object const headers: Record = {}; // Convert Headers object or array to plain object if (options.headers) { if (options.headers instanceof Headers) { options.headers.forEach((value, key) => { headers[key] = value; }); } else if (Array.isArray(options.headers)) { options.headers.forEach(([key, value]) => { headers[key] = value; }); } else { Object.assign(headers, options.headers); } } // Add auth token if available if (token) { headers['Authorization'] = `Bearer ${token}`; } const response = await fetch(url, { ...options, headers, }); if (!response.ok) { if (response.status === 401) removeAuthToken(); const errorMessage = await parseFetchError(response, `Request failed with status ${response.status}`); console.error('API Error:', { status: response.status, statusText: response.statusText, message: errorMessage, url, contentType: response.headers.get('content-type'), }); throw new Error(errorMessage); } return response.json(); } // Health check export async function checkHealth(): Promise { return apiRequest('/health'); } // ==================== AUTHENTICATION ==================== export async function login(credentials: LoginRequest): Promise { const response = await apiRequest('/api/auth/login', { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify(credentials), }); // Store token if (response.access_token) { setAuthToken(response.access_token); } return response; } export async function verifyAuth(): Promise { try { const response = await apiRequest('/api/auth/verify'); return response; } catch { removeAuthToken(); return { username: '', authenticated: false }; } } export async function getCurrentUser(): Promise { try { const response = await apiRequest('/api/auth/me'); return response; } catch { removeAuthToken(); return { username: '', authenticated: false }; } } export function logout(): void { removeAuthToken(); } // ==================== KLING/KIE API ==================== export interface KlingGenerateParams { prompt: string | object; imageUrls?: string[]; model?: string; aspectRatio?: string; generationType?: string; seeds?: number; voiceType?: string; } export interface KlingGenerateResponse { taskId: string; status: string; } export async function klingGenerate(params: KlingGenerateParams): Promise { return apiPostJson('/api/veo/generate', params); } export async function klingExtend(taskId: string, prompt: string | object, seeds?: number, voiceType?: string): Promise { return apiPostJson('/api/veo/extend', { taskId, prompt, seeds, voiceType }); } export async function klingGetStatus(taskId: string): Promise { return apiRequest(`/api/veo/status/${taskId}`); } export async function klingCancel(taskId: string): Promise<{ code: number; msg: string; taskId: string }> { return apiRequest<{ code: number; msg: string; taskId: string }>(`/api/veo/cancel/${taskId}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, }); } export function createKlingEventSource(taskId: string): EventSource { return new EventSource(`${API_BASE}/api/veo/events/${taskId}`); } // ==================== REPLICATE API ==================== export interface ReplicateGenerateParams { prompt: string; imageUrl?: string; model?: string; duration?: number; aspectRatio?: string; } export interface ReplicateGenerateResponse { id: string; status: string; } export async function replicateGenerate(params: ReplicateGenerateParams): Promise { return apiPostJson('/api/replicate/generate', params); } export async function replicateGetStatus(predictionId: string): Promise { return apiRequest(`/api/replicate/status/${predictionId}`); } // Wait for Replicate video completion by polling export async function waitForReplicateVideo( predictionId: string, timeoutMs: number = 600000, // 10 minutes pollIntervalMs: number = 15000 // Poll every 15 seconds (video gen takes 2-5 min) ): Promise { const startTime = Date.now(); while (Date.now() - startTime < timeoutMs) { const status = await replicateGetStatus(predictionId); if (status.status === 'succeeded' && status.url) { return status.url; } else if (status.status === 'failed') { throw new Error(status.error || 'Replicate video generation failed'); } // Wait before next poll await new Promise(resolve => setTimeout(resolve, pollIntervalMs)); } throw new Error('Replicate video generation timed out'); } // ==================== PROMPT GENERATION ==================== // Stream event types export interface StreamEvent { event: 'start' | 'segment' | 'complete' | 'error'; [key: string]: any; } export interface StreamStartEvent extends StreamEvent { event: 'start'; total_segments: number; model: string; } export interface StreamSegmentEvent extends StreamEvent { event: 'segment'; index: number; total: number; progress: number; segment: any; } export interface StreamCompleteEvent extends StreamEvent { event: 'complete'; message: string; prompt_id: string; payload: SegmentsPayload; } export interface StreamErrorEvent extends StreamEvent { event: 'error'; message: string; error_type: string; } export interface GeneratePromptsStreamingOptions { signal?: AbortSignal; } /** * Generate prompts with streaming - segments arrive in real-time * * @param formData - Form data with script, style, image, etc. * @param onEvent - Callback for each stream event * @param options - Optional { signal } for cancellation * @returns Promise that resolves with complete payload */ export async function generatePromptsStreaming( formData: FormData, onEvent: (event: StreamEvent) => void, options?: GeneratePromptsStreamingOptions ): Promise { const response = await fetch(`${API_BASE}/api/generate-prompts-stream`, { method: 'POST', body: formData, signal: options?.signal, }); if (!response.ok) { throw new Error(await parseFetchError(response, 'Failed to generate prompts')); } if (!response.body) { throw new Error('No response body'); } const reader = response.body.getReader(); const decoder = new TextDecoder(); let buffer = ''; let finalPayload: SegmentsPayload | null = null; try { while (true) { const { done, value } = await reader.read(); if (done) break; if (options?.signal?.aborted) { reader.releaseLock(); throw new Error('Generation cancelled by user'); } // Decode chunk and add to buffer buffer += decoder.decode(value, { stream: true }); // Split by newlines (NDJSON format) const lines = buffer.split('\n'); buffer = lines.pop() || ''; // Keep incomplete line in buffer // Process each complete line for (const line of lines) { if (!line.trim()) continue; try { const event = JSON.parse(line) as StreamEvent; // Call event callback onEvent(event); // Store final payload from complete event if (event.event === 'complete') { finalPayload = (event as StreamCompleteEvent).payload; } // If error event, throw if (event.event === 'error') { throw new Error((event as StreamErrorEvent).message); } } catch (parseError) { console.error('Failed to parse stream event:', parseError, 'Line:', line); // Continue processing other events } } } } finally { reader.releaseLock(); } if (!finalPayload) { throw new Error('Stream completed without final payload'); } return finalPayload; } export async function refinePromptContinuity( segmentPrompt: object, lastFrameBlob: Blob ): Promise<{ refined_prompt: object }> { const formData = new FormData(); formData.append('segmentPrompt', JSON.stringify(segmentPrompt)); formData.append('lastFrame', lastFrameBlob, 'last-frame.jpg'); return apiRequest<{ refined_prompt: object }>('/api/refine-prompt-continuity', { method: 'POST', body: formData, }); } // ==================== IMAGE UPLOAD ==================== /** Use reference: true when uploading a last-frame for continuity (keeps higher quality). */ export async function uploadImage( file: File, options?: { reference?: boolean } ): Promise<{ url: string; filename: string }> { const formData = new FormData(); formData.append('file', file); const path = options?.reference ? '/api/upload-image?reference=true' : '/api/upload-image'; return apiRequest<{ url: string; filename: string }>(path, { method: 'POST', body: formData, }); } // ==================== FRAME EXTRACTION ==================== export interface ExtractFramesParams { video_url: string; script?: string; buffer_time?: number; num_frames?: number; model_size?: string; } export interface ExtractFramesResponse { frames: ExtractedFrame[]; } export async function extractFrames(params: ExtractFramesParams): Promise { return apiPostJson('/api/extract-frames', params); } // ==================== WHISPER ANALYSIS ==================== export interface WhisperAnalyzeParams { video_url: string; dialogue: string; buffer_time?: number; model_size?: string; } export interface WhisperAnalyzeResponse { success: boolean; last_word_timestamp: number | null; trim_point: number | null; frame_timestamp: number | null; frame_base64: string | null; video_duration: number; transcribed_text: string | null; // What Whisper actually heard - for prompt refinement error: string | null; } /** * Analyze video with Whisper to find last spoken word and extract frame. * This is the optimized flow that combines Whisper analysis and frame extraction. */ export async function whisperAnalyzeAndExtract(params: WhisperAnalyzeParams): Promise { return apiPostJson('/api/whisper/analyze-and-extract', params); } /** * Refine a segment prompt using the frame AND transcription from the previous segment. * This ensures perfect visual and audio continuity. */ export async function refinePromptWithContext( segmentPrompt: object, frameFile: File, transcribedDialogue: string, expectedDialogue: string ): Promise<{ refined_prompt: object; original_prompt: object }> { const formData = new FormData(); formData.append('segmentPrompt', JSON.stringify(segmentPrompt)); formData.append('lastFrame', frameFile); formData.append('transcribedDialogue', transcribedDialogue); formData.append('expectedDialogue', expectedDialogue); return apiRequest<{ refined_prompt: object; original_prompt: object }>('/api/refine-prompt-continuity', { method: 'POST', body: formData, }); } /** * Check if Whisper is available on the backend */ export async function checkWhisperStatus(): Promise<{ available: boolean; message: string }> { return apiRequest<{ available: boolean; message: string }>('/api/whisper/status'); } // ==================== VIDEO DOWNLOAD ==================== export async function downloadVideo(url: string): Promise { const response = await fetch(`${API_BASE}/api/veo/download?url=${encodeURIComponent(url)}`); if (!response.ok) { throw new Error(`Failed to download video: ${response.status}`); } return response.blob(); } // ==================== UTILITIES ==================== /** Load video from file; caller must revoke object URL when done. */ function loadVideoFromFile(file: File): Promise { return new Promise((resolve, reject) => { const video = document.createElement('video'); video.preload = 'metadata'; video.src = URL.createObjectURL(file); video.onloadedmetadata = () => resolve(video); video.onerror = () => { URL.revokeObjectURL(video.src); reject(new Error('Failed to load video metadata')); }; }); } export async function getVideoDuration(file: File): Promise { const video = await loadVideoFromFile(file); try { return video.duration; } finally { URL.revokeObjectURL(video.src); } } export async function generateThumbnails(file: File, count: number = 5): Promise { const video = await loadVideoFromFile(file); video.muted = true; try { const ctx = document.createElement('canvas').getContext('2d'); if (!ctx) throw new Error('Could not get canvas context'); const maxSize = 400; const videoWidth = video.videoWidth || 1080; const videoHeight = video.videoHeight || 1920; const scale = Math.min(maxSize / videoWidth, maxSize / videoHeight); const w = Math.round(videoWidth * scale); const h = Math.round(videoHeight * scale); ctx.canvas.width = w; ctx.canvas.height = h; const duration = video.duration; const thumbnails: string[] = []; for (let i = 0; i < count; i++) { video.currentTime = (duration / count) * i; await new Promise((res) => { video.onseeked = () => res(); }); ctx.drawImage(video, 0, 0, w, h); thumbnails.push(ctx.canvas.toDataURL('image/jpeg', 0.85)); } return thumbnails; } finally { URL.revokeObjectURL(video.src); } } // Wait for video completion using SSE export function waitForKlingVideo(taskId: string, timeoutMs: number = 300000): Promise { return new Promise((resolve, reject) => { const eventSource = createKlingEventSource(taskId); const startTime = Date.now(); const timeout = setTimeout(() => { eventSource.close(); reject(new Error('Video generation timed out')); }, timeoutMs); eventSource.onmessage = (event) => { try { const data = JSON.parse(event.data); if (data.status === 'succeeded' && data.url) { clearTimeout(timeout); eventSource.close(); resolve(data.url); } else if (data.status === 'cancelled') { clearTimeout(timeout); eventSource.close(); reject(new Error('Video generation cancelled by user')); } else if (data.status === 'failed' || data.code !== undefined && data.code !== 200) { clearTimeout(timeout); eventSource.close(); const errorMsg = data.error || data.msg || `Video generation failed (code: ${data.code || 'unknown'})`; reject(new Error(errorMsg)); } } catch (err) { console.error('Failed to parse SSE data:', err); // If we can't parse, don't reject immediately - might be a partial message } }; eventSource.onerror = () => { // If connection error and we've been waiting a while, reject const elapsed = Date.now() - startTime; if (elapsed > 5000) { // Wait at least 5 seconds before rejecting on connection error clearTimeout(timeout); eventSource.close(); reject(new Error('SSE connection error - video generation may have failed')); } }; }); } // Generate video with automatic retry (retries once on failure) export async function generateVideoWithRetry( generateFn: () => Promise<{ taskId: string }>, timeoutMs: number = 300000, onRetry?: (attempt: number) => void ): Promise { let lastError: Error | null = null; for (let attempt = 0; attempt < MAX_VIDEO_ATTEMPTS; attempt++) { try { if (attempt > 0) { console.log(`🔄 Retrying video generation (attempt ${attempt + 1}/${MAX_VIDEO_ATTEMPTS})...`); if (onRetry) { onRetry(attempt + 1); } } const result = await generateFn(); const videoUrl = await waitForKlingVideo(result.taskId, timeoutMs); return videoUrl; } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); console.error(`❌ Video generation attempt ${attempt + 1} failed:`, lastError.message); if (attempt === 0) { console.log(`⏳ Waiting ${RETRY_DELAY_MS / 1000} seconds before retry...`); await new Promise(resolve => setTimeout(resolve, RETRY_DELAY_MS)); continue; } throw lastError; } } throw lastError || new Error('Video generation failed'); } /** Outcome of flow-level retry: either trigger retry or state was updated (partial/error). */ export type FlowRetryOutcome = 'retry' | 'handled'; /** Shared flow-level retry: on cancel show error; on first failure return 'retry'; else show partial or error. */ export async function handleFlowRetry(options: { attemptCount: number; errorMessage: string; isCancelled: boolean; generatedCount: number; totalCount: number; setError: (s: string) => void; setStep: (s: GenerationStep) => void; setPartialCompletionError: (s: string | null) => void; }): Promise { if (options.isCancelled) { options.setError('Generation cancelled by user'); options.setStep('error'); return 'handled'; } if (options.attemptCount === 0) { await new Promise(resolve => setTimeout(resolve, RETRY_DELAY_MS)); return 'retry'; } if (options.generatedCount > 0) { options.setPartialCompletionError( `Generation stopped at segment ${options.generatedCount + 1} of ${options.totalCount}. Reason: ${options.errorMessage}` ); options.setStep('completed'); } else { options.setError(options.errorMessage); options.setStep('error'); } return 'handled'; } // ==================== CONTENT VALIDATION ==================== export interface ContentValidationRequest { script: string; } export interface ContentValidationResponse { is_valid: boolean; warnings: string[]; guidance?: { avoid_public_figures: string[]; avoid_copyrighted: string[]; avoid_sensitive: string[]; best_practices: string[]; } | null; } /** * Validate script content before generation to catch potential issues early * Checks for public figures, copyrighted content, and other policy violations */ export async function validateContent(script: string): Promise { return apiPostJson('/api/validate-content', { script }); } /** * Get content policy guidance */ export async function getContentGuidance() { return apiRequest('/api/content-guidance'); } // ==================== PROMPT SAFETY ==================== export interface SafetyFixRequest { segment: any; // VeoSegment error_message: string; attempt_count?: number; } export interface SafetyFixResponse { success: boolean; fixed_segment: any | null; changes_made: string | null; error: string | null; } /** * Automatically fix an unsafe prompt using AI * Detects content that triggered safety filters and modifies it to be compliant */ export async function fixUnsafePrompt(request: SafetyFixRequest): Promise { return apiPostJson('/api/safety/fix-unsafe-prompt', request); } /** * Check if error is a Veo 3.1 safety/content policy violation * Based on official Veo 3.1 API documentation * * Official error types: * - PUBLIC_ERROR_MINOR: Generic internal error (wait and retry) * - Support codes (8-digit): Specific safety categories * - "Flagged for containing..." errors: Public figures, unsafe content */ export function isUnsafeSegmentError(errorMessage: string): boolean { const upperError = errorMessage.toUpperCase(); // Official Veo 3.1 safety patterns from documentation const safetyKeywords = [ 'UNSAFE', 'CONTENT_POLICY', 'MODERATION', 'INAPPROPRIATE', 'VIOLATION', 'PROHIBITED', 'SAFETY', 'BLOCKED', // Public figures (MOST COMMON!) 'PROMINENT PUBLIC FIGURE', 'PUBLIC FIGURE', 'CELEBRITY', 'POLITICIAN', 'FLAGGED FOR CONTAINING', 'PROMINENT PUBLIC', ]; // Check for safety keywords const hasKeyword = safetyKeywords.some(keyword => upperError.includes(keyword)); // Check for 8-digit support code (indicates safety filter) // Official Veo 3.1 uses support codes for safety categories const hasSupportCode = /\b\d{8}\b/.test(errorMessage); return hasKeyword || hasSupportCode; } // ==================== VIDEO MERGE/EXPORT ==================== export interface ClipMetadata { index: number; startTime: number; endTime: number; type: 'video' | 'image'; duration?: number; } // Merge multiple video files into a single video export async function mergeVideos( videoBlobs: Blob[], clipMetadata: ClipMetadata[] ): Promise { const formData = new FormData(); // Add clip metadata as JSON formData.append('clips_data', JSON.stringify(clipMetadata)); // Add video files videoBlobs.forEach((blob, index) => { formData.append('files', blob, `video_${index}.mp4`); }); const response = await fetch(`${API_BASE}/api/export/merge`, { method: 'POST', body: formData, }); if (!response.ok) { throw new Error(await parseFetchError(response, 'Failed to merge videos')); } return response.blob(); }