import React, { useState, useCallback, useEffect, useRef } from 'react'; import { motion } from 'framer-motion'; import { useGeneration } from '@/context/GenerationContext'; import type { GenerationInputs, VideoProvider, GeneratedVideo, VeoSegment, KieVideoFlow } from '@/types'; import { SparklesIcon, ArrowLeftIcon, ImageIcon } from './Icons'; import { generatePromptsStreaming, type StreamEvent, type StreamSegmentEvent, uploadImage, klingGenerate, klingExtend, waitForKlingVideo, generateVideoWithRetry, handleFlowRetry, type FlowRetryOutcome, downloadVideo, getVideoDuration, generateThumbnails, replicateGenerate, waitForReplicateVideo, whisperAnalyzeAndExtract, fixUnsafePrompt, isUnsafeSegmentError, validateContent, AUTO_FIX_MAX_ATTEMPTS, MAX_VIDEO_ATTEMPTS, type ContentValidationResponse } from '@/utils/api'; interface GenerationFormProps { provider: VideoProvider; onBack: () => void; } const voiceTypes = ['Deep', 'Warm', 'Crisp']; const energyLevels = ['Low', 'Medium', 'High']; const cameraStyles = ['Standard', 'Handheld', 'Steadicam', 'FPV Drone']; const narrativeStyles = ['Standard', 'Documentary', 'Action', 'Introspective']; const aspectRatios = ['9:16', '16:9', '1:1']; // Generation modes type GenerationMode = 'extend' | 'frame-continuity'; export const GenerationForm: React.FC = ({ provider, onBack }) => { const { startGeneration, updateProgress, addVideo, setStep, setError, updateSegments, addTaskId, removeTaskId, setPartialCompletionError, registerPromptAbortController, state } = useGeneration(); const { generatedVideos, segments, isCancelling } = state; // Check if we're in "reuse mode" (segments pre-loaded from cache) const isReuseMode = segments.length > 0; // Draft storage key const draftKey = `video-gen-draft-${provider}`; // Load draft on mount - initialize state from localStorage const loadDraft = useCallback(() => { try { const savedDraft = localStorage.getItem(draftKey); if (savedDraft) { const draft = JSON.parse(savedDraft); return draft; } } catch (error) { console.warn('Failed to load draft:', error); } return null; }, [draftKey]); const draft = loadDraft(); const [draftRestored, setDraftRestored] = useState(!!draft); const [formState, setFormState] = useState( draft?.formState || { script: '', style: '', voiceType: 'Deep', energyLevel: 'Medium', cameraStyle: 'Standard', narrativeStyle: 'Standard', seedValue: 12005, aspectRatio: '9:16', model: provider === 'kling' ? 'veo3_fast' : 'google/veo-3-fast', } ); const [imageFile, setImageFile] = useState(null); const [imagePreview, setImagePreview] = useState(draft?.imagePreview || null); const [isDragging, setIsDragging] = useState(false); const [isGenerating, setIsGenerating] = useState(false); const [contentWarnings, setContentWarnings] = useState(null); const [showWarnings, setShowWarnings] = useState(false); // Generation mode selection const [generationMode, setGenerationMode] = useState(draft?.generationMode || 'frame-continuity'); /** KIE: prompt style β€” creator talking-head vs product showcase UGC */ const [kieVideoFlow, setKieVideoFlow] = useState(draft?.kieVideoFlow || 'creator_ugc'); // Segment mode: fixed duration (4/6/8s) or AI decides per segment const [segmentMode, setSegmentMode] = useState<'fixed' | 'ai_driven'>(draft?.segmentMode ?? 'fixed'); const [secondsPerSegment, setSecondsPerSegment] = useState<4 | 6 | 8>(draft?.secondsPerSegment ?? 8); // Show notification if draft was restored useEffect(() => { if (draftRestored) { console.log('πŸ“ Draft restored from localStorage'); // Auto-hide notification after 5 seconds const timer = setTimeout(() => setDraftRestored(false), 5000); return () => clearTimeout(timer); } }, [draftRestored]); // Save draft whenever formState, imagePreview, or generationMode changes // Skip saving on initial mount to avoid overwriting with default values const isInitialMount = useRef(true); useEffect(() => { if (isInitialMount.current) { isInitialMount.current = false; return; } try { const draft = { formState, generationMode, kieVideoFlow, segmentMode, secondsPerSegment, savedAt: new Date().toISOString(), }; localStorage.setItem(draftKey, JSON.stringify(draft)); } catch (error) { if (error instanceof DOMException && error.name === 'QuotaExceededError') { try { localStorage.removeItem(draftKey); const fallback = { formState: { ...formState, script: formState.script.slice(0, 5000) }, generationMode, kieVideoFlow, segmentMode, secondsPerSegment, savedAt: new Date().toISOString(), }; localStorage.setItem(draftKey, JSON.stringify(fallback)); } catch { console.warn('Failed to save draft: quota exceeded'); } } else { console.warn('Failed to save draft:', error); } } }, [formState, imagePreview, generationMode, kieVideoFlow, segmentMode, secondsPerSegment, draftKey]); // Clear draft function const clearDraft = useCallback(() => { try { localStorage.removeItem(draftKey); setDraftRestored(false); console.log('πŸ—‘οΈ Draft cleared'); } catch (error) { console.warn('Failed to clear draft:', error); } }, [draftKey]); // Accumulate streamed segments so we can show them live and pass to context const streamedSegmentsRef = useRef([]); // Calculate estimated segments const wordCount = formState.script.trim().split(/\s+/).filter(w => w).length; const estimatedSegments = wordCount > 0 ? Math.max(1, Math.min(Math.ceil(wordCount / 17), 10)) : 0; // Validate content when script changes (debounced) const validateScriptContent = useCallback(async (script: string) => { if (!script.trim() || isReuseMode) { setContentWarnings(null); return; } try { const validation = await validateContent(script); if (!validation.is_valid) { setContentWarnings(validation); setShowWarnings(true); } else { setContentWarnings(null); setShowWarnings(false); } } catch (error) { console.error('Content validation error:', error); // Silently fail - don't block user } }, [isReuseMode]); // Debounce content validation useEffect(() => { const timer = setTimeout(() => { validateScriptContent(formState.script); }, 1000); // Validate 1 second after user stops typing return () => clearTimeout(timer); }, [formState.script, validateScriptContent]); // Handle input changes const handleChange = (e: React.ChangeEvent) => { const { name, value } = e.target; setFormState(prev => ({ ...prev, [name]: value })); }; // Handle image upload const handleImageUpload = useCallback((file: File) => { if (file.type.startsWith('image/')) { setImageFile(file); const reader = new FileReader(); reader.onloadend = () => setImagePreview(reader.result as string); reader.readAsDataURL(file); } }, []); // Drag and drop handlers const handleDragOver = (e: React.DragEvent) => { e.preventDefault(); setIsDragging(true); }; const handleDragLeave = () => setIsDragging(false); const handleDrop = (e: React.DragEvent) => { e.preventDefault(); setIsDragging(false); const file = e.dataTransfer.files[0]; if (file) handleImageUpload(file); }; // Extract last frame from video blob const extractLastFrame = async (videoBlob: Blob): Promise => { return new Promise((resolve, reject) => { const video = document.createElement('video'); video.preload = 'metadata'; video.muted = true; video.src = URL.createObjectURL(videoBlob); video.onloadedmetadata = async () => { // Seek to near the end of the video const targetTime = Math.max(0, video.duration - 0.1); video.currentTime = targetTime; }; video.onseeked = () => { // Create canvas and draw current frame const canvas = document.createElement('canvas'); canvas.width = video.videoWidth; canvas.height = video.videoHeight; const ctx = canvas.getContext('2d'); if (!ctx) { URL.revokeObjectURL(video.src); reject(new Error('Could not get canvas context')); return; } ctx.drawImage(video, 0, 0); // Convert to blob then to file canvas.toBlob((blob) => { URL.revokeObjectURL(video.src); if (!blob) { reject(new Error('Could not extract frame')); return; } const file = new File([blob], `frame-${Date.now()}.jpg`, { type: 'image/jpeg' }); resolve(file); }, 'image/jpeg', 0.95); }; video.onerror = () => { URL.revokeObjectURL(video.src); reject(new Error('Failed to load video for frame extraction')); }; }); }; // ============================================ // KIE GENERATION - FRAME CONTINUITY FLOW // ============================================ // This mirrors the Replicate flow from standalone_video_creator.py: // 1. Generate first video with original reference image // 2. Extract last frame using the whisper analysis from generated video // 3. Use that frame as reference for next segment // 4. Repeat for all segments const handleKlingFrameContinuityFlow = async (attemptCount: number = 0) => { if (!imageFile) return; if (!isReuseMode && !formState.script.trim()) return; setIsGenerating(true); setError(null); let segmentsToUse: VeoSegment[] = []; let currentImageFile: File = imageFile; // Track reference image across segments and catch block let completedVideoCount = 0; // Track locally to avoid stale React state closure try { // Step 1: Get segments - either from cache (reuse mode), existing segments (retry mode), or generate new ones if ((isReuseMode || attemptCount > 0) && segments.length > 0) { // REUSE MODE or RETRY MODE: Skip prompt generation, use existing segments const mode = isReuseMode ? 'REUSE' : 'RETRY'; console.log(`♻️ ${mode} MODE: Using ${segments.length} existing segments (attempt ${attemptCount})`); segmentsToUse = segments; setStep('generating_video'); updateProgress(`Starting video generation with ${segments.length} ${mode === 'RETRY' ? 'cached' : 'pre-loaded'} segments...`, 0, segments.length); } else { // NORMAL MODE: Generate prompts using GPT-5.2 with STREAMING setStep('generating_prompts'); updateProgress('🌊 Streaming prompt generation with GPT-5.2...', 0, estimatedSegments); const formData = new FormData(); formData.append('script', formState.script); formData.append('style', formState.style || 'clean, lifestyle UGC'); formData.append('jsonFormat', 'standard'); formData.append('continuationMode', 'true'); formData.append('voiceType', formState.voiceType || ''); formData.append('energyLevel', formState.energyLevel || ''); formData.append('settingMode', 'single'); formData.append('cameraStyle', formState.cameraStyle || ''); formData.append('narrativeStyle', formState.narrativeStyle || ''); formData.append( 'videoFlow', provider === 'kling' ? kieVideoFlow : 'creator_ugc' ); formData.append('image', imageFile); formData.append('segment_mode', segmentMode); formData.append('seconds_per_segment', String(secondsPerSegment)); if (segmentMode === 'ai_driven') formData.append('allowed_durations', '4,6,8'); streamedSegmentsRef.current = []; const promptAbort = new AbortController(); registerPromptAbortController(promptAbort); try { const payload = await generatePromptsStreaming( formData, (event: StreamEvent) => { switch (event.event) { case 'start': streamedSegmentsRef.current = []; updateSegments([]); console.log(`🌊 Starting streaming generation of ${event.total_segments} segments`); updateProgress(`Streaming ${event.total_segments} segments from GPT-5.2...`, 0, event.total_segments); break; case 'segment': { const seg = (event as StreamSegmentEvent).segment; if (seg) { streamedSegmentsRef.current = [...streamedSegmentsRef.current, seg]; updateSegments(streamedSegmentsRef.current); } console.log(`βœ… Segment ${event.index + 1} complete (${event.progress.toFixed(1)}%)`); updateProgress(`Generated segment ${event.index + 1}... (${event.progress.toFixed(1)}%)`); break; } case 'complete': console.log(`πŸŽ‰ All ${event.payload.segments.length} segments generated!`); updateProgress('All segments generated! Starting video generation...'); break; case 'error': console.error(`❌ Streaming error: ${event.message}`); throw new Error(event.message); } }, { signal: promptAbort.signal } ); if (!payload?.segments?.length) { throw new Error('No segments generated from script'); } segmentsToUse = payload.segments; updateProgress(`Generated ${segmentsToUse.length} segments. Starting video generation...`); startGeneration(segmentsToUse); } finally { registerPromptAbortController(null); } } // Reset reference image to original for this run currentImageFile = imageFile; // RESUME SUPPORT: Start from where we left off if retrying const startIndex = attemptCount > 0 ? generatedVideos.length : 0; // If resuming, extract last frame from previous video for continuity if (startIndex > 0 && generatedVideos[startIndex - 1]?.blobUrl) { updateProgress(`Resuming from segment ${startIndex + 1}...`); console.log(`πŸ”„ Resuming generation from segment ${startIndex + 1} (${generatedVideos.length} videos already generated)`); try { const lastVideoBlob = await fetch(generatedVideos[startIndex - 1].blobUrl!).then(r => r.blob()); const lastFrameFile = await extractLastFrame(lastVideoBlob); currentImageFile = lastFrameFile; console.log(`βœ… Using frame from segment ${startIndex} for continuity`); } catch (frameError) { console.warn('⚠️ Failed to extract frame for resume, using original image:', frameError); } } // Step 2: Generate videos segment by segment with frame continuity for (let i = startIndex; i < segmentsToUse.length; i++) { const isLastSegment = i === segmentsToUse.length - 1; // Per-segment safety retry loop with hard cap (prevents infinite retries) for (let safetyAttempt = 0; safetyAttempt <= AUTO_FIX_MAX_ATTEMPTS; safetyAttempt++) { try { const segment = segmentsToUse[i]; updateProgress( `Generating video ${i + 1} of ${segmentsToUse.length}...${i > 0 ? ' (using last frame from previous)' : ''}${safetyAttempt > 0 ? ` (safety retry ${safetyAttempt}/${AUTO_FIX_MAX_ATTEMPTS})` : ''}`, i, segmentsToUse.length ); // Upload current reference image updateProgress(`Uploading reference image for segment ${i + 1}...`); const uploadResult = await uploadImage(currentImageFile, { reference: true }); const hostedImageUrl = uploadResult.url; console.log(`πŸ–ΌοΈ Segment ${i + 1} using image: ${i === 0 ? 'original' : 'last frame from previous'}`); // Generate video with current reference image updateProgress(`Submitting segment ${i + 1} to KIE Veo 3.1...`); const generateResult = await klingGenerate({ prompt: segment, imageUrls: [hostedImageUrl], model: 'veo3_fast', aspectRatio: formState.aspectRatio, generationType: 'FIRST_AND_LAST_FRAMES_2_VIDEO', seeds: formState.seedValue, voiceType: formState.voiceType, }); // Wait for completion updateProgress(`Processing video ${i + 1}... (this may take 1-2 minutes)`); const videoUrl = await waitForKlingVideo(generateResult.taskId); // Download video updateProgress(`Downloading video ${i + 1}...`); const videoBlob = await downloadVideo(videoUrl); const blobUrl = URL.createObjectURL(videoBlob); // Get video duration const videoFile = new File([videoBlob], `segment-${i + 1}.mp4`, { type: 'video/mp4' }); const duration = await getVideoDuration(videoFile); const thumbnails = await generateThumbnails(videoFile); // Use Whisper to find optimal trim point, extract frame, and get transcription let trimPoint = duration; // Default to full duration let transcribedText = ''; // What Whisper actually heard if (!isLastSegment) { updateProgress(`Analyzing video ${i + 1} with Whisper for optimal continuity...`); try { // Get dialogue from segment for Whisper analysis const dialogue = segment.action_timeline?.dialogue || ''; const whisperResult = await whisperAnalyzeAndExtract({ video_url: videoUrl, dialogue: dialogue, buffer_time: 0.3, model_size: 'base' }); if (whisperResult.success && whisperResult.frame_base64) { // Convert base64 frame to File for next segment (backend may return PNG or JPEG) const dataUrl = whisperResult.frame_base64; const mime = dataUrl.startsWith('data:image/png') ? 'image/png' : 'image/jpeg'; const ext = mime === 'image/png' ? 'png' : 'jpg'; const base64Data = dataUrl.split(',')[1] || dataUrl; const byteCharacters = atob(base64Data); const byteNumbers = new Array(byteCharacters.length); for (let j = 0; j < byteCharacters.length; j++) { byteNumbers[j] = byteCharacters.charCodeAt(j); } const byteArray = new Uint8Array(byteNumbers); const frameBlob = new Blob([byteArray], { type: mime }); currentImageFile = new File([frameBlob], `whisper-frame-${i + 1}.${ext}`, { type: mime }); // Store trim point for later merge if (whisperResult.trim_point) { trimPoint = whisperResult.trim_point; } // Store transcribed text for prompt refinement if (whisperResult.transcribed_text) { transcribedText = whisperResult.transcribed_text; console.log(`πŸ“ Whisper transcription: "${transcribedText.substring(0, 100)}..."`); } console.log(`βœ… Whisper: Last word at ${whisperResult.last_word_timestamp?.toFixed(2)}s, frame at ${whisperResult.frame_timestamp?.toFixed(2)}s, trim at ${trimPoint.toFixed(2)}s`); // REFINE NEXT SEGMENT PROMPT with frame + transcription const nextSegment = segmentsToUse[i + 1]; if (nextSegment && currentImageFile) { updateProgress(`Refining segment ${i + 2} prompt with visual and audio context...`); try { const { refinePromptWithContext } = await import('@/utils/api'); const refined = await refinePromptWithContext( nextSegment, currentImageFile, transcribedText, dialogue ); // Update the next segment with refined prompt segmentsToUse[i + 1] = refined.refined_prompt as typeof nextSegment; console.log(`βœ… Refined segment ${i + 2} prompt for consistency`); } catch (refineError) { console.warn(`⚠️ Prompt refinement failed, using original:`, refineError); } } } else { // Fallback to simple last frame extraction console.log(`⚠️ Whisper failed (${whisperResult.error}), falling back to last frame extraction`); const lastFrameFile = await extractLastFrame(videoBlob); currentImageFile = lastFrameFile; } } catch (frameError) { console.error(`⚠️ Whisper analysis failed, using fallback:`, frameError); try { const lastFrameFile = await extractLastFrame(videoBlob); currentImageFile = lastFrameFile; } catch { // Continue with current image if all extraction fails } } } // Add to generated videos with trim metadata const generatedVideo: GeneratedVideo = { id: `video-${Date.now()}-${i}`, url: videoUrl, blobUrl, segment, duration, thumbnails, trimPoint, // Store trim point for merge }; addVideo(generatedVideo); completedVideoCount++; updateProgress(`Completed video ${i + 1} of ${segmentsToUse.length}`, i + 1, segmentsToUse.length); break; // Segment succeeded - exit safety retry loop, move to next segment } catch (segErr) { const segErrMsg = segErr instanceof Error ? segErr.message : String(segErr); // Safety error with retries remaining: auto-fix the prompt and retry this segment if (isUnsafeSegmentError(segErrMsg) && safetyAttempt < AUTO_FIX_MAX_ATTEMPTS) { console.log(`πŸ›‘οΈ Safety error on segment ${i + 1}, auto-fix attempt ${safetyAttempt + 1}/${AUTO_FIX_MAX_ATTEMPTS}...`); updateProgress(`Safety issue in segment ${i + 1}, auto-fixing (attempt ${safetyAttempt + 1}/${AUTO_FIX_MAX_ATTEMPTS})...`); try { const fixResult = await fixUnsafePrompt({ segment: segmentsToUse[i], error_message: segErrMsg, attempt_count: safetyAttempt + 1 }); if (fixResult.success && fixResult.fixed_segment) { console.log(`βœ… Auto-fix successful: ${fixResult.changes_made}`); updateProgress(`Auto-fix successful: ${fixResult.changes_made}`); segmentsToUse[i] = fixResult.fixed_segment; updateSegments([...segmentsToUse]); await new Promise(resolve => setTimeout(resolve, 1000)); continue; // Retry this segment with the fixed prompt } else { console.warn(`⚠️ Auto-fix returned no usable fix: ${fixResult.error}`); } } catch (fixErr) { console.error('❌ Auto-fix API error:', fixErr); } } // Non-safety error, fix failed, or max safety retries exhausted β†’ propagate to outer handler throw segErr; } } // end per-segment safety retry loop } // All done! clearDraft(); // Clear draft on successful generation setStep('completed'); updateProgress('All videos generated successfully!'); } catch (err) { console.error('Generation error:', err); const errorMessage = err instanceof Error ? err.message : 'Generation failed'; const isUserCancel = (err as Error & { name?: string })?.name === 'AbortError' || errorMessage.includes('cancelled') || isCancelling; // If user cancelled, context already set error with segment/video count; don't overwrite if (isUserCancel) { setStep('error'); setIsGenerating(false); return; } // If some videos were generated, show partial completion immediately // (don't retry from scratch -- that would waste the already-generated videos) if (completedVideoCount > 0) { console.log(`⚠️ Partial completion: ${completedVideoCount}/${segmentsToUse.length} segments generated`); setPartialCompletionError( `Generation stopped at segment ${completedVideoCount + 1} of ${segmentsToUse.length}. ` + `${completedVideoCount} video${completedVideoCount > 1 ? 's' : ''} generated successfully. ` + `Reason: ${errorMessage}` ); setStep('completed'); } else { // No videos generated at all -- use normal retry logic const outcome: FlowRetryOutcome = await handleFlowRetry({ attemptCount, errorMessage, isCancelled: false, generatedCount: 0, totalCount: segmentsToUse.length, setError, setStep, setPartialCompletionError, }); if (outcome === 'retry') { console.log('πŸ”„ No videos generated, auto-retrying...'); updateProgress('Generation failed, automatically retrying...'); return handleKlingFrameContinuityFlow(1); } } } finally { setIsGenerating(false); } }; // ============================================ // KIE GENERATION - EXTEND API FLOW // ============================================ // Original flow using KIE's extend API const handleKlingExtendFlow = async (attemptCount: number = 0) => { if (!imageFile || !formState.script.trim()) return; setIsGenerating(true); setError(null); let payload: { segments: VeoSegment[] } = { segments: [] }; // Declare at function scope let completedVideoCount = 0; // Track locally to avoid stale React state closure try { // Step 1: Get segments - skip prompt generation if segments already exist (retry mode) if (attemptCount > 0 && segments.length > 0) { // RETRY MODE: Use existing segments (they may have been edited) console.log(`♻️ RETRY MODE: Using ${segments.length} existing segments (attempt ${attemptCount})`); payload = { segments }; setStep('generating_video'); updateProgress(`Retrying video generation with ${segments.length} cached segments...`, 0, segments.length); } else { // NORMAL MODE: Generate prompts using streaming (GPT-5.2) setStep('generating_prompts'); updateProgress('Streaming prompt generation with GPT-5.2...', 0, estimatedSegments); const formData = new FormData(); formData.append('script', formState.script); formData.append('style', formState.style || 'clean, lifestyle UGC'); formData.append('jsonFormat', 'standard'); formData.append('continuationMode', 'true'); formData.append('voiceType', formState.voiceType || ''); formData.append('energyLevel', formState.energyLevel || ''); formData.append('settingMode', 'single'); formData.append('cameraStyle', formState.cameraStyle || ''); formData.append('narrativeStyle', formState.narrativeStyle || ''); formData.append( 'videoFlow', provider === 'kling' ? kieVideoFlow : 'creator_ugc' ); formData.append('image', imageFile); formData.append('segment_mode', segmentMode); formData.append('seconds_per_segment', String(secondsPerSegment)); if (segmentMode === 'ai_driven') formData.append('allowed_durations', '4,6,8'); streamedSegmentsRef.current = []; const promptAbortExtend = new AbortController(); registerPromptAbortController(promptAbortExtend); try { payload = await generatePromptsStreaming( formData, (event: StreamEvent) => { switch (event.event) { case 'start': streamedSegmentsRef.current = []; updateSegments([]); updateProgress(`Streaming ${event.total_segments} segments from GPT-5.2...`, 0, event.total_segments); break; case 'segment': { const seg = (event as StreamSegmentEvent).segment; if (seg) { streamedSegmentsRef.current = [...streamedSegmentsRef.current, seg]; updateSegments(streamedSegmentsRef.current); } updateProgress(`Generated segment ${event.index + 1}... (${event.progress.toFixed(1)}%)`); break; } case 'complete': updateProgress('All segments generated! Starting video generation...'); break; case 'error': throw new Error(event.message); } }, { signal: promptAbortExtend.signal } ); if (!payload?.segments?.length) { throw new Error('No segments generated from script'); } updateProgress(`Generated ${payload.segments.length} segments. Starting video generation...`); startGeneration(payload.segments); } finally { registerPromptAbortController(null); } } // Step 2: Upload reference image once updateProgress('Uploading reference image...'); const uploadResult = await uploadImage(imageFile); const hostedImageUrl = uploadResult.url; // Step 3: Generate videos (resume from where we left off if retrying) const startIndex = generatedVideos.length; let currentTaskId: string | null = null; let currentImageUrl = hostedImageUrl; // Start with original image // If resuming, extract last frame from previous video for continuity if (startIndex > 0 && generatedVideos[startIndex - 1]?.blobUrl) { updateProgress(`Extracting last frame from segment ${startIndex} for continuity...`); try { const lastVideoBlob = await fetch(generatedVideos[startIndex - 1].blobUrl!).then(r => r.blob()); const lastFrameFile = await extractLastFrame(lastVideoBlob); const frameUploadResult = await uploadImage(lastFrameFile, { reference: true }); currentImageUrl = frameUploadResult.url; updateProgress(`Using frame from segment ${startIndex} for segment ${startIndex + 1}...`); } catch (frameError) { console.warn('Failed to extract frame, using original image:', frameError); // Continue with original image } } for (let i = startIndex; i < payload.segments.length; i++) { const segment = payload.segments[i]; updateProgress(`Generating video ${i + 1} of ${payload.segments.length}...`, i, payload.segments.length); // Generate video with automatic retry (retries once on failure) updateProgress(`Processing video ${i + 1}... (this may take 1-2 minutes)`); // Check if cancelled if (isCancelling) { throw new Error('Generation cancelled by user'); } const videoUrl = await generateVideoWithRetry(async () => { if (i === 0 || (i === startIndex && startIndex > 0)) { // First segment OR resuming after failure: use generate API with current image const generateResult = await klingGenerate({ prompt: segment, imageUrls: [currentImageUrl], model: 'veo3_fast', aspectRatio: formState.aspectRatio, generationType: 'FIRST_AND_LAST_FRAMES_2_VIDEO', seeds: formState.seedValue, voiceType: formState.voiceType, }); currentTaskId = generateResult.taskId; addTaskId(currentTaskId); return generateResult; } else { // Subsequent segments: use extend API const extendResult = await klingExtend( currentTaskId!, segment, formState.seedValue, formState.voiceType ); currentTaskId = extendResult.taskId; addTaskId(currentTaskId); return extendResult; } }, 300000, (attempt) => { updateProgress(`Retrying video ${i + 1}... (attempt ${attempt}/${MAX_VIDEO_ATTEMPTS})`); }); // Download and save updateProgress(`Downloading video ${i + 1}...`); const videoBlob = await downloadVideo(videoUrl); const blobUrl = URL.createObjectURL(videoBlob); const videoFile = new File([videoBlob], `segment-${i + 1}.mp4`, { type: 'video/mp4' }); const duration = await getVideoDuration(videoFile); const thumbnails = await generateThumbnails(videoFile); addVideo({ id: `video-${Date.now()}-${i}`, url: videoUrl, blobUrl, segment, duration, thumbnails, }); completedVideoCount++; updateProgress(`Completed video ${i + 1} of ${payload.segments.length}`, i + 1, payload.segments.length); // Remove task ID after completion if (currentTaskId) { removeTaskId(currentTaskId); } } clearDraft(); // Clear draft on successful generation setStep('completed'); updateProgress('All videos generated successfully!'); } catch (err) { console.error('Generation error:', err); const errorMessage = err instanceof Error ? err.message : 'Generation failed'; const isUserCancel = (err as Error & { name?: string })?.name === 'AbortError' || errorMessage.includes('cancelled') || isCancelling; if (isUserCancel) { setStep('error'); setIsGenerating(false); return; } // If some videos were generated, show partial completion immediately if (completedVideoCount > 0) { console.log(`⚠️ Partial completion: ${completedVideoCount}/${payload.segments.length} segments generated`); setPartialCompletionError( `Generation stopped at segment ${completedVideoCount + 1} of ${payload.segments.length}. ` + `${completedVideoCount} video${completedVideoCount > 1 ? 's' : ''} generated successfully. ` + `Reason: ${errorMessage}` ); setStep('completed'); } else { const outcome = await handleFlowRetry({ attemptCount, errorMessage, isCancelled: false, generatedCount: 0, totalCount: payload.segments.length, setError, setStep, setPartialCompletionError, }); if (outcome === 'retry') { console.log('πŸ”„ No videos generated, auto-retrying...'); updateProgress('Generation failed, automatically retrying...'); return handleKlingExtendFlow(1); } } } finally { setIsGenerating(false); // Clean up any remaining task IDs state.activeTaskIds.forEach(taskId => removeTaskId(taskId)); } }; // ============================================ // REPLICATE GENERATION - FRAME CONTINUITY FLOW // ============================================ // This mirrors the approach from standalone_video_creator.py: // 1. Generate prompts using streaming (GPT-5.2) // 2. For each segment, generate video with current reference image // 3. Extract last frame from generated video // 4. Use that frame as reference for next segment // 5. Result: Perfect visual continuity across all segments const handleReplicateGeneration = async (attemptCount: number = 0) => { if (!formState.script.trim()) return; setIsGenerating(true); setError(null); let segmentsToUse: VeoSegment[] = []; // Declare at function scope let completedVideoCount = 0; // Track locally to avoid stale React state closure try { // Step 1: Get segments - skip prompt generation if segments already exist (retry mode) if (attemptCount > 0 && segments.length > 0) { // RETRY MODE: Use existing segments console.log(`♻️ RETRY MODE: Using ${segments.length} existing segments (attempt ${attemptCount})`); segmentsToUse = segments; setStep('generating_video'); updateProgress(`Retrying video generation with ${segments.length} cached segments...`, 0, segments.length); } else { // NORMAL MODE: Generate prompts using streaming (GPT-5.2) setStep('generating_prompts'); updateProgress('Streaming prompt generation with GPT-5.2...', 0, estimatedSegments); const formData = new FormData(); formData.append('script', formState.script); formData.append('style', formState.style || 'clean, lifestyle UGC'); formData.append('jsonFormat', 'standard'); formData.append('continuationMode', 'true'); formData.append('voiceType', formState.voiceType || ''); formData.append('energyLevel', formState.energyLevel || ''); formData.append('settingMode', 'single'); formData.append('cameraStyle', formState.cameraStyle || ''); formData.append('narrativeStyle', formState.narrativeStyle || ''); formData.append( 'videoFlow', provider === 'kling' ? kieVideoFlow : 'creator_ugc' ); // If image provided, include it for GPT analysis if (imageFile) { formData.append('image', imageFile); } else { const placeholderBlob = new Blob(['placeholder'], { type: 'image/jpeg' }); formData.append('image', placeholderBlob, 'placeholder.jpg'); } formData.append('segment_mode', segmentMode); formData.append('seconds_per_segment', String(secondsPerSegment)); if (segmentMode === 'ai_driven') formData.append('allowed_durations', '4,6,8'); streamedSegmentsRef.current = []; const promptAbortReplicate = new AbortController(); registerPromptAbortController(promptAbortReplicate); try { const payload = await generatePromptsStreaming( formData, (event: StreamEvent) => { switch (event.event) { case 'start': streamedSegmentsRef.current = []; updateSegments([]); updateProgress(`Streaming ${event.total_segments} segments...`, 0, event.total_segments); break; case 'segment': { const seg = (event as StreamSegmentEvent).segment; if (seg) { streamedSegmentsRef.current = [...streamedSegmentsRef.current, seg]; updateSegments(streamedSegmentsRef.current); } updateProgress(`Generated segment ${event.index + 1}... (${event.progress.toFixed(1)}%)`); break; } case 'complete': updateProgress('All segments generated! Starting Replicate generation...'); break; case 'error': throw new Error(event.message); } }, { signal: promptAbortReplicate.signal } ); if (!payload?.segments?.length) { throw new Error('No segments generated from script'); } segmentsToUse = payload.segments; updateProgress(`Generated ${segmentsToUse.length} segments. Starting Replicate generation...`); startGeneration(segmentsToUse); } finally { registerPromptAbortController(null); } } // Track current reference image (starts with original if provided) let currentImageFile = imageFile; // Step 2: Generate videos segment by segment with frame continuity for (let i = 0; i < segmentsToUse.length; i++) { const segment = segmentsToUse[i]; const isLastSegment = i === segmentsToUse.length - 1; updateProgress( `Generating video ${i + 1} of ${segmentsToUse.length} with Replicate...${i > 0 ? ' (using last frame)' : ''}`, i, segmentsToUse.length ); // Convert structured segment to text prompt for Replicate // Replicate models typically expect text prompts const textPrompt = convertSegmentToTextPrompt(segment); console.log(`🎬 Segment ${i + 1} prompt:`, textPrompt.substring(0, 100) + '...'); // Upload current reference image if available let imageUrl: string | undefined; if (currentImageFile) { updateProgress(`Uploading reference image for segment ${i + 1}...`); const uploadResult = await uploadImage(currentImageFile, { reference: true }); imageUrl = uploadResult.url; console.log(`πŸ–ΌοΈ Segment ${i + 1} using image: ${i === 0 ? 'original' : 'last frame from previous'}`); } // Generate video with Replicate updateProgress(`Submitting segment ${i + 1} to Replicate...`); const generateResult = await replicateGenerate({ prompt: textPrompt, imageUrl: imageUrl, model: formState.model || 'google/veo-3-fast', aspectRatio: formState.aspectRatio, }); // Wait for completion (polling) updateProgress(`Processing video ${i + 1}... (this may take 2-5 minutes)`); const videoUrl = await waitForReplicateVideo(generateResult.id); // Download video updateProgress(`Downloading video ${i + 1}...`); const videoBlob = await downloadVideo(videoUrl); const blobUrl = URL.createObjectURL(videoBlob); // Get video duration and thumbnails const videoFile = new File([videoBlob], `segment-${i + 1}.mp4`, { type: 'video/mp4' }); const duration = await getVideoDuration(videoFile); const thumbnails = await generateThumbnails(videoFile); // Use Whisper to find optimal trim point, extract frame, and get transcription // This is more accurate than extracting the very last frame let trimPoint = duration; // Default to full duration let transcribedText = ''; // What Whisper actually heard if (!isLastSegment) { updateProgress(`Analyzing video ${i + 1} with Whisper for optimal continuity...`); try { // Get dialogue from segment for Whisper analysis const dialogue = segment.action_timeline?.dialogue || textPrompt; const whisperResult = await whisperAnalyzeAndExtract({ video_url: videoUrl, dialogue: dialogue, buffer_time: 0.3, model_size: 'base' }); if (whisperResult.success && whisperResult.frame_base64) { // Convert base64 frame to File for next segment (backend may return PNG or JPEG) const dataUrl = whisperResult.frame_base64; const mime = dataUrl.startsWith('data:image/png') ? 'image/png' : 'image/jpeg'; const ext = mime === 'image/png' ? 'png' : 'jpg'; const base64Data = dataUrl.split(',')[1] || dataUrl; const byteCharacters = atob(base64Data); const byteNumbers = new Array(byteCharacters.length); for (let j = 0; j < byteCharacters.length; j++) { byteNumbers[j] = byteCharacters.charCodeAt(j); } const byteArray = new Uint8Array(byteNumbers); const frameBlob = new Blob([byteArray], { type: mime }); currentImageFile = new File([frameBlob], `whisper-frame-${i + 1}.${ext}`, { type: mime }); // Store trim point for later merge if (whisperResult.trim_point) { trimPoint = whisperResult.trim_point; } // Store transcribed text for prompt refinement if (whisperResult.transcribed_text) { transcribedText = whisperResult.transcribed_text; console.log(`πŸ“ Whisper transcription: "${transcribedText.substring(0, 100)}..."`); } console.log(`βœ… Whisper: Last word at ${whisperResult.last_word_timestamp?.toFixed(2)}s, frame at ${whisperResult.frame_timestamp?.toFixed(2)}s, trim at ${trimPoint.toFixed(2)}s`); // REFINE NEXT SEGMENT PROMPT with frame + transcription const nextSegment = segmentsToUse[i + 1]; if (nextSegment && currentImageFile) { updateProgress(`Refining segment ${i + 2} prompt with visual and audio context...`); try { const { refinePromptWithContext } = await import('@/utils/api'); const refined = await refinePromptWithContext( nextSegment, currentImageFile, transcribedText, dialogue ); // Update the next segment with refined prompt segmentsToUse[i + 1] = refined.refined_prompt as typeof nextSegment; console.log(`βœ… Refined segment ${i + 2} prompt for consistency`); } catch (refineError) { console.warn(`⚠️ Prompt refinement failed, using original:`, refineError); } } } else { // Fallback to simple last frame extraction console.log(`⚠️ Whisper failed (${whisperResult.error}), falling back to last frame extraction`); const lastFrameFile = await extractLastFrame(videoBlob); currentImageFile = lastFrameFile; } } catch (frameError) { console.error(`⚠️ Whisper analysis failed, using fallback:`, frameError); try { const lastFrameFile = await extractLastFrame(videoBlob); currentImageFile = lastFrameFile; } catch { // Continue with current image if all extraction fails } } } // Add to generated videos with trim metadata const generatedVideo: GeneratedVideo = { id: `video-${Date.now()}-${i}`, url: videoUrl, blobUrl, segment, duration, thumbnails, trimPoint, // Store trim point for merge }; addVideo(generatedVideo); completedVideoCount++; updateProgress(`Completed video ${i + 1} of ${segmentsToUse.length}`, i + 1, segmentsToUse.length); } // All done! setStep('completed'); updateProgress('All videos generated successfully with Replicate!'); } catch (err) { console.error('Replicate generation error:', err); const errorMessage = err instanceof Error ? err.message : 'Replicate generation failed'; const isUserCancel = (err as Error & { name?: string })?.name === 'AbortError' || errorMessage.includes('cancelled') || isCancelling; if (isUserCancel) { setStep('error'); setIsGenerating(false); return; } // If some videos were generated, show partial completion immediately if (completedVideoCount > 0) { console.log(`⚠️ Partial completion: ${completedVideoCount}/${segmentsToUse.length} segments generated`); setPartialCompletionError( `Generation stopped at segment ${completedVideoCount + 1} of ${segmentsToUse.length}. ` + `${completedVideoCount} video${completedVideoCount > 1 ? 's' : ''} generated successfully. ` + `Reason: ${errorMessage}` ); setStep('completed'); } else { const outcome = await handleFlowRetry({ attemptCount, errorMessage, isCancelled: false, generatedCount: 0, totalCount: segmentsToUse.length, setError, setStep, setPartialCompletionError, }); if (outcome === 'retry') { console.log('πŸ”„ No videos generated, auto-retrying...'); updateProgress('Generation failed, automatically retrying...'); return handleReplicateGeneration(1); } } } finally { setIsGenerating(false); } }; // Helper: Convert structured segment JSON to text prompt for Replicate // Replicate models typically expect plain text, not structured JSON const convertSegmentToTextPrompt = (segment: VeoSegment): string => { const parts: string[] = []; // Extract dialogue const dialogue = segment.action_timeline?.dialogue; if (dialogue) { parts.push(`"${dialogue}"`); } // Extract character description const character = segment.character_description; if (character?.current_state) { parts.push(`Character: ${character.current_state}`); } // Extract scene description const scene = segment.scene_continuity; if (scene?.environment) { parts.push(`Scene: ${scene.environment}`); } if (scene?.lighting_state) { parts.push(`Lighting: ${scene.lighting_state}`); } if (scene?.camera_position) { parts.push(`Camera: ${scene.camera_position}`); } if (scene?.camera_movement) { parts.push(`Movement: ${scene.camera_movement}`); } // Extract synchronized actions const syncedActions = segment.action_timeline?.synchronized_actions; if (syncedActions) { const actionsList = Object.entries(syncedActions) .filter(([, value]) => value) .map(([key, value]) => `${key}: ${value}`) .join('; '); if (actionsList) { parts.push(`Actions: ${actionsList}`); } } // Add instruction to not include captions/subtitles parts.push('Do not include any captions, subtitles, or text overlays in the video'); return parts.join('. '); }; // Main submit handler const handleSubmit = (e: React.FormEvent) => { e.preventDefault(); if (provider === 'kling') { if (generationMode === 'frame-continuity') { handleKlingFrameContinuityFlow(); } else { handleKlingExtendFlow(); } } else { handleReplicateGeneration(); } }; // Handler for reuse mode - skip prompt generation, go directly to video generation const handleReuseGeneration = (e: React.FormEvent) => { e.preventDefault(); if (!imageFile) { alert('Please upload a character image to start video generation'); return; } console.log(`πŸš€ Starting video generation with ${segments.length} pre-loaded segments (REUSE MODE)`); // Skip prompt generation entirely - go directly to video generation if (provider === 'kling') { if (generationMode === 'frame-continuity') { // Start from the video generation part of the flow setIsGenerating(true); setError(null); startGeneration(segments); handleKlingFrameContinuityFlow(0); } else { setIsGenerating(true); setError(null); startGeneration(segments); handleKlingExtendFlow(0); } } else { setIsGenerating(true); setError(null); startGeneration(segments); handleReplicateGeneration(0); } }; const isValid = provider === 'kling' ? !!imageFile && (isReuseMode || formState.script.trim().length > 0) : (isReuseMode || formState.script.trim().length > 0); return ( {/* Header */}

{provider === 'kling' ? 'KIE API' : 'Replicate'} Video Generation

{isReuseMode ? ( ♻️ Reusing {segments.length} cached segments - Upload image and click Generate to start ) : ( provider === 'kling' ? ( kieVideoFlow === 'product_ugc' ? 'Product-focused UGC on KIE β€” prompts emphasize demos, product-in-hand, and authentic selfie-style shots' : 'Generate professional UGC videos with AI-powered segmentation' ) : 'Create unique videos with open-source models' )}

{/* Content Warnings */} {contentWarnings && showWarnings && !isReuseMode && (
⚠️

Content Policy Warnings

{contentWarnings.warnings.map((warning, idx) => (

{warning}

))}
{contentWarnings.guidance && (
View Content Guidelines
Avoid Public Figures:
    {contentWarnings.guidance.avoid_public_figures.map((item, idx) => (
  • {item}
  • ))}
Avoid Copyrighted Content:
    {contentWarnings.guidance.avoid_copyrighted.map((item, idx) => (
  • {item}
  • ))}
Best Practices:
    {contentWarnings.guidance.best_practices.map((item, idx) => (
  • {item}
  • ))}
)}
)} {/* Reuse Mode Banner */} {isReuseMode && (
♻️

Reusing Cached Prompts

{segments.length} segments are pre-loaded from your saved prompt. Prompt generation will be skipped.

Simply upload your character image and click "Generate Video" to start creating videos immediately!

)} {/* Draft Restored Notification */} {draftRestored && (

Draft restored

Your previous inputs have been loaded

)}
{/* Left Column - Script & Style */}
{/* Script Input - Hidden in reuse mode */} {!isReuseMode && (