sushilideaclan01's picture
Enhance prompt validation and safety features
82a1419
Raw History Blame Contribute Delete
24.1 kB
import type {
VideoStatusResponse,
SegmentsPayload,
HealthStatus,
ExtractedFrame,
LoginRequest,
LoginResponse,
AuthUser,
GenerationStep
} from '@/types';
// ==================== RETRY CONSTANTS ====================
export const RETRY_DELAY_MS = 2000;
export const MAX_VIDEO_ATTEMPTS = 2;
/** Max attempts before skipping auto-fix and using flow retry (attemptCount 0, 1 = 2 tries). */
export const AUTO_FIX_MAX_ATTEMPTS = 3;
const API_BASE = import.meta.env.VITE_API_BASE_URL || 'http://localhost:4000';
// Get auth token from localStorage
function getAuthToken(): string | null {
return localStorage.getItem('auth_token');
}
// Set auth token in localStorage
export function setAuthToken(token: string): void {
localStorage.setItem('auth_token', token);
}
// Remove auth token from localStorage
export function removeAuthToken(): void {
localStorage.removeItem('auth_token');
}
function getStatusBasedErrorMessage(status: number): string {
if (status === 401) return 'Incorrect username or password.';
if (status === 403) return 'Access forbidden.';
if (status === 404) return 'Resource not found.';
if (status >= 500) return 'Server error. Please try again later.';
return `Request failed with status ${status}`;
}
/** Parse error message from a non-ok fetch response (JSON or text). */
async function parseFetchError(response: Response, defaultMessage: string): Promise<string> {
const contentType = response.headers.get('content-type');
const isJson = contentType?.includes('application/json');
try {
if (isJson) {
const errorData = await response.json();
return errorData.detail || errorData.message || errorData.error ||
(typeof errorData === 'string' ? errorData : defaultMessage);
}
const text = await response.text();
return text?.trim() ? text : getStatusBasedErrorMessage(response.status);
} catch {
return getStatusBasedErrorMessage(response.status);
}
}
/** POST JSON to API with auth. */
async function apiPostJson<T>(path: string, body: object): Promise<T> {
return apiRequest<T>(path, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
});
}
// Generic API request handler
async function apiRequest<T>(
path: string,
options: RequestInit = {}
): Promise<T> {
const url = path.startsWith('http') ? path : `${API_BASE}${path}`;
const token = getAuthToken();
// Normalize headers to a plain object
const headers: Record<string, string> = {};
// Convert Headers object or array to plain object
if (options.headers) {
if (options.headers instanceof Headers) {
options.headers.forEach((value, key) => {
headers[key] = value;
});
} else if (Array.isArray(options.headers)) {
options.headers.forEach(([key, value]) => {
headers[key] = value;
});
} else {
Object.assign(headers, options.headers);
}
}
// Add auth token if available
if (token) {
headers['Authorization'] = `Bearer ${token}`;
}
const response = await fetch(url, {
...options,
headers,
});
if (!response.ok) {
if (response.status === 401) removeAuthToken();
const errorMessage = await parseFetchError(response, `Request failed with status ${response.status}`);
console.error('API Error:', {
status: response.status,
statusText: response.statusText,
message: errorMessage,
url,
contentType: response.headers.get('content-type'),
});
throw new Error(errorMessage);
}
return response.json();
}
// Health check
export async function checkHealth(): Promise<HealthStatus> {
return apiRequest<HealthStatus>('/health');
}
// ==================== AUTHENTICATION ====================
export async function login(credentials: LoginRequest): Promise<LoginResponse> {
const response = await apiRequest<LoginResponse>('/api/auth/login', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(credentials),
});
// Store token
if (response.access_token) {
setAuthToken(response.access_token);
}
return response;
}
export async function verifyAuth(): Promise<AuthUser> {
try {
const response = await apiRequest<AuthUser>('/api/auth/verify');
return response;
} catch {
removeAuthToken();
return { username: '', authenticated: false };
}
}
export async function getCurrentUser(): Promise<AuthUser> {
try {
const response = await apiRequest<AuthUser>('/api/auth/me');
return response;
} catch {
removeAuthToken();
return { username: '', authenticated: false };
}
}
export function logout(): void {
removeAuthToken();
}
// ==================== KLING/KIE API ====================
export interface KlingGenerateParams {
prompt: string | object;
imageUrls?: string[];
model?: string;
aspectRatio?: string;
generationType?: string;
seeds?: number;
voiceType?: string;
}
export interface KlingGenerateResponse {
taskId: string;
status: string;
}
export async function klingGenerate(params: KlingGenerateParams): Promise<KlingGenerateResponse> {
return apiPostJson<KlingGenerateResponse>('/api/veo/generate', params);
}
export async function klingExtend(taskId: string, prompt: string | object, seeds?: number, voiceType?: string): Promise<KlingGenerateResponse> {
return apiPostJson<KlingGenerateResponse>('/api/veo/extend', { taskId, prompt, seeds, voiceType });
}
export async function klingGetStatus(taskId: string): Promise<VideoStatusResponse> {
return apiRequest<VideoStatusResponse>(`/api/veo/status/${taskId}`);
}
export async function klingCancel(taskId: string): Promise<{ code: number; msg: string; taskId: string }> {
return apiRequest<{ code: number; msg: string; taskId: string }>(`/api/veo/cancel/${taskId}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
});
}
export function createKlingEventSource(taskId: string): EventSource {
return new EventSource(`${API_BASE}/api/veo/events/${taskId}`);
}
// ==================== REPLICATE API ====================
export interface ReplicateGenerateParams {
prompt: string;
imageUrl?: string;
model?: string;
duration?: number;
aspectRatio?: string;
}
export interface ReplicateGenerateResponse {
id: string;
status: string;
}
export async function replicateGenerate(params: ReplicateGenerateParams): Promise<ReplicateGenerateResponse> {
return apiPostJson<ReplicateGenerateResponse>('/api/replicate/generate', params);
}
export async function replicateGetStatus(predictionId: string): Promise<VideoStatusResponse> {
return apiRequest<VideoStatusResponse>(`/api/replicate/status/${predictionId}`);
}
// Wait for Replicate video completion by polling
export async function waitForReplicateVideo(
predictionId: string,
timeoutMs: number = 600000, // 10 minutes
pollIntervalMs: number = 15000 // Poll every 15 seconds (video gen takes 2-5 min)
): Promise<string> {
const startTime = Date.now();
while (Date.now() - startTime < timeoutMs) {
const status = await replicateGetStatus(predictionId);
if (status.status === 'succeeded' && status.url) {
return status.url;
} else if (status.status === 'failed') {
throw new Error(status.error || 'Replicate video generation failed');
}
// Wait before next poll
await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
}
throw new Error('Replicate video generation timed out');
}
// ==================== PROMPT GENERATION ====================
// Stream event types
export interface StreamEvent {
event: 'start' | 'segment' | 'complete' | 'error';
[key: string]: any;
}
export interface StreamStartEvent extends StreamEvent {
event: 'start';
total_segments: number;
model: string;
}
export interface StreamSegmentEvent extends StreamEvent {
event: 'segment';
index: number;
total: number;
progress: number;
segment: any;
}
export interface StreamCompleteEvent extends StreamEvent {
event: 'complete';
message: string;
prompt_id: string;
payload: SegmentsPayload;
}
export interface StreamErrorEvent extends StreamEvent {
event: 'error';
message: string;
error_type: string;
}
export interface GeneratePromptsStreamingOptions {
signal?: AbortSignal;
}
/**
* Generate prompts with streaming - segments arrive in real-time
*
* @param formData - Form data with script, style, image, etc.
* @param onEvent - Callback for each stream event
* @param options - Optional { signal } for cancellation
* @returns Promise that resolves with complete payload
*/
export async function generatePromptsStreaming(
formData: FormData,
onEvent: (event: StreamEvent) => void,
options?: GeneratePromptsStreamingOptions
): Promise<SegmentsPayload> {
const response = await fetch(`${API_BASE}/api/generate-prompts-stream`, {
method: 'POST',
body: formData,
signal: options?.signal,
});
if (!response.ok) {
throw new Error(await parseFetchError(response, 'Failed to generate prompts'));
}
if (!response.body) {
throw new Error('No response body');
}
const reader = response.body.getReader();
const decoder = new TextDecoder();
let buffer = '';
let finalPayload: SegmentsPayload | null = null;
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (options?.signal?.aborted) {
reader.releaseLock();
throw new Error('Generation cancelled by user');
}
// Decode chunk and add to buffer
buffer += decoder.decode(value, { stream: true });
// Split by newlines (NDJSON format)
const lines = buffer.split('\n');
buffer = lines.pop() || ''; // Keep incomplete line in buffer
// Process each complete line
for (const line of lines) {
if (!line.trim()) continue;
try {
const event = JSON.parse(line) as StreamEvent;
// Call event callback
onEvent(event);
// Store final payload from complete event
if (event.event === 'complete') {
finalPayload = (event as StreamCompleteEvent).payload;
}
// If error event, throw
if (event.event === 'error') {
throw new Error((event as StreamErrorEvent).message);
}
} catch (parseError) {
console.error('Failed to parse stream event:', parseError, 'Line:', line);
// Continue processing other events
}
}
}
} finally {
reader.releaseLock();
}
if (!finalPayload) {
throw new Error('Stream completed without final payload');
}
return finalPayload;
}
export async function refinePromptContinuity(
segmentPrompt: object,
lastFrameBlob: Blob
): Promise<{ refined_prompt: object }> {
const formData = new FormData();
formData.append('segmentPrompt', JSON.stringify(segmentPrompt));
formData.append('lastFrame', lastFrameBlob, 'last-frame.jpg');
return apiRequest<{ refined_prompt: object }>('/api/refine-prompt-continuity', {
method: 'POST',
body: formData,
});
}
// ==================== IMAGE UPLOAD ====================
/** Use reference: true when uploading a last-frame for continuity (keeps higher quality). */
export async function uploadImage(
file: File,
options?: { reference?: boolean }
): Promise<{ url: string; filename: string }> {
const formData = new FormData();
formData.append('file', file);
const path = options?.reference ? '/api/upload-image?reference=true' : '/api/upload-image';
return apiRequest<{ url: string; filename: string }>(path, {
method: 'POST',
body: formData,
});
}
// ==================== FRAME EXTRACTION ====================
export interface ExtractFramesParams {
video_url: string;
script?: string;
buffer_time?: number;
num_frames?: number;
model_size?: string;
}
export interface ExtractFramesResponse {
frames: ExtractedFrame[];
}
export async function extractFrames(params: ExtractFramesParams): Promise<ExtractFramesResponse> {
return apiPostJson<ExtractFramesResponse>('/api/extract-frames', params);
}
// ==================== WHISPER ANALYSIS ====================
export interface WhisperAnalyzeParams {
video_url: string;
dialogue: string;
buffer_time?: number;
model_size?: string;
}
export interface WhisperAnalyzeResponse {
success: boolean;
last_word_timestamp: number | null;
trim_point: number | null;
frame_timestamp: number | null;
frame_base64: string | null;
video_duration: number;
transcribed_text: string | null; // What Whisper actually heard - for prompt refinement
error: string | null;
}
/**
* Analyze video with Whisper to find last spoken word and extract frame.
* This is the optimized flow that combines Whisper analysis and frame extraction.
*/
export async function whisperAnalyzeAndExtract(params: WhisperAnalyzeParams): Promise<WhisperAnalyzeResponse> {
return apiPostJson<WhisperAnalyzeResponse>('/api/whisper/analyze-and-extract', params);
}
/**
* Refine a segment prompt using the frame AND transcription from the previous segment.
* This ensures perfect visual and audio continuity.
*/
export async function refinePromptWithContext(
segmentPrompt: object,
frameFile: File,
transcribedDialogue: string,
expectedDialogue: string
): Promise<{ refined_prompt: object; original_prompt: object }> {
const formData = new FormData();
formData.append('segmentPrompt', JSON.stringify(segmentPrompt));
formData.append('lastFrame', frameFile);
formData.append('transcribedDialogue', transcribedDialogue);
formData.append('expectedDialogue', expectedDialogue);
return apiRequest<{ refined_prompt: object; original_prompt: object }>('/api/refine-prompt-continuity', {
method: 'POST',
body: formData,
});
}
/**
* Check if Whisper is available on the backend
*/
export async function checkWhisperStatus(): Promise<{ available: boolean; message: string }> {
return apiRequest<{ available: boolean; message: string }>('/api/whisper/status');
}
// ==================== VIDEO DOWNLOAD ====================
export async function downloadVideo(url: string): Promise<Blob> {
const response = await fetch(`${API_BASE}/api/veo/download?url=${encodeURIComponent(url)}`);
if (!response.ok) {
throw new Error(`Failed to download video: ${response.status}`);
}
return response.blob();
}
// ==================== UTILITIES ====================
/** Load video from file; caller must revoke object URL when done. */
function loadVideoFromFile(file: File): Promise<HTMLVideoElement> {
return new Promise((resolve, reject) => {
const video = document.createElement('video');
video.preload = 'metadata';
video.src = URL.createObjectURL(file);
video.onloadedmetadata = () => resolve(video);
video.onerror = () => {
URL.revokeObjectURL(video.src);
reject(new Error('Failed to load video metadata'));
};
});
}
export async function getVideoDuration(file: File): Promise<number> {
const video = await loadVideoFromFile(file);
try {
return video.duration;
} finally {
URL.revokeObjectURL(video.src);
}
}
export async function generateThumbnails(file: File, count: number = 5): Promise<string[]> {
const video = await loadVideoFromFile(file);
video.muted = true;
try {
const ctx = document.createElement('canvas').getContext('2d');
if (!ctx) throw new Error('Could not get canvas context');
const maxSize = 400;
const videoWidth = video.videoWidth || 1080;
const videoHeight = video.videoHeight || 1920;
const scale = Math.min(maxSize / videoWidth, maxSize / videoHeight);
const w = Math.round(videoWidth * scale);
const h = Math.round(videoHeight * scale);
ctx.canvas.width = w;
ctx.canvas.height = h;
const duration = video.duration;
const thumbnails: string[] = [];
for (let i = 0; i < count; i++) {
video.currentTime = (duration / count) * i;
await new Promise<void>((res) => { video.onseeked = () => res(); });
ctx.drawImage(video, 0, 0, w, h);
thumbnails.push(ctx.canvas.toDataURL('image/jpeg', 0.85));
}
return thumbnails;
} finally {
URL.revokeObjectURL(video.src);
}
}
// Wait for video completion using SSE
export function waitForKlingVideo(taskId: string, timeoutMs: number = 300000): Promise<string> {
return new Promise((resolve, reject) => {
const eventSource = createKlingEventSource(taskId);
const startTime = Date.now();
const timeout = setTimeout(() => {
eventSource.close();
reject(new Error('Video generation timed out'));
}, timeoutMs);
eventSource.onmessage = (event) => {
try {
const data = JSON.parse(event.data);
if (data.status === 'succeeded' && data.url) {
clearTimeout(timeout);
eventSource.close();
resolve(data.url);
} else if (data.status === 'cancelled') {
clearTimeout(timeout);
eventSource.close();
reject(new Error('Video generation cancelled by user'));
} else if (data.status === 'failed' || data.code !== undefined && data.code !== 200) {
clearTimeout(timeout);
eventSource.close();
const errorMsg = data.error || data.msg || `Video generation failed (code: ${data.code || 'unknown'})`;
reject(new Error(errorMsg));
}
} catch (err) {
console.error('Failed to parse SSE data:', err);
// If we can't parse, don't reject immediately - might be a partial message
}
};
eventSource.onerror = () => {
// If connection error and we've been waiting a while, reject
const elapsed = Date.now() - startTime;
if (elapsed > 5000) { // Wait at least 5 seconds before rejecting on connection error
clearTimeout(timeout);
eventSource.close();
reject(new Error('SSE connection error - video generation may have failed'));
}
};
});
}
// Generate video with automatic retry (retries once on failure)
export async function generateVideoWithRetry(
generateFn: () => Promise<{ taskId: string }>,
timeoutMs: number = 300000,
onRetry?: (attempt: number) => void
): Promise<string> {
let lastError: Error | null = null;
for (let attempt = 0; attempt < MAX_VIDEO_ATTEMPTS; attempt++) {
try {
if (attempt > 0) {
console.log(`🔄 Retrying video generation (attempt ${attempt + 1}/${MAX_VIDEO_ATTEMPTS})...`);
if (onRetry) {
onRetry(attempt + 1);
}
}
const result = await generateFn();
const videoUrl = await waitForKlingVideo(result.taskId, timeoutMs);
return videoUrl;
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
console.error(`❌ Video generation attempt ${attempt + 1} failed:`, lastError.message);
if (attempt === 0) {
console.log(`⏳ Waiting ${RETRY_DELAY_MS / 1000} seconds before retry...`);
await new Promise(resolve => setTimeout(resolve, RETRY_DELAY_MS));
continue;
}
throw lastError;
}
}
throw lastError || new Error('Video generation failed');
}
/** Outcome of flow-level retry: either trigger retry or state was updated (partial/error). */
export type FlowRetryOutcome = 'retry' | 'handled';
/** Shared flow-level retry: on cancel show error; on first failure return 'retry'; else show partial or error. */
export async function handleFlowRetry(options: {
attemptCount: number;
errorMessage: string;
isCancelled: boolean;
generatedCount: number;
totalCount: number;
setError: (s: string) => void;
setStep: (s: GenerationStep) => void;
setPartialCompletionError: (s: string | null) => void;
}): Promise<FlowRetryOutcome> {
if (options.isCancelled) {
options.setError('Generation cancelled by user');
options.setStep('error');
return 'handled';
}
if (options.attemptCount === 0) {
await new Promise(resolve => setTimeout(resolve, RETRY_DELAY_MS));
return 'retry';
}
if (options.generatedCount > 0) {
options.setPartialCompletionError(
`Generation stopped at segment ${options.generatedCount + 1} of ${options.totalCount}. Reason: ${options.errorMessage}`
);
options.setStep('completed');
} else {
options.setError(options.errorMessage);
options.setStep('error');
}
return 'handled';
}
// ==================== CONTENT VALIDATION ====================
export interface ContentValidationRequest {
script: string;
}
export interface ContentValidationResponse {
is_valid: boolean;
warnings: string[];
guidance?: {
avoid_public_figures: string[];
avoid_copyrighted: string[];
avoid_sensitive: string[];
best_practices: string[];
} | null;
}
/**
* Validate script content before generation to catch potential issues early
* Checks for public figures, copyrighted content, and other policy violations
*/
export async function validateContent(script: string): Promise<ContentValidationResponse> {
return apiPostJson<ContentValidationResponse>('/api/validate-content', { script });
}
/**
* Get content policy guidance
*/
export async function getContentGuidance() {
return apiRequest('/api/content-guidance');
}
// ==================== PROMPT SAFETY ====================
export interface SafetyFixRequest {
segment: any; // VeoSegment
error_message: string;
attempt_count?: number;
}
export interface SafetyFixResponse {
success: boolean;
fixed_segment: any | null;
changes_made: string | null;
error: string | null;
}
/**
* Automatically fix an unsafe prompt using AI
* Detects content that triggered safety filters and modifies it to be compliant
*/
export async function fixUnsafePrompt(request: SafetyFixRequest): Promise<SafetyFixResponse> {
return apiPostJson<SafetyFixResponse>('/api/safety/fix-unsafe-prompt', request);
}
/**
* Check if error is a Veo 3.1 safety/content policy violation
* Based on official Veo 3.1 API documentation
*
* Official error types:
* - PUBLIC_ERROR_MINOR: Generic internal error (wait and retry)
* - Support codes (8-digit): Specific safety categories
* - "Flagged for containing..." errors: Public figures, unsafe content
*/
export function isUnsafeSegmentError(errorMessage: string): boolean {
const upperError = errorMessage.toUpperCase();
// Official Veo 3.1 safety patterns from documentation
const safetyKeywords = [
'UNSAFE',
'CONTENT_POLICY',
'MODERATION',
'INAPPROPRIATE',
'VIOLATION',
'PROHIBITED',
'SAFETY',
'BLOCKED',
// Public figures (MOST COMMON!)
'PROMINENT PUBLIC FIGURE',
'PUBLIC FIGURE',
'CELEBRITY',
'POLITICIAN',
'FLAGGED FOR CONTAINING',
'PROMINENT PUBLIC',
];
// Check for safety keywords
const hasKeyword = safetyKeywords.some(keyword => upperError.includes(keyword));
// Check for 8-digit support code (indicates safety filter)
// Official Veo 3.1 uses support codes for safety categories
const hasSupportCode = /\b\d{8}\b/.test(errorMessage);
return hasKeyword || hasSupportCode;
}
// ==================== VIDEO MERGE/EXPORT ====================
export interface ClipMetadata {
index: number;
startTime: number;
endTime: number;
type: 'video' | 'image';
duration?: number;
}
// Merge multiple video files into a single video
export async function mergeVideos(
videoBlobs: Blob[],
clipMetadata: ClipMetadata[]
): Promise<Blob> {
const formData = new FormData();
// Add clip metadata as JSON
formData.append('clips_data', JSON.stringify(clipMetadata));
// Add video files
videoBlobs.forEach((blob, index) => {
formData.append('files', blob, `video_${index}.mp4`);
});
const response = await fetch(`${API_BASE}/api/export/merge`, {
method: 'POST',
body: formData,
});
if (!response.ok) {
throw new Error(await parseFetchError(response, 'Failed to merge videos'));
}
return response.blob();
}