Image-to-Video
Diffusers
LTX-2
text-to-video
video-to-video
image-text-to-video
audio-to-video
text-to-audio
video-to-audio
audio-to-audio
text-to-audio-video
image-to-audio-video
image-text-to-audio-video
ltx-2
ltx-video
ltxv
lightricks
ltx-2.3
Eval Results
Instructions to use Lightricks/LTX-2.3 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use Lightricks/LTX-2.3 with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline from diffusers.utils import load_image, export_to_video # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("Lightricks/LTX-2.3", dtype=torch.bfloat16, device_map="cuda") pipe.to("cuda") prompt = "A man with short gray hair plays a red electric guitar." image = load_image( "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/guitar-man.png" ) output = pipe(image=image, prompt=prompt).frames[0] export_to_video(output, "output.mp4") - LTX-2
How to use Lightricks/LTX-2.3 with LTX-2:
# Install the LTX-2 pipelines git clone https://github.com/Lightricks/LTX-2.git cd LTX-2 uv sync --frozen
# Download the weights from this repo, plus the Gemma text encoder hf download Lightricks/LTX-2.3 --local-dir models/LTX-2.3 hf download google/gemma-3-12b-it-qat-q4_0-unquantized --local-dir models/gemma-3-12b
# Fast pipeline (distilled model, no distilled LoRA needed) uv run python -m ltx_pipelines.distilled \ --distilled-checkpoint-path models/LTX-2.3/<distilled-checkpoint>.safetensors \ --spatial-upsampler-path models/LTX-2.3/<spatial-upsampler>.safetensors \ --gemma-root models/gemma-3-12b \ --prompt "A beautiful sunset over the ocean" \ --output-path output.mp4 # For image-to-video, add: --image path/to/image.jpg 0 0.8# HQ pipeline (two-stage, higher quality) uv run python -m ltx_pipelines.ti2vid_two_stages_hq \ --checkpoint-path models/LTX-2.3/<checkpoint>.safetensors \ --distilled-lora models/LTX-2.3/<distilled-lora>.safetensors 0.8 \ --spatial-upsampler-path models/LTX-2.3/<spatial-upsampler>.safetensors \ --gemma-root models/gemma-3-12b \ --prompt "A beautiful sunset over the ocean" \ --output-path output.mp4 # For image-to-video, add: --image path/to/image.jpg 0 0.8 - Notebooks
- Google Colab
- Kaggle
<!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <title>OX AI - Video Studio</title> <script src="https://tailwindcss.com"></script> </head> <body class="bg-gray-950 text-gray-100 min-h-screen font-sans"> <!-- Navbar --> <header class="border-b border-gray-800 bg-gray-900 px-6 py-4 flex justify-between items-center"> <h1 class="text-2xl font-bold tracking-wider text-purple-400">OX AI <span class="text-xs text-gray-400 font-normal">v1.0 Studio</span></h1> <div class="flex items-center gap-4"> <span class="text-sm bg-gray-800 px-3 py-1.5 rounded-full text-purple-300 font-semibold" id="creditDisplay">Credits: 500</span> </div> </header> <!-- Main Workspace --> <main class="grid grid-cols-1 lg:grid-cols-3 gap-6 p-6 max-w-7xl mx-auto"> <!-- Left & Center: Configuration & Generation --> <div class="lg:col-span-2 space-y-6"> <!-- Prompt Settings --> <div class="bg-gray-900 border border-gray-800 rounded-xl p-5 space-y-4"> <label class="block text-sm font-medium text-gray-400">Text Prompt (High Accuracy Physics Engine)</label> <textarea id="prompt" rows="3" class="w-full bg-gray-950 border border-gray-700 rounded-lg p-3 text-white focus:ring-2 focus:ring-purple-500 focus:outline-none placeholder-gray-600" placeholder="Cinematic shot, cinematic lighting, continuous motion, flawless physics..."></textarea> <!-- Aspect Ratio & Resolution --> <div class="grid grid-cols-2 gap-4"> <div> <label class="block text-xs font-medium text-gray-400 mb-1">Aspect Ratio</label> <select id="aspectRatio" class="w-full bg-gray-950 border border-gray-700 rounded-lg p-2.5 text-white"> <option value="16:9">Widescreen (16:9)</option> <option value="9:16">Vertical Social (9:16)</option> <option value="1:1">Square (1:1)</option> </select> </div> <div> <label class="block text-xs font-medium text-gray-400 mb-1">Target Resolution</label> <select id="resolution" class="w-full bg-gray-950 border border-gray-700 rounded-lg p-2.5 text-white"> <option value="1080p">1080p Full HD</option> <option value="4k">4K Ultra HD (Premium)</option> </select> </div> </div> </div> <!-- Advanced Controls (Consistency & Native Audio) --> <div class="bg-gray-900 border border-gray-800 rounded-xl p-5 space-y-4"> <h3 class="text-sm font-semibold text-purple-400">Creative Control Overlays</h3> <div class="grid grid-cols-1 md:grid-cols-2 gap-4"> <div> <label class="block text-xs text-gray-400 mb-1">Character & Scene Consistency Token</label> <input type="text" id="characterToken" placeholder="e.g., actor_id_492" class="w-full bg-gray-950 border border-gray-700 rounded-lg p-2 text-white text-sm"> </div> <div> <label class="block text-xs text-gray-400 mb-1">Motion Intensity (Physics Anchor)</label> <input type="range" id="motionScale" min="1" max="10" value="5" class="w-full h-2 bg-gray-700 rounded-lg appearance-none cursor-pointer accent-purple-500 mt-3"> </div> </div> <div class="flex items-center gap-6 pt-2"> <label class="flex items-center gap-2 text-sm text-gray-300"> <input type="checkbox" id="nativeAudio" class="rounded bg-gray-950 border-gray-700 text-purple-600 focus:ring-purple-500"> Generate Native Audio & Lip-Sync </label> <label class="flex items-center gap-2 text-sm text-gray-300"> <input type="checkbox" id="commercialSafety" checked disabled class="rounded bg-gray-950 border-gray-700 text-purple-600"> Commercial Safety Guarantee </label> </div> </div> <!-- Action Button --> <button onclick="generateVideo()" id="generateBtn" class="w-full bg-purple-600 hover:bg-purple-700 text-white font-bold py-3.5 px-6 rounded-xl transition duration-200 shadow-lg shadow-purple-900/30"> Render Video Asset </button> </div> <!-- Right Side: Real-time Output & Rendering Monitoring --> <div class="bg-gray-900 border border-gray-800 rounded-xl p-5 flex flex-col justify-between min-h-[400px]"> <div> <h2 class="text-lg font-bold text-gray-200 mb-4">Live Preview Pipeline</h2> <div id="videoContainer" class="w-full aspect-video bg-gray-950 rounded-lg flex items-center justify-center border border-gray-800 relative overflow-hidden"> <div id="statusPlaceholder" class="text-gray-500 text-sm text-center px-4"> <p>No video rendered yet.</p> <p class="text-xs text-gray-600 mt-1">Configure parameters and hit generate.</p> </div> <!-- Real Video Element hidden initially --> <video id="outputVideo" controls class="w-full h-full object-cover hidden"></video> <!-- Loading Mask --> <div id="loader" class="absolute inset-0 bg-gray-950/80 hidden flex-col items-center justify-center gap-3"> <div class="w-10 h-10 border-4 border-purple-500 border-t-transparent rounded-full animate-spin"></div> <p class="text-sm text-purple-400 font-medium animate-pulse" id="progressText">Simulating Physics Engine...</p> </div> </div> </div> <div class="border-t border-gray-800 pt-4 mt-4 space-y-2 text-xs text-gray-400"> <div class="flex justify-between"><span>Render Cost:</span><span id="costEstimation" class="text-white">10 Credits</span></div> <div class="flex justify-between"><span>Licensing Status:</span><span class="text-green-400 font-medium">100% Globally Cleared</span></div> </div> </div> </main> <script> async function generateVideo() { const btn = document.getElementById('generateBtn'); const loader = document.getElementById('loader'); const videoElement = document.getElementById('outputVideo'); const placeholder = document.getElementById('statusPlaceholder'); btn.disabled = true; loader.classList.remove('hidden'); placeholder.classList.add('hidden'); videoElement.classList.add('hidden'); const payload = { prompt: document.getElementById('prompt').value, aspect_ratio: document.getElementById('aspectRatio').value, resolution: document.getElementById('resolution').value, character_token: document.getElementById('characterToken').value, motion_intensity: parseInt(document.getElementById('motionScale').value), native_audio: document.getElementById('nativeAudio').checked }; try { const response = await fetch('http://127.0.0', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }); const data = await response.json(); if(data.success) { document.getElementById('creditDisplay').innerText = `Credits: ${data.remaining_credits}`; videoElement.src = data.video_url; videoElement.classList.remove('hidden'); } else { alert("Error: " + data.message); placeholder.classList.remove('hidden'); } } catch (error) { console.error("Pipeline failure:", error); placeholder.classList.remove('hidden'); } finally { btn.disabled = false; loader.classList.add('hidden'); } } </script> </body> </html>
#74
by UserOXAI - opened
Xonai
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import time
|
| 2 |
+
from fastapi import FastAPI, HTTPException
|
| 3 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 4 |
+
from pydantic import BaseModel, Field
|
| 5 |
+
|
| 6 |
+
app = FastAPI(title="OX AI Engine Core")
|
| 7 |
+
|
| 8 |
+
# Configure CORS for local frontend testing
|
| 9 |
+
app.add_middleware(
|
| 10 |
+
CORSMiddleware,
|
| 11 |
+
allow_origins=["*"],
|
| 12 |
+
allow_credentials=True,
|
| 13 |
+
allow_methods=["*"],
|
| 14 |
+
allow_headers=["*"],
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
# Mock in-memory user tracking database
|
| 18 |
+
USER_WALLET = {"available_credits": 500}
|
| 19 |
+
|
| 20 |
+
class VideoGenerationRequest(BaseModel):
|
| 21 |
+
prompt: str = Field(..., example="Cinematic tracking shot of an astronaut navigating flowing neon rivers.")
|
| 22 |
+
aspect_ratio: str = Field("16:9", example="16:9")
|
| 23 |
+
resolution: str = Field("1080p", example="1080p")
|
| 24 |
+
character_token: str = Field(None, description="Maintains face/asset identity signatures across generations")
|
| 25 |
+
motion_intensity: int = Field(5, ge=1, le=10)
|
| 26 |
+
native_audio: bool = Field(False)
|
| 27 |
+
|
| 28 |
+
@app.post("/api/generate-video")
|
| 29 |
+
async def process_video_generation(request: VideoGenerationRequest):
|
| 30 |
+
if not request.prompt.strip():
|
| 31 |
+
raise HTTPException(status_code=400, detail="Generation prompt cannot be empty.")
|
| 32 |
+
|
| 33 |
+
# Calculate explicit cost matrices based on operational footprint
|
| 34 |
+
base_cost = 10
|
| 35 |
+
resolution_multiplier = 2.5 if request.resolution == "4k" else 1.0
|
| 36 |
+
audio_addon = 5 if request.native_audio else 0
|
| 37 |
+
total_cost = int((base_cost * resolution_multiplier) + audio_addon)
|
| 38 |
+
|
| 39 |
+
if USER_WALLET["available_credits"] < total_cost:
|
| 40 |
+
return {"success": False, "message": "Insufficient rendering credits inside user profile."}
|
| 41 |
+
|
| 42 |
+
# Deduct credits transparently
|
| 43 |
+
USER_WALLET["available_credits"] -= total_cost
|
| 44 |
+
|
| 45 |
+
# Simulated AI Model Generation Time Pipeline (Diffusion step calculations)
|
| 46 |
+
# Inside a live environment, you would call requests.post("https://openai.com", ...) here
|
| 47 |
+
time.sleep(3.5)
|
| 48 |
+
|
| 49 |
+
# Standard dummy placeholder video for visual asset returning
|
| 50 |
+
simulated_output_url = "https://googleapis.com"
|
| 51 |
+
|
| 52 |
+
return {
|
| 53 |
+
"success": True,
|
| 54 |
+
"job_status": "COMPLETED",
|
| 55 |
+
"video_url": simulated_output_url,
|
| 56 |
+
"credits_deducted": total_cost,
|
| 57 |
+
"remaining_credits": USER_WALLET["available_credits"],
|
| 58 |
+
"metadata": {
|
| 59 |
+
"commercial_clearance": "VERIFIED_PASSED",
|
| 60 |
+
"resolution_rendered": request.resolution,
|
| 61 |
+
"consistency_token_applied": request.character_token or "NONE",
|
| 62 |
+
"physics_stabilizer_index": f"v1_intensity_{request.motion_intensity}"
|
| 63 |
+
}
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
if __name__ == "__main__":
|
| 67 |
+
import uvicorn
|
| 68 |
+
uvicorn.run(app, host="127.0.0.1", port=8000)
|