Instructions to use mzx/NEvo with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use mzx/NEvo with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("mzx/NEvo", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
Split into image_num_generations (50) / video_num_generations (25)
Browse files- README.md +6 -4
- stimulus_synthesis/config.py +4 -2
- stimulus_synthesis/pipeline.py +16 -10
- stimulus_synthesis_config.json +2 -1
- tests/test_roi_search_space.py +2 -2
- tests/test_smoke_pipeline.py +1 -1
README.md
CHANGED
|
@@ -82,7 +82,7 @@ from stimulus_synthesis.media import save_video, video_to_t_c_h_w # importable
|
|
| 82 |
save_video(video_to_t_c_h_w(out.best.video), "best_stimulus.mp4") # save the synthesized video
|
| 83 |
```
|
| 84 |
|
| 85 |
-
This runs the two-stage search with the defaults — a population of 20 evolved for
|
| 86 |
|
| 87 |
### Faster run
|
| 88 |
|
|
@@ -92,7 +92,8 @@ For a quicker first result, shrink the search and the video:
|
|
| 92 |
out = pipe(
|
| 93 |
roi="FFA",
|
| 94 |
progress=True,
|
| 95 |
-
|
|
|
|
| 96 |
population_size=8, # GA population per generation (default: 20)
|
| 97 |
seed=0, # RNG seed, for reproducibility
|
| 98 |
video_kwargs={ # passed straight to LTX-Video; omit to use the model's own (heavier) defaults
|
|
@@ -144,14 +145,15 @@ Set in `stimulus_synthesis_config.json`:
|
|
| 144 |
|
| 145 |
| Param | Default | Notes |
|
| 146 |
|-------|---------|-------|
|
| 147 |
-
| `
|
|
|
|
| 148 |
| `default_population_size` | 20 | GA population per generation (= `n_init`) |
|
| 149 |
| `default_mutation_rate` | 0.25 | |
|
| 150 |
| `default_elite_frac` | 0.35 | |
|
| 151 |
| `default_objective` | `indices_mean` | |
|
| 152 |
| `default_score_transform` | disabled | robust augmentation off by default (clean single pass) |
|
| 153 |
|
| 154 |
-
Each stage evaluates a population of `population_size` (default 20) for `
|
| 155 |
|
| 156 |
### Robust scoring
|
| 157 |
|
|
|
|
| 82 |
save_video(video_to_t_c_h_w(out.best.video), "best_stimulus.mp4") # save the synthesized video
|
| 83 |
```
|
| 84 |
|
| 85 |
+
This runs the two-stage search with the defaults — a population of 20 evolved for 50 generations (image) then 25 (video), at full-resolution LTX video — which is thorough but heavy (a single run can take a while and a lot of GPU memory).
|
| 86 |
|
| 87 |
### Faster run
|
| 88 |
|
|
|
|
| 92 |
out = pipe(
|
| 93 |
roi="FFA",
|
| 94 |
progress=True,
|
| 95 |
+
image_num_generations=4, # stage-1 (image) generations (default: 50)
|
| 96 |
+
video_num_generations=4, # stage-2 (video) generations (default: 25)
|
| 97 |
population_size=8, # GA population per generation (default: 20)
|
| 98 |
seed=0, # RNG seed, for reproducibility
|
| 99 |
video_kwargs={ # passed straight to LTX-Video; omit to use the model's own (heavier) defaults
|
|
|
|
| 145 |
|
| 146 |
| Param | Default | Notes |
|
| 147 |
|-------|---------|-------|
|
| 148 |
+
| `default_image_generations` | 50 | stage-1 (image) GA generations (generation 1 = the random init) |
|
| 149 |
+
| `default_video_generations` | 25 | stage-2 (video) GA generations |
|
| 150 |
| `default_population_size` | 20 | GA population per generation (= `n_init`) |
|
| 151 |
| `default_mutation_rate` | 0.25 | |
|
| 152 |
| `default_elite_frac` | 0.35 | |
|
| 153 |
| `default_objective` | `indices_mean` | |
|
| 154 |
| `default_score_transform` | disabled | robust augmentation off by default (clean single pass) |
|
| 155 |
|
| 156 |
+
Each stage evaluates a population of `population_size` (default 20) for its own number of generations — `image_num_generations` (default 50) and `video_num_generations` (default 25) — so total generate→score work per stage is about `population_size × generations`. Video generation dominates runtime — pass `video_kwargs={"num_inference_steps": 8, "num_frames": 25, ...}` to keep it fast (the distilled LTX model runs in few steps); otherwise LTX's own defaults apply.
|
| 157 |
|
| 158 |
### Robust scoring
|
| 159 |
|
stimulus_synthesis/config.py
CHANGED
|
@@ -15,7 +15,8 @@ class StimulusSynthesisConfig:
|
|
| 15 |
default_objective: str = "indices_mean"
|
| 16 |
default_device: str = "cuda"
|
| 17 |
default_population_size: int = 20
|
| 18 |
-
|
|
|
|
| 19 |
default_score_frames: int = 24
|
| 20 |
default_image_batch_size: int = 16
|
| 21 |
default_video_batch_size: int = 8
|
|
@@ -52,7 +53,8 @@ class StimulusSynthesisConfig:
|
|
| 52 |
"default_objective": self.default_objective,
|
| 53 |
"default_device": self.default_device,
|
| 54 |
"default_population_size": self.default_population_size,
|
| 55 |
-
"
|
|
|
|
| 56 |
"default_score_frames": self.default_score_frames,
|
| 57 |
"default_image_batch_size": self.default_image_batch_size,
|
| 58 |
"default_video_batch_size": self.default_video_batch_size,
|
|
|
|
| 15 |
default_objective: str = "indices_mean"
|
| 16 |
default_device: str = "cuda"
|
| 17 |
default_population_size: int = 20
|
| 18 |
+
default_image_generations: int = 50
|
| 19 |
+
default_video_generations: int = 25
|
| 20 |
default_score_frames: int = 24
|
| 21 |
default_image_batch_size: int = 16
|
| 22 |
default_video_batch_size: int = 8
|
|
|
|
| 53 |
"default_objective": self.default_objective,
|
| 54 |
"default_device": self.default_device,
|
| 55 |
"default_population_size": self.default_population_size,
|
| 56 |
+
"default_image_generations": self.default_image_generations,
|
| 57 |
+
"default_video_generations": self.default_video_generations,
|
| 58 |
"default_score_frames": self.default_score_frames,
|
| 59 |
"default_image_batch_size": self.default_image_batch_size,
|
| 60 |
"default_video_batch_size": self.default_video_batch_size,
|
stimulus_synthesis/pipeline.py
CHANGED
|
@@ -133,7 +133,8 @@ class NevoPipeline(DiffusionPipeline):
|
|
| 133 |
roi,
|
| 134 |
enforce_general_search_space,
|
| 135 |
progress,
|
| 136 |
-
|
|
|
|
| 137 |
image_batch_size,
|
| 138 |
video_batch_size,
|
| 139 |
population_size,
|
|
@@ -155,12 +156,13 @@ class NevoPipeline(DiffusionPipeline):
|
|
| 155 |
cfg = self.synthesis_config
|
| 156 |
space_roi = None if enforce_general_search_space else roi
|
| 157 |
pop = max(2, int(population_size or cfg.default_population_size))
|
| 158 |
-
|
|
|
|
| 159 |
score_frames = num_frames if num_frames is not None else cfg.default_score_frames
|
| 160 |
|
| 161 |
# ---- Stage 1: image prompt search ----
|
| 162 |
image_search = GeneticSearch(
|
| 163 |
-
num_generations=
|
| 164 |
population_size=pop,
|
| 165 |
n_init=pop,
|
| 166 |
mutation_rate=cfg.default_mutation_rate,
|
|
@@ -189,7 +191,7 @@ class NevoPipeline(DiffusionPipeline):
|
|
| 189 |
|
| 190 |
# ---- Stage 2: motion prompt search on the fixed best image ----
|
| 191 |
video_search = GeneticSearch(
|
| 192 |
-
num_generations=
|
| 193 |
population_size=pop,
|
| 194 |
n_init=pop,
|
| 195 |
mutation_rate=cfg.default_mutation_rate,
|
|
@@ -240,7 +242,8 @@ class NevoPipeline(DiffusionPipeline):
|
|
| 240 |
"image_to_video_model_id": cfg.image_to_video_model_id,
|
| 241 |
"objective": cfg.default_objective,
|
| 242 |
"two_stage": True,
|
| 243 |
-
"
|
|
|
|
| 244 |
"population_size": pop,
|
| 245 |
},
|
| 246 |
)
|
|
@@ -255,7 +258,8 @@ class NevoPipeline(DiffusionPipeline):
|
|
| 255 |
progress: bool = False,
|
| 256 |
image_batch_size: int | None = None,
|
| 257 |
video_batch_size: int | None = None,
|
| 258 |
-
|
|
|
|
| 259 |
population_size: int | None = None,
|
| 260 |
seed: int = 0,
|
| 261 |
prompt_banks: dict[str, list[str]] | None = None,
|
|
@@ -278,7 +282,8 @@ class NevoPipeline(DiffusionPipeline):
|
|
| 278 |
img_bs = image_batch_size if image_batch_size is not None else cfg.default_image_batch_size
|
| 279 |
vid_bs = video_batch_size if video_batch_size is not None else cfg.default_video_batch_size
|
| 280 |
pop = max(2, population_size if population_size is not None else cfg.default_population_size)
|
| 281 |
-
|
|
|
|
| 282 |
|
| 283 |
if roi is not None and target is None:
|
| 284 |
from .neuro import resolve_driving_voxels
|
|
@@ -296,7 +301,8 @@ class NevoPipeline(DiffusionPipeline):
|
|
| 296 |
roi=roi,
|
| 297 |
enforce_general_search_space=enforce_general_search_space,
|
| 298 |
progress=progress,
|
| 299 |
-
|
|
|
|
| 300 |
image_batch_size=img_bs,
|
| 301 |
video_batch_size=vid_bs,
|
| 302 |
population_size=pop,
|
|
@@ -320,7 +326,7 @@ class NevoPipeline(DiffusionPipeline):
|
|
| 320 |
seed_prompts=seed_prompts,
|
| 321 |
)
|
| 322 |
search = GeneticSearch(
|
| 323 |
-
num_generations=
|
| 324 |
population_size=pop,
|
| 325 |
n_init=pop,
|
| 326 |
mutation_rate=cfg.default_mutation_rate,
|
|
@@ -358,7 +364,7 @@ class NevoPipeline(DiffusionPipeline):
|
|
| 358 |
"text_to_image_model_id": cfg.text_to_image_model_id,
|
| 359 |
"image_to_video_model_id": cfg.image_to_video_model_id,
|
| 360 |
"objective": cfg.default_objective,
|
| 361 |
-
"num_generations":
|
| 362 |
},
|
| 363 |
)
|
| 364 |
|
|
|
|
| 133 |
roi,
|
| 134 |
enforce_general_search_space,
|
| 135 |
progress,
|
| 136 |
+
image_num_generations,
|
| 137 |
+
video_num_generations,
|
| 138 |
image_batch_size,
|
| 139 |
video_batch_size,
|
| 140 |
population_size,
|
|
|
|
| 156 |
cfg = self.synthesis_config
|
| 157 |
space_roi = None if enforce_general_search_space else roi
|
| 158 |
pop = max(2, int(population_size or cfg.default_population_size))
|
| 159 |
+
image_gens = max(1, int(image_num_generations or cfg.default_image_generations))
|
| 160 |
+
video_gens = max(1, int(video_num_generations or cfg.default_video_generations))
|
| 161 |
score_frames = num_frames if num_frames is not None else cfg.default_score_frames
|
| 162 |
|
| 163 |
# ---- Stage 1: image prompt search ----
|
| 164 |
image_search = GeneticSearch(
|
| 165 |
+
num_generations=image_gens,
|
| 166 |
population_size=pop,
|
| 167 |
n_init=pop,
|
| 168 |
mutation_rate=cfg.default_mutation_rate,
|
|
|
|
| 191 |
|
| 192 |
# ---- Stage 2: motion prompt search on the fixed best image ----
|
| 193 |
video_search = GeneticSearch(
|
| 194 |
+
num_generations=video_gens,
|
| 195 |
population_size=pop,
|
| 196 |
n_init=pop,
|
| 197 |
mutation_rate=cfg.default_mutation_rate,
|
|
|
|
| 242 |
"image_to_video_model_id": cfg.image_to_video_model_id,
|
| 243 |
"objective": cfg.default_objective,
|
| 244 |
"two_stage": True,
|
| 245 |
+
"image_generations": image_gens,
|
| 246 |
+
"video_generations": video_gens,
|
| 247 |
"population_size": pop,
|
| 248 |
},
|
| 249 |
)
|
|
|
|
| 258 |
progress: bool = False,
|
| 259 |
image_batch_size: int | None = None,
|
| 260 |
video_batch_size: int | None = None,
|
| 261 |
+
image_num_generations: int | None = None,
|
| 262 |
+
video_num_generations: int | None = None,
|
| 263 |
population_size: int | None = None,
|
| 264 |
seed: int = 0,
|
| 265 |
prompt_banks: dict[str, list[str]] | None = None,
|
|
|
|
| 282 |
img_bs = image_batch_size if image_batch_size is not None else cfg.default_image_batch_size
|
| 283 |
vid_bs = video_batch_size if video_batch_size is not None else cfg.default_video_batch_size
|
| 284 |
pop = max(2, population_size if population_size is not None else cfg.default_population_size)
|
| 285 |
+
image_gens = max(1, image_num_generations if image_num_generations is not None else cfg.default_image_generations)
|
| 286 |
+
video_gens = max(1, video_num_generations if video_num_generations is not None else cfg.default_video_generations)
|
| 287 |
|
| 288 |
if roi is not None and target is None:
|
| 289 |
from .neuro import resolve_driving_voxels
|
|
|
|
| 301 |
roi=roi,
|
| 302 |
enforce_general_search_space=enforce_general_search_space,
|
| 303 |
progress=progress,
|
| 304 |
+
image_num_generations=image_gens,
|
| 305 |
+
video_num_generations=video_gens,
|
| 306 |
image_batch_size=img_bs,
|
| 307 |
video_batch_size=vid_bs,
|
| 308 |
population_size=pop,
|
|
|
|
| 326 |
seed_prompts=seed_prompts,
|
| 327 |
)
|
| 328 |
search = GeneticSearch(
|
| 329 |
+
num_generations=image_gens,
|
| 330 |
population_size=pop,
|
| 331 |
n_init=pop,
|
| 332 |
mutation_rate=cfg.default_mutation_rate,
|
|
|
|
| 364 |
"text_to_image_model_id": cfg.text_to_image_model_id,
|
| 365 |
"image_to_video_model_id": cfg.image_to_video_model_id,
|
| 366 |
"objective": cfg.default_objective,
|
| 367 |
+
"num_generations": image_gens,
|
| 368 |
},
|
| 369 |
)
|
| 370 |
|
stimulus_synthesis_config.json
CHANGED
|
@@ -19,5 +19,6 @@
|
|
| 19 |
"default_image_batch_size": 16,
|
| 20 |
"default_video_batch_size": 8,
|
| 21 |
"default_population_size": 20,
|
| 22 |
-
"
|
|
|
|
| 23 |
}
|
|
|
|
| 19 |
"default_image_batch_size": 16,
|
| 20 |
"default_video_batch_size": 8,
|
| 21 |
"default_population_size": 20,
|
| 22 |
+
"default_image_generations": 50,
|
| 23 |
+
"default_video_generations": 25
|
| 24 |
}
|
tests/test_roi_search_space.py
CHANGED
|
@@ -43,7 +43,7 @@ def test_no_roi_uses_general_prompt_space():
|
|
| 43 |
|
| 44 |
def test_pipeline_runs_with_roi_and_general_override():
|
| 45 |
pipe = _pipe()
|
| 46 |
-
out = pipe(roi="FFA",
|
| 47 |
assert out.best_prompt and isinstance(out.best_score, float)
|
| 48 |
-
out2 = pipe(roi="FFA", enforce_general_search_space=True,
|
| 49 |
assert out2.best_prompt is not None
|
|
|
|
| 43 |
|
| 44 |
def test_pipeline_runs_with_roi_and_general_override():
|
| 45 |
pipe = _pipe()
|
| 46 |
+
out = pipe(roi="FFA", image_num_generations=2, video_num_generations=2, population_size=2, seed=0, video_size=None)
|
| 47 |
assert out.best_prompt and isinstance(out.best_score, float)
|
| 48 |
+
out2 = pipe(roi="FFA", enforce_general_search_space=True, image_num_generations=2, video_num_generations=2, population_size=2, seed=0, video_size=None)
|
| 49 |
assert out2.best_prompt is not None
|
tests/test_smoke_pipeline.py
CHANGED
|
@@ -41,7 +41,7 @@ def test_pipeline_smoke_with_mock_components():
|
|
| 41 |
search_space=space,
|
| 42 |
)
|
| 43 |
|
| 44 |
-
out = pipe(target=[0],
|
| 45 |
|
| 46 |
assert out.best_prompt
|
| 47 |
assert isinstance(out.best_score, float)
|
|
|
|
| 41 |
search_space=space,
|
| 42 |
)
|
| 43 |
|
| 44 |
+
out = pipe(target=[0], image_num_generations=2, population_size=2, seed=123, video_size=None)
|
| 45 |
|
| 46 |
assert out.best_prompt
|
| 47 |
assert isinstance(out.best_score, float)
|