Update inference solvers and versioning
Browse filesAdd ER-SDE, corrected finite endpoint scheduling, Euler-Maruyama endpoint handling, independent code version provenance, and current-code/older-checkpoint documentation. Existing checkpoint and text-encoder weights are unchanged.
- API.md +19 -9
- README.md +42 -12
- RELEASES.md +22 -12
- canter/__init__.py +3 -0
- canter/inference.py +11 -1
- canter/loading.py +9 -0
- canter/pipeline.py +2 -0
- canter/schedules.py +46 -0
- canter/solvers.py +186 -27
- canter/version.py +4 -0
- canter/webui.py +18 -5
- pyproject.toml +4 -1
API.md
CHANGED
|
@@ -6,6 +6,11 @@
|
|
| 6 |
Canter exposes a high-level image pipeline and a lower-level latent inference
|
| 7 |
engine. Configuration uses frozen dataclasses and enums.
|
| 8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
## Minimal use
|
| 10 |
|
| 11 |
```python
|
|
@@ -56,7 +61,7 @@ pipe = CanterPipeline.from_pretrained(
|
|
| 56 |
| `dtype` | `WeightDType.BFLOAT16` | Weight storage dtype. Compute still uses bfloat16 AMP with explicit float32 operations. |
|
| 57 |
| `text_backend` | `TextAttentionBackend.JAGGED` | Text refinement and cross-attention layout. |
|
| 58 |
| `device` | `"cuda"` | CUDA device string or `torch.device`. |
|
| 59 |
-
| `revision` | `None` | Hugging Face branch, commit, or immutable release tag. `None` uses the package's pinned default
|
| 60 |
| `cache_dir` | `None` | Optional Hugging Face cache directory. |
|
| 61 |
| `compile_model` | `True` | Compile the selected inference kernels. Compilation errors are reported directly. |
|
| 62 |
| `vae` | `None` | Optional compatible `CanterVae` instance. `None` downloads DINAC-AE-D2 automatically. |
|
|
@@ -152,6 +157,7 @@ config = CanterInferenceConfig(
|
|
| 152 |
),
|
| 153 |
self_attention_gain=-0.03,
|
| 154 |
euler_maruyama_multiplier=1.0,
|
|
|
|
| 155 |
seed=42,
|
| 156 |
generator=None,
|
| 157 |
)
|
|
@@ -170,7 +176,8 @@ config = CanterInferenceConfig(
|
|
| 170 |
| `cfg` | disabled | Classifier-free guidance settings. |
|
| 171 |
| `pdg` | full-path PDG 2.5 | Path-drop guidance settings. |
|
| 172 |
| `self_attention_gain` | `-0.03` | Gain applied exclusively to image self-attention on the main denoiser path. |
|
| 173 |
-
| `euler_maruyama_multiplier` | `1.0` | Non-negative stochastic noise multiplier used
|
|
|
|
| 174 |
| `seed` | `42` | Random seed. Set to `None` when supplying `generator`. |
|
| 175 |
| `generator` | `None` | Optional CUDA `torch.Generator` on the same device as the model. Exactly one of `seed` and `generator` is required. |
|
| 176 |
|
|
@@ -182,8 +189,9 @@ operations. The VAE decoder retains its published seed behavior.
|
|
| 182 |
Images downloaded from the Gradio interface contain a `canter` PNG text field
|
| 183 |
with compact JSON. The object begins with the prompt, effective per-image seed,
|
| 184 |
width, height, steps, solver, and schedule. It then records PDG, CFG,
|
| 185 |
-
self-attention gain, logSNR shift, Euler-Maruyama
|
| 186 |
-
and weight
|
|
|
|
| 187 |
|
| 188 |
### Solvers
|
| 189 |
|
|
@@ -191,7 +199,8 @@ and weight dtype.
|
|
| 191 |
| --- | --- | --- |
|
| 192 |
| `Solver.EULER` | `"euler"` | First-order deterministic Euler updates. |
|
| 193 |
| `Solver.EULER_MARUYAMA` | `"euler_maruyama"` | Stochastic reverse-SDE updates. Uses `euler_maruyama_multiplier`. |
|
| 194 |
-
| `Solver.
|
|
|
|
| 195 |
| `Solver.ABM2` | `"abm2"` | Variable-step Adams-Bashforth-Moulton updates with corrected-state reevaluation. |
|
| 196 |
|
| 197 |
ABM2 is the default and performs additional denoiser evaluations for its
|
|
@@ -367,6 +376,7 @@ inference = CanterInferenceConfig(
|
|
| 367 |
),
|
| 368 |
self_attention_gain=-0.03,
|
| 369 |
euler_maruyama_multiplier=1.0,
|
|
|
|
| 370 |
seed=123,
|
| 371 |
generator=None,
|
| 372 |
)
|
|
@@ -431,7 +441,7 @@ The returned latents use float32 and channels-last memory format.
|
|
| 431 |
|
| 432 |
## Pipeline metadata
|
| 433 |
|
| 434 |
-
`pipe.metadata` records the
|
| 435 |
-
digests, text-encoder revision, VAE repository,
|
| 436 |
-
revision. Applications that require reproducibility
|
| 437 |
-
|
|
|
|
| 6 |
Canter exposes a high-level image pipeline and a lower-level latent inference
|
| 7 |
engine. Configuration uses frozen dataclasses and enums.
|
| 8 |
|
| 9 |
+
The installed `canter` package always supplies the inference implementation.
|
| 10 |
+
For remote model IDs, `revision` selects checkpoint artifacts only; selecting
|
| 11 |
+
an older checkpoint does not load or execute the Python package bundled in
|
| 12 |
+
that historical repository snapshot.
|
| 13 |
+
|
| 14 |
## Minimal use
|
| 15 |
|
| 16 |
```python
|
|
|
|
| 61 |
| `dtype` | `WeightDType.BFLOAT16` | Weight storage dtype. Compute still uses bfloat16 AMP with explicit float32 operations. |
|
| 62 |
| `text_backend` | `TextAttentionBackend.JAGGED` | Text refinement and cross-attention layout. |
|
| 63 |
| `device` | `"cuda"` | CUDA device string or `torch.device`. |
|
| 64 |
+
| `revision` | `None` | Hugging Face checkpoint branch, commit, or immutable release tag. `None` uses the installed package's pinned default checkpoint. An explicit tag such as `"v0001"` loads those weights with the currently installed code. |
|
| 65 |
| `cache_dir` | `None` | Optional Hugging Face cache directory. |
|
| 66 |
| `compile_model` | `True` | Compile the selected inference kernels. Compilation errors are reported directly. |
|
| 67 |
| `vae` | `None` | Optional compatible `CanterVae` instance. `None` downloads DINAC-AE-D2 automatically. |
|
|
|
|
| 157 |
),
|
| 158 |
self_attention_gain=-0.03,
|
| 159 |
euler_maruyama_multiplier=1.0,
|
| 160 |
+
er_sde_noise_multiplier=1.0,
|
| 161 |
seed=42,
|
| 162 |
generator=None,
|
| 163 |
)
|
|
|
|
| 176 |
| `cfg` | disabled | Classifier-free guidance settings. |
|
| 177 |
| `pdg` | full-path PDG 2.5 | Path-drop guidance settings. |
|
| 178 |
| `self_attention_gain` | `-0.03` | Gain applied exclusively to image self-attention on the main denoiser path. |
|
| 179 |
+
| `euler_maruyama_multiplier` | `1.0` | Non-negative stochastic noise multiplier used by Euler-Maruyama. |
|
| 180 |
+
| `er_sde_noise_multiplier` | `1.0` | Non-negative stochastic noise multiplier used by ER-SDE. |
|
| 181 |
| `seed` | `42` | Random seed. Set to `None` when supplying `generator`. |
|
| 182 |
| `generator` | `None` | Optional CUDA `torch.Generator` on the same device as the model. Exactly one of `seed` and `generator` is required. |
|
| 183 |
|
|
|
|
| 189 |
Images downloaded from the Gradio interface contain a `canter` PNG text field
|
| 190 |
with compact JSON. The object begins with the prompt, effective per-image seed,
|
| 191 |
width, height, steps, solver, and schedule. It then records PDG, CFG,
|
| 192 |
+
self-attention gain, logSNR shift, the Euler-Maruyama and ER-SDE noise
|
| 193 |
+
multipliers, installed code version, numbered checkpoint release, and weight
|
| 194 |
+
dtype.
|
| 195 |
|
| 196 |
### Solvers
|
| 197 |
|
|
|
|
| 199 |
| --- | --- | --- |
|
| 200 |
| `Solver.EULER` | `"euler"` | First-order deterministic Euler updates. |
|
| 201 |
| `Solver.EULER_MARUYAMA` | `"euler_maruyama"` | Stochastic reverse-SDE updates. Uses `euler_maruyama_multiplier`. |
|
| 202 |
+
| `Solver.ER_SDE` | `"er_sde"` | Third-stage VP ER-SDE with 16-point Gauss-Legendre correction quadrature. Uses `er_sde_noise_multiplier`. |
|
| 203 |
+
| `Solver.DPMPP_2M` | `"dpmpp_2m"` | Flow-matching DPM++ 2M updates with finite pre-shift endpoint handling. |
|
| 204 |
| `Solver.ABM2` | `"abm2"` | Variable-step Adams-Bashforth-Moulton updates with corrected-state reevaluation. |
|
| 205 |
|
| 206 |
ABM2 is the default and performs additional denoiser evaluations for its
|
|
|
|
| 376 |
),
|
| 377 |
self_attention_gain=-0.03,
|
| 378 |
euler_maruyama_multiplier=1.0,
|
| 379 |
+
er_sde_noise_multiplier=1.0,
|
| 380 |
seed=123,
|
| 381 |
generator=None,
|
| 382 |
)
|
|
|
|
| 441 |
|
| 442 |
## Pipeline metadata
|
| 443 |
|
| 444 |
+
`pipe.metadata` records the installed Canter code version, resolved checkpoint
|
| 445 |
+
release, weight dtype, source digests, text-encoder revision, VAE repository,
|
| 446 |
+
and resolved immutable VAE revision. Applications that require reproducibility
|
| 447 |
+
should store this metadata and pin both the package version and checkpoint tag.
|
README.md
CHANGED
|
@@ -22,7 +22,7 @@ base_model:
|
|
| 22 |
> The model is still training. Checkpoints and behavior may change during
|
| 23 |
> the preview period, and generation quality is still quite variable.
|
| 24 |
|
| 25 |
-
**Current
|
| 26 |
|
| 27 |
[Example gallery](GALLERY.md) 路 [Getting started](#getting-started) 路
|
| 28 |
[API and inference parameters](API.md) 路
|
|
@@ -55,21 +55,38 @@ Install a CUDA-enabled PyTorch build for your system first. The
|
|
| 55 |
[PyTorch installation selector](https://pytorch.org/get-started/locally/)
|
| 56 |
provides the appropriate command.
|
| 57 |
|
| 58 |
-
###
|
| 59 |
|
| 60 |
-
Install the Hugging Face CLI, download the
|
| 61 |
-
|
| 62 |
|
| 63 |
```bash
|
| 64 |
python -m pip install "huggingface-hub>=1.15,<2"
|
| 65 |
-
hf download data-archetype/canter --revision
|
| 66 |
cd canter
|
| 67 |
-
python -m pip install .
|
| 68 |
```
|
| 69 |
|
| 70 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
parameters remain in float32.
|
| 72 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
### Start the Gradio interface
|
| 74 |
|
| 75 |
Run the application from the downloaded repository:
|
|
@@ -82,8 +99,8 @@ python app.py --in-browser
|
|
| 82 |
downloads the latest compatible DINAC-AE-D2 VAE.
|
| 83 |
The interface appears immediately and reports model loading and pytorch dynamo compilation
|
| 84 |
progress.
|
| 85 |
-
Downloaded PNG files contain the prompt, effective per-image settings,
|
| 86 |
-
numbered
|
| 87 |
|
| 88 |
The server listens on port 7860. To select the bind address explicitly:
|
| 89 |
|
|
@@ -148,14 +165,14 @@ rights.
|
|
| 148 |
|
| 149 |
## Releases
|
| 150 |
|
| 151 |
-
|
| 152 |
-
|
| 153 |
|
| 154 |
```python
|
| 155 |
pipe = CanterPipeline.from_pretrained("data-archetype/canter")
|
| 156 |
```
|
| 157 |
|
| 158 |
-
|
| 159 |
|
| 160 |
```python
|
| 161 |
pipe = CanterPipeline.from_pretrained(
|
|
@@ -164,6 +181,19 @@ pipe = CanterPipeline.from_pretrained(
|
|
| 164 |
)
|
| 165 |
```
|
| 166 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
Release tags follow the `v0001`, `v0002`, and later numbering scheme. Optional
|
| 168 |
full-float32 releases use tags such as `v0001-fp32`.
|
| 169 |
|
|
|
|
| 22 |
> The model is still training. Checkpoints and behavior may change during
|
| 23 |
> the preview period, and generation quality is still quite variable.
|
| 24 |
|
| 25 |
+
**Current checkpoint:** [`v0001`](RELEASES.md#v0001)
|
| 26 |
|
| 27 |
[Example gallery](GALLERY.md) 路 [Getting started](#getting-started) 路
|
| 28 |
[API and inference parameters](API.md) 路
|
|
|
|
| 55 |
[PyTorch installation selector](https://pytorch.org/get-started/locally/)
|
| 56 |
provides the appropriate command.
|
| 57 |
|
| 58 |
+
### Install the latest code and checkpoint
|
| 59 |
|
| 60 |
+
Install the Hugging Face CLI, download the moving `main` revision, and install
|
| 61 |
+
the package in editable mode:
|
| 62 |
|
| 63 |
```bash
|
| 64 |
python -m pip install "huggingface-hub>=1.15,<2"
|
| 65 |
+
hf download data-archetype/canter --revision main --local-dir canter
|
| 66 |
cd canter
|
| 67 |
+
python -m pip install -e .
|
| 68 |
```
|
| 69 |
|
| 70 |
+
`main` contains the latest Canter code and the current default checkpoint.
|
| 71 |
+
Editable installation means that refreshing the same directory updates the
|
| 72 |
+
code used by the installed `canter-web` command.
|
| 73 |
+
|
| 74 |
+
The default checkpoint stores most weights in bfloat16. Numerically sensitive
|
| 75 |
parameters remain in float32.
|
| 76 |
|
| 77 |
+
### Update an existing download
|
| 78 |
+
|
| 79 |
+
Refresh a directory created with `hf download` by running:
|
| 80 |
+
|
| 81 |
+
```bash
|
| 82 |
+
hf download data-archetype/canter --revision main --local-dir canter
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
If the package was installed without `-e`, reinstall it afterwards with
|
| 86 |
+
`python -m pip install --upgrade ./canter`. A Git clone on the `main` branch can
|
| 87 |
+
instead be updated with `git pull`; an editable installation immediately uses
|
| 88 |
+
the updated checkout.
|
| 89 |
+
|
| 90 |
### Start the Gradio interface
|
| 91 |
|
| 92 |
Run the application from the downloaded repository:
|
|
|
|
| 99 |
downloads the latest compatible DINAC-AE-D2 VAE.
|
| 100 |
The interface appears immediately and reports model loading and pytorch dynamo compilation
|
| 101 |
progress.
|
| 102 |
+
Downloaded PNG files contain the prompt, effective per-image settings, Canter
|
| 103 |
+
code version, and numbered checkpoint release as JSON metadata.
|
| 104 |
|
| 105 |
The server listens on port 7860. To select the bind address explicitly:
|
| 106 |
|
|
|
|
| 165 |
|
| 166 |
## Releases
|
| 167 |
|
| 168 |
+
The installed package supplies the inference code. Remote loading without a
|
| 169 |
+
revision uses the checkpoint pinned by that package:
|
| 170 |
|
| 171 |
```python
|
| 172 |
pipe = CanterPipeline.from_pretrained("data-archetype/canter")
|
| 173 |
```
|
| 174 |
|
| 175 |
+
Select an immutable older checkpoint while retaining the installed code:
|
| 176 |
|
| 177 |
```python
|
| 178 |
pipe = CanterPipeline.from_pretrained(
|
|
|
|
| 181 |
)
|
| 182 |
```
|
| 183 |
|
| 184 |
+
The web interface supports the same separation:
|
| 185 |
+
|
| 186 |
+
```bash
|
| 187 |
+
canter-web \
|
| 188 |
+
--model data-archetype/canter \
|
| 189 |
+
--revision v0001 \
|
| 190 |
+
--in-browser
|
| 191 |
+
```
|
| 192 |
+
|
| 193 |
+
Running `app.py` from a tagged standalone download intentionally uses the code
|
| 194 |
+
bundled with that historical snapshot. Use the installed `canter-web` command
|
| 195 |
+
as above when testing old weights with current code.
|
| 196 |
+
|
| 197 |
Release tags follow the `v0001`, `v0002`, and later numbering scheme. Optional
|
| 198 |
full-float32 releases use tags such as `v0001-fp32`.
|
| 199 |
|
RELEASES.md
CHANGED
|
@@ -3,38 +3,48 @@
|
|
| 3 |
[Model card](README.md) 路 [Getting started](README.md#getting-started) 路
|
| 4 |
[API and inference parameters](API.md)
|
| 5 |
|
| 6 |
-
|
| 7 |
-
|
|
|
|
| 8 |
|
| 9 |
| Release | Date | Weight storage | Status |
|
| 10 |
| --- | --- | --- | --- |
|
| 11 |
| [`v0001`](https://huggingface.co/data-archetype/canter/tree/v0001) | July 2026 | bfloat16 with float32 precision islands | Preview |
|
| 12 |
|
| 13 |
-
## Updating
|
| 14 |
|
| 15 |
-
|
| 16 |
|
| 17 |
```bash
|
| 18 |
-
hf download data-archetype/canter --revision
|
| 19 |
-
|
| 20 |
-
python -m pip install --upgrade .
|
| 21 |
```
|
| 22 |
|
| 23 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
|
| 25 |
```python
|
| 26 |
pipe = CanterPipeline.from_pretrained(
|
| 27 |
"data-archetype/canter",
|
| 28 |
-
revision="
|
| 29 |
)
|
| 30 |
```
|
| 31 |
|
| 32 |
```bash
|
| 33 |
-
canter-web --model data-archetype/canter --revision
|
| 34 |
```
|
| 35 |
|
| 36 |
-
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
|
| 39 |
## v0001
|
| 40 |
|
|
|
|
| 3 |
[Model card](README.md) 路 [Getting started](README.md#getting-started) 路
|
| 4 |
[API and inference parameters](API.md)
|
| 5 |
|
| 6 |
+
Checkpoint tags are immutable snapshots. The installed package supplies the
|
| 7 |
+
inference code, so current code can load any compatible checkpoint tag without
|
| 8 |
+
executing the Python files stored in that historical snapshot.
|
| 9 |
|
| 10 |
| Release | Date | Weight storage | Status |
|
| 11 |
| --- | --- | --- | --- |
|
| 12 |
| [`v0001`](https://huggingface.co/data-archetype/canter/tree/v0001) | July 2026 | bfloat16 with float32 precision islands | Preview |
|
| 13 |
|
| 14 |
+
## Updating code
|
| 15 |
|
| 16 |
+
Refresh the moving `main` directory and use an editable installation:
|
| 17 |
|
| 18 |
```bash
|
| 19 |
+
hf download data-archetype/canter --revision main --local-dir canter
|
| 20 |
+
python -m pip install -e ./canter
|
|
|
|
| 21 |
```
|
| 22 |
|
| 23 |
+
For a Git clone on `main`, use `git pull`. If the checkout was installed
|
| 24 |
+
without `-e`, reinstall it after updating.
|
| 25 |
+
|
| 26 |
+
## Selecting a checkpoint
|
| 27 |
+
|
| 28 |
+
Remote API and web UI loading can select any compatible checkpoint explicitly:
|
| 29 |
|
| 30 |
```python
|
| 31 |
pipe = CanterPipeline.from_pretrained(
|
| 32 |
"data-archetype/canter",
|
| 33 |
+
revision="v0001",
|
| 34 |
)
|
| 35 |
```
|
| 36 |
|
| 37 |
```bash
|
| 38 |
+
canter-web --model data-archetype/canter --revision v0001 --in-browser
|
| 39 |
```
|
| 40 |
|
| 41 |
+
These commands use the currently installed code and download only the selected
|
| 42 |
+
checkpoint artifacts. Running `app.py` inside a tagged standalone directory
|
| 43 |
+
instead uses the historical code bundled with that snapshot.
|
| 44 |
+
|
| 45 |
+
For exact reproduction, pin both the Canter package version and checkpoint
|
| 46 |
+
tag. Passing `revision="main"` explicitly selects the moving repository head
|
| 47 |
+
and is not an immutable checkpoint reference.
|
| 48 |
|
| 49 |
## v0001
|
| 50 |
|
canter/__init__.py
CHANGED
|
@@ -34,12 +34,14 @@ from .schedules import Schedule
|
|
| 34 |
from .solvers import Solver, SolverProgress
|
| 35 |
from .text_encoder import CanterTextEncoder, TextBackboneOutput
|
| 36 |
from .vae import CanterVae
|
|
|
|
| 37 |
|
| 38 |
__all__ = [
|
| 39 |
"CANTER_CONFIG",
|
| 40 |
"CANTER_DEFAULT_RELEASE",
|
| 41 |
"CANTER_LICENSE",
|
| 42 |
"CANTER_LICENSE_URL",
|
|
|
|
| 43 |
"CanterComponents",
|
| 44 |
"CanterConfig",
|
| 45 |
"CanterInferenceConfig",
|
|
@@ -66,4 +68,5 @@ __all__ = [
|
|
| 66 |
"TextAttentionBackend",
|
| 67 |
"TextBackboneOutput",
|
| 68 |
"WeightDType",
|
|
|
|
| 69 |
]
|
|
|
|
| 34 |
from .solvers import Solver, SolverProgress
|
| 35 |
from .text_encoder import CanterTextEncoder, TextBackboneOutput
|
| 36 |
from .vae import CanterVae
|
| 37 |
+
from .version import CANTER_VERSION, __version__
|
| 38 |
|
| 39 |
__all__ = [
|
| 40 |
"CANTER_CONFIG",
|
| 41 |
"CANTER_DEFAULT_RELEASE",
|
| 42 |
"CANTER_LICENSE",
|
| 43 |
"CANTER_LICENSE_URL",
|
| 44 |
+
"CANTER_VERSION",
|
| 45 |
"CanterComponents",
|
| 46 |
"CanterConfig",
|
| 47 |
"CanterInferenceConfig",
|
|
|
|
| 68 |
"TextAttentionBackend",
|
| 69 |
"TextBackboneOutput",
|
| 70 |
"WeightDType",
|
| 71 |
+
"__version__",
|
| 72 |
]
|
canter/inference.py
CHANGED
|
@@ -15,7 +15,7 @@ from torch.amp import autocast
|
|
| 15 |
from .modeling_canter import CanterPath, PreparedText
|
| 16 |
from .runtime import CANTER_AMP_DTYPE, validate_common_runtime
|
| 17 |
from .schedules import Schedule, build_schedule
|
| 18 |
-
from .solvers import Solver, SolverProgress, solve
|
| 19 |
|
| 20 |
if TYPE_CHECKING:
|
| 21 |
from .loading import CanterComponents
|
|
@@ -141,6 +141,7 @@ class CanterInferenceConfig:
|
|
| 141 |
euler_maruyama_multiplier: float = 1.0
|
| 142 |
seed: int | None = 42
|
| 143 |
generator: torch.Generator | None = None
|
|
|
|
| 144 |
|
| 145 |
def __post_init__(self) -> None:
|
| 146 |
"""Validate all public inference parameters before model execution."""
|
|
@@ -170,11 +171,14 @@ class CanterInferenceConfig:
|
|
| 170 |
("log_snr_shift", self.log_snr_shift),
|
| 171 |
("self_attention_gain", self.self_attention_gain),
|
| 172 |
("euler_maruyama_multiplier", self.euler_maruyama_multiplier),
|
|
|
|
| 173 |
):
|
| 174 |
if not math.isfinite(float(value)):
|
| 175 |
raise ValueError(f"{name} must be finite.")
|
| 176 |
if float(self.euler_maruyama_multiplier) < 0.0:
|
| 177 |
raise ValueError("euler_maruyama_multiplier must be non-negative.")
|
|
|
|
|
|
|
| 178 |
_validate_rng(self.seed, self.generator)
|
| 179 |
_resolve_window(self.cfg.start_step, self.cfg.stop_step, self.steps, "CFG")
|
| 180 |
_resolve_window(self.pdg.start_step, self.pdg.stop_step, self.steps, "PDG")
|
|
@@ -568,6 +572,11 @@ class CanterInferenceEngine:
|
|
| 568 |
config.schedule,
|
| 569 |
steps=config.steps,
|
| 570 |
log_snr_shift=config.log_snr_shift,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 571 |
device=self.device,
|
| 572 |
)
|
| 573 |
self.model.eval()
|
|
@@ -621,6 +630,7 @@ class CanterInferenceEngine:
|
|
| 621 |
solver=config.solver,
|
| 622 |
generator=execution_generator,
|
| 623 |
euler_maruyama_multiplier=config.euler_maruyama_multiplier,
|
|
|
|
| 624 |
progress=progress,
|
| 625 |
)
|
| 626 |
return CanterLatentOutput(latents=latents, schedule=schedule)
|
|
|
|
| 15 |
from .modeling_canter import CanterPath, PreparedText
|
| 16 |
from .runtime import CANTER_AMP_DTYPE, validate_common_runtime
|
| 17 |
from .schedules import Schedule, build_schedule
|
| 18 |
+
from .solvers import LOGSNR_SOLVER_START_EPS, Solver, SolverProgress, solve
|
| 19 |
|
| 20 |
if TYPE_CHECKING:
|
| 21 |
from .loading import CanterComponents
|
|
|
|
| 141 |
euler_maruyama_multiplier: float = 1.0
|
| 142 |
seed: int | None = 42
|
| 143 |
generator: torch.Generator | None = None
|
| 144 |
+
er_sde_noise_multiplier: float = 1.0
|
| 145 |
|
| 146 |
def __post_init__(self) -> None:
|
| 147 |
"""Validate all public inference parameters before model execution."""
|
|
|
|
| 171 |
("log_snr_shift", self.log_snr_shift),
|
| 172 |
("self_attention_gain", self.self_attention_gain),
|
| 173 |
("euler_maruyama_multiplier", self.euler_maruyama_multiplier),
|
| 174 |
+
("er_sde_noise_multiplier", self.er_sde_noise_multiplier),
|
| 175 |
):
|
| 176 |
if not math.isfinite(float(value)):
|
| 177 |
raise ValueError(f"{name} must be finite.")
|
| 178 |
if float(self.euler_maruyama_multiplier) < 0.0:
|
| 179 |
raise ValueError("euler_maruyama_multiplier must be non-negative.")
|
| 180 |
+
if float(self.er_sde_noise_multiplier) < 0.0:
|
| 181 |
+
raise ValueError("er_sde_noise_multiplier must be non-negative.")
|
| 182 |
_validate_rng(self.seed, self.generator)
|
| 183 |
_resolve_window(self.cfg.start_step, self.cfg.stop_step, self.steps, "CFG")
|
| 184 |
_resolve_window(self.pdg.start_step, self.pdg.stop_step, self.steps, "PDG")
|
|
|
|
| 572 |
config.schedule,
|
| 573 |
steps=config.steps,
|
| 574 |
log_snr_shift=config.log_snr_shift,
|
| 575 |
+
finite_noisy_endpoint_epsilon=(
|
| 576 |
+
float(LOGSNR_SOLVER_START_EPS)
|
| 577 |
+
if config.solver in (Solver.DPMPP_2M, Solver.ER_SDE)
|
| 578 |
+
else None
|
| 579 |
+
),
|
| 580 |
device=self.device,
|
| 581 |
)
|
| 582 |
self.model.eval()
|
|
|
|
| 630 |
solver=config.solver,
|
| 631 |
generator=execution_generator,
|
| 632 |
euler_maruyama_multiplier=config.euler_maruyama_multiplier,
|
| 633 |
+
er_sde_noise_multiplier=config.er_sde_noise_multiplier,
|
| 634 |
progress=progress,
|
| 635 |
)
|
| 636 |
return CanterLatentOutput(latents=latents, schedule=schedule)
|
canter/loading.py
CHANGED
|
@@ -25,6 +25,14 @@ from .runtime import (
|
|
| 25 |
)
|
| 26 |
from .text_encoder import CanterTextEncoder
|
| 27 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
|
| 29 |
class WeightDType(Enum):
|
| 30 |
"""Public Canter checkpoint storage dtypes."""
|
|
@@ -167,6 +175,7 @@ def _resolve_model_dir(
|
|
| 167 |
str(path_or_repo_id),
|
| 168 |
revision=resolved_revision,
|
| 169 |
cache_dir=cache,
|
|
|
|
| 170 |
)
|
| 171 |
)
|
| 172 |
|
|
|
|
| 25 |
)
|
| 26 |
from .text_encoder import CanterTextEncoder
|
| 27 |
|
| 28 |
+
_CHECKPOINT_ALLOW_PATTERNS = (
|
| 29 |
+
"config.json",
|
| 30 |
+
"release.json",
|
| 31 |
+
"weights_manifest.json",
|
| 32 |
+
"model*.safetensors*",
|
| 33 |
+
"text_encoder/*",
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
|
| 37 |
class WeightDType(Enum):
|
| 38 |
"""Public Canter checkpoint storage dtypes."""
|
|
|
|
| 175 |
str(path_or_repo_id),
|
| 176 |
revision=resolved_revision,
|
| 177 |
cache_dir=cache,
|
| 178 |
+
allow_patterns=list(_CHECKPOINT_ALLOW_PATTERNS),
|
| 179 |
)
|
| 180 |
)
|
| 181 |
|
canter/pipeline.py
CHANGED
|
@@ -25,6 +25,7 @@ from .loading import (
|
|
| 25 |
WeightDType,
|
| 26 |
)
|
| 27 |
from .vae import CanterVae
|
|
|
|
| 28 |
|
| 29 |
if TYPE_CHECKING:
|
| 30 |
from .solvers import SolverProgress
|
|
@@ -61,6 +62,7 @@ class CanterPipelineMetadata:
|
|
| 61 |
canter: CanterReleaseMetadata
|
| 62 |
vae_repository: str
|
| 63 |
vae_revision: str
|
|
|
|
| 64 |
|
| 65 |
|
| 66 |
@dataclass(frozen=True)
|
|
|
|
| 25 |
WeightDType,
|
| 26 |
)
|
| 27 |
from .vae import CanterVae
|
| 28 |
+
from .version import CANTER_VERSION
|
| 29 |
|
| 30 |
if TYPE_CHECKING:
|
| 31 |
from .solvers import SolverProgress
|
|
|
|
| 62 |
canter: CanterReleaseMetadata
|
| 63 |
vae_repository: str
|
| 64 |
vae_revision: str
|
| 65 |
+
code_version: str = field(default=CANTER_VERSION, init=False)
|
| 66 |
|
| 67 |
|
| 68 |
@dataclass(frozen=True)
|
canter/schedules.py
CHANGED
|
@@ -63,11 +63,51 @@ def _beta_quantiles(points: int) -> Tensor:
|
|
| 63 |
return torch.from_numpy(values)
|
| 64 |
|
| 65 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
def build_schedule(
|
| 67 |
kind: Schedule,
|
| 68 |
*,
|
| 69 |
steps: int,
|
| 70 |
log_snr_shift: float,
|
|
|
|
| 71 |
device: torch.device,
|
| 72 |
) -> Tensor:
|
| 73 |
"""Build a descending FP32 schedule with ``steps + 1`` endpoint samples."""
|
|
@@ -78,6 +118,7 @@ def build_schedule(
|
|
| 78 |
raise TypeError("steps must be an int.")
|
| 79 |
if steps < 1:
|
| 80 |
raise ValueError("steps must be positive.")
|
|
|
|
| 81 |
points = steps + 1
|
| 82 |
match kind:
|
| 83 |
case Schedule.LINEAR:
|
|
@@ -93,5 +134,10 @@ def build_schedule(
|
|
| 93 |
case _ as unreachable:
|
| 94 |
raise RuntimeError(f"Unsupported Canter schedule: {unreachable}")
|
| 95 |
ascending = ascending.to(device=device, dtype=torch.float32)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
shifted = apply_log_snr_shift(ascending, float(log_snr_shift))
|
| 97 |
return torch.flip(shifted, dims=(0,)).contiguous()
|
|
|
|
| 63 |
return torch.from_numpy(values)
|
| 64 |
|
| 65 |
|
| 66 |
+
def _finite_noisy_endpoint_epsilon(value: float | None) -> float | None:
|
| 67 |
+
"""Validate an optional pre-shift endpoint trimming distance."""
|
| 68 |
+
|
| 69 |
+
if value is None:
|
| 70 |
+
return None
|
| 71 |
+
epsilon = float(value)
|
| 72 |
+
if not math.isfinite(epsilon) or not 0.0 < epsilon < 1.0:
|
| 73 |
+
raise ValueError(
|
| 74 |
+
"finite_noisy_endpoint_epsilon must be finite and lie in (0, 1)."
|
| 75 |
+
)
|
| 76 |
+
return epsilon
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _cap_ascending_noisy_endpoint(timesteps: Tensor, *, epsilon: float) -> Tensor:
|
| 80 |
+
"""Replace the exact t=1 endpoint before applying a log-SNR shift."""
|
| 81 |
+
|
| 82 |
+
if int(timesteps.numel()) < 2 or float(timesteps[-1].item()) != 1.0:
|
| 83 |
+
return timesteps
|
| 84 |
+
candidate = torch.tensor(
|
| 85 |
+
1.0 - float(epsilon),
|
| 86 |
+
device=timesteps.device,
|
| 87 |
+
dtype=timesteps.dtype,
|
| 88 |
+
)
|
| 89 |
+
neighbor = timesteps[-2]
|
| 90 |
+
if float(neighbor.item()) >= float(candidate.item()):
|
| 91 |
+
candidate = torch.nextafter(
|
| 92 |
+
neighbor,
|
| 93 |
+
torch.tensor(1.0, device=timesteps.device, dtype=timesteps.dtype),
|
| 94 |
+
)
|
| 95 |
+
if float(candidate.item()) >= 1.0:
|
| 96 |
+
raise ValueError(
|
| 97 |
+
"Cannot construct a finite noisy endpoint strictly between "
|
| 98 |
+
"the adjacent schedule point and t=1."
|
| 99 |
+
)
|
| 100 |
+
capped = timesteps.clone()
|
| 101 |
+
capped[-1] = candidate
|
| 102 |
+
return capped
|
| 103 |
+
|
| 104 |
+
|
| 105 |
def build_schedule(
|
| 106 |
kind: Schedule,
|
| 107 |
*,
|
| 108 |
steps: int,
|
| 109 |
log_snr_shift: float,
|
| 110 |
+
finite_noisy_endpoint_epsilon: float | None,
|
| 111 |
device: torch.device,
|
| 112 |
) -> Tensor:
|
| 113 |
"""Build a descending FP32 schedule with ``steps + 1`` endpoint samples."""
|
|
|
|
| 118 |
raise TypeError("steps must be an int.")
|
| 119 |
if steps < 1:
|
| 120 |
raise ValueError("steps must be positive.")
|
| 121 |
+
endpoint_epsilon = _finite_noisy_endpoint_epsilon(finite_noisy_endpoint_epsilon)
|
| 122 |
points = steps + 1
|
| 123 |
match kind:
|
| 124 |
case Schedule.LINEAR:
|
|
|
|
| 134 |
case _ as unreachable:
|
| 135 |
raise RuntimeError(f"Unsupported Canter schedule: {unreachable}")
|
| 136 |
ascending = ascending.to(device=device, dtype=torch.float32)
|
| 137 |
+
if endpoint_epsilon is not None:
|
| 138 |
+
ascending = _cap_ascending_noisy_endpoint(
|
| 139 |
+
ascending,
|
| 140 |
+
epsilon=float(endpoint_epsilon),
|
| 141 |
+
)
|
| 142 |
shifted = apply_log_snr_shift(ascending, float(log_snr_shift))
|
| 143 |
return torch.flip(shifted, dims=(0,)).contiguous()
|
canter/solvers.py
CHANGED
|
@@ -6,11 +6,15 @@ import math
|
|
| 6 |
from enum import Enum
|
| 7 |
from typing import Protocol
|
| 8 |
|
|
|
|
| 9 |
import torch
|
| 10 |
import torch.nn.functional as F
|
| 11 |
from torch import Tensor
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
| 14 |
_SCORE_EPS = 1.0e-4
|
| 15 |
|
| 16 |
|
|
@@ -19,6 +23,7 @@ class Solver(Enum):
|
|
| 19 |
|
| 20 |
EULER = "euler"
|
| 21 |
EULER_MARUYAMA = "euler_maruyama"
|
|
|
|
| 22 |
DPMPP_2M = "dpmpp_2m"
|
| 23 |
ABM2 = "abm2"
|
| 24 |
|
|
@@ -121,7 +126,7 @@ def _euler_maruyama(
|
|
| 121 |
dtype=torch.float32,
|
| 122 |
)
|
| 123 |
predicted = velocity(state, time, index).float()
|
| 124 |
-
terminal = index == intervals - 1 and
|
| 125 |
if terminal:
|
| 126 |
state = state - time_value * predicted
|
| 127 |
progress(index + 1, intervals)
|
|
@@ -144,29 +149,17 @@ def _euler_maruyama(
|
|
| 144 |
return state
|
| 145 |
|
| 146 |
|
| 147 |
-
def
|
| 148 |
-
"""
|
| 149 |
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
1.0 - _DPM_START_EPS,
|
| 154 |
-
device=adjusted.device,
|
| 155 |
-
dtype=adjusted.dtype,
|
| 156 |
)
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
)
|
| 162 |
-
if float(candidate.item()) >= 1.0:
|
| 163 |
-
raise ValueError("DPM++ 2M cannot construct a finite start timestep.")
|
| 164 |
-
adjusted[0] = candidate
|
| 165 |
-
if bool(((adjusted[:-1] <= 0.0) | (adjusted[:-1] >= 1.0)).any().item()):
|
| 166 |
-
raise ValueError("DPM++ 2M evaluation times must lie strictly inside (0, 1).")
|
| 167 |
-
if float(adjusted[-1].item()) < 0.0:
|
| 168 |
-
raise ValueError("DPM++ 2M final time must be non-negative.")
|
| 169 |
-
return adjusted
|
| 170 |
|
| 171 |
|
| 172 |
def _half_log_snr(time: Tensor) -> Tensor:
|
|
@@ -210,7 +203,7 @@ def _dpmpp_2m(
|
|
| 210 |
) -> Tensor:
|
| 211 |
"""Integrate with the flow-matching DPM++ 2M formulation."""
|
| 212 |
|
| 213 |
-
times =
|
| 214 |
lambdas = _half_log_snr(times.clamp_min(torch.finfo(times.dtype).tiny))
|
| 215 |
batch = int(state.shape[0])
|
| 216 |
previous_denoised: Tensor | None = None
|
|
@@ -241,6 +234,159 @@ def _dpmpp_2m(
|
|
| 241 |
return state
|
| 242 |
|
| 243 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 244 |
def _ab2_predict(
|
| 245 |
state: Tensor,
|
| 246 |
current_step: Tensor,
|
|
@@ -316,6 +462,7 @@ def solve(
|
|
| 316 |
solver: Solver,
|
| 317 |
generator: torch.Generator,
|
| 318 |
euler_maruyama_multiplier: float,
|
|
|
|
| 319 |
progress: SolverProgress | None = None,
|
| 320 |
) -> Tensor:
|
| 321 |
"""Integrate Canter velocity predictions over one validated schedule."""
|
|
@@ -323,9 +470,12 @@ def solve(
|
|
| 323 |
_validate_inputs(initial_state, schedule)
|
| 324 |
if not isinstance(solver, Solver):
|
| 325 |
raise TypeError("solver must be a Solver.")
|
| 326 |
-
|
| 327 |
-
if not math.isfinite(
|
| 328 |
raise ValueError("euler_maruyama_multiplier must be finite and non-negative.")
|
|
|
|
|
|
|
|
|
|
| 329 |
resolved_progress = _ignore_progress if progress is None else progress
|
| 330 |
match solver:
|
| 331 |
case Solver.EULER:
|
|
@@ -336,7 +486,16 @@ def solve(
|
|
| 336 |
initial_state,
|
| 337 |
schedule,
|
| 338 |
generator=generator,
|
| 339 |
-
multiplier=
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 340 |
progress=resolved_progress,
|
| 341 |
)
|
| 342 |
case Solver.DPMPP_2M:
|
|
|
|
| 6 |
from enum import Enum
|
| 7 |
from typing import Protocol
|
| 8 |
|
| 9 |
+
import numpy as np
|
| 10 |
import torch
|
| 11 |
import torch.nn.functional as F
|
| 12 |
from torch import Tensor
|
| 13 |
|
| 14 |
+
LOGSNR_SOLVER_START_EPS = 1.0e-4
|
| 15 |
+
_ER_QUADRATURE_POINTS = 16
|
| 16 |
+
_ER_NOISE_EXPONENT = 0.3
|
| 17 |
+
_ER_NOISE_OFFSET = 10.0
|
| 18 |
_SCORE_EPS = 1.0e-4
|
| 19 |
|
| 20 |
|
|
|
|
| 23 |
|
| 24 |
EULER = "euler"
|
| 25 |
EULER_MARUYAMA = "euler_maruyama"
|
| 26 |
+
ER_SDE = "er_sde"
|
| 27 |
DPMPP_2M = "dpmpp_2m"
|
| 28 |
ABM2 = "abm2"
|
| 29 |
|
|
|
|
| 126 |
dtype=torch.float32,
|
| 127 |
)
|
| 128 |
predicted = velocity(state, time, index).float()
|
| 129 |
+
terminal = index == intervals - 1 and next_value == 0.0
|
| 130 |
if terminal:
|
| 131 |
state = state - time_value * predicted
|
| 132 |
progress(index + 1, intervals)
|
|
|
|
| 149 |
return state
|
| 150 |
|
| 151 |
|
| 152 |
+
def _prepare_logsnr_schedule(schedule: Tensor, *, solver_name: str) -> Tensor:
|
| 153 |
+
"""Require finite evaluation times for a flow log-SNR solver."""
|
| 154 |
|
| 155 |
+
if bool(((schedule[:-1] <= 0.0) | (schedule[:-1] >= 1.0)).any().item()):
|
| 156 |
+
raise ValueError(
|
| 157 |
+
f"{solver_name} evaluation times must lie strictly inside (0, 1)."
|
|
|
|
|
|
|
|
|
|
| 158 |
)
|
| 159 |
+
final_time = float(schedule[-1].item())
|
| 160 |
+
if final_time < 0.0 or final_time >= 1.0:
|
| 161 |
+
raise ValueError(f"{solver_name} final time must lie in [0, 1).")
|
| 162 |
+
return schedule
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
|
| 164 |
|
| 165 |
def _half_log_snr(time: Tensor) -> Tensor:
|
|
|
|
| 203 |
) -> Tensor:
|
| 204 |
"""Integrate with the flow-matching DPM++ 2M formulation."""
|
| 205 |
|
| 206 |
+
times = _prepare_logsnr_schedule(schedule, solver_name="DPM++ 2M")
|
| 207 |
lambdas = _half_log_snr(times.clamp_min(torch.finfo(times.dtype).tiny))
|
| 208 |
batch = int(state.shape[0])
|
| 209 |
previous_denoised: Tensor | None = None
|
|
|
|
| 234 |
return state
|
| 235 |
|
| 236 |
|
| 237 |
+
def _er_noise_scaler(er_lambda: Tensor) -> Tensor:
|
| 238 |
+
"""Evaluate the paper-selected ER-SDE noise-scaling function."""
|
| 239 |
+
|
| 240 |
+
return er_lambda * (torch.exp(er_lambda.pow(_ER_NOISE_EXPONENT)) + _ER_NOISE_OFFSET)
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def _er_quadrature_rule(*, device: torch.device) -> tuple[Tensor, Tensor]:
|
| 244 |
+
"""Return float32 Gauss-Legendre nodes and weights on the solver device."""
|
| 245 |
+
|
| 246 |
+
nodes_np, weights_np = np.polynomial.legendre.leggauss(_ER_QUADRATURE_POINTS)
|
| 247 |
+
nodes = torch.as_tensor(nodes_np, device=device, dtype=torch.float32)
|
| 248 |
+
weights = torch.as_tensor(weights_np, device=device, dtype=torch.float32)
|
| 249 |
+
return nodes, weights
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def _er_quadrature_terms(
|
| 253 |
+
er_lambda_s: Tensor,
|
| 254 |
+
er_lambda_t: Tensor,
|
| 255 |
+
nodes: Tensor,
|
| 256 |
+
weights: Tensor,
|
| 257 |
+
) -> tuple[Tensor, Tensor]:
|
| 258 |
+
"""Evaluate the ER-SDE correction integrals with Gauss-Legendre quadrature."""
|
| 259 |
+
|
| 260 |
+
midpoint = (er_lambda_s + er_lambda_t) / 2.0
|
| 261 |
+
half_span = (er_lambda_s - er_lambda_t) / 2.0
|
| 262 |
+
positions = midpoint + half_span * nodes
|
| 263 |
+
scaled_positions = _er_noise_scaler(positions)
|
| 264 |
+
second = half_span * torch.sum(weights / scaled_positions)
|
| 265 |
+
third = half_span * torch.sum(
|
| 266 |
+
weights * (positions - er_lambda_s) / scaled_positions
|
| 267 |
+
)
|
| 268 |
+
return second, third
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def _er_sde_step(
|
| 272 |
+
*,
|
| 273 |
+
state: Tensor,
|
| 274 |
+
denoised: Tensor,
|
| 275 |
+
old_denoised: Tensor | None,
|
| 276 |
+
old_denoised_derivative: Tensor | None,
|
| 277 |
+
times: Tensor,
|
| 278 |
+
er_lambdas: Tensor,
|
| 279 |
+
index: int,
|
| 280 |
+
nodes: Tensor,
|
| 281 |
+
weights: Tensor,
|
| 282 |
+
generator: torch.Generator,
|
| 283 |
+
noise_multiplier: float,
|
| 284 |
+
) -> tuple[Tensor, Tensor | None]:
|
| 285 |
+
"""Apply one nonterminal third-stage ER-SDE update."""
|
| 286 |
+
|
| 287 |
+
er_lambda_s = er_lambdas[index]
|
| 288 |
+
er_lambda_t = er_lambdas[index + 1]
|
| 289 |
+
alpha_s = times[index] / er_lambda_s
|
| 290 |
+
alpha_t = times[index + 1] / er_lambda_t
|
| 291 |
+
ratio = _er_noise_scaler(er_lambda_t) / _er_noise_scaler(er_lambda_s)
|
| 292 |
+
next_state = (alpha_t / alpha_s) * ratio * state
|
| 293 |
+
next_state = next_state + alpha_t * (1.0 - ratio) * denoised
|
| 294 |
+
|
| 295 |
+
denoised_derivative: Tensor | None = None
|
| 296 |
+
stage = min(3, int(index) + 1)
|
| 297 |
+
if stage >= 2:
|
| 298 |
+
if old_denoised is None:
|
| 299 |
+
raise RuntimeError("ER-SDE stage 2 requires denoised history.")
|
| 300 |
+
second, third = _er_quadrature_terms(
|
| 301 |
+
er_lambda_s,
|
| 302 |
+
er_lambda_t,
|
| 303 |
+
nodes,
|
| 304 |
+
weights,
|
| 305 |
+
)
|
| 306 |
+
delta = er_lambda_t - er_lambda_s
|
| 307 |
+
previous_delta = er_lambda_s - er_lambdas[index - 1]
|
| 308 |
+
denoised_derivative = (denoised - old_denoised) / previous_delta
|
| 309 |
+
next_state = (
|
| 310 |
+
next_state
|
| 311 |
+
+ alpha_t
|
| 312 |
+
* (delta + second * _er_noise_scaler(er_lambda_t))
|
| 313 |
+
* denoised_derivative
|
| 314 |
+
)
|
| 315 |
+
if stage >= 3:
|
| 316 |
+
if old_denoised_derivative is None:
|
| 317 |
+
raise RuntimeError("ER-SDE stage 3 requires derivative history.")
|
| 318 |
+
derivative_span = (er_lambda_s - er_lambdas[index - 2]) / 2.0
|
| 319 |
+
second_derivative = (
|
| 320 |
+
denoised_derivative - old_denoised_derivative
|
| 321 |
+
) / derivative_span
|
| 322 |
+
next_state = (
|
| 323 |
+
next_state
|
| 324 |
+
+ alpha_t
|
| 325 |
+
* (delta.square() / 2.0 + third * _er_noise_scaler(er_lambda_t))
|
| 326 |
+
* second_derivative
|
| 327 |
+
)
|
| 328 |
+
|
| 329 |
+
if float(noise_multiplier) > 0.0:
|
| 330 |
+
noise = torch.randn(
|
| 331 |
+
state.shape,
|
| 332 |
+
device=state.device,
|
| 333 |
+
dtype=torch.float32,
|
| 334 |
+
generator=generator,
|
| 335 |
+
)
|
| 336 |
+
variance = er_lambda_t.square() - er_lambda_s.square() * ratio.square()
|
| 337 |
+
stochastic_scale = alpha_t * torch.sqrt(torch.clamp(variance, min=0.0))
|
| 338 |
+
next_state = next_state + float(noise_multiplier) * stochastic_scale * noise
|
| 339 |
+
return next_state.float(), denoised_derivative
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
def _er_sde(
|
| 343 |
+
velocity: VelocityFunction,
|
| 344 |
+
state: Tensor,
|
| 345 |
+
schedule: Tensor,
|
| 346 |
+
*,
|
| 347 |
+
generator: torch.Generator,
|
| 348 |
+
noise_multiplier: float,
|
| 349 |
+
progress: SolverProgress,
|
| 350 |
+
) -> Tensor:
|
| 351 |
+
"""Integrate with third-stage VP ER-SDE and Gauss-Legendre quadrature."""
|
| 352 |
+
|
| 353 |
+
times = _prepare_logsnr_schedule(schedule, solver_name="ER-SDE")
|
| 354 |
+
half_log_snr = torch.log1p(-times.float()) - torch.log(times.float())
|
| 355 |
+
er_lambdas = torch.exp(-half_log_snr)
|
| 356 |
+
nodes, weights = _er_quadrature_rule(device=state.device)
|
| 357 |
+
old_denoised: Tensor | None = None
|
| 358 |
+
old_denoised_derivative: Tensor | None = None
|
| 359 |
+
batch = int(state.shape[0])
|
| 360 |
+
intervals = int(times.numel()) - 1
|
| 361 |
+
for index in range(intervals):
|
| 362 |
+
current_time = times[index]
|
| 363 |
+
predicted = velocity(
|
| 364 |
+
state,
|
| 365 |
+
_time_batch(current_time, batch, state.device),
|
| 366 |
+
index,
|
| 367 |
+
).float()
|
| 368 |
+
denoised = state - current_time * predicted
|
| 369 |
+
if float(times[index + 1].item()) == 0.0:
|
| 370 |
+
state = denoised
|
| 371 |
+
else:
|
| 372 |
+
state, old_denoised_derivative = _er_sde_step(
|
| 373 |
+
state=state,
|
| 374 |
+
denoised=denoised,
|
| 375 |
+
old_denoised=old_denoised,
|
| 376 |
+
old_denoised_derivative=old_denoised_derivative,
|
| 377 |
+
times=times,
|
| 378 |
+
er_lambdas=er_lambdas,
|
| 379 |
+
index=index,
|
| 380 |
+
nodes=nodes,
|
| 381 |
+
weights=weights,
|
| 382 |
+
generator=generator,
|
| 383 |
+
noise_multiplier=float(noise_multiplier),
|
| 384 |
+
)
|
| 385 |
+
old_denoised = denoised
|
| 386 |
+
progress(index + 1, intervals)
|
| 387 |
+
return state
|
| 388 |
+
|
| 389 |
+
|
| 390 |
def _ab2_predict(
|
| 391 |
state: Tensor,
|
| 392 |
current_step: Tensor,
|
|
|
|
| 462 |
solver: Solver,
|
| 463 |
generator: torch.Generator,
|
| 464 |
euler_maruyama_multiplier: float,
|
| 465 |
+
er_sde_noise_multiplier: float = 1.0,
|
| 466 |
progress: SolverProgress | None = None,
|
| 467 |
) -> Tensor:
|
| 468 |
"""Integrate Canter velocity predictions over one validated schedule."""
|
|
|
|
| 470 |
_validate_inputs(initial_state, schedule)
|
| 471 |
if not isinstance(solver, Solver):
|
| 472 |
raise TypeError("solver must be a Solver.")
|
| 473 |
+
em_multiplier = float(euler_maruyama_multiplier)
|
| 474 |
+
if not math.isfinite(em_multiplier) or em_multiplier < 0.0:
|
| 475 |
raise ValueError("euler_maruyama_multiplier must be finite and non-negative.")
|
| 476 |
+
er_multiplier = float(er_sde_noise_multiplier)
|
| 477 |
+
if not math.isfinite(er_multiplier) or er_multiplier < 0.0:
|
| 478 |
+
raise ValueError("er_sde_noise_multiplier must be finite and non-negative.")
|
| 479 |
resolved_progress = _ignore_progress if progress is None else progress
|
| 480 |
match solver:
|
| 481 |
case Solver.EULER:
|
|
|
|
| 486 |
initial_state,
|
| 487 |
schedule,
|
| 488 |
generator=generator,
|
| 489 |
+
multiplier=em_multiplier,
|
| 490 |
+
progress=resolved_progress,
|
| 491 |
+
)
|
| 492 |
+
case Solver.ER_SDE:
|
| 493 |
+
return _er_sde(
|
| 494 |
+
velocity,
|
| 495 |
+
initial_state,
|
| 496 |
+
schedule,
|
| 497 |
+
generator=generator,
|
| 498 |
+
noise_multiplier=er_multiplier,
|
| 499 |
progress=resolved_progress,
|
| 500 |
)
|
| 501 |
case Solver.DPMPP_2M:
|
canter/version.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Single source of truth for the installable Canter code version."""
|
| 2 |
+
|
| 3 |
+
__version__ = "0.2.0"
|
| 4 |
+
CANTER_VERSION = __version__
|
canter/webui.py
CHANGED
|
@@ -59,6 +59,7 @@ _TEXT_BACKENDS = {value.value: value for value in TextAttentionBackend}
|
|
| 59 |
_SOLVER_CHOICES = (
|
| 60 |
("ABM2", Solver.ABM2.value),
|
| 61 |
("DPM++ 2M", Solver.DPMPP_2M.value),
|
|
|
|
| 62 |
("Euler", Solver.EULER.value),
|
| 63 |
("Euler-Maruyama", Solver.EULER_MARUYAMA.value),
|
| 64 |
)
|
|
@@ -1048,7 +1049,11 @@ def build_web_request(
|
|
| 1048 |
),
|
| 1049 |
euler_maruyama_multiplier=_number(
|
| 1050 |
euler_maruyama_multiplier,
|
| 1051 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1052 |
),
|
| 1053 |
seed=resolved_seed,
|
| 1054 |
generator=None,
|
|
@@ -1130,6 +1135,8 @@ def _png_metadata_json(
|
|
| 1130 |
"self_attention_gain": inference.self_attention_gain,
|
| 1131 |
"log_snr_shift": inference.log_snr_shift,
|
| 1132 |
"euler_maruyama_multiplier": inference.euler_maruyama_multiplier,
|
|
|
|
|
|
|
| 1133 |
"release": metadata.canter.release,
|
| 1134 |
"weight_dtype": metadata.canter.weight_dtype.value,
|
| 1135 |
}
|
|
@@ -1164,7 +1171,9 @@ def _model_summary(metadata: CanterPipelineMetadata) -> str:
|
|
| 1164 |
|
| 1165 |
canter = metadata.canter
|
| 1166 |
return (
|
| 1167 |
-
f"**Canter {canter.release}** 路
|
|
|
|
|
|
|
| 1168 |
f"VAE `{metadata.vae_repository}` @ `{metadata.vae_revision[:12]}`"
|
| 1169 |
)
|
| 1170 |
|
|
@@ -1238,7 +1247,8 @@ def _generation_status(
|
|
| 1238 |
f"{inference.width}脳{inference.height} 路 "
|
| 1239 |
f"{inference.steps} {inference.solver.value} updates 路 "
|
| 1240 |
f"{_pdg_status(inference)} 路 "
|
| 1241 |
-
f"Canter `{metadata.
|
|
|
|
| 1242 |
)
|
| 1243 |
|
| 1244 |
|
|
@@ -1363,7 +1373,7 @@ def _solver_inputs() -> tuple[
|
|
| 1363 |
maximum=2.0,
|
| 1364 |
value=_DEFAULT_INFERENCE.euler_maruyama_multiplier,
|
| 1365 |
step=0.05,
|
| 1366 |
-
label="
|
| 1367 |
)
|
| 1368 |
return (
|
| 1369 |
steps,
|
|
@@ -1794,7 +1804,10 @@ def _argument_parser() -> argparse.ArgumentParser:
|
|
| 1794 |
description="Launch the bundled Canter Gradio inference application.",
|
| 1795 |
)
|
| 1796 |
parser.add_argument("--model", default=_DEFAULT_MODEL)
|
| 1797 |
-
parser.add_argument(
|
|
|
|
|
|
|
|
|
|
| 1798 |
parser.add_argument(
|
| 1799 |
"--dtype",
|
| 1800 |
choices=tuple(_WEIGHT_DTYPES),
|
|
|
|
| 59 |
_SOLVER_CHOICES = (
|
| 60 |
("ABM2", Solver.ABM2.value),
|
| 61 |
("DPM++ 2M", Solver.DPMPP_2M.value),
|
| 62 |
+
("ER-SDE", Solver.ER_SDE.value),
|
| 63 |
("Euler", Solver.EULER.value),
|
| 64 |
("Euler-Maruyama", Solver.EULER_MARUYAMA.value),
|
| 65 |
)
|
|
|
|
| 1049 |
),
|
| 1050 |
euler_maruyama_multiplier=_number(
|
| 1051 |
euler_maruyama_multiplier,
|
| 1052 |
+
"SDE noise multiplier",
|
| 1053 |
+
),
|
| 1054 |
+
er_sde_noise_multiplier=_number(
|
| 1055 |
+
euler_maruyama_multiplier,
|
| 1056 |
+
"SDE noise multiplier",
|
| 1057 |
),
|
| 1058 |
seed=resolved_seed,
|
| 1059 |
generator=None,
|
|
|
|
| 1135 |
"self_attention_gain": inference.self_attention_gain,
|
| 1136 |
"log_snr_shift": inference.log_snr_shift,
|
| 1137 |
"euler_maruyama_multiplier": inference.euler_maruyama_multiplier,
|
| 1138 |
+
"er_sde_noise_multiplier": inference.er_sde_noise_multiplier,
|
| 1139 |
+
"code_version": metadata.code_version,
|
| 1140 |
"release": metadata.canter.release,
|
| 1141 |
"weight_dtype": metadata.canter.weight_dtype.value,
|
| 1142 |
}
|
|
|
|
| 1171 |
|
| 1172 |
canter = metadata.canter
|
| 1173 |
return (
|
| 1174 |
+
f"**Canter checkpoint {canter.release}** 路 "
|
| 1175 |
+
f"code `{metadata.code_version}` 路 "
|
| 1176 |
+
f"{canter.weight_dtype.value} EMA weights \n"
|
| 1177 |
f"VAE `{metadata.vae_repository}` @ `{metadata.vae_revision[:12]}`"
|
| 1178 |
)
|
| 1179 |
|
|
|
|
| 1247 |
f"{inference.width}脳{inference.height} 路 "
|
| 1248 |
f"{inference.steps} {inference.solver.value} updates 路 "
|
| 1249 |
f"{_pdg_status(inference)} 路 "
|
| 1250 |
+
f"Canter code `{metadata.code_version}` 路 "
|
| 1251 |
+
f"checkpoint `{metadata.canter.release}`"
|
| 1252 |
)
|
| 1253 |
|
| 1254 |
|
|
|
|
| 1373 |
maximum=2.0,
|
| 1374 |
value=_DEFAULT_INFERENCE.euler_maruyama_multiplier,
|
| 1375 |
step=0.05,
|
| 1376 |
+
label="SDE noise multiplier",
|
| 1377 |
)
|
| 1378 |
return (
|
| 1379 |
steps,
|
|
|
|
| 1804 |
description="Launch the bundled Canter Gradio inference application.",
|
| 1805 |
)
|
| 1806 |
parser.add_argument("--model", default=_DEFAULT_MODEL)
|
| 1807 |
+
parser.add_argument(
|
| 1808 |
+
"--revision",
|
| 1809 |
+
help="Checkpoint branch, commit, or immutable tag such as v0001.",
|
| 1810 |
+
)
|
| 1811 |
parser.add_argument(
|
| 1812 |
"--dtype",
|
| 1813 |
choices=tuple(_WEIGHT_DTYPES),
|
pyproject.toml
CHANGED
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
| 4 |
|
| 5 |
[project]
|
| 6 |
name = "canter"
|
| 7 |
-
|
| 8 |
description = "Lean inference package for the Canter text-to-image flow model"
|
| 9 |
readme = "README.md"
|
| 10 |
requires-python = ">=3.10,<3.14"
|
|
@@ -31,3 +31,6 @@ test = [
|
|
| 31 |
|
| 32 |
[tool.hatch.build.targets.wheel]
|
| 33 |
packages = ["canter"]
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
[project]
|
| 6 |
name = "canter"
|
| 7 |
+
dynamic = ["version"]
|
| 8 |
description = "Lean inference package for the Canter text-to-image flow model"
|
| 9 |
readme = "README.md"
|
| 10 |
requires-python = ">=3.10,<3.14"
|
|
|
|
| 31 |
|
| 32 |
[tool.hatch.build.targets.wheel]
|
| 33 |
packages = ["canter"]
|
| 34 |
+
|
| 35 |
+
[tool.hatch.version]
|
| 36 |
+
path = "canter/version.py"
|