Instructions to use dvdface/next-frame-predict with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- TF-Keras
How to use dvdface/next-frame-predict with TF-Keras:
# Note: 'keras<3.x' or 'tf_keras' must be installed (legacy), and from_pretrained_keras was removed in huggingface_hub 1.0. # See https://github.com/keras-team/tf-keras for more details. # !pip install "huggingface_hub<1.0" tf_keras from huggingface_hub import from_pretrained_keras model = from_pretrained_keras("dvdface/next-frame-predict") - Notebooks
- Google Colab
- Kaggle
Simplify project structure: rename dirs and files
Browse files- .gitattributes +1 -0
- README.md +71 -0
- anomaly.py +87 -0
- config.json +16 -0
- configuration.py +10 -0
- dataset_test.py +511 -0
- infer/__init__.py +4 -0
- infer/anomaly.py +65 -0
- infer/io.py +75 -0
- infer/predictor.py +210 -0
- modeling.py +39 -0
- pipeline.py +43 -0
- predict.py +66 -0
- requirements.txt +6 -0
- savedmodel/fingerprint.pb +3 -0
- savedmodel/saved_model.pb +3 -0
- savedmodel/variables/variables.data-00000-of-00001 +3 -0
- savedmodel/variables/variables.index +0 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
savedmodel/variables/variables.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## PredNet denoising (black-box SavedModel)
|
| 2 |
+
|
| 3 |
+
This repository packages a TensorFlow **SavedModel** for video-frame prediction/denoising **without publishing any model architecture source code**. Inference calls the SavedModel signature as a black box and produces predicted frames.
|
| 4 |
+
|
| 5 |
+
### Files
|
| 6 |
+
|
| 7 |
+
- `savedmodel/`: TensorFlow SavedModel directory (weights + graph)
|
| 8 |
+
- `infer/`: minimal inference helpers (black-box loader + IO)
|
| 9 |
+
- `predict.py`: CLI that outputs prediction images
|
| 10 |
+
- `anomaly.py`: CLI that outputs per-frame anomaly scores (ahat_error + 1-SSIM vectors)
|
| 11 |
+
|
| 12 |
+
### Install
|
| 13 |
+
|
| 14 |
+
```bash
|
| 15 |
+
pip install -r requirements.txt
|
| 16 |
+
```
|
| 17 |
+
|
| 18 |
+
### Run (from a directory of frames)
|
| 19 |
+
|
| 20 |
+
`frames_dir` should contain ordered image frames (e.g. `0001.png`, `0002.png`, ...).
|
| 21 |
+
|
| 22 |
+
```bash
|
| 23 |
+
python predict.py --model_dir savedmodel --frames_dir /path/to/frames --out_dir outputs --save_sequence_grid
|
| 24 |
+
```
|
| 25 |
+
|
| 26 |
+
Outputs:
|
| 27 |
+
- `outputs/pred_last.png`
|
| 28 |
+
- `outputs/pred_sequence_grid.png` (optional)
|
| 29 |
+
|
| 30 |
+
### Run (from npy/npz)
|
| 31 |
+
|
| 32 |
+
Accepts `frames` stored as `[T,H,W,C]` or `[B,T,H,W,C]`.
|
| 33 |
+
|
| 34 |
+
```bash
|
| 35 |
+
python predict.py --array frames.npy --out_dir outputs
|
| 36 |
+
```
|
| 37 |
+
|
| 38 |
+
### Anomaly scores (4-frame window)
|
| 39 |
+
|
| 40 |
+
For each window of 4 frames ending at time `t`, this computes two vectors of length 4:
|
| 41 |
+
- `ahat_error[1..4]`: mean absolute error between `pred_frame_k` and `GT_frame_4`
|
| 42 |
+
- `dissim_1mssim[1..4]`: `1 - SSIM(pred_frame_k, GT_frame_4)`
|
| 43 |
+
|
| 44 |
+
```bash
|
| 45 |
+
python anomaly.py --model_dir savedmodel --frames_dir /path/to/frames --out_json anomaly.json --out_csv anomaly.csv
|
| 46 |
+
```
|
| 47 |
+
|
| 48 |
+
### Hugging Face Hub usage
|
| 49 |
+
|
| 50 |
+
After you upload this repo to the Hub, users can download it via `huggingface_hub` and point `--model_dir` at the downloaded `savedmodel/` directory.
|
| 51 |
+
|
| 52 |
+
### HuggingFace Pipeline Usage
|
| 53 |
+
|
| 54 |
+
```python
|
| 55 |
+
from transformers import pipeline
|
| 56 |
+
import numpy as np
|
| 57 |
+
|
| 58 |
+
pipe = pipeline(
|
| 59 |
+
"video-frame-prediction",
|
| 60 |
+
model="dvdface/denoising-prednet",
|
| 61 |
+
trust_remote_code=True,
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
# frames: [T, H, W, C] numpy array, uint8 (0-255) or float (0-1)
|
| 65 |
+
frames = np.random.randint(0, 255, (4, 128, 128, 3), dtype=np.uint8)
|
| 66 |
+
result = pipe(frames)
|
| 67 |
+
|
| 68 |
+
print(result["sequence"].shape) # (4, 128, 128, 3)
|
| 69 |
+
print(result["last_frame"].shape) # (128, 128, 3)
|
| 70 |
+
```
|
| 71 |
+
|
anomaly.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import csv
|
| 5 |
+
import json
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Any, Dict, List, Optional
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
|
| 11 |
+
from infer.io import load_array, load_frames_from_dir
|
| 12 |
+
from infer.predictor import Predictor
|
| 13 |
+
from infer.anomaly import compute_anomaly_from_4frames
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _parse_args() -> argparse.Namespace:
|
| 17 |
+
ap = argparse.ArgumentParser(description="Compute per-window anomaly scores (ahat_error and 1-SSIM vectors).")
|
| 18 |
+
ap.add_argument("--model_dir", type=str, default="savedmodel")
|
| 19 |
+
ap.add_argument("--frames_dir", type=str, default=None)
|
| 20 |
+
ap.add_argument("--array", type=str, default=None)
|
| 21 |
+
ap.add_argument("--out_json", type=str, default="anomaly.json")
|
| 22 |
+
ap.add_argument("--out_csv", type=str, default="anomaly.csv")
|
| 23 |
+
ap.add_argument("--start", type=int, default=0, help="Start frame index (inclusive) on the full sequence")
|
| 24 |
+
ap.add_argument("--end", type=int, default=-1, help="End frame index (exclusive); -1 means to the end")
|
| 25 |
+
return ap.parse_args()
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def main() -> None:
|
| 29 |
+
args = _parse_args()
|
| 30 |
+
if (args.frames_dir is None) == (args.array is None):
|
| 31 |
+
raise SystemExit("Provide exactly one of --frames_dir or --array")
|
| 32 |
+
|
| 33 |
+
if args.frames_dir is not None:
|
| 34 |
+
frames = load_frames_from_dir(args.frames_dir) # [T,H,W,C] 0..255
|
| 35 |
+
else:
|
| 36 |
+
frames = load_array(args.array)
|
| 37 |
+
|
| 38 |
+
if frames.ndim == 5:
|
| 39 |
+
if frames.shape[0] != 1:
|
| 40 |
+
raise SystemExit("Batch arrays are not supported in CLI; please provide a single sequence.")
|
| 41 |
+
frames = frames[0]
|
| 42 |
+
if frames.ndim != 4 or frames.shape[0] < 4:
|
| 43 |
+
raise SystemExit(f"Expected frames with shape [T,H,W,C] and T>=4, got {frames.shape}")
|
| 44 |
+
|
| 45 |
+
T = frames.shape[0]
|
| 46 |
+
start = max(0, int(args.start))
|
| 47 |
+
end = T if int(args.end) == -1 else min(T, int(args.end))
|
| 48 |
+
if end - start < 4:
|
| 49 |
+
raise SystemExit("Range too small; need at least 4 frames.")
|
| 50 |
+
|
| 51 |
+
pred = Predictor(args.model_dir)
|
| 52 |
+
if pred.sequence_len is not None and pred.sequence_len != 4:
|
| 53 |
+
raise SystemExit(f"This tool assumes model sequence_len=4, got {pred.sequence_len}")
|
| 54 |
+
|
| 55 |
+
rows: List[Dict[str, Any]] = []
|
| 56 |
+
for t in range(start + 3, end):
|
| 57 |
+
win = frames[t - 3 : t + 1] # [4,H,W,C] where win[3] is GT frame4
|
| 58 |
+
res = compute_anomaly_from_4frames(pred, win)
|
| 59 |
+
rows.append(
|
| 60 |
+
{
|
| 61 |
+
"t": int(t),
|
| 62 |
+
"ahat_error": res.ahat_error.tolist(),
|
| 63 |
+
"dissim_1mssim": res.dissim_1mssim.tolist(),
|
| 64 |
+
}
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
out_json = Path(args.out_json)
|
| 68 |
+
out_json.parent.mkdir(parents=True, exist_ok=True)
|
| 69 |
+
out_json.write_text(json.dumps({"items": rows}, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 70 |
+
|
| 71 |
+
out_csv = Path(args.out_csv)
|
| 72 |
+
out_csv.parent.mkdir(parents=True, exist_ok=True)
|
| 73 |
+
with out_csv.open("w", newline="", encoding="utf-8") as f:
|
| 74 |
+
w = csv.writer(f)
|
| 75 |
+
w.writerow(["t", "ahat_error_1", "ahat_error_2", "ahat_error_3", "ahat_error_4", "dissim_1", "dissim_2", "dissim_3", "dissim_4"])
|
| 76 |
+
for r in rows:
|
| 77 |
+
a = r["ahat_error"]
|
| 78 |
+
d = r["dissim_1mssim"]
|
| 79 |
+
w.writerow([r["t"], *a, *d])
|
| 80 |
+
|
| 81 |
+
print(f"Wrote: {out_json}")
|
| 82 |
+
print(f"Wrote: {out_csv}")
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
if __name__ == "__main__":
|
| 86 |
+
main()
|
| 87 |
+
|
config.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_type": "prednet",
|
| 3 |
+
"architectures": ["PredNetModel"],
|
| 4 |
+
"auto_map": {
|
| 5 |
+
"AutoModel": "modeling.PredNetModel"
|
| 6 |
+
},
|
| 7 |
+
"custom_pipelines": {
|
| 8 |
+
"video-frame-prediction": {
|
| 9 |
+
"impl": "pipeline.VideoFramePredictionPipeline",
|
| 10 |
+
"pt": [],
|
| 11 |
+
"tf": ["AutoModel"]
|
| 12 |
+
}
|
| 13 |
+
},
|
| 14 |
+
"sequence_len": 4,
|
| 15 |
+
"resize_hw": [128, 128]
|
| 16 |
+
}
|
configuration.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from transformers import PretrainedConfig
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class PredNetConfig(PretrainedConfig):
|
| 5 |
+
model_type = "prednet"
|
| 6 |
+
|
| 7 |
+
def __init__(self, sequence_len=4, resize_hw=(128, 128), **kwargs):
|
| 8 |
+
super().__init__(**kwargs)
|
| 9 |
+
self.sequence_len = sequence_len
|
| 10 |
+
self.resize_hw = list(resize_hw)
|
dataset_test.py
ADDED
|
@@ -0,0 +1,511 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import math
|
| 5 |
+
import re
|
| 6 |
+
from dataclasses import dataclass
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import List, Optional, Sequence, Tuple, Union
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import tensorflow as tf
|
| 12 |
+
|
| 13 |
+
from infer.io import save_image, save_sequence_grid
|
| 14 |
+
from infer.predictor import Predictor
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _natural_key(s: str):
|
| 18 |
+
return [int(t) if t.isdigit() else t.lower() for t in re.split(r"(\d+)", s)]
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _timestamp_from_name(p: Path) -> Optional[int]:
|
| 22 |
+
m = re.match(r"^(\d+)", p.stem)
|
| 23 |
+
if not m:
|
| 24 |
+
return None
|
| 25 |
+
try:
|
| 26 |
+
return int(m.group(1))
|
| 27 |
+
except ValueError:
|
| 28 |
+
return None
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _parse_unix_timestamp_ms(path: Union[str, Path]) -> Optional[int]:
|
| 32 |
+
"""
|
| 33 |
+
Mirror notebook logic:
|
| 34 |
+
- >=19 digits: ns -> ms
|
| 35 |
+
- >=16 digits: us -> ms
|
| 36 |
+
- >=13 digits: ms
|
| 37 |
+
- >=10 digits: s -> ms
|
| 38 |
+
else: None
|
| 39 |
+
"""
|
| 40 |
+
stem = Path(path).stem
|
| 41 |
+
if not stem.isdigit():
|
| 42 |
+
return None
|
| 43 |
+
raw = int(stem)
|
| 44 |
+
digits = len(stem)
|
| 45 |
+
if digits >= 19:
|
| 46 |
+
return raw // 1_000_000
|
| 47 |
+
if digits >= 16:
|
| 48 |
+
return raw // 1_000
|
| 49 |
+
if digits >= 13:
|
| 50 |
+
return raw
|
| 51 |
+
if digits >= 10:
|
| 52 |
+
return raw * 1_000
|
| 53 |
+
return None
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _has_valid_adjacent_timestamp_gap(paths: Sequence[Path], max_gap_ms: int = 32) -> bool:
|
| 57 |
+
timestamps = [_parse_unix_timestamp_ms(p) for p in paths]
|
| 58 |
+
if any(ts is None for ts in timestamps):
|
| 59 |
+
return True
|
| 60 |
+
for prev_ts, cur_ts in zip(timestamps, timestamps[1:]):
|
| 61 |
+
if cur_ts < prev_ts or (cur_ts - prev_ts) > max_gap_ms:
|
| 62 |
+
return False
|
| 63 |
+
return True
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _gap_ms(ts0: int, ts1: int) -> float:
|
| 67 |
+
diff = ts1 - ts0
|
| 68 |
+
if diff <= 0:
|
| 69 |
+
return 0.0
|
| 70 |
+
# Treat timestamps as ms already (they come from filenames); this is only used
|
| 71 |
+
# by the legacy top-k mode which splits clips first.
|
| 72 |
+
return float(diff)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _resize_np(x: np.ndarray, hw: Tuple[int, int]) -> np.ndarray:
|
| 76 |
+
t, h, w, c = x.shape
|
| 77 |
+
y = tf.image.resize(tf.convert_to_tensor(x, tf.float32), hw, method="bilinear", antialias=True)
|
| 78 |
+
return y.numpy()
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _ssim_adjacent(frames01: np.ndarray, ssim_hw: Tuple[int, int]) -> np.ndarray:
|
| 82 |
+
# frames01: [T,H,W,C] float32 0..1
|
| 83 |
+
x = frames01
|
| 84 |
+
if x.shape[1:3] != (ssim_hw[0], ssim_hw[1]):
|
| 85 |
+
x = _resize_np(x, ssim_hw)
|
| 86 |
+
# tf.image.ssim expects [N,H,W,C]
|
| 87 |
+
a = tf.convert_to_tensor(x[:-1], tf.float32)
|
| 88 |
+
b = tf.convert_to_tensor(x[1:], tf.float32)
|
| 89 |
+
s = tf.image.ssim(a, b, max_val=1.0) # [T-1]
|
| 90 |
+
return s.numpy()
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _find_motion_segments_in_clip(frames01: np.ndarray, threshold: float, ssim_hw: Tuple[int, int]) -> List[Tuple[int, int, float]]:
|
| 94 |
+
"""
|
| 95 |
+
Return segments [start,end) where (1-SSIM) > threshold on adjacent pairs.
|
| 96 |
+
Used by legacy top-k selection mode.
|
| 97 |
+
"""
|
| 98 |
+
if frames01.shape[0] < 2:
|
| 99 |
+
return []
|
| 100 |
+
ssim = _ssim_adjacent(frames01, ssim_hw) # len T-1
|
| 101 |
+
motion = 1.0 - ssim
|
| 102 |
+
mask = motion > threshold
|
| 103 |
+
segs: List[Tuple[int, int, float]] = []
|
| 104 |
+
i = 0
|
| 105 |
+
while i < len(mask):
|
| 106 |
+
if not mask[i]:
|
| 107 |
+
i += 1
|
| 108 |
+
continue
|
| 109 |
+
j = i
|
| 110 |
+
while j < len(mask) and mask[j]:
|
| 111 |
+
j += 1
|
| 112 |
+
start = i
|
| 113 |
+
end = j + 1
|
| 114 |
+
score = float(np.mean(motion[i:j])) if j > i else 0.0
|
| 115 |
+
segs.append((start, end, score))
|
| 116 |
+
i = j
|
| 117 |
+
return segs
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def _split_by_timestamp_gap(paths: List[Path], max_gap_ms: int) -> List[List[Path]]:
|
| 121 |
+
clips: List[List[Path]] = []
|
| 122 |
+
cur: List[Path] = []
|
| 123 |
+
prev_ts_ms: Optional[int] = None
|
| 124 |
+
for p in paths:
|
| 125 |
+
ts_ms = _parse_unix_timestamp_ms(p)
|
| 126 |
+
if ts_ms is None:
|
| 127 |
+
# If we can't parse timestamp, keep as continuous.
|
| 128 |
+
if not cur:
|
| 129 |
+
cur = [p]
|
| 130 |
+
else:
|
| 131 |
+
cur.append(p)
|
| 132 |
+
continue
|
| 133 |
+
if prev_ts_ms is None:
|
| 134 |
+
cur = [p]
|
| 135 |
+
prev_ts_ms = ts_ms
|
| 136 |
+
continue
|
| 137 |
+
if (ts_ms - prev_ts_ms) > int(max_gap_ms):
|
| 138 |
+
if cur:
|
| 139 |
+
clips.append(cur)
|
| 140 |
+
cur = [p]
|
| 141 |
+
else:
|
| 142 |
+
cur.append(p)
|
| 143 |
+
prev_ts_ms = ts_ms
|
| 144 |
+
if cur:
|
| 145 |
+
clips.append(cur)
|
| 146 |
+
return clips
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
@dataclass
|
| 150 |
+
class Segment:
|
| 151 |
+
clip_index: int
|
| 152 |
+
start: int # inclusive index in clip
|
| 153 |
+
end: int # exclusive index in clip
|
| 154 |
+
score: float
|
| 155 |
+
|
| 156 |
+
@property
|
| 157 |
+
def length(self) -> int:
|
| 158 |
+
return max(0, self.end - self.start)
|
| 159 |
+
|
| 160 |
+
def sort_key(self) -> Tuple[float, int]:
|
| 161 |
+
# Higher score first, then longer
|
| 162 |
+
return (self.score, self.length)
|
| 163 |
+
|
| 164 |
+
def overlaps(self, other: "Segment") -> bool:
|
| 165 |
+
if self.clip_index != other.clip_index:
|
| 166 |
+
return False
|
| 167 |
+
return not (self.end <= other.start or other.end <= self.start)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _ssim_score_np(img_a: np.ndarray, img_b: np.ndarray, c1: float = 0.01**2, c2: float = 0.03**2) -> float:
|
| 171 |
+
# Mirror notebook SSIM (mean/var based, no gaussian window)
|
| 172 |
+
mu_a = float(img_a.mean())
|
| 173 |
+
mu_b = float(img_b.mean())
|
| 174 |
+
sigma_a = float(img_a.var())
|
| 175 |
+
sigma_b = float(img_b.var())
|
| 176 |
+
sigma_ab = float(((img_a - mu_a) * (img_b - mu_b)).mean())
|
| 177 |
+
numerator = (2.0 * mu_a * mu_b + c1) * (2.0 * sigma_ab + c2)
|
| 178 |
+
denominator = (mu_a * mu_a + mu_b * mu_b + c1) * (sigma_a + sigma_b + c2) + 1e-8
|
| 179 |
+
return float(numerator / denominator)
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def _to_gray_resized01(rgb01: np.ndarray, hw: Tuple[int, int]) -> np.ndarray:
|
| 183 |
+
# rgb01: [H,W,3] 0..1
|
| 184 |
+
x = tf.image.resize(tf.convert_to_tensor(rgb01[None, ...], tf.float32), hw, method="bilinear", antialias=True)[0]
|
| 185 |
+
g = tf.reduce_mean(x, axis=-1) # [H,W]
|
| 186 |
+
return g.numpy()
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def _find_motion_segments_notebook_style(
|
| 190 |
+
clip_paths: Sequence[Path],
|
| 191 |
+
threshold: float,
|
| 192 |
+
ssim_hw: Tuple[int, int],
|
| 193 |
+
) -> List[Tuple[int, int, float]]:
|
| 194 |
+
"""
|
| 195 |
+
Mirror notebook find_motion_segments:
|
| 196 |
+
- compute delta_ssim = 1 - ssim_score_np(gray[i-1], gray[i])
|
| 197 |
+
- enter motion when delta > threshold (start=i-1)
|
| 198 |
+
- exit when delta <= threshold (append (start, i))
|
| 199 |
+
- if still in motion at end, append (start, len-1)
|
| 200 |
+
We also compute score as mean delta over the segment transitions.
|
| 201 |
+
"""
|
| 202 |
+
if len(clip_paths) < 2:
|
| 203 |
+
return []
|
| 204 |
+
|
| 205 |
+
# Streamed implementation to avoid loading the whole clip into memory.
|
| 206 |
+
from PIL import Image
|
| 207 |
+
|
| 208 |
+
def load_gray(p: Path) -> np.ndarray:
|
| 209 |
+
rgb = np.asarray(Image.open(p).convert("RGB"), dtype=np.float32) / 255.0
|
| 210 |
+
return _to_gray_resized01(rgb, ssim_hw)
|
| 211 |
+
|
| 212 |
+
segments: List[Tuple[int, int, float]] = []
|
| 213 |
+
in_motion = False
|
| 214 |
+
start_idx = -1
|
| 215 |
+
acc = 0.0
|
| 216 |
+
acc_n = 0
|
| 217 |
+
|
| 218 |
+
prev_gray = load_gray(clip_paths[0])
|
| 219 |
+
for i in range(1, len(clip_paths)):
|
| 220 |
+
cur_gray = load_gray(clip_paths[i])
|
| 221 |
+
delta_ssim = 1.0 - _ssim_score_np(prev_gray, cur_gray)
|
| 222 |
+
|
| 223 |
+
if (not in_motion) and delta_ssim > threshold:
|
| 224 |
+
in_motion = True
|
| 225 |
+
start_idx = i - 1
|
| 226 |
+
acc = float(delta_ssim)
|
| 227 |
+
acc_n = 1
|
| 228 |
+
elif in_motion and delta_ssim > threshold:
|
| 229 |
+
acc += float(delta_ssim)
|
| 230 |
+
acc_n += 1
|
| 231 |
+
elif in_motion and delta_ssim <= threshold:
|
| 232 |
+
end_idx = i # as notebook
|
| 233 |
+
score = (acc / acc_n) if acc_n else 0.0
|
| 234 |
+
segments.append((start_idx, end_idx, score))
|
| 235 |
+
in_motion = False
|
| 236 |
+
acc = 0.0
|
| 237 |
+
acc_n = 0
|
| 238 |
+
|
| 239 |
+
prev_gray = cur_gray
|
| 240 |
+
|
| 241 |
+
if in_motion:
|
| 242 |
+
end_idx = len(clip_paths) - 1
|
| 243 |
+
score = (acc / acc_n) if acc_n else 0.0
|
| 244 |
+
segments.append((start_idx, end_idx, score))
|
| 245 |
+
|
| 246 |
+
return segments
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def _topk_motion_windows(
|
| 250 |
+
frames01: np.ndarray,
|
| 251 |
+
ssim_hw: Tuple[int, int],
|
| 252 |
+
window_len: int,
|
| 253 |
+
step: int = 1,
|
| 254 |
+
) -> List[Tuple[int, int, float]]:
|
| 255 |
+
"""
|
| 256 |
+
Sliding windows scored by mean motion (1-SSIM) inside the window.
|
| 257 |
+
Returns [(start,end,score)] where end=start+window_len.
|
| 258 |
+
"""
|
| 259 |
+
if window_len < 2 or frames01.shape[0] < window_len:
|
| 260 |
+
return []
|
| 261 |
+
ssim = _ssim_adjacent(frames01, ssim_hw) # len T-1
|
| 262 |
+
motion = 1.0 - ssim
|
| 263 |
+
|
| 264 |
+
out: List[Tuple[int, int, float]] = []
|
| 265 |
+
last_start = frames01.shape[0] - window_len
|
| 266 |
+
for start in range(0, last_start + 1, step):
|
| 267 |
+
# motion indices cover transitions; for frames [start, end) with end=start+L,
|
| 268 |
+
# the relevant motion slice is [start, end-1) (length L-1)
|
| 269 |
+
end = start + window_len
|
| 270 |
+
m = motion[start : end - 1]
|
| 271 |
+
score = float(np.mean(m)) if m.size else 0.0
|
| 272 |
+
out.append((start, end, score))
|
| 273 |
+
return out
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def pick_best_segments(
|
| 277 |
+
frames_dir: Path,
|
| 278 |
+
threshold: float = 0.005,
|
| 279 |
+
ssim_hw: Tuple[int, int] = (64, 64),
|
| 280 |
+
max_gap_ms: int = 32,
|
| 281 |
+
min_len: int = 8,
|
| 282 |
+
topk: int = 50,
|
| 283 |
+
k: int = 4,
|
| 284 |
+
) -> Tuple[List[List[Path]], List[Segment]]:
|
| 285 |
+
paths = [p for p in frames_dir.iterdir() if p.is_file() and p.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp", ".bmp"}]
|
| 286 |
+
paths.sort(key=lambda p: _natural_key(p.name))
|
| 287 |
+
if not paths:
|
| 288 |
+
raise ValueError(f"No frames found in {frames_dir}")
|
| 289 |
+
|
| 290 |
+
clips = _split_by_timestamp_gap(paths, max_gap_ms=max_gap_ms)
|
| 291 |
+
|
| 292 |
+
candidates: List[Tuple[List[Path], Segment]] = []
|
| 293 |
+
for ci, clip in enumerate(clips):
|
| 294 |
+
if len(clip) < min_len:
|
| 295 |
+
continue
|
| 296 |
+
arr = []
|
| 297 |
+
from PIL import Image
|
| 298 |
+
|
| 299 |
+
for p in clip:
|
| 300 |
+
arr.append(np.asarray(Image.open(p).convert("RGB"), dtype=np.float32))
|
| 301 |
+
x = np.stack(arr, axis=0)
|
| 302 |
+
x01 = np.clip(x / 255.0, 0.0, 1.0)
|
| 303 |
+
|
| 304 |
+
# Primary: threshold-based motion segments (not always present)
|
| 305 |
+
cand: List[Segment] = []
|
| 306 |
+
for s, e, sc in _find_motion_segments_in_clip(x01, threshold=threshold, ssim_hw=ssim_hw):
|
| 307 |
+
seg = Segment(clip_index=ci, start=s, end=e, score=sc)
|
| 308 |
+
if seg.length >= min_len:
|
| 309 |
+
cand.append(seg)
|
| 310 |
+
|
| 311 |
+
# Fallback/augment: always take top motion windows so we can pick "bigger change"
|
| 312 |
+
# even when thresholding yields nothing.
|
| 313 |
+
windows = _topk_motion_windows(x01, ssim_hw=ssim_hw, window_len=max(min_len, 8), step=1)
|
| 314 |
+
windows.sort(key=lambda t: t[2], reverse=True)
|
| 315 |
+
for s, e, sc in windows[:topk]:
|
| 316 |
+
cand.append(Segment(clip_index=ci, start=s, end=e, score=sc))
|
| 317 |
+
|
| 318 |
+
for seg in cand:
|
| 319 |
+
candidates.append((clip, seg))
|
| 320 |
+
|
| 321 |
+
if not candidates:
|
| 322 |
+
clip = max(clips, key=len)
|
| 323 |
+
seg = Segment(clip_index=clips.index(clip), start=0, end=len(clip), score=0.0)
|
| 324 |
+
return [clip], [seg]
|
| 325 |
+
|
| 326 |
+
# Sort all candidates by score then length
|
| 327 |
+
candidates.sort(key=lambda cs: (cs[1].score, cs[1].length), reverse=True)
|
| 328 |
+
|
| 329 |
+
picked_clips: List[List[Path]] = []
|
| 330 |
+
picked_segs: List[Segment] = []
|
| 331 |
+
for clip, seg in candidates:
|
| 332 |
+
if any(seg.overlaps(s) for s in picked_segs):
|
| 333 |
+
continue
|
| 334 |
+
picked_clips.append(clip)
|
| 335 |
+
picked_segs.append(seg)
|
| 336 |
+
if len(picked_segs) >= k:
|
| 337 |
+
break
|
| 338 |
+
|
| 339 |
+
return picked_clips, picked_segs
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
def find_all_motion_segments(
|
| 343 |
+
frames_dir: Path,
|
| 344 |
+
threshold: float,
|
| 345 |
+
ssim_hw: Tuple[int, int],
|
| 346 |
+
max_gap_ms: int,
|
| 347 |
+
min_len: int,
|
| 348 |
+
) -> Tuple[List[List[Path]], List[Segment]]:
|
| 349 |
+
# Mirror notebook: do NOT split into clips by timestamp gap.
|
| 350 |
+
paths = [p for p in frames_dir.iterdir() if p.is_file() and p.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp", ".bmp"}]
|
| 351 |
+
paths.sort(key=lambda p: _natural_key(p.name))
|
| 352 |
+
if not paths:
|
| 353 |
+
raise ValueError(f"No frames found in {frames_dir}")
|
| 354 |
+
|
| 355 |
+
# Notebook has a single clip here (unless directory-of-directories).
|
| 356 |
+
clip = paths
|
| 357 |
+
segs = _find_motion_segments_notebook_style(clip, threshold=threshold, ssim_hw=ssim_hw)
|
| 358 |
+
|
| 359 |
+
out_clips: List[List[Path]] = []
|
| 360 |
+
out_segs: List[Segment] = []
|
| 361 |
+
for s, e, sc in segs:
|
| 362 |
+
# In notebook segments are tuples (start, end) with mixed inclusivity; for our slicing,
|
| 363 |
+
# treat as [start, end] inclusive if end < len, else [start, end).
|
| 364 |
+
# We'll convert to Python slice end-exclusive conservatively.
|
| 365 |
+
end_excl = min(e + 1, len(clip))
|
| 366 |
+
seg_len = end_excl - s
|
| 367 |
+
if seg_len < min_len:
|
| 368 |
+
continue
|
| 369 |
+
# Also require there exists at least one valid sequence of length (sequence_len) within this segment
|
| 370 |
+
# that passes the adjacent timestamp gap filter, matching notebook behavior.
|
| 371 |
+
out_clips.append(clip)
|
| 372 |
+
out_segs.append(Segment(clip_index=0, start=s, end=end_excl, score=float(sc)))
|
| 373 |
+
|
| 374 |
+
order = sorted(range(len(out_segs)), key=lambda i: (out_segs[i].score, out_segs[i].length), reverse=True)
|
| 375 |
+
return [out_clips[i] for i in order], [out_segs[i] for i in order]
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
def _parse_args() -> argparse.Namespace:
|
| 379 |
+
ap = argparse.ArgumentParser(description="SSIM-filter dataset and generate teacher-forcing predictions.")
|
| 380 |
+
ap.add_argument("--data_dir", type=str, required=True, help="Directory with frame images (e.g. .../data/data)")
|
| 381 |
+
ap.add_argument("--model_dir", type=str, default="savedmodel", help="SavedModel directory")
|
| 382 |
+
ap.add_argument("--out_dir", type=str, default="dataset_outputs", help="Output directory")
|
| 383 |
+
ap.add_argument("--threshold", type=float, default=0.005, help="Motion threshold on (1-SSIM)")
|
| 384 |
+
ap.add_argument("--ssim_hw", type=str, default="64,64", help="SSIM resize H,W")
|
| 385 |
+
ap.add_argument("--max_gap_ms", type=int, default=32, help="Max adjacent timestamp gap (ms)")
|
| 386 |
+
ap.add_argument("--min_len", type=int, default=8, help="Min segment length (frames)")
|
| 387 |
+
ap.add_argument("--sequence_len", type=int, default=4, help="Model input sequence length (frames)")
|
| 388 |
+
ap.add_argument(
|
| 389 |
+
"--pad_last_frame",
|
| 390 |
+
type=str,
|
| 391 |
+
default="none",
|
| 392 |
+
choices=["none", "zero", "one", "repeat"],
|
| 393 |
+
help="When model expects 4 frames but you provide 3, pad the last frame with this mode.",
|
| 394 |
+
)
|
| 395 |
+
ap.add_argument("--grid_cols", type=int, default=8, help="Columns for saved grids")
|
| 396 |
+
ap.add_argument("--topk_windows", type=int, default=50, help="How many top motion windows to consider per clip")
|
| 397 |
+
ap.add_argument("--num_tests", type=int, default=4, help="How many motion regions to run")
|
| 398 |
+
ap.add_argument(
|
| 399 |
+
"--mode",
|
| 400 |
+
type=str,
|
| 401 |
+
default="topk",
|
| 402 |
+
choices=["topk", "all_motion"],
|
| 403 |
+
help="topk: pick non-overlapping top regions; all_motion: run every threshold-based motion segment",
|
| 404 |
+
)
|
| 405 |
+
return ap.parse_args()
|
| 406 |
+
|
| 407 |
+
|
| 408 |
+
def main() -> None:
|
| 409 |
+
args = _parse_args()
|
| 410 |
+
data_dir = Path(args.data_dir)
|
| 411 |
+
out_dir = Path(args.out_dir)
|
| 412 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 413 |
+
|
| 414 |
+
ssim_hw = tuple(int(x.strip()) for x in args.ssim_hw.split(",")) # type: ignore[assignment]
|
| 415 |
+
|
| 416 |
+
if args.mode == "all_motion":
|
| 417 |
+
clips, segs = find_all_motion_segments(
|
| 418 |
+
data_dir,
|
| 419 |
+
threshold=args.threshold,
|
| 420 |
+
ssim_hw=ssim_hw, # type: ignore[arg-type]
|
| 421 |
+
max_gap_ms=args.max_gap_ms,
|
| 422 |
+
min_len=args.min_len,
|
| 423 |
+
)
|
| 424 |
+
print(f"Found motion segments: {len(segs)}")
|
| 425 |
+
else:
|
| 426 |
+
clips, segs = pick_best_segments(
|
| 427 |
+
data_dir,
|
| 428 |
+
threshold=args.threshold,
|
| 429 |
+
ssim_hw=ssim_hw, # type: ignore[arg-type]
|
| 430 |
+
max_gap_ms=args.max_gap_ms,
|
| 431 |
+
min_len=args.min_len,
|
| 432 |
+
topk=args.topk_windows,
|
| 433 |
+
k=args.num_tests,
|
| 434 |
+
)
|
| 435 |
+
|
| 436 |
+
pred = Predictor(args.model_dir)
|
| 437 |
+
L = int(args.sequence_len)
|
| 438 |
+
|
| 439 |
+
from PIL import Image
|
| 440 |
+
|
| 441 |
+
test_root = out_dir / "test"
|
| 442 |
+
test_root.mkdir(parents=True, exist_ok=True)
|
| 443 |
+
|
| 444 |
+
for idx, (clip, seg) in enumerate(zip(clips, segs), start=1):
|
| 445 |
+
seg_paths = clip[seg.start : seg.end]
|
| 446 |
+
if len(seg_paths) < L:
|
| 447 |
+
continue
|
| 448 |
+
|
| 449 |
+
test_dir = test_root / str(idx)
|
| 450 |
+
(test_dir / "pred").mkdir(parents=True, exist_ok=True)
|
| 451 |
+
(test_dir / "gt").mkdir(parents=True, exist_ok=True)
|
| 452 |
+
|
| 453 |
+
print(f"[test/{idx}] clip={seg.clip_index} start={seg.start} end={seg.end} len={len(seg_paths)} score={seg.score:.6f}")
|
| 454 |
+
print(f"[test/{idx}] first={seg_paths[0].name} last={seg_paths[-1].name}")
|
| 455 |
+
|
| 456 |
+
frames = np.stack([np.asarray(Image.open(p).convert("RGB"), dtype=np.float32) for p in seg_paths], axis=0)
|
| 457 |
+
frames01 = np.clip(frames / 255.0, 0.0, 1.0).astype(np.float32)
|
| 458 |
+
|
| 459 |
+
preds_last: List[np.ndarray] = []
|
| 460 |
+
gt_last: List[np.ndarray] = []
|
| 461 |
+
|
| 462 |
+
kept = 0
|
| 463 |
+
skipped = 0
|
| 464 |
+
for i in range(L - 1, len(frames01)):
|
| 465 |
+
# Teacher forcing with the same "max adjacent timestamp gap" constraint as notebook.
|
| 466 |
+
seq_paths = seg_paths[i - (L - 1) : i + 1]
|
| 467 |
+
if not _has_valid_adjacent_timestamp_gap(seq_paths, args.max_gap_ms):
|
| 468 |
+
skipped += 1
|
| 469 |
+
continue
|
| 470 |
+
|
| 471 |
+
window = frames01[i - (L - 1) : i + 1]
|
| 472 |
+
y = pred.predict_last_frame(window, pad_last_frame=args.pad_last_frame)[0]
|
| 473 |
+
preds_last.append(y)
|
| 474 |
+
gt_last.append(frames01[i])
|
| 475 |
+
kept += 1
|
| 476 |
+
|
| 477 |
+
stem = seg_paths[i].stem
|
| 478 |
+
save_image(test_dir / "pred" / f"pred_{i:04d}_{stem}.png", y)
|
| 479 |
+
save_image(test_dir / "gt" / f"gt_{i:04d}_{stem}.png", frames01[i])
|
| 480 |
+
|
| 481 |
+
if preds_last:
|
| 482 |
+
save_sequence_grid(test_dir / "pred_last_grid.png", np.stack(preds_last, axis=0), cols=args.grid_cols)
|
| 483 |
+
save_sequence_grid(test_dir / "gt_grid.png", np.stack(gt_last, axis=0), cols=args.grid_cols)
|
| 484 |
+
(test_dir / "meta.txt").write_text(
|
| 485 |
+
"\n".join(
|
| 486 |
+
[
|
| 487 |
+
f"clip_index={seg.clip_index}",
|
| 488 |
+
f"start={seg.start}",
|
| 489 |
+
f"end={seg.end}",
|
| 490 |
+
f"len={len(seg_paths)}",
|
| 491 |
+
f"score_mean_1_minus_ssim={seg.score:.6f}",
|
| 492 |
+
f"first_frame={seg_paths[0].name}",
|
| 493 |
+
f"last_frame={seg_paths[-1].name}",
|
| 494 |
+
f"sequence_len={L}",
|
| 495 |
+
f"threshold={args.threshold}",
|
| 496 |
+
f"ssim_hw={ssim_hw[0]},{ssim_hw[1]}",
|
| 497 |
+
f"max_gap_ms={args.max_gap_ms}",
|
| 498 |
+
f"kept_predictions={kept}",
|
| 499 |
+
f"skipped_by_gap={skipped}",
|
| 500 |
+
"",
|
| 501 |
+
]
|
| 502 |
+
),
|
| 503 |
+
encoding="utf-8",
|
| 504 |
+
)
|
| 505 |
+
|
| 506 |
+
print(f"Wrote tests under: {test_root}")
|
| 507 |
+
|
| 508 |
+
|
| 509 |
+
if __name__ == "__main__":
|
| 510 |
+
main()
|
| 511 |
+
|
infer/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .predictor import Predictor
|
| 2 |
+
from .anomaly import AnomalyResult, compute_anomaly_from_4frames
|
| 3 |
+
|
| 4 |
+
__all__ = ["Predictor", "AnomalyResult", "compute_anomaly_from_4frames"]
|
infer/anomaly.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from typing import Dict, Literal, Optional, Tuple
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
|
| 8 |
+
from .predictor import Predictor, PadMode
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@dataclass
|
| 12 |
+
class AnomalyResult:
|
| 13 |
+
# All arrays are float32
|
| 14 |
+
ahat_error: np.ndarray # [4] mean absolute error(pred_t, gt4)
|
| 15 |
+
dissim_1mssim: np.ndarray # [4] 1-SSIM(pred_t, gt4)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _ensure_batched(frames: np.ndarray) -> np.ndarray:
|
| 19 |
+
if frames.ndim == 4:
|
| 20 |
+
return frames[None, ...]
|
| 21 |
+
return frames
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def compute_anomaly_from_4frames(
|
| 25 |
+
predictor: Predictor,
|
| 26 |
+
frames4: np.ndarray,
|
| 27 |
+
) -> AnomalyResult:
|
| 28 |
+
"""
|
| 29 |
+
frames4: [4,H,W,C] or [B,4,H,W,C] in 0..1 or 0..255
|
| 30 |
+
Compares each predicted frame t=1..4 to GT frame4 (index 3).
|
| 31 |
+
Outputs two vectors length 4:
|
| 32 |
+
- ahat_error[t] = mean(abs(pred_t - gt4))
|
| 33 |
+
- dissim_1mssim[t] = 1 - ssim(pred_t, gt4)
|
| 34 |
+
"""
|
| 35 |
+
outs = predictor.predict_outputs(frames4, pad_last_frame="none")
|
| 36 |
+
if "predictions" not in outs:
|
| 37 |
+
raise RuntimeError("SavedModel did not return 'predictions'.")
|
| 38 |
+
preds = outs["predictions"] # [B,4,H,W,C]
|
| 39 |
+
preds = np.asarray(preds, dtype=np.float32)
|
| 40 |
+
|
| 41 |
+
x = predictor._prep(frames4, pad_last_frame="none") # normalized+resized, [B,4,H,W,C]
|
| 42 |
+
x = np.asarray(x, dtype=np.float32)
|
| 43 |
+
gt4 = x[:, 3] # [B,H,W,C]
|
| 44 |
+
|
| 45 |
+
# ahat_error: mean absolute error per time step vs gt4
|
| 46 |
+
abs_err = np.mean(np.abs(preds - gt4[:, None, ...]), axis=(2, 3, 4)) # [B,4]
|
| 47 |
+
ahat_error = np.mean(abs_err, axis=0) # [4]
|
| 48 |
+
|
| 49 |
+
# 1-SSIM per time step vs gt4 (use TF for consistent SSIM)
|
| 50 |
+
tf = predictor._tf
|
| 51 |
+
pred_t = tf.convert_to_tensor(preds, dtype=tf.float32) # [B,4,H,W,C]
|
| 52 |
+
gt_t = tf.convert_to_tensor(gt4[:, None, ...], dtype=tf.float32) # [B,1,H,W,C]
|
| 53 |
+
gt_t = tf.repeat(gt_t, repeats=4, axis=1)
|
| 54 |
+
pred_flat = tf.reshape(pred_t, (-1,) + tuple(preds.shape[2:]))
|
| 55 |
+
gt_flat = tf.reshape(gt_t, (-1,) + tuple(gt4.shape[1:]))
|
| 56 |
+
ssim = tf.image.ssim(pred_flat, gt_flat, max_val=1.0)
|
| 57 |
+
ssim = tf.reshape(ssim, (preds.shape[0], 4))
|
| 58 |
+
dissim = 1.0 - ssim
|
| 59 |
+
dissim_1mssim = tf.reduce_mean(dissim, axis=0).numpy().astype(np.float32) # [4]
|
| 60 |
+
|
| 61 |
+
return AnomalyResult(
|
| 62 |
+
ahat_error=np.asarray(ahat_error, dtype=np.float32),
|
| 63 |
+
dissim_1mssim=np.asarray(dissim_1mssim, dtype=np.float32),
|
| 64 |
+
)
|
| 65 |
+
|
infer/io.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import re
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Dict, Iterable, List, Optional, Sequence, Tuple, Union
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
from PIL import Image
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
_IMG_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".bmp"}
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _natural_key(s: str):
|
| 15 |
+
return [int(t) if t.isdigit() else t.lower() for t in re.split(r"(\d+)", s)]
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def load_frames_from_dir(frames_dir: Union[str, Path]) -> np.ndarray:
|
| 19 |
+
d = Path(frames_dir)
|
| 20 |
+
if not d.exists():
|
| 21 |
+
raise FileNotFoundError(str(d))
|
| 22 |
+
paths = [p for p in d.iterdir() if p.is_file() and p.suffix.lower() in _IMG_EXTS]
|
| 23 |
+
paths.sort(key=lambda p: _natural_key(p.name))
|
| 24 |
+
if not paths:
|
| 25 |
+
raise ValueError(f"No image frames found in {d}")
|
| 26 |
+
|
| 27 |
+
arrs: List[np.ndarray] = []
|
| 28 |
+
for p in paths:
|
| 29 |
+
im = Image.open(p).convert("RGB")
|
| 30 |
+
arrs.append(np.asarray(im, dtype=np.float32))
|
| 31 |
+
x = np.stack(arrs, axis=0) # [T,H,W,C] in 0..255
|
| 32 |
+
return x
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def load_array(path: Union[str, Path]) -> np.ndarray:
|
| 36 |
+
p = Path(path)
|
| 37 |
+
if not p.exists():
|
| 38 |
+
raise FileNotFoundError(str(p))
|
| 39 |
+
if p.suffix.lower() == ".npy":
|
| 40 |
+
return np.load(p)
|
| 41 |
+
if p.suffix.lower() == ".npz":
|
| 42 |
+
z = np.load(p)
|
| 43 |
+
if "frames" in z:
|
| 44 |
+
return z["frames"]
|
| 45 |
+
# fall back to first array
|
| 46 |
+
k = next(iter(z.files))
|
| 47 |
+
return z[k]
|
| 48 |
+
raise ValueError("Expected .npy or .npz")
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def to_uint8_rgb(img01: np.ndarray) -> np.ndarray:
|
| 52 |
+
x = np.clip(img01, 0.0, 1.0)
|
| 53 |
+
return (x * 255.0 + 0.5).astype(np.uint8)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def save_image(path: Union[str, Path], img01: np.ndarray) -> None:
|
| 57 |
+
p = Path(path)
|
| 58 |
+
p.parent.mkdir(parents=True, exist_ok=True)
|
| 59 |
+
Image.fromarray(to_uint8_rgb(img01)).save(p)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def save_sequence_grid(path: Union[str, Path], seq01: np.ndarray, cols: int = 8) -> None:
|
| 63 |
+
# seq01: [T,H,W,C]
|
| 64 |
+
p = Path(path)
|
| 65 |
+
p.parent.mkdir(parents=True, exist_ok=True)
|
| 66 |
+
t, h, w, c = seq01.shape
|
| 67 |
+
cols = max(1, int(cols))
|
| 68 |
+
rows = (t + cols - 1) // cols
|
| 69 |
+
canvas = np.zeros((rows * h, cols * w, c), dtype=np.uint8)
|
| 70 |
+
for i in range(t):
|
| 71 |
+
r = i // cols
|
| 72 |
+
col = i % cols
|
| 73 |
+
canvas[r * h : (r + 1) * h, col * w : (col + 1) * w] = to_uint8_rgb(seq01[i])
|
| 74 |
+
Image.fromarray(canvas).save(p)
|
| 75 |
+
|
infer/predictor.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Any, Dict, Mapping, Optional, Tuple, Union, Literal
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
ArrayLike = Union[np.ndarray]
|
| 11 |
+
PadMode = Literal["none", "zero", "one", "repeat"]
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _as_5d(frames: np.ndarray) -> np.ndarray:
|
| 15 |
+
if frames.ndim == 4:
|
| 16 |
+
return frames[None, ...]
|
| 17 |
+
if frames.ndim == 5:
|
| 18 |
+
return frames
|
| 19 |
+
raise ValueError(f"Expected frames with rank 4 or 5, got shape={frames.shape}")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _normalize01(x: np.ndarray) -> np.ndarray:
|
| 23 |
+
x = x.astype(np.float32, copy=False)
|
| 24 |
+
if x.max() > 1.5:
|
| 25 |
+
x = x / 255.0
|
| 26 |
+
return np.clip(x, 0.0, 1.0)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _pad_last_frame(frames: np.ndarray, pad_mode: PadMode, target_len: int) -> np.ndarray:
|
| 30 |
+
"""
|
| 31 |
+
frames: [B,T,H,W,C] float in 0..1
|
| 32 |
+
Returns [B,target_len,H,W,C].
|
| 33 |
+
"""
|
| 34 |
+
if pad_mode == "none":
|
| 35 |
+
return frames
|
| 36 |
+
b, t, h, w, c = frames.shape
|
| 37 |
+
if t > target_len:
|
| 38 |
+
return frames[:, :target_len]
|
| 39 |
+
if t == target_len:
|
| 40 |
+
return frames
|
| 41 |
+
if t < target_len - 1:
|
| 42 |
+
raise ValueError(f"pad_last_frame supports T=={target_len-1} or T=={target_len}, got T={t}")
|
| 43 |
+
|
| 44 |
+
if pad_mode == "repeat":
|
| 45 |
+
last = frames[:, -1:, :, :, :]
|
| 46 |
+
elif pad_mode == "zero":
|
| 47 |
+
last = np.zeros((b, 1, h, w, c), dtype=frames.dtype)
|
| 48 |
+
elif pad_mode == "one":
|
| 49 |
+
last = np.ones((b, 1, h, w, c), dtype=frames.dtype)
|
| 50 |
+
else:
|
| 51 |
+
raise ValueError(f"Unknown pad_mode={pad_mode}")
|
| 52 |
+
return np.concatenate([frames, last], axis=1)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _pick_signature(module) -> Tuple[str, Any]:
|
| 56 |
+
sigs = getattr(module, "signatures", None)
|
| 57 |
+
if not isinstance(sigs, Mapping) or not sigs:
|
| 58 |
+
raise RuntimeError("SavedModel has no callable signatures.")
|
| 59 |
+
if "serving_default" in sigs:
|
| 60 |
+
return "serving_default", sigs["serving_default"]
|
| 61 |
+
name = next(iter(sigs.keys()))
|
| 62 |
+
return name, sigs[name]
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _signature_io(fn) -> Tuple[Dict[str, Any], Dict[str, Any]]:
|
| 66 |
+
# structured_input_signature = (args, kwargs)
|
| 67 |
+
args, kwargs = fn.structured_input_signature
|
| 68 |
+
if args:
|
| 69 |
+
raise RuntimeError("This SavedModel signature expects positional args; unsupported.")
|
| 70 |
+
return dict(kwargs), dict(fn.structured_outputs)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _infer_hw_from_inputs(input_specs: Mapping[str, Any]) -> Optional[Tuple[int, int]]:
|
| 74 |
+
# Try to find a spec with shape [B,T,H,W,C] or [B,H,W,C]
|
| 75 |
+
for spec in input_specs.values():
|
| 76 |
+
shape = getattr(spec, "shape", None)
|
| 77 |
+
if shape is None:
|
| 78 |
+
continue
|
| 79 |
+
dims = list(shape)
|
| 80 |
+
if len(dims) == 5 and dims[2] is not None and dims[3] is not None:
|
| 81 |
+
return int(dims[2]), int(dims[3])
|
| 82 |
+
if len(dims) == 4 and dims[1] is not None and dims[2] is not None:
|
| 83 |
+
return int(dims[1]), int(dims[2])
|
| 84 |
+
return None
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _select_prediction_tensor(outputs: Mapping[str, Any]):
|
| 88 |
+
# Heuristic: prefer key contains 'pred', else first rank-5 tensor, else first tensor-like output
|
| 89 |
+
items = list(outputs.items())
|
| 90 |
+
for k, v in items:
|
| 91 |
+
if "pred" in k.lower():
|
| 92 |
+
return k, v
|
| 93 |
+
for k, v in items:
|
| 94 |
+
shape = getattr(v, "shape", None)
|
| 95 |
+
if shape is not None and len(shape) == 5:
|
| 96 |
+
return k, v
|
| 97 |
+
return items[0]
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
@dataclass
|
| 101 |
+
class Predictor:
|
| 102 |
+
model_dir: Union[str, Path]
|
| 103 |
+
signature_name: Optional[str] = None
|
| 104 |
+
input_key: Optional[str] = None
|
| 105 |
+
resize_hw: Optional[Tuple[int, int]] = None
|
| 106 |
+
|
| 107 |
+
def __post_init__(self) -> None:
|
| 108 |
+
import tensorflow as tf
|
| 109 |
+
|
| 110 |
+
self._tf = tf
|
| 111 |
+
|
| 112 |
+
# Prefer GPU if available; otherwise CPU is used automatically.
|
| 113 |
+
try:
|
| 114 |
+
gpus = tf.config.list_physical_devices("GPU")
|
| 115 |
+
for gpu in gpus:
|
| 116 |
+
# Avoid TF grabbing all GPU memory up front.
|
| 117 |
+
tf.config.experimental.set_memory_growth(gpu, True)
|
| 118 |
+
self.device: str = "GPU" if gpus else "CPU"
|
| 119 |
+
except Exception:
|
| 120 |
+
self.device = "CPU"
|
| 121 |
+
|
| 122 |
+
self.model_dir = Path(self.model_dir)
|
| 123 |
+
self._module = tf.saved_model.load(str(self.model_dir))
|
| 124 |
+
name, fn = _pick_signature(self._module)
|
| 125 |
+
if self.signature_name is None:
|
| 126 |
+
self.signature_name = name
|
| 127 |
+
self._fn = self._module.signatures[self.signature_name]
|
| 128 |
+
|
| 129 |
+
in_specs, out_specs = _signature_io(self._fn)
|
| 130 |
+
self._in_specs = in_specs
|
| 131 |
+
self._out_specs = out_specs
|
| 132 |
+
|
| 133 |
+
if self.input_key is None:
|
| 134 |
+
# Prefer single input; otherwise pick first
|
| 135 |
+
if len(in_specs) == 1:
|
| 136 |
+
self.input_key = next(iter(in_specs.keys()))
|
| 137 |
+
else:
|
| 138 |
+
self.input_key = next(iter(in_specs.keys()))
|
| 139 |
+
|
| 140 |
+
if self.resize_hw is None:
|
| 141 |
+
self.resize_hw = _infer_hw_from_inputs(in_specs) or (128, 128)
|
| 142 |
+
|
| 143 |
+
# Infer expected sequence length if fixed (e.g. 4)
|
| 144 |
+
spec = self._in_specs.get(self.input_key) if self.input_key else None
|
| 145 |
+
shape = getattr(spec, "shape", None)
|
| 146 |
+
self.sequence_len: Optional[int] = None
|
| 147 |
+
if shape is not None and len(shape) == 5 and shape[1] is not None:
|
| 148 |
+
self.sequence_len = int(shape[1])
|
| 149 |
+
|
| 150 |
+
def _prep(self, frames: np.ndarray, pad_last_frame: PadMode = "none") -> np.ndarray:
|
| 151 |
+
frames = _as_5d(frames)
|
| 152 |
+
frames = _normalize01(frames)
|
| 153 |
+
|
| 154 |
+
# Resize with TF to match model expectation (keeps it black-box)
|
| 155 |
+
b, t, h, w, c = frames.shape
|
| 156 |
+
rh, rw = self.resize_hw or (h, w)
|
| 157 |
+
if (h, w) != (rh, rw):
|
| 158 |
+
tf = self._tf
|
| 159 |
+
x = tf.convert_to_tensor(frames, dtype=tf.float32)
|
| 160 |
+
x = tf.reshape(x, (b * t, h, w, c))
|
| 161 |
+
x = tf.image.resize(x, (rh, rw), method="bilinear", antialias=True)
|
| 162 |
+
x = tf.reshape(x, (b, t, rh, rw, c))
|
| 163 |
+
frames = x.numpy()
|
| 164 |
+
|
| 165 |
+
if self.sequence_len is not None:
|
| 166 |
+
frames = _pad_last_frame(frames, pad_last_frame, target_len=self.sequence_len)
|
| 167 |
+
return frames
|
| 168 |
+
|
| 169 |
+
def predict_sequence(self, frames: np.ndarray, pad_last_frame: PadMode = "none") -> np.ndarray:
|
| 170 |
+
tf = self._tf
|
| 171 |
+
x = self._prep(frames, pad_last_frame=pad_last_frame)
|
| 172 |
+
inp = {self.input_key: tf.convert_to_tensor(x, dtype=tf.float32)}
|
| 173 |
+
outs = self._fn(**inp)
|
| 174 |
+
pred = outs.get("predictions")
|
| 175 |
+
if pred is None:
|
| 176 |
+
_, pred = _select_prediction_tensor(outs)
|
| 177 |
+
pred_np = pred.numpy()
|
| 178 |
+
return np.clip(pred_np, 0.0, 1.0)
|
| 179 |
+
|
| 180 |
+
def predict_outputs(self, frames: np.ndarray, pad_last_frame: PadMode = "none") -> Dict[str, np.ndarray]:
|
| 181 |
+
tf = self._tf
|
| 182 |
+
x = self._prep(frames, pad_last_frame=pad_last_frame)
|
| 183 |
+
inp = {self.input_key: tf.convert_to_tensor(x, dtype=tf.float32)}
|
| 184 |
+
outs = self._fn(**inp)
|
| 185 |
+
out_np: Dict[str, np.ndarray] = {}
|
| 186 |
+
for k, v in outs.items():
|
| 187 |
+
try:
|
| 188 |
+
out_np[k] = v.numpy()
|
| 189 |
+
except Exception:
|
| 190 |
+
pass
|
| 191 |
+
if "predictions" in out_np:
|
| 192 |
+
out_np["predictions"] = np.clip(out_np["predictions"], 0.0, 1.0)
|
| 193 |
+
return out_np
|
| 194 |
+
|
| 195 |
+
def predict_last_frame(self, frames: np.ndarray, pad_last_frame: PadMode = "none") -> np.ndarray:
|
| 196 |
+
seq = self.predict_sequence(frames, pad_last_frame=pad_last_frame)
|
| 197 |
+
if seq.ndim != 5:
|
| 198 |
+
raise RuntimeError(f"Expected prediction rank 5, got shape={seq.shape}")
|
| 199 |
+
return seq[:, -1]
|
| 200 |
+
|
| 201 |
+
def predict_next_from_3(self, frames3: np.ndarray, pad_last_frame: PadMode = "zero") -> np.ndarray:
|
| 202 |
+
"""
|
| 203 |
+
Predict next frame from 3 context frames by padding the 4th input slot.
|
| 204 |
+
Returns [B,H,W,C].
|
| 205 |
+
"""
|
| 206 |
+
if self.sequence_len is not None and self.sequence_len != 4:
|
| 207 |
+
raise RuntimeError(f"predict_next_from_3 expects sequence_len=4, got {self.sequence_len}")
|
| 208 |
+
last = self.predict_last_frame(frames3, pad_last_frame=pad_last_frame)
|
| 209 |
+
return last
|
| 210 |
+
|
modeling.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
from huggingface_hub import snapshot_download
|
| 5 |
+
|
| 6 |
+
from .configuration import PredNetConfig
|
| 7 |
+
from .infer.predictor import Predictor
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class PredNetModel:
|
| 11 |
+
"""Black-box TF SavedModel wrapper compatible with AutoModel via trust_remote_code."""
|
| 12 |
+
|
| 13 |
+
def __init__(self, config: PredNetConfig, predictor: Predictor):
|
| 14 |
+
self.config = config
|
| 15 |
+
self._predictor = predictor
|
| 16 |
+
|
| 17 |
+
@classmethod
|
| 18 |
+
def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
|
| 19 |
+
if os.path.isdir(pretrained_model_name_or_path):
|
| 20 |
+
local_dir = pretrained_model_name_or_path
|
| 21 |
+
else:
|
| 22 |
+
local_dir = snapshot_download(pretrained_model_name_or_path)
|
| 23 |
+
|
| 24 |
+
config = PredNetConfig.from_pretrained(local_dir)
|
| 25 |
+
savedmodel_dir = str(Path(local_dir) / "savedmodel")
|
| 26 |
+
predictor = Predictor(
|
| 27 |
+
model_dir=savedmodel_dir,
|
| 28 |
+
resize_hw=tuple(config.resize_hw),
|
| 29 |
+
)
|
| 30 |
+
return cls(config, predictor)
|
| 31 |
+
|
| 32 |
+
def predict_sequence(self, frames, pad_last_frame="none"):
|
| 33 |
+
return self._predictor.predict_sequence(frames, pad_last_frame=pad_last_frame)
|
| 34 |
+
|
| 35 |
+
def predict_last_frame(self, frames, pad_last_frame="none"):
|
| 36 |
+
return self._predictor.predict_last_frame(frames, pad_last_frame=pad_last_frame)
|
| 37 |
+
|
| 38 |
+
def predict_outputs(self, frames, pad_last_frame="none"):
|
| 39 |
+
return self._predictor.predict_outputs(frames, pad_last_frame=pad_last_frame)
|
pipeline.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
from transformers import Pipeline
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class VideoFramePredictionPipeline(Pipeline):
|
| 8 |
+
"""
|
| 9 |
+
HuggingFace Pipeline for black-box video frame prediction.
|
| 10 |
+
|
| 11 |
+
Usage:
|
| 12 |
+
pipe = pipeline("video-frame-prediction", model="dvdface/denoising-prednet",
|
| 13 |
+
trust_remote_code=True)
|
| 14 |
+
# frames: numpy [T,H,W,C] uint8/float, or path to frame directory
|
| 15 |
+
result = pipe(frames)
|
| 16 |
+
# Returns dict: {"sequence": [T,H,W,C], "last_frame": [H,W,C]}
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
def _sanitize_parameters(self, pad_last_frame="none", return_last_only=False, **kwargs):
|
| 20 |
+
forward_params = {"pad_last_frame": pad_last_frame}
|
| 21 |
+
postprocess_params = {"return_last_only": return_last_only}
|
| 22 |
+
return {}, forward_params, postprocess_params
|
| 23 |
+
|
| 24 |
+
def preprocess(self, inputs):
|
| 25 |
+
if isinstance(inputs, (str, Path)):
|
| 26 |
+
from .infer.io import load_frames_from_dir
|
| 27 |
+
inputs = load_frames_from_dir(inputs)
|
| 28 |
+
return {"frames": np.asarray(inputs)}
|
| 29 |
+
|
| 30 |
+
def _forward(self, model_inputs, pad_last_frame="none"):
|
| 31 |
+
frames = model_inputs["frames"]
|
| 32 |
+
seq = self.model.predict_sequence(frames, pad_last_frame=pad_last_frame)
|
| 33 |
+
return {"predictions": seq}
|
| 34 |
+
|
| 35 |
+
def postprocess(self, model_outputs, return_last_only=False):
|
| 36 |
+
seq = model_outputs["predictions"]
|
| 37 |
+
# Remove batch dim if present
|
| 38 |
+
if seq.ndim == 5:
|
| 39 |
+
seq = seq[0] # [T,H,W,C]
|
| 40 |
+
last = seq[-1] # [H,W,C]
|
| 41 |
+
if return_last_only:
|
| 42 |
+
return last
|
| 43 |
+
return {"sequence": seq, "last_frame": last}
|
predict.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
|
| 9 |
+
from infer.io import load_array, load_frames_from_dir, save_image, save_sequence_grid
|
| 10 |
+
from infer.predictor import Predictor
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def _parse_args() -> argparse.Namespace:
|
| 14 |
+
ap = argparse.ArgumentParser(description="Black-box SavedModel inference: output predicted frames.")
|
| 15 |
+
ap.add_argument("--model_dir", type=str, default="savedmodel", help="Path to SavedModel directory")
|
| 16 |
+
ap.add_argument("--frames_dir", type=str, default=None, help="Directory containing input frames (images)")
|
| 17 |
+
ap.add_argument("--array", type=str, default=None, help="Path to .npy/.npz containing frames")
|
| 18 |
+
ap.add_argument(
|
| 19 |
+
"--pad_last_frame",
|
| 20 |
+
type=str,
|
| 21 |
+
default="none",
|
| 22 |
+
choices=["none", "zero", "one", "repeat"],
|
| 23 |
+
help="If model expects 4 frames but you provide 3, pad the last frame with: zero/one/repeat",
|
| 24 |
+
)
|
| 25 |
+
ap.add_argument("--out_dir", type=str, default="outputs", help="Output directory")
|
| 26 |
+
ap.add_argument("--save_sequence_grid", action="store_true", help="Save a grid of the predicted sequence")
|
| 27 |
+
ap.add_argument("--grid_cols", type=int, default=8, help="Columns for sequence grid")
|
| 28 |
+
return ap.parse_args()
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def main() -> None:
|
| 32 |
+
args = _parse_args()
|
| 33 |
+
if (args.frames_dir is None) == (args.array is None):
|
| 34 |
+
raise SystemExit("Provide exactly one of --frames_dir or --array")
|
| 35 |
+
|
| 36 |
+
if args.frames_dir is not None:
|
| 37 |
+
frames = load_frames_from_dir(args.frames_dir) # [T,H,W,C] 0..255
|
| 38 |
+
else:
|
| 39 |
+
frames = load_array(args.array)
|
| 40 |
+
|
| 41 |
+
pred = Predictor(args.model_dir)
|
| 42 |
+
seq = pred.predict_sequence(frames, pad_last_frame=args.pad_last_frame) # [B,T,H,W,C]
|
| 43 |
+
if seq.ndim == 5:
|
| 44 |
+
seq0 = seq[0]
|
| 45 |
+
last = seq[0, -1]
|
| 46 |
+
elif seq.ndim == 4:
|
| 47 |
+
seq0 = seq
|
| 48 |
+
last = seq[-1]
|
| 49 |
+
else:
|
| 50 |
+
raise RuntimeError(f"Unexpected prediction shape: {seq.shape}")
|
| 51 |
+
|
| 52 |
+
out_dir = Path(args.out_dir)
|
| 53 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 54 |
+
save_image(out_dir / "pred_last.png", last)
|
| 55 |
+
|
| 56 |
+
if args.save_sequence_grid:
|
| 57 |
+
save_sequence_grid(out_dir / "pred_sequence_grid.png", seq0, cols=args.grid_cols)
|
| 58 |
+
|
| 59 |
+
print(f"Wrote: {out_dir / 'pred_last.png'}")
|
| 60 |
+
if args.save_sequence_grid:
|
| 61 |
+
print(f"Wrote: {out_dir / 'pred_sequence_grid.png'}")
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
if __name__ == "__main__":
|
| 65 |
+
main()
|
| 66 |
+
|
requirements.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
tensorflow
|
| 2 |
+
numpy
|
| 3 |
+
pillow
|
| 4 |
+
huggingface_hub
|
| 5 |
+
imageio
|
| 6 |
+
transformers
|
savedmodel/fingerprint.pb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:528aadaff82e9283d2fefdf35b8245289f8ebd0107c02ff0673bca994cab2085
|
| 3 |
+
size 77
|
savedmodel/saved_model.pb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5edf975b0705a826d17a3af45afc313096096784b0725c3432722eec855bdf30
|
| 3 |
+
size 1108400
|
savedmodel/variables/variables.data-00000-of-00001
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:824880996b8fd1dfb5c57d685a78e8b322d21338fdd5d92c797522b5d4c0bcfa
|
| 3 |
+
size 84082031
|
savedmodel/variables/variables.index
ADDED
|
Binary file (3.56 kB). View file
|
|
|