dvdface commited on
Commit
8b78124
·
verified ·
1 Parent(s): a66cdf8

Simplify project structure: rename dirs and files

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ savedmodel/variables/variables.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ## PredNet denoising (black-box SavedModel)
2
+
3
+ This repository packages a TensorFlow **SavedModel** for video-frame prediction/denoising **without publishing any model architecture source code**. Inference calls the SavedModel signature as a black box and produces predicted frames.
4
+
5
+ ### Files
6
+
7
+ - `savedmodel/`: TensorFlow SavedModel directory (weights + graph)
8
+ - `infer/`: minimal inference helpers (black-box loader + IO)
9
+ - `predict.py`: CLI that outputs prediction images
10
+ - `anomaly.py`: CLI that outputs per-frame anomaly scores (ahat_error + 1-SSIM vectors)
11
+
12
+ ### Install
13
+
14
+ ```bash
15
+ pip install -r requirements.txt
16
+ ```
17
+
18
+ ### Run (from a directory of frames)
19
+
20
+ `frames_dir` should contain ordered image frames (e.g. `0001.png`, `0002.png`, ...).
21
+
22
+ ```bash
23
+ python predict.py --model_dir savedmodel --frames_dir /path/to/frames --out_dir outputs --save_sequence_grid
24
+ ```
25
+
26
+ Outputs:
27
+ - `outputs/pred_last.png`
28
+ - `outputs/pred_sequence_grid.png` (optional)
29
+
30
+ ### Run (from npy/npz)
31
+
32
+ Accepts `frames` stored as `[T,H,W,C]` or `[B,T,H,W,C]`.
33
+
34
+ ```bash
35
+ python predict.py --array frames.npy --out_dir outputs
36
+ ```
37
+
38
+ ### Anomaly scores (4-frame window)
39
+
40
+ For each window of 4 frames ending at time `t`, this computes two vectors of length 4:
41
+ - `ahat_error[1..4]`: mean absolute error between `pred_frame_k` and `GT_frame_4`
42
+ - `dissim_1mssim[1..4]`: `1 - SSIM(pred_frame_k, GT_frame_4)`
43
+
44
+ ```bash
45
+ python anomaly.py --model_dir savedmodel --frames_dir /path/to/frames --out_json anomaly.json --out_csv anomaly.csv
46
+ ```
47
+
48
+ ### Hugging Face Hub usage
49
+
50
+ After you upload this repo to the Hub, users can download it via `huggingface_hub` and point `--model_dir` at the downloaded `savedmodel/` directory.
51
+
52
+ ### HuggingFace Pipeline Usage
53
+
54
+ ```python
55
+ from transformers import pipeline
56
+ import numpy as np
57
+
58
+ pipe = pipeline(
59
+ "video-frame-prediction",
60
+ model="dvdface/denoising-prednet",
61
+ trust_remote_code=True,
62
+ )
63
+
64
+ # frames: [T, H, W, C] numpy array, uint8 (0-255) or float (0-1)
65
+ frames = np.random.randint(0, 255, (4, 128, 128, 3), dtype=np.uint8)
66
+ result = pipe(frames)
67
+
68
+ print(result["sequence"].shape) # (4, 128, 128, 3)
69
+ print(result["last_frame"].shape) # (128, 128, 3)
70
+ ```
71
+
anomaly.py ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import csv
5
+ import json
6
+ from pathlib import Path
7
+ from typing import Any, Dict, List, Optional
8
+
9
+ import numpy as np
10
+
11
+ from infer.io import load_array, load_frames_from_dir
12
+ from infer.predictor import Predictor
13
+ from infer.anomaly import compute_anomaly_from_4frames
14
+
15
+
16
+ def _parse_args() -> argparse.Namespace:
17
+ ap = argparse.ArgumentParser(description="Compute per-window anomaly scores (ahat_error and 1-SSIM vectors).")
18
+ ap.add_argument("--model_dir", type=str, default="savedmodel")
19
+ ap.add_argument("--frames_dir", type=str, default=None)
20
+ ap.add_argument("--array", type=str, default=None)
21
+ ap.add_argument("--out_json", type=str, default="anomaly.json")
22
+ ap.add_argument("--out_csv", type=str, default="anomaly.csv")
23
+ ap.add_argument("--start", type=int, default=0, help="Start frame index (inclusive) on the full sequence")
24
+ ap.add_argument("--end", type=int, default=-1, help="End frame index (exclusive); -1 means to the end")
25
+ return ap.parse_args()
26
+
27
+
28
+ def main() -> None:
29
+ args = _parse_args()
30
+ if (args.frames_dir is None) == (args.array is None):
31
+ raise SystemExit("Provide exactly one of --frames_dir or --array")
32
+
33
+ if args.frames_dir is not None:
34
+ frames = load_frames_from_dir(args.frames_dir) # [T,H,W,C] 0..255
35
+ else:
36
+ frames = load_array(args.array)
37
+
38
+ if frames.ndim == 5:
39
+ if frames.shape[0] != 1:
40
+ raise SystemExit("Batch arrays are not supported in CLI; please provide a single sequence.")
41
+ frames = frames[0]
42
+ if frames.ndim != 4 or frames.shape[0] < 4:
43
+ raise SystemExit(f"Expected frames with shape [T,H,W,C] and T>=4, got {frames.shape}")
44
+
45
+ T = frames.shape[0]
46
+ start = max(0, int(args.start))
47
+ end = T if int(args.end) == -1 else min(T, int(args.end))
48
+ if end - start < 4:
49
+ raise SystemExit("Range too small; need at least 4 frames.")
50
+
51
+ pred = Predictor(args.model_dir)
52
+ if pred.sequence_len is not None and pred.sequence_len != 4:
53
+ raise SystemExit(f"This tool assumes model sequence_len=4, got {pred.sequence_len}")
54
+
55
+ rows: List[Dict[str, Any]] = []
56
+ for t in range(start + 3, end):
57
+ win = frames[t - 3 : t + 1] # [4,H,W,C] where win[3] is GT frame4
58
+ res = compute_anomaly_from_4frames(pred, win)
59
+ rows.append(
60
+ {
61
+ "t": int(t),
62
+ "ahat_error": res.ahat_error.tolist(),
63
+ "dissim_1mssim": res.dissim_1mssim.tolist(),
64
+ }
65
+ )
66
+
67
+ out_json = Path(args.out_json)
68
+ out_json.parent.mkdir(parents=True, exist_ok=True)
69
+ out_json.write_text(json.dumps({"items": rows}, ensure_ascii=False, indent=2), encoding="utf-8")
70
+
71
+ out_csv = Path(args.out_csv)
72
+ out_csv.parent.mkdir(parents=True, exist_ok=True)
73
+ with out_csv.open("w", newline="", encoding="utf-8") as f:
74
+ w = csv.writer(f)
75
+ w.writerow(["t", "ahat_error_1", "ahat_error_2", "ahat_error_3", "ahat_error_4", "dissim_1", "dissim_2", "dissim_3", "dissim_4"])
76
+ for r in rows:
77
+ a = r["ahat_error"]
78
+ d = r["dissim_1mssim"]
79
+ w.writerow([r["t"], *a, *d])
80
+
81
+ print(f"Wrote: {out_json}")
82
+ print(f"Wrote: {out_csv}")
83
+
84
+
85
+ if __name__ == "__main__":
86
+ main()
87
+
config.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_type": "prednet",
3
+ "architectures": ["PredNetModel"],
4
+ "auto_map": {
5
+ "AutoModel": "modeling.PredNetModel"
6
+ },
7
+ "custom_pipelines": {
8
+ "video-frame-prediction": {
9
+ "impl": "pipeline.VideoFramePredictionPipeline",
10
+ "pt": [],
11
+ "tf": ["AutoModel"]
12
+ }
13
+ },
14
+ "sequence_len": 4,
15
+ "resize_hw": [128, 128]
16
+ }
configuration.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import PretrainedConfig
2
+
3
+
4
+ class PredNetConfig(PretrainedConfig):
5
+ model_type = "prednet"
6
+
7
+ def __init__(self, sequence_len=4, resize_hw=(128, 128), **kwargs):
8
+ super().__init__(**kwargs)
9
+ self.sequence_len = sequence_len
10
+ self.resize_hw = list(resize_hw)
dataset_test.py ADDED
@@ -0,0 +1,511 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import math
5
+ import re
6
+ from dataclasses import dataclass
7
+ from pathlib import Path
8
+ from typing import List, Optional, Sequence, Tuple, Union
9
+
10
+ import numpy as np
11
+ import tensorflow as tf
12
+
13
+ from infer.io import save_image, save_sequence_grid
14
+ from infer.predictor import Predictor
15
+
16
+
17
+ def _natural_key(s: str):
18
+ return [int(t) if t.isdigit() else t.lower() for t in re.split(r"(\d+)", s)]
19
+
20
+
21
+ def _timestamp_from_name(p: Path) -> Optional[int]:
22
+ m = re.match(r"^(\d+)", p.stem)
23
+ if not m:
24
+ return None
25
+ try:
26
+ return int(m.group(1))
27
+ except ValueError:
28
+ return None
29
+
30
+
31
+ def _parse_unix_timestamp_ms(path: Union[str, Path]) -> Optional[int]:
32
+ """
33
+ Mirror notebook logic:
34
+ - >=19 digits: ns -> ms
35
+ - >=16 digits: us -> ms
36
+ - >=13 digits: ms
37
+ - >=10 digits: s -> ms
38
+ else: None
39
+ """
40
+ stem = Path(path).stem
41
+ if not stem.isdigit():
42
+ return None
43
+ raw = int(stem)
44
+ digits = len(stem)
45
+ if digits >= 19:
46
+ return raw // 1_000_000
47
+ if digits >= 16:
48
+ return raw // 1_000
49
+ if digits >= 13:
50
+ return raw
51
+ if digits >= 10:
52
+ return raw * 1_000
53
+ return None
54
+
55
+
56
+ def _has_valid_adjacent_timestamp_gap(paths: Sequence[Path], max_gap_ms: int = 32) -> bool:
57
+ timestamps = [_parse_unix_timestamp_ms(p) for p in paths]
58
+ if any(ts is None for ts in timestamps):
59
+ return True
60
+ for prev_ts, cur_ts in zip(timestamps, timestamps[1:]):
61
+ if cur_ts < prev_ts or (cur_ts - prev_ts) > max_gap_ms:
62
+ return False
63
+ return True
64
+
65
+
66
+ def _gap_ms(ts0: int, ts1: int) -> float:
67
+ diff = ts1 - ts0
68
+ if diff <= 0:
69
+ return 0.0
70
+ # Treat timestamps as ms already (they come from filenames); this is only used
71
+ # by the legacy top-k mode which splits clips first.
72
+ return float(diff)
73
+
74
+
75
+ def _resize_np(x: np.ndarray, hw: Tuple[int, int]) -> np.ndarray:
76
+ t, h, w, c = x.shape
77
+ y = tf.image.resize(tf.convert_to_tensor(x, tf.float32), hw, method="bilinear", antialias=True)
78
+ return y.numpy()
79
+
80
+
81
+ def _ssim_adjacent(frames01: np.ndarray, ssim_hw: Tuple[int, int]) -> np.ndarray:
82
+ # frames01: [T,H,W,C] float32 0..1
83
+ x = frames01
84
+ if x.shape[1:3] != (ssim_hw[0], ssim_hw[1]):
85
+ x = _resize_np(x, ssim_hw)
86
+ # tf.image.ssim expects [N,H,W,C]
87
+ a = tf.convert_to_tensor(x[:-1], tf.float32)
88
+ b = tf.convert_to_tensor(x[1:], tf.float32)
89
+ s = tf.image.ssim(a, b, max_val=1.0) # [T-1]
90
+ return s.numpy()
91
+
92
+
93
+ def _find_motion_segments_in_clip(frames01: np.ndarray, threshold: float, ssim_hw: Tuple[int, int]) -> List[Tuple[int, int, float]]:
94
+ """
95
+ Return segments [start,end) where (1-SSIM) > threshold on adjacent pairs.
96
+ Used by legacy top-k selection mode.
97
+ """
98
+ if frames01.shape[0] < 2:
99
+ return []
100
+ ssim = _ssim_adjacent(frames01, ssim_hw) # len T-1
101
+ motion = 1.0 - ssim
102
+ mask = motion > threshold
103
+ segs: List[Tuple[int, int, float]] = []
104
+ i = 0
105
+ while i < len(mask):
106
+ if not mask[i]:
107
+ i += 1
108
+ continue
109
+ j = i
110
+ while j < len(mask) and mask[j]:
111
+ j += 1
112
+ start = i
113
+ end = j + 1
114
+ score = float(np.mean(motion[i:j])) if j > i else 0.0
115
+ segs.append((start, end, score))
116
+ i = j
117
+ return segs
118
+
119
+
120
+ def _split_by_timestamp_gap(paths: List[Path], max_gap_ms: int) -> List[List[Path]]:
121
+ clips: List[List[Path]] = []
122
+ cur: List[Path] = []
123
+ prev_ts_ms: Optional[int] = None
124
+ for p in paths:
125
+ ts_ms = _parse_unix_timestamp_ms(p)
126
+ if ts_ms is None:
127
+ # If we can't parse timestamp, keep as continuous.
128
+ if not cur:
129
+ cur = [p]
130
+ else:
131
+ cur.append(p)
132
+ continue
133
+ if prev_ts_ms is None:
134
+ cur = [p]
135
+ prev_ts_ms = ts_ms
136
+ continue
137
+ if (ts_ms - prev_ts_ms) > int(max_gap_ms):
138
+ if cur:
139
+ clips.append(cur)
140
+ cur = [p]
141
+ else:
142
+ cur.append(p)
143
+ prev_ts_ms = ts_ms
144
+ if cur:
145
+ clips.append(cur)
146
+ return clips
147
+
148
+
149
+ @dataclass
150
+ class Segment:
151
+ clip_index: int
152
+ start: int # inclusive index in clip
153
+ end: int # exclusive index in clip
154
+ score: float
155
+
156
+ @property
157
+ def length(self) -> int:
158
+ return max(0, self.end - self.start)
159
+
160
+ def sort_key(self) -> Tuple[float, int]:
161
+ # Higher score first, then longer
162
+ return (self.score, self.length)
163
+
164
+ def overlaps(self, other: "Segment") -> bool:
165
+ if self.clip_index != other.clip_index:
166
+ return False
167
+ return not (self.end <= other.start or other.end <= self.start)
168
+
169
+
170
+ def _ssim_score_np(img_a: np.ndarray, img_b: np.ndarray, c1: float = 0.01**2, c2: float = 0.03**2) -> float:
171
+ # Mirror notebook SSIM (mean/var based, no gaussian window)
172
+ mu_a = float(img_a.mean())
173
+ mu_b = float(img_b.mean())
174
+ sigma_a = float(img_a.var())
175
+ sigma_b = float(img_b.var())
176
+ sigma_ab = float(((img_a - mu_a) * (img_b - mu_b)).mean())
177
+ numerator = (2.0 * mu_a * mu_b + c1) * (2.0 * sigma_ab + c2)
178
+ denominator = (mu_a * mu_a + mu_b * mu_b + c1) * (sigma_a + sigma_b + c2) + 1e-8
179
+ return float(numerator / denominator)
180
+
181
+
182
+ def _to_gray_resized01(rgb01: np.ndarray, hw: Tuple[int, int]) -> np.ndarray:
183
+ # rgb01: [H,W,3] 0..1
184
+ x = tf.image.resize(tf.convert_to_tensor(rgb01[None, ...], tf.float32), hw, method="bilinear", antialias=True)[0]
185
+ g = tf.reduce_mean(x, axis=-1) # [H,W]
186
+ return g.numpy()
187
+
188
+
189
+ def _find_motion_segments_notebook_style(
190
+ clip_paths: Sequence[Path],
191
+ threshold: float,
192
+ ssim_hw: Tuple[int, int],
193
+ ) -> List[Tuple[int, int, float]]:
194
+ """
195
+ Mirror notebook find_motion_segments:
196
+ - compute delta_ssim = 1 - ssim_score_np(gray[i-1], gray[i])
197
+ - enter motion when delta > threshold (start=i-1)
198
+ - exit when delta <= threshold (append (start, i))
199
+ - if still in motion at end, append (start, len-1)
200
+ We also compute score as mean delta over the segment transitions.
201
+ """
202
+ if len(clip_paths) < 2:
203
+ return []
204
+
205
+ # Streamed implementation to avoid loading the whole clip into memory.
206
+ from PIL import Image
207
+
208
+ def load_gray(p: Path) -> np.ndarray:
209
+ rgb = np.asarray(Image.open(p).convert("RGB"), dtype=np.float32) / 255.0
210
+ return _to_gray_resized01(rgb, ssim_hw)
211
+
212
+ segments: List[Tuple[int, int, float]] = []
213
+ in_motion = False
214
+ start_idx = -1
215
+ acc = 0.0
216
+ acc_n = 0
217
+
218
+ prev_gray = load_gray(clip_paths[0])
219
+ for i in range(1, len(clip_paths)):
220
+ cur_gray = load_gray(clip_paths[i])
221
+ delta_ssim = 1.0 - _ssim_score_np(prev_gray, cur_gray)
222
+
223
+ if (not in_motion) and delta_ssim > threshold:
224
+ in_motion = True
225
+ start_idx = i - 1
226
+ acc = float(delta_ssim)
227
+ acc_n = 1
228
+ elif in_motion and delta_ssim > threshold:
229
+ acc += float(delta_ssim)
230
+ acc_n += 1
231
+ elif in_motion and delta_ssim <= threshold:
232
+ end_idx = i # as notebook
233
+ score = (acc / acc_n) if acc_n else 0.0
234
+ segments.append((start_idx, end_idx, score))
235
+ in_motion = False
236
+ acc = 0.0
237
+ acc_n = 0
238
+
239
+ prev_gray = cur_gray
240
+
241
+ if in_motion:
242
+ end_idx = len(clip_paths) - 1
243
+ score = (acc / acc_n) if acc_n else 0.0
244
+ segments.append((start_idx, end_idx, score))
245
+
246
+ return segments
247
+
248
+
249
+ def _topk_motion_windows(
250
+ frames01: np.ndarray,
251
+ ssim_hw: Tuple[int, int],
252
+ window_len: int,
253
+ step: int = 1,
254
+ ) -> List[Tuple[int, int, float]]:
255
+ """
256
+ Sliding windows scored by mean motion (1-SSIM) inside the window.
257
+ Returns [(start,end,score)] where end=start+window_len.
258
+ """
259
+ if window_len < 2 or frames01.shape[0] < window_len:
260
+ return []
261
+ ssim = _ssim_adjacent(frames01, ssim_hw) # len T-1
262
+ motion = 1.0 - ssim
263
+
264
+ out: List[Tuple[int, int, float]] = []
265
+ last_start = frames01.shape[0] - window_len
266
+ for start in range(0, last_start + 1, step):
267
+ # motion indices cover transitions; for frames [start, end) with end=start+L,
268
+ # the relevant motion slice is [start, end-1) (length L-1)
269
+ end = start + window_len
270
+ m = motion[start : end - 1]
271
+ score = float(np.mean(m)) if m.size else 0.0
272
+ out.append((start, end, score))
273
+ return out
274
+
275
+
276
+ def pick_best_segments(
277
+ frames_dir: Path,
278
+ threshold: float = 0.005,
279
+ ssim_hw: Tuple[int, int] = (64, 64),
280
+ max_gap_ms: int = 32,
281
+ min_len: int = 8,
282
+ topk: int = 50,
283
+ k: int = 4,
284
+ ) -> Tuple[List[List[Path]], List[Segment]]:
285
+ paths = [p for p in frames_dir.iterdir() if p.is_file() and p.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp", ".bmp"}]
286
+ paths.sort(key=lambda p: _natural_key(p.name))
287
+ if not paths:
288
+ raise ValueError(f"No frames found in {frames_dir}")
289
+
290
+ clips = _split_by_timestamp_gap(paths, max_gap_ms=max_gap_ms)
291
+
292
+ candidates: List[Tuple[List[Path], Segment]] = []
293
+ for ci, clip in enumerate(clips):
294
+ if len(clip) < min_len:
295
+ continue
296
+ arr = []
297
+ from PIL import Image
298
+
299
+ for p in clip:
300
+ arr.append(np.asarray(Image.open(p).convert("RGB"), dtype=np.float32))
301
+ x = np.stack(arr, axis=0)
302
+ x01 = np.clip(x / 255.0, 0.0, 1.0)
303
+
304
+ # Primary: threshold-based motion segments (not always present)
305
+ cand: List[Segment] = []
306
+ for s, e, sc in _find_motion_segments_in_clip(x01, threshold=threshold, ssim_hw=ssim_hw):
307
+ seg = Segment(clip_index=ci, start=s, end=e, score=sc)
308
+ if seg.length >= min_len:
309
+ cand.append(seg)
310
+
311
+ # Fallback/augment: always take top motion windows so we can pick "bigger change"
312
+ # even when thresholding yields nothing.
313
+ windows = _topk_motion_windows(x01, ssim_hw=ssim_hw, window_len=max(min_len, 8), step=1)
314
+ windows.sort(key=lambda t: t[2], reverse=True)
315
+ for s, e, sc in windows[:topk]:
316
+ cand.append(Segment(clip_index=ci, start=s, end=e, score=sc))
317
+
318
+ for seg in cand:
319
+ candidates.append((clip, seg))
320
+
321
+ if not candidates:
322
+ clip = max(clips, key=len)
323
+ seg = Segment(clip_index=clips.index(clip), start=0, end=len(clip), score=0.0)
324
+ return [clip], [seg]
325
+
326
+ # Sort all candidates by score then length
327
+ candidates.sort(key=lambda cs: (cs[1].score, cs[1].length), reverse=True)
328
+
329
+ picked_clips: List[List[Path]] = []
330
+ picked_segs: List[Segment] = []
331
+ for clip, seg in candidates:
332
+ if any(seg.overlaps(s) for s in picked_segs):
333
+ continue
334
+ picked_clips.append(clip)
335
+ picked_segs.append(seg)
336
+ if len(picked_segs) >= k:
337
+ break
338
+
339
+ return picked_clips, picked_segs
340
+
341
+
342
+ def find_all_motion_segments(
343
+ frames_dir: Path,
344
+ threshold: float,
345
+ ssim_hw: Tuple[int, int],
346
+ max_gap_ms: int,
347
+ min_len: int,
348
+ ) -> Tuple[List[List[Path]], List[Segment]]:
349
+ # Mirror notebook: do NOT split into clips by timestamp gap.
350
+ paths = [p for p in frames_dir.iterdir() if p.is_file() and p.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp", ".bmp"}]
351
+ paths.sort(key=lambda p: _natural_key(p.name))
352
+ if not paths:
353
+ raise ValueError(f"No frames found in {frames_dir}")
354
+
355
+ # Notebook has a single clip here (unless directory-of-directories).
356
+ clip = paths
357
+ segs = _find_motion_segments_notebook_style(clip, threshold=threshold, ssim_hw=ssim_hw)
358
+
359
+ out_clips: List[List[Path]] = []
360
+ out_segs: List[Segment] = []
361
+ for s, e, sc in segs:
362
+ # In notebook segments are tuples (start, end) with mixed inclusivity; for our slicing,
363
+ # treat as [start, end] inclusive if end < len, else [start, end).
364
+ # We'll convert to Python slice end-exclusive conservatively.
365
+ end_excl = min(e + 1, len(clip))
366
+ seg_len = end_excl - s
367
+ if seg_len < min_len:
368
+ continue
369
+ # Also require there exists at least one valid sequence of length (sequence_len) within this segment
370
+ # that passes the adjacent timestamp gap filter, matching notebook behavior.
371
+ out_clips.append(clip)
372
+ out_segs.append(Segment(clip_index=0, start=s, end=end_excl, score=float(sc)))
373
+
374
+ order = sorted(range(len(out_segs)), key=lambda i: (out_segs[i].score, out_segs[i].length), reverse=True)
375
+ return [out_clips[i] for i in order], [out_segs[i] for i in order]
376
+
377
+
378
+ def _parse_args() -> argparse.Namespace:
379
+ ap = argparse.ArgumentParser(description="SSIM-filter dataset and generate teacher-forcing predictions.")
380
+ ap.add_argument("--data_dir", type=str, required=True, help="Directory with frame images (e.g. .../data/data)")
381
+ ap.add_argument("--model_dir", type=str, default="savedmodel", help="SavedModel directory")
382
+ ap.add_argument("--out_dir", type=str, default="dataset_outputs", help="Output directory")
383
+ ap.add_argument("--threshold", type=float, default=0.005, help="Motion threshold on (1-SSIM)")
384
+ ap.add_argument("--ssim_hw", type=str, default="64,64", help="SSIM resize H,W")
385
+ ap.add_argument("--max_gap_ms", type=int, default=32, help="Max adjacent timestamp gap (ms)")
386
+ ap.add_argument("--min_len", type=int, default=8, help="Min segment length (frames)")
387
+ ap.add_argument("--sequence_len", type=int, default=4, help="Model input sequence length (frames)")
388
+ ap.add_argument(
389
+ "--pad_last_frame",
390
+ type=str,
391
+ default="none",
392
+ choices=["none", "zero", "one", "repeat"],
393
+ help="When model expects 4 frames but you provide 3, pad the last frame with this mode.",
394
+ )
395
+ ap.add_argument("--grid_cols", type=int, default=8, help="Columns for saved grids")
396
+ ap.add_argument("--topk_windows", type=int, default=50, help="How many top motion windows to consider per clip")
397
+ ap.add_argument("--num_tests", type=int, default=4, help="How many motion regions to run")
398
+ ap.add_argument(
399
+ "--mode",
400
+ type=str,
401
+ default="topk",
402
+ choices=["topk", "all_motion"],
403
+ help="topk: pick non-overlapping top regions; all_motion: run every threshold-based motion segment",
404
+ )
405
+ return ap.parse_args()
406
+
407
+
408
+ def main() -> None:
409
+ args = _parse_args()
410
+ data_dir = Path(args.data_dir)
411
+ out_dir = Path(args.out_dir)
412
+ out_dir.mkdir(parents=True, exist_ok=True)
413
+
414
+ ssim_hw = tuple(int(x.strip()) for x in args.ssim_hw.split(",")) # type: ignore[assignment]
415
+
416
+ if args.mode == "all_motion":
417
+ clips, segs = find_all_motion_segments(
418
+ data_dir,
419
+ threshold=args.threshold,
420
+ ssim_hw=ssim_hw, # type: ignore[arg-type]
421
+ max_gap_ms=args.max_gap_ms,
422
+ min_len=args.min_len,
423
+ )
424
+ print(f"Found motion segments: {len(segs)}")
425
+ else:
426
+ clips, segs = pick_best_segments(
427
+ data_dir,
428
+ threshold=args.threshold,
429
+ ssim_hw=ssim_hw, # type: ignore[arg-type]
430
+ max_gap_ms=args.max_gap_ms,
431
+ min_len=args.min_len,
432
+ topk=args.topk_windows,
433
+ k=args.num_tests,
434
+ )
435
+
436
+ pred = Predictor(args.model_dir)
437
+ L = int(args.sequence_len)
438
+
439
+ from PIL import Image
440
+
441
+ test_root = out_dir / "test"
442
+ test_root.mkdir(parents=True, exist_ok=True)
443
+
444
+ for idx, (clip, seg) in enumerate(zip(clips, segs), start=1):
445
+ seg_paths = clip[seg.start : seg.end]
446
+ if len(seg_paths) < L:
447
+ continue
448
+
449
+ test_dir = test_root / str(idx)
450
+ (test_dir / "pred").mkdir(parents=True, exist_ok=True)
451
+ (test_dir / "gt").mkdir(parents=True, exist_ok=True)
452
+
453
+ print(f"[test/{idx}] clip={seg.clip_index} start={seg.start} end={seg.end} len={len(seg_paths)} score={seg.score:.6f}")
454
+ print(f"[test/{idx}] first={seg_paths[0].name} last={seg_paths[-1].name}")
455
+
456
+ frames = np.stack([np.asarray(Image.open(p).convert("RGB"), dtype=np.float32) for p in seg_paths], axis=0)
457
+ frames01 = np.clip(frames / 255.0, 0.0, 1.0).astype(np.float32)
458
+
459
+ preds_last: List[np.ndarray] = []
460
+ gt_last: List[np.ndarray] = []
461
+
462
+ kept = 0
463
+ skipped = 0
464
+ for i in range(L - 1, len(frames01)):
465
+ # Teacher forcing with the same "max adjacent timestamp gap" constraint as notebook.
466
+ seq_paths = seg_paths[i - (L - 1) : i + 1]
467
+ if not _has_valid_adjacent_timestamp_gap(seq_paths, args.max_gap_ms):
468
+ skipped += 1
469
+ continue
470
+
471
+ window = frames01[i - (L - 1) : i + 1]
472
+ y = pred.predict_last_frame(window, pad_last_frame=args.pad_last_frame)[0]
473
+ preds_last.append(y)
474
+ gt_last.append(frames01[i])
475
+ kept += 1
476
+
477
+ stem = seg_paths[i].stem
478
+ save_image(test_dir / "pred" / f"pred_{i:04d}_{stem}.png", y)
479
+ save_image(test_dir / "gt" / f"gt_{i:04d}_{stem}.png", frames01[i])
480
+
481
+ if preds_last:
482
+ save_sequence_grid(test_dir / "pred_last_grid.png", np.stack(preds_last, axis=0), cols=args.grid_cols)
483
+ save_sequence_grid(test_dir / "gt_grid.png", np.stack(gt_last, axis=0), cols=args.grid_cols)
484
+ (test_dir / "meta.txt").write_text(
485
+ "\n".join(
486
+ [
487
+ f"clip_index={seg.clip_index}",
488
+ f"start={seg.start}",
489
+ f"end={seg.end}",
490
+ f"len={len(seg_paths)}",
491
+ f"score_mean_1_minus_ssim={seg.score:.6f}",
492
+ f"first_frame={seg_paths[0].name}",
493
+ f"last_frame={seg_paths[-1].name}",
494
+ f"sequence_len={L}",
495
+ f"threshold={args.threshold}",
496
+ f"ssim_hw={ssim_hw[0]},{ssim_hw[1]}",
497
+ f"max_gap_ms={args.max_gap_ms}",
498
+ f"kept_predictions={kept}",
499
+ f"skipped_by_gap={skipped}",
500
+ "",
501
+ ]
502
+ ),
503
+ encoding="utf-8",
504
+ )
505
+
506
+ print(f"Wrote tests under: {test_root}")
507
+
508
+
509
+ if __name__ == "__main__":
510
+ main()
511
+
infer/__init__.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ from .predictor import Predictor
2
+ from .anomaly import AnomalyResult, compute_anomaly_from_4frames
3
+
4
+ __all__ = ["Predictor", "AnomalyResult", "compute_anomaly_from_4frames"]
infer/anomaly.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Dict, Literal, Optional, Tuple
5
+
6
+ import numpy as np
7
+
8
+ from .predictor import Predictor, PadMode
9
+
10
+
11
+ @dataclass
12
+ class AnomalyResult:
13
+ # All arrays are float32
14
+ ahat_error: np.ndarray # [4] mean absolute error(pred_t, gt4)
15
+ dissim_1mssim: np.ndarray # [4] 1-SSIM(pred_t, gt4)
16
+
17
+
18
+ def _ensure_batched(frames: np.ndarray) -> np.ndarray:
19
+ if frames.ndim == 4:
20
+ return frames[None, ...]
21
+ return frames
22
+
23
+
24
+ def compute_anomaly_from_4frames(
25
+ predictor: Predictor,
26
+ frames4: np.ndarray,
27
+ ) -> AnomalyResult:
28
+ """
29
+ frames4: [4,H,W,C] or [B,4,H,W,C] in 0..1 or 0..255
30
+ Compares each predicted frame t=1..4 to GT frame4 (index 3).
31
+ Outputs two vectors length 4:
32
+ - ahat_error[t] = mean(abs(pred_t - gt4))
33
+ - dissim_1mssim[t] = 1 - ssim(pred_t, gt4)
34
+ """
35
+ outs = predictor.predict_outputs(frames4, pad_last_frame="none")
36
+ if "predictions" not in outs:
37
+ raise RuntimeError("SavedModel did not return 'predictions'.")
38
+ preds = outs["predictions"] # [B,4,H,W,C]
39
+ preds = np.asarray(preds, dtype=np.float32)
40
+
41
+ x = predictor._prep(frames4, pad_last_frame="none") # normalized+resized, [B,4,H,W,C]
42
+ x = np.asarray(x, dtype=np.float32)
43
+ gt4 = x[:, 3] # [B,H,W,C]
44
+
45
+ # ahat_error: mean absolute error per time step vs gt4
46
+ abs_err = np.mean(np.abs(preds - gt4[:, None, ...]), axis=(2, 3, 4)) # [B,4]
47
+ ahat_error = np.mean(abs_err, axis=0) # [4]
48
+
49
+ # 1-SSIM per time step vs gt4 (use TF for consistent SSIM)
50
+ tf = predictor._tf
51
+ pred_t = tf.convert_to_tensor(preds, dtype=tf.float32) # [B,4,H,W,C]
52
+ gt_t = tf.convert_to_tensor(gt4[:, None, ...], dtype=tf.float32) # [B,1,H,W,C]
53
+ gt_t = tf.repeat(gt_t, repeats=4, axis=1)
54
+ pred_flat = tf.reshape(pred_t, (-1,) + tuple(preds.shape[2:]))
55
+ gt_flat = tf.reshape(gt_t, (-1,) + tuple(gt4.shape[1:]))
56
+ ssim = tf.image.ssim(pred_flat, gt_flat, max_val=1.0)
57
+ ssim = tf.reshape(ssim, (preds.shape[0], 4))
58
+ dissim = 1.0 - ssim
59
+ dissim_1mssim = tf.reduce_mean(dissim, axis=0).numpy().astype(np.float32) # [4]
60
+
61
+ return AnomalyResult(
62
+ ahat_error=np.asarray(ahat_error, dtype=np.float32),
63
+ dissim_1mssim=np.asarray(dissim_1mssim, dtype=np.float32),
64
+ )
65
+
infer/io.py ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import re
4
+ from pathlib import Path
5
+ from typing import Dict, Iterable, List, Optional, Sequence, Tuple, Union
6
+
7
+ import numpy as np
8
+ from PIL import Image
9
+
10
+
11
+ _IMG_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".bmp"}
12
+
13
+
14
+ def _natural_key(s: str):
15
+ return [int(t) if t.isdigit() else t.lower() for t in re.split(r"(\d+)", s)]
16
+
17
+
18
+ def load_frames_from_dir(frames_dir: Union[str, Path]) -> np.ndarray:
19
+ d = Path(frames_dir)
20
+ if not d.exists():
21
+ raise FileNotFoundError(str(d))
22
+ paths = [p for p in d.iterdir() if p.is_file() and p.suffix.lower() in _IMG_EXTS]
23
+ paths.sort(key=lambda p: _natural_key(p.name))
24
+ if not paths:
25
+ raise ValueError(f"No image frames found in {d}")
26
+
27
+ arrs: List[np.ndarray] = []
28
+ for p in paths:
29
+ im = Image.open(p).convert("RGB")
30
+ arrs.append(np.asarray(im, dtype=np.float32))
31
+ x = np.stack(arrs, axis=0) # [T,H,W,C] in 0..255
32
+ return x
33
+
34
+
35
+ def load_array(path: Union[str, Path]) -> np.ndarray:
36
+ p = Path(path)
37
+ if not p.exists():
38
+ raise FileNotFoundError(str(p))
39
+ if p.suffix.lower() == ".npy":
40
+ return np.load(p)
41
+ if p.suffix.lower() == ".npz":
42
+ z = np.load(p)
43
+ if "frames" in z:
44
+ return z["frames"]
45
+ # fall back to first array
46
+ k = next(iter(z.files))
47
+ return z[k]
48
+ raise ValueError("Expected .npy or .npz")
49
+
50
+
51
+ def to_uint8_rgb(img01: np.ndarray) -> np.ndarray:
52
+ x = np.clip(img01, 0.0, 1.0)
53
+ return (x * 255.0 + 0.5).astype(np.uint8)
54
+
55
+
56
+ def save_image(path: Union[str, Path], img01: np.ndarray) -> None:
57
+ p = Path(path)
58
+ p.parent.mkdir(parents=True, exist_ok=True)
59
+ Image.fromarray(to_uint8_rgb(img01)).save(p)
60
+
61
+
62
+ def save_sequence_grid(path: Union[str, Path], seq01: np.ndarray, cols: int = 8) -> None:
63
+ # seq01: [T,H,W,C]
64
+ p = Path(path)
65
+ p.parent.mkdir(parents=True, exist_ok=True)
66
+ t, h, w, c = seq01.shape
67
+ cols = max(1, int(cols))
68
+ rows = (t + cols - 1) // cols
69
+ canvas = np.zeros((rows * h, cols * w, c), dtype=np.uint8)
70
+ for i in range(t):
71
+ r = i // cols
72
+ col = i % cols
73
+ canvas[r * h : (r + 1) * h, col * w : (col + 1) * w] = to_uint8_rgb(seq01[i])
74
+ Image.fromarray(canvas).save(p)
75
+
infer/predictor.py ADDED
@@ -0,0 +1,210 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from pathlib import Path
5
+ from typing import Any, Dict, Mapping, Optional, Tuple, Union, Literal
6
+
7
+ import numpy as np
8
+
9
+
10
+ ArrayLike = Union[np.ndarray]
11
+ PadMode = Literal["none", "zero", "one", "repeat"]
12
+
13
+
14
+ def _as_5d(frames: np.ndarray) -> np.ndarray:
15
+ if frames.ndim == 4:
16
+ return frames[None, ...]
17
+ if frames.ndim == 5:
18
+ return frames
19
+ raise ValueError(f"Expected frames with rank 4 or 5, got shape={frames.shape}")
20
+
21
+
22
+ def _normalize01(x: np.ndarray) -> np.ndarray:
23
+ x = x.astype(np.float32, copy=False)
24
+ if x.max() > 1.5:
25
+ x = x / 255.0
26
+ return np.clip(x, 0.0, 1.0)
27
+
28
+
29
+ def _pad_last_frame(frames: np.ndarray, pad_mode: PadMode, target_len: int) -> np.ndarray:
30
+ """
31
+ frames: [B,T,H,W,C] float in 0..1
32
+ Returns [B,target_len,H,W,C].
33
+ """
34
+ if pad_mode == "none":
35
+ return frames
36
+ b, t, h, w, c = frames.shape
37
+ if t > target_len:
38
+ return frames[:, :target_len]
39
+ if t == target_len:
40
+ return frames
41
+ if t < target_len - 1:
42
+ raise ValueError(f"pad_last_frame supports T=={target_len-1} or T=={target_len}, got T={t}")
43
+
44
+ if pad_mode == "repeat":
45
+ last = frames[:, -1:, :, :, :]
46
+ elif pad_mode == "zero":
47
+ last = np.zeros((b, 1, h, w, c), dtype=frames.dtype)
48
+ elif pad_mode == "one":
49
+ last = np.ones((b, 1, h, w, c), dtype=frames.dtype)
50
+ else:
51
+ raise ValueError(f"Unknown pad_mode={pad_mode}")
52
+ return np.concatenate([frames, last], axis=1)
53
+
54
+
55
+ def _pick_signature(module) -> Tuple[str, Any]:
56
+ sigs = getattr(module, "signatures", None)
57
+ if not isinstance(sigs, Mapping) or not sigs:
58
+ raise RuntimeError("SavedModel has no callable signatures.")
59
+ if "serving_default" in sigs:
60
+ return "serving_default", sigs["serving_default"]
61
+ name = next(iter(sigs.keys()))
62
+ return name, sigs[name]
63
+
64
+
65
+ def _signature_io(fn) -> Tuple[Dict[str, Any], Dict[str, Any]]:
66
+ # structured_input_signature = (args, kwargs)
67
+ args, kwargs = fn.structured_input_signature
68
+ if args:
69
+ raise RuntimeError("This SavedModel signature expects positional args; unsupported.")
70
+ return dict(kwargs), dict(fn.structured_outputs)
71
+
72
+
73
+ def _infer_hw_from_inputs(input_specs: Mapping[str, Any]) -> Optional[Tuple[int, int]]:
74
+ # Try to find a spec with shape [B,T,H,W,C] or [B,H,W,C]
75
+ for spec in input_specs.values():
76
+ shape = getattr(spec, "shape", None)
77
+ if shape is None:
78
+ continue
79
+ dims = list(shape)
80
+ if len(dims) == 5 and dims[2] is not None and dims[3] is not None:
81
+ return int(dims[2]), int(dims[3])
82
+ if len(dims) == 4 and dims[1] is not None and dims[2] is not None:
83
+ return int(dims[1]), int(dims[2])
84
+ return None
85
+
86
+
87
+ def _select_prediction_tensor(outputs: Mapping[str, Any]):
88
+ # Heuristic: prefer key contains 'pred', else first rank-5 tensor, else first tensor-like output
89
+ items = list(outputs.items())
90
+ for k, v in items:
91
+ if "pred" in k.lower():
92
+ return k, v
93
+ for k, v in items:
94
+ shape = getattr(v, "shape", None)
95
+ if shape is not None and len(shape) == 5:
96
+ return k, v
97
+ return items[0]
98
+
99
+
100
+ @dataclass
101
+ class Predictor:
102
+ model_dir: Union[str, Path]
103
+ signature_name: Optional[str] = None
104
+ input_key: Optional[str] = None
105
+ resize_hw: Optional[Tuple[int, int]] = None
106
+
107
+ def __post_init__(self) -> None:
108
+ import tensorflow as tf
109
+
110
+ self._tf = tf
111
+
112
+ # Prefer GPU if available; otherwise CPU is used automatically.
113
+ try:
114
+ gpus = tf.config.list_physical_devices("GPU")
115
+ for gpu in gpus:
116
+ # Avoid TF grabbing all GPU memory up front.
117
+ tf.config.experimental.set_memory_growth(gpu, True)
118
+ self.device: str = "GPU" if gpus else "CPU"
119
+ except Exception:
120
+ self.device = "CPU"
121
+
122
+ self.model_dir = Path(self.model_dir)
123
+ self._module = tf.saved_model.load(str(self.model_dir))
124
+ name, fn = _pick_signature(self._module)
125
+ if self.signature_name is None:
126
+ self.signature_name = name
127
+ self._fn = self._module.signatures[self.signature_name]
128
+
129
+ in_specs, out_specs = _signature_io(self._fn)
130
+ self._in_specs = in_specs
131
+ self._out_specs = out_specs
132
+
133
+ if self.input_key is None:
134
+ # Prefer single input; otherwise pick first
135
+ if len(in_specs) == 1:
136
+ self.input_key = next(iter(in_specs.keys()))
137
+ else:
138
+ self.input_key = next(iter(in_specs.keys()))
139
+
140
+ if self.resize_hw is None:
141
+ self.resize_hw = _infer_hw_from_inputs(in_specs) or (128, 128)
142
+
143
+ # Infer expected sequence length if fixed (e.g. 4)
144
+ spec = self._in_specs.get(self.input_key) if self.input_key else None
145
+ shape = getattr(spec, "shape", None)
146
+ self.sequence_len: Optional[int] = None
147
+ if shape is not None and len(shape) == 5 and shape[1] is not None:
148
+ self.sequence_len = int(shape[1])
149
+
150
+ def _prep(self, frames: np.ndarray, pad_last_frame: PadMode = "none") -> np.ndarray:
151
+ frames = _as_5d(frames)
152
+ frames = _normalize01(frames)
153
+
154
+ # Resize with TF to match model expectation (keeps it black-box)
155
+ b, t, h, w, c = frames.shape
156
+ rh, rw = self.resize_hw or (h, w)
157
+ if (h, w) != (rh, rw):
158
+ tf = self._tf
159
+ x = tf.convert_to_tensor(frames, dtype=tf.float32)
160
+ x = tf.reshape(x, (b * t, h, w, c))
161
+ x = tf.image.resize(x, (rh, rw), method="bilinear", antialias=True)
162
+ x = tf.reshape(x, (b, t, rh, rw, c))
163
+ frames = x.numpy()
164
+
165
+ if self.sequence_len is not None:
166
+ frames = _pad_last_frame(frames, pad_last_frame, target_len=self.sequence_len)
167
+ return frames
168
+
169
+ def predict_sequence(self, frames: np.ndarray, pad_last_frame: PadMode = "none") -> np.ndarray:
170
+ tf = self._tf
171
+ x = self._prep(frames, pad_last_frame=pad_last_frame)
172
+ inp = {self.input_key: tf.convert_to_tensor(x, dtype=tf.float32)}
173
+ outs = self._fn(**inp)
174
+ pred = outs.get("predictions")
175
+ if pred is None:
176
+ _, pred = _select_prediction_tensor(outs)
177
+ pred_np = pred.numpy()
178
+ return np.clip(pred_np, 0.0, 1.0)
179
+
180
+ def predict_outputs(self, frames: np.ndarray, pad_last_frame: PadMode = "none") -> Dict[str, np.ndarray]:
181
+ tf = self._tf
182
+ x = self._prep(frames, pad_last_frame=pad_last_frame)
183
+ inp = {self.input_key: tf.convert_to_tensor(x, dtype=tf.float32)}
184
+ outs = self._fn(**inp)
185
+ out_np: Dict[str, np.ndarray] = {}
186
+ for k, v in outs.items():
187
+ try:
188
+ out_np[k] = v.numpy()
189
+ except Exception:
190
+ pass
191
+ if "predictions" in out_np:
192
+ out_np["predictions"] = np.clip(out_np["predictions"], 0.0, 1.0)
193
+ return out_np
194
+
195
+ def predict_last_frame(self, frames: np.ndarray, pad_last_frame: PadMode = "none") -> np.ndarray:
196
+ seq = self.predict_sequence(frames, pad_last_frame=pad_last_frame)
197
+ if seq.ndim != 5:
198
+ raise RuntimeError(f"Expected prediction rank 5, got shape={seq.shape}")
199
+ return seq[:, -1]
200
+
201
+ def predict_next_from_3(self, frames3: np.ndarray, pad_last_frame: PadMode = "zero") -> np.ndarray:
202
+ """
203
+ Predict next frame from 3 context frames by padding the 4th input slot.
204
+ Returns [B,H,W,C].
205
+ """
206
+ if self.sequence_len is not None and self.sequence_len != 4:
207
+ raise RuntimeError(f"predict_next_from_3 expects sequence_len=4, got {self.sequence_len}")
208
+ last = self.predict_last_frame(frames3, pad_last_frame=pad_last_frame)
209
+ return last
210
+
modeling.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from pathlib import Path
3
+
4
+ from huggingface_hub import snapshot_download
5
+
6
+ from .configuration import PredNetConfig
7
+ from .infer.predictor import Predictor
8
+
9
+
10
+ class PredNetModel:
11
+ """Black-box TF SavedModel wrapper compatible with AutoModel via trust_remote_code."""
12
+
13
+ def __init__(self, config: PredNetConfig, predictor: Predictor):
14
+ self.config = config
15
+ self._predictor = predictor
16
+
17
+ @classmethod
18
+ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
19
+ if os.path.isdir(pretrained_model_name_or_path):
20
+ local_dir = pretrained_model_name_or_path
21
+ else:
22
+ local_dir = snapshot_download(pretrained_model_name_or_path)
23
+
24
+ config = PredNetConfig.from_pretrained(local_dir)
25
+ savedmodel_dir = str(Path(local_dir) / "savedmodel")
26
+ predictor = Predictor(
27
+ model_dir=savedmodel_dir,
28
+ resize_hw=tuple(config.resize_hw),
29
+ )
30
+ return cls(config, predictor)
31
+
32
+ def predict_sequence(self, frames, pad_last_frame="none"):
33
+ return self._predictor.predict_sequence(frames, pad_last_frame=pad_last_frame)
34
+
35
+ def predict_last_frame(self, frames, pad_last_frame="none"):
36
+ return self._predictor.predict_last_frame(frames, pad_last_frame=pad_last_frame)
37
+
38
+ def predict_outputs(self, frames, pad_last_frame="none"):
39
+ return self._predictor.predict_outputs(frames, pad_last_frame=pad_last_frame)
pipeline.py ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+
3
+ import numpy as np
4
+ from transformers import Pipeline
5
+
6
+
7
+ class VideoFramePredictionPipeline(Pipeline):
8
+ """
9
+ HuggingFace Pipeline for black-box video frame prediction.
10
+
11
+ Usage:
12
+ pipe = pipeline("video-frame-prediction", model="dvdface/denoising-prednet",
13
+ trust_remote_code=True)
14
+ # frames: numpy [T,H,W,C] uint8/float, or path to frame directory
15
+ result = pipe(frames)
16
+ # Returns dict: {"sequence": [T,H,W,C], "last_frame": [H,W,C]}
17
+ """
18
+
19
+ def _sanitize_parameters(self, pad_last_frame="none", return_last_only=False, **kwargs):
20
+ forward_params = {"pad_last_frame": pad_last_frame}
21
+ postprocess_params = {"return_last_only": return_last_only}
22
+ return {}, forward_params, postprocess_params
23
+
24
+ def preprocess(self, inputs):
25
+ if isinstance(inputs, (str, Path)):
26
+ from .infer.io import load_frames_from_dir
27
+ inputs = load_frames_from_dir(inputs)
28
+ return {"frames": np.asarray(inputs)}
29
+
30
+ def _forward(self, model_inputs, pad_last_frame="none"):
31
+ frames = model_inputs["frames"]
32
+ seq = self.model.predict_sequence(frames, pad_last_frame=pad_last_frame)
33
+ return {"predictions": seq}
34
+
35
+ def postprocess(self, model_outputs, return_last_only=False):
36
+ seq = model_outputs["predictions"]
37
+ # Remove batch dim if present
38
+ if seq.ndim == 5:
39
+ seq = seq[0] # [T,H,W,C]
40
+ last = seq[-1] # [H,W,C]
41
+ if return_last_only:
42
+ return last
43
+ return {"sequence": seq, "last_frame": last}
predict.py ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ from pathlib import Path
5
+ from typing import Optional
6
+
7
+ import numpy as np
8
+
9
+ from infer.io import load_array, load_frames_from_dir, save_image, save_sequence_grid
10
+ from infer.predictor import Predictor
11
+
12
+
13
+ def _parse_args() -> argparse.Namespace:
14
+ ap = argparse.ArgumentParser(description="Black-box SavedModel inference: output predicted frames.")
15
+ ap.add_argument("--model_dir", type=str, default="savedmodel", help="Path to SavedModel directory")
16
+ ap.add_argument("--frames_dir", type=str, default=None, help="Directory containing input frames (images)")
17
+ ap.add_argument("--array", type=str, default=None, help="Path to .npy/.npz containing frames")
18
+ ap.add_argument(
19
+ "--pad_last_frame",
20
+ type=str,
21
+ default="none",
22
+ choices=["none", "zero", "one", "repeat"],
23
+ help="If model expects 4 frames but you provide 3, pad the last frame with: zero/one/repeat",
24
+ )
25
+ ap.add_argument("--out_dir", type=str, default="outputs", help="Output directory")
26
+ ap.add_argument("--save_sequence_grid", action="store_true", help="Save a grid of the predicted sequence")
27
+ ap.add_argument("--grid_cols", type=int, default=8, help="Columns for sequence grid")
28
+ return ap.parse_args()
29
+
30
+
31
+ def main() -> None:
32
+ args = _parse_args()
33
+ if (args.frames_dir is None) == (args.array is None):
34
+ raise SystemExit("Provide exactly one of --frames_dir or --array")
35
+
36
+ if args.frames_dir is not None:
37
+ frames = load_frames_from_dir(args.frames_dir) # [T,H,W,C] 0..255
38
+ else:
39
+ frames = load_array(args.array)
40
+
41
+ pred = Predictor(args.model_dir)
42
+ seq = pred.predict_sequence(frames, pad_last_frame=args.pad_last_frame) # [B,T,H,W,C]
43
+ if seq.ndim == 5:
44
+ seq0 = seq[0]
45
+ last = seq[0, -1]
46
+ elif seq.ndim == 4:
47
+ seq0 = seq
48
+ last = seq[-1]
49
+ else:
50
+ raise RuntimeError(f"Unexpected prediction shape: {seq.shape}")
51
+
52
+ out_dir = Path(args.out_dir)
53
+ out_dir.mkdir(parents=True, exist_ok=True)
54
+ save_image(out_dir / "pred_last.png", last)
55
+
56
+ if args.save_sequence_grid:
57
+ save_sequence_grid(out_dir / "pred_sequence_grid.png", seq0, cols=args.grid_cols)
58
+
59
+ print(f"Wrote: {out_dir / 'pred_last.png'}")
60
+ if args.save_sequence_grid:
61
+ print(f"Wrote: {out_dir / 'pred_sequence_grid.png'}")
62
+
63
+
64
+ if __name__ == "__main__":
65
+ main()
66
+
requirements.txt ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ tensorflow
2
+ numpy
3
+ pillow
4
+ huggingface_hub
5
+ imageio
6
+ transformers
savedmodel/fingerprint.pb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:528aadaff82e9283d2fefdf35b8245289f8ebd0107c02ff0673bca994cab2085
3
+ size 77
savedmodel/saved_model.pb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5edf975b0705a826d17a3af45afc313096096784b0725c3432722eec855bdf30
3
+ size 1108400
savedmodel/variables/variables.data-00000-of-00001 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:824880996b8fd1dfb5c57d685a78e8b322d21338fdd5d92c797522b5d4c0bcfa
3
+ size 84082031
savedmodel/variables/variables.index ADDED
Binary file (3.56 kB). View file