feat(pose)!: use coco17 3d router features
Browse filesBREAKING CHANGE: exercise router artifacts must be regenerated with the coco17_3d_v1 feature schema.
- README.md +11 -2
- docs/huggingface-router-release.md +14 -0
- docs/implementation-plan.md +1 -1
- docs/pozify-project-documentation.md +9 -9
- scripts/exercise_router_modal.py +276 -31
- scripts/upload_exercise_router_to_hf.py +4 -0
- src/pozify/exercises/shared/analyzer.py +9 -8
- src/pozify/ml/exercise_router_features.py +29 -17
- src/pozify/ml/exercise_router_inference.py +52 -7
- src/pozify/steps/pose_backends/__init__.py +7 -1
- src/pozify/steps/pose_backends/landmarks.py +25 -16
- src/pozify/steps/pose_cleaning.py +96 -33
- src/pozify/steps/pose_landmarker.py +15 -0
- src/pozify/steps/rep_signals.py +52 -10
- tests/test_exercise_classifier.py +43 -24
- tests/test_pose_steps.py +11 -7
- tests/test_rep_counter.py +16 -0
- web/app.js +1 -1
README.md
CHANGED
|
@@ -95,7 +95,7 @@ When running in real mode, the UI summary now shows:
|
|
| 95 |
```text
|
| 96 |
user profile + input video
|
| 97 |
-> video QC
|
| 98 |
-
->
|
| 99 |
-> pose cleaning and normalization
|
| 100 |
-> exercise classifier
|
| 101 |
-> exercise-specific rep counter
|
|
@@ -248,6 +248,14 @@ uv run modal run scripts/exercise_router_modal.py --stage train-temporal
|
|
| 248 |
uv run modal run scripts/exercise_router_modal.py --stage evaluate
|
| 249 |
```
|
| 250 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
The latest training metrics and selected-artifact result are recorded in
|
| 252 |
[docs/exercise-router-training-report.md](docs/exercise-router-training-report.md).
|
| 253 |
Custom data collection is documented in
|
|
@@ -265,7 +273,8 @@ compact BiLSTM over 30-frame feature tensors on a Modal A10 GPU and writes `temp
|
|
| 265 |
`temporal_metrics.json`. Its default hyperparameters follow the Riccio exercise-classification paper:
|
| 266 |
73 hidden units, 0.2174 dropout, 0.0004 learning rate, batch size 54, and 73 epochs
|
| 267 |
(https://arxiv.org/abs/2411.11548). Evaluation scores every available trained artifact, writes
|
| 268 |
-
per-model metrics into `evaluation.json`,
|
|
|
|
| 269 |
The current selection policy prefers the BiLSTM temporal model when it is available; the baseline is
|
| 270 |
kept as a fallback/reference artifact.
|
| 271 |
|
|
|
|
| 95 |
```text
|
| 96 |
user profile + input video
|
| 97 |
-> video QC
|
| 98 |
+
-> 17-point 3D pose landmarker
|
| 99 |
-> pose cleaning and normalization
|
| 100 |
-> exercise classifier
|
| 101 |
-> exercise-specific rep counter
|
|
|
|
| 248 |
uv run modal run scripts/exercise_router_modal.py --stage evaluate
|
| 249 |
```
|
| 250 |
|
| 251 |
+
The `evaluate` and `all` stages automatically publish the selected router artifacts to Hugging Face
|
| 252 |
+
after evaluation. Set `POZIFY_ROUTER_HF_REPO_ID` and `HF_TOKEN` in `.env` or pass a repo explicitly:
|
| 253 |
+
|
| 254 |
+
```bash
|
| 255 |
+
uv run modal run scripts/exercise_router_modal.py --stage all --repo-id owner/pozify-exercise-router
|
| 256 |
+
uv run modal run scripts/exercise_router_modal.py --stage publish --repo-id owner/pozify-exercise-router
|
| 257 |
+
```
|
| 258 |
+
|
| 259 |
The latest training metrics and selected-artifact result are recorded in
|
| 260 |
[docs/exercise-router-training-report.md](docs/exercise-router-training-report.md).
|
| 261 |
Custom data collection is documented in
|
|
|
|
| 273 |
`temporal_metrics.json`. Its default hyperparameters follow the Riccio exercise-classification paper:
|
| 274 |
73 hidden units, 0.2174 dropout, 0.0004 learning rate, batch size 54, and 73 epochs
|
| 275 |
(https://arxiv.org/abs/2411.11548). Evaluation scores every available trained artifact, writes
|
| 276 |
+
per-model metrics into `evaluation.json`, records the active artifact in `router_selection.json`,
|
| 277 |
+
and publishes artifacts to Hugging Face when the `evaluate` or `all` stage completes.
|
| 278 |
The current selection policy prefers the BiLSTM temporal model when it is available; the baseline is
|
| 279 |
kept as a fallback/reference artifact.
|
| 280 |
|
docs/huggingface-router-release.md
CHANGED
|
@@ -59,6 +59,17 @@ uv run python scripts/upload_exercise_router_to_hf.py \
|
|
| 59 |
|
| 60 |
Omit `--private` if the model repository should be public.
|
| 61 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
The upload script creates the repository if needed and uploads:
|
| 63 |
|
| 64 |
- `README.md` model card
|
|
@@ -66,6 +77,9 @@ The upload script creates the repository if needed and uploads:
|
|
| 66 |
- `router_selection.json`
|
| 67 |
- `temporal.pt`
|
| 68 |
- `training_report.md`
|
|
|
|
|
|
|
|
|
|
| 69 |
|
| 70 |
## Runtime Configuration
|
| 71 |
|
|
|
|
| 59 |
|
| 60 |
Omit `--private` if the model repository should be public.
|
| 61 |
|
| 62 |
+
The Modal training script can also publish directly from the `pozify-router-models` volume after
|
| 63 |
+
evaluation. It reads `HF_TOKEN`, `POZIFY_ROUTER_HF_REPO_ID`, and optional
|
| 64 |
+
`POZIFY_ROUTER_HF_PRIVATE=1` from `.env` via `modal.Secret.from_dotenv()`:
|
| 65 |
+
|
| 66 |
+
```bash
|
| 67 |
+
uv run modal run scripts/exercise_router_modal.py --stage all
|
| 68 |
+
uv run modal run scripts/exercise_router_modal.py --stage publish --repo-id build-small-hackathon/pozify-exercise-router
|
| 69 |
+
```
|
| 70 |
+
|
| 71 |
+
`stage evaluate` and `stage all` publish automatically after writing `router_selection.json`.
|
| 72 |
+
|
| 73 |
The upload script creates the repository if needed and uploads:
|
| 74 |
|
| 75 |
- `README.md` model card
|
|
|
|
| 77 |
- `router_selection.json`
|
| 78 |
- `temporal.pt`
|
| 79 |
- `training_report.md`
|
| 80 |
+
- `evaluation.json`
|
| 81 |
+
- `baseline_metrics.json`
|
| 82 |
+
- `temporal_metrics.json`
|
| 83 |
|
| 84 |
## Runtime Configuration
|
| 85 |
|
docs/implementation-plan.md
CHANGED
|
@@ -28,7 +28,7 @@ Deliverables:
|
|
| 28 |
|
| 29 |
## Phase 3: Pose Extraction And Cleaning
|
| 30 |
|
| 31 |
-
Goal: convert videos into a clean
|
| 32 |
|
| 33 |
Deliverables:
|
| 34 |
|
|
|
|
| 28 |
|
| 29 |
## Phase 3: Pose Extraction And Cleaning
|
| 30 |
|
| 31 |
+
Goal: convert videos into a clean 17-point 3D pose sequence.
|
| 32 |
|
| 33 |
Deliverables:
|
| 34 |
|
docs/pozify-project-documentation.md
CHANGED
|
@@ -33,7 +33,7 @@ Depth strategy:
|
|
| 33 |
- Upload or record short workout videos.
|
| 34 |
- User profile and training intent form.
|
| 35 |
- Video quality check.
|
| 36 |
-
-
|
| 37 |
- Pose cleaning and normalization.
|
| 38 |
- Exercise classifier for `squat`, `push_up`, `shoulder_press`, and `unknown`.
|
| 39 |
- Exercise-specific rep counter.
|
|
@@ -79,7 +79,7 @@ Main pipeline:
|
|
| 79 |
```text
|
| 80 |
user profile + input video
|
| 81 |
-> video QC
|
| 82 |
-
->
|
| 83 |
-> pose cleaning and normalization
|
| 84 |
-> exercise classifier
|
| 85 |
-> exercise-specific rep counter
|
|
@@ -98,7 +98,7 @@ End-to-end flow:
|
|
| 98 |
flowchart TD
|
| 99 |
A["User uploads or records workout video"] --> B["Video QC"]
|
| 100 |
U["User profile and training intent"] --> G["Grounded coach summary"]
|
| 101 |
-
B --> C["
|
| 102 |
C --> D["Pose cleaning and normalization"]
|
| 103 |
D --> E["Exercise classifier"]
|
| 104 |
E --> F{"Exercise type"}
|
|
@@ -402,7 +402,7 @@ Current implementation:
|
|
| 402 |
|
| 403 |
- Returns mocked video metadata and warnings.
|
| 404 |
|
| 405 |
-
### Step 2:
|
| 406 |
|
| 407 |
Goal:
|
| 408 |
|
|
@@ -415,8 +415,8 @@ Recommended model:
|
|
| 415 |
Reasons:
|
| 416 |
|
| 417 |
- Works on decoded video frames.
|
| 418 |
-
- Outputs
|
| 419 |
-
- Provides image coordinates
|
| 420 |
- Practical for a Gradio Space.
|
| 421 |
|
| 422 |
Fine-tuning:
|
|
@@ -431,8 +431,8 @@ Current implementation:
|
|
| 431 |
Future implementation:
|
| 432 |
|
| 433 |
- Run MediaPipe on sampled frames.
|
| 434 |
-
- Store all
|
| 435 |
-
- Store visibility
|
| 436 |
|
| 437 |
### Step 3: Pose Cleaning And Normalization
|
| 438 |
|
|
@@ -715,7 +715,7 @@ Recommended model:
|
|
| 715 |
|
| 716 |
Role:
|
| 717 |
|
| 718 |
-
- Extract
|
| 719 |
|
| 720 |
Fine-tuning:
|
| 721 |
|
|
|
|
| 33 |
- Upload or record short workout videos.
|
| 34 |
- User profile and training intent form.
|
| 35 |
- Video quality check.
|
| 36 |
+
- 17-point 3D pose extraction.
|
| 37 |
- Pose cleaning and normalization.
|
| 38 |
- Exercise classifier for `squat`, `push_up`, `shoulder_press`, and `unknown`.
|
| 39 |
- Exercise-specific rep counter.
|
|
|
|
| 79 |
```text
|
| 80 |
user profile + input video
|
| 81 |
-> video QC
|
| 82 |
+
-> 17-point 3D pose landmarker
|
| 83 |
-> pose cleaning and normalization
|
| 84 |
-> exercise classifier
|
| 85 |
-> exercise-specific rep counter
|
|
|
|
| 98 |
flowchart TD
|
| 99 |
A["User uploads or records workout video"] --> B["Video QC"]
|
| 100 |
U["User profile and training intent"] --> G["Grounded coach summary"]
|
| 101 |
+
B --> C["17-point 3D Pose Landmarker"]
|
| 102 |
C --> D["Pose cleaning and normalization"]
|
| 103 |
D --> E["Exercise classifier"]
|
| 104 |
E --> F{"Exercise type"}
|
|
|
|
| 402 |
|
| 403 |
- Returns mocked video metadata and warnings.
|
| 404 |
|
| 405 |
+
### Step 2: 17-Point 3D Pose Landmarker
|
| 406 |
|
| 407 |
Goal:
|
| 408 |
|
|
|
|
| 415 |
Reasons:
|
| 416 |
|
| 417 |
- Works on decoded video frames.
|
| 418 |
+
- Outputs COCO-17 body landmarks.
|
| 419 |
+
- Provides image coordinates for rendering and world coordinates for 3D exercise metrics.
|
| 420 |
- Practical for a Gradio Space.
|
| 421 |
|
| 422 |
Fine-tuning:
|
|
|
|
| 431 |
Future implementation:
|
| 432 |
|
| 433 |
- Run MediaPipe on sampled frames.
|
| 434 |
+
- Store all 17 COCO body landmarks.
|
| 435 |
+
- Store visibility, pose quality, and 3D world coordinates per frame.
|
| 436 |
|
| 437 |
### Step 3: Pose Cleaning And Normalization
|
| 438 |
|
|
|
|
| 715 |
|
| 716 |
Role:
|
| 717 |
|
| 718 |
+
- Extract 17 COCO body landmarks per valid frame, with 3D world coordinates when available.
|
| 719 |
|
| 720 |
Fine-tuning:
|
| 721 |
|
scripts/exercise_router_modal.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import json
|
|
|
|
| 4 |
from pathlib import Path
|
| 5 |
import shutil
|
| 6 |
import sys
|
|
@@ -12,8 +13,12 @@ import modal
|
|
| 12 |
|
| 13 |
APP_NAME = "pozify-exercise-router"
|
| 14 |
DATASET_ID = "RickyRiccio/Real_Time_Exercise_Recognition_Dataset"
|
|
|
|
|
|
|
|
|
|
| 15 |
DATA_ROOT = Path("/data")
|
| 16 |
MODEL_ROOT = Path("/models")
|
|
|
|
| 17 |
RAW_ROOT = DATA_ROOT / "raw"
|
| 18 |
RICCIO_ROOT = RAW_ROOT / "riccio"
|
| 19 |
CUSTOM_UNKNOWN_ROOT = RAW_ROOT / "custom_unknown"
|
|
@@ -25,6 +30,14 @@ RICCIO_VIDEO_COLLECTIONS = (
|
|
| 25 |
"final_kaggle_with_additional_video",
|
| 26 |
"my_test_video_1",
|
| 27 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
|
| 29 |
image = (
|
| 30 |
modal.Image.debian_slim(python_version="3.10")
|
|
@@ -40,11 +53,25 @@ image = (
|
|
| 40 |
"torch==2.11.0",
|
| 41 |
)
|
| 42 |
.add_local_dir("src", "/root/src", copy=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
)
|
| 44 |
|
| 45 |
app = modal.App(APP_NAME, image=image)
|
| 46 |
-
data_volume = modal.Volume.from_name(
|
| 47 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
|
| 49 |
|
| 50 |
def _write_json(path: Path, payload: Any) -> None:
|
|
@@ -70,17 +97,25 @@ def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
| 70 |
]
|
| 71 |
|
| 72 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
def _video_files(root: Path) -> list[Path]:
|
| 74 |
if not root.exists():
|
| 75 |
return []
|
| 76 |
-
return sorted(
|
|
|
|
|
|
|
| 77 |
|
| 78 |
|
| 79 |
def _riccio_video_files(root: Path) -> list[Path]:
|
| 80 |
files: list[Path] = []
|
| 81 |
dataset_roots = [
|
| 82 |
root / "Real-Time Exercise Recognition Dataset",
|
| 83 |
-
root
|
|
|
|
|
|
|
| 84 |
root,
|
| 85 |
]
|
| 86 |
for collection_name in RICCIO_VIDEO_COLLECTIONS:
|
|
@@ -88,7 +123,9 @@ def _riccio_video_files(root: Path) -> list[Path]:
|
|
| 88 |
collection_root = dataset_root / collection_name
|
| 89 |
if not collection_root.exists():
|
| 90 |
continue
|
| 91 |
-
for class_dir in sorted(
|
|
|
|
|
|
|
| 92 |
files.extend(_video_files(class_dir))
|
| 93 |
if files:
|
| 94 |
return sorted(files)
|
|
@@ -124,17 +161,27 @@ def _label_from_path(path: Path) -> str:
|
|
| 124 |
label = normalize_router_label(part)
|
| 125 |
if label != "unknown":
|
| 126 |
return label
|
| 127 |
-
if any(
|
|
|
|
|
|
|
|
|
|
| 128 |
return "unknown"
|
| 129 |
return "unknown"
|
| 130 |
|
| 131 |
|
| 132 |
def _example_id(path: Path) -> str:
|
| 133 |
safe = "_".join(path.relative_to(RAW_ROOT).parts)
|
| 134 |
-
return "".join(
|
|
|
|
|
|
|
| 135 |
|
| 136 |
|
| 137 |
-
@app.function(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
def ingest() -> dict[str, Any]:
|
| 139 |
from huggingface_hub import snapshot_download
|
| 140 |
|
|
@@ -148,13 +195,22 @@ def ingest() -> dict[str, Any]:
|
|
| 148 |
extracted_archives = _extract_archives(RICCIO_ROOT)
|
| 149 |
|
| 150 |
examples: list[dict[str, Any]] = []
|
| 151 |
-
for video_path in [
|
| 152 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
examples.append(
|
| 154 |
{
|
| 155 |
"id": _example_id(video_path),
|
| 156 |
"video_path": str(video_path),
|
| 157 |
-
"label":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
"source": source,
|
| 159 |
}
|
| 160 |
)
|
|
@@ -186,7 +242,13 @@ def extract_example_features(example: dict[str, Any]) -> dict[str, Any]:
|
|
| 186 |
|
| 187 |
import numpy as np
|
| 188 |
|
| 189 |
-
from pozify.ml.exercise_router_features import
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 190 |
from pozify.steps import pose_cleaning, pose_landmarker, video_qc
|
| 191 |
|
| 192 |
video_path = Path(example["video_path"])
|
|
@@ -203,6 +265,10 @@ def extract_example_features(example: dict[str, Any]) -> dict[str, Any]:
|
|
| 203 |
vectors=np.stack([window.vector for window in windows]),
|
| 204 |
tensors=np.stack([window.tensor for window in windows]),
|
| 205 |
label=example["label"],
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
)
|
| 207 |
else:
|
| 208 |
np.savez_compressed(
|
|
@@ -210,6 +276,10 @@ def extract_example_features(example: dict[str, Any]) -> dict[str, Any]:
|
|
| 210 |
vectors=np.empty((0, 0), dtype=np.float32),
|
| 211 |
tensors=np.empty((0, 0, 0), dtype=np.float32),
|
| 212 |
label=example["label"],
|
|
|
|
|
|
|
|
|
|
|
|
|
| 213 |
)
|
| 214 |
|
| 215 |
data_volume.commit()
|
|
@@ -220,6 +290,8 @@ def extract_example_features(example: dict[str, Any]) -> dict[str, Any]:
|
|
| 220 |
"feature_path": str(feature_path),
|
| 221 |
"window_count": len(windows),
|
| 222 |
"pose_valid_ratio": sequence.pose_valid_ratio,
|
|
|
|
|
|
|
| 223 |
}
|
| 224 |
|
| 225 |
|
|
@@ -232,15 +304,26 @@ def write_feature_manifest(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
|
| 232 |
"example_count": len(rows),
|
| 233 |
"window_count": sum(int(row.get("window_count", 0)) for row in rows),
|
| 234 |
"failed_count": sum(1 for row in rows if not row.get("ok", True)),
|
|
|
|
|
|
|
| 235 |
},
|
| 236 |
)
|
| 237 |
data_volume.commit()
|
| 238 |
-
return {
|
|
|
|
|
|
|
|
|
|
| 239 |
|
| 240 |
|
| 241 |
def _load_feature_arrays() -> tuple[Any, Any, Any]:
|
| 242 |
import numpy as np
|
| 243 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 244 |
vectors: list[np.ndarray] = []
|
| 245 |
tensors: list[np.ndarray] = []
|
| 246 |
labels: list[str] = []
|
|
@@ -248,6 +331,15 @@ def _load_feature_arrays() -> tuple[Any, Any, Any]:
|
|
| 248 |
if not row.get("ok", True) or int(row.get("window_count", 0)) <= 0:
|
| 249 |
continue
|
| 250 |
data = np.load(row["feature_path"], allow_pickle=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
row_vectors = data["vectors"]
|
| 252 |
row_tensors = data["tensors"]
|
| 253 |
label = str(data["label"])
|
|
@@ -260,7 +352,10 @@ def _load_feature_arrays() -> tuple[Any, Any, Any]:
|
|
| 260 |
return np.stack(vectors), np.stack(tensors), labels
|
| 261 |
|
| 262 |
|
| 263 |
-
@app.function(
|
|
|
|
|
|
|
|
|
|
| 264 |
def train_baseline() -> dict[str, Any]:
|
| 265 |
import joblib
|
| 266 |
from sklearn.ensemble import HistGradientBoostingClassifier
|
|
@@ -270,11 +365,18 @@ def train_baseline() -> dict[str, Any]:
|
|
| 270 |
from sklearn.preprocessing import StandardScaler
|
| 271 |
|
| 272 |
sys.path.insert(0, "/root/src")
|
| 273 |
-
from pozify.ml.exercise_router_features import
|
|
|
|
|
|
|
|
|
|
|
|
|
| 274 |
|
| 275 |
vectors, _tensors, labels = _load_feature_arrays()
|
| 276 |
if vectors is None or len(set(labels)) < 2:
|
| 277 |
-
result = {
|
|
|
|
|
|
|
|
|
|
| 278 |
_write_json(MODEL_ROOT / "baseline_metrics.json", result)
|
| 279 |
model_volume.commit()
|
| 280 |
return result
|
|
@@ -316,6 +418,9 @@ def train_baseline() -> dict[str, Any]:
|
|
| 316 |
"labels": [str(label) for label in model.classes_],
|
| 317 |
"model_kind": "baseline",
|
| 318 |
"feature_names": window_vector_feature_names(),
|
|
|
|
|
|
|
|
|
|
| 319 |
"metrics": metrics,
|
| 320 |
},
|
| 321 |
MODEL_ROOT / "baseline.joblib",
|
|
@@ -325,7 +430,11 @@ def train_baseline() -> dict[str, Any]:
|
|
| 325 |
return metrics
|
| 326 |
|
| 327 |
|
| 328 |
-
@app.function(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 329 |
def train_temporal(epochs: int = 73) -> dict[str, Any]:
|
| 330 |
import torch
|
| 331 |
from torch import nn
|
|
@@ -333,7 +442,12 @@ def train_temporal(epochs: int = 73) -> dict[str, Any]:
|
|
| 333 |
from sklearn.model_selection import train_test_split
|
| 334 |
|
| 335 |
sys.path.insert(0, "/root/src")
|
| 336 |
-
from pozify.ml.exercise_router_features import
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
from pozify.ml.exercise_router_evaluation import (
|
| 338 |
evaluation_to_dict,
|
| 339 |
evaluate_router_predictions,
|
|
@@ -351,13 +465,18 @@ def train_temporal(epochs: int = 73) -> dict[str, Any]:
|
|
| 351 |
|
| 352 |
_vectors, tensors, labels = _load_feature_arrays()
|
| 353 |
if tensors is None or len(set(labels)) < 2:
|
| 354 |
-
result = {
|
|
|
|
|
|
|
|
|
|
| 355 |
_write_json(MODEL_ROOT / "temporal_metrics.json", result)
|
| 356 |
model_volume.commit()
|
| 357 |
return result
|
| 358 |
|
| 359 |
label_to_index = {label: index for index, label in enumerate(ROUTER_LABELS)}
|
| 360 |
-
label_indices = [
|
|
|
|
|
|
|
| 361 |
label_counts = {label: labels.count(label) for label in sorted(set(labels))}
|
| 362 |
stratify = labels if min(label_counts.values()) >= 2 else None
|
| 363 |
train_x, valid_x, train_y, valid_y, _train_labels, valid_labels = train_test_split(
|
|
@@ -373,7 +492,9 @@ def train_temporal(epochs: int = 73) -> dict[str, Any]:
|
|
| 373 |
torch.tensor(train_y, dtype=torch.long),
|
| 374 |
)
|
| 375 |
generator = torch.Generator().manual_seed(42)
|
| 376 |
-
loader = DataLoader(
|
|
|
|
|
|
|
| 377 |
|
| 378 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 379 |
torch.manual_seed(42)
|
|
@@ -388,7 +509,9 @@ def train_temporal(epochs: int = 73) -> dict[str, Any]:
|
|
| 388 |
dropout_rate=TEMPORAL_DROPOUT_RATE,
|
| 389 |
)
|
| 390 |
model = build_temporal_router_model(config).to(device)
|
| 391 |
-
optimizer = torch.optim.AdamW(
|
|
|
|
|
|
|
| 392 |
loss_fn = nn.CrossEntropyLoss()
|
| 393 |
model.train()
|
| 394 |
last_loss = 0.0
|
|
@@ -408,7 +531,9 @@ def train_temporal(epochs: int = 73) -> dict[str, Any]:
|
|
| 408 |
labels=ROUTER_LABELS,
|
| 409 |
device=device,
|
| 410 |
)
|
| 411 |
-
validation = evaluation_to_dict(
|
|
|
|
|
|
|
| 412 |
|
| 413 |
torch.save(
|
| 414 |
{
|
|
@@ -417,6 +542,9 @@ def train_temporal(epochs: int = 73) -> dict[str, Any]:
|
|
| 417 |
"state_dict": model.cpu().state_dict(),
|
| 418 |
"labels": list(ROUTER_LABELS),
|
| 419 |
"input_size": int(tensors.shape[-1]),
|
|
|
|
|
|
|
|
|
|
| 420 |
"hidden_size": TEMPORAL_HIDDEN_SIZE,
|
| 421 |
"num_layers": TEMPORAL_NUM_LAYERS,
|
| 422 |
"bidirectional": True,
|
|
@@ -471,7 +599,10 @@ def _evaluate_temporal_checkpoint(checkpoint_path: Path, tensors: Any) -> list[s
|
|
| 471 |
import torch
|
| 472 |
|
| 473 |
sys.path.insert(0, "/root/src")
|
| 474 |
-
from pozify.ml.exercise_router_temporal import
|
|
|
|
|
|
|
|
|
|
| 475 |
|
| 476 |
checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False)
|
| 477 |
labels = tuple(str(label) for label in checkpoint["labels"])
|
|
@@ -519,7 +650,9 @@ def evaluate() -> dict[str, Any]:
|
|
| 519 |
if baseline_path.exists():
|
| 520 |
artifact = joblib.load(baseline_path)
|
| 521 |
predictions = list(artifact["model"].predict(vectors))
|
| 522 |
-
baseline_evaluation = evaluation_to_dict(
|
|
|
|
|
|
|
| 523 |
model_results["baseline"] = {
|
| 524 |
"ok": True,
|
| 525 |
"artifact": "baseline.joblib",
|
|
@@ -536,7 +669,9 @@ def evaluate() -> dict[str, Any]:
|
|
| 536 |
|
| 537 |
if temporal_path.exists():
|
| 538 |
predictions = _evaluate_temporal_checkpoint(temporal_path, tensors)
|
| 539 |
-
temporal_evaluation = evaluation_to_dict(
|
|
|
|
|
|
|
| 540 |
model_results["temporal"] = {
|
| 541 |
"ok": True,
|
| 542 |
"artifact": "temporal.pt",
|
|
@@ -552,7 +687,10 @@ def evaluate() -> dict[str, Any]:
|
|
| 552 |
)
|
| 553 |
|
| 554 |
if not candidates:
|
| 555 |
-
result = {
|
|
|
|
|
|
|
|
|
|
| 556 |
_write_json(MODEL_ROOT / "evaluation.json", result)
|
| 557 |
model_volume.commit()
|
| 558 |
return result
|
|
@@ -571,15 +709,114 @@ def evaluate() -> dict[str, Any]:
|
|
| 571 |
"selected_model": selection["selected_model"],
|
| 572 |
"selected_artifact": selection["selected_artifact"],
|
| 573 |
"models": model_results,
|
| 574 |
-
**{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 575 |
}
|
| 576 |
_write_json(MODEL_ROOT / "evaluation.json", result)
|
| 577 |
model_volume.commit()
|
| 578 |
return result
|
| 579 |
|
| 580 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 581 |
@app.local_entrypoint()
|
| 582 |
-
def main(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 583 |
if stage == "ingest":
|
| 584 |
print(ingest.remote())
|
| 585 |
elif stage == "features":
|
|
@@ -597,17 +834,25 @@ def main(stage: str = "evaluate", limit: int | None = None, epochs: int = 73) ->
|
|
| 597 |
print(train_temporal.remote(epochs=epochs))
|
| 598 |
elif stage == "evaluate":
|
| 599 |
print(evaluate.remote())
|
|
|
|
|
|
|
|
|
|
| 600 |
elif stage == "all":
|
| 601 |
print(ingest.remote())
|
| 602 |
examples = load_feature_examples.remote(limit)
|
| 603 |
rows = []
|
| 604 |
for result in extract_example_features.map(examples, return_exceptions=True):
|
| 605 |
-
rows.append(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 606 |
print(write_feature_manifest.remote(rows))
|
| 607 |
print(train_baseline.remote())
|
| 608 |
print(train_temporal.remote(epochs=epochs))
|
| 609 |
print(evaluate.remote())
|
|
|
|
| 610 |
else:
|
| 611 |
raise ValueError(
|
| 612 |
-
"stage must be one of: ingest, features, train-baseline, train-temporal, evaluate, all"
|
| 613 |
)
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import json
|
| 4 |
+
import os
|
| 5 |
from pathlib import Path
|
| 6 |
import shutil
|
| 7 |
import sys
|
|
|
|
| 13 |
|
| 14 |
APP_NAME = "pozify-exercise-router"
|
| 15 |
DATASET_ID = "RickyRiccio/Real_Time_Exercise_Recognition_Dataset"
|
| 16 |
+
DEFAULT_HF_REPO_ID = "build-small-hackathon/pozify-exercise-router"
|
| 17 |
+
HF_REPO_ID_ENV = "POZIFY_ROUTER_HF_REPO_ID"
|
| 18 |
+
HF_PRIVATE_ENV = "POZIFY_ROUTER_HF_PRIVATE"
|
| 19 |
DATA_ROOT = Path("/data")
|
| 20 |
MODEL_ROOT = Path("/models")
|
| 21 |
+
DOCS_ROOT = Path("/root/docs")
|
| 22 |
RAW_ROOT = DATA_ROOT / "raw"
|
| 23 |
RICCIO_ROOT = RAW_ROOT / "riccio"
|
| 24 |
CUSTOM_UNKNOWN_ROOT = RAW_ROOT / "custom_unknown"
|
|
|
|
| 30 |
"final_kaggle_with_additional_video",
|
| 31 |
"my_test_video_1",
|
| 32 |
)
|
| 33 |
+
HF_ARTIFACT_FILENAMES = (
|
| 34 |
+
"router.joblib",
|
| 35 |
+
"router_selection.json",
|
| 36 |
+
"temporal.pt",
|
| 37 |
+
"evaluation.json",
|
| 38 |
+
"baseline_metrics.json",
|
| 39 |
+
"temporal_metrics.json",
|
| 40 |
+
)
|
| 41 |
|
| 42 |
image = (
|
| 43 |
modal.Image.debian_slim(python_version="3.10")
|
|
|
|
| 53 |
"torch==2.11.0",
|
| 54 |
)
|
| 55 |
.add_local_dir("src", "/root/src", copy=True)
|
| 56 |
+
.add_local_file(
|
| 57 |
+
"docs/huggingface-router-model-card.md",
|
| 58 |
+
"/root/docs/huggingface-router-model-card.md",
|
| 59 |
+
copy=True,
|
| 60 |
+
)
|
| 61 |
+
.add_local_file(
|
| 62 |
+
"docs/exercise-router-training-report.md",
|
| 63 |
+
"/root/docs/exercise-router-training-report.md",
|
| 64 |
+
copy=True,
|
| 65 |
+
)
|
| 66 |
)
|
| 67 |
|
| 68 |
app = modal.App(APP_NAME, image=image)
|
| 69 |
+
data_volume = modal.Volume.from_name(
|
| 70 |
+
"pozify-router-data", create_if_missing=True, version=2
|
| 71 |
+
)
|
| 72 |
+
model_volume = modal.Volume.from_name(
|
| 73 |
+
"pozify-router-models", create_if_missing=True, version=2
|
| 74 |
+
)
|
| 75 |
|
| 76 |
|
| 77 |
def _write_json(path: Path, payload: Any) -> None:
|
|
|
|
| 97 |
]
|
| 98 |
|
| 99 |
|
| 100 |
+
def _env_truthy(value: str | None) -> bool:
|
| 101 |
+
return value is not None and value.strip().lower() in {"1", "true", "yes", "on"}
|
| 102 |
+
|
| 103 |
+
|
| 104 |
def _video_files(root: Path) -> list[Path]:
|
| 105 |
if not root.exists():
|
| 106 |
return []
|
| 107 |
+
return sorted(
|
| 108 |
+
path for path in root.rglob("*") if path.suffix.lower() in VIDEO_SUFFIXES
|
| 109 |
+
)
|
| 110 |
|
| 111 |
|
| 112 |
def _riccio_video_files(root: Path) -> list[Path]:
|
| 113 |
files: list[Path] = []
|
| 114 |
dataset_roots = [
|
| 115 |
root / "Real-Time Exercise Recognition Dataset",
|
| 116 |
+
root
|
| 117 |
+
/ "Real-Time Exercise Recognition Dataset"
|
| 118 |
+
/ "Real-Time Exercise Recognition Dataset",
|
| 119 |
root,
|
| 120 |
]
|
| 121 |
for collection_name in RICCIO_VIDEO_COLLECTIONS:
|
|
|
|
| 123 |
collection_root = dataset_root / collection_name
|
| 124 |
if not collection_root.exists():
|
| 125 |
continue
|
| 126 |
+
for class_dir in sorted(
|
| 127 |
+
path for path in collection_root.iterdir() if path.is_dir()
|
| 128 |
+
):
|
| 129 |
files.extend(_video_files(class_dir))
|
| 130 |
if files:
|
| 131 |
return sorted(files)
|
|
|
|
| 161 |
label = normalize_router_label(part)
|
| 162 |
if label != "unknown":
|
| 163 |
return label
|
| 164 |
+
if any(
|
| 165 |
+
token in part.lower()
|
| 166 |
+
for token in ("unknown", "curl", "idle", "walk", "stretch")
|
| 167 |
+
):
|
| 168 |
return "unknown"
|
| 169 |
return "unknown"
|
| 170 |
|
| 171 |
|
| 172 |
def _example_id(path: Path) -> str:
|
| 173 |
safe = "_".join(path.relative_to(RAW_ROOT).parts)
|
| 174 |
+
return "".join(
|
| 175 |
+
char if char.isalnum() or char in {"_", "-", "."} else "_" for char in safe
|
| 176 |
+
)
|
| 177 |
|
| 178 |
|
| 179 |
+
@app.function(
|
| 180 |
+
volumes={str(DATA_ROOT): data_volume},
|
| 181 |
+
timeout=60 * 60,
|
| 182 |
+
secrets=[modal.Secret.from_name("huggingface-secret")],
|
| 183 |
+
gpu="any",
|
| 184 |
+
)
|
| 185 |
def ingest() -> dict[str, Any]:
|
| 186 |
from huggingface_hub import snapshot_download
|
| 187 |
|
|
|
|
| 195 |
extracted_archives = _extract_archives(RICCIO_ROOT)
|
| 196 |
|
| 197 |
examples: list[dict[str, Any]] = []
|
| 198 |
+
for video_path in [
|
| 199 |
+
*_riccio_video_files(RICCIO_ROOT),
|
| 200 |
+
*_video_files(CUSTOM_UNKNOWN_ROOT),
|
| 201 |
+
]:
|
| 202 |
+
source = (
|
| 203 |
+
"custom_unknown" if CUSTOM_UNKNOWN_ROOT in video_path.parents else "riccio"
|
| 204 |
+
)
|
| 205 |
examples.append(
|
| 206 |
{
|
| 207 |
"id": _example_id(video_path),
|
| 208 |
"video_path": str(video_path),
|
| 209 |
+
"label": (
|
| 210 |
+
"unknown"
|
| 211 |
+
if source == "custom_unknown"
|
| 212 |
+
else _label_from_path(video_path)
|
| 213 |
+
),
|
| 214 |
"source": source,
|
| 215 |
}
|
| 216 |
)
|
|
|
|
| 242 |
|
| 243 |
import numpy as np
|
| 244 |
|
| 245 |
+
from pozify.ml.exercise_router_features import (
|
| 246 |
+
FEATURE_SCHEMA,
|
| 247 |
+
ROUTER_LANDMARK_SCHEMA,
|
| 248 |
+
extract_router_windows,
|
| 249 |
+
window_tensor_feature_names,
|
| 250 |
+
window_vector_feature_names,
|
| 251 |
+
)
|
| 252 |
from pozify.steps import pose_cleaning, pose_landmarker, video_qc
|
| 253 |
|
| 254 |
video_path = Path(example["video_path"])
|
|
|
|
| 265 |
vectors=np.stack([window.vector for window in windows]),
|
| 266 |
tensors=np.stack([window.tensor for window in windows]),
|
| 267 |
label=example["label"],
|
| 268 |
+
feature_schema=FEATURE_SCHEMA,
|
| 269 |
+
landmark_schema=ROUTER_LANDMARK_SCHEMA,
|
| 270 |
+
vector_feature_names=np.asarray(window_vector_feature_names()),
|
| 271 |
+
tensor_feature_names=np.asarray(window_tensor_feature_names()),
|
| 272 |
)
|
| 273 |
else:
|
| 274 |
np.savez_compressed(
|
|
|
|
| 276 |
vectors=np.empty((0, 0), dtype=np.float32),
|
| 277 |
tensors=np.empty((0, 0, 0), dtype=np.float32),
|
| 278 |
label=example["label"],
|
| 279 |
+
feature_schema=FEATURE_SCHEMA,
|
| 280 |
+
landmark_schema=ROUTER_LANDMARK_SCHEMA,
|
| 281 |
+
vector_feature_names=np.asarray(window_vector_feature_names()),
|
| 282 |
+
tensor_feature_names=np.asarray(window_tensor_feature_names()),
|
| 283 |
)
|
| 284 |
|
| 285 |
data_volume.commit()
|
|
|
|
| 290 |
"feature_path": str(feature_path),
|
| 291 |
"window_count": len(windows),
|
| 292 |
"pose_valid_ratio": sequence.pose_valid_ratio,
|
| 293 |
+
"feature_schema": FEATURE_SCHEMA,
|
| 294 |
+
"landmark_schema": ROUTER_LANDMARK_SCHEMA,
|
| 295 |
}
|
| 296 |
|
| 297 |
|
|
|
|
| 304 |
"example_count": len(rows),
|
| 305 |
"window_count": sum(int(row.get("window_count", 0)) for row in rows),
|
| 306 |
"failed_count": sum(1 for row in rows if not row.get("ok", True)),
|
| 307 |
+
"feature_schema": rows[0].get("feature_schema") if rows else None,
|
| 308 |
+
"landmark_schema": rows[0].get("landmark_schema") if rows else None,
|
| 309 |
},
|
| 310 |
)
|
| 311 |
data_volume.commit()
|
| 312 |
+
return {
|
| 313 |
+
"feature_manifest_path": str(FEATURE_MANIFEST_PATH),
|
| 314 |
+
"example_count": len(rows),
|
| 315 |
+
}
|
| 316 |
|
| 317 |
|
| 318 |
def _load_feature_arrays() -> tuple[Any, Any, Any]:
|
| 319 |
import numpy as np
|
| 320 |
|
| 321 |
+
sys.path.insert(0, "/root/src")
|
| 322 |
+
from pozify.ml.exercise_router_features import (
|
| 323 |
+
FEATURE_SCHEMA,
|
| 324 |
+
ROUTER_LANDMARK_SCHEMA,
|
| 325 |
+
)
|
| 326 |
+
|
| 327 |
vectors: list[np.ndarray] = []
|
| 328 |
tensors: list[np.ndarray] = []
|
| 329 |
labels: list[str] = []
|
|
|
|
| 331 |
if not row.get("ok", True) or int(row.get("window_count", 0)) <= 0:
|
| 332 |
continue
|
| 333 |
data = np.load(row["feature_path"], allow_pickle=False)
|
| 334 |
+
feature_schema = str(data["feature_schema"]) if "feature_schema" in data else ""
|
| 335 |
+
landmark_schema = (
|
| 336 |
+
str(data["landmark_schema"]) if "landmark_schema" in data else ""
|
| 337 |
+
)
|
| 338 |
+
if (
|
| 339 |
+
feature_schema != FEATURE_SCHEMA
|
| 340 |
+
or landmark_schema != ROUTER_LANDMARK_SCHEMA
|
| 341 |
+
):
|
| 342 |
+
continue
|
| 343 |
row_vectors = data["vectors"]
|
| 344 |
row_tensors = data["tensors"]
|
| 345 |
label = str(data["label"])
|
|
|
|
| 352 |
return np.stack(vectors), np.stack(tensors), labels
|
| 353 |
|
| 354 |
|
| 355 |
+
@app.function(
|
| 356 |
+
volumes={str(DATA_ROOT): data_volume, str(MODEL_ROOT): model_volume},
|
| 357 |
+
timeout=30 * 60,
|
| 358 |
+
)
|
| 359 |
def train_baseline() -> dict[str, Any]:
|
| 360 |
import joblib
|
| 361 |
from sklearn.ensemble import HistGradientBoostingClassifier
|
|
|
|
| 365 |
from sklearn.preprocessing import StandardScaler
|
| 366 |
|
| 367 |
sys.path.insert(0, "/root/src")
|
| 368 |
+
from pozify.ml.exercise_router_features import (
|
| 369 |
+
FEATURE_SCHEMA,
|
| 370 |
+
ROUTER_LANDMARK_SCHEMA,
|
| 371 |
+
window_vector_feature_names,
|
| 372 |
+
)
|
| 373 |
|
| 374 |
vectors, _tensors, labels = _load_feature_arrays()
|
| 375 |
if vectors is None or len(set(labels)) < 2:
|
| 376 |
+
result = {
|
| 377 |
+
"ok": False,
|
| 378 |
+
"error": "At least two labels with extracted windows are required",
|
| 379 |
+
}
|
| 380 |
_write_json(MODEL_ROOT / "baseline_metrics.json", result)
|
| 381 |
model_volume.commit()
|
| 382 |
return result
|
|
|
|
| 418 |
"labels": [str(label) for label in model.classes_],
|
| 419 |
"model_kind": "baseline",
|
| 420 |
"feature_names": window_vector_feature_names(),
|
| 421 |
+
"feature_schema": FEATURE_SCHEMA,
|
| 422 |
+
"landmark_schema": ROUTER_LANDMARK_SCHEMA,
|
| 423 |
+
"input_size": int(vectors.shape[-1]),
|
| 424 |
"metrics": metrics,
|
| 425 |
},
|
| 426 |
MODEL_ROOT / "baseline.joblib",
|
|
|
|
| 430 |
return metrics
|
| 431 |
|
| 432 |
|
| 433 |
+
@app.function(
|
| 434 |
+
gpu="A10",
|
| 435 |
+
volumes={str(DATA_ROOT): data_volume, str(MODEL_ROOT): model_volume},
|
| 436 |
+
timeout=60 * 60,
|
| 437 |
+
)
|
| 438 |
def train_temporal(epochs: int = 73) -> dict[str, Any]:
|
| 439 |
import torch
|
| 440 |
from torch import nn
|
|
|
|
| 442 |
from sklearn.model_selection import train_test_split
|
| 443 |
|
| 444 |
sys.path.insert(0, "/root/src")
|
| 445 |
+
from pozify.ml.exercise_router_features import (
|
| 446 |
+
FEATURE_SCHEMA,
|
| 447 |
+
ROUTER_LABELS,
|
| 448 |
+
ROUTER_LANDMARK_SCHEMA,
|
| 449 |
+
window_tensor_feature_names,
|
| 450 |
+
)
|
| 451 |
from pozify.ml.exercise_router_evaluation import (
|
| 452 |
evaluation_to_dict,
|
| 453 |
evaluate_router_predictions,
|
|
|
|
| 465 |
|
| 466 |
_vectors, tensors, labels = _load_feature_arrays()
|
| 467 |
if tensors is None or len(set(labels)) < 2:
|
| 468 |
+
result = {
|
| 469 |
+
"ok": False,
|
| 470 |
+
"error": "At least two labels with extracted windows are required",
|
| 471 |
+
}
|
| 472 |
_write_json(MODEL_ROOT / "temporal_metrics.json", result)
|
| 473 |
model_volume.commit()
|
| 474 |
return result
|
| 475 |
|
| 476 |
label_to_index = {label: index for index, label in enumerate(ROUTER_LABELS)}
|
| 477 |
+
label_indices = [
|
| 478 |
+
label_to_index.get(label, label_to_index["unknown"]) for label in labels
|
| 479 |
+
]
|
| 480 |
label_counts = {label: labels.count(label) for label in sorted(set(labels))}
|
| 481 |
stratify = labels if min(label_counts.values()) >= 2 else None
|
| 482 |
train_x, valid_x, train_y, valid_y, _train_labels, valid_labels = train_test_split(
|
|
|
|
| 492 |
torch.tensor(train_y, dtype=torch.long),
|
| 493 |
)
|
| 494 |
generator = torch.Generator().manual_seed(42)
|
| 495 |
+
loader = DataLoader(
|
| 496 |
+
train_dataset, batch_size=TEMPORAL_BATCH_SIZE, shuffle=True, generator=generator
|
| 497 |
+
)
|
| 498 |
|
| 499 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 500 |
torch.manual_seed(42)
|
|
|
|
| 509 |
dropout_rate=TEMPORAL_DROPOUT_RATE,
|
| 510 |
)
|
| 511 |
model = build_temporal_router_model(config).to(device)
|
| 512 |
+
optimizer = torch.optim.AdamW(
|
| 513 |
+
model.parameters(), lr=TEMPORAL_LEARNING_RATE, weight_decay=1e-4
|
| 514 |
+
)
|
| 515 |
loss_fn = nn.CrossEntropyLoss()
|
| 516 |
model.train()
|
| 517 |
last_loss = 0.0
|
|
|
|
| 531 |
labels=ROUTER_LABELS,
|
| 532 |
device=device,
|
| 533 |
)
|
| 534 |
+
validation = evaluation_to_dict(
|
| 535 |
+
evaluate_router_predictions(list(valid_labels), valid_predictions)
|
| 536 |
+
)
|
| 537 |
|
| 538 |
torch.save(
|
| 539 |
{
|
|
|
|
| 542 |
"state_dict": model.cpu().state_dict(),
|
| 543 |
"labels": list(ROUTER_LABELS),
|
| 544 |
"input_size": int(tensors.shape[-1]),
|
| 545 |
+
"feature_names": window_tensor_feature_names(),
|
| 546 |
+
"feature_schema": FEATURE_SCHEMA,
|
| 547 |
+
"landmark_schema": ROUTER_LANDMARK_SCHEMA,
|
| 548 |
"hidden_size": TEMPORAL_HIDDEN_SIZE,
|
| 549 |
"num_layers": TEMPORAL_NUM_LAYERS,
|
| 550 |
"bidirectional": True,
|
|
|
|
| 599 |
import torch
|
| 600 |
|
| 601 |
sys.path.insert(0, "/root/src")
|
| 602 |
+
from pozify.ml.exercise_router_temporal import (
|
| 603 |
+
TemporalRouterConfig,
|
| 604 |
+
build_temporal_router_model,
|
| 605 |
+
)
|
| 606 |
|
| 607 |
checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False)
|
| 608 |
labels = tuple(str(label) for label in checkpoint["labels"])
|
|
|
|
| 650 |
if baseline_path.exists():
|
| 651 |
artifact = joblib.load(baseline_path)
|
| 652 |
predictions = list(artifact["model"].predict(vectors))
|
| 653 |
+
baseline_evaluation = evaluation_to_dict(
|
| 654 |
+
evaluate_router_predictions(labels, predictions)
|
| 655 |
+
)
|
| 656 |
model_results["baseline"] = {
|
| 657 |
"ok": True,
|
| 658 |
"artifact": "baseline.joblib",
|
|
|
|
| 669 |
|
| 670 |
if temporal_path.exists():
|
| 671 |
predictions = _evaluate_temporal_checkpoint(temporal_path, tensors)
|
| 672 |
+
temporal_evaluation = evaluation_to_dict(
|
| 673 |
+
evaluate_router_predictions(labels, predictions)
|
| 674 |
+
)
|
| 675 |
model_results["temporal"] = {
|
| 676 |
"ok": True,
|
| 677 |
"artifact": "temporal.pt",
|
|
|
|
| 687 |
)
|
| 688 |
|
| 689 |
if not candidates:
|
| 690 |
+
result = {
|
| 691 |
+
"ok": False,
|
| 692 |
+
"error": "At least one trained router artifact is required",
|
| 693 |
+
}
|
| 694 |
_write_json(MODEL_ROOT / "evaluation.json", result)
|
| 695 |
model_volume.commit()
|
| 696 |
return result
|
|
|
|
| 709 |
"selected_model": selection["selected_model"],
|
| 710 |
"selected_artifact": selection["selected_artifact"],
|
| 711 |
"models": model_results,
|
| 712 |
+
**{
|
| 713 |
+
key: selected[key]
|
| 714 |
+
for key in (
|
| 715 |
+
"accuracy",
|
| 716 |
+
"precision",
|
| 717 |
+
"recall",
|
| 718 |
+
"unknown_rejection_rate",
|
| 719 |
+
"confusion_matrix",
|
| 720 |
+
)
|
| 721 |
+
},
|
| 722 |
}
|
| 723 |
_write_json(MODEL_ROOT / "evaluation.json", result)
|
| 724 |
model_volume.commit()
|
| 725 |
return result
|
| 726 |
|
| 727 |
|
| 728 |
+
def _upload_hf_file(
|
| 729 |
+
api: Any,
|
| 730 |
+
*,
|
| 731 |
+
repo_id: str,
|
| 732 |
+
local_path: Path,
|
| 733 |
+
path_in_repo: str,
|
| 734 |
+
) -> dict[str, Any]:
|
| 735 |
+
if not local_path.exists():
|
| 736 |
+
return {
|
| 737 |
+
"path": str(local_path),
|
| 738 |
+
"path_in_repo": path_in_repo,
|
| 739 |
+
"uploaded": False,
|
| 740 |
+
"reason": "missing",
|
| 741 |
+
}
|
| 742 |
+
api.upload_file(
|
| 743 |
+
repo_id=repo_id,
|
| 744 |
+
repo_type="model",
|
| 745 |
+
path_or_fileobj=str(local_path),
|
| 746 |
+
path_in_repo=path_in_repo,
|
| 747 |
+
)
|
| 748 |
+
return {
|
| 749 |
+
"path": str(local_path),
|
| 750 |
+
"path_in_repo": path_in_repo,
|
| 751 |
+
"uploaded": True,
|
| 752 |
+
}
|
| 753 |
+
|
| 754 |
+
|
| 755 |
+
@app.function(
|
| 756 |
+
volumes={str(MODEL_ROOT): model_volume},
|
| 757 |
+
secrets=[modal.Secret.from_name("huggingface-secret")],
|
| 758 |
+
timeout=20 * 60,
|
| 759 |
+
)
|
| 760 |
+
def publish_to_hf(
|
| 761 |
+
repo_id: str | None = None,
|
| 762 |
+
private: bool | None = None,
|
| 763 |
+
) -> dict[str, Any]:
|
| 764 |
+
from huggingface_hub import HfApi
|
| 765 |
+
|
| 766 |
+
repo_id = repo_id or os.getenv(HF_REPO_ID_ENV) or DEFAULT_HF_REPO_ID
|
| 767 |
+
private = _env_truthy(os.getenv(HF_PRIVATE_ENV)) if private is None else private
|
| 768 |
+
if not os.getenv("HF_TOKEN"):
|
| 769 |
+
return {
|
| 770 |
+
"ok": False,
|
| 771 |
+
"error": "HF_TOKEN is required in the Modal environment or local .env",
|
| 772 |
+
"repo_id": repo_id,
|
| 773 |
+
}
|
| 774 |
+
|
| 775 |
+
api = HfApi()
|
| 776 |
+
api.create_repo(repo_id=repo_id, repo_type="model", private=private, exist_ok=True)
|
| 777 |
+
uploads = [
|
| 778 |
+
_upload_hf_file(
|
| 779 |
+
api,
|
| 780 |
+
repo_id=repo_id,
|
| 781 |
+
local_path=DOCS_ROOT / "huggingface-router-model-card.md",
|
| 782 |
+
path_in_repo="README.md",
|
| 783 |
+
),
|
| 784 |
+
_upload_hf_file(
|
| 785 |
+
api,
|
| 786 |
+
repo_id=repo_id,
|
| 787 |
+
local_path=DOCS_ROOT / "exercise-router-training-report.md",
|
| 788 |
+
path_in_repo="training_report.md",
|
| 789 |
+
),
|
| 790 |
+
]
|
| 791 |
+
uploads.extend(
|
| 792 |
+
_upload_hf_file(
|
| 793 |
+
api,
|
| 794 |
+
repo_id=repo_id,
|
| 795 |
+
local_path=MODEL_ROOT / filename,
|
| 796 |
+
path_in_repo=filename,
|
| 797 |
+
)
|
| 798 |
+
for filename in HF_ARTIFACT_FILENAMES
|
| 799 |
+
)
|
| 800 |
+
|
| 801 |
+
result = {
|
| 802 |
+
"ok": any(item["uploaded"] for item in uploads),
|
| 803 |
+
"repo_id": repo_id,
|
| 804 |
+
"private": private,
|
| 805 |
+
"uploads": uploads,
|
| 806 |
+
}
|
| 807 |
+
_write_json(MODEL_ROOT / "hf_upload.json", result)
|
| 808 |
+
model_volume.commit()
|
| 809 |
+
return result
|
| 810 |
+
|
| 811 |
+
|
| 812 |
@app.local_entrypoint()
|
| 813 |
+
def main(
|
| 814 |
+
stage: str = "evaluate",
|
| 815 |
+
limit: int | None = None,
|
| 816 |
+
epochs: int = 73,
|
| 817 |
+
repo_id: str | None = None,
|
| 818 |
+
private: bool | None = None,
|
| 819 |
+
) -> None:
|
| 820 |
if stage == "ingest":
|
| 821 |
print(ingest.remote())
|
| 822 |
elif stage == "features":
|
|
|
|
| 834 |
print(train_temporal.remote(epochs=epochs))
|
| 835 |
elif stage == "evaluate":
|
| 836 |
print(evaluate.remote())
|
| 837 |
+
print(publish_to_hf.remote(repo_id=repo_id, private=private))
|
| 838 |
+
elif stage == "publish":
|
| 839 |
+
print(publish_to_hf.remote(repo_id=repo_id, private=private))
|
| 840 |
elif stage == "all":
|
| 841 |
print(ingest.remote())
|
| 842 |
examples = load_feature_examples.remote(limit)
|
| 843 |
rows = []
|
| 844 |
for result in extract_example_features.map(examples, return_exceptions=True):
|
| 845 |
+
rows.append(
|
| 846 |
+
{"ok": False, "error": repr(result)}
|
| 847 |
+
if isinstance(result, Exception)
|
| 848 |
+
else {"ok": True, **result}
|
| 849 |
+
)
|
| 850 |
print(write_feature_manifest.remote(rows))
|
| 851 |
print(train_baseline.remote())
|
| 852 |
print(train_temporal.remote(epochs=epochs))
|
| 853 |
print(evaluate.remote())
|
| 854 |
+
print(publish_to_hf.remote(repo_id=repo_id, private=private))
|
| 855 |
else:
|
| 856 |
raise ValueError(
|
| 857 |
+
"stage must be one of: ingest, features, train-baseline, train-temporal, evaluate, publish, all"
|
| 858 |
)
|
scripts/upload_exercise_router_to_hf.py
CHANGED
|
@@ -13,6 +13,10 @@ ARTIFACT_FILENAMES = (
|
|
| 13 |
"router.joblib",
|
| 14 |
"router_selection.json",
|
| 15 |
"temporal.pt",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
)
|
| 17 |
|
| 18 |
|
|
|
|
| 13 |
"router.joblib",
|
| 14 |
"router_selection.json",
|
| 15 |
"temporal.pt",
|
| 16 |
+
"evaluation.json",
|
| 17 |
+
"baseline_metrics.json",
|
| 18 |
+
"temporal_metrics.json",
|
| 19 |
+
"hf_upload.json",
|
| 20 |
)
|
| 21 |
|
| 22 |
|
src/pozify/exercises/shared/analyzer.py
CHANGED
|
@@ -4,7 +4,7 @@ from statistics import pstdev
|
|
| 4 |
from typing import Any, Callable, Protocol
|
| 5 |
|
| 6 |
from pozify.contracts import PoseFrame
|
| 7 |
-
from pozify.steps.rep_signals import angle_deg, average_axis, landmark_axis
|
| 8 |
|
| 9 |
|
| 10 |
NumberGetter = Callable[[PoseFrame], float | None]
|
|
@@ -68,11 +68,7 @@ def safe_ratio(numerator: float | None, denominator: float | None) -> float | No
|
|
| 68 |
|
| 69 |
|
| 70 |
def width(frame: PoseFrame, left: str, right: str) -> float | None:
|
| 71 |
-
|
| 72 |
-
right_x = landmark_axis(frame, right, "x")
|
| 73 |
-
if left_x is None or right_x is None:
|
| 74 |
-
return None
|
| 75 |
-
return abs(right_x - left_x)
|
| 76 |
|
| 77 |
|
| 78 |
def mean_pair(
|
|
@@ -99,11 +95,16 @@ def side_delta(
|
|
| 99 |
def torso_lean_deg(frame: PoseFrame, side: str) -> float | None:
|
| 100 |
shoulder_x = landmark_axis(frame, f"{side}_shoulder", "x")
|
| 101 |
shoulder_y = landmark_axis(frame, f"{side}_shoulder", "y")
|
|
|
|
| 102 |
hip_x = landmark_axis(frame, f"{side}_hip", "x")
|
| 103 |
hip_y = landmark_axis(frame, f"{side}_hip", "y")
|
| 104 |
-
|
|
|
|
| 105 |
return None
|
| 106 |
-
horizontal_offset =
|
|
|
|
|
|
|
|
|
|
| 107 |
vertical_offset = abs(float(shoulder_y) - float(hip_y))
|
| 108 |
if vertical_offset <= 1e-6:
|
| 109 |
return None
|
|
|
|
| 4 |
from typing import Any, Callable, Protocol
|
| 5 |
|
| 6 |
from pozify.contracts import PoseFrame
|
| 7 |
+
from pozify.steps.rep_signals import angle_deg, average_axis, distance, landmark_axis
|
| 8 |
|
| 9 |
|
| 10 |
NumberGetter = Callable[[PoseFrame], float | None]
|
|
|
|
| 68 |
|
| 69 |
|
| 70 |
def width(frame: PoseFrame, left: str, right: str) -> float | None:
|
| 71 |
+
return distance(frame, left, right)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
|
| 73 |
|
| 74 |
def mean_pair(
|
|
|
|
| 95 |
def torso_lean_deg(frame: PoseFrame, side: str) -> float | None:
|
| 96 |
shoulder_x = landmark_axis(frame, f"{side}_shoulder", "x")
|
| 97 |
shoulder_y = landmark_axis(frame, f"{side}_shoulder", "y")
|
| 98 |
+
shoulder_z = landmark_axis(frame, f"{side}_shoulder", "z")
|
| 99 |
hip_x = landmark_axis(frame, f"{side}_hip", "x")
|
| 100 |
hip_y = landmark_axis(frame, f"{side}_hip", "y")
|
| 101 |
+
hip_z = landmark_axis(frame, f"{side}_hip", "z")
|
| 102 |
+
if None in {shoulder_x, shoulder_y, shoulder_z, hip_x, hip_y, hip_z}:
|
| 103 |
return None
|
| 104 |
+
horizontal_offset = (
|
| 105 |
+
(float(shoulder_x) - float(hip_x)) ** 2
|
| 106 |
+
+ (float(shoulder_z) - float(hip_z)) ** 2
|
| 107 |
+
) ** 0.5
|
| 108 |
vertical_offset = abs(float(shoulder_y) - float(hip_y))
|
| 109 |
if vertical_offset <= 1e-6:
|
| 110 |
return None
|
src/pozify/ml/exercise_router_features.py
CHANGED
|
@@ -6,7 +6,13 @@ import math
|
|
| 6 |
import numpy as np
|
| 7 |
|
| 8 |
from pozify.contracts import PoseFrame, PoseSequence
|
| 9 |
-
from pozify.steps.pose_backends.landmarks import LANDMARK_NAMES
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
|
| 12 |
ROUTER_LABELS = ("squat", "push_up", "shoulder_press", "unknown")
|
|
@@ -77,9 +83,9 @@ def frame_feature_names() -> list[str]:
|
|
| 77 |
for landmark in LANDMARK_NAMES:
|
| 78 |
base_names.extend(
|
| 79 |
[
|
| 80 |
-
f"{landmark}
|
| 81 |
-
f"{landmark}
|
| 82 |
-
f"{landmark}
|
| 83 |
f"{landmark}_visibility",
|
| 84 |
]
|
| 85 |
)
|
|
@@ -123,14 +129,16 @@ def _axis(values: dict[str, float] | None, axis: str) -> float:
|
|
| 123 |
def _visibility(frame: PoseFrame, values: dict[str, float] | None) -> float:
|
| 124 |
if values is not None and "visibility" in values:
|
| 125 |
return max(0.0, min(1.0, float(values["visibility"])))
|
| 126 |
-
return
|
| 127 |
|
| 128 |
|
| 129 |
-
def _point(frame: PoseFrame, name: str) -> tuple[float, float] | None:
|
| 130 |
-
|
| 131 |
-
|
|
|
|
|
|
|
| 132 |
return None
|
| 133 |
-
return
|
| 134 |
|
| 135 |
|
| 136 |
def _distance(frame: PoseFrame, first: str, second: str) -> float | None:
|
|
@@ -138,7 +146,7 @@ def _distance(frame: PoseFrame, first: str, second: str) -> float | None:
|
|
| 138 |
second_point = _point(frame, second)
|
| 139 |
if first_point is None or second_point is None:
|
| 140 |
return None
|
| 141 |
-
return math.
|
| 142 |
|
| 143 |
|
| 144 |
def _safe_ratio(numerator: float | None, denominator: float | None) -> float:
|
|
@@ -156,12 +164,16 @@ def _angle_deg(frame: PoseFrame, first: str, middle: str, last: str) -> float:
|
|
| 156 |
|
| 157 |
abx = first_point[0] - middle_point[0]
|
| 158 |
aby = first_point[1] - middle_point[1]
|
|
|
|
| 159 |
cbx = last_point[0] - middle_point[0]
|
| 160 |
cby = last_point[1] - middle_point[1]
|
| 161 |
-
|
|
|
|
|
|
|
|
|
|
| 162 |
if denom <= 1e-6:
|
| 163 |
return 0.0
|
| 164 |
-
cosine = max(-1.0, min(1.0, (abx * cbx + aby * cby) / denom))
|
| 165 |
return math.degrees(math.acos(cosine))
|
| 166 |
|
| 167 |
|
|
@@ -174,12 +186,13 @@ def _frame_mean_visibility(frame: PoseFrame) -> float:
|
|
| 174 |
def frame_feature_vector(frame: PoseFrame) -> np.ndarray:
|
| 175 |
values: list[float] = []
|
| 176 |
for landmark in LANDMARK_NAMES:
|
| 177 |
-
landmark_values = frame.landmarks.get(landmark)
|
|
|
|
| 178 |
values.extend(
|
| 179 |
[
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
_visibility(frame, landmark_values),
|
| 184 |
]
|
| 185 |
)
|
|
@@ -269,4 +282,3 @@ def extract_router_windows(
|
|
| 269 |
)
|
| 270 |
)
|
| 271 |
return windows
|
| 272 |
-
|
|
|
|
| 6 |
import numpy as np
|
| 7 |
|
| 8 |
from pozify.contracts import PoseFrame, PoseSequence
|
| 9 |
+
from pozify.steps.pose_backends.landmarks import LANDMARK_NAMES, LANDMARK_SCHEMA
|
| 10 |
+
from pozify.steps.rep_signals import landmark_axis
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
FEATURE_SCHEMA = "coco17_3d_v1"
|
| 14 |
+
ROUTER_LANDMARK_SCHEMA = LANDMARK_SCHEMA
|
| 15 |
+
ROUTER_INPUT_SIZE = 3 * (len(LANDMARK_NAMES) * 4 + 8 + 3)
|
| 16 |
|
| 17 |
|
| 18 |
ROUTER_LABELS = ("squat", "push_up", "shoulder_press", "unknown")
|
|
|
|
| 83 |
for landmark in LANDMARK_NAMES:
|
| 84 |
base_names.extend(
|
| 85 |
[
|
| 86 |
+
f"{landmark}_pose3d_x",
|
| 87 |
+
f"{landmark}_pose3d_y",
|
| 88 |
+
f"{landmark}_pose3d_z",
|
| 89 |
f"{landmark}_visibility",
|
| 90 |
]
|
| 91 |
)
|
|
|
|
| 129 |
def _visibility(frame: PoseFrame, values: dict[str, float] | None) -> float:
|
| 130 |
if values is not None and "visibility" in values:
|
| 131 |
return max(0.0, min(1.0, float(values["visibility"])))
|
| 132 |
+
return 0.0
|
| 133 |
|
| 134 |
|
| 135 |
+
def _point(frame: PoseFrame, name: str) -> tuple[float, float, float] | None:
|
| 136 |
+
x = landmark_axis(frame, name, "x")
|
| 137 |
+
y = landmark_axis(frame, name, "y")
|
| 138 |
+
z = landmark_axis(frame, name, "z")
|
| 139 |
+
if None in {x, y, z}:
|
| 140 |
return None
|
| 141 |
+
return float(x), float(y), float(z)
|
| 142 |
|
| 143 |
|
| 144 |
def _distance(frame: PoseFrame, first: str, second: str) -> float | None:
|
|
|
|
| 146 |
second_point = _point(frame, second)
|
| 147 |
if first_point is None or second_point is None:
|
| 148 |
return None
|
| 149 |
+
return math.sqrt(sum((first_point[index] - second_point[index]) ** 2 for index in range(3)))
|
| 150 |
|
| 151 |
|
| 152 |
def _safe_ratio(numerator: float | None, denominator: float | None) -> float:
|
|
|
|
| 164 |
|
| 165 |
abx = first_point[0] - middle_point[0]
|
| 166 |
aby = first_point[1] - middle_point[1]
|
| 167 |
+
abz = first_point[2] - middle_point[2]
|
| 168 |
cbx = last_point[0] - middle_point[0]
|
| 169 |
cby = last_point[1] - middle_point[1]
|
| 170 |
+
cbz = last_point[2] - middle_point[2]
|
| 171 |
+
denom = math.sqrt(abx * abx + aby * aby + abz * abz) * math.sqrt(
|
| 172 |
+
cbx * cbx + cby * cby + cbz * cbz
|
| 173 |
+
)
|
| 174 |
if denom <= 1e-6:
|
| 175 |
return 0.0
|
| 176 |
+
cosine = max(-1.0, min(1.0, (abx * cbx + aby * cby + abz * cbz) / denom))
|
| 177 |
return math.degrees(math.acos(cosine))
|
| 178 |
|
| 179 |
|
|
|
|
| 186 |
def frame_feature_vector(frame: PoseFrame) -> np.ndarray:
|
| 187 |
values: list[float] = []
|
| 188 |
for landmark in LANDMARK_NAMES:
|
| 189 |
+
landmark_values = frame.world_landmarks.get(landmark) or frame.landmarks.get(landmark)
|
| 190 |
+
point = _point(frame, landmark)
|
| 191 |
values.extend(
|
| 192 |
[
|
| 193 |
+
point[0] if point is not None else 0.0,
|
| 194 |
+
point[1] if point is not None else 0.0,
|
| 195 |
+
point[2] if point is not None else 0.0,
|
| 196 |
_visibility(frame, landmark_values),
|
| 197 |
]
|
| 198 |
)
|
|
|
|
| 282 |
)
|
| 283 |
)
|
| 284 |
return windows
|
|
|
src/pozify/ml/exercise_router_inference.py
CHANGED
|
@@ -9,7 +9,14 @@ from typing import Any
|
|
| 9 |
import numpy as np
|
| 10 |
|
| 11 |
from pozify.hf_spaces import router_torch_device
|
| 12 |
-
from pozify.ml.exercise_router_features import
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
from pozify.ml.exercise_router_temporal import TorchTemporalRouter
|
| 14 |
|
| 15 |
|
|
@@ -39,6 +46,8 @@ class RouterModelBundle:
|
|
| 39 |
labels: tuple[str, ...] = ROUTER_LABELS
|
| 40 |
scaler: Any | None = None
|
| 41 |
model_kind: str = "baseline"
|
|
|
|
|
|
|
| 42 |
|
| 43 |
|
| 44 |
@dataclass(frozen=True)
|
|
@@ -137,6 +146,8 @@ def load_router_model_file(path: Path) -> RouterModelBundle:
|
|
| 137 |
model = artifact.get("model")
|
| 138 |
if model is None:
|
| 139 |
raise ValueError(f"Router artifact {path} is missing a 'model' entry")
|
|
|
|
|
|
|
| 140 |
label_source = getattr(model, "classes_", None)
|
| 141 |
if label_source is None:
|
| 142 |
label_source = artifact.get("labels") or artifact.get("classes")
|
|
@@ -145,13 +156,12 @@ def load_router_model_file(path: Path) -> RouterModelBundle:
|
|
| 145 |
model=model,
|
| 146 |
labels=labels,
|
| 147 |
scaler=artifact.get("scaler"),
|
| 148 |
-
model_kind=
|
|
|
|
|
|
|
| 149 |
)
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
labels=_labels_from_artifact(None, artifact),
|
| 153 |
-
scaler=None,
|
| 154 |
-
model_kind="baseline",
|
| 155 |
)
|
| 156 |
|
| 157 |
|
|
@@ -162,6 +172,7 @@ def _load_temporal_router_model_file(path: Path) -> RouterModelBundle:
|
|
| 162 |
raise RuntimeError("torch is required to load temporal exercise router artifacts") from exc
|
| 163 |
|
| 164 |
checkpoint = torch.load(path, map_location="cpu", weights_only=False)
|
|
|
|
| 165 |
labels = _labels_from_artifact(checkpoint.get("labels"), checkpoint)
|
| 166 |
model = TorchTemporalRouter(
|
| 167 |
checkpoint=checkpoint,
|
|
@@ -173,9 +184,39 @@ def _load_temporal_router_model_file(path: Path) -> RouterModelBundle:
|
|
| 173 |
labels=labels,
|
| 174 |
scaler=None,
|
| 175 |
model_kind=str(checkpoint.get("model_kind", "temporal")),
|
|
|
|
|
|
|
| 176 |
)
|
| 177 |
|
| 178 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 179 |
def predict_window_probabilities(
|
| 180 |
bundle: RouterModelBundle,
|
| 181 |
windows: list[RouterWindow],
|
|
@@ -184,6 +225,10 @@ def predict_window_probabilities(
|
|
| 184 |
return []
|
| 185 |
|
| 186 |
labels = _labels_from_artifact(bundle.labels, bundle.model)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
if bundle.model_kind == "temporal":
|
| 188 |
inputs = np.stack([window.tensor for window in windows]).astype(np.float32)
|
| 189 |
else:
|
|
|
|
| 9 |
import numpy as np
|
| 10 |
|
| 11 |
from pozify.hf_spaces import router_torch_device
|
| 12 |
+
from pozify.ml.exercise_router_features import (
|
| 13 |
+
FEATURE_SCHEMA,
|
| 14 |
+
ROUTER_LABELS,
|
| 15 |
+
ROUTER_LANDMARK_SCHEMA,
|
| 16 |
+
window_tensor_feature_names,
|
| 17 |
+
window_vector_feature_names,
|
| 18 |
+
RouterWindow,
|
| 19 |
+
)
|
| 20 |
from pozify.ml.exercise_router_temporal import TorchTemporalRouter
|
| 21 |
|
| 22 |
|
|
|
|
| 46 |
labels: tuple[str, ...] = ROUTER_LABELS
|
| 47 |
scaler: Any | None = None
|
| 48 |
model_kind: str = "baseline"
|
| 49 |
+
feature_schema: str = FEATURE_SCHEMA
|
| 50 |
+
landmark_schema: str = ROUTER_LANDMARK_SCHEMA
|
| 51 |
|
| 52 |
|
| 53 |
@dataclass(frozen=True)
|
|
|
|
| 146 |
model = artifact.get("model")
|
| 147 |
if model is None:
|
| 148 |
raise ValueError(f"Router artifact {path} is missing a 'model' entry")
|
| 149 |
+
model_kind = str(artifact.get("model_kind", artifact.get("kind", "baseline")))
|
| 150 |
+
_validate_router_artifact_metadata(artifact, model_kind=model_kind, path=path)
|
| 151 |
label_source = getattr(model, "classes_", None)
|
| 152 |
if label_source is None:
|
| 153 |
label_source = artifact.get("labels") or artifact.get("classes")
|
|
|
|
| 156 |
model=model,
|
| 157 |
labels=labels,
|
| 158 |
scaler=artifact.get("scaler"),
|
| 159 |
+
model_kind=model_kind,
|
| 160 |
+
feature_schema=str(artifact.get("feature_schema", "")),
|
| 161 |
+
landmark_schema=str(artifact.get("landmark_schema", "")),
|
| 162 |
)
|
| 163 |
+
raise ValueError(
|
| 164 |
+
f"Router artifact {path} is missing required schema metadata for {FEATURE_SCHEMA}"
|
|
|
|
|
|
|
|
|
|
| 165 |
)
|
| 166 |
|
| 167 |
|
|
|
|
| 172 |
raise RuntimeError("torch is required to load temporal exercise router artifacts") from exc
|
| 173 |
|
| 174 |
checkpoint = torch.load(path, map_location="cpu", weights_only=False)
|
| 175 |
+
_validate_router_artifact_metadata(checkpoint, model_kind="temporal", path=path)
|
| 176 |
labels = _labels_from_artifact(checkpoint.get("labels"), checkpoint)
|
| 177 |
model = TorchTemporalRouter(
|
| 178 |
checkpoint=checkpoint,
|
|
|
|
| 184 |
labels=labels,
|
| 185 |
scaler=None,
|
| 186 |
model_kind=str(checkpoint.get("model_kind", "temporal")),
|
| 187 |
+
feature_schema=str(checkpoint.get("feature_schema", "")),
|
| 188 |
+
landmark_schema=str(checkpoint.get("landmark_schema", "")),
|
| 189 |
)
|
| 190 |
|
| 191 |
|
| 192 |
+
def _validate_router_artifact_metadata(
|
| 193 |
+
artifact: dict[str, Any],
|
| 194 |
+
*,
|
| 195 |
+
model_kind: str,
|
| 196 |
+
path: Path,
|
| 197 |
+
) -> None:
|
| 198 |
+
feature_schema = artifact.get("feature_schema")
|
| 199 |
+
landmark_schema = artifact.get("landmark_schema")
|
| 200 |
+
if feature_schema != FEATURE_SCHEMA or landmark_schema != ROUTER_LANDMARK_SCHEMA:
|
| 201 |
+
raise ValueError(
|
| 202 |
+
f"Router artifact {path} uses incompatible schema: "
|
| 203 |
+
f"feature_schema={feature_schema!r}, landmark_schema={landmark_schema!r}"
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
input_size = artifact.get("input_size")
|
| 207 |
+
if input_size is None:
|
| 208 |
+
return
|
| 209 |
+
expected_size = (
|
| 210 |
+
len(window_tensor_feature_names())
|
| 211 |
+
if model_kind == "temporal"
|
| 212 |
+
else len(window_vector_feature_names())
|
| 213 |
+
)
|
| 214 |
+
if int(input_size) != expected_size:
|
| 215 |
+
raise ValueError(
|
| 216 |
+
f"Router artifact {path} input_size={input_size} does not match {expected_size}"
|
| 217 |
+
)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
def predict_window_probabilities(
|
| 221 |
bundle: RouterModelBundle,
|
| 222 |
windows: list[RouterWindow],
|
|
|
|
| 225 |
return []
|
| 226 |
|
| 227 |
labels = _labels_from_artifact(bundle.labels, bundle.model)
|
| 228 |
+
if bundle.feature_schema != FEATURE_SCHEMA or bundle.landmark_schema != ROUTER_LANDMARK_SCHEMA:
|
| 229 |
+
raise ValueError(
|
| 230 |
+
"Router model schema does not match current pose feature schema"
|
| 231 |
+
)
|
| 232 |
if bundle.model_kind == "temporal":
|
| 233 |
inputs = np.stack([window.tensor for window in windows]).astype(np.float32)
|
| 234 |
else:
|
src/pozify/steps/pose_backends/__init__.py
CHANGED
|
@@ -1,7 +1,12 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
from pozify.steps.pose_backends.base import PoseBackend, PoseDetection
|
| 4 |
-
from pozify.steps.pose_backends.landmarks import
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
from pozify.steps.pose_backends.mediapipe import MediaPipePoseBackend
|
| 6 |
from pozify.steps.pose_backends.mmpose import MMPoseBackend
|
| 7 |
from pozify.steps.pose_backends.mock import MockPoseBackend
|
|
@@ -10,6 +15,7 @@ from pozify.steps.pose_backends.registry import create_pose_backend
|
|
| 10 |
|
| 11 |
__all__ = [
|
| 12 |
"LANDMARK_NAMES",
|
|
|
|
| 13 |
"MMPoseBackend",
|
| 14 |
"MediaPipePoseBackend",
|
| 15 |
"MockPoseBackend",
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
from pozify.steps.pose_backends.base import PoseBackend, PoseDetection
|
| 4 |
+
from pozify.steps.pose_backends.landmarks import (
|
| 5 |
+
LANDMARK_NAMES,
|
| 6 |
+
LANDMARK_SCHEMA,
|
| 7 |
+
landmark_list_to_dict,
|
| 8 |
+
landmark_to_dict,
|
| 9 |
+
)
|
| 10 |
from pozify.steps.pose_backends.mediapipe import MediaPipePoseBackend
|
| 11 |
from pozify.steps.pose_backends.mmpose import MMPoseBackend
|
| 12 |
from pozify.steps.pose_backends.mock import MockPoseBackend
|
|
|
|
| 15 |
|
| 16 |
__all__ = [
|
| 17 |
"LANDMARK_NAMES",
|
| 18 |
+
"LANDMARK_SCHEMA",
|
| 19 |
"MMPoseBackend",
|
| 20 |
"MediaPipePoseBackend",
|
| 21 |
"MockPoseBackend",
|
src/pozify/steps/pose_backends/landmarks.py
CHANGED
|
@@ -3,40 +3,46 @@ from __future__ import annotations
|
|
| 3 |
from typing import Any
|
| 4 |
|
| 5 |
|
|
|
|
|
|
|
| 6 |
LANDMARK_NAMES = [
|
| 7 |
"nose",
|
| 8 |
-
"left_eye_inner",
|
| 9 |
"left_eye",
|
| 10 |
-
"left_eye_outer",
|
| 11 |
-
"right_eye_inner",
|
| 12 |
"right_eye",
|
| 13 |
-
"right_eye_outer",
|
| 14 |
"left_ear",
|
| 15 |
"right_ear",
|
| 16 |
-
"mouth_left",
|
| 17 |
-
"mouth_right",
|
| 18 |
"left_shoulder",
|
| 19 |
"right_shoulder",
|
| 20 |
"left_elbow",
|
| 21 |
"right_elbow",
|
| 22 |
"left_wrist",
|
| 23 |
"right_wrist",
|
| 24 |
-
"left_pinky",
|
| 25 |
-
"right_pinky",
|
| 26 |
-
"left_index",
|
| 27 |
-
"right_index",
|
| 28 |
-
"left_thumb",
|
| 29 |
-
"right_thumb",
|
| 30 |
"left_hip",
|
| 31 |
"right_hip",
|
| 32 |
"left_knee",
|
| 33 |
"right_knee",
|
| 34 |
"left_ankle",
|
| 35 |
"right_ankle",
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
]
|
| 41 |
|
| 42 |
|
|
@@ -57,6 +63,9 @@ def landmark_list_to_dict(landmarks: Any | None) -> dict[str, dict[str, float]]:
|
|
| 57 |
if landmarks is None:
|
| 58 |
return {}
|
| 59 |
landmark_values = landmarks.landmark if hasattr(landmarks, "landmark") else landmarks
|
|
|
|
|
|
|
|
|
|
| 60 |
return {
|
| 61 |
name: landmark_to_dict(landmark)
|
| 62 |
for name, landmark in zip(LANDMARK_NAMES, landmark_values, strict=False)
|
|
|
|
| 3 |
from typing import Any
|
| 4 |
|
| 5 |
|
| 6 |
+
LANDMARK_SCHEMA = "coco17"
|
| 7 |
+
|
| 8 |
LANDMARK_NAMES = [
|
| 9 |
"nose",
|
|
|
|
| 10 |
"left_eye",
|
|
|
|
|
|
|
| 11 |
"right_eye",
|
|
|
|
| 12 |
"left_ear",
|
| 13 |
"right_ear",
|
|
|
|
|
|
|
| 14 |
"left_shoulder",
|
| 15 |
"right_shoulder",
|
| 16 |
"left_elbow",
|
| 17 |
"right_elbow",
|
| 18 |
"left_wrist",
|
| 19 |
"right_wrist",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
"left_hip",
|
| 21 |
"right_hip",
|
| 22 |
"left_knee",
|
| 23 |
"right_knee",
|
| 24 |
"left_ankle",
|
| 25 |
"right_ankle",
|
| 26 |
+
]
|
| 27 |
+
|
| 28 |
+
MEDIAPIPE_COCO17_INDICES = [
|
| 29 |
+
0, # nose
|
| 30 |
+
2, # left_eye
|
| 31 |
+
5, # right_eye
|
| 32 |
+
7, # left_ear
|
| 33 |
+
8, # right_ear
|
| 34 |
+
11, # left_shoulder
|
| 35 |
+
12, # right_shoulder
|
| 36 |
+
13, # left_elbow
|
| 37 |
+
14, # right_elbow
|
| 38 |
+
15, # left_wrist
|
| 39 |
+
16, # right_wrist
|
| 40 |
+
23, # left_hip
|
| 41 |
+
24, # right_hip
|
| 42 |
+
25, # left_knee
|
| 43 |
+
26, # right_knee
|
| 44 |
+
27, # left_ankle
|
| 45 |
+
28, # right_ankle
|
| 46 |
]
|
| 47 |
|
| 48 |
|
|
|
|
| 63 |
if landmarks is None:
|
| 64 |
return {}
|
| 65 |
landmark_values = landmarks.landmark if hasattr(landmarks, "landmark") else landmarks
|
| 66 |
+
landmark_values = list(landmark_values)
|
| 67 |
+
if len(landmark_values) >= 33:
|
| 68 |
+
landmark_values = [landmark_values[index] for index in MEDIAPIPE_COCO17_INDICES]
|
| 69 |
return {
|
| 70 |
name: landmark_to_dict(landmark)
|
| 71 |
for name, landmark in zip(LANDMARK_NAMES, landmark_values, strict=False)
|
src/pozify/steps/pose_cleaning.py
CHANGED
|
@@ -14,12 +14,18 @@ def _copy_landmarks(
|
|
| 14 |
return {name: dict(values) for name, values in landmarks.items()}
|
| 15 |
|
| 16 |
|
| 17 |
-
def _copy_frame(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
return PoseFrame(
|
| 19 |
frame_index=frame.frame_index,
|
| 20 |
timestamp_sec=frame.timestamp_sec,
|
| 21 |
landmarks=_copy_landmarks(landmarks if landmarks is not None else frame.landmarks),
|
| 22 |
-
world_landmarks=_copy_landmarks(
|
|
|
|
|
|
|
| 23 |
pose_quality=dict(frame.pose_quality),
|
| 24 |
)
|
| 25 |
|
|
@@ -78,11 +84,16 @@ def _interpolate_short_gaps(frames: list[PoseFrame]) -> list[PoseFrame]:
|
|
| 78 |
cleaned[next_index].landmarks,
|
| 79 |
fraction,
|
| 80 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
cleaned[frame_index] = PoseFrame(
|
| 82 |
frame_index=cleaned[frame_index].frame_index,
|
| 83 |
timestamp_sec=cleaned[frame_index].timestamp_sec,
|
| 84 |
landmarks=landmarks,
|
| 85 |
-
world_landmarks=
|
| 86 |
pose_quality={
|
| 87 |
**cleaned[frame_index].pose_quality,
|
| 88 |
"interpolated": bool(landmarks),
|
|
@@ -93,41 +104,51 @@ def _interpolate_short_gaps(frames: list[PoseFrame]) -> list[PoseFrame]:
|
|
| 93 |
|
| 94 |
|
| 95 |
def _add_smoothed_fields(frames: list[PoseFrame]) -> list[PoseFrame]:
|
| 96 |
-
|
|
|
|
| 97 |
smoothed_frames: list[PoseFrame] = []
|
| 98 |
|
| 99 |
for frame in frames:
|
| 100 |
landmarks = _copy_landmarks(frame.landmarks)
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
for field in SMOOTHED_FIELDS:
|
| 105 |
-
if field not in values:
|
| 106 |
-
continue
|
| 107 |
-
previous_value = previous_values.get(f"smoothed_{field}", values[field])
|
| 108 |
-
smoothed = previous_value * (1.0 - SMOOTHING_ALPHA) + values[field] * SMOOTHING_ALPHA
|
| 109 |
-
values[f"smoothed_{field}"] = round(smoothed, 6)
|
| 110 |
-
current_smoothed[f"smoothed_{field}"] = values[f"smoothed_{field}"]
|
| 111 |
-
previous[name] = current_smoothed
|
| 112 |
|
| 113 |
smoothed_frames.append(
|
| 114 |
PoseFrame(
|
| 115 |
frame_index=frame.frame_index,
|
| 116 |
timestamp_sec=frame.timestamp_sec,
|
| 117 |
landmarks=landmarks,
|
| 118 |
-
world_landmarks=
|
| 119 |
pose_quality=frame.pose_quality,
|
| 120 |
)
|
| 121 |
)
|
| 122 |
return smoothed_frames
|
| 123 |
|
| 124 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
def _normalization_origin_and_scale(
|
| 126 |
landmarks: dict[str, dict[str, float]],
|
| 127 |
-
) -> tuple[float, float, float, bool]:
|
| 128 |
required = ("left_hip", "right_hip", "left_shoulder", "right_shoulder")
|
| 129 |
if not all(name in landmarks for name in required):
|
| 130 |
-
return 0.0, 0.0, 1.0, False
|
| 131 |
|
| 132 |
left_hip = landmarks["left_hip"]
|
| 133 |
right_hip = landmarks["right_hip"]
|
|
@@ -135,41 +156,83 @@ def _normalization_origin_and_scale(
|
|
| 135 |
right_shoulder = landmarks["right_shoulder"]
|
| 136 |
origin_x = (left_hip["x"] + right_hip["x"]) / 2.0
|
| 137 |
origin_y = (left_hip["y"] + right_hip["y"]) / 2.0
|
|
|
|
| 138 |
mid_shoulder_x = (left_shoulder["x"] + right_shoulder["x"]) / 2.0
|
| 139 |
mid_shoulder_y = (left_shoulder["y"] + right_shoulder["y"]) / 2.0
|
| 140 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
if torso_length <= 1e-6:
|
| 142 |
-
return origin_x, origin_y, 1.0, False
|
| 143 |
-
return origin_x, origin_y, torso_length, True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
|
| 145 |
|
| 146 |
def _add_normalized_fields(frames: list[PoseFrame]) -> list[PoseFrame]:
|
| 147 |
normalized_frames: list[PoseFrame] = []
|
|
|
|
|
|
|
| 148 |
for frame in frames:
|
| 149 |
-
landmarks =
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
values["normalized_y"] = round((source_y - origin_y) / scale, 6)
|
| 159 |
-
values["normalized_z"] = round(source_z / scale, 6)
|
| 160 |
|
| 161 |
normalized_frames.append(
|
| 162 |
PoseFrame(
|
| 163 |
frame_index=frame.frame_index,
|
| 164 |
timestamp_sec=frame.timestamp_sec,
|
| 165 |
landmarks=landmarks,
|
| 166 |
-
world_landmarks=
|
| 167 |
pose_quality={
|
| 168 |
**frame.pose_quality,
|
| 169 |
"cleaned": True,
|
| 170 |
"normalized": normalized,
|
| 171 |
"normalization_origin": "mid_hip",
|
| 172 |
"normalization_scale": "mid_shoulder_to_mid_hip",
|
|
|
|
| 173 |
},
|
| 174 |
)
|
| 175 |
)
|
|
|
|
| 14 |
return {name: dict(values) for name, values in landmarks.items()}
|
| 15 |
|
| 16 |
|
| 17 |
+
def _copy_frame(
|
| 18 |
+
frame: PoseFrame,
|
| 19 |
+
landmarks: dict[str, dict[str, float]] | None = None,
|
| 20 |
+
world_landmarks: dict[str, dict[str, float]] | None = None,
|
| 21 |
+
) -> PoseFrame:
|
| 22 |
return PoseFrame(
|
| 23 |
frame_index=frame.frame_index,
|
| 24 |
timestamp_sec=frame.timestamp_sec,
|
| 25 |
landmarks=_copy_landmarks(landmarks if landmarks is not None else frame.landmarks),
|
| 26 |
+
world_landmarks=_copy_landmarks(
|
| 27 |
+
world_landmarks if world_landmarks is not None else frame.world_landmarks
|
| 28 |
+
),
|
| 29 |
pose_quality=dict(frame.pose_quality),
|
| 30 |
)
|
| 31 |
|
|
|
|
| 84 |
cleaned[next_index].landmarks,
|
| 85 |
fraction,
|
| 86 |
)
|
| 87 |
+
world_landmarks = _interpolate_landmarks(
|
| 88 |
+
cleaned[previous_index].world_landmarks,
|
| 89 |
+
cleaned[next_index].world_landmarks,
|
| 90 |
+
fraction,
|
| 91 |
+
)
|
| 92 |
cleaned[frame_index] = PoseFrame(
|
| 93 |
frame_index=cleaned[frame_index].frame_index,
|
| 94 |
timestamp_sec=cleaned[frame_index].timestamp_sec,
|
| 95 |
landmarks=landmarks,
|
| 96 |
+
world_landmarks=world_landmarks,
|
| 97 |
pose_quality={
|
| 98 |
**cleaned[frame_index].pose_quality,
|
| 99 |
"interpolated": bool(landmarks),
|
|
|
|
| 104 |
|
| 105 |
|
| 106 |
def _add_smoothed_fields(frames: list[PoseFrame]) -> list[PoseFrame]:
|
| 107 |
+
previous_landmarks: dict[str, dict[str, float]] = {}
|
| 108 |
+
previous_world_landmarks: dict[str, dict[str, float]] = {}
|
| 109 |
smoothed_frames: list[PoseFrame] = []
|
| 110 |
|
| 111 |
for frame in frames:
|
| 112 |
landmarks = _copy_landmarks(frame.landmarks)
|
| 113 |
+
world_landmarks = _copy_landmarks(frame.world_landmarks)
|
| 114 |
+
_smooth_landmarks(landmarks, previous_landmarks)
|
| 115 |
+
_smooth_landmarks(world_landmarks, previous_world_landmarks)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
|
| 117 |
smoothed_frames.append(
|
| 118 |
PoseFrame(
|
| 119 |
frame_index=frame.frame_index,
|
| 120 |
timestamp_sec=frame.timestamp_sec,
|
| 121 |
landmarks=landmarks,
|
| 122 |
+
world_landmarks=world_landmarks,
|
| 123 |
pose_quality=frame.pose_quality,
|
| 124 |
)
|
| 125 |
)
|
| 126 |
return smoothed_frames
|
| 127 |
|
| 128 |
|
| 129 |
+
def _smooth_landmarks(
|
| 130 |
+
landmarks: dict[str, dict[str, float]],
|
| 131 |
+
previous: dict[str, dict[str, float]],
|
| 132 |
+
) -> None:
|
| 133 |
+
for name, values in landmarks.items():
|
| 134 |
+
previous_values = previous.get(name, {})
|
| 135 |
+
current_smoothed: dict[str, float] = {}
|
| 136 |
+
for field in SMOOTHED_FIELDS:
|
| 137 |
+
if field not in values:
|
| 138 |
+
continue
|
| 139 |
+
previous_value = previous_values.get(f"smoothed_{field}", values[field])
|
| 140 |
+
smoothed = previous_value * (1.0 - SMOOTHING_ALPHA) + values[field] * SMOOTHING_ALPHA
|
| 141 |
+
values[f"smoothed_{field}"] = round(smoothed, 6)
|
| 142 |
+
current_smoothed[f"smoothed_{field}"] = values[f"smoothed_{field}"]
|
| 143 |
+
previous[name] = current_smoothed
|
| 144 |
+
|
| 145 |
+
|
| 146 |
def _normalization_origin_and_scale(
|
| 147 |
landmarks: dict[str, dict[str, float]],
|
| 148 |
+
) -> tuple[float, float, float, float, bool]:
|
| 149 |
required = ("left_hip", "right_hip", "left_shoulder", "right_shoulder")
|
| 150 |
if not all(name in landmarks for name in required):
|
| 151 |
+
return 0.0, 0.0, 0.0, 1.0, False
|
| 152 |
|
| 153 |
left_hip = landmarks["left_hip"]
|
| 154 |
right_hip = landmarks["right_hip"]
|
|
|
|
| 156 |
right_shoulder = landmarks["right_shoulder"]
|
| 157 |
origin_x = (left_hip["x"] + right_hip["x"]) / 2.0
|
| 158 |
origin_y = (left_hip["y"] + right_hip["y"]) / 2.0
|
| 159 |
+
origin_z = (left_hip.get("z", 0.0) + right_hip.get("z", 0.0)) / 2.0
|
| 160 |
mid_shoulder_x = (left_shoulder["x"] + right_shoulder["x"]) / 2.0
|
| 161 |
mid_shoulder_y = (left_shoulder["y"] + right_shoulder["y"]) / 2.0
|
| 162 |
+
mid_shoulder_z = (left_shoulder.get("z", 0.0) + right_shoulder.get("z", 0.0)) / 2.0
|
| 163 |
+
torso_length = (
|
| 164 |
+
(mid_shoulder_x - origin_x) ** 2
|
| 165 |
+
+ (mid_shoulder_y - origin_y) ** 2
|
| 166 |
+
+ (mid_shoulder_z - origin_z) ** 2
|
| 167 |
+
) ** 0.5
|
| 168 |
if torso_length <= 1e-6:
|
| 169 |
+
return origin_x, origin_y, origin_z, 1.0, False
|
| 170 |
+
return origin_x, origin_y, origin_z, torso_length, True
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _vertical_sign(frames: list[PoseFrame], *, use_world_landmarks: bool) -> float:
|
| 174 |
+
for frame in frames:
|
| 175 |
+
landmarks = frame.world_landmarks if use_world_landmarks else frame.landmarks
|
| 176 |
+
required = ("left_hip", "right_hip", "left_shoulder", "right_shoulder")
|
| 177 |
+
if not all(name in landmarks for name in required):
|
| 178 |
+
continue
|
| 179 |
+
origin_y = (landmarks["left_hip"]["y"] + landmarks["right_hip"]["y"]) / 2.0
|
| 180 |
+
shoulder_y = (
|
| 181 |
+
landmarks["left_shoulder"]["y"] + landmarks["right_shoulder"]["y"]
|
| 182 |
+
) / 2.0
|
| 183 |
+
return -1.0 if shoulder_y > origin_y else 1.0
|
| 184 |
+
return 1.0
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def _add_normalized_landmarks(
|
| 188 |
+
landmarks: dict[str, dict[str, float]],
|
| 189 |
+
*,
|
| 190 |
+
vertical_sign: float,
|
| 191 |
+
) -> tuple[dict[str, dict[str, float]], bool]:
|
| 192 |
+
normalized_landmarks = _copy_landmarks(landmarks)
|
| 193 |
+
origin_x, origin_y, origin_z, scale, normalized = _normalization_origin_and_scale(
|
| 194 |
+
normalized_landmarks
|
| 195 |
+
)
|
| 196 |
+
for values in normalized_landmarks.values():
|
| 197 |
+
source_x = values.get("smoothed_x", values.get("x"))
|
| 198 |
+
source_y = values.get("smoothed_y", values.get("y"))
|
| 199 |
+
source_z = values.get("smoothed_z", values.get("z"))
|
| 200 |
+
if source_x is None or source_y is None or source_z is None:
|
| 201 |
+
continue
|
| 202 |
+
values["normalized_x"] = round((source_x - origin_x) / scale, 6)
|
| 203 |
+
values["normalized_y"] = round(((source_y - origin_y) * vertical_sign) / scale, 6)
|
| 204 |
+
values["normalized_z"] = round((source_z - origin_z) / scale, 6)
|
| 205 |
+
return normalized_landmarks, normalized
|
| 206 |
|
| 207 |
|
| 208 |
def _add_normalized_fields(frames: list[PoseFrame]) -> list[PoseFrame]:
|
| 209 |
normalized_frames: list[PoseFrame] = []
|
| 210 |
+
landmark_vertical_sign = _vertical_sign(frames, use_world_landmarks=False)
|
| 211 |
+
world_vertical_sign = _vertical_sign(frames, use_world_landmarks=True)
|
| 212 |
for frame in frames:
|
| 213 |
+
landmarks, landmarks_normalized = _add_normalized_landmarks(
|
| 214 |
+
frame.landmarks,
|
| 215 |
+
vertical_sign=landmark_vertical_sign,
|
| 216 |
+
)
|
| 217 |
+
world_landmarks, world_normalized = _add_normalized_landmarks(
|
| 218 |
+
frame.world_landmarks,
|
| 219 |
+
vertical_sign=world_vertical_sign,
|
| 220 |
+
)
|
| 221 |
+
normalized = world_normalized if world_landmarks else landmarks_normalized
|
|
|
|
|
|
|
| 222 |
|
| 223 |
normalized_frames.append(
|
| 224 |
PoseFrame(
|
| 225 |
frame_index=frame.frame_index,
|
| 226 |
timestamp_sec=frame.timestamp_sec,
|
| 227 |
landmarks=landmarks,
|
| 228 |
+
world_landmarks=world_landmarks,
|
| 229 |
pose_quality={
|
| 230 |
**frame.pose_quality,
|
| 231 |
"cleaned": True,
|
| 232 |
"normalized": normalized,
|
| 233 |
"normalization_origin": "mid_hip",
|
| 234 |
"normalization_scale": "mid_shoulder_to_mid_hip",
|
| 235 |
+
"world_landmarks_normalized": world_normalized,
|
| 236 |
},
|
| 237 |
)
|
| 238 |
)
|
src/pozify/steps/pose_landmarker.py
CHANGED
|
@@ -8,6 +8,7 @@ import cv2
|
|
| 8 |
|
| 9 |
from pozify.contracts import PoseFrame, PoseSequence, VideoManifest
|
| 10 |
from pozify.steps.pose_backends import (
|
|
|
|
| 11 |
MockPoseBackend,
|
| 12 |
PoseBackend,
|
| 13 |
create_pose_backend,
|
|
@@ -104,6 +105,14 @@ def _pose_quality(landmarks: dict[str, dict[str, float]]) -> dict[str, Any]:
|
|
| 104 |
}
|
| 105 |
|
| 106 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
def _empty_sequence() -> PoseSequence:
|
| 108 |
return PoseSequence(
|
| 109 |
frames=[], normalized=False, smoothing_method="none", pose_valid_ratio=0.0
|
|
@@ -134,6 +143,10 @@ def _run_with_backend(manifest: VideoManifest, backend: PoseBackend) -> PoseSequ
|
|
| 134 |
pose_quality={
|
| 135 |
**_pose_quality(detection.landmarks),
|
| 136 |
"source": detection.source,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
},
|
| 138 |
)
|
| 139 |
)
|
|
@@ -166,6 +179,8 @@ def _run_mock(manifest: VideoManifest) -> PoseSequence:
|
|
| 166 |
**_pose_quality(detection.landmarks),
|
| 167 |
"source": detection.source,
|
| 168 |
"mock": True,
|
|
|
|
|
|
|
| 169 |
},
|
| 170 |
)
|
| 171 |
)
|
|
|
|
| 8 |
|
| 9 |
from pozify.contracts import PoseFrame, PoseSequence, VideoManifest
|
| 10 |
from pozify.steps.pose_backends import (
|
| 11 |
+
LANDMARK_SCHEMA,
|
| 12 |
MockPoseBackend,
|
| 13 |
PoseBackend,
|
| 14 |
create_pose_backend,
|
|
|
|
| 105 |
}
|
| 106 |
|
| 107 |
|
| 108 |
+
def _coordinate_source(detection_source: str, world_landmarks: dict[str, dict[str, float]]) -> str:
|
| 109 |
+
if not world_landmarks:
|
| 110 |
+
return "image_landmarks"
|
| 111 |
+
if detection_source.startswith("mediapipe"):
|
| 112 |
+
return "mediapipe_world_landmarks"
|
| 113 |
+
return "world_landmarks"
|
| 114 |
+
|
| 115 |
+
|
| 116 |
def _empty_sequence() -> PoseSequence:
|
| 117 |
return PoseSequence(
|
| 118 |
frames=[], normalized=False, smoothing_method="none", pose_valid_ratio=0.0
|
|
|
|
| 143 |
pose_quality={
|
| 144 |
**_pose_quality(detection.landmarks),
|
| 145 |
"source": detection.source,
|
| 146 |
+
"landmark_schema": LANDMARK_SCHEMA,
|
| 147 |
+
"coordinate_source": _coordinate_source(
|
| 148 |
+
detection.source, detection.world_landmarks
|
| 149 |
+
),
|
| 150 |
},
|
| 151 |
)
|
| 152 |
)
|
|
|
|
| 179 |
**_pose_quality(detection.landmarks),
|
| 180 |
"source": detection.source,
|
| 181 |
"mock": True,
|
| 182 |
+
"landmark_schema": LANDMARK_SCHEMA,
|
| 183 |
+
"coordinate_source": "mock_landmarks",
|
| 184 |
},
|
| 185 |
)
|
| 186 |
)
|
src/pozify/steps/rep_signals.py
CHANGED
|
@@ -14,14 +14,18 @@ class SignalSample:
|
|
| 14 |
|
| 15 |
|
| 16 |
def _landmark_values(frame: PoseFrame, name: str) -> dict[str, float] | None:
|
| 17 |
-
return frame.landmarks.get(name)
|
| 18 |
|
| 19 |
|
| 20 |
def landmark_axis(frame: PoseFrame, name: str, axis: str) -> float | None:
|
| 21 |
values = _landmark_values(frame, name)
|
| 22 |
if values is None:
|
| 23 |
return None
|
| 24 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
|
| 26 |
|
| 27 |
def average_axis(frame: PoseFrame, names: tuple[str, ...], axis: str) -> float | None:
|
|
@@ -35,31 +39,69 @@ def average_axis(frame: PoseFrame, names: tuple[str, ...], axis: str) -> float |
|
|
| 35 |
def angle_deg(frame: PoseFrame, first: str, middle: str, last: str) -> float | None:
|
| 36 |
ax = landmark_axis(frame, first, "x")
|
| 37 |
ay = landmark_axis(frame, first, "y")
|
|
|
|
| 38 |
bx = landmark_axis(frame, middle, "x")
|
| 39 |
by = landmark_axis(frame, middle, "y")
|
|
|
|
| 40 |
cx = landmark_axis(frame, last, "x")
|
| 41 |
cy = landmark_axis(frame, last, "y")
|
| 42 |
-
|
|
|
|
| 43 |
return None
|
| 44 |
|
| 45 |
abx = ax - bx
|
| 46 |
aby = ay - by
|
|
|
|
| 47 |
cbx = cx - bx
|
| 48 |
cby = cy - by
|
| 49 |
-
|
|
|
|
|
|
|
|
|
|
| 50 |
if denom <= 1e-6:
|
| 51 |
return None
|
| 52 |
-
cosine = max(-1.0, min(1.0, (abx * cbx + aby * cby) / denom))
|
| 53 |
return math.degrees(math.acos(cosine))
|
| 54 |
|
| 55 |
|
| 56 |
def body_line_score(frame: PoseFrame) -> float | None:
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
if None
|
| 61 |
return None
|
| 62 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
|
| 64 |
|
| 65 |
def smooth_signal(values: list[float | None], window_radius: int = 2) -> list[float | None]:
|
|
|
|
| 14 |
|
| 15 |
|
| 16 |
def _landmark_values(frame: PoseFrame, name: str) -> dict[str, float] | None:
|
| 17 |
+
return frame.world_landmarks.get(name) or frame.landmarks.get(name)
|
| 18 |
|
| 19 |
|
| 20 |
def landmark_axis(frame: PoseFrame, name: str, axis: str) -> float | None:
|
| 21 |
values = _landmark_values(frame, name)
|
| 22 |
if values is None:
|
| 23 |
return None
|
| 24 |
+
fallback = 0.0 if axis == "z" else None
|
| 25 |
+
return values.get(
|
| 26 |
+
f"normalized_{axis}",
|
| 27 |
+
values.get(f"smoothed_{axis}", values.get(axis, fallback)),
|
| 28 |
+
)
|
| 29 |
|
| 30 |
|
| 31 |
def average_axis(frame: PoseFrame, names: tuple[str, ...], axis: str) -> float | None:
|
|
|
|
| 39 |
def angle_deg(frame: PoseFrame, first: str, middle: str, last: str) -> float | None:
|
| 40 |
ax = landmark_axis(frame, first, "x")
|
| 41 |
ay = landmark_axis(frame, first, "y")
|
| 42 |
+
az = landmark_axis(frame, first, "z")
|
| 43 |
bx = landmark_axis(frame, middle, "x")
|
| 44 |
by = landmark_axis(frame, middle, "y")
|
| 45 |
+
bz = landmark_axis(frame, middle, "z")
|
| 46 |
cx = landmark_axis(frame, last, "x")
|
| 47 |
cy = landmark_axis(frame, last, "y")
|
| 48 |
+
cz = landmark_axis(frame, last, "z")
|
| 49 |
+
if None in {ax, ay, az, bx, by, bz, cx, cy, cz}:
|
| 50 |
return None
|
| 51 |
|
| 52 |
abx = ax - bx
|
| 53 |
aby = ay - by
|
| 54 |
+
abz = az - bz
|
| 55 |
cbx = cx - bx
|
| 56 |
cby = cy - by
|
| 57 |
+
cbz = cz - bz
|
| 58 |
+
ab_length = math.sqrt(abx * abx + aby * aby + abz * abz)
|
| 59 |
+
cb_length = math.sqrt(cbx * cbx + cby * cby + cbz * cbz)
|
| 60 |
+
denom = ab_length * cb_length
|
| 61 |
if denom <= 1e-6:
|
| 62 |
return None
|
| 63 |
+
cosine = max(-1.0, min(1.0, (abx * cbx + aby * cby + abz * cbz) / denom))
|
| 64 |
return math.degrees(math.acos(cosine))
|
| 65 |
|
| 66 |
|
| 67 |
def body_line_score(frame: PoseFrame) -> float | None:
|
| 68 |
+
shoulder = average_point(frame, ("left_shoulder", "right_shoulder"))
|
| 69 |
+
hip = average_point(frame, ("left_hip", "right_hip"))
|
| 70 |
+
ankle = average_point(frame, ("left_ankle", "right_ankle"))
|
| 71 |
+
if shoulder is None or hip is None or ankle is None:
|
| 72 |
return None
|
| 73 |
+
midline = tuple((shoulder[index] + ankle[index]) / 2.0 for index in range(3))
|
| 74 |
+
deviation = math.sqrt(sum((midline[index] - hip[index]) ** 2 for index in range(3)))
|
| 75 |
+
return 1.0 - deviation
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def average_point(frame: PoseFrame, names: tuple[str, ...]) -> tuple[float, float, float] | None:
|
| 79 |
+
points: list[tuple[float, float, float]] = []
|
| 80 |
+
for name in names:
|
| 81 |
+
x = landmark_axis(frame, name, "x")
|
| 82 |
+
y = landmark_axis(frame, name, "y")
|
| 83 |
+
z = landmark_axis(frame, name, "z")
|
| 84 |
+
if None in {x, y, z}:
|
| 85 |
+
continue
|
| 86 |
+
points.append((float(x), float(y), float(z)))
|
| 87 |
+
if not points:
|
| 88 |
+
return None
|
| 89 |
+
count = len(points)
|
| 90 |
+
return (
|
| 91 |
+
sum(point[0] for point in points) / count,
|
| 92 |
+
sum(point[1] for point in points) / count,
|
| 93 |
+
sum(point[2] for point in points) / count,
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def distance(frame: PoseFrame, first: str, second: str) -> float | None:
|
| 98 |
+
first_point = average_point(frame, (first,))
|
| 99 |
+
second_point = average_point(frame, (second,))
|
| 100 |
+
if first_point is None or second_point is None:
|
| 101 |
+
return None
|
| 102 |
+
return math.sqrt(
|
| 103 |
+
sum((first_point[index] - second_point[index]) ** 2 for index in range(3))
|
| 104 |
+
)
|
| 105 |
|
| 106 |
|
| 107 |
def smooth_signal(values: list[float | None], window_radius: int = 2) -> list[float | None]:
|
tests/test_exercise_classifier.py
CHANGED
|
@@ -17,7 +17,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
| 17 |
from pozify.contracts import PoseFrame, PoseSequence, UserProfile, validate_contract
|
| 18 |
from pozify.ml.exercise_router_evaluation import evaluate_router_predictions, select_router_candidate
|
| 19 |
from pozify.ml.exercise_router_features import (
|
|
|
|
| 20 |
ROUTER_LABELS,
|
|
|
|
| 21 |
extract_router_windows,
|
| 22 |
window_tensor_feature_names,
|
| 23 |
window_vector_feature_names,
|
|
@@ -142,6 +144,17 @@ class _FakePushUpModel:
|
|
| 142 |
return np.tile(np.asarray([[0.03, 0.91, 0.02, 0.04]]), (values.shape[0], 1))
|
| 143 |
|
| 144 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
class ExerciseRouterFeatureTests(unittest.TestCase):
|
| 146 |
def test_extracts_windows_for_supported_exercise_motion(self) -> None:
|
| 147 |
for exercise in ("squat", "push_up", "shoulder_press"):
|
|
@@ -164,6 +177,17 @@ class ExerciseRouterFeatureTests(unittest.TestCase):
|
|
| 164 |
self.assertEqual(extract_router_windows(empty), [])
|
| 165 |
self.assertEqual(extract_router_windows(_sequence(visibility=0.05)), [])
|
| 166 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
|
| 168 |
class ExerciseRouterAggregationTests(unittest.TestCase):
|
| 169 |
def test_aggregates_confident_windows(self) -> None:
|
|
@@ -200,14 +224,7 @@ class ExerciseRouterModelLoadingTests(unittest.TestCase):
|
|
| 200 |
with tempfile.TemporaryDirectory() as temp_dir:
|
| 201 |
model_dir = Path(temp_dir)
|
| 202 |
artifact_path = model_dir / "selected.joblib"
|
| 203 |
-
joblib.dump(
|
| 204 |
-
{
|
| 205 |
-
"model": _FakePushUpModel(),
|
| 206 |
-
"labels": ["unknown", "squat", "push_up", "shoulder_press"],
|
| 207 |
-
"model_kind": "baseline",
|
| 208 |
-
},
|
| 209 |
-
artifact_path,
|
| 210 |
-
)
|
| 211 |
(model_dir / "router_selection.json").write_text(
|
| 212 |
json.dumps({"selected_artifact": artifact_path.name}),
|
| 213 |
encoding="utf-8",
|
|
@@ -233,14 +250,7 @@ class ExerciseRouterModelLoadingTests(unittest.TestCase):
|
|
| 233 |
model_dir = Path(temp_dir)
|
| 234 |
artifact_path = model_dir / "router.joblib"
|
| 235 |
selection_path = model_dir / "router_selection.json"
|
| 236 |
-
joblib.dump(
|
| 237 |
-
{
|
| 238 |
-
"model": _FakePushUpModel(),
|
| 239 |
-
"labels": list(ROUTER_LABELS),
|
| 240 |
-
"model_kind": "baseline",
|
| 241 |
-
},
|
| 242 |
-
artifact_path,
|
| 243 |
-
)
|
| 244 |
selection_path.write_text(
|
| 245 |
json.dumps({"selected_artifact": artifact_path.name}),
|
| 246 |
encoding="utf-8",
|
|
@@ -267,14 +277,7 @@ class ExerciseRouterModelLoadingTests(unittest.TestCase):
|
|
| 267 |
model_dir = Path(temp_dir)
|
| 268 |
artifact_path = model_dir / "router.joblib"
|
| 269 |
selection_path = model_dir / "router_selection.json"
|
| 270 |
-
joblib.dump(
|
| 271 |
-
{
|
| 272 |
-
"model": _FakePushUpModel(),
|
| 273 |
-
"labels": list(ROUTER_LABELS),
|
| 274 |
-
"model_kind": "baseline",
|
| 275 |
-
},
|
| 276 |
-
artifact_path,
|
| 277 |
-
)
|
| 278 |
selection_path.write_text(
|
| 279 |
json.dumps({"selected_artifact": artifact_path.name}),
|
| 280 |
encoding="utf-8",
|
|
@@ -299,6 +302,22 @@ class ExerciseRouterModelLoadingTests(unittest.TestCase):
|
|
| 299 |
self.assertEqual(bundle.model_kind, "baseline")
|
| 300 |
self.assertEqual(bundle.labels, tuple(ROUTER_LABELS))
|
| 301 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 302 |
|
| 303 |
class ExerciseClassifierStepTests(unittest.TestCase):
|
| 304 |
def test_manual_override_bypasses_model_and_validates_contract(self) -> None:
|
|
|
|
| 17 |
from pozify.contracts import PoseFrame, PoseSequence, UserProfile, validate_contract
|
| 18 |
from pozify.ml.exercise_router_evaluation import evaluate_router_predictions, select_router_candidate
|
| 19 |
from pozify.ml.exercise_router_features import (
|
| 20 |
+
FEATURE_SCHEMA,
|
| 21 |
ROUTER_LABELS,
|
| 22 |
+
ROUTER_LANDMARK_SCHEMA,
|
| 23 |
extract_router_windows,
|
| 24 |
window_tensor_feature_names,
|
| 25 |
window_vector_feature_names,
|
|
|
|
| 144 |
return np.tile(np.asarray([[0.03, 0.91, 0.02, 0.04]]), (values.shape[0], 1))
|
| 145 |
|
| 146 |
|
| 147 |
+
def _fake_router_artifact() -> dict[str, object]:
|
| 148 |
+
return {
|
| 149 |
+
"model": _FakePushUpModel(),
|
| 150 |
+
"labels": list(ROUTER_LABELS),
|
| 151 |
+
"model_kind": "baseline",
|
| 152 |
+
"feature_schema": FEATURE_SCHEMA,
|
| 153 |
+
"landmark_schema": ROUTER_LANDMARK_SCHEMA,
|
| 154 |
+
"input_size": len(window_vector_feature_names()),
|
| 155 |
+
}
|
| 156 |
+
|
| 157 |
+
|
| 158 |
class ExerciseRouterFeatureTests(unittest.TestCase):
|
| 159 |
def test_extracts_windows_for_supported_exercise_motion(self) -> None:
|
| 160 |
for exercise in ("squat", "push_up", "shoulder_press"):
|
|
|
|
| 177 |
self.assertEqual(extract_router_windows(empty), [])
|
| 178 |
self.assertEqual(extract_router_windows(_sequence(visibility=0.05)), [])
|
| 179 |
|
| 180 |
+
def test_missing_landmark_visibility_is_zero(self) -> None:
|
| 181 |
+
sequence = _sequence("push_up")
|
| 182 |
+
frame = sequence.frames[0]
|
| 183 |
+
frame.landmarks.pop("nose")
|
| 184 |
+
|
| 185 |
+
windows = extract_router_windows(sequence, min_mean_visibility=0.0)
|
| 186 |
+
|
| 187 |
+
self.assertGreater(len(windows), 0)
|
| 188 |
+
nose_visibility_index = window_tensor_feature_names().index("nose_visibility")
|
| 189 |
+
self.assertEqual(windows[0].tensor[0, nose_visibility_index], 0.0)
|
| 190 |
+
|
| 191 |
|
| 192 |
class ExerciseRouterAggregationTests(unittest.TestCase):
|
| 193 |
def test_aggregates_confident_windows(self) -> None:
|
|
|
|
| 224 |
with tempfile.TemporaryDirectory() as temp_dir:
|
| 225 |
model_dir = Path(temp_dir)
|
| 226 |
artifact_path = model_dir / "selected.joblib"
|
| 227 |
+
joblib.dump(_fake_router_artifact(), artifact_path)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 228 |
(model_dir / "router_selection.json").write_text(
|
| 229 |
json.dumps({"selected_artifact": artifact_path.name}),
|
| 230 |
encoding="utf-8",
|
|
|
|
| 250 |
model_dir = Path(temp_dir)
|
| 251 |
artifact_path = model_dir / "router.joblib"
|
| 252 |
selection_path = model_dir / "router_selection.json"
|
| 253 |
+
joblib.dump(_fake_router_artifact(), artifact_path)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 254 |
selection_path.write_text(
|
| 255 |
json.dumps({"selected_artifact": artifact_path.name}),
|
| 256 |
encoding="utf-8",
|
|
|
|
| 277 |
model_dir = Path(temp_dir)
|
| 278 |
artifact_path = model_dir / "router.joblib"
|
| 279 |
selection_path = model_dir / "router_selection.json"
|
| 280 |
+
joblib.dump(_fake_router_artifact(), artifact_path)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 281 |
selection_path.write_text(
|
| 282 |
json.dumps({"selected_artifact": artifact_path.name}),
|
| 283 |
encoding="utf-8",
|
|
|
|
| 302 |
self.assertEqual(bundle.model_kind, "baseline")
|
| 303 |
self.assertEqual(bundle.labels, tuple(ROUTER_LABELS))
|
| 304 |
|
| 305 |
+
def test_old_router_artifact_without_schema_is_rejected(self) -> None:
|
| 306 |
+
with tempfile.TemporaryDirectory() as temp_dir:
|
| 307 |
+
model_dir = Path(temp_dir)
|
| 308 |
+
joblib.dump(
|
| 309 |
+
{
|
| 310 |
+
"model": _FakePushUpModel(),
|
| 311 |
+
"labels": list(ROUTER_LABELS),
|
| 312 |
+
"model_kind": "baseline",
|
| 313 |
+
},
|
| 314 |
+
model_dir / "router.joblib",
|
| 315 |
+
)
|
| 316 |
+
|
| 317 |
+
with patch.dict(os.environ, {HF_DISABLE_ENV: "1"}):
|
| 318 |
+
with self.assertRaises(ValueError):
|
| 319 |
+
load_router_model(model_dir)
|
| 320 |
+
|
| 321 |
|
| 322 |
class ExerciseClassifierStepTests(unittest.TestCase):
|
| 323 |
def test_manual_override_bypasses_model_and_validates_contract(self) -> None:
|
tests/test_pose_steps.py
CHANGED
|
@@ -81,7 +81,7 @@ class PoseStepTests(unittest.TestCase):
|
|
| 81 |
writer.release()
|
| 82 |
return path
|
| 83 |
|
| 84 |
-
def
|
| 85 |
path = self._write_video()
|
| 86 |
manifest = VideoManifest(
|
| 87 |
video_path=str(path),
|
|
@@ -101,10 +101,12 @@ class PoseStepTests(unittest.TestCase):
|
|
| 101 |
sequence = pose_landmarker.run(manifest, backend=_FakePose())
|
| 102 |
|
| 103 |
self.assertEqual(len(sequence.frames), 4)
|
| 104 |
-
self.assertEqual(len(sequence.frames[0].landmarks),
|
| 105 |
-
self.assertIn("
|
| 106 |
-
self.
|
|
|
|
| 107 |
self.assertEqual(sequence.frames[0].pose_quality["source"], "fake_pose")
|
|
|
|
| 108 |
self.assertGreater(sequence.frames[0].pose_quality["mean_visibility"], 0.9)
|
| 109 |
self.assertTrue(sequence.frames[0].pose_quality["critical_landmarks_visible"])
|
| 110 |
self.assertEqual(sequence.pose_valid_ratio, 1.0)
|
|
@@ -184,9 +186,9 @@ class PoseStepTests(unittest.TestCase):
|
|
| 184 |
last = landmark_list_to_dict(last_landmarks)
|
| 185 |
sequence = PoseSequence(
|
| 186 |
frames=[
|
| 187 |
-
PoseFrame(0, 0.0, first,
|
| 188 |
PoseFrame(1, 0.033, {}, {}, {"mean_visibility": 0.0}),
|
| 189 |
-
PoseFrame(2, 0.067, last,
|
| 190 |
],
|
| 191 |
normalized=False,
|
| 192 |
smoothing_method="none",
|
|
@@ -204,6 +206,8 @@ class PoseStepTests(unittest.TestCase):
|
|
| 204 |
self.assertIn("smoothed_x", shoulder)
|
| 205 |
self.assertIn("normalized_x", shoulder)
|
| 206 |
self.assertTrue(cleaned.frames[1].pose_quality["normalized"])
|
|
|
|
|
|
|
| 207 |
|
| 208 |
@unittest.skipUnless(
|
| 209 |
os.getenv("POZIFY_RUN_REAL_POSE_TESTS") == "1",
|
|
@@ -222,7 +226,7 @@ class PoseStepTests(unittest.TestCase):
|
|
| 222 |
self.assertTrue(manifest.analysis_allowed)
|
| 223 |
self.assertGreater(len(sequence.frames), 0)
|
| 224 |
self.assertGreater(sequence.pose_valid_ratio, 0.0)
|
| 225 |
-
self.assertEqual(len(sequence.frames[0].landmarks),
|
| 226 |
self.assertEqual(sequence.frames[0].pose_quality["source"], "mediapipe_pose")
|
| 227 |
self.assertTrue(cleaned.normalized)
|
| 228 |
|
|
|
|
| 81 |
writer.release()
|
| 82 |
return path
|
| 83 |
|
| 84 |
+
def test_pose_landmarker_maps_to_coco17_landmarks_and_quality(self) -> None:
|
| 85 |
path = self._write_video()
|
| 86 |
manifest = VideoManifest(
|
| 87 |
video_path=str(path),
|
|
|
|
| 101 |
sequence = pose_landmarker.run(manifest, backend=_FakePose())
|
| 102 |
|
| 103 |
self.assertEqual(len(sequence.frames), 4)
|
| 104 |
+
self.assertEqual(len(sequence.frames[0].landmarks), 17)
|
| 105 |
+
self.assertIn("left_ankle", sequence.frames[0].landmarks)
|
| 106 |
+
self.assertNotIn("left_foot_index", sequence.frames[0].landmarks)
|
| 107 |
+
self.assertEqual(len(sequence.frames[0].world_landmarks), 17)
|
| 108 |
self.assertEqual(sequence.frames[0].pose_quality["source"], "fake_pose")
|
| 109 |
+
self.assertEqual(sequence.frames[0].pose_quality["landmark_schema"], "coco17")
|
| 110 |
self.assertGreater(sequence.frames[0].pose_quality["mean_visibility"], 0.9)
|
| 111 |
self.assertTrue(sequence.frames[0].pose_quality["critical_landmarks_visible"])
|
| 112 |
self.assertEqual(sequence.pose_valid_ratio, 1.0)
|
|
|
|
| 186 |
last = landmark_list_to_dict(last_landmarks)
|
| 187 |
sequence = PoseSequence(
|
| 188 |
frames=[
|
| 189 |
+
PoseFrame(0, 0.0, first, first, {"mean_visibility": 0.9}),
|
| 190 |
PoseFrame(1, 0.033, {}, {}, {"mean_visibility": 0.0}),
|
| 191 |
+
PoseFrame(2, 0.067, last, last, {"mean_visibility": 0.9}),
|
| 192 |
],
|
| 193 |
normalized=False,
|
| 194 |
smoothing_method="none",
|
|
|
|
| 206 |
self.assertIn("smoothed_x", shoulder)
|
| 207 |
self.assertIn("normalized_x", shoulder)
|
| 208 |
self.assertTrue(cleaned.frames[1].pose_quality["normalized"])
|
| 209 |
+
self.assertIn("smoothed_x", cleaned.frames[1].world_landmarks["left_shoulder"])
|
| 210 |
+
self.assertIn("normalized_z", cleaned.frames[1].world_landmarks["left_shoulder"])
|
| 211 |
|
| 212 |
@unittest.skipUnless(
|
| 213 |
os.getenv("POZIFY_RUN_REAL_POSE_TESTS") == "1",
|
|
|
|
| 226 |
self.assertTrue(manifest.analysis_allowed)
|
| 227 |
self.assertGreater(len(sequence.frames), 0)
|
| 228 |
self.assertGreater(sequence.pose_valid_ratio, 0.0)
|
| 229 |
+
self.assertEqual(len(sequence.frames[0].landmarks), 17)
|
| 230 |
self.assertEqual(sequence.frames[0].pose_quality["source"], "mediapipe_pose")
|
| 231 |
self.assertTrue(cleaned.normalized)
|
| 232 |
|
tests/test_rep_counter.py
CHANGED
|
@@ -12,6 +12,7 @@ from pozify.exercises import create_exercise_strategy
|
|
| 12 |
from pozify.exercises.push_up import PushUpExercise
|
| 13 |
from pozify.exercises.shoulder_press import ShoulderPressExercise
|
| 14 |
from pozify.exercises.squat import SquatExercise
|
|
|
|
| 15 |
|
| 16 |
|
| 17 |
def _frame(frame_index: int, landmarks: dict[str, dict[str, float]]) -> PoseFrame:
|
|
@@ -145,6 +146,21 @@ def _exercise_strategy(exercise: str, sequence: PoseSequence):
|
|
| 145 |
|
| 146 |
|
| 147 |
class RepCounterTests(unittest.TestCase):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
def test_segments_squat_reps(self) -> None:
|
| 149 |
sequence = _sequence_for_exercise("squat", 3)
|
| 150 |
reps, debug = _exercise_strategy("squat", sequence).count()
|
|
|
|
| 12 |
from pozify.exercises.push_up import PushUpExercise
|
| 13 |
from pozify.exercises.shoulder_press import ShoulderPressExercise
|
| 14 |
from pozify.exercises.squat import SquatExercise
|
| 15 |
+
from pozify.steps.rep_signals import angle_deg
|
| 16 |
|
| 17 |
|
| 18 |
def _frame(frame_index: int, landmarks: dict[str, dict[str, float]]) -> PoseFrame:
|
|
|
|
| 146 |
|
| 147 |
|
| 148 |
class RepCounterTests(unittest.TestCase):
|
| 149 |
+
def test_angle_deg_uses_3d_world_landmarks(self) -> None:
|
| 150 |
+
frame = PoseFrame(
|
| 151 |
+
frame_index=0,
|
| 152 |
+
timestamp_sec=0.0,
|
| 153 |
+
landmarks={},
|
| 154 |
+
world_landmarks={
|
| 155 |
+
"left_shoulder": {"x": 1.0, "y": 0.0, "z": 0.0},
|
| 156 |
+
"left_elbow": {"x": 0.0, "y": 0.0, "z": 0.0},
|
| 157 |
+
"left_wrist": {"x": 0.0, "y": 0.0, "z": 1.0},
|
| 158 |
+
},
|
| 159 |
+
pose_quality={"mean_visibility": 1.0},
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
self.assertEqual(round(angle_deg(frame, "left_shoulder", "left_elbow", "left_wrist") or 0.0), 90)
|
| 163 |
+
|
| 164 |
def test_segments_squat_reps(self) -> None:
|
| 165 |
sequence = _sequence_for_exercise("squat", 3)
|
| 166 |
reps, debug = _exercise_strategy("squat", sequence).count()
|
web/app.js
CHANGED
|
@@ -912,7 +912,7 @@ function App() {
|
|
| 912 |
h(
|
| 913 |
"div",
|
| 914 |
{ className: "metric" },
|
| 915 |
-
h("strong", null, "
|
| 916 |
h("span", null, "pose landmarks"),
|
| 917 |
),
|
| 918 |
h(
|
|
|
|
| 912 |
h(
|
| 913 |
"div",
|
| 914 |
{ className: "metric" },
|
| 915 |
+
h("strong", null, "17"),
|
| 916 |
h("span", null, "pose landmarks"),
|
| 917 |
),
|
| 918 |
h(
|