Download scripts/third_party_control_refit.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 13.1 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/third_party_control_refit.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/third_party_control_refit.py
-
curl -L -o third_party_control_refit.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/third_party_control_refit.py
13.1 kB
| """Bounded control-head-only experiments on deployment-aligned frozen features. | |
| This module never changes the parent model, backbone, audio, or GT. Results are | |
| training-set diagnostics, not held-out generalization or complete checkpoints. | |
| """ | |
| from __future__ import annotations | |
| import copy | |
| import math | |
| from typing import Any | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| def _normal_tensor(value, *, dtype): | |
| # clone outside inference_mode produces tensors safe to save for backward. | |
| with torch.inference_mode(False): | |
| return torch.as_tensor(value).detach().to(device="cpu", dtype=dtype).clone() | |
| def _targets(labels, background_nonmotion_mask, sample_weights=None): | |
| raw = _normal_tensor(labels, dtype=torch.float64) | |
| if raw.ndim != 1 or not raw.numel() or not torch.isfinite(raw).all(): | |
| raise ValueError("Labels must be a finite nonempty vector") | |
| if not torch.all((raw >= 0) & (raw <= 2) & (raw == raw.round())): | |
| raise ValueError("Labels must be integer listen/motion/idle IDs 0..2") | |
| labels = raw.long() | |
| raw_mask = _normal_tensor(background_nonmotion_mask, dtype=torch.float64) | |
| if raw_mask.shape != labels.shape or not torch.all((raw_mask == 0) | (raw_mask == 1)): | |
| raise ValueError("Background mask must have one boolean value per label") | |
| mask = raw_mask.bool() | |
| if torch.any(mask & labels.eq(1)): | |
| raise ValueError("Background nonmotion mask cannot include GT motion") | |
| weights = (torch.ones(len(labels), dtype=torch.float32) if sample_weights is None | |
| else _normal_tensor(sample_weights, dtype=torch.float32)) | |
| if (weights.shape != labels.shape or not torch.isfinite(weights).all() | |
| or torch.any(weights < 0) or weights.sum() <= 0): | |
| raise ValueError("Sample weights must be finite, nonnegative, and have positive total") | |
| return labels, mask, weights | |
| def score_control_predictions(logits_or_ids, labels, background_nonmotion_mask, | |
| sample_weights=None) -> dict[str, Any]: | |
| """Score sample/frame-weighted control only; no pose or causal attribution.""" | |
| labels, background, weights = _targets(labels, background_nonmotion_mask, sample_weights) | |
| values = _normal_tensor(logits_or_ids, dtype=torch.float32) | |
| if not torch.isfinite(values).all(): | |
| raise ValueError("Predictions must be finite") | |
| if values.ndim == 2 and values.shape == (len(labels), 3): | |
| predictions = values.argmax(-1) | |
| elif values.shape == labels.shape and torch.all((values >= 0) & (values <= 2) & (values == values.round())): | |
| predictions = values.long() | |
| else: | |
| raise ValueError("Expected N by 3 logits or N integer predictions") | |
| confusion = torch.zeros(9, dtype=torch.float64) | |
| confusion.scatter_add_(0, labels * 3 + predictions, weights.double()) | |
| confusion = confusion.reshape(3, 3) | |
| total = float(weights.sum()) | |
| denominator = float(weights[background].sum()) | |
| numerator = float(weights[background & predictions.eq(1)].sum()) | |
| recalls = {} | |
| for index, name in enumerate(("listen", "motion", "idle")): | |
| support = float(confusion[index].sum()) | |
| recalls[name] = float(confusion[index, index] / support) if support else None | |
| return {"count": len(labels), "weight_sum": total, | |
| "control_accuracy": float(weights[predictions.eq(labels)].sum()) / total, | |
| "control_confusion_rows_gt_columns_prediction": confusion.tolist(), | |
| "class_recall": recalls, | |
| "background_nonmotion_weight": denominator, | |
| "background_false_motion_weight": numerator, | |
| "background_false_motion_rate": numerator / denominator if denominator else None, | |
| "predicted_motion_fraction": float(weights[predictions.eq(1)].sum()) / total} | |
| def balanced_control_weights(labels, background_nonmotion_mask, sample_weights=None): | |
| """Moderate sqrt inverse-frequency balancing plus 2x background negatives.""" | |
| labels, background, weights = _targets(labels, background_nonmotion_mask, sample_weights) | |
| counts = torch.zeros(3).scatter_add_(0, labels, weights) | |
| present = counts > 0 | |
| factors = torch.ones(3) | |
| factors[present] = torch.sqrt(counts[present].mean() / counts[present]).clamp(0.5, 3.0) | |
| result = weights * factors[labels] * torch.where(background, 2.0, 1.0) | |
| return result / result.mean() | |
| def _logits(head, features, device, batch_size, *, deployment_autocast=False): | |
| head.eval() | |
| with torch.no_grad(), torch.autocast(device_type=device.type, dtype=torch.bfloat16, | |
| enabled=deployment_autocast): | |
| return torch.cat([head(chunk.to(device)).float().cpu() | |
| for chunk in features.split(batch_size)]) | |
| def score_deployment_head(head, features, labels, background_nonmotion_mask, *, | |
| sample_weights=None, device="cpu", batch_size=512, | |
| deployment_autocast=None): | |
| """Score a copied head using deployment BF16 autocast on CUDA by default. | |
| Returns metrics and CPU FP32 logits (values may have been BF16-computed). | |
| This is control-only scoring, not a new motion rollout. | |
| """ | |
| if not isinstance(batch_size, int) or batch_size <= 0: | |
| raise ValueError("Positive batch size required") | |
| device = torch.device(device) | |
| if deployment_autocast is None: | |
| deployment_autocast = device.type == "cuda" | |
| features = _normal_tensor(features, dtype=torch.float32) | |
| labels, mask, weights = _targets(labels, background_nonmotion_mask, sample_weights) | |
| if features.ndim != 2 or len(features) != len(labels) or not torch.isfinite(features).all(): | |
| raise ValueError("Features must be finite N by D and aligned with labels") | |
| copied = copy.deepcopy(head).to(device=device, dtype=torch.float32).eval() | |
| logits = _logits(copied, features, device, batch_size, | |
| deployment_autocast=bool(deployment_autocast)) | |
| return {"metrics": score_control_predictions(logits, labels, mask, weights), | |
| "logits": logits, | |
| "precision": "bf16_autocast" if deployment_autocast else "float32"} | |
| def _state(head, original_state): | |
| return {name: tensor.detach().cpu().to(dtype=original_state[name].dtype).clone() | |
| for name, tensor in head.state_dict().items()} | |
| def recommend_control_candidate(baseline, candidates, recall_tolerance=0.02): | |
| """Conservative training diagnostic recommendation, never a deployment gate. | |
| Require improved background control, no >2pp motion/listen recall loss, and | |
| no >1pp global accuracy loss. Absence of background support retains baseline. | |
| """ | |
| reference = baseline["metrics"] | |
| if reference["background_false_motion_rate"] is None: | |
| return "baseline" | |
| accepted = [] | |
| for name, candidate in candidates.items(): | |
| metric = candidate["metrics"] | |
| if metric["background_false_motion_rate"] is None: | |
| continue | |
| if metric["background_false_motion_rate"] >= reference["background_false_motion_rate"] - 1e-6: | |
| continue | |
| if metric["control_accuracy"] < reference["control_accuracy"] - 0.01: | |
| continue | |
| if any(reference["class_recall"][label] is not None and | |
| (metric["class_recall"][label] is None or | |
| metric["class_recall"][label] < reference["class_recall"][label] - recall_tolerance) | |
| for label in ("listen", "motion")): | |
| continue | |
| accepted.append((metric["background_false_motion_rate"], -metric["control_accuracy"], name)) | |
| return min(accepted)[2] if accepted else "baseline" | |
| def fit_control_candidates(base_head: nn.Module, features, labels, background_nonmotion_mask, | |
| *, sample_weights=None, device="cpu", seed=26100607, | |
| epochs=20, batch_size=512, learning_rate=2e-4): | |
| """Fit two independent copied heads; return CPU states, histories and scores. | |
| `features` must be the pre-control-head N x D streamed features of one fixed | |
| backbone checkpoint. Sample weights may encode frame duration. Every epoch | |
| consumes every feature once. No backbone, action or duration parameter is | |
| touched, and no decision threshold or GT context is introduced. | |
| """ | |
| if (not isinstance(epochs, int) or epochs <= 0 or not isinstance(batch_size, int) | |
| or batch_size <= 0 or not math.isfinite(learning_rate) or learning_rate <= 0): | |
| raise ValueError("Positive epochs, batch size and learning rate required") | |
| features = _normal_tensor(features, dtype=torch.float32) | |
| labels, background, weights = _targets(labels, background_nonmotion_mask, sample_weights) | |
| if features.ndim != 2 or features.shape[0] != len(labels) or features.shape[1] == 0 or not torch.isfinite(features).all(): | |
| raise ValueError("Features must be finite N by D and aligned with labels") | |
| if any(not torch.isfinite(value).all() for value in base_head.state_dict().values()): | |
| raise ValueError("Parent head contains nonfinite state") | |
| original_state = {name: value.detach().cpu().clone() for name, value in base_head.state_dict().items()} | |
| device = torch.device(device) | |
| baseline_head = copy.deepcopy(base_head).to(device=device, dtype=torch.float32).eval() | |
| try: | |
| baseline_logits = _logits(baseline_head, features, device, batch_size) | |
| except (RuntimeError, TypeError) as error: | |
| raise ValueError("Features are incompatible with the control head") from error | |
| if baseline_logits.shape != (len(labels), 3): | |
| raise ValueError("Control head must produce three logits per feature") | |
| baseline = {"state_dict": original_state, | |
| "metrics": score_control_predictions(baseline_logits, labels, background, weights)} | |
| del baseline_head | |
| candidate_weights = {"uniform_ce": weights / weights.mean(), | |
| "balanced_ce": balanced_control_weights(labels, background, weights)} | |
| candidates = {} | |
| # fork_rng avoids changing the caller's random state; each branch also uses | |
| # an independent CPU generator so its sample permutations are identical. | |
| cuda_devices = [device.index if device.index is not None else torch.cuda.current_device()] if device.type == "cuda" else [] | |
| for name, objective_weights in candidate_weights.items(): | |
| with torch.random.fork_rng(devices=cuda_devices): | |
| torch.manual_seed(int(seed)) | |
| head = copy.deepcopy(base_head).to(device=device, dtype=torch.float32) | |
| head.requires_grad_(True) | |
| optimizer = torch.optim.AdamW(head.parameters(), lr=learning_rate, weight_decay=0.0) | |
| generator = torch.Generator(device="cpu").manual_seed(int(seed)) | |
| history = [] | |
| for epoch in range(epochs): | |
| head.train() | |
| permutation = torch.randperm(len(labels), generator=generator) | |
| loss_sum = weight_sum = 0.0 | |
| for indices in permutation.split(batch_size): | |
| local_weights = objective_weights[indices].to(device) | |
| if local_weights.sum() <= 0: | |
| continue | |
| optimizer.zero_grad(set_to_none=True) | |
| logits = head(features[indices].to(device)) | |
| losses = F.cross_entropy(logits, labels[indices].to(device), reduction="none") | |
| loss = (losses * local_weights).sum() / local_weights.sum() | |
| if not torch.isfinite(loss): | |
| raise FloatingPointError(f"Nonfinite {name} control loss") | |
| loss.backward() | |
| norm = nn.utils.clip_grad_norm_(head.parameters(), 1.0, error_if_nonfinite=True) | |
| optimizer.step() | |
| loss_sum += float((losses.detach() * local_weights).sum()) | |
| weight_sum += float(local_weights.sum()) | |
| logits = _logits(head, features, device, batch_size) | |
| metrics = score_control_predictions(logits, labels, background, weights) | |
| history.append({"epoch": epoch + 1, "weighted_ce": loss_sum / weight_sum, | |
| "last_grad_norm": float(norm), **metrics}) | |
| candidates[name] = {"state_dict": _state(head, original_state), "metrics": metrics, | |
| "history": history, "epochs": epochs} | |
| return {"baseline": baseline, "candidates": candidates, | |
| "recommended_name": recommend_control_candidate(baseline, candidates), | |
| "selection_policy": "training-only; background improvement, listen/motion recall loss <=2pp, accuracy loss <=1pp", | |
| "config": {"seed": int(seed), "epochs": epochs, "batch_size": batch_size, | |
| "learning_rate": learning_rate, "feature_count": len(labels), | |
| "trainable_scope": "copied_control_head_only", "precision": "float32", | |
| "balanced_class_weight": "sqrt inverse frequency clipped [0.5,3]", | |
| "background_nonmotion_boost": 2.0}} | |