File size: 12,488 Bytes
8d7b4b1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
"""Feature validation and train/inference consistency checking.

This module ensures that training and inference pipelines produce identical
feature representations.
"""

from __future__ import annotations

from pathlib import Path
from typing import Dict, List, Optional, Tuple
import numpy as np
import torch
from dataclasses import dataclass

from src.features.specs import FeatureSpecification, ModalitySpec
from src.features.preprocessing import FeaturePreprocessor, PCATransform, FeatureNormalizer


@dataclass
class FeatureValidationResult:
    """Result of feature validation."""

    is_valid: bool
    errors: List[str]
    warnings: List[str]
    feature_dims: Dict[str, int]

    def report(self) -> str:
        """Generate validation report."""
        lines = []
        lines.append("=" * 60)
        lines.append("FEATURE VALIDATION REPORT")
        lines.append("=" * 60)
        
        if self.is_valid:
            lines.append("✓ All validations passed")
        else:
            lines.append("✗ Validation failed")
            for error in self.errors:
                lines.append(f"  ERROR: {error}")
        
        if self.warnings:
            lines.append("\nWarnings:")
            for warning in self.warnings:
                lines.append(f"  WARNING: {warning}")
        
        lines.append("\nFeature Dimensions:")
        for modality, dim in self.feature_dims.items():
            lines.append(f"  {modality}: {dim}")
        
        lines.append("=" * 60)
        return "\n".join(lines)


class FeatureValidator:
    """Validate features match specification."""

    def __init__(self, spec: FeatureSpecification):
        """Initialize validator.

        Args:
            spec: Feature specification to validate against
        """
        self.spec = spec

    def validate_single_modality(
        self,
        features: np.ndarray,
        modality: str,
    ) -> Tuple[bool, List[str], List[str]]:
        """Validate single modality features.

        Args:
            features: Feature array
            modality: Modality name ("audio", "video", "text")

        Returns:
            Tuple of (is_valid, errors, warnings)
        """
        errors = []
        warnings = []

        try:
            modal_spec = self.spec.get_spec(modality)
        except ValueError:
            errors.append(f"Unknown modality: {modality}")
            return False, errors, warnings

        # Check dtype
        if features.dtype != np.float32:
            errors.append(f"{modality}: dtype is {features.dtype}, expected float32")

        # Check dimensionality
        if modal_spec.temporal:
            if features.ndim != 2:
                errors.append(f"{modality}: expected 2D array [time, dim], got {features.ndim}D")
            else:
                if features.shape[1] != modal_spec.expected_dim:
                    errors.append(
                        f"{modality}: dimension mismatch. "
                        f"Expected {modal_spec.expected_dim}, got {features.shape[1]}"
                    )
                if features.shape[0] == 0:
                    errors.append(f"{modality}: zero-length temporal dimension")
        else:
            if features.ndim != 1:
                errors.append(f"{modality}: expected 1D array, got {features.ndim}D")
            if features.shape[0] != modal_spec.expected_dim:
                errors.append(
                    f"{modality}: dimension mismatch. "
                    f"Expected {modal_spec.expected_dim}, got {features.shape[0]}"
                )

        # Check for NaN/Inf
        if np.isnan(features).any():
            errors.append(f"{modality}: contains NaN values")
        if np.isinf(features).any():
            errors.append(f"{modality}: contains Inf values")

        # Warnings for extreme values
        if np.abs(features).max() > 100:
            warnings.append(f"{modality}: contains very large values (max={np.abs(features).max():.2f})")
        if np.abs(features[features != 0]).min() < 1e-6:
            warnings.append(f"{modality}: contains very small non-zero values")

        is_valid = len(errors) == 0
        return is_valid, errors, warnings

    def validate_batch(
        self,
        features: Dict[str, np.ndarray],
    ) -> FeatureValidationResult:
        """Validate complete batch of features.

        Args:
            features: Dict with "audio", "video", "text" keys

        Returns:
            FeatureValidationResult with detailed report
        """
        all_errors = []
        all_warnings = []
        feature_dims = {}

        for modality in ["audio", "video", "text"]:
            if modality in features and features[modality] is not None:
                array = features[modality]
                is_valid, errors, warnings = self.validate_single_modality(array, modality)
                all_errors.extend(errors)
                all_warnings.extend(warnings)
                
                if array.ndim == 2:
                    feature_dims[modality] = array.shape[1]
                elif array.ndim == 1:
                    feature_dims[modality] = array.shape[0]

        is_valid = len(all_errors) == 0
        return FeatureValidationResult(
            is_valid=is_valid,
            errors=all_errors,
            warnings=all_warnings,
            feature_dims=feature_dims,
        )


class TrainInferenceConsistencyChecker:
    """Check that training and inference use consistent features."""

    def __init__(
        self,
        spec: FeatureSpecification,
        train_preprocessors: Optional[Dict[str, FeaturePreprocessor]] = None,
        inference_preprocessors: Optional[Dict[str, FeaturePreprocessor]] = None,
    ):
        """Initialize consistency checker.

        Args:
            spec: Feature specification
            train_preprocessors: Preprocessors used during training
            inference_preprocessors: Preprocessors used during inference
        """
        self.spec = spec
        self.train_preprocessors = train_preprocessors or {}
        self.inference_preprocessors = inference_preprocessors or {}
        self.validator = FeatureValidator(spec)

    def check_preprocessor_compatibility(self) -> Tuple[bool, List[str]]:
        """Check that training and inference preprocessors are compatible.

        Returns:
            Tuple of (is_compatible, issues)
        """
        issues = []

        for modality in ["audio", "video", "text"]:
            train_prep = self.train_preprocessors.get(modality)
            infer_prep = self.inference_preprocessors.get(modality)

            # Both should exist if either exists
            if (train_prep is None) != (infer_prep is None):
                issues.append(
                    f"{modality}: preprocessor exists in training but not in "
                    f"inference (or vice versa)"
                )
                continue

            # If both exist, check compatibility
            if train_prep and infer_prep:
                if train_prep.normalizer_type != infer_prep.normalizer_type:
                    issues.append(
                        f"{modality}: different normalizer types "
                        f"(train={train_prep.normalizer_type}, "
                        f"infer={infer_prep.normalizer_type})"
                    )

                if train_prep.apply_pca != infer_prep.apply_pca:
                    issues.append(
                        f"{modality}: PCA enabled in training but not in inference "
                        f"(or vice versa)"
                    )

                if train_prep.apply_pca and infer_prep.apply_pca:
                    if train_prep.pca_components != infer_prep.pca_components:
                        issues.append(
                            f"{modality}: different PCA components "
                            f"(train={train_prep.pca_components}, "
                            f"infer={infer_prep.pca_components})"
                        )

        is_compatible = len(issues) == 0
        return is_compatible, issues

    def validate_training_batch(self, batch: Dict[str, np.ndarray]) -> FeatureValidationResult:
        """Validate training batch."""
        return self.validator.validate_batch(batch)

    def validate_inference_batch(self, batch: Dict[str, np.ndarray]) -> FeatureValidationResult:
        """Validate inference batch."""
        return self.validator.validate_batch(batch)

    def generate_report(self) -> str:
        """Generate comprehensive consistency report."""
        lines = []
        lines.append("\n" + "=" * 80)
        lines.append("TRAIN/INFERENCE CONSISTENCY CHECK")
        lines.append("=" * 80)

        # Check preprocessor compatibility
        is_compatible, issues = self.check_preprocessor_compatibility()
        if is_compatible:
            lines.append("\n✓ Preprocessors are compatible between training and inference")
        else:
            lines.append("\n✗ Preprocessor compatibility issues found:")
            for issue in issues:
                lines.append(f"  - {issue}")

        # Feature spec
        lines.append(f"\nFeature Specification:")
        lines.append(f"  Audio: {self.spec.audio_dim}D")
        lines.append(f"  Video: {self.spec.video_dim}D")
        lines.append(f"  Text: {self.spec.text_dim}D")

        lines.append("\n" + "=" * 80)
        return "\n".join(lines)


class FeatureCheckpoint:
    """Save and restore preprocessors for exact reproducibility."""

    def __init__(self, checkpoint_dir: Path):
        """Initialize checkpoint manager.

        Args:
            checkpoint_dir: Directory for storing checkpoints
        """
        self.checkpoint_dir = Path(checkpoint_dir)
        self.checkpoint_dir.mkdir(parents=True, exist_ok=True)

    def save_preprocessors(
        self,
        preprocessors: Dict[str, FeaturePreprocessor],
        experiment_name: str,
    ) -> None:
        """Save preprocessors for later use.

        Args:
            preprocessors: Dict of modality -> preprocessor
            experiment_name: Name for this experiment
        """
        exp_dir = self.checkpoint_dir / experiment_name / "preprocessors"
        exp_dir.mkdir(parents=True, exist_ok=True)

        for modality, prep in preprocessors.items():
            prep_path = exp_dir / f"{modality}_preprocessor.pkl"
            prep.save(prep_path)

    def load_preprocessors(
        self,
        experiment_name: str,
    ) -> Dict[str, FeaturePreprocessor]:
        """Load preprocessors from checkpoint.

        Args:
            experiment_name: Experiment name

        Returns:
            Dict of modality -> preprocessor
        """
        exp_dir = self.checkpoint_dir / experiment_name / "preprocessors"
        
        if not exp_dir.exists():
            raise FileNotFoundError(f"Preprocessor checkpoint not found: {exp_dir}")

        preprocessors = {}
        for modality in ["audio", "video", "text"]:
            prep_path = exp_dir / f"{modality}_preprocessor.pkl"
            if prep_path.exists():
                preprocessors[modality] = FeaturePreprocessor.load(prep_path)

        return preprocessors

    def save_feature_spec(
        self,
        spec: FeatureSpecification,
        experiment_name: str,
    ) -> None:
        """Save feature specification."""
        import json
        
        exp_dir = self.checkpoint_dir / experiment_name
        exp_dir.mkdir(parents=True, exist_ok=True)
        
        spec_path = exp_dir / "feature_spec.json"
        with open(spec_path, "w") as f:
            json.dump(spec.to_dict(), f, indent=2)

    def load_feature_spec(self, experiment_name: str) -> FeatureSpecification:
        """Load feature specification."""
        import json
        
        spec_path = self.checkpoint_dir / experiment_name / "feature_spec.json"
        if not spec_path.exists():
            raise FileNotFoundError(f"Feature spec not found: {spec_path}")
        
        with open(spec_path, "r") as f:
            spec_dict = json.load(f)
        
        # Reconstruct FeatureSpecification from dict
        from src.features.specs import ModalitySpec
        
        audio_spec = ModalitySpec.from_dict(spec_dict["audio"])
        video_spec = ModalitySpec.from_dict(spec_dict["video"])
        text_spec = ModalitySpec.from_dict(spec_dict["text"])
        
        return FeatureSpecification(audio_spec, video_spec, text_spec)