Buckets:
| """Read-only adapter for existing real evaluation.npz mappings.""" | |
| import numpy as np | |
| from .composition import paired_composition_bootstrap | |
| def bootstrap_composition_evaluations(evaluations_a, evaluations_b, *, seed_ids_a, seed_ids_b, **kwargs): | |
| """Validate saved real-evaluation axes, then bootstrap B - A. | |
| Pass lists of open np.load(..., allow_pickle=False) mappings in seed order. | |
| No files are opened or written here. Seeds come from caller-verified run | |
| metadata, because evaluation.npz does not itself store seed IDs. | |
| This adapter intentionally rejects synthetic cross-probe pair artifacts. | |
| """ | |
| left, right = list(evaluations_a), list(evaluations_b) | |
| if not left or len(left) != len(right): | |
| raise ValueError("evaluation lists must have the same nonzero seed count") | |
| reference = left[0] | |
| shared = ("composition_pairs", "composition_prompt_ids", "composition_weights", "test_prompt_ids", "attributes") | |
| for i, evaluation in enumerate(left + right): | |
| for key in (*shared, "composition_pair_correct", "composition_pair_valid"): | |
| if key not in evaluation: | |
| raise ValueError(f"evaluation {i} missing {key}; real same-prompt artifacts required") | |
| for key in shared: | |
| if not np.array_equal(reference[key], evaluation[key]): | |
| raise ValueError(f"evaluation {i} axis/data alignment error in {key}") | |
| pairs = np.asarray(reference["composition_pairs"]) | |
| pids = np.asarray(reference["composition_prompt_ids"]) | |
| test_pids = np.asarray(reference["test_prompt_ids"]) | |
| attributes = np.asarray(reference["attributes"]) | |
| weights = np.asarray(reference["composition_weights"]) | |
| if test_pids.ndim != 1 or attributes.ndim != 2 or len(attributes) != len(test_pids): | |
| raise ValueError("TEST prompt/attribute response axis mismatch") | |
| if pairs.ndim != 2 or pairs.shape[1] != 2 or pairs.dtype.kind not in "iu": | |
| raise ValueError("composition_pairs must be an integer [pair, 2] endpoint array") | |
| if pids.shape != (len(pairs),) or (pairs < 0).any() or (pairs >= len(test_pids)).any(): | |
| raise ValueError("composition pair endpoints or prompt axis are invalid") | |
| if (pairs[:, 0] == pairs[:, 1]).any(): | |
| raise ValueError("self-pairs are not within-prompt response comparisons") | |
| if not np.array_equal(test_pids[pairs[:, 0]], pids) or not np.array_equal(test_pids[pairs[:, 1]], pids): | |
| raise ValueError("composition pairs must have both endpoints in the stated prompt") | |
| if len(np.unique(np.sort(pairs, axis=1), axis=0)) != len(pairs): | |
| raise ValueError("duplicate or reversed duplicate response pairs") | |
| # Real evaluation enumerates *all* unordered within-prompt response pairs. | |
| _, prompt_sizes = np.unique(test_pids, return_counts=True) | |
| if int((prompt_sizes * (prompt_sizes - 1) // 2).sum()) != len(pairs): | |
| raise ValueError("composition_pairs must contain all within-prompt TEST pairs") | |
| shape = np.asarray(reference["composition_pair_correct"]).shape | |
| if len(shape) != 2 or shape[0] != len(pairs) or weights.ndim != 2 or weights.shape != (shape[1], attributes.shape[1]): | |
| raise ValueError("pair/rubric/attribute axis dimensions do not match") | |
| if not np.isfinite(weights).all() or not np.isfinite(attributes).all(): | |
| raise ValueError("rubric weights and human attributes must be finite") | |
| result = paired_composition_bootstrap( | |
| np.stack([e["composition_pair_correct"] for e in left]), | |
| np.stack([e["composition_pair_correct"] for e in right]), | |
| np.stack([e["composition_pair_valid"] for e in left]), | |
| np.stack([e["composition_pair_valid"] for e in right]), | |
| pids, seed_ids_a=seed_ids_a, seed_ids_b=seed_ids_b, | |
| pair_ids_a=np.arange(len(pairs)), pair_ids_b=np.arange(len(pairs)), | |
| rubric_ids_a=np.arange(len(weights)), rubric_ids_b=np.arange(len(weights)), | |
| pair_prompt_ids_b=right[0]["composition_prompt_ids"], **kwargs, | |
| ) | |
| result["alignment_checked"]["all_evaluation_pairs_weights_prompts_attributes"] = True | |
| result["assumptions"].append("Adapter verified all unordered within-prompt pairs and shared weights/attributes; caller must verify run seeds and TEST response IDs in predictions.npz.") | |
| return result | |
Xet Storage Details
- Size:
- 4.3 kB
- Xet hash:
- c5b049d897b9841475c2efce6ab2b1f31783af25dafde3b5362f64736922f75c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.