Buckets:
| # 中文源码导读(2026-09-15)|阅读顺序 048|本地工程 | |
| # 来源:官方同路径无此文件。 | |
| # 作用:检查模型文件、资源、拓扑和磁盘等运行前提,不执行候选搜索。 | |
| # 对照基准:test-time-training/discover@6c40e82;完整路线见 docs/CODE_READING_ORDER_ZH.md。 | |
| """Fail early on missing local dependencies, model assets or occupied GPUs.""" | |
| import hashlib | |
| import importlib.metadata as metadata | |
| import json | |
| from pathlib import Path | |
| import subprocess | |
| import sys | |
| import os | |
| from ttt_discover.runtime.topology import validate_topology | |
| ROOT = Path(__file__).resolve().parents[2] | |
| def check_model_files(c): | |
| """CPU-only asset/layout check, including zero-byte cache placeholders.""" | |
| model = Path(c["model_path"]) | |
| for filename in ("config.json", "tokenizer_config.json", "tokenizer.json", "model.safetensors.index.json"): | |
| file = model / filename | |
| if not file.is_file() or file.stat().st_size == 0 or not os.access(file, os.R_OK): | |
| raise RuntimeError(f"Missing, empty or unreadable model file: {file}") | |
| model_config = json.loads((model / "config.json").read_text()) | |
| if c["model_name"] == "Qwen/Qwen3-8B": | |
| expected = dict(model_type="qwen3", num_hidden_layers=36, hidden_size=4096, | |
| intermediate_size=12288, vocab_size=151936, num_attention_heads=32, | |
| num_key_value_heads=8, tie_word_embeddings=False) | |
| if any(model_config.get(key) != value for key, value in expected.items()): | |
| raise RuntimeError("Expected the aligned Qwen3-8B dense model layout") | |
| if model_config.get("quantization_config"): | |
| raise RuntimeError("Qwen profile expects BF16 base weights, not a quantized checkpoint") | |
| elif c["model_name"] == "openai/gpt-oss-120b": | |
| if model_config.get("model_type") != "gpt_oss" or model_config.get("num_hidden_layers") != 36: | |
| raise RuntimeError("Expected the GPT-OSS-120B model") | |
| else: | |
| raise RuntimeError("Unsupported model profile") | |
| index = json.loads((model / "model.safetensors.index.json").read_text()) | |
| shards = set(index["weight_map"].values()) | |
| if not shards: | |
| raise RuntimeError("Model weight index is empty") | |
| for shard in shards: | |
| file = model / shard | |
| if not file.is_file() or file.stat().st_size < 1024 or not os.access(file, os.R_OK): | |
| raise RuntimeError(f"Missing, empty or unreadable model shard: {shard}") | |
| return {"model_name": c["model_name"], "shards": len(shards), | |
| "config_sha256": hashlib.sha256((model / "config.json").read_bytes()).hexdigest()} | |
| def check(c, require_idle=False): | |
| gpu_indices = validate_topology(c) | |
| inherited = os.environ.get("CUDA_VISIBLE_DEVICES") | |
| if inherited is not None and set(inherited.split(",")) != set(map(str, c["train_gpus"])): | |
| if not set(map(str, gpu_indices)) <= set(inherited.split(",")): | |
| raise RuntimeError("Saved topology would use GPUs excluded by CUDA_VISIBLE_DEVICES") | |
| os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, c["train_gpus"])) | |
| import torch | |
| model_evidence = check_model_files(c) | |
| qwen = c["model_name"] == "Qwen/Qwen3-8B" | |
| if metadata.version("torch").split("+")[0] != "2.8.0" or metadata.version("transformers") != "4.56.2": | |
| raise RuntimeError("Activate the locked training environment via scripts/run.sh") | |
| expected = {"numpy": "2.2.6", "scipy": "1.15.3", "cvxpy": "1.7.4"} if c["task"] != "denoising" else {"numpy": "2.3.5", "scipy": "1.17.1"} | |
| versions = {name: metadata.version(name) for name in expected} | |
| if versions != expected: | |
| raise RuntimeError(f"Scientific package mismatch: {versions} != {expected}") | |
| if not torch.cuda.is_available() or torch.cuda.device_count() != len(c["train_gpus"]): | |
| raise RuntimeError(f'This profile requires {len(c["train_gpus"])} visible training GPUs') | |
| for i in range(len(c["train_gpus"])): | |
| props = torch.cuda.get_device_properties(i) | |
| if props.major != 9 or props.total_memory < 79 * 1024**3: | |
| raise RuntimeError(f"Unsupported GPU {i}: {props}") | |
| # OSS requires its pinned kernel. Qwen explicitly uses its historical SDPA | |
| # path, not a fallback after an OSS-kernel loading failure. | |
| os.environ["HF_HOME"] = str(ROOT / ".cache/huggingface") | |
| os.environ["HF_HUB_OFFLINE"] = "1" | |
| if not qwen: | |
| from kernels import get_kernel | |
| repo, revision = c["attention"].split("@") | |
| get_kernel(repo, revision=revision) | |
| if c["task"] == "denoising": | |
| import openproblems.data | |
| openproblems.data.no_cleanup() | |
| if not (ROOT / ".cache/data_ready.json").is_file(): | |
| raise RuntimeError("Run scripts/prepare.sh first to download the pancreas evaluation data") | |
| else: | |
| import cvxpy | |
| required = {"CBC", "CLARABEL", "CVXOPT", "ECOS", "GLOP", "GLPK", "GLPK_MI", "GUROBI", "MOSEK", "OSQP", "PDLP", "SCIPY", "SCS", "XPRESS"} | |
| missing = required - set(cvxpy.installed_solvers()) | |
| if missing: | |
| raise RuntimeError(f"Missing solver packages: {sorted(missing)}") | |
| from ttt_discover.runtime.train import environment_for | |
| if not qwen: | |
| subprocess.run([str(ROOT / ".venv-serve/bin/python"), str(ROOT / "scripts/patch_vllm.py"), "--check"], check=True) | |
| env_type = environment_for(c) | |
| problem_type = ( | |
| "26" | |
| if c["task"] == "cp26" | |
| else c["task"] if c["task"] in {"ac1", "ac2"} else "" | |
| ) | |
| state = env_type.create_initial_state(problem_type) | |
| gpu_csv = subprocess.check_output(["nvidia-smi", "--query-gpu=index,name,memory.used,temperature.gpu,clocks.sm", "--format=csv,noheader,nounits"], text=True) | |
| if require_idle: | |
| selected_uuids = set(subprocess.check_output([ | |
| "nvidia-smi", "-i", ",".join(map(str, gpu_indices)), "--query-gpu=uuid", | |
| "--format=csv,noheader,nounits"], text=True).splitlines()) | |
| apps = subprocess.check_output(["nvidia-smi", "--query-compute-apps=pid,gpu_uuid", "--format=csv,noheader,nounits"], text=True).strip() | |
| # torch's device inspection above does not allocate a CUDA context. | |
| apps = [row for row in apps.splitlines() if row.split(",")[-1].strip() in selected_uuids | |
| and row.split(",")[0].strip() != str(os.getpid())] | |
| if apps: | |
| raise RuntimeError(f"GPU compute processes already exist. No process was stopped: {apps}") | |
| result = {"task": c["task"], "python": sys.version.split()[0], "versions": versions, "model": model_evidence, | |
| "root_value": getattr(state, "value", None), "gpu_status": gpu_csv.splitlines()} | |
| print(json.dumps({"preflight": result}, indent=2), flush=True) | |
| return result | |
Xet Storage Details
- Size:
- 6.77 kB
- Xet hash:
- 49c0b610d31cacf0b2412a77a55b1347c5ba4d2a56f5078f36b880e6ace1817c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.