zefancainlp's picture
download
raw
6.77 kB
# 中文源码导读(2026-09-15)|阅读顺序 048|本地工程
# 来源:官方同路径无此文件。
# 作用:检查模型文件、资源、拓扑和磁盘等运行前提,不执行候选搜索。
# 对照基准:test-time-training/discover@6c40e82;完整路线见 docs/CODE_READING_ORDER_ZH.md。
"""Fail early on missing local dependencies, model assets or occupied GPUs."""
import hashlib
import importlib.metadata as metadata
import json
from pathlib import Path
import subprocess
import sys
import os
from ttt_discover.runtime.topology import validate_topology
ROOT = Path(__file__).resolve().parents[2]
def check_model_files(c):
"""CPU-only asset/layout check, including zero-byte cache placeholders."""
model = Path(c["model_path"])
for filename in ("config.json", "tokenizer_config.json", "tokenizer.json", "model.safetensors.index.json"):
file = model / filename
if not file.is_file() or file.stat().st_size == 0 or not os.access(file, os.R_OK):
raise RuntimeError(f"Missing, empty or unreadable model file: {file}")
model_config = json.loads((model / "config.json").read_text())
if c["model_name"] == "Qwen/Qwen3-8B":
expected = dict(model_type="qwen3", num_hidden_layers=36, hidden_size=4096,
intermediate_size=12288, vocab_size=151936, num_attention_heads=32,
num_key_value_heads=8, tie_word_embeddings=False)
if any(model_config.get(key) != value for key, value in expected.items()):
raise RuntimeError("Expected the aligned Qwen3-8B dense model layout")
if model_config.get("quantization_config"):
raise RuntimeError("Qwen profile expects BF16 base weights, not a quantized checkpoint")
elif c["model_name"] == "openai/gpt-oss-120b":
if model_config.get("model_type") != "gpt_oss" or model_config.get("num_hidden_layers") != 36:
raise RuntimeError("Expected the GPT-OSS-120B model")
else:
raise RuntimeError("Unsupported model profile")
index = json.loads((model / "model.safetensors.index.json").read_text())
shards = set(index["weight_map"].values())
if not shards:
raise RuntimeError("Model weight index is empty")
for shard in shards:
file = model / shard
if not file.is_file() or file.stat().st_size < 1024 or not os.access(file, os.R_OK):
raise RuntimeError(f"Missing, empty or unreadable model shard: {shard}")
return {"model_name": c["model_name"], "shards": len(shards),
"config_sha256": hashlib.sha256((model / "config.json").read_bytes()).hexdigest()}
def check(c, require_idle=False):
gpu_indices = validate_topology(c)
inherited = os.environ.get("CUDA_VISIBLE_DEVICES")
if inherited is not None and set(inherited.split(",")) != set(map(str, c["train_gpus"])):
if not set(map(str, gpu_indices)) <= set(inherited.split(",")):
raise RuntimeError("Saved topology would use GPUs excluded by CUDA_VISIBLE_DEVICES")
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, c["train_gpus"]))
import torch
model_evidence = check_model_files(c)
qwen = c["model_name"] == "Qwen/Qwen3-8B"
if metadata.version("torch").split("+")[0] != "2.8.0" or metadata.version("transformers") != "4.56.2":
raise RuntimeError("Activate the locked training environment via scripts/run.sh")
expected = {"numpy": "2.2.6", "scipy": "1.15.3", "cvxpy": "1.7.4"} if c["task"] != "denoising" else {"numpy": "2.3.5", "scipy": "1.17.1"}
versions = {name: metadata.version(name) for name in expected}
if versions != expected:
raise RuntimeError(f"Scientific package mismatch: {versions} != {expected}")
if not torch.cuda.is_available() or torch.cuda.device_count() != len(c["train_gpus"]):
raise RuntimeError(f'This profile requires {len(c["train_gpus"])} visible training GPUs')
for i in range(len(c["train_gpus"])):
props = torch.cuda.get_device_properties(i)
if props.major != 9 or props.total_memory < 79 * 1024**3:
raise RuntimeError(f"Unsupported GPU {i}: {props}")
# OSS requires its pinned kernel. Qwen explicitly uses its historical SDPA
# path, not a fallback after an OSS-kernel loading failure.
os.environ["HF_HOME"] = str(ROOT / ".cache/huggingface")
os.environ["HF_HUB_OFFLINE"] = "1"
if not qwen:
from kernels import get_kernel
repo, revision = c["attention"].split("@")
get_kernel(repo, revision=revision)
if c["task"] == "denoising":
import openproblems.data
openproblems.data.no_cleanup()
if not (ROOT / ".cache/data_ready.json").is_file():
raise RuntimeError("Run scripts/prepare.sh first to download the pancreas evaluation data")
else:
import cvxpy
required = {"CBC", "CLARABEL", "CVXOPT", "ECOS", "GLOP", "GLPK", "GLPK_MI", "GUROBI", "MOSEK", "OSQP", "PDLP", "SCIPY", "SCS", "XPRESS"}
missing = required - set(cvxpy.installed_solvers())
if missing:
raise RuntimeError(f"Missing solver packages: {sorted(missing)}")
from ttt_discover.runtime.train import environment_for
if not qwen:
subprocess.run([str(ROOT / ".venv-serve/bin/python"), str(ROOT / "scripts/patch_vllm.py"), "--check"], check=True)
env_type = environment_for(c)
problem_type = (
"26"
if c["task"] == "cp26"
else c["task"] if c["task"] in {"ac1", "ac2"} else ""
)
state = env_type.create_initial_state(problem_type)
gpu_csv = subprocess.check_output(["nvidia-smi", "--query-gpu=index,name,memory.used,temperature.gpu,clocks.sm", "--format=csv,noheader,nounits"], text=True)
if require_idle:
selected_uuids = set(subprocess.check_output([
"nvidia-smi", "-i", ",".join(map(str, gpu_indices)), "--query-gpu=uuid",
"--format=csv,noheader,nounits"], text=True).splitlines())
apps = subprocess.check_output(["nvidia-smi", "--query-compute-apps=pid,gpu_uuid", "--format=csv,noheader,nounits"], text=True).strip()
# torch's device inspection above does not allocate a CUDA context.
apps = [row for row in apps.splitlines() if row.split(",")[-1].strip() in selected_uuids
and row.split(",")[0].strip() != str(os.getpid())]
if apps:
raise RuntimeError(f"GPU compute processes already exist. No process was stopped: {apps}")
result = {"task": c["task"], "python": sys.version.split()[0], "versions": versions, "model": model_evidence,
"root_value": getattr(state, "value", None), "gpu_status": gpu_csv.splitlines()}
print(json.dumps({"preflight": result}, indent=2), flush=True)
return result

Xet Storage Details

Size:
6.77 kB
·
Xet hash:
49c0b610d31cacf0b2412a77a55b1347c5ba4d2a56f5078f36b880e6ace1817c

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.