File size: 2,093 Bytes
21b75e3 a18cd40 21b75e3 a18cd40 d08c744 21b75e3 19a2d83 3caced1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | from pathlib import Path
REMOTE_TRAIN = Path("scripts/remote_train.sh").read_text()
def test_remote_train_pins_hf_dataset_revision_for_split_download():
assert "DATA_REVISION=" in REMOTE_TRAIN
assert "dataset_info('$DATA_DATASET').sha" in REMOTE_TRAIN
assert "DATA_REVISION was not set; using mutable dataset HEAD" in REMOTE_TRAIN
assert "production HF jobs should pin DATA_REVISION" in REMOTE_TRAIN
assert "revision='$DATA_REVISION'" in REMOTE_TRAIN
assert "allow_patterns=['train/*.parquet', 'val/*.parquet', 'test/*.parquet', 'split_summary.json']" in REMOTE_TRAIN
assert "allow_patterns=['te_seqdata.parquet']" not in REMOTE_TRAIN
assert "root te_seqdata.parquet is intentionally ignored" in REMOTE_TRAIN
def test_remote_train_lmdb_signature_includes_dataset_revision():
assert "DATA_REVISION_SHORT" in REMOTE_TRAIN
assert "immutable split-input" in REMOTE_TRAIN
assert 'PROCESSED_SIGNATURE="${LOCAL_PROCESSED_SIGNATURE}_ds${DATA_REVISION_SHORT}"' in REMOTE_TRAIN
def test_remote_train_uses_trainer_processed_dir_for_local_artifacts():
assert 'PROCESSED_DIR="data/processed/te_visuals/$LOCAL_PROCESSED_SIGNATURE"' in REMOTE_TRAIN
assert 'PROCESSED_DIR="data/processed/te_visuals/auto"' not in REMOTE_TRAIN
def test_remote_train_supports_forced_lmdb_rebuild():
assert "FORCE_ETL_REBUILD" in REMOTE_TRAIN
assert "skipping LMDB cache pull" in REMOTE_TRAIN
assert "rm -f \"$PROCESSED_DIR/tensors.lmdb\"" in REMOTE_TRAIN
def test_remote_train_requires_lmdb_cache_manifest_for_cache_hit():
assert "etl_cache_manifest.json" in REMOTE_TRAIN
assert "[ -f \"$PROCESSED_DIR/etl_cache_manifest.json\" ]" in REMOTE_TRAIN
assert "root te_seqdata.parquet is intentionally ignored" in REMOTE_TRAIN
def test_remote_train_disables_full_embedding_export():
assert "--no-export-embeddings" in REMOTE_TRAIN
def test_remote_train_defaults_amp_off_for_hf_jobs():
assert 'DEEPGENOPIX_USE_AMP="${DEEPGENOPIX_USE_AMP:-0}"' in REMOTE_TRAIN
assert "using DEEPGENOPIX_USE_AMP=$DEEPGENOPIX_USE_AMP" in REMOTE_TRAIN
|