code-HFD / scripts /data /manage_source_split_artifacts_test.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
2.61 kB
from __future__ import annotations
import json
import pytest
import scripts.data.manage_source_split_artifacts as artifacts
from scripts.data.manage_source_split_artifacts import (
EPHEMERAL_MARKER_NAME,
MANIFEST_NAME,
build_manifest,
cleanup_ephemeral_artifacts,
stage_artifacts,
verify_artifacts,
)
def test_manifest_stage_and_verify_repair_same_size_corruption(tmp_path):
source = tmp_path / "mounted_oss"
destination = tmp_path / "cpfs_cache"
(source / "records").mkdir(parents=True)
(source / "runtime_indexes").mkdir()
(source / "records/data.bin").write_bytes(b"verified-record-data")
(source / "runtime_indexes/index.npy").write_bytes(b"verified-index")
build_manifest(source)
stage_artifacts(source, destination)
verify_artifacts(destination)
record = destination / "records/data.bin"
record.write_bytes(b"x" * len(b"verified-record-data"))
stage_artifacts(source, destination)
assert record.read_bytes() == b"verified-record-data"
verify_artifacts(destination)
def test_stage_rejects_manifest_path_traversal(tmp_path):
source = tmp_path / "source"
destination = tmp_path / "destination"
source.mkdir()
manifest = {
"version": 1,
"source_split_version": 1,
"files": [
{
"path": "../escape",
"size": 0,
"mtime_ns": 0,
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
}
],
"file_count": 1,
"total_bytes": 0,
}
(source / MANIFEST_NAME).write_text(json.dumps(manifest), encoding="utf-8")
with pytest.raises(ValueError, match="Unsafe artifact path"):
stage_artifacts(source, destination)
def test_ephemeral_stage_is_owned_and_cleanup_is_confined(tmp_path, monkeypatch):
ephemeral_root = tmp_path / "dev_shm"
ephemeral_root.mkdir()
monkeypatch.setattr(artifacts, "EPHEMERAL_ROOTS", (ephemeral_root,))
source = tmp_path / "source"
destination = ephemeral_root / "psi0_source_split.test"
source.mkdir()
(source / "records.bin").write_bytes(b"shared-memory-records")
build_manifest(source)
stage_artifacts(source, destination, ephemeral=True)
assert (destination / EPHEMERAL_MARKER_NAME).is_file()
verify_artifacts(destination)
cleanup_ephemeral_artifacts(destination)
assert not destination.exists()
unowned = ephemeral_root / "unowned"
unowned.mkdir()
with pytest.raises(RuntimeError, match="unowned"):
cleanup_ephemeral_artifacts(unowned)