File size: 2,611 Bytes
8562e9b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | from __future__ import annotations
import json
import pytest
import scripts.data.manage_source_split_artifacts as artifacts
from scripts.data.manage_source_split_artifacts import (
EPHEMERAL_MARKER_NAME,
MANIFEST_NAME,
build_manifest,
cleanup_ephemeral_artifacts,
stage_artifacts,
verify_artifacts,
)
def test_manifest_stage_and_verify_repair_same_size_corruption(tmp_path):
source = tmp_path / "mounted_oss"
destination = tmp_path / "cpfs_cache"
(source / "records").mkdir(parents=True)
(source / "runtime_indexes").mkdir()
(source / "records/data.bin").write_bytes(b"verified-record-data")
(source / "runtime_indexes/index.npy").write_bytes(b"verified-index")
build_manifest(source)
stage_artifacts(source, destination)
verify_artifacts(destination)
record = destination / "records/data.bin"
record.write_bytes(b"x" * len(b"verified-record-data"))
stage_artifacts(source, destination)
assert record.read_bytes() == b"verified-record-data"
verify_artifacts(destination)
def test_stage_rejects_manifest_path_traversal(tmp_path):
source = tmp_path / "source"
destination = tmp_path / "destination"
source.mkdir()
manifest = {
"version": 1,
"source_split_version": 1,
"files": [
{
"path": "../escape",
"size": 0,
"mtime_ns": 0,
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
}
],
"file_count": 1,
"total_bytes": 0,
}
(source / MANIFEST_NAME).write_text(json.dumps(manifest), encoding="utf-8")
with pytest.raises(ValueError, match="Unsafe artifact path"):
stage_artifacts(source, destination)
def test_ephemeral_stage_is_owned_and_cleanup_is_confined(tmp_path, monkeypatch):
ephemeral_root = tmp_path / "dev_shm"
ephemeral_root.mkdir()
monkeypatch.setattr(artifacts, "EPHEMERAL_ROOTS", (ephemeral_root,))
source = tmp_path / "source"
destination = ephemeral_root / "psi0_source_split.test"
source.mkdir()
(source / "records.bin").write_bytes(b"shared-memory-records")
build_manifest(source)
stage_artifacts(source, destination, ephemeral=True)
assert (destination / EPHEMERAL_MARKER_NAME).is_file()
verify_artifacts(destination)
cleanup_ephemeral_artifacts(destination)
assert not destination.exists()
unowned = ephemeral_root / "unowned"
unowned.mkdir()
with pytest.raises(RuntimeError, match="unowned"):
cleanup_ephemeral_artifacts(unowned)
|