Download scripts/data/manage_source_split_artifacts_test.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 2.61 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/manage_source_split_artifacts_test.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/manage_source_split_artifacts_test.py
-
curl -L -o manage_source_split_artifacts_test.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/manage_source_split_artifacts_test.py
2.61 kB
| from __future__ import annotations | |
| import json | |
| import pytest | |
| import scripts.data.manage_source_split_artifacts as artifacts | |
| from scripts.data.manage_source_split_artifacts import ( | |
| EPHEMERAL_MARKER_NAME, | |
| MANIFEST_NAME, | |
| build_manifest, | |
| cleanup_ephemeral_artifacts, | |
| stage_artifacts, | |
| verify_artifacts, | |
| ) | |
| def test_manifest_stage_and_verify_repair_same_size_corruption(tmp_path): | |
| source = tmp_path / "mounted_oss" | |
| destination = tmp_path / "cpfs_cache" | |
| (source / "records").mkdir(parents=True) | |
| (source / "runtime_indexes").mkdir() | |
| (source / "records/data.bin").write_bytes(b"verified-record-data") | |
| (source / "runtime_indexes/index.npy").write_bytes(b"verified-index") | |
| build_manifest(source) | |
| stage_artifacts(source, destination) | |
| verify_artifacts(destination) | |
| record = destination / "records/data.bin" | |
| record.write_bytes(b"x" * len(b"verified-record-data")) | |
| stage_artifacts(source, destination) | |
| assert record.read_bytes() == b"verified-record-data" | |
| verify_artifacts(destination) | |
| def test_stage_rejects_manifest_path_traversal(tmp_path): | |
| source = tmp_path / "source" | |
| destination = tmp_path / "destination" | |
| source.mkdir() | |
| manifest = { | |
| "version": 1, | |
| "source_split_version": 1, | |
| "files": [ | |
| { | |
| "path": "../escape", | |
| "size": 0, | |
| "mtime_ns": 0, | |
| "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", | |
| } | |
| ], | |
| "file_count": 1, | |
| "total_bytes": 0, | |
| } | |
| (source / MANIFEST_NAME).write_text(json.dumps(manifest), encoding="utf-8") | |
| with pytest.raises(ValueError, match="Unsafe artifact path"): | |
| stage_artifacts(source, destination) | |
| def test_ephemeral_stage_is_owned_and_cleanup_is_confined(tmp_path, monkeypatch): | |
| ephemeral_root = tmp_path / "dev_shm" | |
| ephemeral_root.mkdir() | |
| monkeypatch.setattr(artifacts, "EPHEMERAL_ROOTS", (ephemeral_root,)) | |
| source = tmp_path / "source" | |
| destination = ephemeral_root / "psi0_source_split.test" | |
| source.mkdir() | |
| (source / "records.bin").write_bytes(b"shared-memory-records") | |
| build_manifest(source) | |
| stage_artifacts(source, destination, ephemeral=True) | |
| assert (destination / EPHEMERAL_MARKER_NAME).is_file() | |
| verify_artifacts(destination) | |
| cleanup_ephemeral_artifacts(destination) | |
| assert not destination.exists() | |
| unowned = ephemeral_root / "unowned" | |
| unowned.mkdir() | |
| with pytest.raises(RuntimeError, match="unowned"): | |
| cleanup_ephemeral_artifacts(unowned) | |