code-HFD / src /psi /data /seed /storage_paths.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
1.81 kB
from __future__ import annotations
import os
from pathlib import Path
YETINGHONG_WORKSPACE_ROOT = Path(
os.environ.get(
"HFD_YETINGHONG_WORKSPACE_ROOT",
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/"
"user_workspace/yetinghong",
)
)
HFD_VIDEOGEN_ROOT = Path(
os.environ.get(
"HFD_VIDEOGEN_ROOT",
str(YETINGHONG_WORKSPACE_ROOT / "videogen"),
)
)
BUWENDONG_CODE_ROOT = Path(
os.environ.get(
"HFD_BUWENDONG_CODE_ROOT",
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/"
"user_workspace/buwendong/user/code",
)
)
_LEGACY_STORAGE_PREFIXES = (
# These left-hand paths are intentionally the pre-migration prefixes
# embedded in immutable manifests and historical run configs. Do not
# mechanically rewrite them when the destination workspace moves.
(Path("/cpfs01/videogen/yetinghong/datasets"), HFD_VIDEOGEN_ROOT),
(Path("/cpfs01/users/buwendong/code"), BUWENDONG_CODE_ROOT),
(Path("/cpfs01/users/yetinghong"), YETINGHONG_WORKSPACE_ROOT),
)
def relocate_hfd_storage_path(value: str | os.PathLike[str]) -> Path:
"""Map an old HFD absolute path to the migrated storage namespace.
Existing manifests stay byte-identical so their checkpoint-bound SHA256
values remain valid. The mapping is lexical and deliberately does not
probe the filesystem, which makes historical run configs work even after
the old mount is removed.
"""
path = Path(value).expanduser()
if not path.is_absolute():
return path
for legacy_root, migrated_root in _LEGACY_STORAGE_PREFIXES:
try:
relative = path.relative_to(legacy_root)
except ValueError:
continue
return migrated_root / relative
return path