File size: 5,529 Bytes
d6c5624 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | """Create a reviewed environment-only Docker context from a fixed Conda prefix."""
import argparse
import hashlib
import importlib.metadata
import io
import json
import os
import shutil
import tarfile
from pathlib import Path
WEIGHT_SUFFIXES = {".pt", ".ckpt", ".safetensors", ".onnx", ".model"}
SKIP_DIRS = {"__pycache__", ".git", ".cache", "conda-meta", "tests", "test"}
SKIP_NAMES = {"direct_url.json", "pip.conf", ".netrc", ".condarc", "hfkey.txt", "token", "stored_tokens", "credentials", "id_rsa", "id_ed25519", ".npmrc", ".pypirc"}
REVIEWED_BINARY_RESOURCES = {
"matplotlib/mpl-data/sample_data/goog.npz", "matplotlib/mpl-data/sample_data/jacksboro_fault_dem.npz",
"matplotlib/mpl-data/sample_data/topobathy.npz", "scipy/stats/_sobol_direction_numbers.npz",
"sentencepiece/package_data/nfkc.bin", "sentencepiece/package_data/nfkc_cf.bin",
"sentencepiece/package_data/nmt_nfkc.bin", "sentencepiece/package_data/nmt_nfkc_cf.bin",
"text_unidecode/data.bin", "whisper/assets/mel_filters.npz",
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--prefix", type=Path, default=Path("/opt/conda/envs/hybrid"))
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
prefix = args.prefix.resolve()
assert prefix == Path("/opt/conda/envs/hybrid"), "Snapshot preserves its original prefix"
args.out.mkdir(parents=True, exist_ok=False)
files, excluded = [], []
for directory, dirs, names in os.walk(prefix, followlinks=False):
dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS)
for d in dirs:
path = Path(directory) / d
if path.is_symlink():
assert path.resolve().is_relative_to(prefix), f"External directory link: {path}"
files.append(path)
for name in sorted(names):
path = Path(directory) / name
rel = path.relative_to(prefix)
if name in SKIP_NAMES or name.startswith(".env") or path.suffix in WEIGHT_SUFFIXES or path.suffix == ".pyc":
excluded.append(str(rel))
continue
if path.suffix == ".pth" and path.parent.name != "site-packages":
excluded.append(str(rel))
continue
if path.suffix.lower() in {".bin", ".pkl", ".npz", ".h5", ".hdf5"}:
resource = str(rel).removeprefix("lib/python3.12/site-packages/")
if resource not in REVIEWED_BINARY_RESOURCES:
raise ValueError(f"Review potential model/data before including: {rel}")
if path.is_symlink():
assert path.resolve().is_relative_to(prefix), f"External file link: {path}"
files.append(path)
manifest = []
patch_prefix = b"import inspect\nif not hasattr(inspect, 'getargspec'):\n inspect.getargspec = inspect.getfullargspec\n"
with tarfile.open(args.out / "environment.tar", "x", dereference=False) as archive:
for path in sorted(set(files)):
rel = path.relative_to(prefix)
info = archive.gettarinfo(str(path), arcname=str(rel))
info.uid = info.gid = 0
info.uname = info.gname = "root"
if info.issym():
archive.addfile(info)
manifest.append({"file": str(rel), "link": info.linkname})
continue
if not info.isfile():
raise ValueError(f"Unsupported file: {rel}")
modified = str(rel) == "lib/python3.12/site-packages/chumpy/__init__.py"
if modified:
original = path.read_bytes()
data = patch_prefix + original
info.size = len(data)
archive.addfile(info, io.BytesIO(data))
digest = hashlib.sha256(data).hexdigest()
else:
h = hashlib.sha256()
with path.open("rb") as source:
for block in iter(lambda: source.read(1024 * 1024), b""):
h.update(block)
digest = h.hexdigest()
with path.open("rb") as source:
archive.addfile(info, source)
entry = {"file": str(rel), "bytes": info.size, "sha256": digest}
if modified:
entry["patch"] = "chumpy-local inspect.getargspec fallback for Python 3.12"
entry["original_sha256"] = hashlib.sha256(original).hexdigest()
manifest.append(entry)
versions = sorted({f"{dist.metadata['Name']}=={dist.version}" for dist in importlib.metadata.distributions() if dist.metadata.get("Name")})
(args.out / "requirements.snapshot.txt").write_text("\n".join(versions) + "\n")
report = {
"prefix": str(prefix), "snapshot_kind": "installed dependencies, not a pip-rebuild lock",
"excluded": excluded, "skipped_directories": sorted(SKIP_DIRS), "files": manifest,
"scope": "No project, user home, HF cache, datasets, pretrained weights or credentials; optional package test/LPIPS weights removed",
}
(args.out / "environment-inventory.json").write_text(json.dumps(report, indent=2))
source = Path(__file__).resolve().parent
for name in ("Dockerfile", "smoke_test.py"):
shutil.copy2(source / name, args.out / name)
print(json.dumps({"context": str(args.out), "tar_bytes": (args.out / "environment.tar").stat().st_size, "entries": len(manifest), "excluded_files": len(excluded)}), flush=True)
if __name__ == "__main__":
main()
|