"""Summarise a 7z archive from its file list, without extracting anything (plan ยง7, Phase 1). Files are grouped into patterns by replacing the case ID in their path, so a layout of thousands of files reads as a short table. In a solid archive one file cannot be pulled out alone: 7z decodes its block from the start up to that file. `decode_bytes` is that cost for a whole pattern, which is what decides whether single modalities can be extracted cheaply. """ import hashlib import json import os import re import subprocess import time from collections import defaultdict CASE_RE = re.compile(r"sub-[A-Za-z]*\d+") PERFUSION_MAPS = ("cbf", "cbv", "mtt", "tmax") SECTION_RULE = "----------" MAX_PATTERNS_SHOWN = 60 MAX_SHARED_SHOWN = 40 MAX_MISSING_SHOWN = 20 def _run_with_heartbeat(cmd, stdout, timeout_s, heartbeat_s, doing): """Run a 7z command, printing progress, and return the seconds it took.""" started = last = time.time() proc = subprocess.Popen(cmd, stdout=stdout, stderr=subprocess.PIPE, text=True) while proc.poll() is None: time.sleep(min(heartbeat_s, 1)) now = time.time() if now - started > timeout_s: proc.kill() raise TimeoutError(f"7z took over {timeout_s} s {doing}. Reconnect and run again.") if now - last >= heartbeat_s: print(f"Still {doing}: {now - started:.0f} s", flush=True) last = now if proc.returncode: raise RuntimeError(f"7z exited with {proc.returncode}: {proc.stderr.read()[-800:]}") return time.time() - started def list_archive(archive, out_path, seven_zip="7z", timeout_s=1800, heartbeat_s=30): """Write `7z l -slt` output to out_path. Reads the archive only; out_path appears only on success.""" part = f"{out_path}.part" with open(part, "w") as out: seconds = _run_with_heartbeat([seven_zip, "l", "-slt", archive], out, timeout_s, heartbeat_s, "reading the file list") os.replace(part, out_path) return seconds def extract_members(archive, dest, members, exclude=(), seven_zip="7z", timeout_s=1800, heartbeat_s=30): """Extract paths or wildcards from the archive into dest, keeping their folders.""" cmd = [seven_zip, "x", archive, f"-o{dest}", "-y", "-bd", *members, *(f"-xr!{pattern}" for pattern in exclude)] return _run_with_heartbeat(cmd, subprocess.DEVNULL, timeout_s, heartbeat_s, "extracting") def copy_with_progress(src, dst, every_s=60, chunk_bytes=16 * 1024 ** 2): """Copy a large file, printing progress, and return the seconds it took. Checks the final size.""" total = os.path.getsize(src) started = last = time.time() done = 0 with open(src, "rb") as reader, open(f"{dst}.part", "wb") as writer: while chunk := reader.read(chunk_bytes): writer.write(chunk) done += len(chunk) now = time.time() if now - last >= every_s: print(f"Copied {done / 1024 ** 3:.1f} of {total / 1024 ** 3:.1f} GiB, {done / 1024 ** 2 / (now - started):.0f} MiB/s", flush=True) last = now if os.path.getsize(f"{dst}.part") != total: raise OSError(f"Copy of {src} is the wrong size.") os.replace(f"{dst}.part", dst) return time.time() - started def sha256_of(path, chunk_bytes=16 * 1024 ** 2): digest = hashlib.sha256() with open(path, "rb") as f: while chunk := f.read(chunk_bytes): digest.update(chunk) return digest.hexdigest() def _fields(lines): fields = {} for line in lines: key, sep, value = line.partition(" =") if sep: fields[key] = value.strip() return fields def parse_slt(text): """Parse `7z l -slt` output into (archive properties, entries in stored order).""" lines = text.splitlines() if "--" not in lines or SECTION_RULE not in lines: raise ValueError("Not a 7z -slt listing: section markers are missing.") start, rule = lines.index("--"), lines.index(SECTION_RULE) archive = _fields(lines[start + 1 : rule]) entries = [] for chunk in "\n".join(lines[rule + 1 :]).split("\n\n"): fields = _fields(chunk.splitlines()) if "Path" not in fields: continue attributes = fields.get("Attributes", "") entries.append( { "path": fields["Path"], "size": int(fields.get("Size") or 0), "block": fields.get("Block") or None, "encrypted": fields.get("Encrypted") == "+", "is_dir": fields.get("Folder") == "+" or attributes.startswith("D"), } ) return archive, entries def _extension(name): if name.endswith(".nii.gz"): return ".nii.gz" return os.path.splitext(name)[1] or "(none)" def decode_bytes(files): """Bytes 7z must decode to extract `files`: each block from its start up to the last wanted file.""" return sum(f["decode_to"] for f in {f["block"] or f["path"]: f for f in sorted(files, key=lambda f: f["decode_to"])}.values()) def _with_decode_offsets(entries): position = defaultdict(int) files = [] for entry in entries: if entry["is_dir"]: continue key = entry["block"] or entry["path"] position[key] += entry["size"] files.append({**entry, "decode_to": position[key]}) return files def summarise(archive, entries): files = _with_decode_offsets(entries) cases = sorted({m.group() for f in files if (m := CASE_RE.search(f["path"]))}) top_level = defaultdict(lambda: {"files": 0, "bytes": 0}) extensions = defaultdict(lambda: {"files": 0, "bytes": 0}) by_pattern = defaultdict(list) shared = [] for f in files: for bucket in (top_level[f["path"].split("/")[0]], extensions[_extension(f["path"].rsplit("/", 1)[-1])]): bucket["files"] += 1 bucket["bytes"] += f["size"] if CASE_RE.search(f["path"]): by_pattern[CASE_RE.sub("{case}", f["path"])].append(f) else: shared.append({"path": f["path"], "bytes": f["size"], "decode_bytes": f["decode_to"]}) patterns = [] for pattern, group in sorted(by_pattern.items()): present = {CASE_RE.search(f["path"]).group() for f in group} patterns.append( { "pattern": pattern, "files": len(group), "cases": len(present), "bytes": sum(f["size"] for f in group), "decode_bytes": decode_bytes(group), "missing_cases": sorted(set(cases) - present), } ) perfusion_maps = {} for name in PERFUSION_MAPS: word = re.compile(rf"(? MAX_SHARED_SHOWN: out.append(f" ... {len(shared) - MAX_SHARED_SHOWN} more in the summary JSON") patterns = summary["patterns"] out += ["", f"PATTERNS ({len(patterns)}): cases, files, GiB, GiB to decode, pattern"] for p in patterns[:MAX_PATTERNS_SHOWN]: out.append(f" {p['cases']:>4} {p['files']:>6} {_gib(p['bytes']):>9} {_gib(p['decode_bytes']):>9} {p['pattern']}") missing = p["missing_cases"] if missing: shown = ", ".join(missing[:MAX_MISSING_SHOWN]) more = f" and {len(missing) - MAX_MISSING_SHOWN} more" if len(missing) > MAX_MISSING_SHOWN else "" out.append(f" missing: {shown}{more}") if len(patterns) > MAX_PATTERNS_SHOWN: out.append(f" ... {len(patterns) - MAX_PATTERNS_SHOWN} more in the summary JSON") return "\n".join(out) def write_summary(summary, path): with open(path, "w") as f: json.dump(summary, f, indent=2)