File size: 11,354 Bytes
713f736
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
"""Summarise a 7z archive from its file list, without extracting anything (plan §7, Phase 1).

Files are grouped into patterns by replacing the case ID in their path, so a layout of thousands of
files reads as a short table. In a solid archive one file cannot be pulled out alone: 7z decodes its
block from the start up to that file. `decode_bytes` is that cost for a whole pattern, which is what
decides whether single modalities can be extracted cheaply.
"""

import hashlib
import json
import os
import re
import subprocess
import time
from collections import defaultdict

CASE_RE = re.compile(r"sub-[A-Za-z]*\d+")
PERFUSION_MAPS = ("cbf", "cbv", "mtt", "tmax")
SECTION_RULE = "----------"
MAX_PATTERNS_SHOWN = 60
MAX_SHARED_SHOWN = 40
MAX_MISSING_SHOWN = 20


def _run_with_heartbeat(cmd, stdout, timeout_s, heartbeat_s, doing):
    """Run a 7z command, printing progress, and return the seconds it took."""
    started = last = time.time()
    proc = subprocess.Popen(cmd, stdout=stdout, stderr=subprocess.PIPE, text=True)
    while proc.poll() is None:
        time.sleep(min(heartbeat_s, 1))
        now = time.time()
        if now - started > timeout_s:
            proc.kill()
            raise TimeoutError(f"7z took over {timeout_s} s {doing}. Reconnect and run again.")
        if now - last >= heartbeat_s:
            print(f"Still {doing}: {now - started:.0f} s", flush=True)
            last = now
    if proc.returncode:
        raise RuntimeError(f"7z exited with {proc.returncode}: {proc.stderr.read()[-800:]}")
    return time.time() - started


def list_archive(archive, out_path, seven_zip="7z", timeout_s=1800, heartbeat_s=30):
    """Write `7z l -slt` output to out_path. Reads the archive only; out_path appears only on success."""
    part = f"{out_path}.part"
    with open(part, "w") as out:
        seconds = _run_with_heartbeat([seven_zip, "l", "-slt", archive], out, timeout_s, heartbeat_s, "reading the file list")
    os.replace(part, out_path)
    return seconds


def extract_members(archive, dest, members, exclude=(), seven_zip="7z", timeout_s=1800, heartbeat_s=30):
    """Extract paths or wildcards from the archive into dest, keeping their folders."""
    cmd = [seven_zip, "x", archive, f"-o{dest}", "-y", "-bd", *members, *(f"-xr!{pattern}" for pattern in exclude)]
    return _run_with_heartbeat(cmd, subprocess.DEVNULL, timeout_s, heartbeat_s, "extracting")


def copy_with_progress(src, dst, every_s=60, chunk_bytes=16 * 1024 ** 2):
    """Copy a large file, printing progress, and return the seconds it took. Checks the final size."""
    total = os.path.getsize(src)
    started = last = time.time()
    done = 0
    with open(src, "rb") as reader, open(f"{dst}.part", "wb") as writer:
        while chunk := reader.read(chunk_bytes):
            writer.write(chunk)
            done += len(chunk)
            now = time.time()
            if now - last >= every_s:
                print(f"Copied {done / 1024 ** 3:.1f} of {total / 1024 ** 3:.1f} GiB, {done / 1024 ** 2 / (now - started):.0f} MiB/s", flush=True)
                last = now
    if os.path.getsize(f"{dst}.part") != total:
        raise OSError(f"Copy of {src} is the wrong size.")
    os.replace(f"{dst}.part", dst)
    return time.time() - started


def sha256_of(path, chunk_bytes=16 * 1024 ** 2):
    digest = hashlib.sha256()
    with open(path, "rb") as f:
        while chunk := f.read(chunk_bytes):
            digest.update(chunk)
    return digest.hexdigest()


def _fields(lines):
    fields = {}
    for line in lines:
        key, sep, value = line.partition(" =")
        if sep:
            fields[key] = value.strip()
    return fields


def parse_slt(text):
    """Parse `7z l -slt` output into (archive properties, entries in stored order)."""
    lines = text.splitlines()
    if "--" not in lines or SECTION_RULE not in lines:
        raise ValueError("Not a 7z -slt listing: section markers are missing.")
    start, rule = lines.index("--"), lines.index(SECTION_RULE)
    archive = _fields(lines[start + 1 : rule])

    entries = []
    for chunk in "\n".join(lines[rule + 1 :]).split("\n\n"):
        fields = _fields(chunk.splitlines())
        if "Path" not in fields:
            continue
        attributes = fields.get("Attributes", "")
        entries.append(
            {
                "path": fields["Path"],
                "size": int(fields.get("Size") or 0),
                "block": fields.get("Block") or None,
                "encrypted": fields.get("Encrypted") == "+",
                "is_dir": fields.get("Folder") == "+" or attributes.startswith("D"),
            }
        )
    return archive, entries


def _extension(name):
    if name.endswith(".nii.gz"):
        return ".nii.gz"
    return os.path.splitext(name)[1] or "(none)"


def decode_bytes(files):
    """Bytes 7z must decode to extract `files`: each block from its start up to the last wanted file."""
    return sum(f["decode_to"] for f in {f["block"] or f["path"]: f for f in sorted(files, key=lambda f: f["decode_to"])}.values())


def _with_decode_offsets(entries):
    position = defaultdict(int)
    files = []
    for entry in entries:
        if entry["is_dir"]:
            continue
        key = entry["block"] or entry["path"]
        position[key] += entry["size"]
        files.append({**entry, "decode_to": position[key]})
    return files


def summarise(archive, entries):
    files = _with_decode_offsets(entries)
    cases = sorted({m.group() for f in files if (m := CASE_RE.search(f["path"]))})

    top_level = defaultdict(lambda: {"files": 0, "bytes": 0})
    extensions = defaultdict(lambda: {"files": 0, "bytes": 0})
    by_pattern = defaultdict(list)
    shared = []
    for f in files:
        for bucket in (top_level[f["path"].split("/")[0]], extensions[_extension(f["path"].rsplit("/", 1)[-1])]):
            bucket["files"] += 1
            bucket["bytes"] += f["size"]
        if CASE_RE.search(f["path"]):
            by_pattern[CASE_RE.sub("{case}", f["path"])].append(f)
        else:
            shared.append({"path": f["path"], "bytes": f["size"], "decode_bytes": f["decode_to"]})

    patterns = []
    for pattern, group in sorted(by_pattern.items()):
        present = {CASE_RE.search(f["path"]).group() for f in group}
        patterns.append(
            {
                "pattern": pattern,
                "files": len(group),
                "cases": len(present),
                "bytes": sum(f["size"] for f in group),
                "decode_bytes": decode_bytes(group),
                "missing_cases": sorted(set(cases) - present),
            }
        )

    perfusion_maps = {}
    for name in PERFUSION_MAPS:
        word = re.compile(rf"(?<![a-z]){name}(?![a-z])", re.IGNORECASE)
        hits = [p for p in patterns if word.search(p["pattern"].rsplit("/", 1)[-1])]
        perfusion_maps[name] = {"max_cases": max((p["cases"] for p in hits), default=0), "patterns": [p["pattern"] for p in hits]}

    return {
        "archive": {
            "type": archive.get("Type"),
            "physical_size": int(archive.get("Physical Size") or 0),
            "headers_size": int(archive.get("Headers Size") or 0),
            "method": archive.get("Method"),
            "solid": archive.get("Solid") == "+",
            "blocks": int(archive.get("Blocks") or 0),
        },
        "n_files": len(files),
        "n_dirs": len(entries) - len(files),
        "unpacked_bytes": sum(f["size"] for f in files),
        "encrypted_files": sum(f["encrypted"] for f in files),
        "n_cases": len(cases),
        "first_case": cases[0] if cases else None,
        "last_case": cases[-1] if cases else None,
        "top_level": dict(top_level),
        "extensions": dict(extensions),
        "perfusion_maps": perfusion_maps,
        "shared_files": shared,
        "patterns": patterns,
    }


def find_problems(summary, expected_size, expected_cases):
    """Things that disagree with what we expected. These are findings to report, not crashes."""
    problems = []
    if summary["n_files"] == 0:
        problems.append("The listing has no files.")
    if summary["archive"]["physical_size"] != expected_size:
        problems.append(f"Archive size {summary['archive']['physical_size']} is not the verified {expected_size}.")
    if summary["n_cases"] != expected_cases:
        problems.append(f"Found {summary['n_cases']} case IDs, expected {expected_cases}.")
    if summary["encrypted_files"]:
        problems.append(f"{summary['encrypted_files']} files are encrypted.")
    for name, found in summary["perfusion_maps"].items():
        if found["max_cases"] < summary["n_cases"]:
            problems.append(f"{name} map: best pattern covers {found['max_cases']} of {summary['n_cases']} cases.")
    return problems


def _gib(n):
    return f"{n / 1024 ** 3:.3f}"


def format_report(summary, problems, local_free_bytes=None):
    a = summary["archive"]
    out = [
        f"ARCHIVE  type={a['type']} method={a['method']} solid={a['solid']} blocks={a['blocks']} "
        f"size={_gib(a['physical_size'])} GiB headers={a['headers_size']} bytes",
        f"CONTENTS files={summary['n_files']} dirs={summary['n_dirs']} unpacked={_gib(summary['unpacked_bytes'])} GiB",
        f"CASES    {summary['n_cases']} ({summary['first_case']} to {summary['last_case']})",
    ]
    if local_free_bytes is not None:
        fits = "fits" if summary["unpacked_bytes"] < local_free_bytes else "DOES NOT FIT"
        out.append(f"DISK     local free={_gib(local_free_bytes)} GiB: a full unpack {fits}")

    out += ["", f"PROBLEMS ({len(problems)})"] + [f"  {p}" for p in problems]

    out += ["", "PERFUSION MAPS"]
    for name, found in summary["perfusion_maps"].items():
        out.append(f"  {name}: up to {found['max_cases']} cases in {found['patterns'] or 'no pattern'}")

    out += ["", "TOP LEVEL (files, GiB)"]
    out += [f"  {k}: {v['files']}, {_gib(v['bytes'])}" for k, v in sorted(summary["top_level"].items())]
    out += ["", "EXTENSIONS (files, GiB)"]
    out += [f"  {k}: {v['files']}, {_gib(v['bytes'])}" for k, v in sorted(summary["extensions"].items())]

    shared = summary["shared_files"]
    out += ["", f"FILES NOT TIED TO A CASE ({len(shared)}): GiB, GiB to decode, path"]
    out += [f"  {_gib(f['bytes'])}  {_gib(f['decode_bytes'])}  {f['path']}" for f in shared[:MAX_SHARED_SHOWN]]
    if len(shared) > MAX_SHARED_SHOWN:
        out.append(f"  ... {len(shared) - MAX_SHARED_SHOWN} more in the summary JSON")

    patterns = summary["patterns"]
    out += ["", f"PATTERNS ({len(patterns)}): cases, files, GiB, GiB to decode, pattern"]
    for p in patterns[:MAX_PATTERNS_SHOWN]:
        out.append(f"  {p['cases']:>4} {p['files']:>6} {_gib(p['bytes']):>9} {_gib(p['decode_bytes']):>9}  {p['pattern']}")
        missing = p["missing_cases"]
        if missing:
            shown = ", ".join(missing[:MAX_MISSING_SHOWN])
            more = f" and {len(missing) - MAX_MISSING_SHOWN} more" if len(missing) > MAX_MISSING_SHOWN else ""
            out.append(f"       missing: {shown}{more}")
    if len(patterns) > MAX_PATTERNS_SHOWN:
        out.append(f"  ... {len(patterns) - MAX_PATTERNS_SHOWN} more in the summary JSON")
    return "\n".join(out)


def write_summary(summary, path):
    with open(path, "w") as f:
        json.dump(summary, f, indent=2)