Verify uploaded Source identities and provide a pinned 27-Source downloader

basis/releases/source_ratios_20260917/README.md

@@ -14,18 +14,18 @@ Added 2026-09-17. These files fill the seven missing method/backbone/ratio candi
 
 ## Download and load
 
-Existing BASIS checkouts may pin an older manifest. This standalone command resolves the current repository commit once and downloads one complete Source; change `source_id` for another row above. Install `huggingface_hub` and the BASIS package in the usual environment.
+Existing BASIS checkouts may pin an older manifest. This standalone command pins the verified release commit and downloads one complete Source; change `source_id` for another row above. Install `huggingface_hub` and the BASIS package in the usual environment.
 
 ```python
 import json
 from pathlib import Path
-from huggingface_hub import HfApi, snapshot_download
+from huggingface_hub import snapshot_download
 from basis.compression.checkpoints import load_checkpoint
 from basis.utils.io import sha256_file
 
 repo = "Duke-CEI-SVD/FlashSVD"
 source_id = "qwen3/swiftsvd_r050"
-revision = HfApi().repo_info(repo).sha
+revision = "08d28e1337c4fec6a106c959c4ca8a6e2a0d6acb"  # verified release commit
 root = Path("data/flashsvd")
 snapshot_download(repo, revision=revision, local_dir=root,
                   allow_patterns=[f"basis/sources/{source_id}/*"])
@@ -47,3 +47,13 @@ All seven use `kind=null` in the Source record and a native BASIS `model.pt`; th
 - Suggested BASIS capacity is recorded and lies within 11M ±5%. Full recovery preflight, Avg7, pressure, and three-seed recovery remain subsequent experiments. New Sources must not silently reuse old gradient caches.
 - The seven checkpoints total 67.324 GiB. Per-Source `provenance/manifest.json` retains construction/calibration lineage; `validation.json` records the local checks. [Aggregate audit](validation.json).
 - All prior Source entries and identities are preserved. The package now has 27 Sources: 23 hosted checkpoints and four pinned LowRankArena imports. Prior frozen revisions remain available.
+
+## Updated selective downloader
+
+Download [download_inputs.py](download_inputs.py) and [data_bundle.json](data_bundle.json) into the same folder, then run:
+
+```bash
+python download_inputs.py --source qwen3/swiftsvd_r050 --output data/flashsvd
+```
+
+This pinned registry includes all 27 Sources. [Remote publication checks](publication_validation.json) verify the new weights and unchanged prior files.

basis/releases/source_ratios_20260917/data_bundle.json

@@ -0,0 +1,15 @@
+{
+  "status": "published",
+  "repo_id": "Duke-CEI-SVD/FlashSVD",
+  "revision": "08d28e1337c4fec6a106c959c4ca8a6e2a0d6acb",
+  "manifest_path": "basis/manifest.json",
+  "manifest_sha256": "81b74f1fab6c7aaf040e174a1a8f5046f0f8dafee035ec2cf01b76502e4fe0c7",
+  "publication": "pull_request",
+  "pr_url": "https://huggingface.co/Duke-CEI-SVD/FlashSVD/discussions/2",
+  "data_bytes": 4728774592,
+  "source_bytes": 264095245023,
+  "external_weight_bytes": 53054325445,
+  "source_count": 27,
+  "full_checkpoints_in_flashsvd": 23,
+  "sources_reusing_lowrankarena": 4
+}

basis/releases/source_ratios_20260917/download_inputs.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""Download selected frozen BASIS inputs and verify their SHA-256 checksums."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import shutil
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+BUNDLE = Path(__file__).with_name('data_bundle.json')
+
+
+def sha256(path):
+    digest = hashlib.sha256()
+    with Path(path).open('rb') as stream:
+        for block in iter(lambda: stream.read(8 * 1024 * 1024), b''):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def select_files(manifest, groups, source_ids):
+    available_groups = {r['group'] for r in manifest['files']} - {'sources'}
+    unknown = set(groups) - available_groups
+    sources = {s['id']: s for s in manifest['sources']}
+    if unknown or set(source_ids) - sources.keys():
+        raise ValueError(f'Unknown groups or Sources: {sorted(unknown | (set(source_ids) - sources.keys()))}')
+    selected = []
+    prefixes = tuple(f'basis/sources/{s}/' for s in source_ids)
+    for record in manifest['files']:
+        path = Path(record['path'])
+        if path.is_absolute() or '..' in path.parts or path.parts[0] != 'basis':
+            raise ValueError(f'Invalid package path: {path}')
+        if record['group'] in set(groups) | {'provenance'} or (prefixes and record['path'].startswith(prefixes)):
+            selected.append(record)
+    return selected
+
+
+def verify_files(records, root):
+    for record in records:
+        path = root / record['path']
+        if path.stat().st_size != record['bytes'] or sha256(path) != record['sha256']:
+            raise ValueError(f'Checksum mismatch: {path}')
+
+
+def download_external(records, root):
+    """Reuse pinned LowRankArena files and assemble the declared local layout."""
+    from huggingface_hub import hf_hub_download
+
+    def fetch(record):
+        remote = record['download']
+        target = root / record['path']
+        if target.is_file() and target.stat().st_size == record['bytes'] and sha256(target) == record['sha256']:
+            return
+        cached = Path(hf_hub_download(remote['repo_id'], remote['path'],
+                                     revision=remote['revision'], cache_dir=root / '.cache' / 'external')).resolve()
+        if sha256(cached) != record['sha256']:
+            raise ValueError(f'External checksum mismatch: {remote["path"]}')
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if target.exists() or target.is_symlink():
+            target.unlink()
+        try:
+            os.link(cached, target)
+        except OSError:
+            shutil.copyfile(cached, target)
+
+    with ThreadPoolExecutor(max_workers=4) as pool:
+        list(pool.map(fetch, records))
+
+
+def write_path_map(manifest, root, revision):
+    # This maps the full inventory; only requested groups have been downloaded.
+    paths = {r['original_path']: str(root / r['path']) for r in manifest['files']}
+    paths.update({s['original_path']: str(root / s['path']) for s in manifest['sources']})
+    target = root / 'basis' / 'path_map.json'
+    target.write_text(json.dumps({
+        'repo_id': manifest['repo_id'], 'revision': revision,
+        'note': 'Full inventory mapping; check that an input was downloaded before use.',
+        'paths': paths, 'dense_parents': manifest['dense_parents'],
+    }, indent=2) + '\n')
+    return target
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--group', action='append', default=[], help='Repeat for c4, c4_probe_shard, q64, q128, q64_gradients, q128_gradients, tokenizers, or provenance')
+    parser.add_argument('--source', action='append', default=[], help='Exact Source ID, e.g. llama31/swiftsvd_r080; repeat as needed')
+    parser.add_argument('--output', type=Path, default=Path('data/flashsvd'))
+    parser.add_argument('--list', action='store_true', help='List available input groups and Sources')
+    parser.add_argument('--dry-run', action='store_true', help='Show selected sizes; download only the small manifest')
+    args = parser.parse_args()
+    bundle = json.loads(BUNDLE.read_text())
+    if bundle.get('status') != 'published' or not bundle.get('revision'):
+        parser.error('The package is prepared but has not yet been published. See paper/DOWNLOADS.md.')
+    from huggingface_hub import hf_hub_download, snapshot_download
+    root = args.output.expanduser().resolve()
+    manifest_path = hf_hub_download(bundle['repo_id'], bundle['manifest_path'],
+                                    revision=bundle['revision'], local_dir=root)
+    if sha256(manifest_path) != bundle['manifest_sha256']:
+        raise ValueError('Package manifest checksum mismatch')
+    manifest = json.loads(Path(manifest_path).read_text())
+    if args.list:
+        for group in sorted({r['group'] for r in manifest['files']} - {'sources'}):
+            records = [r for r in manifest['files'] if r['group'] == group]
+            print(f'{group}: {len(records)} files, {sum(r["bytes"] for r in records) / 2**30:.3f} GiB')
+        for source in manifest['sources']:
+            print(f'Source {source["id"]}: Tasks {", ".join(source["tasks"])}')
+        return
+    if not args.group and not args.source:
+        parser.error('Select --group / --source, or use --list.')
+    selected = select_files(manifest, args.group, args.source)
+    print(f'Selected {len(selected)} files, {sum(r["bytes"] for r in selected) / 2**30:.3f} GiB')
+    if args.dry_run:
+        return
+    local = [r for r in selected if 'download' not in r]
+    external = [r for r in selected if 'download' in r]
+    if local:
+        snapshot_download(bundle['repo_id'], revision=bundle['revision'], local_dir=root,
+                          allow_patterns=[r['path'] for r in local], max_workers=4)
+    if external:
+        print(f'Reusing {len(external)} weight files from LowRankArena.')
+        download_external(external, root)
+    verify_files(selected, root)
+    print('Verified every selected file. Path map:', write_path_map(manifest, root, bundle['revision']))
+    print('Create fresh task manifests using these paths; preserve the original frozen manifests.')
+
+
+if __name__ == '__main__':
+    main()

basis/releases/source_ratios_20260917/publication_validation.json

@@ -0,0 +1,14 @@
+{
+  "status": "passed",
+  "verified_revision": "08d28e1337c4fec6a106c959c4ca8a6e2a0d6acb",
+  "uploaded_objects_verified": 35,
+  "new_weight_files_verified": 7,
+  "new_source_count": 7,
+  "source_count": 27,
+  "weight_verification": "All seven remote LFS byte counts and SHA256 identities match freshly hashed local artifacts",
+  "metadata_verification": "Git blob IDs or LFS identities match for every uploaded metadata object",
+  "original_files_preserved": 586,
+  "original_source_records_preserved": 20,
+  "anonymous_manifest_and_seven_source_records_downloaded": true,
+  "manifest_sha256": "81b74f1fab6c7aaf040e174a1a8f5046f0f8dafee035ec2cf01b76502e4fe0c7"
+}