Spaces:
Running
Running
File size: 4,982 Bytes
1ad509e c724b78 1ad509e c724b78 1ad509e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | """Fetch immutable, hash-checked runtime data without growing Space Git history.
Record after committing assets: python scripts/fetch_runtime_assets.py --record-commit SHA
Docker build: python scripts/fetch_runtime_assets.py --output /assets/data
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path, PurePosixPath
import re
import tempfile
import time
import urllib.request
REPO = 'gauravmeena0708/pf-circulars'
MANIFEST = Path('deploy/runtime-assets.json')
def digest(path):
value = hashlib.sha256()
with Path(path).open('rb') as stream:
for block in iter(lambda: stream.read(1024 * 1024), b''):
value.update(block)
return value.hexdigest()
def validate(manifest):
if manifest['repo'] != REPO or not re.fullmatch('[a-f0-9]{40}', manifest['revision']):
raise ValueError('Runtime assets require the configured repository and an immutable commit SHA')
seen = set()
for entry in manifest['files']:
path = PurePosixPath(entry['path'])
if path.is_absolute() or '..' in path.parts or '\\' in entry['path'] or str(path) in seen:
raise ValueError('Invalid or duplicate runtime asset path')
# Only the search index is a build-time asset. Original policy PDFs are fetched
# on demand from their public URLs (see policy_pdfs.py) and are not stored in Git.
if not entry['path'].startswith('vector_store/data_index/'):
raise ValueError('Unexpected runtime asset path')
if not re.fullmatch('[a-f0-9]{64}', entry['sha256']) or not 0 < entry['size'] <= 500*1024*1024:
raise ValueError('Invalid runtime asset hash or size')
seen.add(str(path))
def fetch(entry, revision, output):
target = Path(output) / entry['path']
if target.is_file() and target.stat().st_size == entry['size'] and digest(target) == entry['sha256']:
return
target.parent.mkdir(parents=True, exist_ok=True)
url = f'https://media.githubusercontent.com/media/{REPO}/{revision}/{entry["path"]}'
for attempt in range(3):
temporary = None
try:
checksum = hashlib.sha256()
size = 0
request = urllib.request.Request(url, headers={'User-Agent': 'EPFO-Workspace-Build', 'Accept-Encoding': 'identity'})
with urllib.request.urlopen(request, timeout=90) as response, tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as out:
temporary = Path(out.name)
for block in iter(lambda: response.read(1024*1024), b''):
size += len(block)
if size > entry['size']:
raise ValueError('Asset exceeds its recorded size')
checksum.update(block)
out.write(block)
if size != entry['size'] or checksum.hexdigest() != entry['sha256']:
raise ValueError('Runtime asset size or SHA-256 mismatch')
temporary.chmod(0o644)
temporary.replace(target)
print('Verified ' + entry['path'], flush=True)
return
except Exception:
if temporary and temporary.exists():
temporary.unlink()
if attempt == 2:
raise
time.sleep(2 * (attempt+1))
def record(revision, root=Path('.'), path=MANIFEST, preserve_revision=False):
files = sorted(root/'vector_store/data_index'/name for name in
('faiss_index.index', 'faiss_index.passages.db', 'faiss_index.bm25.json.gz'))
manifest = dict(repo=REPO, revision=revision, files=[dict(path=p.relative_to(root).as_posix(), size=p.stat().st_size, sha256=digest(p)) for p in files])
if preserve_revision and path.exists():
previous = json.loads(path.read_text())
validate(previous)
if previous['files'] == manifest['files']:
manifest['revision'] = previous['revision']
validate(manifest)
path.write_text(json.dumps(manifest, indent=2)+'\n')
return manifest
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--manifest', type=Path, default=MANIFEST)
parser.add_argument('--output', type=Path, default=Path('.'))
mode = parser.add_mutually_exclusive_group()
mode.add_argument('--record-commit')
mode.add_argument('--refresh-commit', help='CI: keep the pinned revision when asset bytes have not changed')
args = parser.parse_args()
if args.record_commit or args.refresh_commit:
record(args.record_commit or args.refresh_commit, path=args.manifest, preserve_revision=bool(args.refresh_commit))
return
manifest = json.loads(args.manifest.read_text())
validate(manifest)
with ThreadPoolExecutor(max_workers=4) as pool:
list(pool.map(lambda entry: fetch(entry, manifest['revision'], args.output), manifest['files']))
if __name__ == '__main__':
main()
|