File size: 2,438 Bytes
ab18ce0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a47010e
ab18ce0
 
 
 
 
 
 
 
 
 
 
a47010e
 
ab18ce0
 
 
 
 
 
 
 
a47010e
 
 
 
 
 
ab18ce0
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
"""Reconstruct the hash-pinned evaluation Transformers tree."""
import argparse
import hashlib
import json
from pathlib import Path
import shutil
import subprocess
import sys
from zipfile import ZipFile

ROOT = Path(__file__).resolve().parents[1]


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--mode', choices=['eval', 'qwen3'], default='eval')
    parser.add_argument('--wheel', type=Path)
    args = parser.parse_args()
    manifest = json.loads((ROOT / 'runtime/manifest.json').read_text())
    cache = ROOT / '.runtime/downloads'
    cache.mkdir(parents=True, exist_ok=True)
    wheel = args.wheel or cache / 'transformers-4.57.0-py3-none-any.whl'
    if not wheel.exists():
        subprocess.run([sys.executable, '-m', 'pip', 'download', '--no-deps',
                        '--dest', str(cache), 'transformers==4.57.0'], check=True)
    if hashlib.sha256(wheel.read_bytes()).hexdigest() != manifest['wheel_sha256']:
        raise ValueError('Transformers wheel hash mismatch')
    target = ROOT / f'.runtime/{args.mode}/src'
    source_hashes = {}
    with ZipFile(wheel) as archive:
        for member in archive.infolist():
            if member.filename.startswith('transformers/') and not member.is_dir():
                if '..' in Path(member.filename).parts:
                    raise ValueError('Unsafe wheel member')
                dest = target / member.filename
                dest.parent.mkdir(parents=True, exist_ok=True)
                dest.write_bytes(archive.read(member))
                if args.mode == 'qwen3' and member.filename.endswith('.py'):
                    source_hashes[member.filename] = hashlib.sha256(archive.read(member)).hexdigest()
    if args.mode == 'qwen3':
        (target.parent / 'source_manifest.json').write_text(json.dumps({
            'wheel_sha256': manifest['wheel_sha256'], 'files': source_hashes}, indent=2) + '\n')
    for relative, expected in (manifest['overlays']['eval'] if args.mode == 'eval' else {}).items():
        src = ROOT / 'runtime/overlays/eval' / relative
        if hashlib.sha256(src.read_bytes()).hexdigest() != expected:
            raise ValueError(f'Overlay hash mismatch: {relative}')
        dest = target / relative
        dest.parent.mkdir(parents=True, exist_ok=True)
        shutil.copyfile(src, dest)
    print('Evaluation runtime prepared and verified')


if __name__ == '__main__':
    main()