File size: 6,803 Bytes
ab18ce0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
"""Shared, resumable, mismatch-only Qwen27B post-processing for saved outputs."""
from __future__ import annotations

import argparse
from concurrent.futures import ThreadPoolExecutor, as_completed
import fcntl
import hashlib
import json
import os
from pathlib import Path
import time
import urllib.request

from .protocol import DEFAULT_MODEL, PROTOCOL, exact_match, make_request, validate
from .scoring import summarize


def atomic_json(path, value):
    tmp = path.with_suffix(path.suffix + '.tmp')
    tmp.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + '\n')
    tmp.replace(path)


def query(base_url, payload, timeout=180):
    headers = {'Content-Type': 'application/json'}
    if os.environ.get('JUDGE_API_KEY'):
        headers['Authorization'] = 'Bearer ' + os.environ['JUDGE_API_KEY']
    request = urllib.request.Request(base_url.rstrip('/') + '/chat/completions',
                                     data=json.dumps(payload).encode(), headers=headers)
    for attempt in range(3):
        try:
            with urllib.request.urlopen(request, timeout=timeout) as response:
                body = json.load(response)
            choice = body['choices'][0]
            text = choice['message']['content'].strip()
            if choice.get('finish_reason') == 'length' or text not in ('CORRECT', 'INCORRECT'):
                raise ValueError(f'Invalid judge verdict: {text!r}')
            return text == 'CORRECT', body
        except Exception:
            if attempt == 2:
                raise
            time.sleep(2 ** attempt)


def run(input_path, output_dir, *, image_root, base_url, model, revision, concurrency=4, dry_run=False):
    if revision == 'unspecified' or not revision.strip():
        raise ValueError('Pin the judge checkpoint revision (or local content hash)')
    output_dir.mkdir(parents=True, exist_ok=True)
    with (output_dir / '.lock').open('w') as lock:
        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
        raw = input_path.read_bytes()
        records = [json.loads(line) for line in raw.splitlines() if line.strip()]
        if not records or len({(r['dataset'], str(r['sample_id'])) for r in records}) != len(records):
            raise ValueError('Empty input or duplicate sample IDs')
        contract = {'protocol': PROTOCOL, 'input_sha256': hashlib.sha256(raw).hexdigest(),
                    'model': model, 'judge_revision': revision, 'rows': len(records)}
        manifest = output_dir / 'manifest.json'
        if manifest.exists() and json.loads(manifest.read_text()) != contract:
            raise ValueError('Existing outputs have a different input/model/protocol; use a new directory')
        atomic_json(manifest, contract)
        cache_path = output_dir / 'judge_cache.jsonl'
        cache = {}
        if cache_path.exists():
            for line in cache_path.read_text().splitlines():
                cached = json.loads(line)
                if cached['fingerprint'] in cache:
                    raise ValueError('Duplicate judge-cache entry')
                cache[cached['fingerprint']] = cached
        results, pending = [], []
        for row in records:
            validate(row)
            matched = exact_match(row)
            result = {**row, 'exact_match': matched, 'correct': matched,
                      'source': 'deterministic_match' if matched else 'qwen27_pending', 'protocol': PROTOCOL}
            results.append(result)
            if matched:
                continue
            payload, key, identity = make_request(row, image_root, model, revision)
            result['judge_fingerprint'] = key
            if key in cache:
                result.update(correct=cache[key]['correct'], source='qwen27_visual_gt')
            else:
                pending.append((result, payload, key, identity))
        preflight = {**contract, 'exact_matches': sum(r['exact_match'] for r in results),
                     'fallback_total': sum(not r['exact_match'] for r in results),
                     'pending_requests': len(pending), 'status': 'prepared'}
        atomic_json(output_dir / 'preflight.json', preflight)
        if dry_run:
            return preflight
        errors = []
        with ThreadPoolExecutor(max_workers=concurrency) as pool, cache_path.open('a') as handle:
            futures = {pool.submit(query, base_url, payload): (result, key, identity)
                       for result, payload, key, identity in pending}
            for future in as_completed(futures):
                result, key, identity = futures[future]
                try:
                    correct, body = future.result()
                    cached = {'fingerprint': key, 'correct': correct, 'identity': identity, 'response': body}
                    handle.write(json.dumps(cached, ensure_ascii=False) + '\n')
                    handle.flush()
                    os.fsync(handle.fileno())
                    result.update(correct=correct, source='qwen27_visual_gt')
                except Exception as error:
                    errors.append({'sample_id': result['sample_id'], 'error': str(error)})
        if errors:
            atomic_json(output_dir / 'summary.json', {**preflight, 'status': 'incomplete', 'errors': errors})
            raise RuntimeError(f'{len(errors)} judge requests failed; successful entries cached, no score published')
        temp = output_dir / 'judged.jsonl.tmp'
        with temp.open('w') as handle:
            for result in results:
                handle.write(json.dumps(result, ensure_ascii=False, allow_nan=False) + '\n')
        temp.replace(output_dir / 'judged.jsonl')
        report = {**preflight, 'status': 'completed', 'pending_requests': 0,
                  'scores': summarize(results), 'official_protocol_equivalent': False}
        atomic_json(output_dir / 'summary.json', report)
        return report


def main():
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('--input', type=Path, required=True)
    p.add_argument('--output-dir', type=Path, required=True)
    p.add_argument('--image-root', type=Path, default=Path('.'))
    p.add_argument('--base-url', default='http://127.0.0.1:8000/v1')
    p.add_argument('--model', default=DEFAULT_MODEL)
    p.add_argument('--judge-revision', required=True)
    p.add_argument('--concurrency', type=int, default=4)
    p.add_argument('--dry-run', action='store_true')
    a = p.parse_args()
    if a.concurrency < 1:
        p.error('concurrency must be positive')
    print(json.dumps(run(a.input, a.output_dir, image_root=a.image_root, base_url=a.base_url,
                         model=a.model, revision=a.judge_revision, concurrency=a.concurrency,
                         dry_run=a.dry_run), indent=2))


if __name__ == '__main__':
    main()