File size: 5,231 Bytes
104376c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
"""Standalone Nima inference. Imported MLX and weights only inside load/predict."""
import argparse
import hashlib
import json
from pathlib import Path
import time
from encoding import RULES, choices, encode
from head import create_head, pack

HERE = Path(__file__).resolve().parent


def digest(path):
    h = hashlib.sha256()
    with Path(path).open('rb') as handle:
        for block in iter(lambda: handle.read(1024 * 1024), b''):
            h.update(block)
    return h.hexdigest()


def load(release_dir=HERE, local_files_only=False):
    import mlx.core as mx
    import mlx.nn as nn
    from mlx.utils import tree_flatten
    from mlx_lm import load as load_qwen
    from huggingface_hub import snapshot_download

    folder = Path(release_dir)
    config = json.loads((folder / 'config.json').read_text())
    architecture = json.loads((folder / 'architecture.json').read_text())
    head_path = folder / config['head_file']
    if digest(head_path) != config['head_sha256']:
        raise ValueError('Nima head checksum mismatch')
    snapshot = Path(snapshot_download(
        config['base_model'], revision=config['base_revision'],
        allow_patterns=list(config['base_file_sha256']),
        local_files_only=local_files_only,
    ))
    for name, expected in config['base_file_sha256'].items():
        if digest(snapshot / name) != expected:
            raise ValueError(f'Frozen backbone file checksum mismatch: {name}')
    model, tokenizer, base_config = load_qwen(str(snapshot), return_config=True)
    if model.model_type != 'qwen3' or base_config.get('hidden_size') != architecture['hidden_size']:
        raise ValueError('Expected the pinned Qwen3 architecture')
    if base_config.get('quantization', {}).get('bits') != 4:
        raise ValueError('Expected a 4-bit backbone')
    if not any(isinstance(module, nn.QuantizedLinear) for _, module in model.named_modules()):
        raise ValueError('Quantized backbone layers missing')
    model.freeze()
    model.eval()
    if tree_flatten(model.trainable_parameters()):
        raise ValueError('Backbone must be completely frozen')
    head = create_head(architecture)
    head.load_weights(str(head_path))
    head.eval()
    if sum(value.size for _, value in tree_flatten(head.parameters())) != config['head_parameters']:
        raise ValueError('Head parameter count mismatch')
    mx.eval(model.parameters(), head.parameters())
    return Nima(model, tokenizer, head, architecture, config)


class Nima:
    def __init__(self, model, tokenizer, head, architecture, config):
        self.model, self.tokenizer, self.head = model, tokenizer, head
        self.architecture, self.config = architecture, config

    def predict(self, request):
        import mlx.core as mx
        if not isinstance(request.get('prompt'), str) or not request['prompt'].strip():
            raise ValueError('A nonempty prompt is required')
        if not isinstance(request.get('context'), dict) or not {'current_date', 'timezone'} <= request['context'].keys():
            raise ValueError('context requires current_date and timezone')
        names = choices(request)
        if 'no_tool' not in names:
            raise ValueError('Include an explicit no_tool choice to allow abstention')
        case = {**request, 'id': 'custom', 'target': names[0]}
        started = time.perf_counter()
        encoded = encode(case, names, self.tokenizer, RULES, self.architecture['max_tokens'])
        # Torso only: Model.__call__, vocabulary projection and generation are never used.
        hidden = self.model.model(mx.array([encoded['token_ids']], dtype=mx.int32), cache=None)
        evidence = mx.stop_gradient(hidden[0, mx.array(encoded['evidence_positions']), :].astype(mx.float32))
        candidates = mx.stack([mx.mean(hidden[0, span['start']:span['end'], :], axis=0) for span in encoded['candidate_spans']])
        candidates = mx.stop_gradient(candidates.astype(mx.float32))
        mx.eval(evidence, candidates)
        q, e, qv, ev, _ = pack([{'candidates': candidates, 'evidence': evidence, 'label': 0}])
        logits = self.head(q, e, qv, ev)[0, :len(names)]
        probabilities = mx.softmax(logits / self.config['temperature'])
        mx.eval(logits, probabilities)
        elapsed = time.perf_counter() - started
        scores = dict(zip(names, probabilities.tolist()))
        winner = min(scores, key=lambda name: (-scores[name], name))
        return {'prediction': winner, 'probabilities': scores,
                'logits': dict(zip(names, logits.tolist())), 'latency_seconds': elapsed,
                'input_tokens': len(encoded['token_ids']), 'qwen_forward_passes': 1,
                'note': 'Probabilities are uncalibrated and relative to the supplied choices; no tool executes.'}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--input', type=Path, required=True)
    parser.add_argument('--local-files-only', action='store_true')
    args = parser.parse_args()
    request = json.loads(args.input.read_text())
    router = load(local_files_only=args.local_files_only)
    print(json.dumps(router.predict(request), indent=2, ensure_ascii=False))


if __name__ == '__main__':
    main()