File size: 1,548 Bytes
79df115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
"""Real local Gradio HTTP/API round trip with the tiny architecture, CPU only."""
import json
from pathlib import Path
import torch
from gradio_client import Client
import app
from engine import generate, POLICIES
from test_engine import Tokenizer
from test_model import tiny_model

def test_http_generate_and_download(monkeypatch):
    monkeypatch.setenv("NO_PROXY", "localhost,127.0.0.1,::1")
    monkeypatch.setenv("no_proxy", "localhost,127.0.0.1,::1")
    monkeypatch.setenv("GRADIO_ANALYTICS_ENABLED", "False")
    # All application callbacks, serialization, queueing and file download remain real.
    # Only the inference device is changed; no 9B checkpoint is loaded.
    monkeypatch.setattr(app, "generate", lambda m, t, ids, settings: generate(m, t, ids, settings, device="cpu"))
    torch.set_num_threads(1)
    demo = app.create_demo(tiny_model(), Tokenizer(), {"test_only": True})
    try:
        _, url, _ = demo.queue().launch(server_name="127.0.0.1", prevent_thread_lock=True, quiet=True)
        client = Client(url, verbose=False)
        response, metrics, table, report = client.predict("hello", "", 17, 16, 2, POLICIES[0], 112, True, api_name="/generate")
        assert isinstance(response, str)
        assert 0 < metrics["output_tokens"] <= 17
        assert len(table["data"]) >= 2
        saved = json.loads(Path(report).read_text())
        assert saved["prompt"] == "hello"
        assert saved["environment"]["test_only"] is True
        assert saved["settings"]["max_tokens"] == 17
    finally:
        demo.close()