File size: 6,007 Bytes
2bff5be d004991 2bff5be 2b9255b 2bff5be eb373ad 2bff5be | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | """Exercise source selection, highlighting, persistence and mobile layout in the actual WASM UI."""
import argparse
from contextlib import contextmanager
from functools import partial
import hashlib
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
import json
from pathlib import Path
from tempfile import TemporaryDirectory
import threading
from playwright.sync_api import sync_playwright, expect
@contextmanager
def fixture(export, head, config):
spec = json.loads((head / "config.json").read_text())
with TemporaryDirectory(prefix="franok-reader-ui-") as folder:
root = Path(folder)
for p in Path("web").iterdir():
if p.name not in ("model.json", "release.json", "reader.safetensors"):
(root / p.name).symlink_to(p.resolve())
(root / "model.json").symlink_to((export / "model.json").resolve())
(root / "fixture.q4").symlink_to((export / "model.q4").resolve())
weights = head / "head.safetensors"
(root / "reader.safetensors").symlink_to(weights.resolve())
release = json.loads(Path("web/release.json").read_text())
release["formats"]["q4"]["url"] = "./fixture.q4"
release["formats"].pop("q8", None) # the fixture is Q4, so the page must not choose Q8
# The test fixture exercises both possible decisions independently of task accuracy.
release["reader"] = dict(
**spec,
url="./reader.safetensors",
baseSha256=(release["formats"].get("fp16") or release["formats"]["fp32"])["sha256"],
sha256=hashlib.sha256(weights.read_bytes()).hexdigest(),
)
release["reader"]["threshold"] = config
(root / "release.json").write_text(json.dumps(release))
server = ThreadingHTTPServer(
("127.0.0.1", 0), partial(SimpleHTTPRequestHandler, directory=folder)
)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
yield f"http://127.0.0.1:{server.server_port}/"
finally:
server.shutdown()
server.server_close()
def verify(browser, url, output, expect_answer):
page = browser.new_page(viewport=dict(width=1100, height=780))
errors = []
page.on("pageerror", lambda e: errors.append(str(e)))
page.goto(url)
page.locator("#lang").select_option("tr")
page.locator("#readerOpen").click()
expect(page.get_by_role("dialog", name="Metin hakkında soru sor")).to_be_visible()
passage = (
"Salonun kapısındaki küçük kartta IŞIK yazıyor. Toplantı cuma sabahı başlıyor."
)
page.locator("#sourceText").fill(passage)
page.locator("#sourceQuestion").fill("Kartta ne yazıyor?")
page.screenshot(path=output / "reader-form.png")
page.locator("#readSource").click()
expect(page.locator("#status")).to_have_text(
"Metin bu cihazda okundu.", timeout=250000
)
result = page.locator(".message.assistant").last
answer = result.locator(".body").inner_text()
accepted = bool(result.locator("blockquote").count())
if expect_answer is not None:
assert accepted == expect_answer, (accepted, answer)
if accepted:
assert answer and answer in passage
result.locator("summary").click()
expect(result.locator("mark")).to_have_text(answer)
assert result.locator(".sourcepassage p").inner_text() == passage
else:
assert "bulamadı" in answer
page.reload()
expect(page.locator(".message.assistant .body")).to_have_text(answer)
page.set_viewport_size(dict(width=360, height=740))
assert page.evaluate("document.documentElement.scrollWidth <= innerWidth")
page.locator("#readerOpen").click()
expect(page.locator("#readSource")).to_be_in_viewport()
page.screenshot(path=output / "reader-mobile.png")
page.keyboard.press("Escape")
expect(page.locator("#readingDialog")).not_to_be_visible()
page.locator("#find").click()
page.locator("#query").fill("Kartta")
page.get_by_role("button", name="Attach turn 1 to next message").click()
page.locator("#readerOpen").click()
expect(page.locator("#sourceText")).to_have_value(passage)
second = "Toplantı ne zaman başlıyor?"
page.locator("#sourceQuestion").fill(second)
page.locator("#readSource").click()
expect(page.locator("#status")).to_have_text(
"Metin bu cihazda okundu.", timeout=180000
)
page.reload()
expect(page.locator(".message.user").last.locator(".body")).to_have_text(second)
expect(
page.locator(".message.user").last.get_by_text("Attached turn 1")
).to_be_visible()
details = page.locator(".message.assistant").last.locator(".sourcepassage")
details.locator("summary").click()
assert details.locator("p").inner_text() == passage
assert not errors, errors
page.close()
return dict(accepted=accepted, answer=answer)
if __name__ == "__main__":
p = argparse.ArgumentParser()
p.add_argument("--export", type=Path)
p.add_argument("--head", type=Path)
p.add_argument("--url")
p.add_argument("--out", type=Path, required=True)
a = p.parse_args()
a.out.mkdir(parents=True, exist_ok=True)
with sync_playwright() as pw:
browser = pw.chromium.launch(executable_path="/usr/bin/google-chrome")
receipts = []
if a.url:
receipts.append(verify(browser, a.url, a.out, None))
else:
for threshold, accepted in ((-1e6, True), (1e6, False)):
directory = a.out / ("quote" if accepted else "abstain")
directory.mkdir(exist_ok=True)
with fixture(a.export, a.head, threshold) as url:
receipts.append(verify(browser, url, directory, accepted))
browser.close()
(a.out / "results.json").write_text(
json.dumps(receipts, ensure_ascii=False, indent=2) + "\n"
)
print(receipts, flush=True)
|