lucataco's picture
Clairic, MLX 4-bit, with runtime, server and conversion script
1a48d6b verified
Raw History Blame Contribute Delete
2.02 kB
from __future__ import annotations
import re
from typing import Any
_CHAR_LIST_REQUEST = re.compile(
"\\b(?:(?:extract|list|return|split|read)\\s+(?:(?:the|of)\\s+)?|(?:each|every|all|individual|separate)\\s+)(?:(?:single|individual)\\s+)?(?:characters?|letters?|digits?|symbols?)\\b",
re.IGNORECASE,
)
class RejectRecord(ValueError):
def __init__(self, reason: str, field: str | None = None) -> None:
(self.reason, self.field) = (reason, field)
super().__init__(reason + (f": {field}" if field else ""))
def list_query_budget(text: str, spec: dict[str, Any]) -> tuple[int, str]:
maximum = spec.get("max_items")
if maximum is not None:
if isinstance(maximum, bool) or not isinstance(maximum, int) or maximum < 0:
raise RejectRecord("invalid_explicit_list_budget")
return (maximum + 1, "request_max_items")
instructions = str(spec.get("instructions", ""))
for match in _CHAR_LIST_REQUEST.finditer(instructions):
if not re.search(
"\\b(?:preserv\\w*|keep|retain\\w*|including|with)\\s*$",
instructions[max(0, match.start() - 25) : match.start()],
re.IGNORECASE,
):
return (sum((not char.isspace() for char in text)) + 1, "source_characters")
lines = [line for line in text.splitlines() if line.strip()]
row_list = bool(
re.search(
"\\b(?:rows?|top to bottom|line items?|item lines?|receipt order|printed row order)\\b",
instructions,
re.IGNORECASE,
)
or (
re.search("\\b(?:receipt|invoice|table)\\b", instructions, re.IGNORECASE)
and re.search(
"\\b(?:items?|products?|quantit\\w*|prices?|amounts?|names?)\\b",
instructions,
re.IGNORECASE,
)
)
)
if row_list and len(lines) > 1:
return (len(lines) + 1, "nonblank_source_lines")
return (len(re.findall("\\w+|[^\\w\\s]", text)) + 1, "source_lexical_units")