satquery-agentic / satquery_routing_patch.py
rudhratej09's picture
Phase 9 stability + Phase 5 tiling
4cffba0 verified
Raw History Blame Contribute Delete
6.45 kB
"""Task-conditional LoRA routing for SatQuery. v2 - written against the real
source, not guesses.
What the recon established, and why v1 could not have worked:
* `satquery.models.qwen_vl.load_model()` returns a **ModelHandle dataclass**,
not the model. The model is at `handle.model`. v1 looked for
`disable_adapter` on the returned object and found nothing.
* `qwen_vl.generate()` has no `task` argument - it only sees a finished
prompt string. The task lives one layer up, in
`satquery.tools.qwen._run(assets, query, params, task)`, which is what
`vlm_single` / `vlm_temporal` / `vlm_crossmodal` all call. That is the
routing hook.
* The `SATQUERY_LORA_ADAPTER` env-var patch is NOT in the cloned repo - it
was only ever an uncommitted Colab working-tree edit. So this module has
to attach the adapter itself.
Nothing in the repo is edited. Delete this file and the patch is gone.
import satquery_routing_patch as srp
srp.install(adapter_path="/path/to/best_epoch5")
"""
from __future__ import annotations
import functools
import inspect
import os
from contextlib import nullcontext
# Only these tasks get the fine-tuned adapter. Everything else runs on the
# base model, because the LoRA only ever saw count examples and answers
# yes/no questions with count-style text (measured: 61% -> 11% overall).
ADAPTER_TASKS = {"VQA_COUNT"}
_DEPTH = [0] # ROUTING_FIX reentrancy depth
_STATE = {
"installed": False,
"wrapped": [],
"handle": None,
"adapter_path": None,
"log": [],
}
def _task_name(task):
if task is None:
return None
for attr in ("value", "name"):
v = getattr(task, attr, None)
if isinstance(v, str):
return v
return str(task)
# ---------------------------------------------------------------------------
# Adapter attachment
# ---------------------------------------------------------------------------
def attach_adapter(adapter_path=None, **load_kwargs):
"""Load the model and wrap handle.model in a PeftModel, once.
ModelHandle is a plain (non-frozen) dataclass and load_model is
lru_cache'd, so mutating handle.model here is seen by every later caller
that hits the same cache entry.
"""
import satquery.models.qwen_vl as qvl
if _STATE["handle"] is not None:
return _STATE["handle"]
adapter_path = adapter_path or os.environ.get("SATQUERY_LORA_ADAPTER")
handle = qvl.load_model(**load_kwargs)
if adapter_path:
if hasattr(handle.model, "disable_adapter"):
handle.notes.append("adapter already attached")
else:
from peft import PeftModel
handle.model = PeftModel.from_pretrained(handle.model, adapter_path)
handle.notes.append(f"LoRA adapter attached from {adapter_path}")
_STATE["adapter_path"] = adapter_path
_STATE["handle"] = handle
return handle
def _peft_model():
h = _STATE["handle"]
if h is None:
return None
m = getattr(h, "model", None)
return m if hasattr(m, "disable_adapter") else None
# ---------------------------------------------------------------------------
# The hook
# ---------------------------------------------------------------------------
def _wrap(fn):
sig = inspect.signature(fn)
@functools.wraps(fn)
def inner(*args, **kwargs):
try:
bound = sig.bind_partial(*args, **kwargs)
tname = _task_name(bound.arguments.get("task"))
except Exception:
tname = None
use_adapter = tname in ADAPTER_TASKS
m = _peft_model()
if m is None:
ctx, mode = nullcontext(), "no-adapter"
elif use_adapter:
ctx, mode = nullcontext(), "ADAPTER"
else:
ctx, mode = m.disable_adapter(), "base"
_STATE["log"].append((tname, mode))
if os.environ.get("SATQUERY_ROUTING_VERBOSE"):
print(f"[routing] task={tname} -> {mode}", flush=True)
# ROUTING_FIX: PEFT's disable_adapter() is not reentrant -- its exit
# re-ENABLES the adapter even inside an outer disable. install() also
# wraps build_prompt (it takes `task`), so for every non-count task
# the adapter came back on between building the prompt and
# generating. Only the outermost routed call now switches weights.
if _DEPTH[0] > 0:
return fn(*args, **kwargs)
_DEPTH[0] += 1
try:
with ctx:
return fn(*args, **kwargs)
finally:
_DEPTH[0] -= 1
inner.__satquery_routed__ = True
return inner
def find_targets():
"""Functions in satquery.tools.qwen that carry a `task` argument."""
import satquery.tools.qwen as tq
found = []
for name, fn in list(vars(tq).items()):
if not inspect.isfunction(fn):
continue
params = list(inspect.signature(fn).parameters)
found.append({"name": name, "has_task": "task" in params,
"params": params})
return found
def install(adapter_path=None, verbose=True, **load_kwargs):
if _STATE["installed"]:
return _STATE
import satquery.tools.qwen as tq
attach_adapter(adapter_path, **load_kwargs)
targets = find_targets()
for t in targets:
if not t["has_task"]:
continue
fn = getattr(tq, t["name"])
if getattr(fn, "__satquery_routed__", False):
continue
setattr(tq, t["name"], _wrap(fn))
_STATE["wrapped"].append(t["name"])
_STATE["installed"] = True
_STATE["targets"] = targets
if verbose:
print("[routing] functions in satquery.tools.qwen:")
for t in targets:
flag = "HOOKED" if t["has_task"] else "-"
print(f" {t['name']:22s} {flag:8s} params={t['params']}")
m = _peft_model()
print(f"[routing] wrapped : {_STATE['wrapped'] or 'NOTHING'}")
print(f"[routing] adapter path : {_STATE['adapter_path']}")
print(f"[routing] peft model : {type(m).__name__ if m else None}")
if not _STATE["wrapped"]:
print("[routing] !! no function with a `task` arg - nothing routed")
if m is None:
print("[routing] !! adapter NOT attached - every task runs on base")
return _STATE
def report():
from collections import Counter
return Counter(_STATE["log"])