            fqn=fqn,
            name=node.name,
            qualname=self._cur_qualname(),
            kind="class",
            module=self.module,
            file=self.path,
            line=getattr(node, "lineno", 1),
            end_line=getattr(node, "end_lineno", getattr(node, "lineno", 1)),
            doc=doc_s,
        )
        self.symbols.append(sym)
        self.stack.append(node.name)
        self.class_stack.append(node.name)
        self.generic_visit(node)
        self.class_stack.pop()
        self.stack.pop()

    def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:  # type: ignore[override]
        self._visit_func_like(node)

    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Any:  # type: ignore[override]
        self._visit_func_like(node)

    def _visit_func_like(self, node: Any) -> None:
        fqn = self._fqn(node.name)
        # Signature & returns
        sig_s, ret_s = None, None
        try:
            params = []
            for a in getattr(node, "args", None).args or []:
                nm = getattr(a, "arg", "")
                ann = getattr(a, "annotation", None)
                params.append(f"{nm}:{ast.unparse(ann)}" if ann is not None else nm)
            ret = getattr(node, "returns", None)
            ret_s = ast.unparse(ret) if ret is not None else None
            sig_s = f"({', '.join(params)})"
        except Exception:
            sig_s, ret_s = None, None
        try:
            doc_s = ast.get_docstring(node) or None
        except Exception:
            doc_s = None


            kind="class",
            module=self.module,
            file=self.path,
            line=getattr(node, "lineno", 1),
            end_line=getattr(node, "end_lineno", getattr(node, "lineno", 1)),
            doc=doc_s,
        )
        self.symbols.append(sym)
        self.stack.append(node.name)
        self.class_stack.append(node.name)
        self.generic_visit(node)
        self.class_stack.pop()
        self.stack.pop()

    def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:  # type: ignore[override]
        self._visit_func_like(node)

    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Any:  # type: ignore[override]
        self._visit_func_like(node)

    def _visit_func_like(self, node: Any) -> None:
        fqn = self._fqn(node.name)
        # Signature & returns
        sig_s, ret_s = None, None
        try:
            params = []
            for a in getattr(node, "args", None).args or []:
                nm = getattr(a, "arg", "")
                ann = getattr(a, "annotation", None)
                params.append(f"{nm}:{ast.unparse(ann)}" if ann is not None else nm)
            ret = getattr(node, "returns", None)
            ret_s = ast.unparse(ret) if ret is not None else None
            sig_s = f"({', '.join(params)})"
        except Exception:
            sig_s, ret_s = None, None
        try:
            doc_s = ast.get_docstring(node) or None
        except Exception:
            doc_s = None
        sym = Symbol(
            fqn=fqn,
            name=node.name,
            qualname=self._cur_qualname(),
            kind="function",
            module=self.module,
            file=self.path,
            line=getattr(node, "lineno", 1),
            end_line=getattr(node, "end_lineno", getattr(node, "lineno", 1)),
            doc=doc_s,
            signature=sig_s,
            returns=ret_s,
        )
        self.symbols.append(sym)
        self.stack.append(node.name)
        # Traverse body to collect calls
        for sub in ast.walk(node):
            if isinstance(sub, ast.Call):
                callee_key = self._extract_callee_key(sub.func)
                if callee_key:
                    self.calls.append((fqn, callee_key))
        # Decorators as calls
        for dec in getattr(node, "decorator_list", []) or []:
            callee_key = self._extract_callee_key(dec)
            if callee_key:
                self.calls.append((fqn, callee_key))
        self.stack.pop()

    def visit_Assign(self, node: ast.Assign) -> Any:  # type: ignore[override]
        for t in getattr(node, "targets", []) or []:
            if isinstance(t, ast.Name):
                fqn = self._fqn(t.id)
                sym = Symbol(
                    fqn=fqn,
                    name=t.id,
                    qualname=self._cur_qualname(),
                    kind="variable",
                    module=self.module,
                    file=self.path,
                    line=getattr(node, "lineno", 1),
                    end_line=getattr(node, "end_lineno", getattr(node, "lineno", 1)),
                )
                self.symbols.append(sym)
        # capture __all__ = ["..."]
        try:
            names = []
            is_all = any(


            end_line=getattr(node, "end_lineno", getattr(node, "lineno", 1)),
            doc=doc_s,
            signature=sig_s,
            returns=ret_s,
        )
        self.symbols.append(sym)
        self.stack.append(node.name)
        # Traverse body to collect calls
        for sub in ast.walk(node):
            if isinstance(sub, ast.Call):
                callee_key = self._extract_callee_key(sub.func)
                if callee_key:
                    self.calls.append((fqn, callee_key))
        # Decorators as calls
        for dec in getattr(node, "decorator_list", []) or []:
            callee_key = self._extract_callee_key(dec)
            if callee_key:
                self.calls.append((fqn, callee_key))
        self.stack.pop()

    def visit_Assign(self, node: ast.Assign) -> Any:  # type: ignore[override]
        for t in getattr(node, "targets", []) or []:
            if isinstance(t, ast.Name):
                fqn = self._fqn(t.id)
                sym = Symbol(
                    fqn=fqn,
                    name=t.id,
                    qualname=self._cur_qualname(),
                    kind="variable",
                    module=self.module,
                    file=self.path,
                    line=getattr(node, "lineno", 1),
                    end_line=getattr(node, "end_lineno", getattr(node, "lineno", 1)),
                )
                self.symbols.append(sym)
        # capture __all__ = ["..."]
        try:
            names = []
            is_all = any(
                (isinstance(t, ast.Name) and t.id == "__all__") for t in node.targets
            )
            if is_all and isinstance(node.value, (ast.List, ast.Tuple)):
                for el in node.value.elts:
                    if isinstance(el, ast.Constant) and isinstance(el.value, str):
                        names.append(el.value)
            if names:
                self.exports.extend(names)
        except Exception:
            pass
        self.generic_visit(node)

    def _extract_callee_key(self, fn: ast.AST) -> Optional[str]:
        # simple name
        if isinstance(fn, ast.Name):
            return fn.id

        # super().method()
        if (
            isinstance(fn, ast.Attribute)
            and isinstance(fn.value, ast.Call)
            and isinstance(fn.value.func, ast.Name)
            and fn.value.func.id == "super"
        ):
            meth = fn.attr
            cur_cls = self._cur_class()
            if cur_cls:
                return f"{self.module}.{cur_cls}.{meth}"
            return meth

        # obj.attr chain


                    line=getattr(node, "lineno", 1),
                    end_line=getattr(node, "end_lineno", getattr(node, "lineno", 1)),
                )
                self.symbols.append(sym)
        # capture __all__ = ["..."]
        try:
            names = []
            is_all = any(
                (isinstance(t, ast.Name) and t.id == "__all__") for t in node.targets
            )
            if is_all and isinstance(node.value, (ast.List, ast.Tuple)):
                for el in node.value.elts:
                    if isinstance(el, ast.Constant) and isinstance(el.value, str):
                        names.append(el.value)
            if names:
                self.exports.extend(names)
        except Exception:
            pass
        self.generic_visit(node)

    def _extract_callee_key(self, fn: ast.AST) -> Optional[str]:
        # simple name
        if isinstance(fn, ast.Name):
            return fn.id

        # super().method()
        if (
            isinstance(fn, ast.Attribute)
            and isinstance(fn.value, ast.Call)
            and isinstance(fn.value.func, ast.Name)
            and fn.value.func.id == "super"
        ):
            meth = fn.attr
            cur_cls = self._cur_class()
            if cur_cls:
                return f"{self.module}.{cur_cls}.{meth}"
            return meth

        # obj.attr chain
        if isinstance(fn, ast.Attribute):
            parts: List[str] = []
            cur = fn
            while isinstance(cur, ast.Attribute):
                parts.append(cur.attr)
                cur = cur.value
            parts.reverse()

            if isinstance(cur, ast.Name):
                base = cur.id
                if base in ("self", "cls"):
                    cur_cls = self._cur_class()
                    if cur_cls and parts:
                        return f"{self.module}.{cur_cls}.{parts[-1]}"
                    return f"{self.module}.{cur_cls}" if cur_cls else parts[-1]
                if base in self.imports:
                    return f"{base}:{parts[-1]}" if parts else base
                return f"{self.module}.{base}.{parts[-1]}" if parts else base
        # getattr(module, "name") heuristic
        if (
            isinstance(fn, ast.Call)
            and isinstance(fn.func, ast.Name)
            and fn.func.id == "getattr"
            and fn.args
            and len(fn.args) >= 2
            and isinstance(fn.args[0], ast.Name)
            and isinstance(fn.args[1], ast.Constant)
            and isinstance(fn.args[1].value, str)
        ):
            base = fn.args[0].id
            name = fn.args[1].value
            if base in self.imports:
                return f"{self.imports[base]}.{name}"
            cur_cls = self._cur_class()
            if base in ("self", "cls") and cur_cls:
                return f"{self.module}.{cur_cls}.{name}"
            return f"{self.module}.{base}.{name}"
        # importlib.import_module("pkg.mod") heuristic
        if (
            isinstance(fn, ast.Call)
            and isinstance(fn.func, ast.Attribute)
            and isinstance(fn.func.value, ast.Name)
            and fn.func.value.id == "importlib"
            and fn.func.attr == "import_module"
            and fn.args
            and isinstance(fn.args[0], ast.Constant)
            and isinstance(fn.args[0].value, str)
        ):
            mod = str(fn.args[0].value)
            return mod
        return None

if __name__ == "__main__":
    _cli()


from __future__ import annotations

import os
import json
from pathlib import Path
from typing import Dict, List, Tuple, Any, Iterable

def _read_json(path: Path) -> Dict[str, Any]:
	try:
		with open(path, "r", encoding="utf-8") as fh:
			return json.loads(fh.read())
	except Exception:
		return {}

def _load_local_jsonl(fp: Path, text_key: str = "text", max_n: int | None = None) -> List[str]:
	texts: List[str] = []
	if not fp.exists():
		return texts
	with open(fp, "r", encoding="utf-8") as fh:
		for line in fh:
			line = line.strip()
			if not line:
				continue
			try:
				obj = json.loads(line)
			except Exception:
				continue
			txt = str(obj.get(text_key) or "").strip()
			if txt:
				texts.append(txt)
			if max_n is not None and len(texts) >= int(max_n):
				break
	return texts

def _load_hf(repo: str, subset: str | None, split: str, max_n: int | None, cache_dir: Path, streaming: bool) -> List[str]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception:
		return []
	try:
		if subset:
			ds = load_dataset(repo, subset, split=split, cache_dir=str(cache_dir), streaming=streaming)
		else:
			ds = load_dataset(repo, split=split, cache_dir=str(cache_dir), streaming=streaming)
	except Exception:
		return []
	texts: List[str] = []
	if streaming:
		count = 0
		for row in ds:  # type: ignore
			prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
			code = str(row.get("code") or row.get("solution") or "").strip()
			combined = (prompt + ("\n" + code if code else "")).strip()
			if combined:
				texts.append(combined)
				count += 1
			if max_n is not None and count >= int(max_n):
				break
		return texts
	# non-streaming
	total = len(ds)  # type: ignore
	for i in range(total):
		row = ds[i]  # type: ignore
		prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
		code = str(row.get("code") or row.get("solution") or "").strip()
		combined = (prompt + ("\n" + code if code else "")).strip()
		if combined:
			texts.append(combined)
		if max_n is not None and len(texts) >= int(max_n):
			break
	return texts

def load_program_texts(example_dir: str, config_path: str | None = None, train_new_only: bool = False, state_path: str | None = None) -> Tuple[List[str], List[str]]:
	"""
	Load training texts from a config that can reference local and HF datasets.
	Priority: local first; if missing, fetch from HF (optionally streaming) and store under local_dir.

	Returns (texts, loaded_source_names)
	"""
	ex_dir = Path(example_dir).resolve()
	# Config discovery
	cfg_path = Path(config_path) if config_path else (ex_dir / "datasets" / "config.json")
	cfg = _read_json(cfg_path) if cfg_path.exists() else {}
	local_dir = Path(cfg.get("local_dir") or (ex_dir / "datasets")).resolve()
	local_dir.mkdir(parents=True, exist_ok=True)
	cache_dir = local_dir / "hf_cache"
	cache_dir.mkdir(parents=True, exist_ok=True)
	sources: List[Dict[str, Any]] = list(cfg.get("sources") or [])
	streaming = os.getenv("HF_DATASETS_STREAMING", "1") == "1"
	full_datasets = os.getenv("FULL_DATASETS", "1") == "1"
	# Empty/0 -> unlimited by default
	_default_env = os.getenv("DATASET_MAX_N_PER_SOURCE", "") or "0"
	try:
		default_max_n_val = int(_default_env)
	except Exception:
		default_max_n_val = 0

	# Optional program state to support "train new only"
	seen: List[str] = []
	if train_new_only and state_path:
		try:
			st = _read_json(Path(state_path))
			seen = list(st.get("datasets_seen") or [])
		except Exception:
			seen = []

	texts_acc: List[str] = []
	loaded_names: List[str] = []
	for src in sources:
		name = str(src.get("name") or "").strip()
		if not name:
			continue
		if train_new_only and name in seen:
			continue
		if full_datasets:
			max_n = None
		else:
			max_n = src.get("max_n")
			if max_n is None and default_max_n_val > 0:
				max_n = default_max_n_val
		text_key = str(src.get("text_key") or "text").strip()
		subset = src.get("subset")
		split = str(src.get("split") or "train").strip()
		# Determine local file path: explicit 'path' or derived from name+split
		rel = str(src.get("path") or "").strip()
		if rel:
			fp = Path(rel)
			if not fp.is_absolute():
				fp = (local_dir / rel).resolve()
		else:
			suffix = f"_{split}" if split else ""
			fp = (local_dir / f"{name.replace('/', '_')}{suffix}.jsonl").resolve()
		# Try local first
		out_texts: List[str] = _load_local_jsonl(fp, text_key=text_key, max_n=max_n)
		# If not present, try HF using 'name' as repo id
		if not out_texts:
			out_texts = _load_hf(name, subset, split, max_n, cache_dir, streaming=streaming)
			# Persist to local for future runs
			if out_texts:
				try:
					with open(fp, "w", encoding="utf-8") as fh:
						for t in out_texts:
							fh.write(json.dumps({"text": t}) + "\n")
				except Exception:
					pass
		if out_texts:
			texts_acc.extend(out_texts)
			loaded_names.append(f"{name}:{split}" if split else name)
	return texts_acc, loaded_names

def iter_program_texts(example_dir: str, config_path: str | None = None, train_new_only: bool = False, state_path: str | None = None) -> Iterable[Tuple[List[str], str]]:
	"""
	Yields (texts, source_name) one dataset at a time using the same resolution rules as load_program_texts.
	Respects HF streaming and per-source max limits.
	"""
	ex_dir = Path(example_dir).resolve()
	cfg_path = Path(config_path) if config_path else (ex_dir / "datasets" / "config.json")
	cfg = _read_json(cfg_path) if cfg_path.exists() else {}
	local_dir = Path(cfg.get("local_dir") or (ex_dir / "datasets")).resolve()
	local_dir.mkdir(parents=True, exist_ok=True)
	cache_dir = local_dir / "hf_cache"
	cache_dir.mkdir(parents=True, exist_ok=True)
	sources: List[Dict[str, Any]] = list(cfg.get("sources") or [])
	streaming = os.getenv("HF_DATASETS_STREAMING", "1") == "1"
	full_datasets = os.getenv("FULL_DATASETS", "1") == "1"
	_default_env = os.getenv("DATASET_MAX_N_PER_SOURCE", "") or "0"
	try:
		default_max_n_val = int(_default_env)
	except Exception:
		default_max_n_val = 0
	max_sources = int(os.getenv("DATASET_LIMIT_SOURCES", "0") or "0")  # 0 means no limit

	seen: List[str] = []
	if train_new_only and state_path:
		try:
			st = _read_json(Path(state_path))
			seen = list(st.get("datasets_seen") or [])
		except Exception:
			seen = []

	count_sources = 0
	for src in sources:
		name = str(src.get("name") or "").strip()
		if not name:
			continue
		if train_new_only and name in seen:
			continue
		if full_datasets:
			max_n = None
		else:
			max_n = src.get("max_n")
			if max_n is None and default_max_n_val > 0:
				max_n = default_max_n_val
		text_key = str(src.get("text_key") or "text").strip()
		subset = src.get("subset")
		split = str(src.get("split") or "train").strip()
		rel = str(src.get("path") or "").strip()
		if rel:
			fp = Path(rel)
			if not fp.is_absolute():
				fp = (local_dir / rel).resolve()
		else:
			suffix = f"_{split}" if split else ""
			fp = (local_dir / f"{name.replace('/', '_')}{suffix}.jsonl").resolve()
		out_texts: List[str] = _load_local_jsonl(fp, text_key=text_key, max_n=max_n)
		if not out_texts:
			out_texts = _load_hf(name, subset, split, max_n, cache_dir, streaming=streaming)
			if out_texts:
				try:
					with open(fp, "w", encoding="utf-8") as fh:
						for t in out_texts:
							fh.write(json.dumps({"text": t}) + "\n")
				except Exception:
					pass
		if out_texts:
			yield out_texts, (f"{name}:{split}" if split else name)
			count_sources += 1
			if max_sources and count_sources >= max_sources:
				break


from __future__ import annotations

import os
import json
from pathlib import Path
from typing import Dict, List, Tuple, Any, Iterable

def _read_json(path: Path) -> Dict[str, Any]:
	try:
		with open(path, "r", encoding="utf-8") as fh:
			return json.loads(fh.read())
	except Exception:
		return {}

def _load_local_jsonl(fp: Path, text_key: str = "text", max_n: int | None = None) -> List[str]:
	texts: List[str] = []
	if not fp.exists():
		return texts
	with open(fp, "r", encoding="utf-8") as fh:
		for line in fh:
			line = line.strip()
			if not line:
				continue
			try:
				obj = json.loads(line)
			except Exception:
				continue
			txt = str(obj.get(text_key) or "").strip()
			if txt:
				texts.append(txt)
			if max_n is not None and len(texts) >= int(max_n):
				break


from __future__ import annotations

import os
import json
from pathlib import Path
from typing import Dict, List, Tuple, Any, Iterable

def _read_json(path: Path) -> Dict[str, Any]:
	try:
		with open(path, "r", encoding="utf-8") as fh:
			return json.loads(fh.read())
	except Exception:
		return {}

def _load_local_jsonl(fp: Path, text_key: str = "text", max_n: int | None = None) -> List[str]:
	texts: List[str] = []
	if not fp.exists():
		return texts
	with open(fp, "r", encoding="utf-8") as fh:
		for line in fh:
			line = line.strip()
			if not line:
				continue
			try:
				obj = json.loads(line)
			except Exception:
				continue
			txt = str(obj.get(text_key) or "").strip()
			if txt:
				texts.append(txt)
			if max_n is not None and len(texts) >= int(max_n):
				break
	return texts

def _load_hf(repo: str, subset: str | None, split: str, max_n: int | None, cache_dir: Path, streaming: bool) -> List[str]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception:
		return []
	try:
		if subset:
			ds = load_dataset(repo, subset, split=split, cache_dir=str(cache_dir), streaming=streaming)
		else:
			ds = load_dataset(repo, split=split, cache_dir=str(cache_dir), streaming=streaming)
	except Exception:
		return []
	texts: List[str] = []
	if streaming:
		count = 0
		for row in ds:  # type: ignore
			prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
			code = str(row.get("code") or row.get("solution") or "").strip()


	texts: List[str] = []
	if not fp.exists():
		return texts
	with open(fp, "r", encoding="utf-8") as fh:
		for line in fh:
			line = line.strip()
			if not line:
				continue
			try:
				obj = json.loads(line)
			except Exception:
				continue
			txt = str(obj.get(text_key) or "").strip()
			if txt:
				texts.append(txt)
			if max_n is not None and len(texts) >= int(max_n):
				break
	return texts

def _load_hf(repo: str, subset: str | None, split: str, max_n: int | None, cache_dir: Path, streaming: bool) -> List[str]:
	try:
		from datasets import load_dataset  # type: ignore
	except Exception:
		return []
	try:
		if subset:
			ds = load_dataset(repo, subset, split=split, cache_dir=str(cache_dir), streaming=streaming)
		else:
			ds = load_dataset(repo, split=split, cache_dir=str(cache_dir), streaming=streaming)
	except Exception:
		return []
	texts: List[str] = []
	if streaming:
		count = 0
		for row in ds:  # type: ignore
			prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
			code = str(row.get("code") or row.get("solution") or "").strip()
			combined = (prompt + ("\n" + code if code else "")).strip()
			if combined:
				texts.append(combined)
				count += 1
			if max_n is not None and count >= int(max_n):
				break
		return texts
	# non-streaming
	total = len(ds)  # type: ignore
	for i in range(total):
		row = ds[i]  # type: ignore
		prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
		code = str(row.get("code") or row.get("solution") or "").strip()
		combined = (prompt + ("\n" + code if code else "")).strip()
		if combined:
			texts.append(combined)
		if max_n is not None and len(texts) >= int(max_n):
			break
	return texts

def load_program_texts(example_dir: str, config_path: str | None = None, train_new_only: bool = False, state_path: str | None = None) -> Tuple[List[str], List[str]]:
	"""
	Load training texts from a config that can reference local and HF datasets.
	Priority: local first; if missing, fetch from HF (optionally streaming) and store under local_dir.

	Returns (texts, loaded_source_names)
	"""
	ex_dir = Path(example_dir).resolve()
	# Config discovery
	cfg_path = Path(config_path) if config_path else (ex_dir / "datasets" / "config.json")
	cfg = _read_json(cfg_path) if cfg_path.exists() else {}
	local_dir = Path(cfg.get("local_dir") or (ex_dir / "datasets")).resolve()
	local_dir.mkdir(parents=True, exist_ok=True)
	cache_dir = local_dir / "hf_cache"
	cache_dir.mkdir(parents=True, exist_ok=True)
	sources: List[Dict[str, Any]] = list(cfg.get("sources") or [])
	streaming = os.getenv("HF_DATASETS_STREAMING", "1") == "1"
	full_datasets = os.getenv("FULL_DATASETS", "1") == "1"


			if combined:
				texts.append(combined)
				count += 1
			if max_n is not None and count >= int(max_n):
				break
		return texts
	# non-streaming
	total = len(ds)  # type: ignore
	for i in range(total):
		row = ds[i]  # type: ignore
		prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
		code = str(row.get("code") or row.get("solution") or "").strip()
		combined = (prompt + ("\n" + code if code else "")).strip()
		if combined:
			texts.append(combined)
		if max_n is not None and len(texts) >= int(max_n):
			break
	return texts

def load_program_texts(example_dir: str, config_path: str | None = None, train_new_only: bool = False, state_path: str | None = None) -> Tuple[List[str], List[str]]:
	"""
	Load training texts from a config that can reference local and HF datasets.
	Priority: local first; if missing, fetch from HF (optionally streaming) and store under local_dir.

	Returns (texts, loaded_source_names)
	"""
	ex_dir = Path(example_dir).resolve()
	# Config discovery
	cfg_path = Path(config_path) if config_path else (ex_dir / "datasets" / "config.json")
	cfg = _read_json(cfg_path) if cfg_path.exists() else {}
	local_dir = Path(cfg.get("local_dir") or (ex_dir / "datasets")).resolve()
	local_dir.mkdir(parents=True, exist_ok=True)
	cache_dir = local_dir / "hf_cache"
	cache_dir.mkdir(parents=True, exist_ok=True)
	sources: List[Dict[str, Any]] = list(cfg.get("sources") or [])
	streaming = os.getenv("HF_DATASETS_STREAMING", "1") == "1"
	full_datasets = os.getenv("FULL_DATASETS", "1") == "1"
	# Empty/0 -> unlimited by default
	_default_env = os.getenv("DATASET_MAX_N_PER_SOURCE", "") or "0"
	try:
		default_max_n_val = int(_default_env)
	except Exception:
		default_max_n_val = 0

	# Optional program state to support "train new only"
	seen: List[str] = []
	if train_new_only and state_path:
		try:
			st = _read_json(Path(state_path))
			seen = list(st.get("datasets_seen") or [])
		except Exception:
			seen = []

	texts_acc: List[str] = []
	loaded_names: List[str] = []
	for src in sources:
		name = str(src.get("name") or "").strip()
		if not name:
			continue
		if train_new_only and name in seen:
			continue
		if full_datasets:
			max_n = None
		else:
			max_n = src.get("max_n")
			if max_n is None and default_max_n_val > 0:
				max_n = default_max_n_val
		text_key = str(src.get("text_key") or "text").strip()
		subset = src.get("subset")
		split = str(src.get("split") or "train").strip()
		# Determine local file path: explicit 'path' or derived from name+split
		rel = str(src.get("path") or "").strip()
		if rel:
			fp = Path(rel)
			if not fp.is_absolute():
				fp = (local_dir / rel).resolve()
		else:
			suffix = f"_{split}" if split else ""
			fp = (local_dir / f"{name.replace('/', '_')}{suffix}.jsonl").resolve()
		# Try local first
		out_texts: List[str] = _load_local_jsonl(fp, text_key=text_key, max_n=max_n)
		# If not present, try HF using 'name' as repo id
		if not out_texts:
			out_texts = _load_hf(name, subset, split, max_n, cache_dir, streaming=streaming)
			# Persist to local for future runs
			if out_texts:
				try:
					with open(fp, "w", encoding="utf-8") as fh:
						for t in out_texts:
							fh.write(json.dumps({"text": t}) + "\n")
				except Exception:
					pass
		if out_texts:
			texts_acc.extend(out_texts)
			loaded_names.append(f"{name}:{split}" if split else name)
	return texts_acc, loaded_names

def iter_program_texts(example_dir: str, config_path: str | None = None, train_new_only: bool = False, state_path: str | None = None) -> Iterable[Tuple[List[str], str]]:
	"""
	Yields (texts, source_name) one dataset at a time using the same resolution rules as load_program_texts.
	Respects HF streaming and per-source max limits.
	"""
	ex_dir = Path(example_dir).resolve()
	cfg_path = Path(config_path) if config_path else (ex_dir / "datasets" / "config.json")
	cfg = _read_json(cfg_path) if cfg_path.exists() else {}
	local_dir = Path(cfg.get("local_dir") or (ex_dir / "datasets")).resolve()
	local_dir.mkdir(parents=True, exist_ok=True)
	cache_dir = local_dir / "hf_cache"
	cache_dir.mkdir(parents=True, exist_ok=True)
	sources: List[Dict[str, Any]] = list(cfg.get("sources") or [])
	streaming = os.getenv("HF_DATASETS_STREAMING", "1") == "1"
	full_datasets = os.getenv("FULL_DATASETS", "1") == "1"
	_default_env = os.getenv("DATASET_MAX_N_PER_SOURCE", "") or "0"
	try:
		default_max_n_val = int(_default_env)


			fp = (local_dir / f"{name.replace('/', '_')}{suffix}.jsonl").resolve()
		# Try local first
		out_texts: List[str] = _load_local_jsonl(fp, text_key=text_key, max_n=max_n)
		# If not present, try HF using 'name' as repo id
		if not out_texts:
			out_texts = _load_hf(name, subset, split, max_n, cache_dir, streaming=streaming)
			# Persist to local for future runs
			if out_texts:
				try:
					with open(fp, "w", encoding="utf-8") as fh:
						for t in out_texts:
							fh.write(json.dumps({"text": t}) + "\n")
				except Exception:
					pass
		if out_texts:
			texts_acc.extend(out_texts)
			loaded_names.append(f"{name}:{split}" if split else name)
	return texts_acc, loaded_names

def iter_program_texts(example_dir: str, config_path: str | None = None, train_new_only: bool = False, state_path: str | None = None) -> Iterable[Tuple[List[str], str]]:
	"""
	Yields (texts, source_name) one dataset at a time using the same resolution rules as load_program_texts.
	Respects HF streaming and per-source max limits.
	"""
	ex_dir = Path(example_dir).resolve()
	cfg_path = Path(config_path) if config_path else (ex_dir / "datasets" / "config.json")
	cfg = _read_json(cfg_path) if cfg_path.exists() else {}
	local_dir = Path(cfg.get("local_dir") or (ex_dir / "datasets")).resolve()
	local_dir.mkdir(parents=True, exist_ok=True)
	cache_dir = local_dir / "hf_cache"
	cache_dir.mkdir(parents=True, exist_ok=True)
	sources: List[Dict[str, Any]] = list(cfg.get("sources") or [])
	streaming = os.getenv("HF_DATASETS_STREAMING", "1") == "1"
	full_datasets = os.getenv("FULL_DATASETS", "1") == "1"
	_default_env = os.getenv("DATASET_MAX_N_PER_SOURCE", "") or "0"
	try:
		default_max_n_val = int(_default_env)
	except Exception:
		default_max_n_val = 0
	max_sources = int(os.getenv("DATASET_LIMIT_SOURCES", "0") or "0")  # 0 means no limit

	seen: List[str] = []
	if train_new_only and state_path:
		try:
			st = _read_json(Path(state_path))
			seen = list(st.get("datasets_seen") or [])
		except Exception:
			seen = []

	count_sources = 0
	for src in sources:
		name = str(src.get("name") or "").strip()
		if not name:
			continue
		if train_new_only and name in seen:
			continue
		if full_datasets:
			max_n = None
		else:
			max_n = src.get("max_n")
			if max_n is None and default_max_n_val > 0:
				max_n = default_max_n_val
		text_key = str(src.get("text_key") or "text").strip()
		subset = src.get("subset")
		split = str(src.get("split") or "train").strip()
		rel = str(src.get("path") or "").strip()
		if rel:
			fp = Path(rel)
			if not fp.is_absolute():
				fp = (local_dir / rel).resolve()
		else:
			suffix = f"_{split}" if split else ""
			fp = (local_dir / f"{name.replace('/', '_')}{suffix}.jsonl").resolve()
		out_texts: List[str] = _load_local_jsonl(fp, text_key=text_key, max_n=max_n)
		if not out_texts:
			out_texts = _load_hf(name, subset, split, max_n, cache_dir, streaming=streaming)
			if out_texts:
				try:
					with open(fp, "w", encoding="utf-8") as fh:
						for t in out_texts:
							fh.write(json.dumps({"text": t}) + "\n")
				except Exception:
					pass
		if out_texts:
			yield out_texts, (f"{name}:{split}" if split else name)
			count_sources += 1
			if max_sources and count_sources >= max_sources:
				break


from __future__ import annotations

import os
import sys
import json
from pathlib import Path
from datetime import datetime, timezone
from typing import Dict, Any

def _resolve_pg_ctor(pg_backend: str):
    mod, _, attr = pg_backend.partition(":")
    if not mod or not attr:
        raise ValueError(f"Invalid ProgramGraph backend '{pg_backend}', expected 'module:ClassName'")
    m = __import__(mod, fromlist=[attr])
    return getattr(m, attr)

def emit_repository_knowledge(repo_root: str, out_path: str, pg_backend: str) -> str:
    """Emit a consolidated repository_knowledge.json with entities, edges, and artifact spans."""
    repo_root_abs = os.path.abspath(repo_root)
    ctor = _resolve_pg_ctor(pg_backend)
    pg = ctor(repo_root_abs, ignore=None)
    ents = []
    ents_by_id: Dict[str, Any] = {}
    # Entities
    for e in pg.entities():
        rec = {
            "id": e.id,
            "name": e.name,
            "kind": e.kind,
            "owner": e.owner,
            "uri": e.uri,
        }
        ents.append(rec)
        ents_by_id[e.id] = rec
    # Edges
    eds = []
    try:
        for ed in pg.edges():
            eds.append({"src": ed.src, "dst": ed.dst, "type": ed.type})
    except Exception:
        eds = []
    # Artifact anchors per entity (best-effort)
    anchors = {}
    for e in pg.entities():
        try:
            ra = pg.resolve(e.uri)
        except Exception:
            continue
        anchors[e.id] = {
            "artifact_uri": ra.artifact_uri,
            "path": ra.artifact_uri.split("/artifact/", 1)[-1],
            "span": {"start": int(ra.span.start_line), "end": int(ra.span.end_line)},
            "hash": ra.hash,
        }
    # Artifacts list (lightweight)
    arts = []
    try:
        for a in pg.artifacts("source"):
            arts.append({
                "uri": a.uri,
                "type": a.type,
                "hash": getattr(a, "hash", ""),
                "path": a.uri.split("/artifact/", 1)[-1],
            })
    except Exception:
        arts = []
    obj = {
        "schema_version": 1,
        "program_id": getattr(pg, "program_id", Path(repo_root_abs).name),
        "generated_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
        "repo_root": repo_root_abs,
        "entities": ents,
        "edges": eds,
        "anchors": anchors,
        "artifacts": arts,
    }
    os.makedirs(os.path.dirname(out_path), exist_ok=True)
    with open(out_path, "w", encoding="utf-8") as fh:
        fh.write(json.dumps(obj, indent=2))
    return out_path

def main() -> None:
    # CLI: python emit_repository_knowledge.py <repo_root> <out_path> <pg_backend_module:Class>
    if len(sys.argv) < 4:
        print(
            "usage: python emit_repository_knowledge.py <repo_root> <out_path> <pg_backend module:Class>",
            file=sys.stderr,
        )
        sys.exit(2)
    repo_root = sys.argv[1]
    out_path = sys.argv[2]
    pg_backend = sys.argv[3]
    out = emit_repository_knowledge(repo_root, out_path, pg_backend)
    print(out)

if __name__ == "__main__":
    main()


from __future__ import annotations

import os
import sys
import json
from pathlib import Path
from datetime import datetime, timezone
from typing import Dict, Any

def _resolve_pg_ctor(pg_backend: str):
    mod, _, attr = pg_backend.partition(":")
    if not mod or not attr:
        raise ValueError(f"Invalid ProgramGraph backend '{pg_backend}', expected 'module:ClassName'")
    m = __import__(mod, fromlist=[attr])
    return getattr(m, attr)

def emit_repository_knowledge(repo_root: str, out_path: str, pg_backend: str) -> str:
    """Emit a consolidated repository_knowledge.json with entities, edges, and artifact spans."""
    repo_root_abs = os.path.abspath(repo_root)
    ctor = _resolve_pg_ctor(pg_backend)
    pg = ctor(repo_root_abs, ignore=None)
    ents = []
    ents_by_id: Dict[str, Any] = {}
    # Entities
    for e in pg.entities():
        rec = {
            "id": e.id,
            "name": e.name,
            "kind": e.kind,
            "owner": e.owner,
            "uri": e.uri,
        }
        ents.append(rec)
        ents_by_id[e.id] = rec


main

                "path": a.uri.split("/artifact/", 1)[-1],
            })
    except Exception:
        arts = []
    obj = {
        "schema_version": 1,
        "program_id": getattr(pg, "program_id", Path(repo_root_abs).name),
        "generated_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
        "repo_root": repo_root_abs,
        "entities": ents,
        "edges": eds,
        "anchors": anchors,
        "artifacts": arts,
    }
    os.makedirs(os.path.dirname(out_path), exist_ok=True)
    with open(out_path, "w", encoding="utf-8") as fh:
        fh.write(json.dumps(obj, indent=2))
    return out_path

def main() -> None:
    # CLI: python emit_repository_knowledge.py <repo_root> <out_path> <pg_backend_module:Class>
    if len(sys.argv) < 4:
        print(
            "usage: python emit_repository_knowledge.py <repo_root> <out_path> <pg_backend module:Class>",
            file=sys.stderr,
        )
        sys.exit(2)
    repo_root = sys.argv[1]
    out_path = sys.argv[2]
    pg_backend = sys.argv[3]
    out = emit_repository_knowledge(repo_root, out_path, pg_backend)
    print(out)

if __name__ == "__main__":
    main()


from __future__ import annotations

from dataclasses import dataclass, field, asdict
from typing import Dict, List, Optional, Tuple, Set, Any
import os
import json
import time
import math
import numpy as np

def _now_ts() -> float:
    try:
        return time.time()
    except Exception:
        return 0.0

def _unit(x: np.ndarray) -> np.ndarray:
    n = float(np.linalg.norm(x))
    return (x / n) if n > 0 else x

@dataclass
class RepoState:
    version: int = 1
    repo_root: str = ""
    # Monotone/idempotent components
    candidates_modules: Set[str] = field(default_factory=set)
    candidates_files: Set[str] = field(default_factory=set)
    facts: Set[Tuple[str, int, int]] = field(default_factory=set)  # (rel_path, a, b)
    beh_events: List[Dict[str, Any]] = field(default_factory=list)  # append-only
    # Vector view (anytime): keep a unit vector and a running weight
    vec: Optional[List[float]] = None
    vec_weight: float = 0.0
    # Capacity budget H
    H: float = 0.0
    # Last updated
    updated_at: float = 0.0

    def checksum(self) -> str:
        try:
            base = {
                "m": sorted(list(self.candidates_modules)),
                "f": sorted(list(self.candidates_files)),
                "facts": sorted([(p, int(a), int(b)) for (p, a, b) in self.facts]),
                "vw": float(self.vec_weight),
                "vl": len(self.vec or []),
                "H": float(self.H),
            }
            raw = json.dumps(base, sort_keys=True)
            import hashlib as _hh  # local
            return _hh.sha1(raw.encode("utf-8", errors="ignore")).hexdigest()
        except Exception:
            return ""

def _default_state_path(repo_root: str) -> str:
    try:
        return os.path.join(os.path.abspath(repo_root), ".repo_state.json")
    except Exception:
        return ".repo_state.json"

def load_repo_state(repo_root: str, path: Optional[str] = None) -> RepoState:
    p = path or _default_state_path(repo_root)
    try:
        if os.path.isfile(p):
            obj = json.loads(open(p, "r", encoding="utf-8").read())
            st = RepoState(
                version=int(obj.get("version", 1)),
                repo_root=str(obj.get("repo_root") or repo_root),
                candidates_modules=set(obj.get("candidates_modules", []) or []),
                candidates_files=set(obj.get("candidates_files", []) or []),
                facts=set((t[0], int(t[1]), int(t[2])) for t in (obj.get("facts", []) or [])),
                beh_events=list(obj.get("beh_events", []) or []),
                vec=(obj.get("vec") if isinstance(obj.get("vec"), list) else None),
                vec_weight=float(obj.get("vec_weight", 0.0) or 0.0),
                H=float(obj.get("H", 0.0) or 0.0),
                updated_at=float(obj.get("updated_at", 0.0) or 0.0),
            )
            return st
    except Exception:
        pass
    return RepoState(version=1, repo_root=os.path.abspath(repo_root))

def save_repo_state(state: RepoState, path: Optional[str] = None) -> None:
    p = path or _default_state_path(state.repo_root)
    try:
        state.updated_at = _now_ts()
        os.makedirs(os.path.dirname(p), exist_ok=True)
    except Exception:
        pass
    try:
        obj = {
            "version": state.version,
            "repo_root": state.repo_root,
            "candidates_modules": sorted(list(state.candidates_modules)),
            "candidates_files": sorted(list(state.candidates_files)),
            "facts": sorted([(p, int(a), int(b)) for (p, a, b) in state.facts]),
            "beh_events": state.beh_events,
            "vec": state.vec,
            "vec_weight": float(state.vec_weight),
            "H": float(state.H),
            "updated_at": float(state.updated_at),
        }
        open(p, "w", encoding="utf-8").write(json.dumps(obj, indent=2))
    except Exception:
        pass

def join_repo_states(old: RepoState, new: RepoState) -> RepoState:
    st = RepoState(version=max(int(old.version), int(new.version)))
    st.repo_root = old.repo_root or new.repo_root
    st.candidates_modules = set(old.candidates_modules) | set(new.candidates_modules)
    st.candidates_files = set(old.candidates_files) | set(new.candidates_files)
    st.facts = set(old.facts) | set(new.facts)
    st.beh_events = list(old.beh_events) + list(new.beh_events)
    # Vector join: unit-sum with weights, monotone in weight
    zv_old = np.array(old.vec, dtype=np.float32) if old.vec is not None else None
    zv_new = np.array(new.vec, dtype=np.float32) if new.vec is not None else None
    w_old = float(max(0.0, old.vec_weight))
    w_new = float(max(0.0, new.vec_weight))
    if zv_old is None and zv_new is None:
        st.vec, st.vec_weight = None, float(w_old + w_new)
    elif zv_old is None:
        st.vec, st.vec_weight = list(_unit(zv_new).tolist()), float(w_new + w_old)
    elif zv_new is None:
        st.vec, st.vec_weight = list(_unit(zv_old).tolist()), float(w_old + w_new)
    else:
        try:
            z = (w_old * _unit(zv_old)) + (w_new * _unit(zv_new))
            st.vec = list(_unit(z).tolist())
        except Exception:
            st.vec = list(_unit(zv_old).tolist())
        st.vec_weight = float(w_old + w_new)
    st.H = max(float(old.H), float(new.H))
    st.updated_at = max(float(old.updated_at), float(new.updated_at), _now_ts())
    return st

def changed_bits(prev: RepoState, cur: RepoState) -> bool:
    try:
        if len(cur.candidates_modules) > len(prev.candidates_modules):
            return True
        if len(cur.candidates_files) > len(prev.candidates_files):
            return True
        if len(cur.facts) > len(prev.facts):
            return True
        if float(cur.vec_weight) > float(prev.vec_weight):
            return True
    except Exception:
        return True
    return False

def new_state_from_run(
    repo_root: str,
    *,
    modules: List[str],
    files: List[str],
    citations: List[Tuple[str, int, int]],
    z_vec: Optional[np.ndarray],
    beh_event: Optional[Dict[str, Any]] = None,
    H_increment: float = 0.0,
) -> RepoState:
    st = RepoState(version=1, repo_root=os.path.abspath(repo_root))
    st.candidates_modules = set([m for m in (modules or []) if m])
    st.candidates_files = set([f for f in (files or []) if f])
    st.facts = set([(p, int(a), int(b)) for (p, a, b) in (citations or [])])
    st.beh_events = ([beh_event] if beh_event else [])
    if z_vec is not None:
        try:
            st.vec = list(_unit(z_vec.astype(np.float32)).tolist())
            st.vec_weight = 1.0
        except Exception:
            st.vec, st.vec_weight = None, 0.0
    st.H = float(max(0.0, H_increment))
    st.updated_at = _now_ts()
    return st


from __future__ import annotations

from dataclasses import dataclass, field, asdict
from typing import Dict, List, Optional, Tuple, Set, Any
import os
import json
import time
import math
import numpy as np

def _now_ts() -> float:
    try:
        return time.time()
    except Exception:
        return 0.0

def _unit(x: np.ndarray) -> np.ndarray:
    n = float(np.linalg.norm(x))
    return (x / n) if n > 0 else x

@dataclass
class RepoState:
    version: int = 1
    repo_root: str = ""
    # Monotone/idempotent components
    candidates_modules: Set[str] = field(default_factory=set)
    candidates_files: Set[str] = field(default_factory=set)
    facts: Set[Tuple[str, int, int]] = field(default_factory=set)  # (rel_path, a, b)
    beh_events: List[Dict[str, Any]] = field(default_factory=list)  # append-only
    # Vector view (anytime): keep a unit vector and a running weight
    vec: Optional[List[float]] = None
    vec_weight: float = 0.0
    # Capacity budget H


from __future__ import annotations

from dataclasses import dataclass, field, asdict
from typing import Dict, List, Optional, Tuple, Set, Any
import os
import json
import time
import math
import numpy as np

def _now_ts() -> float:
    try:
        return time.time()
    except Exception:
        return 0.0

def _unit(x: np.ndarray) -> np.ndarray:
    n = float(np.linalg.norm(x))
    return (x / n) if n > 0 else x

@dataclass
class RepoState:
    version: int = 1
    repo_root: str = ""
    # Monotone/idempotent components
    candidates_modules: Set[str] = field(default_factory=set)
    candidates_files: Set[str] = field(default_factory=set)
    facts: Set[Tuple[str, int, int]] = field(default_factory=set)  # (rel_path, a, b)
    beh_events: List[Dict[str, Any]] = field(default_factory=list)  # append-only
    # Vector view (anytime): keep a unit vector and a running weight
    vec: Optional[List[float]] = None
    vec_weight: float = 0.0
    # Capacity budget H
    H: float = 0.0
    # Last updated
    updated_at: float = 0.0

    def checksum(self) -> str:


RepoState

import os
import json
import time
import math
import numpy as np

def _now_ts() -> float:
    try:
        return time.time()
    except Exception:
        return 0.0

def _unit(x: np.ndarray) -> np.ndarray:
    n = float(np.linalg.norm(x))
    return (x / n) if n > 0 else x

@dataclass
class RepoState:
    version: int = 1
    repo_root: str = ""
    # Monotone/idempotent components
    candidates_modules: Set[str] = field(default_factory=set)
    candidates_files: Set[str] = field(default_factory=set)
    facts: Set[Tuple[str, int, int]] = field(default_factory=set)  # (rel_path, a, b)
    beh_events: List[Dict[str, Any]] = field(default_factory=list)  # append-only
    # Vector view (anytime): keep a unit vector and a running weight
    vec: Optional[List[float]] = None
    vec_weight: float = 0.0
    # Capacity budget H
    H: float = 0.0
    # Last updated
    updated_at: float = 0.0

    def checksum(self) -> str:
        try:
            base = {
                "m": sorted(list(self.candidates_modules)),
                "f": sorted(list(self.candidates_files)),
                "facts": sorted([(p, int(a), int(b)) for (p, a, b) in self.facts]),
                "vw": float(self.vec_weight),
                "vl": len(self.vec or []),
                "H": float(self.H),
            }
            raw = json.dumps(base, sort_keys=True)
            import hashlib as _hh  # local
            return _hh.sha1(raw.encode("utf-8", errors="ignore")).hexdigest()
        except Exception:
            return ""

def _default_state_path(repo_root: str) -> str:
    try:
        return os.path.join(os.path.abspath(repo_root), ".repo_state.json")
    except Exception:
        return ".repo_state.json"

def load_repo_state(repo_root: str, path: Optional[str] = None) -> RepoState:
    p = path or _default_state_path(repo_root)
    try:
        if os.path.isfile(p):
            obj = json.loads(open(p, "r", encoding="utf-8").read())
            st = RepoState(
                version=int(obj.get("version", 1)),
                repo_root=str(obj.get("repo_root") or repo_root),
                candidates_modules=set(obj.get("candidates_modules", []) or []),
                candidates_files=set(obj.get("candidates_files", []) or []),
                facts=set((t[0], int(t[1]), int(t[2])) for t in (obj.get("facts", []) or [])),


    # Last updated
    updated_at: float = 0.0

    def checksum(self) -> str:
        try:
            base = {
                "m": sorted(list(self.candidates_modules)),
                "f": sorted(list(self.candidates_files)),
                "facts": sorted([(p, int(a), int(b)) for (p, a, b) in self.facts]),
                "vw": float(self.vec_weight),
                "vl": len(self.vec or []),
                "H": float(self.H),
            }
            raw = json.dumps(base, sort_keys=True)
            import hashlib as _hh  # local
            return _hh.sha1(raw.encode("utf-8", errors="ignore")).hexdigest()
        except Exception:
            return ""

def _default_state_path(repo_root: str) -> str:
    try:
        return os.path.join(os.path.abspath(repo_root), ".repo_state.json")
    except Exception:
        return ".repo_state.json"

def load_repo_state(repo_root: str, path: Optional[str] = None) -> RepoState:
    p = path or _default_state_path(repo_root)
    try:
        if os.path.isfile(p):
            obj = json.loads(open(p, "r", encoding="utf-8").read())
            st = RepoState(
                version=int(obj.get("version", 1)),
                repo_root=str(obj.get("repo_root") or repo_root),
                candidates_modules=set(obj.get("candidates_modules", []) or []),
                candidates_files=set(obj.get("candidates_files", []) or []),
                facts=set((t[0], int(t[1]), int(t[2])) for t in (obj.get("facts", []) or [])),
                beh_events=list(obj.get("beh_events", []) or []),
                vec=(obj.get("vec") if isinstance(obj.get("vec"), list) else None),
                vec_weight=float(obj.get("vec_weight", 0.0) or 0.0),
                H=float(obj.get("H", 0.0) or 0.0),
                updated_at=float(obj.get("updated_at", 0.0) or 0.0),
            )
            return st


                "f": sorted(list(self.candidates_files)),
                "facts": sorted([(p, int(a), int(b)) for (p, a, b) in self.facts]),
                "vw": float(self.vec_weight),
                "vl": len(self.vec or []),
                "H": float(self.H),
            }
            raw = json.dumps(base, sort_keys=True)
            import hashlib as _hh  # local
            return _hh.sha1(raw.encode("utf-8", errors="ignore")).hexdigest()
        except Exception:
            return ""

def _default_state_path(repo_root: str) -> str:
    try:
        return os.path.join(os.path.abspath(repo_root), ".repo_state.json")
    except Exception:
        return ".repo_state.json"

def load_repo_state(repo_root: str, path: Optional[str] = None) -> RepoState:
    p = path or _default_state_path(repo_root)
    try:
        if os.path.isfile(p):
            obj = json.loads(open(p, "r", encoding="utf-8").read())
            st = RepoState(
                version=int(obj.get("version", 1)),
                repo_root=str(obj.get("repo_root") or repo_root),
                candidates_modules=set(obj.get("candidates_modules", []) or []),
                candidates_files=set(obj.get("candidates_files", []) or []),
                facts=set((t[0], int(t[1]), int(t[2])) for t in (obj.get("facts", []) or [])),
                beh_events=list(obj.get("beh_events", []) or []),
                vec=(obj.get("vec") if isinstance(obj.get("vec"), list) else None),
                vec_weight=float(obj.get("vec_weight", 0.0) or 0.0),
                H=float(obj.get("H", 0.0) or 0.0),
                updated_at=float(obj.get("updated_at", 0.0) or 0.0),
            )
            return st
    except Exception:
        pass
    return RepoState(version=1, repo_root=os.path.abspath(repo_root))

def save_repo_state(state: RepoState, path: Optional[str] = None) -> None:
    p = path or _default_state_path(state.repo_root)
    try:
        state.updated_at = _now_ts()
        os.makedirs(os.path.dirname(p), exist_ok=True)
    except Exception:
        pass
    try:
        obj = {
            "version": state.version,
            "repo_root": state.repo_root,
            "candidates_modules": sorted(list(state.candidates_modules)),
            "candidates_files": sorted(list(state.candidates_files)),
            "facts": sorted([(p, int(a), int(b)) for (p, a, b) in state.facts]),
            "beh_events": state.beh_events,
            "vec": state.vec,
            "vec_weight": float(state.vec_weight),
            "H": float(state.H),


        if os.path.isfile(p):
            obj = json.loads(open(p, "r", encoding="utf-8").read())
            st = RepoState(
                version=int(obj.get("version", 1)),
                repo_root=str(obj.get("repo_root") or repo_root),
                candidates_modules=set(obj.get("candidates_modules", []) or []),
                candidates_files=set(obj.get("candidates_files", []) or []),
                facts=set((t[0], int(t[1]), int(t[2])) for t in (obj.get("facts", []) or [])),
                beh_events=list(obj.get("beh_events", []) or []),
                vec=(obj.get("vec") if isinstance(obj.get("vec"), list) else None),
                vec_weight=float(obj.get("vec_weight", 0.0) or 0.0),
                H=float(obj.get("H", 0.0) or 0.0),
                updated_at=float(obj.get("updated_at", 0.0) or 0.0),
            )
            return st
    except Exception:
        pass
    return RepoState(version=1, repo_root=os.path.abspath(repo_root))

def save_repo_state(state: RepoState, path: Optional[str] = None) -> None:
    p = path or _default_state_path(state.repo_root)
    try:
        state.updated_at = _now_ts()
        os.makedirs(os.path.dirname(p), exist_ok=True)
    except Exception:
        pass
    try:
        obj = {
            "version": state.version,
            "repo_root": state.repo_root,
            "candidates_modules": sorted(list(state.candidates_modules)),
            "candidates_files": sorted(list(state.candidates_files)),
            "facts": sorted([(p, int(a), int(b)) for (p, a, b) in state.facts]),
            "beh_events": state.beh_events,
            "vec": state.vec,
            "vec_weight": float(state.vec_weight),
            "H": float(state.H),
            "updated_at": float(state.updated_at),
        }
        open(p, "w", encoding="utf-8").write(json.dumps(obj, indent=2))
    except Exception:
        pass

def join_repo_states(old: RepoState, new: RepoState) -> RepoState:
    st = RepoState(version=max(int(old.version), int(new.version)))
    st.repo_root = old.repo_root or new.repo_root
    st.candidates_modules = set(old.candidates_modules) | set(new.candidates_modules)
    st.candidates_files = set(old.candidates_files) | set(new.candidates_files)
    st.facts = set(old.facts) | set(new.facts)
    st.beh_events = list(old.beh_events) + list(new.beh_events)
    # Vector join: unit-sum with weights, monotone in weight
    zv_old = np.array(old.vec, dtype=np.float32) if old.vec is not None else None
    zv_new = np.array(new.vec, dtype=np.float32) if new.vec is not None else None
    w_old = float(max(0.0, old.vec_weight))
    w_new = float(max(0.0, new.vec_weight))
    if zv_old is None and zv_new is None:
        st.vec, st.vec_weight = None, float(w_old + w_new)
    elif zv_old is None:
        st.vec, st.vec_weight = list(_unit(zv_new).tolist()), float(w_new + w_old)
    elif zv_new is None:
        st.vec, st.vec_weight = list(_unit(zv_old).tolist()), float(w_old + w_new)


    except Exception:
        pass
    try:
        obj = {
            "version": state.version,
            "repo_root": state.repo_root,
            "candidates_modules": sorted(list(state.candidates_modules)),
            "candidates_files": sorted(list(state.candidates_files)),
            "facts": sorted([(p, int(a), int(b)) for (p, a, b) in state.facts]),
            "beh_events": state.beh_events,
            "vec": state.vec,
            "vec_weight": float(state.vec_weight),
            "H": float(state.H),
            "updated_at": float(state.updated_at),
        }
        open(p, "w", encoding="utf-8").write(json.dumps(obj, indent=2))
    except Exception:
        pass

def join_repo_states(old: RepoState, new: RepoState) -> RepoState:
    st = RepoState(version=max(int(old.version), int(new.version)))
    st.repo_root = old.repo_root or new.repo_root
    st.candidates_modules = set(old.candidates_modules) | set(new.candidates_modules)
    st.candidates_files = set(old.candidates_files) | set(new.candidates_files)
    st.facts = set(old.facts) | set(new.facts)
    st.beh_events = list(old.beh_events) + list(new.beh_events)
    # Vector join: unit-sum with weights, monotone in weight
    zv_old = np.array(old.vec, dtype=np.float32) if old.vec is not None else None
    zv_new = np.array(new.vec, dtype=np.float32) if new.vec is not None else None
    w_old = float(max(0.0, old.vec_weight))
    w_new = float(max(0.0, new.vec_weight))
    if zv_old is None and zv_new is None:
        st.vec, st.vec_weight = None, float(w_old + w_new)
    elif zv_old is None:
        st.vec, st.vec_weight = list(_unit(zv_new).tolist()), float(w_new + w_old)
    elif zv_new is None:
        st.vec, st.vec_weight = list(_unit(zv_old).tolist()), float(w_old + w_new)
    else:
        try:
            z = (w_old * _unit(zv_old)) + (w_new * _unit(zv_new))
            st.vec = list(_unit(z).tolist())
        except Exception:
            st.vec = list(_unit(zv_old).tolist())
        st.vec_weight = float(w_old + w_new)
    st.H = max(float(old.H), float(new.H))
    st.updated_at = max(float(old.updated_at), float(new.updated_at), _now_ts())
    return st

def changed_bits(prev: RepoState, cur: RepoState) -> bool:
    try:
        if len(cur.candidates_modules) > len(prev.candidates_modules):
            return True
        if len(cur.candidates_files) > len(prev.candidates_files):
            return True
        if len(cur.facts) > len(prev.facts):
            return True
        if float(cur.vec_weight) > float(prev.vec_weight):
            return True
    except Exception:
        return True
    return False

def new_state_from_run(
    repo_root: str,
    *,


    w_old = float(max(0.0, old.vec_weight))
    w_new = float(max(0.0, new.vec_weight))
    if zv_old is None and zv_new is None:
        st.vec, st.vec_weight = None, float(w_old + w_new)
    elif zv_old is None:
        st.vec, st.vec_weight = list(_unit(zv_new).tolist()), float(w_new + w_old)
    elif zv_new is None:
        st.vec, st.vec_weight = list(_unit(zv_old).tolist()), float(w_old + w_new)
    else:
        try:
            z = (w_old * _unit(zv_old)) + (w_new * _unit(zv_new))
            st.vec = list(_unit(z).tolist())
        except Exception:
            st.vec = list(_unit(zv_old).tolist())
        st.vec_weight = float(w_old + w_new)
    st.H = max(float(old.H), float(new.H))
    st.updated_at = max(float(old.updated_at), float(new.updated_at), _now_ts())
    return st

def changed_bits(prev: RepoState, cur: RepoState) -> bool:
    try:
        if len(cur.candidates_modules) > len(prev.candidates_modules):
            return True
        if len(cur.candidates_files) > len(prev.candidates_files):
            return True
        if len(cur.facts) > len(prev.facts):
            return True
        if float(cur.vec_weight) > float(prev.vec_weight):
            return True
    except Exception:
        return True
    return False

def new_state_from_run(
    repo_root: str,
    *,
    modules: List[str],
    files: List[str],
    citations: List[Tuple[str, int, int]],
    z_vec: Optional[np.ndarray],
    beh_event: Optional[Dict[str, Any]] = None,
    H_increment: float = 0.0,
) -> RepoState:
    st = RepoState(version=1, repo_root=os.path.abspath(repo_root))
    st.candidates_modules = set([m for m in (modules or []) if m])
    st.candidates_files = set([f for f in (files or []) if f])
    st.facts = set([(p, int(a), int(b)) for (p, a, b) in (citations or [])])
    st.beh_events = ([beh_event] if beh_event else [])
    if z_vec is not None:
        try:
            st.vec = list(_unit(z_vec.astype(np.float32)).tolist())


    st.H = max(float(old.H), float(new.H))
    st.updated_at = max(float(old.updated_at), float(new.updated_at), _now_ts())
    return st

def changed_bits(prev: RepoState, cur: RepoState) -> bool:
    try:
        if len(cur.candidates_modules) > len(prev.candidates_modules):
            return True
        if len(cur.candidates_files) > len(prev.candidates_files):
            return True
        if len(cur.facts) > len(prev.facts):
            return True
        if float(cur.vec_weight) > float(prev.vec_weight):
            return True
    except Exception:
        return True
    return False

def new_state_from_run(
    repo_root: str,
    *,
    modules: List[str],
    files: List[str],
    citations: List[Tuple[str, int, int]],
    z_vec: Optional[np.ndarray],
    beh_event: Optional[Dict[str, Any]] = None,
    H_increment: float = 0.0,
) -> RepoState:
    st = RepoState(version=1, repo_root=os.path.abspath(repo_root))
    st.candidates_modules = set([m for m in (modules or []) if m])
    st.candidates_files = set([f for f in (files or []) if f])
    st.facts = set([(p, int(a), int(b)) for (p, a, b) in (citations or [])])
    st.beh_events = ([beh_event] if beh_event else [])
    if z_vec is not None:
        try:
            st.vec = list(_unit(z_vec.astype(np.float32)).tolist())
            st.vec_weight = 1.0
        except Exception:
            st.vec, st.vec_weight = None, 0.0
    st.H = float(max(0.0, H_increment))
    st.updated_at = _now_ts()
    return st


checksum

    return (x / n) if n > 0 else x

@dataclass
class RepoState:
    version: int = 1
    repo_root: str = ""
    # Monotone/idempotent components
    candidates_modules: Set[str] = field(default_factory=set)
    candidates_files: Set[str] = field(default_factory=set)
    facts: Set[Tuple[str, int, int]] = field(default_factory=set)  # (rel_path, a, b)
    beh_events: List[Dict[str, Any]] = field(default_factory=list)  # append-only
    # Vector view (anytime): keep a unit vector and a running weight
    vec: Optional[List[float]] = None
    vec_weight: float = 0.0
    # Capacity budget H
    H: float = 0.0
    # Last updated
    updated_at: float = 0.0

    def checksum(self) -> str:
        try:
            base = {
                "m": sorted(list(self.candidates_modules)),
                "f": sorted(list(self.candidates_files)),
                "facts": sorted([(p, int(a), int(b)) for (p, a, b) in self.facts]),
                "vw": float(self.vec_weight),
                "vl": len(self.vec or []),
                "H": float(self.H),
            }
            raw = json.dumps(base, sort_keys=True)
            import hashlib as _hh  # local
            return _hh.sha1(raw.encode("utf-8", errors="ignore")).hexdigest()
        except Exception:
            return ""

def _default_state_path(repo_root: str) -> str:
    try:
        return os.path.join(os.path.abspath(repo_root), ".repo_state.json")
    except Exception:
        return ".repo_state.json"

def load_repo_state(repo_root: str, path: Optional[str] = None) -> RepoState:
    p = path or _default_state_path(repo_root)
    try:
        if os.path.isfile(p):
            obj = json.loads(open(p, "r", encoding="utf-8").read())
            st = RepoState(
                version=int(obj.get("version", 1)),
                repo_root=str(obj.get("repo_root") or repo_root),
                candidates_modules=set(obj.get("candidates_modules", []) or []),
                candidates_files=set(obj.get("candidates_files", []) or []),
                facts=set((t[0], int(t[1]), int(t[2])) for t in (obj.get("facts", []) or [])),


from __future__ import annotations

import os
import sys
import subprocess
from pathlib import Path

# Resolve program_conditioned_adapter directory directly from this file
EX_DIR = Path(__file__).resolve().parents[2]
SMOKE_REPO = EX_DIR / "smoke_repo"
# Backend class name (implemented in this example package)
PG_BACKEND = "examples.scripts.python_repo_graph:PythonRepoGraph"
# Program configuration loader
from .program_config import load_program_config  # noqa: E402

def _run(cmd: list[str]) -> int:
    print("[run]", " ".join(cmd))
    return subprocess.call(cmd)

def main() -> None:
    cfg = load_program_config(str(SMOKE_REPO))
    adapters_dir = cfg.paths.adapters_dir
    adapters_dir.mkdir(parents=True, exist_ok=True)
    # Resolve backend module path if a bare class name is provided
    pg_backend = PG_BACKEND
    if ":" not in pg_backend:
        pg_backend = cfg.pg_backend
    # Emit consolidated repository knowledge (entities/edges/anchors) for grounding
    knowledge_path = cfg.paths.knowledge_path
    _run([
        sys.executable,
        "-m",
        "examples.python_repo_grounded_qa.emit_repository_knowledge",
        str(SMOKE_REPO),
        str(knowledge_path),
        pg_backend,
    ])
    # Build adapters and caches for the smoke repo (PCA-agnostic; PG injected via --pg-backend)
    rc = _run([
        sys.executable,
        "-m",
        "build",
        "--sources", str(SMOKE_REPO),
        "--model", "meta-llama/Llama-3.1-8B-Instruct",
        "--adapters-dir", str(adapters_dir),
        "--embed-dim", "256",
        "--include-text",
        "--text-max-bytes", "20000",
        "--pg-backend", pg_backend,
        "--contracts-require-citations",
        "--contracts-retrieval-policy", str(cfg.contracts.retrieval_policy),
        "--contracts-retrieval-temp", str(cfg.contracts.retrieval_temp),
        "--init-program-state",
        "--program-state-path", str(cfg.paths.program_state_path),
        "--seed", "0",
        "--verbose",
    ])
    if rc != 0:
        sys.exit(rc)
    # Run grounded QA (selection and stamping via ProgramGraph plugin)
    prompt = "Where is the add function defined and how is it used? Provide citations."
    rc = _run([
        sys.executable,
        "-m",
        "run",
        "--sources", str(SMOKE_REPO),
        "--model", "meta-llama/Llama-3.1-8B-Instruct",
        "--adapters-dir", str(adapters_dir),
        "--prompt", prompt,
        "--of-sources", "question",
        "--pack-context",
        "--pack-mode", "windows",
        "--context-tokens", "800",
        "--require-citations",
        "--structured",
        "--citations-enforce",
        "--pg-backend", pg_backend,
        "--retrieval-policy", str(cfg.contracts.retrieval_policy),
        "--retrieval-temp", str(cfg.contracts.retrieval_temp),
        "--program-state", str(cfg.paths.program_state_path),
        "--delta-cap", "0.05",
        "--verbose",
    ])
    sys.exit(rc)

if __name__ == "__main__":
    main()


from __future__ import annotations

import os
import sys
import subprocess
from pathlib import Path

# Resolve program_conditioned_adapter directory directly from this file
EX_DIR = Path(__file__).resolve().parents[2]
SMOKE_REPO = EX_DIR / "smoke_repo"
# Backend class name (implemented in this example package)
PG_BACKEND = "examples.scripts.python_repo_graph:PythonRepoGraph"
# Program configuration loader
from .program_config import load_program_config  # noqa: E402

def _run(cmd: list[str]) -> int:
    print("[run]", " ".join(cmd))
    return subprocess.call(cmd)

def main() -> None:
    cfg = load_program_config(str(SMOKE_REPO))
    adapters_dir = cfg.paths.adapters_dir
    adapters_dir.mkdir(parents=True, exist_ok=True)
    # Resolve backend module path if a bare class name is provided
    pg_backend = PG_BACKEND
    if ":" not in pg_backend:
        pg_backend = cfg.pg_backend
    # Emit consolidated repository knowledge (entities/edges/anchors) for grounding
    knowledge_path = cfg.paths.knowledge_path
    _run([
        sys.executable,
        "-m",
        "examples.python_repo_grounded_qa.emit_repository_knowledge",
        str(SMOKE_REPO),
        str(knowledge_path),
        pg_backend,
    ])


main

import os
import sys
import subprocess
from pathlib import Path

# Resolve program_conditioned_adapter directory directly from this file
EX_DIR = Path(__file__).resolve().parents[2]
SMOKE_REPO = EX_DIR / "smoke_repo"
# Backend class name (implemented in this example package)
PG_BACKEND = "examples.scripts.python_repo_graph:PythonRepoGraph"
# Program configuration loader
from .program_config import load_program_config  # noqa: E402

def _run(cmd: list[str]) -> int:
    print("[run]", " ".join(cmd))
    return subprocess.call(cmd)

def main() -> None:
    cfg = load_program_config(str(SMOKE_REPO))
    adapters_dir = cfg.paths.adapters_dir
    adapters_dir.mkdir(parents=True, exist_ok=True)
    # Resolve backend module path if a bare class name is provided
    pg_backend = PG_BACKEND
    if ":" not in pg_backend:
        pg_backend = cfg.pg_backend
    # Emit consolidated repository knowledge (entities/edges/anchors) for grounding
    knowledge_path = cfg.paths.knowledge_path
    _run([
        sys.executable,
        "-m",
        "examples.python_repo_grounded_qa.emit_repository_knowledge",
        str(SMOKE_REPO),
        str(knowledge_path),
        pg_backend,
    ])
    # Build adapters and caches for the smoke repo (PCA-agnostic; PG injected via --pg-backend)
    rc = _run([
        sys.executable,
        "-m",
        "build",
        "--sources", str(SMOKE_REPO),
        "--model", "meta-llama/Llama-3.1-8B-Instruct",
        "--adapters-dir", str(adapters_dir),
        "--embed-dim", "256",
        "--include-text",
        "--text-max-bytes", "20000",
        "--pg-backend", pg_backend,
        "--contracts-require-citations",
        "--contracts-retrieval-policy", str(cfg.contracts.retrieval_policy),
        "--contracts-retrieval-temp", str(cfg.contracts.retrieval_temp),
        "--init-program-state",
        "--program-state-path", str(cfg.paths.program_state_path),
        "--seed", "0",
        "--verbose",
    ])
    if rc != 0:
        sys.exit(rc)
    # Run grounded QA (selection and stamping via ProgramGraph plugin)
    prompt = "Where is the add function defined and how is it used? Provide citations."
    rc = _run([
        sys.executable,
        "-m",
        "run",
        "--sources", str(SMOKE_REPO),
        "--model", "meta-llama/Llama-3.1-8B-Instruct",
        "--adapters-dir", str(adapters_dir),
        "--prompt", prompt,
        "--of-sources", "question",
        "--pack-context",
        "--pack-mode", "windows",
        "--context-tokens", "800",
        "--require-citations",
        "--structured",
        "--citations-enforce",
        "--pg-backend", pg_backend,
        "--retrieval-policy", str(cfg.contracts.retrieval_policy),
        "--retrieval-temp", str(cfg.contracts.retrieval_temp),
        "--program-state", str(cfg.paths.program_state_path),
        "--delta-cap", "0.05",
        "--verbose",
    ])
    sys.exit(rc)

if __name__ == "__main__":
    main()


from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Optional

@dataclass(frozen=True)
class ProgramContracts:
    require_citations: bool = True
    citations_per_paragraph: bool = False
    retrieval_policy: str = "sim:0.6,struct:0.4"
    retrieval_temp: float = 0.7

@dataclass(frozen=True)
class ProgramPaths:
    adapters_dir: Path
    knowledge_path: Path
    program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
    program_id: str
    pg_backend: str  # dotted path module:Class
    paths: ProgramPaths
    contracts: ProgramContracts

def _detect_pg_backend(base_dir: Path) -> str:
    pkg = "examples.python_repo_grounded_qa"
    if (base_dir / "python_repo_graph.py").exists():
        return f"{pkg}.python_repo_graph:PythonRepoGraph"
    if (base_dir / "repo_graph.py").exists():
        return f"{pkg}.repo_graph:PythonRepoGraph"
    # Fallback to python_repo_graph by name
    return f"{pkg}.python_repo_graph:PythonRepoGraph"

def load_program_config(repo_root: str) -> ProgramConfig:
    repo = Path(repo_root).resolve()
    ex_dir = Path(__file__).resolve().parents[2]
    example_dir = Path(__file__).resolve().parent
    adapters_dir = ex_dir / "artifacts" / "smoke_base"
    knowledge_path = adapters_dir / "repository_knowledge.json"
    pg_backend = _detect_pg_backend(example_dir)
    program_id = repo.name or "repo"
    return ProgramConfig(
        program_id=program_id,
        pg_backend=pg_backend,
        paths=ProgramPaths(
            adapters_dir=adapters_dir,
            knowledge_path=knowledge_path,
            program_state_path=adapters_dir / ".program_state.json",
        ),
        contracts=ProgramContracts(),
    )


ProgramContracts

from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Optional

@dataclass(frozen=True)
class ProgramContracts:
    require_citations: bool = True
    citations_per_paragraph: bool = False
    retrieval_policy: str = "sim:0.6,struct:0.4"
    retrieval_temp: float = 0.7

@dataclass(frozen=True)
class ProgramPaths:
    adapters_dir: Path
    knowledge_path: Path
    program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
    program_id: str
    pg_backend: str  # dotted path module:Class
    paths: ProgramPaths
    contracts: ProgramContracts

def _detect_pg_backend(base_dir: Path) -> str:
    pkg = "examples.python_repo_grounded_qa"
    if (base_dir / "python_repo_graph.py").exists():


ProgramPaths

from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Optional

@dataclass(frozen=True)
class ProgramContracts:
    require_citations: bool = True
    citations_per_paragraph: bool = False
    retrieval_policy: str = "sim:0.6,struct:0.4"
    retrieval_temp: float = 0.7

@dataclass(frozen=True)
class ProgramPaths:
    adapters_dir: Path
    knowledge_path: Path
    program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
    program_id: str
    pg_backend: str  # dotted path module:Class
    paths: ProgramPaths
    contracts: ProgramContracts

def _detect_pg_backend(base_dir: Path) -> str:
    pkg = "examples.python_repo_grounded_qa"
    if (base_dir / "python_repo_graph.py").exists():
        return f"{pkg}.python_repo_graph:PythonRepoGraph"
    if (base_dir / "repo_graph.py").exists():
        return f"{pkg}.repo_graph:PythonRepoGraph"
    # Fallback to python_repo_graph by name
    return f"{pkg}.python_repo_graph:PythonRepoGraph"


ProgramConfig

from pathlib import Path
from typing import Optional

@dataclass(frozen=True)
class ProgramContracts:
    require_citations: bool = True
    citations_per_paragraph: bool = False
    retrieval_policy: str = "sim:0.6,struct:0.4"
    retrieval_temp: float = 0.7

@dataclass(frozen=True)
class ProgramPaths:
    adapters_dir: Path
    knowledge_path: Path
    program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
    program_id: str
    pg_backend: str  # dotted path module:Class
    paths: ProgramPaths
    contracts: ProgramContracts

def _detect_pg_backend(base_dir: Path) -> str:
    pkg = "examples.python_repo_grounded_qa"
    if (base_dir / "python_repo_graph.py").exists():
        return f"{pkg}.python_repo_graph:PythonRepoGraph"
    if (base_dir / "repo_graph.py").exists():
        return f"{pkg}.repo_graph:PythonRepoGraph"
    # Fallback to python_repo_graph by name
    return f"{pkg}.python_repo_graph:PythonRepoGraph"

def load_program_config(repo_root: str) -> ProgramConfig:
    repo = Path(repo_root).resolve()
    ex_dir = Path(__file__).resolve().parents[2]
    example_dir = Path(__file__).resolve().parent
    adapters_dir = ex_dir / "artifacts" / "smoke_base"
    knowledge_path = adapters_dir / "repository_knowledge.json"
    pg_backend = _detect_pg_backend(example_dir)
    program_id = repo.name or "repo"


    citations_per_paragraph: bool = False
    retrieval_policy: str = "sim:0.6,struct:0.4"
    retrieval_temp: float = 0.7

@dataclass(frozen=True)
class ProgramPaths:
    adapters_dir: Path
    knowledge_path: Path
    program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
    program_id: str
    pg_backend: str  # dotted path module:Class
    paths: ProgramPaths
    contracts: ProgramContracts

def _detect_pg_backend(base_dir: Path) -> str:
    pkg = "examples.python_repo_grounded_qa"
    if (base_dir / "python_repo_graph.py").exists():
        return f"{pkg}.python_repo_graph:PythonRepoGraph"
    if (base_dir / "repo_graph.py").exists():
        return f"{pkg}.repo_graph:PythonRepoGraph"
    # Fallback to python_repo_graph by name
    return f"{pkg}.python_repo_graph:PythonRepoGraph"

def load_program_config(repo_root: str) -> ProgramConfig:
    repo = Path(repo_root).resolve()
    ex_dir = Path(__file__).resolve().parents[2]
    example_dir = Path(__file__).resolve().parent
    adapters_dir = ex_dir / "artifacts" / "smoke_base"
    knowledge_path = adapters_dir / "repository_knowledge.json"
    pg_backend = _detect_pg_backend(example_dir)
    program_id = repo.name or "repo"
    return ProgramConfig(
        program_id=program_id,
        pg_backend=pg_backend,
        paths=ProgramPaths(
            adapters_dir=adapters_dir,
            knowledge_path=knowledge_path,
            program_state_path=adapters_dir / ".program_state.json",
        ),
        contracts=ProgramContracts(),
    )


@dataclass(frozen=True)
class ProgramConfig:
    program_id: str
    pg_backend: str  # dotted path module:Class
    paths: ProgramPaths
    contracts: ProgramContracts

def _detect_pg_backend(base_dir: Path) -> str:
    pkg = "examples.python_repo_grounded_qa"
    if (base_dir / "python_repo_graph.py").exists():
        return f"{pkg}.python_repo_graph:PythonRepoGraph"
    if (base_dir / "repo_graph.py").exists():
        return f"{pkg}.repo_graph:PythonRepoGraph"
    # Fallback to python_repo_graph by name
    return f"{pkg}.python_repo_graph:PythonRepoGraph"

def load_program_config(repo_root: str) -> ProgramConfig:
    repo = Path(repo_root).resolve()
    ex_dir = Path(__file__).resolve().parents[2]
    example_dir = Path(__file__).resolve().parent
    adapters_dir = ex_dir / "artifacts" / "smoke_base"
    knowledge_path = adapters_dir / "repository_knowledge.json"
    pg_backend = _detect_pg_backend(example_dir)
    program_id = repo.name or "repo"
    return ProgramConfig(
        program_id=program_id,
        pg_backend=pg_backend,
        paths=ProgramPaths(
            adapters_dir=adapters_dir,
            knowledge_path=knowledge_path,
            program_state_path=adapters_dir / ".program_state.json",
        ),
        contracts=ProgramContracts(),
    )


from __future__ import annotations

import os
import ast
import hashlib
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple, Iterable, Set

@dataclass(frozen=True)
class FileSpan:
    file: str           # absolute path
    start_line: int     # 1-based inclusive
    end_line: int       # 1-based inclusive

@dataclass
class CGEntity:
    id: str             # stable id (e.g., fqn)
    kind: str           # module|function|class|test
    name: str
    file: str           # absolute path
    owner: Optional[str]
    start_line: int
    end_line: int

@dataclass
class CGEdge:
    src: str            # CGEntity.id
    dst: str            # CGEntity.id
    type: str           # imports|calls|owns|tests

class CodeGraph:
    def __init__(self, repo_root: str, ignore: Optional[List[str]] = None):
        self.root = os.path.abspath(repo_root)
        self.ignore_rules = [s for s in (ignore or []) if s]
        self.entities_by_id: Dict[str, CGEntity] = {}
        self.edges_list: List[CGEdge] = []
        self._file_hash: Dict[str, str] = {}
        self._id_by_module: Dict[str, str] = {}
        self._ids_by_file: Dict[str, List[str]] = {}
        self._index_identifiers: Dict[str, List[str]] = {}

    # Build
    def build(self) -> "CodeGraph":
        py_files = self._discover_py_files(self.root, self.ignore_rules)
        for abs_fp in py_files:
            mod = self._module_name_for(abs_fp)
            mid = f"py:{mod}"
            self._id_by_module[mod] = mid
            ent = CGEntity(
                id=mid, kind="module", name=mod, file=abs_fp, owner=None,
                start_line=1, end_line=self._safe_count_lines(abs_fp),
            )
            self.entities_by_id[mid] = ent
            self._ids_by_file.setdefault(abs_fp, []).append(mid)
            self._index_identifiers.setdefault(mod.lower(), []).append(mid)
            # Parse AST for defs/imports/calls
            try:
                with open(abs_fp, "r", encoding="utf-8", errors="ignore") as fh:
                    src = fh.read()
                tree = ast.parse(src)
            except Exception:
                tree = None
            if tree is None:
                continue
            # functions/classes
            for node in ast.walk(tree):
                if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
                    name = getattr(node, "name", "")
                    a = int(getattr(node, "lineno", 1))
                    b = int(getattr(node, "end_lineno", a))
                    fid = f"py:{mod}.{name}"
                    self.entities_by_id[fid] = CGEntity(
                        id=fid, kind="function", name=name, file=abs_fp, owner=mid,
                        start_line=a, end_line=b,
                    )
                    self._ids_by_file.setdefault(abs_fp, []).append(fid)
                    self.edges_list.append(CGEdge(src=mid, dst=fid, type="owns"))
                    self._index_identifiers.setdefault(name.lower(), []).append(fid)
                elif isinstance(node, ast.ClassDef):
                    name = getattr(node, "name", "")
                    a = int(getattr(node, "lineno", 1))
                    b = int(getattr(node, "end_lineno", a))
                    cid = f"py:{mod}.{name}"
                    self.entities_by_id[cid] = CGEntity(
                        id=cid, kind="class", name=name, file=abs_fp, owner=mid,
                        start_line=a, end_line=b,
                    )
                    self._ids_by_file.setdefault(abs_fp, []).append(cid)
                    self.edges_list.append(CGEdge(src=mid, dst=cid, type="owns"))
                    self._index_identifiers.setdefault(name.lower(), []).append(cid)
            # imports (module-level)
            try:
                for node in ast.walk(tree):
                    if isinstance(node, ast.Import):
                        for alias in node.names:
                            im = alias.name
                            if not im:
                                continue
                            # record import edge to module-name id if present later; store raw as meta by name
                            tgt_mod = im
                            tid = f"py:{tgt_mod}"
                            self.edges_list.append(CGEdge(src=mid, dst=tid, type="imports"))
                    elif isinstance(node, ast.ImportFrom):
                        im = node.module or ""
                        if not im:
                            continue
                        tgt_mod = im
                        tid = f"py:{tgt_mod}"
                        self.edges_list.append(CGEdge(src=mid, dst=tid, type="imports"))
            except Exception:
                pass
            # calls (best-effort): record identifiers used in Call nodes
            try:
                for node in ast.walk(tree):
                    if isinstance(node, ast.Call):
                        fn = getattr(node, "func", None)
                        name = None
                        if isinstance(fn, ast.Attribute):
                            name = getattr(fn.attr, "lower", None)
                            name = fn.attr if fn and hasattr(fn, "attr") else None
                        elif isinstance(fn, ast.Name):
                            name = fn.id
                        if name:
                            lid = name.lower()
                            for cand in self._index_identifiers.get(lid, []):
                                self.edges_list.append(CGEdge(src=mid, dst=cand, type="calls"))
            except Exception:
                pass
            # tests tag
            base = os.path.basename(abs_fp)
            if base.startswith("test_") or base.endswith("_test.py"):
                self.entities_by_id[mid].kind = "test_module"
        # finalize file hashes
        self._precompute_hashes(py_files)
        return self

    # Public accessors
    def entities(self) -> Iterable[CGEntity]:
        return self.entities_by_id.values()

    def edges(self) -> Iterable[CGEdge]:
        # Filter edges whose endpoints are known (post totality)
        known = set(self.entities_by_id.keys())
        for e in self.edges_list:
            if (e.src in known) and (e.dst in known):
                yield e

    def file_hash(self, abs_path: str) -> str:
        return self._file_hash.get(abs_path) or ""

    def ids_for_file(self, abs_path: str) -> List[str]:
        return list(self._ids_by_file.get(abs_path, []))

    def find_identifier_ids(self, token: str) -> List[str]:
        return list(self._index_identifiers.get(token.lower(), []))

    # Helpers
    def _discover_py_files(self, root: str, ignore: List[str]) -> List[str]:
        out: List[str] = []
        for dirpath, dirnames, filenames in os.walk(root):
            # naive ignore: drop segments that contain any ignore pattern
            if any(ig in dirpath for ig in ignore):
                continue
            for fn in filenames:
                if not fn.endswith(".py"):
                    continue
                ap = os.path.abspath(os.path.join(dirpath, fn))
                out.append(ap)
        return out

    def _module_name_for(self, abs_file: str) -> str:
        # repo-relative without extension, path with dots
        rel = os.path.relpath(abs_file, self.root).replace("\\", "/")
        if rel.endswith(".py"):
            rel = rel[:-3]
        parts = [p for p in rel.split("/") if p and p != "__init__"]
        return ".".join(parts) or os.path.splitext(os.path.basename(abs_file))[0]

    def _safe_count_lines(self, abs_file: str) -> int:
        try:
            with open(abs_file, "r", encoding="utf-8", errors="ignore") as fh:
                return sum(1 for _ in fh)
        except Exception:
            return 1

    def _precompute_hashes(self, files: List[str]) -> None:
        for fp in files:
            try:
                with open(fp, "rb") as fh:
                    raw = fh.read()
                h = hashlib.sha256(raw).hexdigest()
            except Exception:
                h = ""
            self._file_hash[fp] = h


FileSpan

from __future__ import annotations

import os
import ast
import hashlib
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple, Iterable, Set

@dataclass(frozen=True)
class FileSpan:
    file: str           # absolute path
    start_line: int     # 1-based inclusive
    end_line: int       # 1-based inclusive

@dataclass
class CGEntity:
    id: str             # stable id (e.g., fqn)
    kind: str           # module|function|class|test
    name: str
    file: str           # absolute path
    owner: Optional[str]
    start_line: int
    end_line: int

@dataclass
class CGEdge:
    src: str            # CGEntity.id
    dst: str            # CGEntity.id
    type: str           # imports|calls|owns|tests


CGEntity

from __future__ import annotations

import os
import ast
import hashlib
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple, Iterable, Set

@dataclass(frozen=True)
class FileSpan:
    file: str           # absolute path
    start_line: int     # 1-based inclusive
    end_line: int       # 1-based inclusive

@dataclass
class CGEntity:
    id: str             # stable id (e.g., fqn)
    kind: str           # module|function|class|test
    name: str
    file: str           # absolute path
    owner: Optional[str]
    start_line: int
    end_line: int

@dataclass
class CGEdge:
    src: str            # CGEntity.id
    dst: str            # CGEntity.id
    type: str           # imports|calls|owns|tests

class CodeGraph:
    def __init__(self, repo_root: str, ignore: Optional[List[str]] = None):
        self.root = os.path.abspath(repo_root)
        self.ignore_rules = [s for s in (ignore or []) if s]
        self.entities_by_id: Dict[str, CGEntity] = {}
        self.edges_list: List[CGEdge] = []
        self._file_hash: Dict[str, str] = {}
        self._id_by_module: Dict[str, str] = {}
        self._ids_by_file: Dict[str, List[str]] = {}
        self._index_identifiers: Dict[str, List[str]] = {}


CGEdge

@dataclass(frozen=True)
class FileSpan:
    file: str           # absolute path
    start_line: int     # 1-based inclusive
    end_line: int       # 1-based inclusive

@dataclass
class CGEntity:
    id: str             # stable id (e.g., fqn)
    kind: str           # module|function|class|test
    name: str
    file: str           # absolute path
    owner: Optional[str]
    start_line: int
    end_line: int

@dataclass
class CGEdge:
    src: str            # CGEntity.id
    dst: str            # CGEntity.id
    type: str           # imports|calls|owns|tests

class CodeGraph:
    def __init__(self, repo_root: str, ignore: Optional[List[str]] = None):
        self.root = os.path.abspath(repo_root)
        self.ignore_rules = [s for s in (ignore or []) if s]
        self.entities_by_id: Dict[str, CGEntity] = {}
        self.edges_list: List[CGEdge] = []
        self._file_hash: Dict[str, str] = {}
        self._id_by_module: Dict[str, str] = {}
        self._ids_by_file: Dict[str, List[str]] = {}
        self._index_identifiers: Dict[str, List[str]] = {}

    # Build
    def build(self) -> "CodeGraph":
        py_files = self._discover_py_files(self.root, self.ignore_rules)
        for abs_fp in py_files:
            mod = self._module_name_for(abs_fp)
            mid = f"py:{mod}"
            self._id_by_module[mod] = mid


CodeGraph

@dataclass
class CGEntity:
    id: str             # stable id (e.g., fqn)
    kind: str           # module|function|class|test
    name: str
    file: str           # absolute path
    owner: Optional[str]
    start_line: int
    end_line: int

@dataclass
class CGEdge:
    src: str            # CGEntity.id
    dst: str            # CGEntity.id
    type: str           # imports|calls|owns|tests

class CodeGraph:
    def __init__(self, repo_root: str, ignore: Optional[List[str]] = None):
        self.root = os.path.abspath(repo_root)
        self.ignore_rules = [s for s in (ignore or []) if s]
        self.entities_by_id: Dict[str, CGEntity] = {}
        self.edges_list: List[CGEdge] = []
        self._file_hash: Dict[str, str] = {}
        self._id_by_module: Dict[str, str] = {}
        self._ids_by_file: Dict[str, List[str]] = {}
        self._index_identifiers: Dict[str, List[str]] = {}

    # Build
    def build(self) -> "CodeGraph":
        py_files = self._discover_py_files(self.root, self.ignore_rules)
        for abs_fp in py_files:
            mod = self._module_name_for(abs_fp)
            mid = f"py:{mod}"
            self._id_by_module[mod] = mid
            ent = CGEntity(
                id=mid, kind="module", name=mod, file=abs_fp, owner=None,
                start_line=1, end_line=self._safe_count_lines(abs_fp),
            )
            self.entities_by_id[mid] = ent
            self._ids_by_file.setdefault(abs_fp, []).append(mid)
            self._index_identifiers.setdefault(mod.lower(), []).append(mid)
            # Parse AST for defs/imports/calls
            try:
                with open(abs_fp, "r", encoding="utf-8", errors="ignore") as fh:
                    src = fh.read()
                tree = ast.parse(src)
            except Exception:
                tree = None
            if tree is None:
                continue
            # functions/classes
            for node in ast.walk(tree):
                if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
                    name = getattr(node, "name", "")
                    a = int(getattr(node, "lineno", 1))
                    b = int(getattr(node, "end_lineno", a))
                    fid = f"py:{mod}.{name}"
                    self.entities_by_id[fid] = CGEntity(
                        id=fid, kind="function", name=name, file=abs_fp, owner=mid,
                        start_line=a, end_line=b,
                    )
                    self._ids_by_file.setdefault(abs_fp, []).append(fid)
                    self.edges_list.append(CGEdge(src=mid, dst=fid, type="owns"))
                    self._index_identifiers.setdefault(name.lower(), []).append(fid)
                elif isinstance(node, ast.ClassDef):
                    name = getattr(node, "name", "")
                    a = int(getattr(node, "lineno", 1))
                    b = int(getattr(node, "end_lineno", a))
                    cid = f"py:{mod}.{name}"
                    self.entities_by_id[cid] = CGEntity(
                        id=cid, kind="class", name=name, file=abs_fp, owner=mid,
                        start_line=a, end_line=b,
                    )
                    self._ids_by_file.setdefault(abs_fp, []).append(cid)
                    self.edges_list.append(CGEdge(src=mid, dst=cid, type="owns"))
                    self._index_identifiers.setdefault(name.lower(), []).append(cid)
            # imports (module-level)
            try:
                for node in ast.walk(tree):
                    if isinstance(node, ast.Import):
                        for alias in node.names:
                            im = alias.name
                            if not im:
                                continue
                            # record import edge to module-name id if present later; store raw as meta by name
                            tgt_mod = im
                            tid = f"py:{tgt_mod}"
                            self.edges_list.append(CGEdge(src=mid, dst=tid, type="imports"))
                    elif isinstance(node, ast.ImportFrom):
                        im = node.module or ""
                        if not im:
                            continue
                        tgt_mod = im
                        tid = f"py:{tgt_mod}"
                        self.edges_list.append(CGEdge(src=mid, dst=tid, type="imports"))
            except Exception:
                pass
            # calls (best-effort): record identifiers used in Call nodes
            try:
                for node in ast.walk(tree):
                    if isinstance(node, ast.Call):
                        fn = getattr(node, "func", None)
                        name = None
                        if isinstance(fn, ast.Attribute):
                            name = getattr(fn.attr, "lower", None)
                            name = fn.attr if fn and hasattr(fn, "attr") else None
                        elif isinstance(fn, ast.Name):
                            name = fn.id
                        if name:
                            lid = name.lower()
                            for cand in self._index_identifiers.get(lid, []):
                                self.edges_list.append(CGEdge(src=mid, dst=cand, type="calls"))
            except Exception:
                pass
            # tests tag
            base = os.path.basename(abs_fp)
            if base.startswith("test_") or base.endswith("_test.py"):
                self.entities_by_id[mid].kind = "test_module"
        # finalize file hashes
        self._precompute_hashes(py_files)
        return self

    # Public accessors
    def entities(self) -> Iterable[CGEntity]:
        return self.entities_by_id.values()

    def edges(self) -> Iterable[CGEdge]:
        # Filter edges whose endpoints are known (post totality)
        known = set(self.entities_by_id.keys())
        for e in self.edges_list:
            if (e.src in known) and (e.dst in known):
                yield e

    def file_hash(self, abs_path: str) -> str:
        return self._file_hash.get(abs_path) or ""

    def ids_for_file(self, abs_path: str) -> List[str]:
        return list(self._ids_by_file.get(abs_path, []))

    def find_identifier_ids(self, token: str) -> List[str]:
        return list(self._index_identifiers.get(token.lower(), []))

    # Helpers
    def _discover_py_files(self, root: str, ignore: List[str]) -> List[str]:
        out: List[str] = []
        for dirpath, dirnames, filenames in os.walk(root):
            # naive ignore: drop segments that contain any ignore pattern
            if any(ig in dirpath for ig in ignore):
                continue
            for fn in filenames:
                if not fn.endswith(".py"):
                    continue
                ap = os.path.abspath(os.path.join(dirpath, fn))
                out.append(ap)
        return out

    def _module_name_for(self, abs_file: str) -> str:
        # repo-relative without extension, path with dots
        rel = os.path.relpath(abs_file, self.root).replace("\\", "/")
        if rel.endswith(".py"):
            rel = rel[:-3]
        parts = [p for p in rel.split("/") if p and p != "__init__"]
        return ".".join(parts) or os.path.splitext(os.path.basename(abs_file))[0]

    def _safe_count_lines(self, abs_file: str) -> int:
        try:
            with open(abs_file, "r", encoding="utf-8", errors="ignore") as fh:
                return sum(1 for _ in fh)
        except Exception:
            return 1

    def _precompute_hashes(self, files: List[str]) -> None:
        for fp in files:
            try:
                with open(fp, "rb") as fh:
                    raw = fh.read()
                h = hashlib.sha256(raw).hexdigest()
            except Exception:
                h = ""
            self._file_hash[fp] = h


@dataclass
class CGEntity:
    id: str             # stable id (e.g., fqn)
    kind: str           # module|function|class|test
    name: str
    file: str           # absolute path
    owner: Optional[str]
    start_line: int
    end_line: int

@dataclass
class CGEdge:
    src: str            # CGEntity.id
    dst: str            # CGEntity.id
    type: str           # imports|calls|owns|tests

class CodeGraph:
    def __init__(self, repo_root: str, ignore: Optional[List[str]] = None):
        self.root = os.path.abspath(repo_root)
        self.ignore_rules = [s for s in (ignore or []) if s]
        self.entities_by_id: Dict[str, CGEntity] = {}
        self.edges_list: List[CGEdge] = []
        self._file_hash: Dict[str, str] = {}
        self._id_by_module: Dict[str, str] = {}
        self._ids_by_file: Dict[str, List[str]] = {}
        self._index_identifiers: Dict[str, List[str]] = {}

    # Build
    def build(self) -> "CodeGraph":
        py_files = self._discover_py_files(self.root, self.ignore_rules)
        for abs_fp in py_files:
            mod = self._module_name_for(abs_fp)
            mid = f"py:{mod}"
            self._id_by_module[mod] = mid
            ent = CGEntity(
                id=mid, kind="module", name=mod, file=abs_fp, owner=None,
                start_line=1, end_line=self._safe_count_lines(abs_fp),
            )
            self.entities_by_id[mid] = ent
            self._ids_by_file.setdefault(abs_fp, []).append(mid)
            self._index_identifiers.setdefault(mod.lower(), []).append(mid)
            # Parse AST for defs/imports/calls
            try:
                with open(abs_fp, "r", encoding="utf-8", errors="ignore") as fh:
                    src = fh.read()
                tree = ast.parse(src)