from __future__ import annotations

import sys
import os
import json
from pathlib import Path
from typing import List, Dict

import torch
from torch.utils.data import Dataset

ROOT = EX_DIR
try:
	from .program_config import load_program_config  # type: ignore  # noqa: E402
except Exception:
	from program_config import load_program_config  # type: ignore  # noqa: E402

class SimpleLmDataset(Dataset):
	def __init__(self, tokenizer, texts: List[str], max_len: int = 128) -> None:
		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)
		attn = (input_ids != pad_id).to(torch.long)
		return {"input_ids": input_ids.unsqueeze(0), "attn_mask": attn.unsqueeze(0)}

def main() -> None:
	# Prefer evaluating the "complete" run by default
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "complete_dataset_trainer"
	if not artifacts_root.exists():
		# Fallback to smoke artifacts if complete doesn't exist
		artifacts_root = example_dir / "artifacts" / "smoke_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	ckpt_path = adapters_dir / "trained_ckpt" / "model.pt"
	out_dir.mkdir(parents=True, exist_ok=True)
	eval_random = os.getenv("EVAL_RANDOM", "0") == "1"
	eval_model = os.getenv("EVAL_MODEL", "local")  # "local" (toy LM) or "llama_hf"

	if str(repo_root) not in sys.path:
		sys.path.insert(0, str(repo_root))
	py_path = os.environ.get("PYTHONPATH") or ""
	os.environ["PYTHONPATH"] = f"{str(repo_root)}{(':' + py_path) if py_path else ''}"

	# Tokenizer(s)
	if eval_model == "llama_hf":
		# Hugging Face Llama tokenizer for baseline (no adapters)
		try:
			from transformers import AutoTokenizer  # type: ignore
			hf_tok = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B-Instruct")
			# Ensure pad token is set for batching; use eos as pad
			if hf_tok.pad_token_id is None and hf_tok.eos_token_id is not None:
				hf_tok.pad_token = hf_tok.eos_token
		except Exception as e:
			print(f"[eval] Failed to load HF tokenizer: {e}")
			return
	else:
		from data.tokenizer import LocalLlamaTokenizer  # type: ignore
		try:
			snap_default = os.environ.get("LLAMA_SNAPSHOT_DIR", str(repo_root / "checkpoints" / "llama"))
			tok = LocalLlamaTokenizer(snap_default)
		except Exception:
			from data.tokenizer import WhitespaceTokenizer  # type: ignore
			tok = WhitespaceTokenizer()

	# Load MBPP test split
	texts: List[str] = []
	total = 0
	try:
		from examples.scripts.mbpp_loader import load_mbpp_texts  # type: ignore
		texts, total = load_mbpp_texts(max_n=1_000_000, split="test")
		print(f"[mbpp:test] loaded {len(texts)} / {total} samples")
	except Exception as e:
		print(f"[mbpp:test] failed to load via HF datasets: {e}")
	if not texts:
		print("[eval] No test texts available; exiting with empty report.")
		report = {
			"split": "test",
			"avg_loss": None,
			"ppl": None,
			"samples": 0,
			"device": "cpu",
			"ckpt_path": str(ckpt_path),
		}
		out_path = out_dir / "MBPPEvalTestReport.json"
		with open(out_path, "w", encoding="utf-8") as fh:
			fh.write(json.dumps({"schema_version": 1, "EvalReport": report}, indent=2))
		print(str(out_path))
		return

	# Build dataset or leave as raw texts depending on eval model
	if eval_model == "llama_hf":
		ds = None  # we will use hf_tok directly on texts per-sample
	else:
		ds = SimpleLmDataset(tok, texts, max_len=128)

	# Model selection
	device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
	if eval_model == "llama_hf":
		try:
			from transformers import AutoModelForCausalLM  # type: ignore
			hf_model = AutoModelForCausalLM.from_pretrained(
				"meta-llama/Llama-3.1-8B-Instruct",
				device_map="auto" if torch.cuda.is_available() else None,
				torch_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
			)
			# Optional: load PEFT/LoRA adapter when provided
			adapter_path = os.getenv("ADAPTER_PATH", "").strip()
			if adapter_path:
				try:
					from peft import PeftModel  # type: ignore
					hf_model = PeftModel.from_pretrained(hf_model, adapter_path)
					print(f"[eval] loaded adapter from {adapter_path}")
				except Exception as e:
					print(f"[eval] Failed to load adapter at {adapter_path}: {e}")
			hf_model.eval()
		except Exception as e:
			print(f"[eval] Failed to load HF model: {e}")
			return
	else:
		# Local tiny LM (trained vs random)
		from specs.config import ModelConfig  # type: ignore
		from model.factory import build_causal_lm  # type: ignore
		cfg_m = ModelConfig(
			d_model=256, n_heads=8, n_layers=2, d_ff=1024, vocab_size=int(getattr(tok, "vocab_size", 32000) or 32000),
			dtype="float32", kv_cache_paged=False, attn_impl="eager",
		)
		model = build_causal_lm(cfg_m, block="llama", compress=None)
		model = model.to(device)
		if eval_random:
			print("[eval] EVAL_RANDOM=1 set, skipping checkpoint load (random-initialized model).")
		else:
			if ckpt_path.exists():
				state = torch.load(ckpt_path, map_location=device)
				model.load_state_dict(state, strict=False)
			else:
				print(f"[eval] Warning: checkpoint not found at {ckpt_path}, evaluating random-initialized model.")
		model.eval()

	# Evaluate
	# Use pad-aware loss
	if eval_model == "llama_hf":
		pad_id = int(getattr(hf_tok, "pad_token_id", 0) or 0)
	else:
		pad_id = int(getattr(tok, "pad_token_id", 0) or 0)
	loss_fn = torch.nn.CrossEntropyLoss(ignore_index=-100)
	total_loss = 0.0
	num = 0
	with torch.no_grad():
		if eval_model == "llama_hf":
			max_len = 128
			for t in texts:
				enc = hf_tok(t, return_tensors="pt", truncation=True, max_length=max_len, padding="max_length")
				input_ids = enc["input_ids"].to(device if torch.cuda.is_available() else "cpu")
				attn_mask = enc["attention_mask"].to(device if torch.cuda.is_available() else "cpu")
				out = hf_model(input_ids=input_ids, attention_mask=attn_mask, return_dict=True)
				logits = out["logits"]
				vocab = logits.size(-1)
				labels = input_ids[..., 1:].contiguous()
				labels = labels.masked_fill(labels == pad_id, -100)
				loss = loss_fn(
					logits[..., :-1, :].contiguous().view(-1, vocab),
					labels.view(-1),
				)
				total_loss += float(loss.detach().item())
				num += 1
		else:
			for i in range(len(ds)):
				batch = ds[i]
				batch = {k: v.to(device) for k, v in batch.items()}
				out = model(input_ids=batch["input_ids"], attention_mask=batch["attn_mask"], return_dict=True)
				logits = out["logits"]  # [B, T, V]
				vocab = logits.size(-1)
				labels = batch["input_ids"][..., 1:].contiguous()
				labels = labels.masked_fill(labels == pad_id, -100)
				loss = loss_fn(
					logits[..., :-1, :].contiguous().view(-1, vocab),
					labels.view(-1),
				)
				total_loss += float(loss.detach().item())
				num += 1
	avg_loss = total_loss / max(1, num)
	try:
		import math
		ppl = float(math.exp(avg_loss))
	except Exception:
		ppl = None

	report = {
		"split": "test",
		"avg_loss": avg_loss,
		"ppl": ppl,
		"samples": num,
		"device": str(device),
		"ckpt_path": str(ckpt_path),
	}
	if eval_model == "llama_hf":
		out_filename = "MBPPEvalTestReport.llama.json"
	else:
		out_filename = "MBPPEvalTestReport.random.json" if eval_random else "MBPPEvalTestReport.json"
	out_path = out_dir / out_filename
	with open(out_path, "w", encoding="utf-8") as fh:
		fh.write(json.dumps({"schema_version": 1, "EvalReport": report}, indent=2))
	print(str(out_path))

if __name__ == "__main__":
	main()


SimpleLmDataset

from __future__ import annotations

import sys
import os
import json
from pathlib import Path
from typing import List, Dict

import torch
from torch.utils.data import Dataset

ROOT = EX_DIR
try:
	from .program_config import load_program_config  # type: ignore  # noqa: E402
except Exception:
	from program_config import load_program_config  # type: ignore  # noqa: E402

class SimpleLmDataset(Dataset):
	def __init__(self, tokenizer, texts: List[str], max_len: int = 128) -> None:
		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)
		attn = (input_ids != pad_id).to(torch.long)
		return {"input_ids": input_ids.unsqueeze(0), "attn_mask": attn.unsqueeze(0)}

def main() -> None:
	# Prefer evaluating the "complete" run by default
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "complete_dataset_trainer"
	if not artifacts_root.exists():
		# Fallback to smoke artifacts if complete doesn't exist
		artifacts_root = example_dir / "artifacts" / "smoke_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	ckpt_path = adapters_dir / "trained_ckpt" / "model.pt"
	out_dir.mkdir(parents=True, exist_ok=True)
	eval_random = os.getenv("EVAL_RANDOM", "0") == "1"
	eval_model = os.getenv("EVAL_MODEL", "local")  # "local" (toy LM) or "llama_hf"

	if str(repo_root) not in sys.path:
		sys.path.insert(0, str(repo_root))
	py_path = os.environ.get("PYTHONPATH") or ""
	os.environ["PYTHONPATH"] = f"{str(repo_root)}{(':' + py_path) if py_path else ''}"


main

	def __init__(self, tokenizer, texts: List[str], max_len: int = 128) -> None:
		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)
		attn = (input_ids != pad_id).to(torch.long)
		return {"input_ids": input_ids.unsqueeze(0), "attn_mask": attn.unsqueeze(0)}

def main() -> None:
	# Prefer evaluating the "complete" run by default
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "complete_dataset_trainer"
	if not artifacts_root.exists():
		# Fallback to smoke artifacts if complete doesn't exist
		artifacts_root = example_dir / "artifacts" / "smoke_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	ckpt_path = adapters_dir / "trained_ckpt" / "model.pt"
	out_dir.mkdir(parents=True, exist_ok=True)
	eval_random = os.getenv("EVAL_RANDOM", "0") == "1"
	eval_model = os.getenv("EVAL_MODEL", "local")  # "local" (toy LM) or "llama_hf"

	if str(repo_root) not in sys.path:
		sys.path.insert(0, str(repo_root))
	py_path = os.environ.get("PYTHONPATH") or ""
	os.environ["PYTHONPATH"] = f"{str(repo_root)}{(':' + py_path) if py_path else ''}"

	# Tokenizer(s)
	if eval_model == "llama_hf":
		# Hugging Face Llama tokenizer for baseline (no adapters)
		try:
			from transformers import AutoTokenizer  # type: ignore
			hf_tok = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B-Instruct")
			# Ensure pad token is set for batching; use eos as pad
			if hf_tok.pad_token_id is None and hf_tok.eos_token_id is not None:
				hf_tok.pad_token = hf_tok.eos_token
		except Exception as e:
			print(f"[eval] Failed to load HF tokenizer: {e}")
			return
	else:
		from data.tokenizer import LocalLlamaTokenizer  # type: ignore
		try:
			snap_default = os.environ.get("LLAMA_SNAPSHOT_DIR", str(repo_root / "checkpoints" / "llama"))
			tok = LocalLlamaTokenizer(snap_default)
		except Exception:
			from data.tokenizer import WhitespaceTokenizer  # type: ignore
			tok = WhitespaceTokenizer()

	# Load MBPP test split
	texts: List[str] = []
	total = 0
	try:
		from examples.scripts.mbpp_loader import load_mbpp_texts  # type: ignore
		texts, total = load_mbpp_texts(max_n=1_000_000, split="test")
		print(f"[mbpp:test] loaded {len(texts)} / {total} samples")
	except Exception as e:
		print(f"[mbpp:test] failed to load via HF datasets: {e}")
	if not texts:
		print("[eval] No test texts available; exiting with empty report.")
		report = {
			"split": "test",
			"avg_loss": None,
			"ppl": None,
			"samples": 0,
			"device": "cpu",
			"ckpt_path": str(ckpt_path),
		}
		out_path = out_dir / "MBPPEvalTestReport.json"
		with open(out_path, "w", encoding="utf-8") as fh:
			fh.write(json.dumps({"schema_version": 1, "EvalReport": report}, indent=2))
		print(str(out_path))
		return

	# Build dataset or leave as raw texts depending on eval model
	if eval_model == "llama_hf":
		ds = None  # we will use hf_tok directly on texts per-sample
	else:
		ds = SimpleLmDataset(tok, texts, max_len=128)

	# Model selection
	device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
	if eval_model == "llama_hf":
		try:
			from transformers import AutoModelForCausalLM  # type: ignore
			hf_model = AutoModelForCausalLM.from_pretrained(
				"meta-llama/Llama-3.1-8B-Instruct",
				device_map="auto" if torch.cuda.is_available() else None,
				torch_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
			)
			# Optional: load PEFT/LoRA adapter when provided
			adapter_path = os.getenv("ADAPTER_PATH", "").strip()
			if adapter_path:
				try:
					from peft import PeftModel  # type: ignore
					hf_model = PeftModel.from_pretrained(hf_model, adapter_path)
					print(f"[eval] loaded adapter from {adapter_path}")
				except Exception as e:
					print(f"[eval] Failed to load adapter at {adapter_path}: {e}")
			hf_model.eval()
		except Exception as e:
			print(f"[eval] Failed to load HF model: {e}")
			return
	else:
		# Local tiny LM (trained vs random)
		from specs.config import ModelConfig  # type: ignore
		from model.factory import build_causal_lm  # type: ignore
		cfg_m = ModelConfig(
			d_model=256, n_heads=8, n_layers=2, d_ff=1024, vocab_size=int(getattr(tok, "vocab_size", 32000) or 32000),
			dtype="float32", kv_cache_paged=False, attn_impl="eager",
		)
		model = build_causal_lm(cfg_m, block="llama", compress=None)
		model = model.to(device)
		if eval_random:
			print("[eval] EVAL_RANDOM=1 set, skipping checkpoint load (random-initialized model).")
		else:
			if ckpt_path.exists():
				state = torch.load(ckpt_path, map_location=device)
				model.load_state_dict(state, strict=False)
			else:
				print(f"[eval] Warning: checkpoint not found at {ckpt_path}, evaluating random-initialized model.")
		model.eval()

	# Evaluate
	# Use pad-aware loss
	if eval_model == "llama_hf":
		pad_id = int(getattr(hf_tok, "pad_token_id", 0) or 0)
	else:
		pad_id = int(getattr(tok, "pad_token_id", 0) or 0)
	loss_fn = torch.nn.CrossEntropyLoss(ignore_index=-100)
	total_loss = 0.0
	num = 0
	with torch.no_grad():
		if eval_model == "llama_hf":
			max_len = 128
			for t in texts:
				enc = hf_tok(t, return_tensors="pt", truncation=True, max_length=max_len, padding="max_length")
				input_ids = enc["input_ids"].to(device if torch.cuda.is_available() else "cpu")
				attn_mask = enc["attention_mask"].to(device if torch.cuda.is_available() else "cpu")
				out = hf_model(input_ids=input_ids, attention_mask=attn_mask, return_dict=True)
				logits = out["logits"]
				vocab = logits.size(-1)
				labels = input_ids[..., 1:].contiguous()
				labels = labels.masked_fill(labels == pad_id, -100)
				loss = loss_fn(
					logits[..., :-1, :].contiguous().view(-1, vocab),
					labels.view(-1),
				)
				total_loss += float(loss.detach().item())
				num += 1
		else:
			for i in range(len(ds)):
				batch = ds[i]
				batch = {k: v.to(device) for k, v in batch.items()}
				out = model(input_ids=batch["input_ids"], attention_mask=batch["attn_mask"], return_dict=True)
				logits = out["logits"]  # [B, T, V]
				vocab = logits.size(-1)
				labels = batch["input_ids"][..., 1:].contiguous()
				labels = labels.masked_fill(labels == pad_id, -100)
				loss = loss_fn(
					logits[..., :-1, :].contiguous().view(-1, vocab),
					labels.view(-1),
				)
				total_loss += float(loss.detach().item())
				num += 1
	avg_loss = total_loss / max(1, num)
	try:
		import math
		ppl = float(math.exp(avg_loss))
	except Exception:
		ppl = None

	report = {
		"split": "test",
		"avg_loss": avg_loss,
		"ppl": ppl,
		"samples": num,
		"device": str(device),
		"ckpt_path": str(ckpt_path),
	}
	if eval_model == "llama_hf":
		out_filename = "MBPPEvalTestReport.llama.json"
	else:
		out_filename = "MBPPEvalTestReport.random.json" if eval_random else "MBPPEvalTestReport.json"
	out_path = out_dir / out_filename
	with open(out_path, "w", encoding="utf-8") as fh:
		fh.write(json.dumps({"schema_version": 1, "EvalReport": report}, indent=2))
	print(str(out_path))

if __name__ == "__main__":
	main()


from __future__ import annotations

import sys
import os
import json
from pathlib import Path
from typing import List, Dict

import torch
from torch.utils.data import Dataset

ROOT = EX_DIR
try:
	from .program_config import load_program_config  # type: ignore  # noqa: E402
except Exception:
	from program_config import load_program_config  # type: ignore  # noqa: E402

class SimpleLmDataset(Dataset):
	def __init__(self, tokenizer, texts: List[str], max_len: int = 128) -> None:
		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)
		attn = (input_ids != pad_id).to(torch.long)
		return {"input_ids": input_ids.unsqueeze(0), "attn_mask": attn.unsqueeze(0)}

def main() -> None:
	# Prefer evaluating the "complete" run by default
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "complete_dataset_trainer"
	if not artifacts_root.exists():
		# Fallback to smoke artifacts if complete doesn't exist
		artifacts_root = example_dir / "artifacts" / "smoke_dataset_trainer"


import torch
from torch.utils.data import Dataset

ROOT = EX_DIR
try:
	from .program_config import load_program_config  # type: ignore  # noqa: E402
except Exception:
	from program_config import load_program_config  # type: ignore  # noqa: E402

class SimpleLmDataset(Dataset):
	def __init__(self, tokenizer, texts: List[str], max_len: int = 128) -> None:
		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)
		attn = (input_ids != pad_id).to(torch.long)
		return {"input_ids": input_ids.unsqueeze(0), "attn_mask": attn.unsqueeze(0)}

def main() -> None:
	# Prefer evaluating the "complete" run by default
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "complete_dataset_trainer"
	if not artifacts_root.exists():
		# Fallback to smoke artifacts if complete doesn't exist
		artifacts_root = example_dir / "artifacts" / "smoke_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	ckpt_path = adapters_dir / "trained_ckpt" / "model.pt"


ROOT = EX_DIR
try:
	from .program_config import load_program_config  # type: ignore  # noqa: E402
except Exception:
	from program_config import load_program_config  # type: ignore  # noqa: E402

class SimpleLmDataset(Dataset):
	def __init__(self, tokenizer, texts: List[str], max_len: int = 128) -> None:
		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)
		attn = (input_ids != pad_id).to(torch.long)
		return {"input_ids": input_ids.unsqueeze(0), "attn_mask": attn.unsqueeze(0)}

def main() -> None:
	# Prefer evaluating the "complete" run by default
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "complete_dataset_trainer"
	if not artifacts_root.exists():
		# Fallback to smoke artifacts if complete doesn't exist
		artifacts_root = example_dir / "artifacts" / "smoke_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	ckpt_path = adapters_dir / "trained_ckpt" / "model.pt"
	out_dir.mkdir(parents=True, exist_ok=True)
	eval_random = os.getenv("EVAL_RANDOM", "0") == "1"
	eval_model = os.getenv("EVAL_MODEL", "local")  # "local" (toy LM) or "llama_hf"

	if str(repo_root) not in sys.path:
		sys.path.insert(0, str(repo_root))
	py_path = os.environ.get("PYTHONPATH") or ""
	os.environ["PYTHONPATH"] = f"{str(repo_root)}{(':' + py_path) if py_path else ''}"


from __future__ import annotations

import sys
import subprocess
from pathlib import Path

# Resolve program_conditioned_adapter directory directly from this file
EX_DIR = Path(__file__).resolve().parents[2]
SMOKE_REPO = EX_DIR / "smoke_repo"
PG_BACKEND = "examples.scripts.python_repo_graph:PythonRepoGraph"
# Program configuration
from .program_config import load_program_config  # noqa: E402

def _run(cmd: list[str]) -> int:
    print("[run]", " ".join(cmd))
    return subprocess.call(cmd)

def main() -> None:
    cfg = load_program_config(str(SMOKE_REPO))
    adapters_dir = cfg.paths.adapters_dir
    adapters_dir.mkdir(parents=True, exist_ok=True)
    # Emit planning knowledge artifacts (entities/edges/artifacts-derived plans)
    rc = _run([
        sys.executable,
        str(Path(__file__).resolve().parent / "emit_planning_knowledge.py"),
        "--program", str(SMOKE_REPO),
        "--pg-backend", (cfg.pg_backend if ":" in cfg.pg_backend else PG_BACKEND),
        "--out-dir", str(cfg.paths.knowledge_dir),
        "--max-modules", "200",
        "--verbose",
    ])
    if rc != 0:
        sys.exit(rc)

    # Build adapters and caches for the smoke repo (PCA-agnostic; PG via --pg-backend)
    rc = _run([
        sys.executable,
        str(EX_DIR / "build.py"),
        "--sources", str(SMOKE_REPO),
        "--model", "meta-llama/Llama-3.1-8B-Instruct",
        "--adapters-dir", str(adapters_dir),
        "--embed-dim", "256",
        "--include-text",
        "--text-max-bytes", "20000",
        "--pg-backend", cfg.pg_backend if ":" in cfg.pg_backend else PG_BACKEND,
        "--graph-prop-hops", "2",
        "--graph-prop-damp", "0.85",
        "--contracts-require-citations",
        "--contracts-retrieval-policy", str(cfg.contracts.retrieval_policy),
        "--contracts-retrieval-temp", str(cfg.contracts.retrieval_temp),
        "--kbann-priors",
        "--knowledge-preset",
        "--auto-rank",
        "--rank-min", "8",
        "--rank-max", "16",
        "--init-program-state",
        "--program-state-path", str(cfg.paths.program_state_path),
        "--seed", "0",
        "--verbose",
    ])
    if rc != 0:
        sys.exit(rc)

    # Grounded planning prompt (facts-seeded steps + citations)
    prompt = (
        "Produce a grounded step-by-step plan to add a new CLI subcommand "
        "that lists all modules and their public functions. Include specific paths and "
        "line ranges to modify or create, referencing code entities with citations."
    )

    # Run grounded planning (structured mode; citations enforced)
    rc = _run([
        sys.executable,
        str(EX_DIR / "run.py"),
        "--sources", str(SMOKE_REPO),
        "--model", "meta-llama/Llama-3.1-8B-Instruct",
        "--adapters-dir", str(adapters_dir),
        "--prompt", prompt,
        "--of-sources", "question",
        "--pack-context",
        "--pack-mode", "windows",
        "--context-tokens", "1200",
        "--require-citations",
        "--structured",
        "--citations-enforce",
        "--pg-backend", cfg.pg_backend if ":" in cfg.pg_backend else PG_BACKEND,
        "--retrieval-policy", cfg.contracts.retrieval_policy,
        "--retrieval-temp", str(cfg.contracts.retrieval_temp),
        "--alpha-warmup",
        "--adapter-aware-decoding",
        "--program-state", str(cfg.paths.program_state_path),
        "--delta-cap", "0.05",
        "--verbose",
    ])
    sys.exit(rc)

if __name__ == "__main__":
    main()


from __future__ import annotations

import sys
import subprocess
from pathlib import Path

# Resolve program_conditioned_adapter directory directly from this file
EX_DIR = Path(__file__).resolve().parents[2]
SMOKE_REPO = EX_DIR / "smoke_repo"
PG_BACKEND = "examples.scripts.python_repo_graph:PythonRepoGraph"
# Program configuration
from .program_config import load_program_config  # noqa: E402

def _run(cmd: list[str]) -> int:
    print("[run]", " ".join(cmd))
    return subprocess.call(cmd)

def main() -> None:
    cfg = load_program_config(str(SMOKE_REPO))
    adapters_dir = cfg.paths.adapters_dir
    adapters_dir.mkdir(parents=True, exist_ok=True)
    # Emit planning knowledge artifacts (entities/edges/artifacts-derived plans)
    rc = _run([
        sys.executable,
        str(Path(__file__).resolve().parent / "emit_planning_knowledge.py"),
        "--program", str(SMOKE_REPO),
        "--pg-backend", (cfg.pg_backend if ":" in cfg.pg_backend else PG_BACKEND),
        "--out-dir", str(cfg.paths.knowledge_dir),
        "--max-modules", "200",
        "--verbose",
    ])
    if rc != 0:
        sys.exit(rc)

    # Build adapters and caches for the smoke repo (PCA-agnostic; PG via --pg-backend)


from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Optional

@dataclass(frozen=True)
class ProgramContracts:
    require_citations: bool = True
    citations_per_paragraph: bool = False
    retrieval_policy: str = "sim:0.6,struct:0.4,plan:0.2"
    retrieval_temp: float = 0.7

@dataclass(frozen=True)
class ProgramPaths:
    adapters_dir: Path
    knowledge_dir: Path
    program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
    program_id: str
    pg_backend: str  # dotted path module:Class
    paths: ProgramPaths
    contracts: ProgramContracts

def _detect_pg_backend(example_dir: Path) -> str:
    # Reuse the QA PythonRepoGraph backend by default
    return "examples.scripts.python_repo_graph:PythonRepoGraph"

def load_program_config(repo_root: str) -> ProgramConfig:
    repo = Path(repo_root).resolve()
    example_dir = Path(__file__).resolve().parent
    adapters_dir = example_dir / "artifacts" / "smoke_planning"
    knowledge_dir = adapters_dir  # colocate planning knowledge
    pg_backend = _detect_pg_backend(example_dir)
    program_id = repo.name or "repo"
    return ProgramConfig(
        program_id=program_id,
        pg_backend=pg_backend,
        paths=ProgramPaths(
            adapters_dir=adapters_dir,
            knowledge_dir=knowledge_dir,
            program_state_path=adapters_dir / ".program_state.json",
        ),
        contracts=ProgramContracts(),
    )


ProgramContracts

from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Optional

@dataclass(frozen=True)
class ProgramContracts:
    require_citations: bool = True
    citations_per_paragraph: bool = False
    retrieval_policy: str = "sim:0.6,struct:0.4,plan:0.2"
    retrieval_temp: float = 0.7

@dataclass(frozen=True)
class ProgramPaths:
    adapters_dir: Path
    knowledge_dir: Path
    program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
    program_id: str
    pg_backend: str  # dotted path module:Class
    paths: ProgramPaths
    contracts: ProgramContracts

def _detect_pg_backend(example_dir: Path) -> str:
    # Reuse the QA PythonRepoGraph backend by default
    return "examples.scripts.python_repo_graph:PythonRepoGraph"


ProgramPaths

from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Optional

@dataclass(frozen=True)
class ProgramContracts:
    require_citations: bool = True
    citations_per_paragraph: bool = False
    retrieval_policy: str = "sim:0.6,struct:0.4,plan:0.2"
    retrieval_temp: float = 0.7

@dataclass(frozen=True)
class ProgramPaths:
    adapters_dir: Path
    knowledge_dir: Path
    program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
    program_id: str
    pg_backend: str  # dotted path module:Class
    paths: ProgramPaths
    contracts: ProgramContracts

def _detect_pg_backend(example_dir: Path) -> str:
    # Reuse the QA PythonRepoGraph backend by default
    return "examples.scripts.python_repo_graph:PythonRepoGraph"

def load_program_config(repo_root: str) -> ProgramConfig:
    repo = Path(repo_root).resolve()
    example_dir = Path(__file__).resolve().parent
    adapters_dir = example_dir / "artifacts" / "smoke_planning"
    knowledge_dir = adapters_dir  # colocate planning knowledge


ProgramConfig

from pathlib import Path
from typing import Optional

@dataclass(frozen=True)
class ProgramContracts:
    require_citations: bool = True
    citations_per_paragraph: bool = False
    retrieval_policy: str = "sim:0.6,struct:0.4,plan:0.2"
    retrieval_temp: float = 0.7

@dataclass(frozen=True)
class ProgramPaths:
    adapters_dir: Path
    knowledge_dir: Path
    program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
    program_id: str
    pg_backend: str  # dotted path module:Class
    paths: ProgramPaths
    contracts: ProgramContracts

def _detect_pg_backend(example_dir: Path) -> str:
    # Reuse the QA PythonRepoGraph backend by default
    return "examples.scripts.python_repo_graph:PythonRepoGraph"

def load_program_config(repo_root: str) -> ProgramConfig:
    repo = Path(repo_root).resolve()
    example_dir = Path(__file__).resolve().parent
    adapters_dir = example_dir / "artifacts" / "smoke_planning"
    knowledge_dir = adapters_dir  # colocate planning knowledge
    pg_backend = _detect_pg_backend(example_dir)
    program_id = repo.name or "repo"
    return ProgramConfig(
        program_id=program_id,
        pg_backend=pg_backend,
        paths=ProgramPaths(
            adapters_dir=adapters_dir,
            knowledge_dir=knowledge_dir,


    citations_per_paragraph: bool = False
    retrieval_policy: str = "sim:0.6,struct:0.4,plan:0.2"
    retrieval_temp: float = 0.7

@dataclass(frozen=True)
class ProgramPaths:
    adapters_dir: Path
    knowledge_dir: Path
    program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
    program_id: str
    pg_backend: str  # dotted path module:Class
    paths: ProgramPaths
    contracts: ProgramContracts

def _detect_pg_backend(example_dir: Path) -> str:
    # Reuse the QA PythonRepoGraph backend by default
    return "examples.scripts.python_repo_graph:PythonRepoGraph"

def load_program_config(repo_root: str) -> ProgramConfig:
    repo = Path(repo_root).resolve()
    example_dir = Path(__file__).resolve().parent
    adapters_dir = example_dir / "artifacts" / "smoke_planning"
    knowledge_dir = adapters_dir  # colocate planning knowledge
    pg_backend = _detect_pg_backend(example_dir)
    program_id = repo.name or "repo"
    return ProgramConfig(
        program_id=program_id,
        pg_backend=pg_backend,
        paths=ProgramPaths(
            adapters_dir=adapters_dir,
            knowledge_dir=knowledge_dir,
            program_state_path=adapters_dir / ".program_state.json",
        ),
        contracts=ProgramContracts(),
    )


@dataclass(frozen=True)
class ProgramPaths:
    adapters_dir: Path
    knowledge_dir: Path
    program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
    program_id: str
    pg_backend: str  # dotted path module:Class
    paths: ProgramPaths
    contracts: ProgramContracts

def _detect_pg_backend(example_dir: Path) -> str:
    # Reuse the QA PythonRepoGraph backend by default
    return "examples.scripts.python_repo_graph:PythonRepoGraph"

def load_program_config(repo_root: str) -> ProgramConfig:
    repo = Path(repo_root).resolve()
    example_dir = Path(__file__).resolve().parent
    adapters_dir = example_dir / "artifacts" / "smoke_planning"
    knowledge_dir = adapters_dir  # colocate planning knowledge
    pg_backend = _detect_pg_backend(example_dir)
    program_id = repo.name or "repo"
    return ProgramConfig(
        program_id=program_id,
        pg_backend=pg_backend,
        paths=ProgramPaths(
            adapters_dir=adapters_dir,
            knowledge_dir=knowledge_dir,
            program_state_path=adapters_dir / ".program_state.json",
        ),
        contracts=ProgramContracts(),
    )


from __future__ import annotations

import sys
from pathlib import Path
import subprocess

EX_DIR = Path("/data/transformer_10/examples/program_conditioned_adapter")
SMOKE_PROG = Path("/data/transformer_10/examples/program_conditioned_adapter/smoke_repo")
ART_DIR = EX_DIR / "examples" / "python_repo_grounded_planning" / "artifacts" / "smoke_planning"

PG_BACKEND = "examples.scripts.python_repo_graph:PythonRepoGraph"
EMIT_MOD = "examples.python_repo_grounded_planning.emit_planning_knowledge"

def _run(argv: list[str]) -> int:
    print("[run]", " ".join(argv))
    return subprocess.call(argv)

def main() -> None:
    ART_DIR.mkdir(parents=True, exist_ok=True)

    rc = _run([
        sys.executable, "-m", "build",
        "--program", str(SMOKE_PROG),
        "--pg-backend", PG_BACKEND,
        "--model", "meta-llama/Llama-3.1-8B-Instruct",
        "--adapters-dir", str(ART_DIR),
        "--embed-dim", "256",
        "--include-text",
        "--text-max-bytes", "20000",
        "--graph-prop-hops", "1",
        "--graph-prop-damp", "0.85",
        "--code-recall-preset",
        "--auto-rank",
        "--rank-min", "4",
        "--rank-max", "16",
        "--seed", "0",
        "--verbose",
    ])
    if rc != 0:
        sys.exit(rc)

    rc = _run([
        sys.executable, "-m", EMIT_MOD,
        "--program", str(SMOKE_PROG),
        "--pg-backend", PG_BACKEND,
        "--out-dir", str(ART_DIR),
        "--max-modules", "200",
        "--verbose",
    ])
    if rc != 0:
        sys.exit(rc)

    prompt = (
        "Produce a grounded step-by-step plan to add a new CLI subcommand that lists all modules "
        "and their public functions. Include exact files or artifact URIs and line ranges to modify/create, "
        "and cite entities/windows."
    )

    rc = _run([
        sys.executable, "-m", "run",
        "--program", str(SMOKE_PROG),
        "--pg-backend", PG_BACKEND,
        "--model", "meta-llama/Llama-3.1-8B-Instruct",
        "--adapters-dir", str(ART_DIR),
        "--prompt", prompt,
        "--of-sources", "question",
        "--pack-context",
        "--pack-mode", "windows",
        "--context-tokens", "1200",
        "--require-citations",
        "--structured",
        "--citations-enforce",
        "--retrieval-policy", "sim:0.45,struct:0.35,plan:0.20",
        "--retrieval-temp", "0.7",
        "--alpha-warmup",
        "--adapter-aware-decoding",
        "--verbose",
    ])
    sys.exit(rc)

if __name__ == "__main__":
    main()


from __future__ import annotations

import sys
from pathlib import Path
import subprocess

EX_DIR = Path("/data/transformer_10/examples/program_conditioned_adapter")
SMOKE_PROG = Path("/data/transformer_10/examples/program_conditioned_adapter/smoke_repo")
ART_DIR = EX_DIR / "examples" / "python_repo_grounded_planning" / "artifacts" / "smoke_planning"

PG_BACKEND = "examples.scripts.python_repo_graph:PythonRepoGraph"
EMIT_MOD = "examples.python_repo_grounded_planning.emit_planning_knowledge"

def _run(argv: list[str]) -> int:
    print("[run]", " ".join(argv))
    return subprocess.call(argv)

def main() -> None:
    ART_DIR.mkdir(parents=True, exist_ok=True)

    rc = _run([
        sys.executable, "-m", "build",
        "--program", str(SMOKE_PROG),
        "--pg-backend", PG_BACKEND,
        "--model", "meta-llama/Llama-3.1-8B-Instruct",
        "--adapters-dir", str(ART_DIR),
        "--embed-dim", "256",
        "--include-text",
        "--text-max-bytes", "20000",
        "--graph-prop-hops", "1",
        "--graph-prop-damp", "0.85",
        "--code-recall-preset",
        "--auto-rank",
        "--rank-min", "4",
        "--rank-max", "16",


from __future__ import annotations

import argparse
import json
import os
from pathlib import Path
from typing import Dict, List, Tuple, Optional, Set
import importlib
import numpy as np

def _load_symbol(path: str):
    mod, _, attr = path.partition(":")
    m = importlib.import_module(mod)
    return getattr(m, attr)

def _write_jsonl(path: Path, rows: List[Dict]) -> None:
    with path.open("w", encoding="utf-8") as f:
        for r in rows:
            f.write(json.dumps(r) + "\n")

def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--program", required=True)
    ap.add_argument("--pg-backend", required=True)
    ap.add_argument("--out-dir", required=True)
    ap.add_argument("--max-modules", type=int, default=200)
    ap.add_argument("--verbose", action="store_true")
    args = ap.parse_args()

    out_dir = Path(args.out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)

    pg_ctor = _load_symbol(args.pg_backend)
    pg = pg_ctor(args.program, ignore=None)

    # Collect entities and a simple owner map
    entities = list(pg.entities())
    modules = [e for e in entities if e.kind == "module"]
    functions = [e for e in entities if e.kind == "function"]
    classes = [e for e in entities if e.kind == "class"]

    # 1) planning_components.jsonl (owners/modules with primary artifact spans)
    comps: List[Dict] = []
    for m in modules[: max(1, int(args.max_modules))]:
        try:
            ra = pg.resolve(m.uri)
            comps.append({
                "owner": (m.name or m.id),
                "kind": "module",
                "artifact_uri": ra.artifact_uri,
                "span": {"start": int(ra.span.start_line), "end": int(ra.span.end_line)},
            })
        except Exception:
            continue
    _write_jsonl(out_dir / "planning_components.jsonl", comps)

    # 2) planning_entrypoints.jsonl (very light heuristic: functions named main or having owner 'cli'/'__main__')
    entries: List[Dict] = []
    for fn in functions:
        name = (fn.name or "").lower()
        own = (fn.owner or "").lower()
        if name == "main" or "__main__" in own or "cli" in own:
            try:
                ra = pg.resolve(fn.uri)
                entries.append({
                    "type": "cli",
                    "name": fn.name,
                    "handler": f"{own}:{fn.name}" if own and fn.name else fn.id,
                    "decl_uri": ra.artifact_uri,
                    "span": {"start": int(ra.span.start_line), "end": int(ra.span.end_line)},
                })
            except Exception:
                continue
    _write_jsonl(out_dir / "planning_entrypoints.jsonl", entries)

    # 3) planning_mutations.jsonl (modules as candidate edit sites with affordances)
    muts: List[Dict] = []
    for m in modules[: max(1, int(args.max_modules))]:
        try:
            ra = pg.resolve(m.uri)
            muts.append({
                "target": (m.name or m.id),
                "kind": "module",
                "artifact_uri": ra.artifact_uri,
                "affordances": ["add_function", "edit_imports", "edit_exports"],
            })
        except Exception:
            continue
    _write_jsonl(out_dir / "planning_mutations.jsonl", muts)

    # 4) planning_tests_map.jsonl (map owners to test files by simple heuristic)
    tests_map: List[Dict] = []
    # harvest artifacts(kind="source") and flag ones with "test" in path
    test_arts: Set[str] = set()
    for art in pg.artifacts("source"):
        p = art.uri.lower()
        if ("test" in p) or ("/tests/" in p):
            test_arts.add(art.uri)
    for m in modules:
        # naive mapping: module owner name in test path
        owner = (m.name or m.id)
        for ta in test_arts:
            if owner.split(".")[-1] in ta:
                tests_map.append({
                    "owner": owner,
                    "test_file": ta,
                    "span": {"start": 1, "end": 1},
                })
                break
    _write_jsonl(out_dir / "planning_tests_map.jsonl", tests_map)

    # 5) planning_dependencies.jsonl (import/call edges at entity granularity)
    deps: List[Dict] = []
    try:
        for e in pg.edges():
            deps.append({"src": e.src, "dst": e.dst, "edge": e.type})
    except Exception:
        deps = []
    _write_jsonl(out_dir / "planning_dependencies.jsonl", deps)

    # 6) planning_rerank_features.npz (very light owner graph centrality and placeholders)
    owners = sorted(list({(m.owner or m.name or m.id) for m in modules}))
    owner_index: Dict[str, int] = {o: i for i, o in enumerate(owners)}
    centrality = np.zeros((len(owners),), dtype=np.float32)
    try:
        owner_edges: Dict[str, Set[str]] = {}
        for e in pg.edges():
            # map entity ids to owners if available
            srco = None
            dsto = None
            # build entity -> owner map lazily
        ent_owner: Dict[str, Optional[str]] = {}
        for ent in entities:
            ent_owner[ent.id] = ent.owner
        for e in pg.edges():
            so = ent_owner.get(e.src)
            do = ent_owner.get(e.dst)
            if so and do:
                owner_edges.setdefault(so, set()).add(do)
                owner_edges.setdefault(do, set()).add(so)
        for o, nbs in owner_edges.items():
            idx = owner_index.get(o)
            if idx is not None:
                centrality[idx] = float(len(nbs))
        if centrality.max() > 0:
            centrality = centrality / centrality.max()
    except Exception:
        centrality = np.zeros((len(owners),), dtype=np.float32)
    registry_score = np.zeros_like(centrality, dtype=np.float32)
    test_coverage = np.zeros_like(centrality, dtype=np.float32)
    np.savez_compressed(
        out_dir / "planning_rerank_features.npz",
        owners=np.array(owners, dtype=object),
        centrality=centrality,
        registry_score=registry_score,
        test_coverage=test_coverage,
    )

    if args.verbose:
        print(json.dumps({
            "components": len(comps),
            "entrypoints": len(entries),
            "mutations": len(muts),
            "tests_map": len(tests_map),
            "dependencies": len(deps),
            "owners": len(owners),
        }, indent=2))

if __name__ == "__main__":
    main()


from __future__ import annotations

import argparse
import json
import os
from pathlib import Path
from typing import Dict, List, Tuple, Optional, Set
import importlib
import numpy as np

def _load_symbol(path: str):
    mod, _, attr = path.partition(":")
    m = importlib.import_module(mod)
    return getattr(m, attr)

def _write_jsonl(path: Path, rows: List[Dict]) -> None:
    with path.open("w", encoding="utf-8") as f:
        for r in rows:
            f.write(json.dumps(r) + "\n")

def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--program", required=True)
    ap.add_argument("--pg-backend", required=True)
    ap.add_argument("--out-dir", required=True)
    ap.add_argument("--max-modules", type=int, default=200)
    ap.add_argument("--verbose", action="store_true")
    args = ap.parse_args()

    out_dir = Path(args.out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)


from __future__ import annotations

import argparse
import json
import os
from pathlib import Path
from typing import Dict, List, Tuple, Optional, Set
import importlib
import numpy as np

def _load_symbol(path: str):
    mod, _, attr = path.partition(":")
    m = importlib.import_module(mod)
    return getattr(m, attr)

def _write_jsonl(path: Path, rows: List[Dict]) -> None:
    with path.open("w", encoding="utf-8") as f:
        for r in rows:
            f.write(json.dumps(r) + "\n")

def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--program", required=True)
    ap.add_argument("--pg-backend", required=True)
    ap.add_argument("--out-dir", required=True)
    ap.add_argument("--max-modules", type=int, default=200)
    ap.add_argument("--verbose", action="store_true")
    args = ap.parse_args()

    out_dir = Path(args.out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)

    pg_ctor = _load_symbol(args.pg_backend)
    pg = pg_ctor(args.program, ignore=None)

    # Collect entities and a simple owner map
    entities = list(pg.entities())
    modules = [e for e in entities if e.kind == "module"]


main

import json
import os
from pathlib import Path
from typing import Dict, List, Tuple, Optional, Set
import importlib
import numpy as np

def _load_symbol(path: str):
    mod, _, attr = path.partition(":")
    m = importlib.import_module(mod)
    return getattr(m, attr)

def _write_jsonl(path: Path, rows: List[Dict]) -> None:
    with path.open("w", encoding="utf-8") as f:
        for r in rows:
            f.write(json.dumps(r) + "\n")

def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--program", required=True)
    ap.add_argument("--pg-backend", required=True)
    ap.add_argument("--out-dir", required=True)
    ap.add_argument("--max-modules", type=int, default=200)
    ap.add_argument("--verbose", action="store_true")
    args = ap.parse_args()

    out_dir = Path(args.out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)

    pg_ctor = _load_symbol(args.pg_backend)
    pg = pg_ctor(args.program, ignore=None)

    # Collect entities and a simple owner map
    entities = list(pg.entities())
    modules = [e for e in entities if e.kind == "module"]
    functions = [e for e in entities if e.kind == "function"]
    classes = [e for e in entities if e.kind == "class"]

    # 1) planning_components.jsonl (owners/modules with primary artifact spans)
    comps: List[Dict] = []
    for m in modules[: max(1, int(args.max_modules))]:
        try:
            ra = pg.resolve(m.uri)
            comps.append({
                "owner": (m.name or m.id),
                "kind": "module",
                "artifact_uri": ra.artifact_uri,
                "span": {"start": int(ra.span.start_line), "end": int(ra.span.end_line)},
            })
        except Exception:
            continue
    _write_jsonl(out_dir / "planning_components.jsonl", comps)

    # 2) planning_entrypoints.jsonl (very light heuristic: functions named main or having owner 'cli'/'__main__')
    entries: List[Dict] = []
    for fn in functions:
        name = (fn.name or "").lower()
        own = (fn.owner or "").lower()
        if name == "main" or "__main__" in own or "cli" in own:
            try:
                ra = pg.resolve(fn.uri)
                entries.append({
                    "type": "cli",
                    "name": fn.name,
                    "handler": f"{own}:{fn.name}" if own and fn.name else fn.id,
                    "decl_uri": ra.artifact_uri,
                    "span": {"start": int(ra.span.start_line), "end": int(ra.span.end_line)},
                })
            except Exception:
                continue
    _write_jsonl(out_dir / "planning_entrypoints.jsonl", entries)

    # 3) planning_mutations.jsonl (modules as candidate edit sites with affordances)
    muts: List[Dict] = []
    for m in modules[: max(1, int(args.max_modules))]:
        try:
            ra = pg.resolve(m.uri)
            muts.append({
                "target": (m.name or m.id),
                "kind": "module",
                "artifact_uri": ra.artifact_uri,
                "affordances": ["add_function", "edit_imports", "edit_exports"],
            })
        except Exception:
            continue
    _write_jsonl(out_dir / "planning_mutations.jsonl", muts)

    # 4) planning_tests_map.jsonl (map owners to test files by simple heuristic)
    tests_map: List[Dict] = []
    # harvest artifacts(kind="source") and flag ones with "test" in path
    test_arts: Set[str] = set()
    for art in pg.artifacts("source"):
        p = art.uri.lower()
        if ("test" in p) or ("/tests/" in p):
            test_arts.add(art.uri)
    for m in modules:
        # naive mapping: module owner name in test path
        owner = (m.name or m.id)
        for ta in test_arts:
            if owner.split(".")[-1] in ta:
                tests_map.append({
                    "owner": owner,
                    "test_file": ta,
                    "span": {"start": 1, "end": 1},
                })
                break
    _write_jsonl(out_dir / "planning_tests_map.jsonl", tests_map)

    # 5) planning_dependencies.jsonl (import/call edges at entity granularity)
    deps: List[Dict] = []
    try:
        for e in pg.edges():
            deps.append({"src": e.src, "dst": e.dst, "edge": e.type})
    except Exception:
        deps = []
    _write_jsonl(out_dir / "planning_dependencies.jsonl", deps)

    # 6) planning_rerank_features.npz (very light owner graph centrality and placeholders)
    owners = sorted(list({(m.owner or m.name or m.id) for m in modules}))
    owner_index: Dict[str, int] = {o: i for i, o in enumerate(owners)}
    centrality = np.zeros((len(owners),), dtype=np.float32)
    try:
        owner_edges: Dict[str, Set[str]] = {}
        for e in pg.edges():
            # map entity ids to owners if available
            srco = None
            dsto = None
            # build entity -> owner map lazily
        ent_owner: Dict[str, Optional[str]] = {}
        for ent in entities:
            ent_owner[ent.id] = ent.owner
        for e in pg.edges():
            so = ent_owner.get(e.src)
            do = ent_owner.get(e.dst)
            if so and do:
                owner_edges.setdefault(so, set()).add(do)
                owner_edges.setdefault(do, set()).add(so)
        for o, nbs in owner_edges.items():
            idx = owner_index.get(o)
            if idx is not None:
                centrality[idx] = float(len(nbs))
        if centrality.max() > 0:
            centrality = centrality / centrality.max()
    except Exception:
        centrality = np.zeros((len(owners),), dtype=np.float32)
    registry_score = np.zeros_like(centrality, dtype=np.float32)
    test_coverage = np.zeros_like(centrality, dtype=np.float32)
    np.savez_compressed(
        out_dir / "planning_rerank_features.npz",
        owners=np.array(owners, dtype=object),
        centrality=centrality,
        registry_score=registry_score,
        test_coverage=test_coverage,
    )

    if args.verbose:
        print(json.dumps({
            "components": len(comps),
            "entrypoints": len(entries),
            "mutations": len(muts),
            "tests_map": len(tests_map),
            "dependencies": len(deps),
            "owners": len(owners),
        }, indent=2))

if __name__ == "__main__":
    main()


examples.scripts.embedding

from typing import Any, Dict, List, Optional, Tuple
import os
import re
import numpy as np
from concurrent.futures import ThreadPoolExecutor, as_completed
try:
    import pathspec  # type: ignore
except Exception:  # pragma: no cover
    pathspec = None  # type: ignore

from examples.scripts.code_graph import CodeGraph

def build_repo_embedding(
    repo_root: str,
    *,
    dim: int = EMBED_DIM_DEFAULT,
    seed: int = 0,
    include_text: bool = False,
    text_max_bytes: int = 0,
    max_text_tokens: int = 0,
    text_weight: float = 0.25,
    calls_weight: float = 0.25,
    types_weight: float = 0.20,
    tests_weight: float = 0.15,
    graph_prop_hops: int = 0,
    graph_prop_damp: float = 0.85,
    ignore: Optional[List[str]] = None,
) -> Dict[str, np.ndarray]:
    g = CodeGraph.load_or_build(repo_root, ignore=ignore)

    # Collect lightweight features
    sym_feats: List[Tuple[str, float]] = []
    doc_feats: List[Tuple[str, float]] = []
    mod_feats: List[Tuple[str, float]] = []
    type_feats: List[Tuple[str, float]] = []
    test_counts: Dict[str, int] = {}

    for mod, nodes in g.pytest_nodes_by_module.items():
        test_counts[mod] = len(nodes or [])

    for _fqn, s in g.symbols_by_fqn.items():
        base = f"{s.kind}:{s.name}"
        sym_feats.append((base, 1.0))
        if s.signature:
            # keep in sym for backward-compat
            sym_feats.append((f"sig:{s.signature}", 0.5))
            type_feats.append((f"sig:{s.signature}", 1.0))
        if s.returns:
            sym_feats.append((f"ret:{s.returns}", 0.5))
            type_feats.append((f"ret:{s.returns}", 1.0))
        if s.doc:
            # downweight docs by length
            d = (s.doc or "").strip()
            if d:
                head = d.splitlines()[0][:160]
                doc_feats.append((f"doc:{head}", 0.25))
        mod_feats.append((f"mod:{s.module}", 0.2))

    # Call-graph features (multi-view)
    call_feats: List[Tuple[str, float]] = []
    try:
        for caller, callee in g.calls:
            call_feats.append((f"call:{caller}->{callee}", 1.0))
    except Exception:
        call_feats = []

    # Module import topology (indegree/outdegree proxies) with optional graph propagation
    indeg: Dict[str, int] = {}
    outdeg: Dict[str, int] = {m: len(deps) for m, deps in g.module_imports.items()}
    modules = list(g.modules.keys())
    mod_idx: Dict[str, int] = {m: i for i, m in enumerate(modules)}
    for m, deps in g.module_imports.items():
        for d in deps:
            indeg[d] = indeg.get(d, 0) + 1
    base_vec = np.zeros((len(modules),), dtype=np.float32)
    for m in modules:
        i = mod_idx[m]
        base_vec[i] = float(indeg.get(m, 0) + outdeg.get(m, 0) + test_counts.get(m, 0))
    prop_vec = base_vec.copy()
    if int(graph_prop_hops) > 0:
        # Build sparse adjacency (imports treated as undirected for smoothing)
        neigh: List[List[int]] = [[] for _ in modules]
        for m, deps in g.module_imports.items():
            i = mod_idx[m]
            for d in deps:
                if d in mod_idx:
                    j = mod_idx[d]
                    neigh[i].append(j)
                    neigh[j].append(i)
        vec = prop_vec
        damp = float(graph_prop_damp)
        for _ in range(max(0, int(graph_prop_hops))):
            nxt = np.zeros_like(vec)
            for i, ns in enumerate(neigh):
                if not ns:
                    continue
                s = 0.0
                for j in ns:
                    s += float(vec[j])
                nxt[i] = (1.0 - damp) * float(base_vec[i]) + damp * (s / float(len(ns)))
            vec = nxt
        prop_vec = vec
        # normalize
        nrm = float(np.linalg.norm(prop_vec))
        if nrm > 0:
            prop_vec = prop_vec / nrm
    topo_feats: List[Tuple[str, float]] = []
    for m in modules:
        topo_feats.append((f"indeg:{m}", float(indeg.get(m, 0))))
        topo_feats.append((f"outdeg:{m}", float(outdeg.get(m, 0))))
        if test_counts.get(m):
            topo_feats.append((f"tests:{m}", float(test_counts[m])))
        if int(graph_prop_hops) > 0:
            topo_feats.append((f"prop:{m}", float(prop_vec[mod_idx[m]])))

    z_sym = _feature_hash(sym_feats, dim, seed + HASH_SEEDS[0])
    z_doc = _feature_hash(doc_feats, dim, seed + HASH_SEEDS[1])
    z_mod = _feature_hash(mod_feats, dim, seed + HASH_SEEDS[2])
    z_top = _feature_hash(topo_feats, dim, seed + HASH_SEEDS[3])
    z_types = _feature_hash(type_feats, dim, seed + HASH_SEEDS[5])
    z_calls = _feature_hash(call_feats, dim, seed + HASH_SEEDS[6])

    # Optional: include raw repository text/code hashed into the embedding
    z_text = np.zeros((dim,), dtype=np.float32)
    if include_text and text_max_bytes and text_max_bytes > 0:
        # Streamed sparse accumulator for large repos
        text_acc: Dict[int, float] = {}
        # Prefer CodeGraph's indexed files if available; otherwise walk the repo
        files: List[str] = []
        try:
            files = list(getattr(g, "indexed_files", []) or [])
        except Exception:
            files = []
        if not files:
            # Fallback: collect common source files
            exts = {".py", ".md", ".rst", ".txt", ".json", ".toml", ".yaml", ".yml", ".ini"}
            # Normalize ignore list + .gitignore pathspec
            ignore_list = [os.path.normpath(p) for p in (ignore or [])]
            pspec = None
            try:
                gi = os.path.join(repo_root, ".gitignore")
                if pathspec is not None and os.path.exists(gi):
                    with open(gi, "r", encoding="utf-8", errors="ignore") as fh:
                        lines = [ln.rstrip("\n") for ln in fh]
                    pspec = pathspec.PathSpec.from_lines("gitwildmatch", lines)
            except Exception:
                pspec = None
            def _is_ignored(rel: str) -> bool:
                r = os.path.normpath(rel)
                if pspec is not None:
                    if pspec.match_file(r.replace(os.sep, "/")):
                        return True
                for pat in ignore_list:
                    if r == pat or r.startswith(pat + os.sep):
                        return True
                return False
            for root, dirs, fnames in os.walk(repo_root):
                rel_root = os.path.relpath(root, repo_root)
                if _is_ignored(rel_root):
                    dirs[:] = []
                    continue
                # prune ignored subdirs
                dirs[:] = [d for d in dirs if not _is_ignored(os.path.join(rel_root, d))]
                for fn in fnames:
                    if os.path.splitext(fn)[1].lower() in exts:
                        fp = os.path.join(root, fn)
                        rel_fp = os.path.relpath(fp, repo_root)
                        if _is_ignored(rel_fp):
                            continue
                        files.append(fp)
        # Normalize file paths and pre-filter by extension
        exts_all = {".py", ".md", ".rst", ".txt", ".json", ".toml", ".yaml", ".yml", ".ini"}
        norm_files: List[str] = []
        for f in files:
            p = f if os.path.isabs(f) else os.path.join(repo_root, f)
            if os.path.splitext(p)[1].lower() in exts_all:
                norm_files.append(p)

        # bytes_budget: None means unlimited; token_cap 0 means unlimited
        bytes_budget: Optional[int] = int(text_max_bytes) if int(text_max_bytes) > 0 else None
        token_cap = int(max(0, int(max_text_tokens))) if max_text_tokens is not None else 0
        tokens_emitted = 0

        def _process_file(fp: str) -> Tuple[int, int, Dict[int, float]]:
            try:
                # Per-file cap: avoid reading giant files fully when many are available
                per_cap = int(text_max_bytes)
                with open(fp, "rb") as fh:
                    raw = fh.read(per_cap)
                if b"\x00" in raw:
                    return (0, 0, {})
                n_bytes = len(raw)
                text = raw.decode("utf-8", errors="ignore").lower()
                toks = re.findall(r"[a-zA-Z0-9_]+", text)
                if not toks:
                    return (n_bytes, 0, {})
                n = 3
                stride = 2
                acc_local: Dict[int, float] = {}
                ng_count = 0
                for i in range(0, max(0, len(toks) - n + 1), stride):
                    key = f"text:{' '.join(toks[i:i+n])}"
                    _accumulate_sparse(acc_local, key, float(text_weight), dim, seed + HASH_SEEDS[4])
                    ng_count += 1
                return (n_bytes, ng_count, acc_local)
            except Exception:
                return (0, 0, {})

        # Concurrent read/tokenize with early short-circuit
        with ThreadPoolExecutor(max_workers=min(8, max(2, os.cpu_count() or 8))) as ex:
            futures = [ex.submit(_process_file, fp) for fp in norm_files]
            for fut in as_completed(futures):
                if bytes_budget is not None and bytes_budget <= 0:
                    break
                n_bytes, ng_count, acc_local = fut.result()
                if bytes_budget is not None:
                    bytes_budget -= n_bytes
                    if bytes_budget is not None and bytes_budget <= 0:
                        # still add what we got, then stop
                        pass
                if not acc_local:
                    continue
                tokens_emitted += int(ng_count)
                for k, v in acc_local.items():
                    text_acc[k] = float(text_acc.get(k, 0.0)) + float(v)
                if token_cap and tokens_emitted >= token_cap:
                    break

        if text_acc:
            z_text = _dense_from_sparse(text_acc, dim)

    # Per-family normalize (layer-norm style), then weighted sum
    def _unit(x: np.ndarray) -> np.ndarray:
        n = float(np.linalg.norm(x))
        return (x / n) if n > 0 else x

    sym_w = 1.0
    doc_w = 1.0
    mod_w = 1.0
    top_w = 1.0
    txt_w = float(text_weight)

    z = (
        sym_w * _unit(z_sym)
        + doc_w * _unit(z_doc)
        + mod_w * _unit(z_mod)
        + top_w * _unit(z_top)
        + txt_w * _unit(z_text)
        + float(types_weight) * _unit(z_types)
        + float(calls_weight) * _unit(z_calls)
        + float(tests_weight) * _unit(np.zeros_like(z_sym))  # tests view added below
    )
    norm = float(np.linalg.norm(z))
    if norm > 0:
        z = z / norm

    # Tests view (counts already influence topology; add dedicated view from test nodes)
    z_tests = np.zeros((dim,), dtype=np.float32)
    try:
        test_feats: List[Tuple[str, float]] = []
        for mod, nodes in g.pytest_nodes_by_module.items():
            if nodes:
                test_feats.append((f"tests:{mod}:{len(nodes)}", 1.0))
        if test_feats:
            z_tests = _feature_hash(test_feats, dim, seed + HASH_SEEDS[7])
    except Exception:
        z_tests = np.zeros((dim,), dtype=np.float32)

    # Re-add tests view (unit) to z
    z = z + float(tests_weight) * _unit(z_tests)

    # Sparsity diagnostics (fraction non-zero)
    def _sparse_frac(x: np.ndarray) -> float:
        if x.size == 0:
            return 0.0
        return float((np.count_nonzero(x) / float(x.size)))

    result = {
        "z": z.astype(np.float32),
        "z_sym": z_sym.astype(np.float32),
        "z_doc": z_doc.astype(np.float32),
        "z_mod": z_mod.astype(np.float32),
        "z_top": z_top.astype(np.float32),
        "z_types": z_types.astype(np.float32),
        "z_calls": z_calls.astype(np.float32),
        "z_tests": z_tests.astype(np.float32),
        "sparsity": {
            "z_sym": _sparse_frac(z_sym),
            "z_doc": _sparse_frac(z_doc),
            "z_mod": _sparse_frac(z_mod),
            "z_top": _sparse_frac(z_top),
            "z_text": _sparse_frac(z_text),
            "z_types": _sparse_frac(z_types),
            "z_calls": _sparse_frac(z_calls),
            "z_tests": _sparse_frac(z_tests),
        },
    }
    if include_text:
        result["z_text"] = z_text.astype(np.float32)
    return result


                if bytes_budget is not None and bytes_budget <= 0:
                    break
                n_bytes, ng_count, acc_local = fut.result()
                if bytes_budget is not None:
                    bytes_budget -= n_bytes
                    if bytes_budget is not None and bytes_budget <= 0:
                        # still add what we got, then stop
                        pass
                if not acc_local:
                    continue
                tokens_emitted += int(ng_count)
                for k, v in acc_local.items():
                    text_acc[k] = float(text_acc.get(k, 0.0)) + float(v)
                if token_cap and tokens_emitted >= token_cap:
                    break

        if text_acc:
            z_text = _dense_from_sparse(text_acc, dim)

    # Per-family normalize (layer-norm style), then weighted sum
    def _unit(x: np.ndarray) -> np.ndarray:
        n = float(np.linalg.norm(x))
        return (x / n) if n > 0 else x

    sym_w = 1.0
    doc_w = 1.0
    mod_w = 1.0
    top_w = 1.0
    txt_w = float(text_weight)

    z = (
        sym_w * _unit(z_sym)
        + doc_w * _unit(z_doc)
        + mod_w * _unit(z_mod)
        + top_w * _unit(z_top)
        + txt_w * _unit(z_text)
        + float(types_weight) * _unit(z_types)
        + float(calls_weight) * _unit(z_calls)
        + float(tests_weight) * _unit(np.zeros_like(z_sym))  # tests view added below
    )
    norm = float(np.linalg.norm(z))
    if norm > 0:
        z = z / norm


    norm = float(np.linalg.norm(z))
    if norm > 0:
        z = z / norm

    # Tests view (counts already influence topology; add dedicated view from test nodes)
    z_tests = np.zeros((dim,), dtype=np.float32)
    try:
        test_feats: List[Tuple[str, float]] = []
        for mod, nodes in g.pytest_nodes_by_module.items():
            if nodes:
                test_feats.append((f"tests:{mod}:{len(nodes)}", 1.0))
        if test_feats:
            z_tests = _feature_hash(test_feats, dim, seed + HASH_SEEDS[7])
    except Exception:
        z_tests = np.zeros((dim,), dtype=np.float32)

    # Re-add tests view (unit) to z
    z = z + float(tests_weight) * _unit(z_tests)

    # Sparsity diagnostics (fraction non-zero)
    def _sparse_frac(x: np.ndarray) -> float:
        if x.size == 0:
            return 0.0
        return float((np.count_nonzero(x) / float(x.size)))

    result = {
        "z": z.astype(np.float32),
        "z_sym": z_sym.astype(np.float32),
        "z_doc": z_doc.astype(np.float32),
        "z_mod": z_mod.astype(np.float32),
        "z_top": z_top.astype(np.float32),
        "z_types": z_types.astype(np.float32),
        "z_calls": z_calls.astype(np.float32),
        "z_tests": z_tests.astype(np.float32),
        "sparsity": {
            "z_sym": _sparse_frac(z_sym),
            "z_doc": _sparse_frac(z_doc),
            "z_mod": _sparse_frac(z_mod),
            "z_top": _sparse_frac(z_top),
            "z_text": _sparse_frac(z_text),
            "z_types": _sparse_frac(z_types),
            "z_calls": _sparse_frac(z_calls),
            "z_tests": _sparse_frac(z_tests),
        },


                for fn in fnames:
                    if os.path.splitext(fn)[1].lower() in exts:
                        fp = os.path.join(root, fn)
                        rel_fp = os.path.relpath(fp, repo_root)
                        if _is_ignored(rel_fp):
                            continue
                        files.append(fp)
        # Normalize file paths and pre-filter by extension
        exts_all = {".py", ".md", ".rst", ".txt", ".json", ".toml", ".yaml", ".yml", ".ini"}
        norm_files: List[str] = []
        for f in files:
            p = f if os.path.isabs(f) else os.path.join(repo_root, f)
            if os.path.splitext(p)[1].lower() in exts_all:
                norm_files.append(p)

        # bytes_budget: None means unlimited; token_cap 0 means unlimited
        bytes_budget: Optional[int] = int(text_max_bytes) if int(text_max_bytes) > 0 else None
        token_cap = int(max(0, int(max_text_tokens))) if max_text_tokens is not None else 0
        tokens_emitted = 0

        def _process_file(fp: str) -> Tuple[int, int, Dict[int, float]]:
            try:
                # Per-file cap: avoid reading giant files fully when many are available
                per_cap = int(text_max_bytes)
                with open(fp, "rb") as fh:
                    raw = fh.read(per_cap)
                if b"\x00" in raw:
                    return (0, 0, {})
                n_bytes = len(raw)
                text = raw.decode("utf-8", errors="ignore").lower()
                toks = re.findall(r"[a-zA-Z0-9_]+", text)
                if not toks:
                    return (n_bytes, 0, {})
                n = 3
                stride = 2
                acc_local: Dict[int, float] = {}
                ng_count = 0
                for i in range(0, max(0, len(toks) - n + 1), stride):
                    key = f"text:{' '.join(toks[i:i+n])}"
                    _accumulate_sparse(acc_local, key, float(text_weight), dim, seed + HASH_SEEDS[4])
                    ng_count += 1
                return (n_bytes, ng_count, acc_local)
            except Exception:
                return (0, 0, {})

        # Concurrent read/tokenize with early short-circuit
        with ThreadPoolExecutor(max_workers=min(8, max(2, os.cpu_count() or 8))) as ex:
            futures = [ex.submit(_process_file, fp) for fp in norm_files]
            for fut in as_completed(futures):
                if bytes_budget is not None and bytes_budget <= 0:
                    break
                n_bytes, ng_count, acc_local = fut.result()
                if bytes_budget is not None:
                    bytes_budget -= n_bytes
                    if bytes_budget is not None and bytes_budget <= 0:
                        # still add what we got, then stop
                        pass
                if not acc_local:
                    continue
                tokens_emitted += int(ng_count)
                for k, v in acc_local.items():
                    text_acc[k] = float(text_acc.get(k, 0.0)) + float(v)
                if token_cap and tokens_emitted >= token_cap:
                    break


        # Prefer CodeGraph's indexed files if available; otherwise walk the repo
        files: List[str] = []
        try:
            files = list(getattr(g, "indexed_files", []) or [])
        except Exception:
            files = []
        if not files:
            # Fallback: collect common source files
            exts = {".py", ".md", ".rst", ".txt", ".json", ".toml", ".yaml", ".yml", ".ini"}
            # Normalize ignore list + .gitignore pathspec
            ignore_list = [os.path.normpath(p) for p in (ignore or [])]
            pspec = None
            try:
                gi = os.path.join(repo_root, ".gitignore")
                if pathspec is not None and os.path.exists(gi):
                    with open(gi, "r", encoding="utf-8", errors="ignore") as fh:
                        lines = [ln.rstrip("\n") for ln in fh]
                    pspec = pathspec.PathSpec.from_lines("gitwildmatch", lines)
            except Exception:
                pspec = None
            def _is_ignored(rel: str) -> bool:
                r = os.path.normpath(rel)
                if pspec is not None:
                    if pspec.match_file(r.replace(os.sep, "/")):
                        return True
                for pat in ignore_list:
                    if r == pat or r.startswith(pat + os.sep):
                        return True
                return False
            for root, dirs, fnames in os.walk(repo_root):
                rel_root = os.path.relpath(root, repo_root)
                if _is_ignored(rel_root):
                    dirs[:] = []
                    continue
                # prune ignored subdirs
                dirs[:] = [d for d in dirs if not _is_ignored(os.path.join(rel_root, d))]
                for fn in fnames:
                    if os.path.splitext(fn)[1].lower() in exts:
                        fp = os.path.join(root, fn)
                        rel_fp = os.path.relpath(fp, repo_root)
                        if _is_ignored(rel_fp):
                            continue
                        files.append(fp)
        # Normalize file paths and pre-filter by extension
        exts_all = {".py", ".md", ".rst", ".txt", ".json", ".toml", ".yaml", ".yml", ".ini"}
        norm_files: List[str] = []
        for f in files:
            p = f if os.path.isabs(f) else os.path.join(repo_root, f)
            if os.path.splitext(p)[1].lower() in exts_all:


from __future__ import annotations

from typing import List, Tuple

def load_mbpp_texts(max_n: int = 128, split: str = "train") -> Tuple[List[str], int]:
	"""
	Load MBPP via Hugging Face datasets and return a list of training texts.
	Each text is prompt + solution combined to form a code-aware LM sample.
	Returns (texts, total_available).
	"""
	try:
		from datasets import load_dataset  # type: ignore
	except Exception:
		return [], 0
	# Try common configs in order of availability
	ds = None
	for name in ("mbpp",):
		for subset in ("sanitized", None):
			try:
				if subset is None:
					ds = load_dataset(name, split=split)
				else:
					ds = load_dataset(name, subset, split=split)
				break
			except Exception:
				continue
		if ds is not None:
			break
	if ds is None:
		return [], 0
	texts: List[str] = []
	total = len(ds)
	for i in range(total):
		row = ds[i]
		# MBPP variants have fields like: "text" or "prompt"/"question", "code" (solution), "test_list"
		prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
		code = str(row.get("code") or row.get("solution") or "").strip()
		if not prompt and not code:
			continue
		# Simple concatenation; training LM on both description and solution
		combined = (prompt + "\n" + code).strip()
		if combined:
			texts.append(combined)
		if len(texts) >= int(max_n):
			break
	return texts, total

def load_mbpp_texts_all_splits(max_n: int = 1_000_000) -> Tuple[List[str], int]:
	"""
	Load MBPP across all available splits (e.g., train/validation/test) and concatenate.
	Returns (texts, total_available_across_splits).
	"""
	try:
		from datasets import load_dataset  # type: ignore
	except Exception:
		return [], 0
	ds_dict = None
	for name in ("mbpp",):
		for subset in ("sanitized", None):
			try:
				if subset is None:
					ds_dict = load_dataset(name)  # DatasetDict
				else:
					ds_dict = load_dataset(name, subset)  # DatasetDict
				break
			except Exception:
				continue
		if ds_dict is not None:
			break
	if ds_dict is None:
		return [], 0
	texts: List[str] = []
	total = 0
	# Iterate splits in a stable order
	for split_name in ("train", "validation", "test"):
		if split_name not in ds_dict:
			continue
		ds = ds_dict[split_name]
		sz = len(ds)
		total += sz
		for i in range(sz):
			row = ds[i]
			prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
			code = str(row.get("code") or row.get("solution") or "").strip()
			if not prompt and not code:
				continue
			combined = (prompt + "\n" + code).strip()
			if combined:
				texts.append(combined)
			if len(texts) >= int(max_n):
				return texts, total
	return texts, total


from __future__ import annotations

from typing import List, Tuple

def load_mbpp_texts(max_n: int = 128, split: str = "train") -> Tuple[List[str], int]:
	"""
	Load MBPP via Hugging Face datasets and return a list of training texts.
	Each text is prompt + solution combined to form a code-aware LM sample.
	Returns (texts, total_available).
	"""
	try:
		from datasets import load_dataset  # type: ignore
	except Exception:
		return [], 0
	# Try common configs in order of availability
	ds = None
	for name in ("mbpp",):
		for subset in ("sanitized", None):
			try:
				if subset is None:
					ds = load_dataset(name, split=split)
				else:
					ds = load_dataset(name, subset, split=split)
				break
			except Exception:
				continue
		if ds is not None:
			break
	if ds is None:
		return [], 0
	texts: List[str] = []
	total = len(ds)
	for i in range(total):
		row = ds[i]
		# MBPP variants have fields like: "text" or "prompt"/"question", "code" (solution), "test_list"
		prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
		code = str(row.get("code") or row.get("solution") or "").strip()
		if not prompt and not code:
			continue
		# Simple concatenation; training LM on both description and solution
		combined = (prompt + "\n" + code).strip()
		if combined:
			texts.append(combined)
		if len(texts) >= int(max_n):
			break
	return texts, total

def load_mbpp_texts_all_splits(max_n: int = 1_000_000) -> Tuple[List[str], int]:
	"""
	Load MBPP across all available splits (e.g., train/validation/test) and concatenate.
	Returns (texts, total_available_across_splits).
	"""
	try:
		from datasets import load_dataset  # type: ignore
	except Exception:
		return [], 0
	ds_dict = None
	for name in ("mbpp",):
		for subset in ("sanitized", None):
			try:
				if subset is None:
					ds_dict = load_dataset(name)  # DatasetDict
				else:
					ds_dict = load_dataset(name, subset)  # DatasetDict
				break


	if ds is None:
		return [], 0
	texts: List[str] = []
	total = len(ds)
	for i in range(total):
		row = ds[i]
		# MBPP variants have fields like: "text" or "prompt"/"question", "code" (solution), "test_list"
		prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
		code = str(row.get("code") or row.get("solution") or "").strip()
		if not prompt and not code:
			continue
		# Simple concatenation; training LM on both description and solution
		combined = (prompt + "\n" + code).strip()
		if combined:
			texts.append(combined)
		if len(texts) >= int(max_n):
			break
	return texts, total

def load_mbpp_texts_all_splits(max_n: int = 1_000_000) -> Tuple[List[str], int]:
	"""
	Load MBPP across all available splits (e.g., train/validation/test) and concatenate.
	Returns (texts, total_available_across_splits).
	"""
	try:
		from datasets import load_dataset  # type: ignore
	except Exception:
		return [], 0
	ds_dict = None
	for name in ("mbpp",):
		for subset in ("sanitized", None):
			try:
				if subset is None:
					ds_dict = load_dataset(name)  # DatasetDict
				else:
					ds_dict = load_dataset(name, subset)  # DatasetDict
				break
			except Exception:
				continue
		if ds_dict is not None:
			break
	if ds_dict is None:
		return [], 0
	texts: List[str] = []
	total = 0
	# Iterate splits in a stable order
	for split_name in ("train", "validation", "test"):
		if split_name not in ds_dict:
			continue
		ds = ds_dict[split_name]
		sz = len(ds)
		total += sz
		for i in range(sz):
			row = ds[i]
			prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
			code = str(row.get("code") or row.get("solution") or "").strip()
			if not prompt and not code:
				continue
			combined = (prompt + "\n" + code).strip()
			if combined:
				texts.append(combined)
			if len(texts) >= int(max_n):
				return texts, total
	return texts, total


from __future__ import annotations

import os
import ast
import hashlib
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple, Iterable, Set

@dataclass(frozen=True)
class FileSpan:
    file: str           # absolute path
    start_line: int     # 1-based inclusive
    end_line: int       # 1-based inclusive

@dataclass
class CGEntity:
    id: str             # stable id (e.g., fqn)
    kind: str           # module|function|class|test
    name: str
    file: str           # absolute path
    owner: Optional[str]
    start_line: int
    end_line: int

@dataclass
class CGEdge:
    src: str            # CGEntity.id
    dst: str            # CGEntity.id
    type: str           # imports|calls|owns|tests

class CodeGraph:
    def __init__(self, repo_root: str, ignore: Optional[List[str]] = None):
        self.root = os.path.abspath(repo_root)
        self.ignore_rules = [s for s in (ignore or []) if s]
        self.entities_by_id: Dict[str, CGEntity] = {}
        self.edges_list: List[CGEdge] = []
        self._file_hash: Dict[str, str] = {}
        self._id_by_module: Dict[str, str] = {}
        self._ids_by_file: Dict[str, List[str]] = {}
        self._index_identifiers: Dict[str, List[str]] = {}

    # Build
    def build(self) -> "CodeGraph":
        py_files = self._discover_py_files(self.root, self.ignore_rules)
        for abs_fp in py_files:
            mod = self._module_name_for(abs_fp)
            mid = f"py:{mod}"
            self._id_by_module[mod] = mid
            ent = CGEntity(
                id=mid, kind="module", name=mod, file=abs_fp, owner=None,
                start_line=1, end_line=self._safe_count_lines(abs_fp),
            )
            self.entities_by_id[mid] = ent
            self._ids_by_file.setdefault(abs_fp, []).append(mid)
            self._index_identifiers.setdefault(mod.lower(), []).append(mid)
            # Parse AST for defs/imports/calls
            try:
                with open(abs_fp, "r", encoding="utf-8", errors="ignore") as fh:
                    src = fh.read()
                tree = ast.parse(src)
            except Exception:
                tree = None
            if tree is None:
                continue
            # functions/classes
            for node in ast.walk(tree):
                if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
                    name = getattr(node, "name", "")
                    a = int(getattr(node, "lineno", 1))
                    b = int(getattr(node, "end_lineno", a))
                    fid = f"py:{mod}.{name}"
                    self.entities_by_id[fid] = CGEntity(
                        id=fid, kind="function", name=name, file=abs_fp, owner=mid,
                        start_line=a, end_line=b,
                    )
                    self._ids_by_file.setdefault(abs_fp, []).append(fid)
                    self.edges_list.append(CGEdge(src=mid, dst=fid, type="owns"))
                    self._index_identifiers.setdefault(name.lower(), []).append(fid)
                elif isinstance(node, ast.ClassDef):
                    name = getattr(node, "name", "")
                    a = int(getattr(node, "lineno", 1))
                    b = int(getattr(node, "end_lineno", a))
                    cid = f"py:{mod}.{name}"
                    self.entities_by_id[cid] = CGEntity(
                        id=cid, kind="class", name=name, file=abs_fp, owner=mid,
                        start_line=a, end_line=b,
                    )
                    self._ids_by_file.setdefault(abs_fp, []).append(cid)
                    self.edges_list.append(CGEdge(src=mid, dst=cid, type="owns"))
                    self._index_identifiers.setdefault(name.lower(), []).append(cid)
            # imports (module-level)
            try:
                for node in ast.walk(tree):
                    if isinstance(node, ast.Import):
                        for alias in node.names:
                            im = alias.name
                            if not im:
                                continue
                            tgt_mod = im
                            tid = f"py:{tgt_mod}"
                            self.edges_list.append(CGEdge(src=mid, dst=tid, type="imports"))
                    elif isinstance(node, ast.ImportFrom):
                        im = node.module or ""
                        if not im:
                            continue
                        tgt_mod = im
                        tid = f"py:{tgt_mod}"
                        self.edges_list.append(CGEdge(src=mid, dst=tid, type="imports"))
            except Exception:
                pass
            # calls (best-effort): record identifiers used in Call nodes
            try:
                for node in ast.walk(tree):
                    if isinstance(node, ast.Call):
                        fn = getattr(node, "func", None)
                        name = None
                        if isinstance(fn, ast.Attribute):
                            name = getattr(fn, "attr", None)
                        elif isinstance(fn, ast.Name):
                            name = fn.id
                        if name:
                            lid = str(name).lower()
                            for cand in self._index_identifiers.get(lid, []):
                                self.edges_list.append(CGEdge(src=mid, dst=cand, type="calls"))
            except Exception:
                pass
            # tests tag
            base = os.path.basename(abs_fp)
            if base.startswith("test_") or base.endswith("_test.py"):
                self.entities_by_id[mid].kind = "test_module"
        # finalize file hashes
        self._precompute_hashes(py_files)
        return self

    # Public accessors
    def entities(self) -> Iterable[CGEntity]:
        return self.entities_by_id.values()

    def edges(self) -> Iterable[CGEdge]:
        # Filter edges whose endpoints are known (post totality)
        known = set(self.entities_by_id.keys())
        for e in self.edges_list:
            if (e.src in known) and (e.dst in known):
                yield e

    def file_hash(self, abs_path: str) -> str:
        return self._file_hash.get(abs_path) or ""

    def ids_for_file(self, abs_path: str) -> List[str]:
        return list(self._ids_by_file.get(abs_path, []))

    def find_identifier_ids(self, token: str) -> List[str]:
        return list(self._index_identifiers.get(token.lower(), []))

    # Helpers
    def _discover_py_files(self, root: str, ignore: List[str]) -> List[str]:
        out: List[str] = []
        for dirpath, dirnames, filenames in os.walk(root):
            # naive ignore: drop segments that contain any ignore pattern
            if any(ig in dirpath for ig in ignore):
                continue
            for fn in filenames:
                if not fn.endswith(".py"):
                    continue
                ap = os.path.abspath(os.path.join(dirpath, fn))
                out.append(ap)
        return out

    def _module_name_for(self, abs_file: str) -> str:
        # repo-relative without extension, path with dots
        rel = os.path.relpath(abs_file, self.root).replace("\\", "/")
        if rel.endswith(".py"):
            rel = rel[:-3]
        parts = [p for p in rel.split("/") if p and p != "__init__"]
        return ".".join(parts) or os.path.splitext(os.path.basename(abs_file))[0]

    def _safe_count_lines(self, abs_file: str) -> int:
        try:
            with open(abs_file, "r", encoding="utf-8", errors="ignore") as fh:
                return sum(1 for _ in fh)
        except Exception:
            return 1

    def _precompute_hashes(self, files: List[str]) -> None:
        for fp in files:
            try:
                with open(fp, "rb") as fh:
                    raw = fh.read()
                h = hashlib.sha256(raw).hexdigest()
            except Exception:
                h = ""
            self._file_hash[fp] = h


FileSpan

from __future__ import annotations

import os
import ast
import hashlib
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple, Iterable, Set

@dataclass(frozen=True)
class FileSpan:
    file: str           # absolute path
    start_line: int     # 1-based inclusive
    end_line: int       # 1-based inclusive

@dataclass
class CGEntity:
    id: str             # stable id (e.g., fqn)
    kind: str           # module|function|class|test
    name: str
    file: str           # absolute path
    owner: Optional[str]
    start_line: int
    end_line: int

@dataclass
class CGEdge:
    src: str            # CGEntity.id
    dst: str            # CGEntity.id
    type: str           # imports|calls|owns|tests


CGEntity

from __future__ import annotations

import os
import ast
import hashlib
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple, Iterable, Set

@dataclass(frozen=True)
class FileSpan:
    file: str           # absolute path
    start_line: int     # 1-based inclusive
    end_line: int       # 1-based inclusive

@dataclass
class CGEntity:
    id: str             # stable id (e.g., fqn)
    kind: str           # module|function|class|test
    name: str
    file: str           # absolute path
    owner: Optional[str]
    start_line: int
    end_line: int

@dataclass
class CGEdge:
    src: str            # CGEntity.id
    dst: str            # CGEntity.id
    type: str           # imports|calls|owns|tests

class CodeGraph:
    def __init__(self, repo_root: str, ignore: Optional[List[str]] = None):
        self.root = os.path.abspath(repo_root)
        self.ignore_rules = [s for s in (ignore or []) if s]
        self.entities_by_id: Dict[str, CGEntity] = {}
        self.edges_list: List[CGEdge] = []
        self._file_hash: Dict[str, str] = {}
        self._id_by_module: Dict[str, str] = {}
        self._ids_by_file: Dict[str, List[str]] = {}
        self._index_identifiers: Dict[str, List[str]] = {}


CGEdge

@dataclass(frozen=True)
class FileSpan:
    file: str           # absolute path
    start_line: int     # 1-based inclusive
    end_line: int       # 1-based inclusive

@dataclass
class CGEntity:
    id: str             # stable id (e.g., fqn)
    kind: str           # module|function|class|test
    name: str
    file: str           # absolute path
    owner: Optional[str]
    start_line: int
    end_line: int

@dataclass
class CGEdge:
    src: str            # CGEntity.id
    dst: str            # CGEntity.id
    type: str           # imports|calls|owns|tests

class CodeGraph:
    def __init__(self, repo_root: str, ignore: Optional[List[str]] = None):
        self.root = os.path.abspath(repo_root)
        self.ignore_rules = [s for s in (ignore or []) if s]
        self.entities_by_id: Dict[str, CGEntity] = {}
        self.edges_list: List[CGEdge] = []
        self._file_hash: Dict[str, str] = {}
        self._id_by_module: Dict[str, str] = {}
        self._ids_by_file: Dict[str, List[str]] = {}
        self._index_identifiers: Dict[str, List[str]] = {}

    # Build
    def build(self) -> "CodeGraph":
        py_files = self._discover_py_files(self.root, self.ignore_rules)
        for abs_fp in py_files:
            mod = self._module_name_for(abs_fp)
            mid = f"py:{mod}"
            self._id_by_module[mod] = mid


CodeGraph

@dataclass
class CGEntity:
    id: str             # stable id (e.g., fqn)
    kind: str           # module|function|class|test
    name: str
    file: str           # absolute path
    owner: Optional[str]
    start_line: int
    end_line: int

@dataclass
class CGEdge:
    src: str            # CGEntity.id
    dst: str            # CGEntity.id
    type: str           # imports|calls|owns|tests

class CodeGraph:
    def __init__(self, repo_root: str, ignore: Optional[List[str]] = None):
        self.root = os.path.abspath(repo_root)
        self.ignore_rules = [s for s in (ignore or []) if s]
        self.entities_by_id: Dict[str, CGEntity] = {}
        self.edges_list: List[CGEdge] = []
        self._file_hash: Dict[str, str] = {}
        self._id_by_module: Dict[str, str] = {}
        self._ids_by_file: Dict[str, List[str]] = {}
        self._index_identifiers: Dict[str, List[str]] = {}

    # Build
    def build(self) -> "CodeGraph":
        py_files = self._discover_py_files(self.root, self.ignore_rules)
        for abs_fp in py_files:
            mod = self._module_name_for(abs_fp)
            mid = f"py:{mod}"
            self._id_by_module[mod] = mid
            ent = CGEntity(
                id=mid, kind="module", name=mod, file=abs_fp, owner=None,
                start_line=1, end_line=self._safe_count_lines(abs_fp),
            )
            self.entities_by_id[mid] = ent
            self._ids_by_file.setdefault(abs_fp, []).append(mid)
            self._index_identifiers.setdefault(mod.lower(), []).append(mid)
            # Parse AST for defs/imports/calls
            try:
                with open(abs_fp, "r", encoding="utf-8", errors="ignore") as fh:
                    src = fh.read()
                tree = ast.parse(src)
            except Exception:
                tree = None
            if tree is None:
                continue
            # functions/classes
            for node in ast.walk(tree):
                if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
                    name = getattr(node, "name", "")
                    a = int(getattr(node, "lineno", 1))
                    b = int(getattr(node, "end_lineno", a))
                    fid = f"py:{mod}.{name}"
                    self.entities_by_id[fid] = CGEntity(
                        id=fid, kind="function", name=name, file=abs_fp, owner=mid,
                        start_line=a, end_line=b,
                    )
                    self._ids_by_file.setdefault(abs_fp, []).append(fid)
                    self.edges_list.append(CGEdge(src=mid, dst=fid, type="owns"))
                    self._index_identifiers.setdefault(name.lower(), []).append(fid)
                elif isinstance(node, ast.ClassDef):
                    name = getattr(node, "name", "")
                    a = int(getattr(node, "lineno", 1))
                    b = int(getattr(node, "end_lineno", a))
                    cid = f"py:{mod}.{name}"
                    self.entities_by_id[cid] = CGEntity(
                        id=cid, kind="class", name=name, file=abs_fp, owner=mid,
                        start_line=a, end_line=b,
                    )
                    self._ids_by_file.setdefault(abs_fp, []).append(cid)
                    self.edges_list.append(CGEdge(src=mid, dst=cid, type="owns"))
                    self._index_identifiers.setdefault(name.lower(), []).append(cid)
            # imports (module-level)
            try:
                for node in ast.walk(tree):
                    if isinstance(node, ast.Import):
                        for alias in node.names:
                            im = alias.name
                            if not im:
                                continue
                            tgt_mod = im
                            tid = f"py:{tgt_mod}"
                            self.edges_list.append(CGEdge(src=mid, dst=tid, type="imports"))
                    elif isinstance(node, ast.ImportFrom):
                        im = node.module or ""
                        if not im:
                            continue
                        tgt_mod = im
                        tid = f"py:{tgt_mod}"
                        self.edges_list.append(CGEdge(src=mid, dst=tid, type="imports"))
            except Exception:
                pass
            # calls (best-effort): record identifiers used in Call nodes
            try:
                for node in ast.walk(tree):
                    if isinstance(node, ast.Call):
                        fn = getattr(node, "func", None)
                        name = None
                        if isinstance(fn, ast.Attribute):
                            name = getattr(fn, "attr", None)
                        elif isinstance(fn, ast.Name):
                            name = fn.id
                        if name:
                            lid = str(name).lower()
                            for cand in self._index_identifiers.get(lid, []):
                                self.edges_list.append(CGEdge(src=mid, dst=cand, type="calls"))
            except Exception:
                pass
            # tests tag
            base = os.path.basename(abs_fp)
            if base.startswith("test_") or base.endswith("_test.py"):
                self.entities_by_id[mid].kind = "test_module"
        # finalize file hashes
        self._precompute_hashes(py_files)
        return self

    # Public accessors
    def entities(self) -> Iterable[CGEntity]:
        return self.entities_by_id.values()

    def edges(self) -> Iterable[CGEdge]:
        # Filter edges whose endpoints are known (post totality)
        known = set(self.entities_by_id.keys())
        for e in self.edges_list:
            if (e.src in known) and (e.dst in known):
                yield e

    def file_hash(self, abs_path: str) -> str:
        return self._file_hash.get(abs_path) or ""

    def ids_for_file(self, abs_path: str) -> List[str]:
        return list(self._ids_by_file.get(abs_path, []))

    def find_identifier_ids(self, token: str) -> List[str]:
        return list(self._index_identifiers.get(token.lower(), []))

    # Helpers
    def _discover_py_files(self, root: str, ignore: List[str]) -> List[str]:
        out: List[str] = []
        for dirpath, dirnames, filenames in os.walk(root):
            # naive ignore: drop segments that contain any ignore pattern
            if any(ig in dirpath for ig in ignore):
                continue
            for fn in filenames:
                if not fn.endswith(".py"):
                    continue
                ap = os.path.abspath(os.path.join(dirpath, fn))
                out.append(ap)
        return out

    def _module_name_for(self, abs_file: str) -> str:
        # repo-relative without extension, path with dots
        rel = os.path.relpath(abs_file, self.root).replace("\\", "/")
        if rel.endswith(".py"):
            rel = rel[:-3]
        parts = [p for p in rel.split("/") if p and p != "__init__"]
        return ".".join(parts) or os.path.splitext(os.path.basename(abs_file))[0]

    def _safe_count_lines(self, abs_file: str) -> int:
        try:
            with open(abs_file, "r", encoding="utf-8", errors="ignore") as fh:
                return sum(1 for _ in fh)
        except Exception:
            return 1

    def _precompute_hashes(self, files: List[str]) -> None:
        for fp in files:
            try:
                with open(fp, "rb") as fh:
                    raw = fh.read()
                h = hashlib.sha256(raw).hexdigest()
            except Exception:
                h = ""
            self._file_hash[fp] = h