main

from pathlib import Path
from typing import Dict, Any, List, Tuple

import torch
import torch.nn.functional as F
from torch.utils.data import DataLoader

@dataclass
class EvalPaths:
	adapters_dir: Path
	out_dir: Path
	state_path: Path

def _ensure_dirs(paths: EvalPaths) -> None:
	paths.adapters_dir.mkdir(parents=True, exist_ok=True)
	paths.out_dir.mkdir(parents=True, exist_ok=True)

def main() -> None:
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "datasets" / "hf_lora_mbpp"
	paths = EvalPaths(
		adapters_dir=artifacts_root / "lora_adapter",
		out_dir=artifacts_root / "outputs",
		state_path=artifacts_root / ".program_state.json",
	)
	_ensure_dirs(paths)

	# If not specified, prefer GPUs 0,1
	if not os.getenv("CUDA_VISIBLE_DEVICES"):
		os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"

	model_id = os.getenv("LLAMA_MODEL_ID", "meta-llama/Llama-3.1-8B-Instruct")
	adapter_path = (os.getenv("ADAPTER_PATH") or str(paths.adapters_dir)).strip()
	context_len = int(os.getenv("MAX_LEN", "256"))
	per_device_batch = int(os.getenv("BATCH_SIZE", "1"))
	grad_accum = int(os.getenv("GRAD_ACCUM_STEPS", "8"))
	logging_steps = int(os.getenv("LOG_STEPS", "25"))
	seed = int(os.getenv("SEED", "0"))
	use_qlora = os.getenv("USE_QLORA", "1") == "1"

	torch.manual_seed(seed)
	if torch.cuda.is_available():
		torch.cuda.manual_seed_all(seed)

	from transformers import AutoTokenizer, AutoModelForCausalLM, DataCollatorForLanguageModeling  # type: ignore
	try:
		from peft import get_peft_model, prepare_model_for_kbit_training, PeftModel  # type: ignore
	except Exception as e:
		print(f"[eval] PEFT not available: {e}")
		print("Install 'peft' if you want to evaluate LoRA adapters.")
		PeftModel = None  # type: ignore

	tok = AutoTokenizer.from_pretrained(model_id)
	if tok.pad_token_id is None and tok.eos_token is not None:
		tok.pad_token = tok.eos_token
	pad_id = int(tok.pad_token_id or 0)

	print(f"[eval] loading base model {model_id}")
	model_kwargs: Dict[str, Any] = {}
	visible = os.environ.get("CUDA_VISIBLE_DEVICES", "")
	num_visible = len([v for v in visible.split(",") if v.strip()]) if visible else torch.cuda.device_count()
	if use_qlora:
		try:
			from transformers import BitsAndBytesConfig  # type: ignore
			bnb_config = BitsAndBytesConfig(
				load_in_4bit=True,
				bnb_4bit_quant_type="nf4",
				bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
			)
			model_kwargs["quantization_config"] = bnb_config
			model_kwargs["device_map"] = "auto" if num_visible > 1 else None
		except Exception as e:
			print(f"[eval] BitsAndBytes not available, evaluating in full precision: {e}")
			model_kwargs["device_map"] = None
			model_kwargs["torch_dtype"] = torch.bfloat16 if torch.cuda.is_available() else torch.float32
	else:
		model_kwargs["device_map"] = "auto" if num_visible > 1 else None
		model_kwargs["torch_dtype"] = torch.bfloat16 if torch.cuda.is_available() else torch.float32

	device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
	base_model = AutoModelForCausalLM.from_pretrained(model_id, **model_kwargs)
	try:
		base_model.config.use_cache = False
	except Exception:
		pass
	device_map_used = model_kwargs.get("device_map", None) is not None
	if not device_map_used:
		base_model = base_model.to(device)
	if use_qlora:
		try:
			base_model = prepare_model_for_kbit_training(base_model)  # type: ignore
		except Exception:
			pass
	base_model.eval()

	@torch.no_grad()
	def evaluate_model(model: Any, tokenized_ds: Any, pad_token_id: int) -> Dict[str, float]:
		loader = DataLoader(tokenized_ds, batch_size=per_device_batch, shuffle=False)
		total_loss = 0.0
		num_batches = 0
		for batch in loader:
			labels = batch.get("labels", None)
			if labels is None:
				# create labels equal to input_ids to do next-token prediction
				labels = batch["input_ids"].clone()
				batch["labels"] = labels
			# Move to device only when not using a sharded device_map
			if not device_map_used:
				for k, v in list(batch.items()):
					if torch.is_tensor(v):
						batch[k] = v.to(device)
			inp = {k: v for k, v in batch.items() if k != "labels"}
			outputs = model(**inp)
			logits = outputs.get("logits")
			vocab = logits.size(-1)
			shift_logits = logits[..., :-1, :].contiguous()
			shift_labels = batch["labels"][..., 1:].contiguous().to(shift_logits.device)
			shift_labels = shift_labels.masked_fill(shift_labels == int(pad_token_id), -100)
			loss = F.cross_entropy(
				shift_logits.view(-1, vocab),
				shift_labels.view(-1),
				ignore_index=-100,
			)
			total_loss += float(loss.item())
			num_batches += 1
		avg_loss = total_loss / max(1, num_batches)
		try:
			ppl = float(torch.exp(torch.tensor(avg_loss)).item())
		except Exception:
			ppl = float("inf")
		return {"loss": avg_loss, "perplexity": ppl}

	# Data util
	from datasets import Dataset as HFDataset  # type: ignore
	def make_dataset(texts: List[str]) -> Any:
		raw = HFDataset.from_dict({"text": texts})
		def _tok_fn(row: Dict[str, Any]) -> Dict[str, Any]:
			out = tok(row["text"], truncation=True, max_length=context_len, padding="max_length")
			return out
		return raw.map(_tok_fn, batched=False, remove_columns=["text"])

	# Collator
	collator = DataCollatorForLanguageModeling(tokenizer=tok, mlm=False)

	# Resolve datasets one-by-one (sequential, full by default)
	datasets_cfg = os.getenv("DATASETS_CONFIG", "")
	train_new_only = False  # eval should not skip previously seen
	iter_program_texts = None
	try:
		from examples.scripts.datasets_loader import iter_program_texts as _ipt  # type: ignore
		iter_program_texts = _ipt
	except Exception:
		try:
			import importlib.util as _imputil  # type: ignore
			_loader_path = (example_dir.parent / "scripts" / "datasets_loader.py").resolve()
			_spec = _imputil.spec_from_file_location("datasets_loader", str(_loader_path))
			if _spec and _spec.loader:
				_mod = _imputil.module_from_spec(_spec)
				_spec.loader.exec_module(_mod)  # type: ignore
				iter_program_texts = getattr(_mod, "iter_program_texts", None)  # type: ignore
		except Exception:
			iter_program_texts = None
	if iter_program_texts is None:
		print("[eval] could not resolve datasets loader")
		sys.exit(2)

	results: Dict[str, Dict[str, Any]] = {}

	# Evaluate baseline first
	for texts, src_name in iter_program_texts(
		example_dir=str(example_dir),
		config_path=datasets_cfg or None,
		train_new_only=train_new_only,
		state_path=str(paths.state_path),
	):
		if not texts:
			continue
		ds = make_dataset(texts)
		ds = ds.with_format("torch")
		print(f"[eval] baseline on {src_name} ({len(texts)} samples)")
		metrics = evaluate_model(base_model, ds, pad_id)
		results[src_name] = results.get(src_name, {})
		results[src_name]["baseline"] = {
			"loss": float(metrics.get("loss", 0.0)),
			"perplexity": float(metrics.get("perplexity", 0.0)),
		}

	# Attach LoRA and evaluate finetuned
	model_ft = base_model
	if PeftModel is not None and adapter_path and Path(adapter_path).exists():
		print(f"[eval] loading adapter from {adapter_path}")
		model_ft = PeftModel.from_pretrained(base_model, adapter_path, is_trainable=False)  # type: ignore
	else:
		print(f"[eval] adapter not found or PEFT missing; skipping finetuned eval: {adapter_path}")
	model_ft.eval()
	for texts, src_name in iter_program_texts(
		example_dir=str(example_dir),
		config_path=datasets_cfg or None,
		train_new_only=train_new_only,
		state_path=str(paths.state_path),
	):
		if not texts:
			continue
		ds = make_dataset(texts)
		ds = ds.with_format("torch")
		print(f"[eval] finetuned on {src_name} ({len(texts)} samples)")
		metrics = evaluate_model(model_ft, ds, pad_id)
		results[src_name] = results.get(src_name, {})
		results[src_name]["finetuned"] = {
			"loss": float(metrics.get("loss", 0.0)),
			"perplexity": float(metrics.get("perplexity", 0.0)),
		}

	# Save combined report
	report = {
		"model_id": model_id,
		"adapter_path": str(adapter_path),
		"results": results,
	}
	with open(paths.out_dir / "EvalComparison.json", "w", encoding="utf-8") as fh:
		fh.write(json.dumps({"schema_version": 1, "EvalComparison": report}, indent=2))
	print(str(paths.out_dir / "EvalComparison.json"))

if __name__ == "__main__":
	main()


		model_kwargs["device_map"] = "auto" if num_visible > 1 else None
		model_kwargs["torch_dtype"] = torch.bfloat16 if torch.cuda.is_available() else torch.float32

	device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
	base_model = AutoModelForCausalLM.from_pretrained(model_id, **model_kwargs)
	try:
		base_model.config.use_cache = False
	except Exception:
		pass
	device_map_used = model_kwargs.get("device_map", None) is not None
	if not device_map_used:
		base_model = base_model.to(device)
	if use_qlora:
		try:
			base_model = prepare_model_for_kbit_training(base_model)  # type: ignore
		except Exception:
			pass
	base_model.eval()

	@torch.no_grad()
	def evaluate_model(model: Any, tokenized_ds: Any, pad_token_id: int) -> Dict[str, float]:
		loader = DataLoader(tokenized_ds, batch_size=per_device_batch, shuffle=False)
		total_loss = 0.0
		num_batches = 0
		for batch in loader:
			labels = batch.get("labels", None)
			if labels is None:
				# create labels equal to input_ids to do next-token prediction
				labels = batch["input_ids"].clone()
				batch["labels"] = labels
			# Move to device only when not using a sharded device_map
			if not device_map_used:
				for k, v in list(batch.items()):
					if torch.is_tensor(v):
						batch[k] = v.to(device)
			inp = {k: v for k, v in batch.items() if k != "labels"}
			outputs = model(**inp)
			logits = outputs.get("logits")
			vocab = logits.size(-1)
			shift_logits = logits[..., :-1, :].contiguous()
			shift_labels = batch["labels"][..., 1:].contiguous().to(shift_logits.device)
			shift_labels = shift_labels.masked_fill(shift_labels == int(pad_token_id), -100)
			loss = F.cross_entropy(
				shift_logits.view(-1, vocab),
				shift_labels.view(-1),
				ignore_index=-100,
			)
			total_loss += float(loss.item())
			num_batches += 1
		avg_loss = total_loss / max(1, num_batches)
		try:
			ppl = float(torch.exp(torch.tensor(avg_loss)).item())
		except Exception:
			ppl = float("inf")
		return {"loss": avg_loss, "perplexity": ppl}

	# Data util
	from datasets import Dataset as HFDataset  # type: ignore
	def make_dataset(texts: List[str]) -> Any:
		raw = HFDataset.from_dict({"text": texts})
		def _tok_fn(row: Dict[str, Any]) -> Dict[str, Any]:
			out = tok(row["text"], truncation=True, max_length=context_len, padding="max_length")
			return out
		return raw.map(_tok_fn, batched=False, remove_columns=["text"])

	# Collator
	collator = DataCollatorForLanguageModeling(tokenizer=tok, mlm=False)

	# Resolve datasets one-by-one (sequential, full by default)
	datasets_cfg = os.getenv("DATASETS_CONFIG", "")
	train_new_only = False  # eval should not skip previously seen
	iter_program_texts = None
	try:
		from examples.scripts.datasets_loader import iter_program_texts as _ipt  # type: ignore
		iter_program_texts = _ipt


			vocab = logits.size(-1)
			shift_logits = logits[..., :-1, :].contiguous()
			shift_labels = batch["labels"][..., 1:].contiguous().to(shift_logits.device)
			shift_labels = shift_labels.masked_fill(shift_labels == int(pad_token_id), -100)
			loss = F.cross_entropy(
				shift_logits.view(-1, vocab),
				shift_labels.view(-1),
				ignore_index=-100,
			)
			total_loss += float(loss.item())
			num_batches += 1
		avg_loss = total_loss / max(1, num_batches)
		try:
			ppl = float(torch.exp(torch.tensor(avg_loss)).item())
		except Exception:
			ppl = float("inf")
		return {"loss": avg_loss, "perplexity": ppl}

	# Data util
	from datasets import Dataset as HFDataset  # type: ignore
	def make_dataset(texts: List[str]) -> Any:
		raw = HFDataset.from_dict({"text": texts})
		def _tok_fn(row: Dict[str, Any]) -> Dict[str, Any]:
			out = tok(row["text"], truncation=True, max_length=context_len, padding="max_length")
			return out
		return raw.map(_tok_fn, batched=False, remove_columns=["text"])

	# Collator
	collator = DataCollatorForLanguageModeling(tokenizer=tok, mlm=False)

	# Resolve datasets one-by-one (sequential, full by default)
	datasets_cfg = os.getenv("DATASETS_CONFIG", "")
	train_new_only = False  # eval should not skip previously seen
	iter_program_texts = None
	try:
		from examples.scripts.datasets_loader import iter_program_texts as _ipt  # type: ignore
		iter_program_texts = _ipt
	except Exception:
		try:
			import importlib.util as _imputil  # type: ignore
			_loader_path = (example_dir.parent / "scripts" / "datasets_loader.py").resolve()
			_spec = _imputil.spec_from_file_location("datasets_loader", str(_loader_path))
			if _spec and _spec.loader:
				_mod = _imputil.module_from_spec(_spec)
				_spec.loader.exec_module(_mod)  # type: ignore
				iter_program_texts = getattr(_mod, "iter_program_texts", None)  # type: ignore


			shift_labels = batch["labels"][..., 1:].contiguous().to(shift_logits.device)
			shift_labels = shift_labels.masked_fill(shift_labels == int(pad_token_id), -100)
			loss = F.cross_entropy(
				shift_logits.view(-1, vocab),
				shift_labels.view(-1),
				ignore_index=-100,
			)
			total_loss += float(loss.item())
			num_batches += 1
		avg_loss = total_loss / max(1, num_batches)
		try:
			ppl = float(torch.exp(torch.tensor(avg_loss)).item())
		except Exception:
			ppl = float("inf")
		return {"loss": avg_loss, "perplexity": ppl}

	# Data util
	from datasets import Dataset as HFDataset  # type: ignore
	def make_dataset(texts: List[str]) -> Any:
		raw = HFDataset.from_dict({"text": texts})
		def _tok_fn(row: Dict[str, Any]) -> Dict[str, Any]:
			out = tok(row["text"], truncation=True, max_length=context_len, padding="max_length")
			return out
		return raw.map(_tok_fn, batched=False, remove_columns=["text"])

	# Collator
	collator = DataCollatorForLanguageModeling(tokenizer=tok, mlm=False)

	# Resolve datasets one-by-one (sequential, full by default)
	datasets_cfg = os.getenv("DATASETS_CONFIG", "")
	train_new_only = False  # eval should not skip previously seen
	iter_program_texts = None
	try:
		from examples.scripts.datasets_loader import iter_program_texts as _ipt  # type: ignore
		iter_program_texts = _ipt
	except Exception:
		try:
			import importlib.util as _imputil  # type: ignore
			_loader_path = (example_dir.parent / "scripts" / "datasets_loader.py").resolve()
			_spec = _imputil.spec_from_file_location("datasets_loader", str(_loader_path))
			if _spec and _spec.loader:
				_mod = _imputil.module_from_spec(_spec)
				_spec.loader.exec_module(_mod)  # type: ignore


from __future__ import annotations

import os
import sys
import json
from dataclasses import dataclass
from pathlib import Path
from typing import List, Dict, Any

import torch
import torch.nn.functional as F

# Minimal dependencies: transformers + peft + datasets

@dataclass
class TrainPaths:
	adapters_dir: Path
	out_dir: Path

def _ensure_dirs(paths: TrainPaths) -> None:
	paths.adapters_dir.mkdir(parents=True, exist_ok=True)
	paths.out_dir.mkdir(parents=True, exist_ok=True)

def _load_mbpp_texts(max_n: int) -> List[str]:
	# Prefer direct HF datasets to avoid import path issues
	try:
		from datasets import load_dataset  # type: ignore
		ds = None
		for subset in ("sanitized", None):
			try:
				ds = load_dataset("mbpp", subset, split="train") if subset else load_dataset("mbpp", split="train")
				break
			except Exception:
				continue
		if ds is None:
			raise RuntimeError("mbpp dataset not available")
		texts: List[str] = []
		total = len(ds)
		for i in range(total):
			row = ds[i]
			prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
			code = str(row.get("code") or row.get("solution") or "").strip()
			combined = (prompt + "\n" + code).strip()
			if combined:
				texts.append(combined)
			if len(texts) >= int(max_n):
				break
		print(f"[mbpp] loaded {len(texts)} / {total} train samples")
		return texts
	except Exception as e:
		print(f"[mbpp] failed via datasets; using tiny fallback: {e}")
		return [
			"def add(x, y): return x + y",
			"class Foo:\n    def bar(self):\n        return 42",
			"for i in range(10): print(i)",
		]

def main() -> None:
	example_dir = Path(__file__).resolve()
	artifacts_root = example_dir / "datasets" / "hf_lora_mbpp"
	paths = TrainPaths(
		adapters_dir=artifacts_root / "lora_adapter",
		out_dir=artifacts_root / "outputs",
	)
	_ensure_dirs(paths)
	state_path = artifacts_root / ".program_state.json"

	# If not specified, prefer GPUs 0,1
	if not os.getenv("CUDA_VISIBLE_DEVICES"):
		os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"

	model_id = os.getenv("LLAMA_MODEL_ID", "meta-llama/Llama-3.1-8B-Instruct")
	max_n = int(os.getenv("MBPP_MAX_N", "120"))
	context_len = int(os.getenv("MAX_LEN", "256"))
	lr = float(os.getenv("LR", "5e-5"))
	num_epochs = float(os.getenv("EPOCHS", "1"))
	per_device_batch = int(os.getenv("BATCH_SIZE", "1"))
	grad_accum = int(os.getenv("GRAD_ACCUM_STEPS", "8"))
	warmup_ratio = float(os.getenv("WARMUP_RATIO", "0.1"))
	weight_decay = float(os.getenv("WEIGHT_DECAY", "0.01"))
	save_steps = int(os.getenv("SAVE_STEPS", "0"))
	logging_steps = int(os.getenv("LOG_STEPS", "25"))
	seed = int(os.getenv("SEED", "0"))
	use_qlora = os.getenv("USE_QLORA", "1") == "1"

	torch.manual_seed(seed)
	if torch.cuda.is_available():
		torch.cuda.manual_seed_all(seed)

	# Tokenizer/model
	from transformers import AutoTokenizer, AutoModelForCausalLM, DataCollatorForLanguageModeling, Trainer, TrainingArguments  # type: ignore
	try:
		from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training, PeftModel  # type: ignore
	except Exception as e:
		print(f"[lora] PEFT not available: {e}")
		print("Please install 'peft' to train adapters: pip install peft")
		sys.exit(2)

	tok = AutoTokenizer.from_pretrained(model_id)
	if tok.pad_token_id is None and tok.eos_token is not None:
		tok.pad_token = tok.eos_token
	pad_id = int(tok.pad_token_id or 0)

	print(f"[model] loading {model_id}")
	model_kwargs: Dict[str, Any] = {}
	device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
	visible = os.environ.get("CUDA_VISIBLE_DEVICES", "")
	num_visible = len([v for v in visible.split(",") if v.strip()]) if visible else torch.cuda.device_count()
	if use_qlora:
		try:
			from transformers import BitsAndBytesConfig  # type: ignore
			bnb_config = BitsAndBytesConfig(
				load_in_4bit=True,
				bnb_4bit_quant_type="nf4",
				bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
			)
			model_kwargs["quantization_config"] = bnb_config
			# Prefer layer-wise sharding over DataParallel
			model_kwargs["device_map"] = "auto" if num_visible > 1 else None
		except Exception as e:
			print(f"[qlora] BitsAndBytes not available, falling back to full precision: {e}")
			model_kwargs["device_map"] = None
			model_kwargs["torch_dtype"] = torch.bfloat16 if torch.cuda.is_available() else torch.float32
	else:
		# If multiple GPUs, shard with accelerate device_map
		model_kwargs["device_map"] = "auto" if num_visible > 1 else None
		model_kwargs["torch_dtype"] = torch.bfloat16 if torch.cuda.is_available() else torch.float32

	model = AutoModelForCausalLM.from_pretrained(model_id, **model_kwargs)
	# Disable cache when using gradient checkpointing
	try:
		model.config.use_cache = False
	except Exception:
		pass
	# Only move to single device when not using device_map auto
	if model_kwargs.get("device_map", None) is None:
		model = model.to(device)
	# Prepare for k-bit training (enables input grads, casts norms, etc.)
	if use_qlora:
		try:
			model = prepare_model_for_kbit_training(model)
		except Exception as e:
			print(f"[qlora] prepare_model_for_kbit_training failed: {e}")
	# Optional: gradient checkpointing for memory
	if hasattr(model, "gradient_checkpointing_enable"):
		try:
			model.gradient_checkpointing_enable()
		except Exception:
			pass

	# Attach LoRA to attention and MLP projections
	target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
	lora_r = int(os.getenv("LORA_R", "8"))
	lora_alpha = int(os.getenv("LORA_ALPHA", "16"))
	lora_dropout = float(os.getenv("LORA_DROPOUT", "0.05"))
	resume = os.getenv("RESUME", "0") == "1"
	adapter_override = os.getenv("ADAPTER_PATH", "").strip()
	loaded_adapter_path = None
	if adapter_override:
		loaded_adapter_path = adapter_override
	elif state_path.exists():
		try:
			prev_state = json.loads(state_path.read_text())
			loaded_adapter_path = str(prev_state.get("last_adapter_path") or "").strip() or None
		except Exception:
			loaded_adapter_path = None
	# If resume is requested but no adapter path is available, fall back to creating a new LoRA
	if resume and not loaded_adapter_path:
		resume = False
	if resume and loaded_adapter_path:
		try:
			model = PeftModel.from_pretrained(model, loaded_adapter_path, is_trainable=True)
			print(f"[resume] loaded adapter from {loaded_adapter_path}")
		except Exception as e:
			print(f"[resume] failed to load adapter at {loaded_adapter_path}: {e}")
			resume = False
	if not resume:
		lora_cfg = LoraConfig(
			r=lora_r,
			lora_alpha=lora_alpha,
			target_modules=target_modules,
			lora_dropout=lora_dropout,
			bias="none",
			task_type="CAUSAL_LM",
		)
		model = get_peft_model(model, lora_cfg)
	# Print trainable parameter summary without relying on PEFT helper
	try:
		trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
		total = sum(p.numel() for p in model.parameters())
		ratio = (float(trainable) / float(max(1, total))) * 100.0
		print(f"trainable params: {trainable:,} || all params: {total:,} || trainable%: {ratio:.4f}")
	except Exception:
		pass

	# Trainer with custom loss that avoids passing labels to model forward
	from transformers import Trainer, TrainingArguments  # type: ignore
	class LossTrainer(Trainer):
		def __init__(self, pad_id: int, *args, **kwargs) -> None:
			super().__init__(*args, **kwargs)
			self._pad_id = int(pad_id)
		def compute_loss(self, model, inputs, return_outputs: bool = False, num_items_in_batch=None):
			labels = inputs.get("labels", None)
			if labels is None:
				return super().compute_loss(model, inputs, return_outputs=return_outputs)
			inp = {k: v for k, v in inputs.items() if k != "labels"}
			outputs = model(**inp)
			logits = outputs.get("logits")
			vocab = logits.size(-1)
			shift_logits = logits[..., :-1, :].contiguous()
			shift_labels = labels[..., 1:].contiguous().to(shift_logits.device)
			shift_labels = shift_labels.masked_fill(shift_labels == int(self._pad_id), -100)
			loss = F.cross_entropy(
				shift_logits.view(-1, vocab),
				shift_labels.view(-1),
				ignore_index=-100,
			)
			return (loss, outputs) if return_outputs else loss

	# Build dataset(s) via configurable loader (local-first, then HF)
	datasets_cfg = os.getenv("DATASETS_CONFIG", "")
	sequential = os.getenv("SEQUENTIAL_TRAIN", "1") == "1"
	# In sequential mode, default to skipping datasets we've already trained
	train_new_only = os.getenv("TRAIN_NEW_ONLY", "1" if sequential else "0") == "1"
	loaded_names: List[str] = []
	total_samples = 0
	from datasets import Dataset as HFDataset  # type: ignore
	if sequential:
		iter_program_texts = None
		try:
			from examples.scripts.datasets_loader import iter_program_texts as _ipt  # type: ignore
			iter_program_texts = _ipt
		except Exception:
			# Fallback: import from file path to avoid relying on PYTHONPATH
			try:
				import importlib.util as _imputil  # type: ignore
				_loader_path = (example_dir.parent / "scripts" / "datasets_loader.py").resolve()
				_spec = _imputil.spec_from_file_location("datasets_loader", str(_loader_path))
				if _spec and _spec.loader:
					_mod = _imputil.module_from_spec(_spec)
					_spec.loader.exec_module(_mod)  # type: ignore
					iter_program_texts = getattr(_mod, "iter_program_texts", None)  # type: ignore
			except Exception:
				iter_program_texts = None
		if iter_program_texts is not None:
			source_iter = iter_program_texts(
				example_dir=str(example_dir),
				config_path=datasets_cfg or None,
				train_new_only=train_new_only,
				state_path=str(state_path),
			)
			saw_any = False
			for ds_texts, src_name in source_iter:
				if not ds_texts:
					continue
				saw_any = True
				raw = HFDataset.from_dict({"text": ds_texts})
				def _tok_fn(row: Dict[str, Any]) -> Dict[str, Any]:
					out = tok(row["text"], truncation=True, max_length=context_len, padding="max_length")
					return out
				tokenized = raw.map(_tok_fn, batched=False, remove_columns=["text"])
				# LM data collator will create labels, and default ignore_index=-100
				collator = DataCollatorForLanguageModeling(tokenizer=tok, mlm=False)
				train_args = TrainingArguments(
					output_dir=str(paths.out_dir / "trainer"),
					per_device_train_batch_size=per_device_batch,
					gradient_accumulation_steps=grad_accum,
					learning_rate=lr,
					weight_decay=weight_decay,
					num_train_epochs=num_epochs,
					warmup_ratio=warmup_ratio,
					logging_steps=logging_steps,
					save_steps=save_steps,
					save_total_limit=1,
					report_to=[],
					seed=seed,
					bf16=torch.cuda.is_available(),
					fp16=False if torch.cuda.is_available() else False,
				)
				trainer = LossTrainer(
					pad_id=pad_id,
					model=model,
					args=train_args,
					train_dataset=tokenized,
					data_collator=collator,
				)
				print(f"[train] starting on {src_name} ({len(ds_texts)} samples)")
				_ = trainer.train()
				print(f"[train] done {src_name}")
				# Save LoRA adapter incrementally and update state
				paths.adapters_dir.mkdir(parents=True, exist_ok=True)
				model.save_pretrained(str(paths.adapters_dir))
				try:
					state = {}
					if state_path.exists():
						state = json.loads(state_path.read_text())
					state["last_adapter_path"] = str(paths.adapters_dir)
					seen = set(state.get("datasets_seen", []))
					seen.add(src_name)
					state["datasets_seen"] = sorted(list(seen))
					state_path.write_text(json.dumps(state, indent=2), encoding="utf-8")
				except Exception:
					pass
				loaded_names.append(src_name)
				total_samples += len(ds_texts)
			if not saw_any:
				# Fallback to MBPP when nothing configured/available
				ds_texts = _load_mbpp_texts(max_n=max_n)
				raw = HFDataset.from_dict({"text": ds_texts})
				def _tok_fn(row: Dict[str, Any]) -> Dict[str, Any]:
					out = tok(row["text"], truncation=True, max_length=context_len, padding="max_length")
					return out
				tokenized = raw.map(_tok_fn, batched=False, remove_columns=["text"])
		else:
			# Fall through to non-sequential path on missing iterator
			sequential = False
	if not sequential:
		try:
			from examples.scripts.datasets_loader import load_program_texts  # type: ignore
		except Exception:
			# Fallback: load loader directly from file path without altering sys.path
			import importlib.util as _imputil  # type: ignore
			_loader_path = (example_dir.parent / "scripts" / "datasets_loader.py").resolve()
			_spec = _imputil.spec_from_file_location("datasets_loader", str(_loader_path))
			if _spec and _spec.loader:
				_mod = _imputil.module_from_spec(_spec)
				_spec.loader.exec_module(_mod)  # type: ignore
				load_program_texts = getattr(_mod, "load_program_texts")  # type: ignore
			else:
				raise
		texts, loaded_names = load_program_texts(
			example_dir=str(example_dir),
			config_path=datasets_cfg or None,
			train_new_only=train_new_only,
			state_path=str(state_path),
		)
		if not texts:
			# Fallback to MBPP train only if config produced nothing
			texts = _load_mbpp_texts(max_n=max_n)
			loaded_names = ["mbpp:train"]
		raw = HFDataset.from_dict({"text": texts})
		def _tok_fn(row: Dict[str, Any]) -> Dict[str, Any]:
			out = tok(row["text"], truncation=True, max_length=context_len, padding="max_length")
			return out
		tokenized = raw.map(_tok_fn, batched=False, remove_columns=["text"])
		total_samples = len(texts)

	def _tok_fn(row: Dict[str, Any]) -> Dict[str, Any]:
		out = tok(row["text"], truncation=True, max_length=context_len, padding="max_length")
		return out

	if not sequential:
		# LM data collator will create labels, and default ignore_index=-100
		collator = DataCollatorForLanguageModeling(tokenizer=tok, mlm=False)
		# Training config
		train_args = TrainingArguments(
			output_dir=str(paths.out_dir / "trainer"),
			per_device_train_batch_size=per_device_batch,
			gradient_accumulation_steps=grad_accum,
			learning_rate=lr,
			weight_decay=weight_decay,
			num_train_epochs=num_epochs,
			warmup_ratio=warmup_ratio,
			logging_steps=logging_steps,
			save_steps=save_steps,
			save_total_limit=1,
			report_to=[],
			seed=seed,
			bf16=torch.cuda.is_available(),
			fp16=False if torch.cuda.is_available() else False,
		)

	if not sequential:
		trainer = LossTrainer(
			pad_id=pad_id,
			model=model,
			args=train_args,
			train_dataset=tokenized,
			data_collator=collator,
		)
		print("[train] starting")
		train_out = trainer.train()
		print("[train] done")
	else:
		train_out = type("obj", (), {"metrics": {}})()  # minimal placeholder

	# Save LoRA adapter
	paths.adapters_dir.mkdir(parents=True, exist_ok=True)
	model.save_pretrained(str(paths.adapters_dir))
	# Update program state for continuous training
	try:
		state = {}
		if state_path.exists():
			state = json.loads(state_path.read_text())
		state["last_adapter_path"] = str(paths.adapters_dir)
		seen = set(state.get("datasets_seen", []))
		seen.update(loaded_names or [])
		state["datasets_seen"] = sorted(list(seen))
		state["last_training"] = {
			"samples": total_samples,
			"epochs": num_epochs,
			"batch_size": per_device_batch,
			"grad_accum": grad_accum,
			"lr": lr,
			"max_len": context_len,
		}
		state_path.write_text(json.dumps(state, indent=2), encoding="utf-8")
		print(f"[state] updated {state_path}")
	except Exception as e:
		print(f"[state] failed to update program state: {e}")
	# Save report
	metrics = train_out.metrics if hasattr(train_out, "metrics") else {}
	report = {
		"model_id": model_id,
		"num_samples": total_samples,
		"epochs": num_epochs,
		"batch_size": per_device_batch,
		"grad_accum": grad_accum,
		"lr": lr,
		"warmup_ratio": warmup_ratio,
		"weight_decay": weight_decay,
		"lora": {
			"r": lora_r,
			"alpha": lora_alpha,
			"dropout": lora_dropout,
			"targets": target_modules,
		},
		"metrics": metrics,
		"adapter_path": str(paths.adapters_dir),
	}
	with open(paths.out_dir / "LoraTrainingReport.json", "w", encoding="utf-8") as fh:
		fh.write(json.dumps({"schema_version": 1, "LoraTrainingReport": report}, indent=2))
	print(str(paths.out_dir / "LoraTrainingReport.json"))

if __name__ == "__main__":
	main()


TrainPaths

from __future__ import annotations

import os
import sys
import json
from dataclasses import dataclass
from pathlib import Path
from typing import List, Dict, Any

import torch
import torch.nn.functional as F

# Minimal dependencies: transformers + peft + datasets

@dataclass
class TrainPaths:
	adapters_dir: Path
	out_dir: Path

def _ensure_dirs(paths: TrainPaths) -> None:
	paths.adapters_dir.mkdir(parents=True, exist_ok=True)
	paths.out_dir.mkdir(parents=True, exist_ok=True)

def _load_mbpp_texts(max_n: int) -> List[str]:
	# Prefer direct HF datasets to avoid import path issues
	try:
		from datasets import load_dataset  # type: ignore
		ds = None
		for subset in ("sanitized", None):
			try:
				ds = load_dataset("mbpp", subset, split="train") if subset else load_dataset("mbpp", split="train")
				break
			except Exception:
				continue
		if ds is None:
			raise RuntimeError("mbpp dataset not available")


import os
import sys
import json
from dataclasses import dataclass
from pathlib import Path
from typing import List, Dict, Any

import torch
import torch.nn.functional as F

# Minimal dependencies: transformers + peft + datasets

@dataclass
class TrainPaths:
	adapters_dir: Path
	out_dir: Path

def _ensure_dirs(paths: TrainPaths) -> None:
	paths.adapters_dir.mkdir(parents=True, exist_ok=True)
	paths.out_dir.mkdir(parents=True, exist_ok=True)

def _load_mbpp_texts(max_n: int) -> List[str]:
	# Prefer direct HF datasets to avoid import path issues
	try:
		from datasets import load_dataset  # type: ignore
		ds = None
		for subset in ("sanitized", None):
			try:
				ds = load_dataset("mbpp", subset, split="train") if subset else load_dataset("mbpp", split="train")
				break
			except Exception:
				continue
		if ds is None:
			raise RuntimeError("mbpp dataset not available")
		texts: List[str] = []
		total = len(ds)
		for i in range(total):
			row = ds[i]
			prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()


from pathlib import Path
from typing import List, Dict, Any

import torch
import torch.nn.functional as F

# Minimal dependencies: transformers + peft + datasets

@dataclass
class TrainPaths:
	adapters_dir: Path
	out_dir: Path

def _ensure_dirs(paths: TrainPaths) -> None:
	paths.adapters_dir.mkdir(parents=True, exist_ok=True)
	paths.out_dir.mkdir(parents=True, exist_ok=True)

def _load_mbpp_texts(max_n: int) -> List[str]:
	# Prefer direct HF datasets to avoid import path issues
	try:
		from datasets import load_dataset  # type: ignore
		ds = None
		for subset in ("sanitized", None):
			try:
				ds = load_dataset("mbpp", subset, split="train") if subset else load_dataset("mbpp", split="train")
				break
			except Exception:
				continue
		if ds is None:
			raise RuntimeError("mbpp dataset not available")
		texts: List[str] = []
		total = len(ds)
		for i in range(total):
			row = ds[i]
			prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
			code = str(row.get("code") or row.get("solution") or "").strip()
			combined = (prompt + "\n" + code).strip()
			if combined:
				texts.append(combined)
			if len(texts) >= int(max_n):
				break
		print(f"[mbpp] loaded {len(texts)} / {total} train samples")
		return texts
	except Exception as e:
		print(f"[mbpp] failed via datasets; using tiny fallback: {e}")
		return [
			"def add(x, y): return x + y",
			"class Foo:\n    def bar(self):\n        return 42",
			"for i in range(10): print(i)",
		]

def main() -> None:
	example_dir = Path(__file__).resolve()
	artifacts_root = example_dir / "datasets" / "hf_lora_mbpp"
	paths = TrainPaths(
		adapters_dir=artifacts_root / "lora_adapter",
		out_dir=artifacts_root / "outputs",
	)
	_ensure_dirs(paths)
	state_path = artifacts_root / ".program_state.json"

	# If not specified, prefer GPUs 0,1
	if not os.getenv("CUDA_VISIBLE_DEVICES"):
		os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"

	model_id = os.getenv("LLAMA_MODEL_ID", "meta-llama/Llama-3.1-8B-Instruct")
	max_n = int(os.getenv("MBPP_MAX_N", "120"))
	context_len = int(os.getenv("MAX_LEN", "256"))
	lr = float(os.getenv("LR", "5e-5"))


main

		for i in range(total):
			row = ds[i]
			prompt = str(row.get("prompt") or row.get("question") or row.get("text") or "").strip()
			code = str(row.get("code") or row.get("solution") or "").strip()
			combined = (prompt + "\n" + code).strip()
			if combined:
				texts.append(combined)
			if len(texts) >= int(max_n):
				break
		print(f"[mbpp] loaded {len(texts)} / {total} train samples")
		return texts
	except Exception as e:
		print(f"[mbpp] failed via datasets; using tiny fallback: {e}")
		return [
			"def add(x, y): return x + y",
			"class Foo:\n    def bar(self):\n        return 42",
			"for i in range(10): print(i)",
		]

def main() -> None:
	example_dir = Path(__file__).resolve()
	artifacts_root = example_dir / "datasets" / "hf_lora_mbpp"
	paths = TrainPaths(
		adapters_dir=artifacts_root / "lora_adapter",
		out_dir=artifacts_root / "outputs",
	)
	_ensure_dirs(paths)
	state_path = artifacts_root / ".program_state.json"

	# If not specified, prefer GPUs 0,1
	if not os.getenv("CUDA_VISIBLE_DEVICES"):
		os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"

	model_id = os.getenv("LLAMA_MODEL_ID", "meta-llama/Llama-3.1-8B-Instruct")
	max_n = int(os.getenv("MBPP_MAX_N", "120"))
	context_len = int(os.getenv("MAX_LEN", "256"))
	lr = float(os.getenv("LR", "5e-5"))
	num_epochs = float(os.getenv("EPOCHS", "1"))
	per_device_batch = int(os.getenv("BATCH_SIZE", "1"))
	grad_accum = int(os.getenv("GRAD_ACCUM_STEPS", "8"))
	warmup_ratio = float(os.getenv("WARMUP_RATIO", "0.1"))
	weight_decay = float(os.getenv("WEIGHT_DECAY", "0.01"))
	save_steps = int(os.getenv("SAVE_STEPS", "0"))
	logging_steps = int(os.getenv("LOG_STEPS", "25"))
	seed = int(os.getenv("SEED", "0"))
	use_qlora = os.getenv("USE_QLORA", "1") == "1"

	torch.manual_seed(seed)
	if torch.cuda.is_available():
		torch.cuda.manual_seed_all(seed)

	# Tokenizer/model
	from transformers import AutoTokenizer, AutoModelForCausalLM, DataCollatorForLanguageModeling, Trainer, TrainingArguments  # type: ignore
	try:
		from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training, PeftModel  # type: ignore
	except Exception as e:
		print(f"[lora] PEFT not available: {e}")
		print("Please install 'peft' to train adapters: pip install peft")
		sys.exit(2)

	tok = AutoTokenizer.from_pretrained(model_id)
	if tok.pad_token_id is None and tok.eos_token is not None:
		tok.pad_token = tok.eos_token
	pad_id = int(tok.pad_token_id or 0)

	print(f"[model] loading {model_id}")
	model_kwargs: Dict[str, Any] = {}
	device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
	visible = os.environ.get("CUDA_VISIBLE_DEVICES", "")
	num_visible = len([v for v in visible.split(",") if v.strip()]) if visible else torch.cuda.device_count()
	if use_qlora:
		try:
			from transformers import BitsAndBytesConfig  # type: ignore
			bnb_config = BitsAndBytesConfig(
				load_in_4bit=True,
				bnb_4bit_quant_type="nf4",
				bnb_4bit_compute_dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
			)
			model_kwargs["quantization_config"] = bnb_config
			# Prefer layer-wise sharding over DataParallel
			model_kwargs["device_map"] = "auto" if num_visible > 1 else None
		except Exception as e:
			print(f"[qlora] BitsAndBytes not available, falling back to full precision: {e}")
			model_kwargs["device_map"] = None
			model_kwargs["torch_dtype"] = torch.bfloat16 if torch.cuda.is_available() else torch.float32
	else:
		# If multiple GPUs, shard with accelerate device_map
		model_kwargs["device_map"] = "auto" if num_visible > 1 else None
		model_kwargs["torch_dtype"] = torch.bfloat16 if torch.cuda.is_available() else torch.float32

	model = AutoModelForCausalLM.from_pretrained(model_id, **model_kwargs)
	# Disable cache when using gradient checkpointing
	try:
		model.config.use_cache = False
	except Exception:
		pass
	# Only move to single device when not using device_map auto
	if model_kwargs.get("device_map", None) is None:
		model = model.to(device)
	# Prepare for k-bit training (enables input grads, casts norms, etc.)
	if use_qlora:
		try:
			model = prepare_model_for_kbit_training(model)
		except Exception as e:
			print(f"[qlora] prepare_model_for_kbit_training failed: {e}")
	# Optional: gradient checkpointing for memory
	if hasattr(model, "gradient_checkpointing_enable"):
		try:
			model.gradient_checkpointing_enable()
		except Exception:
			pass

	# Attach LoRA to attention and MLP projections
	target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
	lora_r = int(os.getenv("LORA_R", "8"))
	lora_alpha = int(os.getenv("LORA_ALPHA", "16"))
	lora_dropout = float(os.getenv("LORA_DROPOUT", "0.05"))
	resume = os.getenv("RESUME", "0") == "1"
	adapter_override = os.getenv("ADAPTER_PATH", "").strip()
	loaded_adapter_path = None
	if adapter_override:
		loaded_adapter_path = adapter_override
	elif state_path.exists():
		try:
			prev_state = json.loads(state_path.read_text())
			loaded_adapter_path = str(prev_state.get("last_adapter_path") or "").strip() or None
		except Exception:
			loaded_adapter_path = None
	# If resume is requested but no adapter path is available, fall back to creating a new LoRA
	if resume and not loaded_adapter_path:
		resume = False
	if resume and loaded_adapter_path:
		try:
			model = PeftModel.from_pretrained(model, loaded_adapter_path, is_trainable=True)
			print(f"[resume] loaded adapter from {loaded_adapter_path}")
		except Exception as e:
			print(f"[resume] failed to load adapter at {loaded_adapter_path}: {e}")
			resume = False
	if not resume:
		lora_cfg = LoraConfig(
			r=lora_r,
			lora_alpha=lora_alpha,
			target_modules=target_modules,
			lora_dropout=lora_dropout,
			bias="none",
			task_type="CAUSAL_LM",
		)
		model = get_peft_model(model, lora_cfg)
	# Print trainable parameter summary without relying on PEFT helper
	try:
		trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
		total = sum(p.numel() for p in model.parameters())
		ratio = (float(trainable) / float(max(1, total))) * 100.0
		print(f"trainable params: {trainable:,} || all params: {total:,} || trainable%: {ratio:.4f}")
	except Exception:
		pass

	# Trainer with custom loss that avoids passing labels to model forward
	from transformers import Trainer, TrainingArguments  # type: ignore
	class LossTrainer(Trainer):
		def __init__(self, pad_id: int, *args, **kwargs) -> None:
			super().__init__(*args, **kwargs)
			self._pad_id = int(pad_id)
		def compute_loss(self, model, inputs, return_outputs: bool = False, num_items_in_batch=None):
			labels = inputs.get("labels", None)
			if labels is None:
				return super().compute_loss(model, inputs, return_outputs=return_outputs)
			inp = {k: v for k, v in inputs.items() if k != "labels"}
			outputs = model(**inp)
			logits = outputs.get("logits")
			vocab = logits.size(-1)
			shift_logits = logits[..., :-1, :].contiguous()
			shift_labels = labels[..., 1:].contiguous().to(shift_logits.device)
			shift_labels = shift_labels.masked_fill(shift_labels == int(self._pad_id), -100)
			loss = F.cross_entropy(
				shift_logits.view(-1, vocab),
				shift_labels.view(-1),
				ignore_index=-100,
			)
			return (loss, outputs) if return_outputs else loss

	# Build dataset(s) via configurable loader (local-first, then HF)
	datasets_cfg = os.getenv("DATASETS_CONFIG", "")
	sequential = os.getenv("SEQUENTIAL_TRAIN", "1") == "1"
	# In sequential mode, default to skipping datasets we've already trained
	train_new_only = os.getenv("TRAIN_NEW_ONLY", "1" if sequential else "0") == "1"
	loaded_names: List[str] = []
	total_samples = 0
	from datasets import Dataset as HFDataset  # type: ignore
	if sequential:
		iter_program_texts = None
		try:
			from examples.scripts.datasets_loader import iter_program_texts as _ipt  # type: ignore
			iter_program_texts = _ipt
		except Exception:
			# Fallback: import from file path to avoid relying on PYTHONPATH
			try:
				import importlib.util as _imputil  # type: ignore
				_loader_path = (example_dir.parent / "scripts" / "datasets_loader.py").resolve()
				_spec = _imputil.spec_from_file_location("datasets_loader", str(_loader_path))
				if _spec and _spec.loader:
					_mod = _imputil.module_from_spec(_spec)
					_spec.loader.exec_module(_mod)  # type: ignore
					iter_program_texts = getattr(_mod, "iter_program_texts", None)  # type: ignore
			except Exception:
				iter_program_texts = None
		if iter_program_texts is not None:
			source_iter = iter_program_texts(
				example_dir=str(example_dir),
				config_path=datasets_cfg or None,
				train_new_only=train_new_only,
				state_path=str(state_path),
			)
			saw_any = False
			for ds_texts, src_name in source_iter:
				if not ds_texts:
					continue
				saw_any = True
				raw = HFDataset.from_dict({"text": ds_texts})
				def _tok_fn(row: Dict[str, Any]) -> Dict[str, Any]:
					out = tok(row["text"], truncation=True, max_length=context_len, padding="max_length")
					return out
				tokenized = raw.map(_tok_fn, batched=False, remove_columns=["text"])
				# LM data collator will create labels, and default ignore_index=-100
				collator = DataCollatorForLanguageModeling(tokenizer=tok, mlm=False)
				train_args = TrainingArguments(
					output_dir=str(paths.out_dir / "trainer"),
					per_device_train_batch_size=per_device_batch,
					gradient_accumulation_steps=grad_accum,
					learning_rate=lr,
					weight_decay=weight_decay,
					num_train_epochs=num_epochs,
					warmup_ratio=warmup_ratio,
					logging_steps=logging_steps,
					save_steps=save_steps,
					save_total_limit=1,
					report_to=[],
					seed=seed,
					bf16=torch.cuda.is_available(),
					fp16=False if torch.cuda.is_available() else False,
				)
				trainer = LossTrainer(
					pad_id=pad_id,
					model=model,
					args=train_args,
					train_dataset=tokenized,
					data_collator=collator,
				)
				print(f"[train] starting on {src_name} ({len(ds_texts)} samples)")
				_ = trainer.train()
				print(f"[train] done {src_name}")
				# Save LoRA adapter incrementally and update state
				paths.adapters_dir.mkdir(parents=True, exist_ok=True)
				model.save_pretrained(str(paths.adapters_dir))
				try:
					state = {}
					if state_path.exists():
						state = json.loads(state_path.read_text())
					state["last_adapter_path"] = str(paths.adapters_dir)
					seen = set(state.get("datasets_seen", []))
					seen.add(src_name)
					state["datasets_seen"] = sorted(list(seen))
					state_path.write_text(json.dumps(state, indent=2), encoding="utf-8")
				except Exception:
					pass
				loaded_names.append(src_name)
				total_samples += len(ds_texts)
			if not saw_any:
				# Fallback to MBPP when nothing configured/available
				ds_texts = _load_mbpp_texts(max_n=max_n)
				raw = HFDataset.from_dict({"text": ds_texts})
				def _tok_fn(row: Dict[str, Any]) -> Dict[str, Any]:
					out = tok(row["text"], truncation=True, max_length=context_len, padding="max_length")
					return out
				tokenized = raw.map(_tok_fn, batched=False, remove_columns=["text"])
		else:
			# Fall through to non-sequential path on missing iterator
			sequential = False
	if not sequential:
		try:
			from examples.scripts.datasets_loader import load_program_texts  # type: ignore
		except Exception:
			# Fallback: load loader directly from file path without altering sys.path
			import importlib.util as _imputil  # type: ignore
			_loader_path = (example_dir.parent / "scripts" / "datasets_loader.py").resolve()
			_spec = _imputil.spec_from_file_location("datasets_loader", str(_loader_path))
			if _spec and _spec.loader:
				_mod = _imputil.module_from_spec(_spec)
				_spec.loader.exec_module(_mod)  # type: ignore
				load_program_texts = getattr(_mod, "load_program_texts")  # type: ignore
			else:
				raise
		texts, loaded_names = load_program_texts(
			example_dir=str(example_dir),
			config_path=datasets_cfg or None,
			train_new_only=train_new_only,
			state_path=str(state_path),
		)
		if not texts:
			# Fallback to MBPP train only if config produced nothing
			texts = _load_mbpp_texts(max_n=max_n)
			loaded_names = ["mbpp:train"]
		raw = HFDataset.from_dict({"text": texts})
		def _tok_fn(row: Dict[str, Any]) -> Dict[str, Any]:
			out = tok(row["text"], truncation=True, max_length=context_len, padding="max_length")
			return out
		tokenized = raw.map(_tok_fn, batched=False, remove_columns=["text"])
		total_samples = len(texts)

	def _tok_fn(row: Dict[str, Any]) -> Dict[str, Any]:
		out = tok(row["text"], truncation=True, max_length=context_len, padding="max_length")
		return out

	if not sequential:
		# LM data collator will create labels, and default ignore_index=-100
		collator = DataCollatorForLanguageModeling(tokenizer=tok, mlm=False)
		# Training config
		train_args = TrainingArguments(
			output_dir=str(paths.out_dir / "trainer"),
			per_device_train_batch_size=per_device_batch,
			gradient_accumulation_steps=grad_accum,
			learning_rate=lr,
			weight_decay=weight_decay,
			num_train_epochs=num_epochs,
			warmup_ratio=warmup_ratio,
			logging_steps=logging_steps,
			save_steps=save_steps,
			save_total_limit=1,
			report_to=[],
			seed=seed,
			bf16=torch.cuda.is_available(),
			fp16=False if torch.cuda.is_available() else False,
		)

	if not sequential:
		trainer = LossTrainer(
			pad_id=pad_id,
			model=model,
			args=train_args,
			train_dataset=tokenized,
			data_collator=collator,
		)
		print("[train] starting")
		train_out = trainer.train()
		print("[train] done")
	else:
		train_out = type("obj", (), {"metrics": {}})()  # minimal placeholder

	# Save LoRA adapter
	paths.adapters_dir.mkdir(parents=True, exist_ok=True)
	model.save_pretrained(str(paths.adapters_dir))
	# Update program state for continuous training
	try:
		state = {}
		if state_path.exists():
			state = json.loads(state_path.read_text())
		state["last_adapter_path"] = str(paths.adapters_dir)
		seen = set(state.get("datasets_seen", []))
		seen.update(loaded_names or [])
		state["datasets_seen"] = sorted(list(seen))
		state["last_training"] = {
			"samples": total_samples,
			"epochs": num_epochs,
			"batch_size": per_device_batch,
			"grad_accum": grad_accum,
			"lr": lr,
			"max_len": context_len,
		}
		state_path.write_text(json.dumps(state, indent=2), encoding="utf-8")
		print(f"[state] updated {state_path}")
	except Exception as e:
		print(f"[state] failed to update program state: {e}")
	# Save report
	metrics = train_out.metrics if hasattr(train_out, "metrics") else {}
	report = {
		"model_id": model_id,
		"num_samples": total_samples,
		"epochs": num_epochs,
		"batch_size": per_device_batch,
		"grad_accum": grad_accum,
		"lr": lr,
		"warmup_ratio": warmup_ratio,
		"weight_decay": weight_decay,
		"lora": {
			"r": lora_r,
			"alpha": lora_alpha,
			"dropout": lora_dropout,
			"targets": target_modules,
		},
		"metrics": metrics,
		"adapter_path": str(paths.adapters_dir),
	}
	with open(paths.out_dir / "LoraTrainingReport.json", "w", encoding="utf-8") as fh:
		fh.write(json.dumps({"schema_version": 1, "LoraTrainingReport": report}, indent=2))
	print(str(paths.out_dir / "LoraTrainingReport.json"))

if __name__ == "__main__":
	main()


LossTrainer

		lora_cfg = LoraConfig(
			r=lora_r,
			lora_alpha=lora_alpha,
			target_modules=target_modules,
			lora_dropout=lora_dropout,
			bias="none",
			task_type="CAUSAL_LM",
		)
		model = get_peft_model(model, lora_cfg)
	# Print trainable parameter summary without relying on PEFT helper
	try:
		trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
		total = sum(p.numel() for p in model.parameters())
		ratio = (float(trainable) / float(max(1, total))) * 100.0
		print(f"trainable params: {trainable:,} || all params: {total:,} || trainable%: {ratio:.4f}")
	except Exception:
		pass

	# Trainer with custom loss that avoids passing labels to model forward
	from transformers import Trainer, TrainingArguments  # type: ignore
	class LossTrainer(Trainer):
		def __init__(self, pad_id: int, *args, **kwargs) -> None:
			super().__init__(*args, **kwargs)
			self._pad_id = int(pad_id)
		def compute_loss(self, model, inputs, return_outputs: bool = False, num_items_in_batch=None):
			labels = inputs.get("labels", None)
			if labels is None:
				return super().compute_loss(model, inputs, return_outputs=return_outputs)
			inp = {k: v for k, v in inputs.items() if k != "labels"}
			outputs = model(**inp)
			logits = outputs.get("logits")
			vocab = logits.size(-1)
			shift_logits = logits[..., :-1, :].contiguous()
			shift_labels = labels[..., 1:].contiguous().to(shift_logits.device)
			shift_labels = shift_labels.masked_fill(shift_labels == int(self._pad_id), -100)
			loss = F.cross_entropy(
				shift_logits.view(-1, vocab),
				shift_labels.view(-1),
				ignore_index=-100,
			)
			return (loss, outputs) if return_outputs else loss

	# Build dataset(s) via configurable loader (local-first, then HF)
	datasets_cfg = os.getenv("DATASETS_CONFIG", "")
	sequential = os.getenv("SEQUENTIAL_TRAIN", "1") == "1"
	# In sequential mode, default to skipping datasets we've already trained
	train_new_only = os.getenv("TRAIN_NEW_ONLY", "1" if sequential else "0") == "1"
	loaded_names: List[str] = []
	total_samples = 0
	from datasets import Dataset as HFDataset  # type: ignore
	if sequential:
		iter_program_texts = None
		try:
			from examples.scripts.datasets_loader import iter_program_texts as _ipt  # type: ignore
			iter_program_texts = _ipt
		except Exception:
			# Fallback: import from file path to avoid relying on PYTHONPATH
			try:
				import importlib.util as _imputil  # type: ignore
				_loader_path = (example_dir.parent / "scripts" / "datasets_loader.py").resolve()
				_spec = _imputil.spec_from_file_location("datasets_loader", str(_loader_path))


				# Save LoRA adapter incrementally and update state
				paths.adapters_dir.mkdir(parents=True, exist_ok=True)
				model.save_pretrained(str(paths.adapters_dir))
				try:
					state = {}
					if state_path.exists():
						state = json.loads(state_path.read_text())
					state["last_adapter_path"] = str(paths.adapters_dir)
					seen = set(state.get("datasets_seen", []))
					seen.add(src_name)
					state["datasets_seen"] = sorted(list(seen))
					state_path.write_text(json.dumps(state, indent=2), encoding="utf-8")
				except Exception:
					pass
				loaded_names.append(src_name)
				total_samples += len(ds_texts)
			if not saw_any:
				# Fallback to MBPP when nothing configured/available
				ds_texts = _load_mbpp_texts(max_n=max_n)
				raw = HFDataset.from_dict({"text": ds_texts})
				def _tok_fn(row: Dict[str, Any]) -> Dict[str, Any]:
					out = tok(row["text"], truncation=True, max_length=context_len, padding="max_length")
					return out
				tokenized = raw.map(_tok_fn, batched=False, remove_columns=["text"])
		else:
			# Fall through to non-sequential path on missing iterator
			sequential = False
	if not sequential:
		try:
			from examples.scripts.datasets_loader import load_program_texts  # type: ignore
		except Exception:
			# Fallback: load loader directly from file path without altering sys.path
			import importlib.util as _imputil  # type: ignore
			_loader_path = (example_dir.parent / "scripts" / "datasets_loader.py").resolve()
			_spec = _imputil.spec_from_file_location("datasets_loader", str(_loader_path))
			if _spec and _spec.loader:
				_mod = _imputil.module_from_spec(_spec)
				_spec.loader.exec_module(_mod)  # type: ignore
				load_program_texts = getattr(_mod, "load_program_texts")  # type: ignore
			else:
				raise
		texts, loaded_names = load_program_texts(
			example_dir=str(example_dir),


			r=lora_r,
			lora_alpha=lora_alpha,
			target_modules=target_modules,
			lora_dropout=lora_dropout,
			bias="none",
			task_type="CAUSAL_LM",
		)
		model = get_peft_model(model, lora_cfg)
	# Print trainable parameter summary without relying on PEFT helper
	try:
		trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
		total = sum(p.numel() for p in model.parameters())
		ratio = (float(trainable) / float(max(1, total))) * 100.0
		print(f"trainable params: {trainable:,} || all params: {total:,} || trainable%: {ratio:.4f}")
	except Exception:
		pass

	# Trainer with custom loss that avoids passing labels to model forward
	from transformers import Trainer, TrainingArguments  # type: ignore
	class LossTrainer(Trainer):
		def __init__(self, pad_id: int, *args, **kwargs) -> None:
			super().__init__(*args, **kwargs)
			self._pad_id = int(pad_id)
		def compute_loss(self, model, inputs, return_outputs: bool = False, num_items_in_batch=None):
			labels = inputs.get("labels", None)
			if labels is None:
				return super().compute_loss(model, inputs, return_outputs=return_outputs)
			inp = {k: v for k, v in inputs.items() if k != "labels"}
			outputs = model(**inp)
			logits = outputs.get("logits")
			vocab = logits.size(-1)
			shift_logits = logits[..., :-1, :].contiguous()
			shift_labels = labels[..., 1:].contiguous().to(shift_logits.device)
			shift_labels = shift_labels.masked_fill(shift_labels == int(self._pad_id), -100)
			loss = F.cross_entropy(
				shift_logits.view(-1, vocab),
				shift_labels.view(-1),
				ignore_index=-100,
			)
			return (loss, outputs) if return_outputs else loss

	# Build dataset(s) via configurable loader (local-first, then HF)
	datasets_cfg = os.getenv("DATASETS_CONFIG", "")


			lora_dropout=lora_dropout,
			bias="none",
			task_type="CAUSAL_LM",
		)
		model = get_peft_model(model, lora_cfg)
	# Print trainable parameter summary without relying on PEFT helper
	try:
		trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
		total = sum(p.numel() for p in model.parameters())
		ratio = (float(trainable) / float(max(1, total))) * 100.0
		print(f"trainable params: {trainable:,} || all params: {total:,} || trainable%: {ratio:.4f}")
	except Exception:
		pass

	# Trainer with custom loss that avoids passing labels to model forward
	from transformers import Trainer, TrainingArguments  # type: ignore
	class LossTrainer(Trainer):
		def __init__(self, pad_id: int, *args, **kwargs) -> None:
			super().__init__(*args, **kwargs)
			self._pad_id = int(pad_id)
		def compute_loss(self, model, inputs, return_outputs: bool = False, num_items_in_batch=None):
			labels = inputs.get("labels", None)
			if labels is None:
				return super().compute_loss(model, inputs, return_outputs=return_outputs)
			inp = {k: v for k, v in inputs.items() if k != "labels"}
			outputs = model(**inp)
			logits = outputs.get("logits")
			vocab = logits.size(-1)
			shift_logits = logits[..., :-1, :].contiguous()
			shift_labels = labels[..., 1:].contiguous().to(shift_logits.device)
			shift_labels = shift_labels.masked_fill(shift_labels == int(self._pad_id), -100)
			loss = F.cross_entropy(
				shift_logits.view(-1, vocab),
				shift_labels.view(-1),
				ignore_index=-100,
			)
			return (loss, outputs) if return_outputs else loss

	# Build dataset(s) via configurable loader (local-first, then HF)
	datasets_cfg = os.getenv("DATASETS_CONFIG", "")
	sequential = os.getenv("SEQUENTIAL_TRAIN", "1") == "1"
	# In sequential mode, default to skipping datasets we've already trained
	train_new_only = os.getenv("TRAIN_NEW_ONLY", "1" if sequential else "0") == "1"
	loaded_names: List[str] = []
	total_samples = 0
	from datasets import Dataset as HFDataset  # type: ignore
	if sequential:
		iter_program_texts = None
		try:
			from examples.scripts.datasets_loader import iter_program_texts as _ipt  # type: ignore
			iter_program_texts = _ipt
		except Exception:
			# Fallback: import from file path to avoid relying on PYTHONPATH
			try:
				import importlib.util as _imputil  # type: ignore
				_loader_path = (example_dir.parent / "scripts" / "datasets_loader.py").resolve()
				_spec = _imputil.spec_from_file_location("datasets_loader", str(_loader_path))


from __future__ import annotations

import sys
import os
import json
from pathlib import Path
from typing import List, Dict, Any
import math
import random
import numpy as np

import torch
from torch.utils.data import Dataset, DataLoader

ROOT = EX_DIR
PG_BACKEND = "examples.scripts.dataset_graph:DatasetProgramGraph"
try:
	from .program_config import load_program_config  # type: ignore  # noqa: E402
except Exception:
	from program_config import load_program_config  # type: ignore  # noqa: E402

def _run(cmd: List[str]) -> int:
	print("[run]", " ".join(cmd))
	return os.system(" ".join(cmd))

class SimpleLmDataset(Dataset):
	def __init__(self, tokenizer, texts: List[str], max_len: int = 128) -> None:
		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		# pad left simple
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)
		attn = (input_ids != pad_id).to(torch.long)
		return {"input_ids": input_ids, "attn_mask": attn}

def main() -> None:
	cfg = load_program_config(str(ROOT))
	adapters_dir = cfg.paths.adapters_dir
	out_dir = cfg.paths.out_dir
	telemetry_json = cfg.paths.telemetry_path
	adapters_dir.mkdir(parents=True, exist_ok=True)
	out_dir.mkdir(parents=True, exist_ok=True)

	if str(repo_root) not in sys.path:
		sys.path.insert(0, str(repo_root))
	py_path = os.environ.get("PYTHONPATH") or ""
	os.environ["PYTHONPATH"] = f"{str(repo_root)}{(':' + py_path) if py_path else ''}"
	# Reproducibility
	seed = int(os.getenv("SEED", "0"))
	random.seed(seed)
	np.random.seed(seed)
	torch.manual_seed(seed)
	if torch.cuda.is_available():
		torch.cuda.manual_seed_all(seed)

	# 1) Optional: Build ProgramGraph caches/adapters (kept for consistent manifest/state)
	rc = _run([
		sys.executable,
		str(EX_DIR / "build.py"),
		"--sources", str(ROOT),
		"--model", "meta-llama/Llama-3.1-8B-Instruct",
		"--adapters-dir", str(adapters_dir),
		"--embed-dim", "256",
		"--include-text",
		"--text-max-bytes", "20000",
		"--pg-backend", PG_BACKEND,
		"--graph-prop-hops", "2",
		"--graph-prop-damp", "0.85",
		"--init-program-state",
		"--program-state-path", str(cfg.paths.program_state_path),
		"--seed", "0",
		"--verbose",
	])
	if rc != 0:
		sys.exit(rc)

	# 2) Build a tokenized dataset: prefer MBPP if enabled, else synthetic
	from data.tokenizer import LocalLlamaTokenizer  # type: ignore
	# Use a local snapshot directory if available; otherwise fallback to a whitespace tokenizer
	snap_default = os.environ.get("LLAMA_SNAPSHOT_DIR", str(repo_root / "checkpoints" / "llama"))
	try:
		tok = LocalLlamaTokenizer(snap_default)
	except Exception:
		from data.tokenizer import WhitespaceTokenizer  # type: ignore
		tok = WhitespaceTokenizer()
	texts = []
	use_mbpp = os.getenv("USE_MBPP", "0") == "1"
	if use_mbpp:
		try:
			from examples.scripts.mbpp_loader import load_mbpp_texts  # type: ignore
			max_n = int(os.getenv("MBPP_MAX_N", "64"))
			mbpp_texts, total = load_mbpp_texts(max_n=max_n, split="train")
			if mbpp_texts:
				print(f"[mbpp] loaded {len(mbpp_texts)} / {total} samples")
				texts = mbpp_texts
		except Exception as e:
			print(f"[mbpp] failed to load via HF datasets: {e}")
	if not texts:
		texts = [
			"def add(x, y): return x + y",
			"class Foo:\n    def bar(self):\n        return 42",
			"for i in range(10): print(i)",
		]
	ds = SimpleLmDataset(tok, texts, max_len=128)

	# 3) Build a small model from our local stack and train for a few steps
	from specs.config import ModelConfig  # type: ignore
	from model.factory import build_causal_lm  # type: ignore

	cfg_m = ModelConfig(
		d_model=256, n_heads=8, n_layers=2, d_ff=1024, vocab_size=int(getattr(tok, "vocab_size", 32000) or 32000),
		dtype="float32", kv_cache_paged=False, attn_impl="eager",
	)
	model = build_causal_lm(cfg_m, block="llama", compress=None)
	model.train()
	device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
	model = model.to(device)

	lr = float(os.getenv("LR", "1e-4"))
	opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01)
	# Ignore padding in loss
	pad_id = int(getattr(tok, "pad_token_id", 0) or 0)
	loss_fn = torch.nn.CrossEntropyLoss(ignore_index=-100)

	# DataLoader with batching and shuffling
	batch_size = int(os.getenv("BATCH_SIZE", "8"))
	dl = DataLoader(ds, batch_size=batch_size, shuffle=True, drop_last=False)

	steps = int(os.getenv("TRAIN_STEPS", "16"))
	grad_accum = int(os.getenv("GRAD_ACCUM_STEPS", "1"))
	warmup_ratio = float(os.getenv("WARMUP_RATIO", "0.1"))
	warmup_steps = int(os.getenv("WARMUP_STEPS", str(max(1, int(steps * warmup_ratio))))))
	min_lr_ratio = float(os.getenv("MIN_LR_RATIO", "0.1"))
	def _lr_lambda(current_step: int) -> float:
		if current_step < warmup_steps:
			return max(1e-6, float(current_step + 1) / float(max(1, warmup_steps)))
		progress = float(current_step - warmup_steps) / float(max(1, steps - warmup_steps))
		cosine = 0.5 * (1.0 + math.cos(math.pi * min(1.0, max(0.0, progress))))
		return float(min_lr_ratio + (1.0 - min_lr_ratio) * cosine)
	scheduler = torch.optim.lr_scheduler.LambdaLR(opt, lr_lambda=_lr_lambda)

	total_loss = 0.0
	print(f"[train] steps={steps}, dataset_size={len(ds)}, batch_size={batch_size}, grad_accum={grad_accum}, lr={lr}")
	def infinite_batches():
		while True:
			for batch in dl:
				yield {k: v.to(device) for k, v in batch.items()}
	batches = infinite_batches()
	for step in range(steps):
		opt.zero_grad(set_to_none=True)
		step_loss = 0.0
		for _ in range(max(1, grad_accum)):
			batch = next(batches)
			out = model(input_ids=batch["input_ids"], attention_mask=batch["attn_mask"], return_dict=True)
			logits = out["logits"]  # [B, T, V]
			vocab = logits.size(-1)
			# Shift logits and labels for next-token prediction
			logits_shifted = logits[..., :-1, :].contiguous()
			labels = batch["input_ids"][..., 1:].contiguous()
			# Mask pad tokens in labels
			labels = labels.masked_fill(labels == pad_id, -100)
			loss = loss_fn(
				logits_shifted.view(-1, vocab),
				labels.view(-1),
			) / float(max(1, grad_accum))
			loss.backward()
			step_loss += float(loss.detach().item())
		torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
		opt.step()
		scheduler.step()
		total_loss += step_loss
		if (step + 1) % max(1, int(os.getenv("LOG_EVERY", "50"))) == 0 or (step + 1) == steps:
			cur_lr = scheduler.get_last_lr()[0] if scheduler is not None else lr
			print(f"[train] step={step+1}/{steps} loss={step_loss:.4f} avg_loss={(total_loss/(step+1)):.4f} lr={cur_lr:.6g}")
	avg_loss = total_loss / max(1, steps)

	# 4) Save a checkpoint stub under adapters_dir and emit a training report
	ckpt_dir = adapters_dir / "trained_ckpt"
	ckpt_dir.mkdir(parents=True, exist_ok=True)
	torch.save(model.state_dict(), ckpt_dir / "model.pt")
	report = {
		"avg_loss": avg_loss,
		"steps": steps,
		"device": str(device),
		"ckpt_path": str(ckpt_dir / "model.pt"),
	}
	out_path = out_dir / "DatasetTrainerReport.json"
	out_dir.mkdir(parents=True, exist_ok=True)
	with open(out_path, "w", encoding="utf-8") as fh:
		fh.write(json.dumps({"schema_version": 1, "DatasetTrainerReport": report}, indent=2))
	print(str(out_path))
	sys.exit(0)

if __name__ == "__main__":
	main()


from __future__ import annotations

import sys
import os
import json
from pathlib import Path
from typing import List, Dict

import torch
from torch.utils.data import Dataset, DataLoader

ROOT = EX_DIR
PG_BACKEND = "examples.scripts.dataset_graph:DatasetProgramGraph"
try:
	from .program_config import load_program_config  # type: ignore  # noqa: E402
except Exception:
	from program_config import load_program_config  # type: ignore  # noqa: E402

def _run(cmd: List[str]) -> int:
	print("[run]", " ".join(cmd))
	return os.system(" ".join(cmd))

class SimpleLmDataset(Dataset):
	def __init__(self, tokenizer, texts: List[str], max_len: int = 128) -> None:
		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		# pad left simple
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)
		attn = (input_ids != pad_id).to(torch.long)
		return {"input_ids": input_ids, "attn_mask": attn}

def main() -> None:
	# Load the base config (for program id/backend), but use a separate artifacts dir
	_ = load_program_config(str(ROOT))
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "complete_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	program_state_path = adapters_dir / ".program_state.json"
	adapters_dir.mkdir(parents=True, exist_ok=True)
	out_dir.mkdir(parents=True, exist_ok=True)

	if str(repo_root) not in sys.path:
		sys.path.insert(0, str(repo_root))
	py_path = os.environ.get("PYTHONPATH") or ""
	os.environ["PYTHONPATH"] = f"{str(repo_root)}{(':' + py_path) if py_path else ''}"

	# 1) Build ProgramGraph caches/adapters
	rc = _run([
		sys.executable,
		str(EX_DIR / "build.py"),
		"--sources", str(ROOT),
		"--model", "meta-llama/Llama-3.1-8B-Instruct",
		"--adapters-dir", str(adapters_dir),
		"--embed-dim", "256",
		"--include-text",
		"--text-max-bytes", "20000",
		"--pg-backend", PG_BACKEND,
		"--graph-prop-hops", "2",
		"--graph-prop-damp", "0.85",
		"--init-program-state",
		"--program-state-path", str(program_state_path),
		"--seed", "0",
		"--verbose",
	])
	if rc != 0:
		sys.exit(rc)

	# 2) Build a tokenized dataset: always use full MBPP train split when available
	from data.tokenizer import LocalLlamaTokenizer  # type: ignore
	# Use a local snapshot directory if available; otherwise fallback to a whitespace tokenizer
	snap_default = os.environ.get("LLAMA_SNAPSHOT_DIR", str(repo_root / "checkpoints" / "llama"))
	try:
		tok = LocalLlamaTokenizer(snap_default)
	except Exception:
		from data.tokenizer import WhitespaceTokenizer  # type: ignore
		tok = WhitespaceTokenizer()
	texts: List[str] = []
	total = 0
	try:
		from examples.scripts.mbpp_loader import load_mbpp_texts_all_splits  # type: ignore
		# Large cap to force "all" samples across all splits
		mbpp_texts, total = load_mbpp_texts_all_splits(max_n=1_000_000_000)
		if mbpp_texts:
			print(f"[mbpp] loaded {len(mbpp_texts)} / {total} samples (all splits)")
			texts = mbpp_texts
	except Exception as e:
		print(f"[mbpp] failed to load via HF datasets: {e}")
	if not texts:
		# Fallback tiny dataset
		texts = [
			"def add(x, y): return x + y",
			"class Foo:\n    def bar(self):\n        return 42",
			"for i in range(10): print(i)",
		]
	ds = SimpleLmDataset(tok, texts, max_len=128)

	# 3) Build a small model from our local stack and train for exactly one pass over ds
	from specs.config import ModelConfig  # type: ignore
	from model.factory import build_causal_lm  # type: ignore

	cfg_m = ModelConfig(
		d_model=256, n_heads=8, n_layers=2, d_ff=1024, vocab_size=int(getattr(tok, "vocab_size", 32000) or 32000),
		dtype="float32", kv_cache_paged=False, attn_impl="eager",
	)
	model = build_causal_lm(cfg_m, block="llama", compress=None)
	model.train()
	device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
	model = model.to(device)

	lr = float(os.getenv("LR", "1e-4"))
	opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01)
	# Ignore padding in loss
	pad_id = int(getattr(tok, "pad_token_id", 0) or 0)
	loss_fn = torch.nn.CrossEntropyLoss(ignore_index=-100)

	# DataLoader epoch training
	batch_size = int(os.getenv("BATCH_SIZE", "8"))
	dl = DataLoader(ds, batch_size=batch_size, shuffle=True, drop_last=False)

	total_loss = 0.0
	steps = 0
	epochs = int(os.getenv("EPOCHS", "1"))
	print(f"[train] epochs={epochs}, dataset_size={len(ds)}, batch_size={batch_size}")
	for _ in range(epochs):
		for batch in dl:
			steps += 1
			opt.zero_grad(set_to_none=True)
			batch = {k: v.to(device) for k, v in batch.items()}
			out = model(input_ids=batch["input_ids"], attention_mask=batch["attn_mask"], return_dict=True)
			logits = out["logits"]  # [B, T, V]
			vocab = logits.size(-1)
			# Shift and mask labels
			logits_shifted = logits[..., :-1, :].contiguous()
			labels = batch["input_ids"][..., 1:].contiguous()
			labels = labels.masked_fill(labels == pad_id, -100)
			loss = loss_fn(
				logits_shifted.view(-1, vocab),
				labels.view(-1),
			)
			loss.backward()
			torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
			opt.step()
			total_loss += float(loss.detach().item())
	avg_loss = total_loss / max(1, steps)

	# 4) Save a checkpoint under adapters_dir and emit a training report
	ckpt_dir = adapters_dir / "trained_ckpt"
	ckpt_dir.mkdir(parents=True, exist_ok=True)
	torch.save(model.state_dict(), ckpt_dir / "model.pt")
	report = {
		"avg_loss": avg_loss,
		"steps": steps,
		"device": str(device),
		"ckpt_path": str(ckpt_dir / "model.pt"),
	}
	out_path = out_dir / "DatasetTrainerReport.json"
	out_dir.mkdir(parents=True, exist_ok=True)
	with open(out_path, "w", encoding="utf-8") as fh:
		fh.write(json.dumps({"schema_version": 1, "DatasetTrainerReport": report}, indent=2))
	print(str(out_path))
	sys.exit(0)

if __name__ == "__main__":
	main()


from __future__ import annotations

import sys
import os
import json
from pathlib import Path
from typing import List, Dict

import torch
from torch.utils.data import Dataset, DataLoader

ROOT = EX_DIR
PG_BACKEND = "examples.scripts.dataset_graph:DatasetProgramGraph"
try:
	from .program_config import load_program_config  # type: ignore  # noqa: E402
except Exception:
	from program_config import load_program_config  # type: ignore  # noqa: E402

def _run(cmd: List[str]) -> int:
	print("[run]", " ".join(cmd))
	return os.system(" ".join(cmd))

class SimpleLmDataset(Dataset):
	def __init__(self, tokenizer, texts: List[str], max_len: int = 128) -> None:
		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		# pad left simple
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)


SimpleLmDataset

import json
from pathlib import Path
from typing import List, Dict

import torch
from torch.utils.data import Dataset, DataLoader

ROOT = EX_DIR
PG_BACKEND = "examples.scripts.dataset_graph:DatasetProgramGraph"
try:
	from .program_config import load_program_config  # type: ignore  # noqa: E402
except Exception:
	from program_config import load_program_config  # type: ignore  # noqa: E402

def _run(cmd: List[str]) -> int:
	print("[run]", " ".join(cmd))
	return os.system(" ".join(cmd))

class SimpleLmDataset(Dataset):
	def __init__(self, tokenizer, texts: List[str], max_len: int = 128) -> None:
		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		# pad left simple
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)
		attn = (input_ids != pad_id).to(torch.long)
		return {"input_ids": input_ids, "attn_mask": attn}

def main() -> None:
	# Load the base config (for program id/backend), but use a separate artifacts dir
	_ = load_program_config(str(ROOT))
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "complete_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	program_state_path = adapters_dir / ".program_state.json"
	adapters_dir.mkdir(parents=True, exist_ok=True)
	out_dir.mkdir(parents=True, exist_ok=True)

	if str(repo_root) not in sys.path:
		sys.path.insert(0, str(repo_root))
	py_path = os.environ.get("PYTHONPATH") or ""
	os.environ["PYTHONPATH"] = f"{str(repo_root)}{(':' + py_path) if py_path else ''}"

	# 1) Build ProgramGraph caches/adapters
	rc = _run([


main

		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		# pad left simple
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)
		attn = (input_ids != pad_id).to(torch.long)
		return {"input_ids": input_ids, "attn_mask": attn}

def main() -> None:
	# Load the base config (for program id/backend), but use a separate artifacts dir
	_ = load_program_config(str(ROOT))
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "complete_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	program_state_path = adapters_dir / ".program_state.json"
	adapters_dir.mkdir(parents=True, exist_ok=True)
	out_dir.mkdir(parents=True, exist_ok=True)

	if str(repo_root) not in sys.path:
		sys.path.insert(0, str(repo_root))
	py_path = os.environ.get("PYTHONPATH") or ""
	os.environ["PYTHONPATH"] = f"{str(repo_root)}{(':' + py_path) if py_path else ''}"

	# 1) Build ProgramGraph caches/adapters
	rc = _run([
		sys.executable,
		str(EX_DIR / "build.py"),
		"--sources", str(ROOT),
		"--model", "meta-llama/Llama-3.1-8B-Instruct",
		"--adapters-dir", str(adapters_dir),
		"--embed-dim", "256",
		"--include-text",
		"--text-max-bytes", "20000",
		"--pg-backend", PG_BACKEND,
		"--graph-prop-hops", "2",
		"--graph-prop-damp", "0.85",
		"--init-program-state",
		"--program-state-path", str(program_state_path),
		"--seed", "0",
		"--verbose",
	])
	if rc != 0:
		sys.exit(rc)

	# 2) Build a tokenized dataset: always use full MBPP train split when available
	from data.tokenizer import LocalLlamaTokenizer  # type: ignore
	# Use a local snapshot directory if available; otherwise fallback to a whitespace tokenizer
	snap_default = os.environ.get("LLAMA_SNAPSHOT_DIR", str(repo_root / "checkpoints" / "llama"))
	try:
		tok = LocalLlamaTokenizer(snap_default)
	except Exception:
		from data.tokenizer import WhitespaceTokenizer  # type: ignore
		tok = WhitespaceTokenizer()
	texts: List[str] = []
	total = 0
	try:
		from examples.scripts.mbpp_loader import load_mbpp_texts_all_splits  # type: ignore
		# Large cap to force "all" samples across all splits
		mbpp_texts, total = load_mbpp_texts_all_splits(max_n=1_000_000_000)
		if mbpp_texts:
			print(f"[mbpp] loaded {len(mbpp_texts)} / {total} samples (all splits)")
			texts = mbpp_texts
	except Exception as e:
		print(f"[mbpp] failed to load via HF datasets: {e}")
	if not texts:
		# Fallback tiny dataset
		texts = [
			"def add(x, y): return x + y",
			"class Foo:\n    def bar(self):\n        return 42",
			"for i in range(10): print(i)",
		]
	ds = SimpleLmDataset(tok, texts, max_len=128)

	# 3) Build a small model from our local stack and train for exactly one pass over ds
	from specs.config import ModelConfig  # type: ignore
	from model.factory import build_causal_lm  # type: ignore

	cfg_m = ModelConfig(
		d_model=256, n_heads=8, n_layers=2, d_ff=1024, vocab_size=int(getattr(tok, "vocab_size", 32000) or 32000),
		dtype="float32", kv_cache_paged=False, attn_impl="eager",
	)
	model = build_causal_lm(cfg_m, block="llama", compress=None)
	model.train()
	device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
	model = model.to(device)

	lr = float(os.getenv("LR", "1e-4"))
	opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01)
	# Ignore padding in loss
	pad_id = int(getattr(tok, "pad_token_id", 0) or 0)
	loss_fn = torch.nn.CrossEntropyLoss(ignore_index=-100)

	# DataLoader epoch training
	batch_size = int(os.getenv("BATCH_SIZE", "8"))
	dl = DataLoader(ds, batch_size=batch_size, shuffle=True, drop_last=False)

	total_loss = 0.0
	steps = 0
	epochs = int(os.getenv("EPOCHS", "1"))
	print(f"[train] epochs={epochs}, dataset_size={len(ds)}, batch_size={batch_size}")
	for _ in range(epochs):
		for batch in dl:
			steps += 1
			opt.zero_grad(set_to_none=True)
			batch = {k: v.to(device) for k, v in batch.items()}
			out = model(input_ids=batch["input_ids"], attention_mask=batch["attn_mask"], return_dict=True)
			logits = out["logits"]  # [B, T, V]
			vocab = logits.size(-1)
			# Shift and mask labels
			logits_shifted = logits[..., :-1, :].contiguous()
			labels = batch["input_ids"][..., 1:].contiguous()
			labels = labels.masked_fill(labels == pad_id, -100)
			loss = loss_fn(
				logits_shifted.view(-1, vocab),
				labels.view(-1),
			)
			loss.backward()
			torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
			opt.step()
			total_loss += float(loss.detach().item())
	avg_loss = total_loss / max(1, steps)

	# 4) Save a checkpoint under adapters_dir and emit a training report
	ckpt_dir = adapters_dir / "trained_ckpt"
	ckpt_dir.mkdir(parents=True, exist_ok=True)
	torch.save(model.state_dict(), ckpt_dir / "model.pt")
	report = {
		"avg_loss": avg_loss,
		"steps": steps,
		"device": str(device),
		"ckpt_path": str(ckpt_dir / "model.pt"),
	}
	out_path = out_dir / "DatasetTrainerReport.json"
	out_dir.mkdir(parents=True, exist_ok=True)
	with open(out_path, "w", encoding="utf-8") as fh:
		fh.write(json.dumps({"schema_version": 1, "DatasetTrainerReport": report}, indent=2))
	print(str(out_path))
	sys.exit(0)

if __name__ == "__main__":
	main()


from pathlib import Path
from typing import List, Dict

import torch
from torch.utils.data import Dataset, DataLoader

ROOT = EX_DIR
PG_BACKEND = "examples.scripts.dataset_graph:DatasetProgramGraph"
try:
	from .program_config import load_program_config  # type: ignore  # noqa: E402
except Exception:
	from program_config import load_program_config  # type: ignore  # noqa: E402

def _run(cmd: List[str]) -> int:
	print("[run]", " ".join(cmd))
	return os.system(" ".join(cmd))

class SimpleLmDataset(Dataset):
	def __init__(self, tokenizer, texts: List[str], max_len: int = 128) -> None:
		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		# pad left simple
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)
		attn = (input_ids != pad_id).to(torch.long)
		return {"input_ids": input_ids, "attn_mask": attn}

def main() -> None:
	# Load the base config (for program id/backend), but use a separate artifacts dir
	_ = load_program_config(str(ROOT))
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "complete_dataset_trainer"
	adapters_dir = artifacts_root


try:
	from .program_config import load_program_config  # type: ignore  # noqa: E402
except Exception:
	from program_config import load_program_config  # type: ignore  # noqa: E402

def _run(cmd: List[str]) -> int:
	print("[run]", " ".join(cmd))
	return os.system(" ".join(cmd))

class SimpleLmDataset(Dataset):
	def __init__(self, tokenizer, texts: List[str], max_len: int = 128) -> None:
		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		# pad left simple
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)
		attn = (input_ids != pad_id).to(torch.long)
		return {"input_ids": input_ids, "attn_mask": attn}

def main() -> None:
	# Load the base config (for program id/backend), but use a separate artifacts dir
	_ = load_program_config(str(ROOT))
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "complete_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	program_state_path = adapters_dir / ".program_state.json"
	adapters_dir.mkdir(parents=True, exist_ok=True)


	from program_config import load_program_config  # type: ignore  # noqa: E402

def _run(cmd: List[str]) -> int:
	print("[run]", " ".join(cmd))
	return os.system(" ".join(cmd))

class SimpleLmDataset(Dataset):
	def __init__(self, tokenizer, texts: List[str], max_len: int = 128) -> None:
		self.tok = tokenizer
		self.max_len = int(max_len)
		self.samples: List[List[int]] = []
		for t in texts:
			ids = [int(i) for i in self.tok.encode(t)]
			self.samples.append(ids[: self.max_len])

	def __len__(self) -> int:
		return len(self.samples)

	def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
		ids = self.samples[idx]
		# pad left simple
		pad_id = int(getattr(self.tok, "pad_token_id", 0) or 0)
		seq = ids + [pad_id] * max(0, self.max_len - len(ids))
		input_ids = torch.tensor(seq[: self.max_len], dtype=torch.long)
		attn = (input_ids != pad_id).to(torch.long)
		return {"input_ids": input_ids, "attn_mask": attn}

def main() -> None:
	# Load the base config (for program id/backend), but use a separate artifacts dir
	_ = load_program_config(str(ROOT))
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "complete_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	program_state_path = adapters_dir / ".program_state.json"
	adapters_dir.mkdir(parents=True, exist_ok=True)
	out_dir.mkdir(parents=True, exist_ok=True)

	if str(repo_root) not in sys.path:
		sys.path.insert(0, str(repo_root))
	py_path = os.environ.get("PYTHONPATH") or ""
	os.environ["PYTHONPATH"] = f"{str(repo_root)}{(':' + py_path) if py_path else ''}"

	# 1) Build ProgramGraph caches/adapters
	rc = _run([


from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Optional

@dataclass(frozen=True)
class ProgramContracts:
	require_citations: bool = False
	citations_per_paragraph: bool = False
	retrieval_policy: str = "sim:0.5,struct:0.5"
	retrieval_temp: float = 0.6

@dataclass(frozen=True)
class ProgramPaths:
	adapters_dir: Path
	out_dir: Path
	telemetry_path: Path
	program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
	program_id: str
	pg_backend: str
	paths: ProgramPaths
	contracts: ProgramContracts

def _detect_pg_backend(example_dir: Path) -> str:
	return "examples.scripts.dataset_graph:DatasetProgramGraph"

def load_program_config(root: str) -> ProgramConfig:
	root_path = Path(root).resolve()
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "smoke_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	telemetry_path = out_dir / "structured_output.json"
	pg_backend = _detect_pg_backend(example_dir)
	program_id = root_path.name or "datasets"
	return ProgramConfig(
		program_id=program_id,
		pg_backend=pg_backend,
		paths=ProgramPaths(
			adapters_dir=adapters_dir,
			out_dir=out_dir,
			telemetry_path=telemetry_path,
			program_state_path=adapters_dir / ".program_state.json",
		),
		contracts=ProgramContracts(),
	)


ProgramContracts

from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Optional

@dataclass(frozen=True)
class ProgramContracts:
	require_citations: bool = False
	citations_per_paragraph: bool = False
	retrieval_policy: str = "sim:0.5,struct:0.5"
	retrieval_temp: float = 0.6

@dataclass(frozen=True)
class ProgramPaths:
	adapters_dir: Path
	out_dir: Path
	telemetry_path: Path
	program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
	program_id: str
	pg_backend: str
	paths: ProgramPaths
	contracts: ProgramContracts

def _detect_pg_backend(example_dir: Path) -> str:
	return "examples.scripts.dataset_graph:DatasetProgramGraph"


ProgramPaths

from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Optional

@dataclass(frozen=True)
class ProgramContracts:
	require_citations: bool = False
	citations_per_paragraph: bool = False
	retrieval_policy: str = "sim:0.5,struct:0.5"
	retrieval_temp: float = 0.6

@dataclass(frozen=True)
class ProgramPaths:
	adapters_dir: Path
	out_dir: Path
	telemetry_path: Path
	program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
	program_id: str
	pg_backend: str
	paths: ProgramPaths
	contracts: ProgramContracts

def _detect_pg_backend(example_dir: Path) -> str:
	return "examples.scripts.dataset_graph:DatasetProgramGraph"

def load_program_config(root: str) -> ProgramConfig:
	root_path = Path(root).resolve()
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "smoke_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"


ProgramConfig

from typing import Optional

@dataclass(frozen=True)
class ProgramContracts:
	require_citations: bool = False
	citations_per_paragraph: bool = False
	retrieval_policy: str = "sim:0.5,struct:0.5"
	retrieval_temp: float = 0.6

@dataclass(frozen=True)
class ProgramPaths:
	adapters_dir: Path
	out_dir: Path
	telemetry_path: Path
	program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
	program_id: str
	pg_backend: str
	paths: ProgramPaths
	contracts: ProgramContracts

def _detect_pg_backend(example_dir: Path) -> str:
	return "examples.scripts.dataset_graph:DatasetProgramGraph"

def load_program_config(root: str) -> ProgramConfig:
	root_path = Path(root).resolve()
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "smoke_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	telemetry_path = out_dir / "structured_output.json"
	pg_backend = _detect_pg_backend(example_dir)
	program_id = root_path.name or "datasets"
	return ProgramConfig(
		program_id=program_id,
		pg_backend=pg_backend,
		paths=ProgramPaths(
			adapters_dir=adapters_dir,


	retrieval_policy: str = "sim:0.5,struct:0.5"
	retrieval_temp: float = 0.6

@dataclass(frozen=True)
class ProgramPaths:
	adapters_dir: Path
	out_dir: Path
	telemetry_path: Path
	program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
	program_id: str
	pg_backend: str
	paths: ProgramPaths
	contracts: ProgramContracts

def _detect_pg_backend(example_dir: Path) -> str:
	return "examples.scripts.dataset_graph:DatasetProgramGraph"

def load_program_config(root: str) -> ProgramConfig:
	root_path = Path(root).resolve()
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "smoke_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	telemetry_path = out_dir / "structured_output.json"
	pg_backend = _detect_pg_backend(example_dir)
	program_id = root_path.name or "datasets"
	return ProgramConfig(
		program_id=program_id,
		pg_backend=pg_backend,
		paths=ProgramPaths(
			adapters_dir=adapters_dir,
			out_dir=out_dir,
			telemetry_path=telemetry_path,
			program_state_path=adapters_dir / ".program_state.json",
		),


@dataclass(frozen=True)
class ProgramPaths:
	adapters_dir: Path
	out_dir: Path
	telemetry_path: Path
	program_state_path: Optional[Path] = None

@dataclass(frozen=True)
class ProgramConfig:
	program_id: str
	pg_backend: str
	paths: ProgramPaths
	contracts: ProgramContracts

def _detect_pg_backend(example_dir: Path) -> str:
	return "examples.scripts.dataset_graph:DatasetProgramGraph"

def load_program_config(root: str) -> ProgramConfig:
	root_path = Path(root).resolve()
	example_dir = Path(__file__).resolve().parent
	artifacts_root = example_dir / "artifacts" / "smoke_dataset_trainer"
	adapters_dir = artifacts_root
	out_dir = artifacts_root / "outputs"
	telemetry_path = out_dir / "structured_output.json"
	pg_backend = _detect_pg_backend(example_dir)
	program_id = root_path.name or "datasets"
	return ProgramConfig(
		program_id=program_id,
		pg_backend=pg_backend,
		paths=ProgramPaths(
			adapters_dir=adapters_dir,
			out_dir=out_dir,
			telemetry_path=telemetry_path,
			program_state_path=adapters_dir / ".program_state.json",
		),
		contracts=ProgramContracts(),
	)