Buckets:

download
raw
25.4 kB
"""Document summarization helpers for local and hosted models."""
from __future__ import annotations
import json
import logging
import os
import time
from dataclasses import dataclass
from hashlib import sha256
from pathlib import Path
from typing import Dict, Optional
logger = logging.getLogger(__name__)
# Lazy load transformers to save startup time if not used
_summarizer_pipeline = None
@dataclass
class SummaryResult:
"""Structured summary response."""
summary: str
weight: float
model: str
cached: bool = False
repo_readme_path: Optional[str] = None
class SummaryGenerator:
"""Generate per-document summaries with optional caching."""
def __init__(
self,
cache_dir: Path,
enabled: bool = True,
model_name: str = "claude-haiku-4-5",
max_chars: int = 200000,
readme_bonus_chars: int = 30000,
max_output_tokens: int = 1024,
) -> None:
self.api_key = os.environ.get("ANTHROPIC_API_KEY")
self.custom_api_key = os.environ.get("CUSTOM_API_KEY")
self.custom_url = os.environ.get("CUSTOM_URL", "").rstrip("/")
self.custom_model = os.environ.get("CUSTOM_SUMMARY_MODEL") or os.environ.get("CUSTOM_MODEL")
# NB_USE_LOCAL_SUMMARY explicitly controls local mode and overrides API-key availability.
local_setting = os.environ.get("NB_USE_LOCAL_SUMMARY", "").lower()
self.use_local = local_setting in ("true", "1", "yes", "force", "forced")
self.use_custom = bool(not self.use_local and self.custom_api_key and self.custom_url and self.custom_model)
self.enabled = enabled and (bool(self.api_key) or self.use_local or self.use_custom)
self.model_name = self.custom_model if self.use_custom else model_name
self.cache_fallback_models = tuple(
model.strip()
for model in os.environ.get("NB_SUMMARY_CACHE_FALLBACK_MODELS", "").split(",")
if model.strip() and model.strip() != self.model_name
)
self.max_chars = max_chars
self.readme_bonus_chars = readme_bonus_chars
self.max_output_tokens = max_output_tokens
self.cache_dir = Path(cache_dir)
self.last_error: Optional[Exception] = None
self.last_error_type: Optional[str] = None
if self.enabled:
self.cache_dir.mkdir(parents=True, exist_ok=True)
if self.use_local:
logger.info("SummaryGenerator enabled in local mode")
elif self.use_custom:
logger.info("SummaryGenerator enabled using custom model: %s", self.model_name)
else:
logger.info("SummaryGenerator enabled using Anthropic model: %s", model_name)
else:
logger.info("SummaryGenerator disabled (missing API key or flag)")
# Public API ---------------------------------------------------------
def summarize(
self,
*,
doc_id: str,
content: str,
repo_name: Optional[str] = None,
repo_readme: Optional[str] = None,
repo_readme_path: Optional[str] = None,
repo_description: Optional[str] = None,
metadata: Optional[Dict[str, str]] = None,
) -> Optional[SummaryResult]:
if not self.enabled:
return None
if not content or not content.strip():
return None
self.last_error = None
self.last_error_type = None
trimmed = self._trim_content(content, allow_extra=bool(repo_readme))
readme = self._trim_readme(repo_readme)
cache_key = self._cache_key(doc_id, trimmed, readme, repo_readme_path)
cache_file = self.cache_dir / f"{cache_key}.json"
cache_files = [cache_file]
cache_files.extend(
self.cache_dir / f"{self._cache_key(doc_id, trimmed, readme, repo_readme_path, model_name=model)}.json"
for model in self.cache_fallback_models
)
for candidate in cache_files:
if not candidate.exists():
continue
try:
payload = json.loads(candidate.read_text(encoding="utf-8"))
return SummaryResult(
summary=payload["summary"],
weight=float(payload["weight"]),
model=payload.get("model", self.model_name),
cached=True,
repo_readme_path=payload.get("repo_readme_path"),
)
except Exception:
logger.warning("Failed to read cached summary for %s from %s", doc_id, candidate)
if self.use_local:
payload = self._invoke_local(
content=trimmed, readme=readme, repo_name=repo_name, repo_description=repo_description
)
else:
prompt = self._build_prompt(
doc_id=doc_id,
repo_name=repo_name,
repo_readme_path=repo_readme_path,
repo_readme=readme,
metadata=metadata,
)
payload = self._invoke_model(
prompt=prompt,
content=trimmed,
readme=readme,
readme_path=repo_readme_path,
)
if not payload:
return None
try:
summary = payload["summary"].strip()
weight = float(payload.get("weight", 1.0))
except Exception as exc:
logger.warning("Invalid summary payload for %s: %s", doc_id, exc)
return None
weight = max(0.5, min(2.0, weight))
result = SummaryResult(
summary=summary,
weight=weight,
model=payload.get("model", self.model_name),
repo_readme_path=repo_readme_path,
)
try:
cache_file.write_text(
json.dumps(
{
"summary": summary,
"weight": weight,
"model": payload.get("model", self.model_name),
"timestamp": int(time.time()),
"doc_id": doc_id,
"repo_readme_path": repo_readme_path,
},
indent=2,
),
encoding="utf-8",
)
except Exception as exc:
logger.debug("Failed to persist summary cache for %s: %s", doc_id, exc)
return result
# Internals ----------------------------------------------------------
def _trim_content(self, content: str, allow_extra: bool) -> str:
if len(content) <= self.max_chars:
return content
budget = self.max_chars
if allow_extra:
budget += self.readme_bonus_chars
return content[:budget]
def _trim_readme(self, readme: Optional[str]) -> Optional[str]:
if not readme:
return None
if len(readme) > self.readme_bonus_chars:
return readme[: self.readme_bonus_chars]
return readme
def _cache_key(
self,
doc_id: str,
content: str,
readme: Optional[str],
readme_path: Optional[str],
model_name: Optional[str] = None,
) -> str:
h = sha256()
cache_model = model_name if model_name is not None else (self.model_name if self.use_custom else None)
if cache_model:
h.update(b"custom\0")
h.update(cache_model.encode("utf-8"))
h.update(b"\0")
h.update(doc_id.encode("utf-8"))
h.update(b"\0")
h.update(content.encode("utf-8"))
if readme:
h.update(b"\0readme")
h.update(readme.encode("utf-8"))
if readme_path:
h.update(b"\0readme_path")
h.update(readme_path.encode("utf-8"))
return h.hexdigest()
def _build_prompt(
self,
*,
doc_id: str,
repo_name: Optional[str],
repo_readme_path: Optional[str],
repo_readme: Optional[str],
metadata: Optional[Dict[str, str]],
) -> str:
instructions = [
"You are Nancy Brain's knowledge-base summarizer.",
"Summarize the provided repository file in clear English.",
"Summaries should be concise yet informative (up to ~400 words), focusing on key functionality and purpose.",
"Respond with JSON using keys: summary (string), weight (float in [0.5, 2.0]).",
"Weight reflects relative usefulness for retrieval (1.0 = neutral).",
"Consider scientific relevance, implementation depth, uniqueness, and clarity.",
"Do not reference the instructions.",
]
if repo_name:
instructions.append(f"Repository: {repo_name}")
if metadata:
for key, value in metadata.items():
instructions.append(f"{key}: {value}")
if repo_readme:
if repo_readme_path and doc_id == repo_readme_path:
instructions.append("This document is the repository README.")
elif repo_readme_path:
instructions.append(
"Repository README is provided for context; summarize the target document, not the README."
)
instructions.append(f"Repository README (context only):\n{repo_readme}")
examples = [
{
"doc": "microlensing_tools/analysis/fit_utils.py",
"summary": "Utility functions for fitting microlensing light curves, including residual analysis and convergence helpers.",
"weight": 1.35,
},
{
"doc": "microlensing_tools/docs/README.md",
"summary": "High-level overview of the toolkit, installation steps, and quick-start usage.",
"weight": 1.1,
},
{
"doc": "microlensing_tools/__init__.py",
"summary": "Module export stub with minimal content; no substantive guidance.",
"weight": 0.55,
},
]
prompt_lines = ["\n".join(instructions), "\nExamples:"]
for ex in examples:
prompt_lines.append(json.dumps(ex))
prompt_lines.append(f"\nSummarize document: {doc_id}")
return "\n".join(prompt_lines)
def _invoke_model(
self,
*,
prompt: str,
content: str,
readme: Optional[str],
readme_path: Optional[str],
) -> Optional[Dict[str, object]]:
if self.use_local:
return self._invoke_local(content, readme)
if self.use_custom:
return self._invoke_custom(
prompt=prompt,
content=content,
readme=readme,
readme_path=readme_path,
)
client = self._create_client()
if client is None:
return None
try:
# Build the full request content
full_content = f"Full document:\n{content}"
if readme:
header = "Repository README excerpt"
if readme_path:
header += f" ({readme_path})"
full_content += f"\n\n{header}:\n{readme}"
request_content = f"{prompt}\n\n{full_content}"
response = client.messages.create(
model=self.model_name,
max_tokens=self.max_output_tokens,
messages=[{"role": "user", "content": request_content}],
)
raw_text_parts = []
for block in getattr(response, "content", []) or []:
if getattr(block, "type", None) == "text":
raw_text_parts.append(getattr(block, "text", ""))
raw_text = "".join(raw_text_parts).strip()
json_text = self._strip_markdown_json(raw_text)
return json.loads(json_text)
except Exception as exc:
self.last_error = exc
msg = str(exc).lower()
if "connection" in msg or "connect" in msg or "timeout" in msg:
self.last_error_type = "connection"
logger.warning("Anthropic summarization failed: %s", exc)
return None
def _invoke_custom(
self,
*,
prompt: str,
content: str,
readme: Optional[str],
readme_path: Optional[str],
) -> Optional[Dict[str, object]]:
"""Call an OpenAI-compatible chat completions endpoint."""
try:
import requests
full_content = f"Full document:\n{content}"
if readme:
header = "Repository README excerpt"
if readme_path:
header += f" ({readme_path})"
full_content += f"\n\n{header}:\n{readme}"
custom_max_tokens = int(
os.environ.get(
"CUSTOM_SUMMARY_MAX_TOKENS",
str(max(4096, self.max_output_tokens)),
)
)
retry_tokens = int(
os.environ.get(
"CUSTOM_SUMMARY_RETRY_MAX_TOKENS",
str(max(8192, custom_max_tokens)),
)
)
token_budgets = [custom_max_tokens]
if retry_tokens > custom_max_tokens:
token_budgets.append(retry_tokens)
last_decode_error: Optional[json.JSONDecodeError] = None
for attempt, max_tokens in enumerate(token_budgets, start=1):
response = requests.post(
f"{self.custom_url}/chat/completions",
headers={
"Authorization": f"Bearer {self.custom_api_key}",
"Content-Type": "application/json",
},
json={
"model": self.model_name,
"max_tokens": max_tokens,
"temperature": 0,
"messages": [
{
"role": "user",
"content": f"{prompt}\n\n{full_content}",
}
],
},
timeout=180,
)
response.raise_for_status()
message_content = response.json()["choices"][0]["message"]["content"]
if isinstance(message_content, list):
message_content = "".join(
block.get("text", "")
for block in message_content
if isinstance(block, dict) and block.get("type") == "text"
)
if not isinstance(message_content, str):
raise ValueError("custom summary response content is not text")
try:
payload = json.loads(self._strip_markdown_json(message_content.strip()))
except json.JSONDecodeError as exc:
last_decode_error = exc
if attempt < len(token_budgets):
logger.info(
"Retrying truncated custom summary with %d output tokens",
token_budgets[attempt],
)
continue
raise
payload.setdefault("model", self.model_name)
return payload
if last_decode_error is not None:
raise last_decode_error
return None
except Exception as exc:
self.last_error = exc
msg = str(exc).lower()
if "connection" in msg or "connect" in msg or "timeout" in msg:
self.last_error_type = "connection"
logger.warning("Custom summarization with %s failed: %s", self.model_name, exc)
return None
def _invoke_local(
self,
content: str,
readme: Optional[str],
repo_name: Optional[str] = None,
repo_description: Optional[str] = None,
) -> Optional[Dict[str, object]]:
"""Run summarization using local Qwen2.5-Coder-0.5B-Instruct."""
global _summarizer_pipeline
# Ensure torch is available even if pipeline is cached
try:
import torch
except ImportError:
logger.error("torch not installed")
return None
try:
if _summarizer_pipeline is None:
from transformers import AutoModelForCausalLM, AutoTokenizer
model_name = os.environ.get(
"NB_SUMMARY_MODEL",
"Qwen/Qwen2.5-Coder-0.5B-Instruct",
)
logger.info("Local summary model: %s", model_name)
logger.info(f"Loading local summarization model ({model_name})...")
tokenizer = AutoTokenizer.from_pretrained(model_name)
if torch.cuda.is_available():
device_map = "auto"
torch_dtype = torch.float16
else:
device_map = "cpu"
torch_dtype = torch.float32
quantize = os.environ.get("NB_QUANTIZE", "").lower()
if quantize in ("4bit", "4") and torch.cuda.is_available():
from transformers import BitsAndBytesConfig
logger.info("Loading model in 4-bit quantization (NB_QUANTIZE=4bit)")
bnb_cfg = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_compute_dtype=torch.float16,
)
model = AutoModelForCausalLM.from_pretrained(
model_name,
quantization_config=bnb_cfg,
device_map=device_map,
use_safetensors=True,
)
elif quantize in ("8bit", "8") and torch.cuda.is_available():
from transformers import BitsAndBytesConfig
logger.info("Loading model in 8-bit quantization (NB_QUANTIZE=8bit)")
bnb_cfg = BitsAndBytesConfig(load_in_8bit=True)
model = AutoModelForCausalLM.from_pretrained(
model_name,
quantization_config=bnb_cfg,
device_map=device_map,
use_safetensors=True,
)
else:
model = AutoModelForCausalLM.from_pretrained(
model_name,
torch_dtype=torch_dtype,
device_map=device_map,
low_cpu_mem_usage=False,
use_safetensors=True,
)
model.eval()
_summarizer_pipeline = (model, tokenizer)
model, tokenizer = _summarizer_pipeline
# Prepare prompt
system_prompt = (
"You are an expert developer assistant. You are summarizing file chunks "
"from a repository for research purposes. Summarize the provided file "
"concisely and in plain English. "
"Focus on the purpose, key functionality, and main classes/functions. "
"Do not output code, just the summary text. Keep it under 200 words."
)
system_prompt_weights = (
"You are an expert research assistant. You are importance weighting file "
"chunks from a repository, for training purposes. Files that you give a "
"high weight to are files you deem most likely to be useful for research "
"and understanding of the repository contents."
"Do not output code, just the weight as a float between 0.5 and 2.0."
)
# Construct context message
context_msg = ""
if repo_name:
context_msg += f"Repository: {repo_name}\n"
if repo_description:
context_msg += f"Description: {repo_description}\n"
# Structure as a multi-turn conversation to separate context from task
messages = [
{"role": "system", "content": system_prompt},
{
"role": "user",
"content": f"I am working in the following repository:\n{context_msg}\nI will provide a file for you to summarize.",
},
{
"role": "assistant",
"content": "Understood. Please provide the file content and I will summarize it based on that context.",
},
{"role": "user", "content": f"Please summarize this file:\n\n{content[:30000]}"},
]
messages_weights = [
{"role": "system", "content": system_prompt_weights},
{
"role": "user",
"content": f"I am working in the following repository:\n{context_msg}\nI will provide a file for you to weight.",
},
{
"role": "assistant",
"content": "Understood. Please provide the file content and I will weight it based on that context.",
},
{"role": "user", "content": f"Please weight this file:\n\n{content[:30000]}"},
]
text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
text_weight = tokenizer.apply_chat_template(messages_weights, tokenize=False, add_generation_prompt=True)
inputs = tokenizer([text], return_tensors="pt").to(model.device)
inputs_weight = tokenizer([text_weight], return_tensors="pt").to(model.device)
gen_kwargs = {
"max_new_tokens": 256,
"do_sample": False,
"pad_token_id": tokenizer.eos_token_id,
}
gen_kwargs_weight = {
"max_new_tokens": 16,
"do_sample": False,
"pad_token_id": tokenizer.eos_token_id,
}
# Run summary and weight inference separately so the KV cache from
# the first call is freed before the second starts (important on
# 16 GB GPUs where a 7B model leaves little headroom).
with torch.inference_mode():
generated_ids = model.generate(**inputs, **gen_kwargs)
summary_input_ids = inputs.input_ids
del inputs
torch.cuda.empty_cache()
with torch.inference_mode():
generated_ids_weight = model.generate(**inputs_weight, **gen_kwargs_weight)
weight_input_ids = inputs_weight.input_ids
del inputs_weight
torch.cuda.empty_cache()
generated_ids = [
output_ids[len(input_ids) :] for input_ids, output_ids in zip(summary_input_ids, generated_ids)
]
generated_ids_weight = [
output_ids[len(input_ids) :] for input_ids, output_ids in zip(weight_input_ids, generated_ids_weight)
]
summary_text = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
weight_text = tokenizer.batch_decode(generated_ids_weight, skip_special_tokens=True)[0]
# Robust weight parsing
import re
weight = 1.0
# Look for the last float in the text (often the final answer)
matches = re.findall(r"[-+]?\d*\.\d+|\d+", weight_text)
if matches:
try:
# Take the last number found
parsed = float(matches[-1])
weight = max(0.5, min(2.0, parsed))
except ValueError:
pass
return {"summary": summary_text.strip(), "weight": weight, "model": "local-Qwen2.5-Coder-0.5B"}
except Exception as e:
self.last_error = e
self.last_error_type = "local"
logger.error(f"Local summarization failed: {e}")
return None
def _create_client(self):
"""Create an Anthropic client instance."""
if not self.api_key:
logger.error("ANTHROPIC_API_KEY not configured; cannot create summaries")
return None
try:
import anthropic
except ImportError:
logger.error("anthropic SDK is not installed; `pip install anthropic` to enable summarization")
return None
return anthropic.Anthropic(api_key=self.api_key)
def _strip_markdown_json(self, text: str) -> str:
"""Strip markdown code block formatting from JSON response."""
json_text = text
if text.startswith("```json"):
json_text = json_text[7:] # Remove "```json"
if json_text.startswith("\n"):
json_text = json_text[1:] # Remove leading newline
if json_text.endswith("```"):
json_text = json_text[:-3] # Remove trailing "```"
if json_text.endswith("\n"):
json_text = json_text[:-1] # Remove trailing newline
return json_text

Xet Storage Details

Size:
25.4 kB
·
Xet hash:
207980f65da558dbe94d044df74b9115c9b6e10766271d1f3613f8157e663523

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.